fix: after updating, check the node group version to determine whethe…

…r the update failed.

Signed-off-by: minhthong582000 <[email protected]>

cmd/addons.go

@@ -36,7 +36,7 @@ var addonsCmd = &cobra.Command{
 		var wg sync.WaitGroup
 		errChan := make(chan error)
 
-		updater := updater.NewEKSUpdater(awsClient.EKS())
+		updater := updater.NewEKSUpdater(awsClient.EKS(), awsClient.SSM())
 		for _, addon := range addonsList {
 			wg.Add(1)
 

cmd/nodegroups.go

@@ -29,7 +29,7 @@ var nodegroupsCmd = &cobra.Command{
 			return err
 		}
 
-		updater := updater.NewEKSUpdater(awsClient.EKS())
+		updater := updater.NewEKSUpdater(awsClient.EKS(), awsClient.SSM())
 		err = updater.UpdateClusterNodeGroup(ctx, &clusterName, &nodegroupName, waitForNodeUpdates)
 		if err != nil {
 			return err

go.mod

@@ -3,24 +3,25 @@ module github.com/armory-io/eks-auto-updater
 go 1.20
 
 require (
-	github.com/aws/aws-sdk-go-v2 v1.17.6
+	github.com/aws/aws-sdk-go-v2 v1.21.2
 	github.com/aws/aws-sdk-go-v2/config v1.18.18
 	github.com/aws/aws-sdk-go-v2/credentials v1.13.17
 	github.com/aws/aws-sdk-go-v2/service/eks v1.27.7
+	github.com/aws/aws-sdk-go-v2/service/ssm v1.39.0
 	github.com/aws/aws-sdk-go-v2/service/sts v1.18.6
 	github.com/hashicorp/go-version v1.6.0
 	github.com/spf13/cobra v1.7.0
 )
 
 require (
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.0 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.30 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.24 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.31 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.24 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sso v1.12.5 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.5 // indirect
-	github.com/aws/smithy-go v1.13.5 // indirect
+	github.com/aws/smithy-go v1.15.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect

go.sum

@@ -1,29 +1,35 @@
-github.com/aws/aws-sdk-go-v2 v1.17.6 h1:Y773UK7OBqhzi5VDXMi1zVGsoj+CVHs2eaC2bDsLwi0=
 github.com/aws/aws-sdk-go-v2 v1.17.6/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
+github.com/aws/aws-sdk-go-v2 v1.21.2 h1:+LXZ0sgo8quN9UOKXXzAWRT3FWd4NxeXWOZom9pE7GA=
+github.com/aws/aws-sdk-go-v2 v1.21.2/go.mod h1:ErQhvNuEMhJjweavOYhxVkn2RUx7kQXVATHrjKtxIpM=
 github.com/aws/aws-sdk-go-v2/config v1.18.18 h1:/ePABXvXl3ESlzUGnkkvvNnRFw3Gh13dyqaq0Qo3JcU=
 github.com/aws/aws-sdk-go-v2/config v1.18.18/go.mod h1:Lj3E7XcxJnxMa+AYo89YiL68s1cFJRGduChynYU67VA=
 github.com/aws/aws-sdk-go-v2/credentials v1.13.17 h1:IubQO/RNeIVKF5Jy77w/LfUvmmCxTnk2TP1UZZIMiF4=
 github.com/aws/aws-sdk-go-v2/credentials v1.13.17/go.mod h1:K9xeFo1g/YPMguMUD69YpwB4Nyi6W/5wn706xIInJFg=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.0 h1:/2Cb3SK3xVOQA7Xfr5nCWCo5H3UiNINtsVvVdk8sQqA=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.0/go.mod h1:neYVaeKr5eT7BzwULuG2YbLhzWZ22lpjKdCybR7AXrQ=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.30 h1:y+8n9AGDjikyXoMBTRaHHHSaFEB8267ykmvyPodJfys=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.30/go.mod h1:LUBAO3zNXQjoONBKn/kR1y0Q4cj/D02Ts0uHYjcCQLM=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.24 h1:r+Kv+SEJquhAZXaJ7G4u44cIwXV3f8K+N482NNAzJZA=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43 h1:nFBQlGtkbPzp/NjZLuFxRqmT91rLJkgvsEQs68h962Y=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43/go.mod h1:auo+PiyLl0n1l8A0e8RIeR8tOzYPfZZH/JNlrJ8igTQ=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.24/go.mod h1:gAuCezX/gob6BSMbItsSlMb6WZGV7K2+fWOvk8xBSto=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37 h1:JRVhO25+r3ar2mKGP7E0LDl8K9/G36gjlqca5iQbaqc=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37/go.mod h1:Qe+2KtKml+FEsQF/DHmDV+xjtche/hwoF75EG4UlHW8=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.3.31 h1:hf+Vhp5WtTdcSdE+yEcUz8L73sAzN0R+0jQv+Z51/mI=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.3.31/go.mod h1:5zUjguZfG5qjhG9/wqmuyHRyUftl2B5Cp6NNxNC6kRA=
 github.com/aws/aws-sdk-go-v2/service/eks v1.27.7 h1:oXita1pMwUidgg1dhQFqNMvRVHealaXeaJfIHhsjR/s=
 github.com/aws/aws-sdk-go-v2/service/eks v1.27.7/go.mod h1:Y/GLnfOgdvVnUKdL4YaF1FvFISbB/jD5gmlCuGI0nPU=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.24 h1:c5qGfdbCHav6viBwiyDns3OXqhqAbGjfIB4uVu2ayhk=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.24/go.mod h1:HMA4FZG6fyib+NDo5bpIxX1EhYjrAOveZJY2YR0xrNE=
+github.com/aws/aws-sdk-go-v2/service/ssm v1.39.0 h1:TDA81vAA0TIaw64bYe0ItJw0gxCZnKa9wcxgr1i9Ow0=
+github.com/aws/aws-sdk-go-v2/service/ssm v1.39.0/go.mod h1:qpnJ98BgJ3YUEvHMgJ1OADwaOgqhgv0nxnqAjTKupeY=
 github.com/aws/aws-sdk-go-v2/service/sso v1.12.5 h1:bdKIX6SVF3nc3xJFw6Nf0igzS6Ff/louGq8Z6VP/3Hs=
 github.com/aws/aws-sdk-go-v2/service/sso v1.12.5/go.mod h1:vuWiaDB30M/QTC+lI3Wj6S/zb7tpUK2MSYgy3Guh2L0=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.5 h1:xLPZMyuZ4GuqRCIec/zWuIhRFPXh2UOJdLXBSi64ZWQ=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.5/go.mod h1:QjxpHmCwAg0ESGtPQnLIVp7SedTOBMYy+Slr3IfMKeI=
 github.com/aws/aws-sdk-go-v2/service/sts v1.18.6 h1:rIFn5J3yDoeuKCE9sESXqM5POTAhOP1du3bv/qTL+tE=
 github.com/aws/aws-sdk-go-v2/service/sts v1.18.6/go.mod h1:48WJ9l3dwP0GSHWGc5sFGGlCkuA82Mc2xnw+T6Q8aDw=
-github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8=
 github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
+github.com/aws/smithy-go v1.15.0 h1:PS/durmlzvAFpQHDs4wi4sNNP9ExsqZh6IlfdHXgKK8=
+github.com/aws/smithy-go v1.15.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

internal/updater/eks.go

@@ -4,21 +4,36 @@ import (
 	"context"
 
 	"github.com/armory-io/eks-auto-updater/pkg/aws/eks"
+	"github.com/armory-io/eks-auto-updater/pkg/aws/ssm"
 )
 
 type EKSUpdater struct {
-	client eks.Interface
+	eksClient eks.Interface
+	ssmClient ssm.Interface
 }
 
-func NewEKSUpdater(client eks.Interface) *EKSUpdater {
+func NewEKSUpdater(eksClient eks.Interface, ssmClient ssm.Interface) *EKSUpdater {
 	return &EKSUpdater{
-		client: client,
+		eksClient: eksClient,
+		ssmClient: ssmClient,
 	}
 }
 
 // UpdateClusterNodeGroup updates the nodegroup of a cluster to the latest version
-func (u EKSUpdater) UpdateClusterNodeGroup(ctx context.Context, clusterName *string, nodegroupName *string, waitForNodeUpdates int) error {
-	err := u.client.UpdateNodegroupVersion(ctx, clusterName, nodegroupName, waitForNodeUpdates)
+func (u EKSUpdater) UpdateClusterNodeGroup(ctx context.Context, clusterName, nodegroupName *string, waitForNodeUpdates int) error {
+	// Retrieve the latest AMI release version from SSM parameter store (managed by AWS)
+	// and use this information to determine if the update is successful
+	version, err := u.eksClient.GetClusterVersion(ctx, clusterName)
+	if err != nil {
+		return err
+	}
+	latestVersion, err := u.ssmClient.GetLatestAMIReleaseVersion(ctx, &version, clusterName)
+	if err != nil {
+		return err
+	}
+
+	// Update the nodegroup to the latest version
+	err = u.eksClient.UpdateNodegroupVersion(ctx, clusterName, nodegroupName, &latestVersion, waitForNodeUpdates)
 	if err != nil {
 		return err
 	}
@@ -28,7 +43,7 @@ func (u EKSUpdater) UpdateClusterNodeGroup(ctx context.Context, clusterName *str
 
 // UpdateAddon updates the addon of a cluster to the latest version
 func (u EKSUpdater) UpdateAddon(ctx context.Context, clusterName *string, addonName *string) (err error) {
-	err = u.client.UpdateAddon(ctx, clusterName, addonName)
+	err = u.eksClient.UpdateAddon(ctx, clusterName, addonName)
 	if err != nil {
 		return err
 	}

pkg/aws/client.go

@@ -2,11 +2,11 @@ package aws
 
 import (
 	"context"
-	"log"
 	"time"
 
 	"github.com/armory-io/eks-auto-updater/pkg/aws/eks"
 	"github.com/armory-io/eks-auto-updater/pkg/aws/options"
+	"github.com/armory-io/eks-auto-updater/pkg/aws/ssm"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/config"
@@ -16,10 +16,12 @@ import (
 
 type Client interface {
 	EKS() eks.Interface
+	SSM() ssm.Interface
 }
 
 type client struct {
 	eks  eks.Interface
+	ssm  ssm.Interface
 	opts options.Options
 }
 
@@ -44,17 +46,23 @@ func NewClient(ctx context.Context, opts ...options.Option) (Client, error) {
 			o.Duration = time.Duration(60) * time.Minute
 		})
 		cfg.Credentials = aws.NewCredentialsCache(provider)
-
-		log.Println("INFO: Assuming role ARN " + AWSRoleArn)
 	}
 
 	if c.eks, err = eks.NewFromConfig(cfg); err != nil {
 		return nil, err
 	}
 
+	if c.ssm, err = ssm.NewFromConfig(cfg, c.eks); err != nil {
+		return nil, err
+	}
+
 	return c, nil
 }
 
 func (c *client) EKS() eks.Interface {
 	return c.eks
 }
+
+func (c *client) SSM() ssm.Interface {
+	return c.ssm
+}

pkg/aws/eks/eks.go

@@ -15,13 +15,16 @@ import (
 type Interface interface {
 	// UpdateNodegroupVersion checks if a nodegroup is in an updatable state and
 	// triggers an update if it is, then waits for the update to complete
-	UpdateNodegroupVersion(ctx context.Context, clusterName *string, nodegroupName *string, maxWaitDur int) error
+	UpdateNodegroupVersion(ctx context.Context, clusterName, nodegroupName, latestReleaseVersion *string, maxWaitDur int) error
 
 	// UpdateAddon updates the addon of a cluster to the latest version
-	UpdateAddon(ctx context.Context, clusterName *string, addonName *string) error
+	UpdateAddon(ctx context.Context, clusterName, addonName *string) error
 
 	// GetAddonsList returns a list of addons in a cluster
 	GetAddonsList(ctx context.Context, clusterName *string) ([]string, error)
+
+	// GetClusterVersion returns the kubernetes version of a cluster
+	GetClusterVersion(ctx context.Context, clusterName *string) (string, error)
 }
 
 type Client struct {
@@ -35,7 +38,7 @@ func NewFromConfig(cfg aws.Config) (Interface, error) {
 	return c, nil
 }
 
-func (c Client) UpdateNodegroupVersion(ctx context.Context, clusterName *string, nodegroupName *string, maxWaitDur int) error {
+func (c Client) UpdateNodegroupVersion(ctx context.Context, clusterName, nodegroupName, latestReleaseVersion *string, maxWaitDur int) error {
 	// Check if there is an update in progress
 	status, err := c.getNodeGroupStatus(ctx, clusterName, nodegroupName)
 	if err != nil {
@@ -48,7 +51,7 @@ func (c Client) UpdateNodegroupVersion(ctx context.Context, clusterName *string,
 	} else if status == types.NodegroupStatusActive {
 		// If it's active, trigger an update
 		log.Println("INFO: Nodegroup is active. Triggering update...")
-		jobID, err := c.updateNodegroupVersion(ctx, clusterName, nodegroupName)
+		jobID, err := c.updateNodegroupVersion(ctx, clusterName, nodegroupName, latestReleaseVersion)
 		if err != nil {
 			return err
 		}
@@ -58,7 +61,7 @@ func (c Client) UpdateNodegroupVersion(ctx context.Context, clusterName *string,
 	}
 
 	// Wait for the update to complete
-	err = c.nodegroupUpdateWaiter(ctx, clusterName, nodegroupName, maxWaitDur)
+	err = c.nodegroupUpdateWaiter(ctx, clusterName, nodegroupName, latestReleaseVersion, maxWaitDur)
 	if err != nil {
 		return err
 	}
@@ -78,16 +81,32 @@ func (c Client) getNodeGroupStatus(ctx context.Context, clusterName *string, nod
 		return "", fmt.Errorf("ERROR: Unable to describe nodegroup: %w", err)
 	}
 
+	fmt.Printf("version: %s\n", *currentNodeGroup.Nodegroup.ReleaseVersion)
+
 	return currentNodeGroup.Nodegroup.Status, nil
 }
 
+// getNodeGroupReleaseVersion returns the release version of a nodegroup
+func (c Client) getNodeGroupReleaseVersion(ctx context.Context, clusterName, nodegroupName *string) (string, error) {
+	currentNodeGroup, err := c.eks.DescribeNodegroup(ctx, &eks.DescribeNodegroupInput{
+		ClusterName:   clusterName,
+		NodegroupName: nodegroupName,
+	})
+	if err != nil {
+		return "", fmt.Errorf("ERROR: Unable to describe nodegroup: %w", err)
+	}
+
+	return *currentNodeGroup.Nodegroup.ReleaseVersion, nil
+}
+
 // updateNodegroupVersion updates the nodegroup of a cluster to the latest version
 // and returns the ID of the update job
-func (c Client) updateNodegroupVersion(ctx context.Context, clusterName *string, nodegroupName *string) (string, error) {
+func (c Client) updateNodegroupVersion(ctx context.Context, clusterName, nodegroupName, latestReleaseVersion *string) (string, error) {
 	// Trigger an update
 	version, err := c.eks.UpdateNodegroupVersion(ctx, &eks.UpdateNodegroupVersionInput{
-		ClusterName:   clusterName,
-		NodegroupName: nodegroupName,
+		ClusterName:    clusterName,
+		NodegroupName:  nodegroupName,
+		ReleaseVersion: latestReleaseVersion,
 	})
 	if err != nil {
 		return "", fmt.Errorf("ERROR: Update call failed %w", err)
@@ -97,7 +116,8 @@ func (c Client) updateNodegroupVersion(ctx context.Context, clusterName *string,
 }
 
 // nodegroupUpdateWaiter waits for the nodegroup of a cluster to finish updating
-func (c Client) nodegroupUpdateWaiter(ctx context.Context, clusterName *string, nodegroupName *string, maxWaitDur int) error {
+// and determines if the update was successful.
+func (c Client) nodegroupUpdateWaiter(ctx context.Context, clusterName, nodegroupName, latestReleaseVersion *string, maxWaitDur int) error {
 	waiter := eks.NewNodegroupActiveWaiter(c.eks)
 	err := waiter.Wait(ctx, &eks.DescribeNodegroupInput{
 		ClusterName:   clusterName,
@@ -109,10 +129,19 @@ func (c Client) nodegroupUpdateWaiter(ctx context.Context, clusterName *string,
 		return fmt.Errorf("ERROR: Update failed to complete in the allotted time: %w", err)
 	}
 
+	// After the update is complete, check if the version is correct
+	currentVersion, err := c.getNodeGroupReleaseVersion(ctx, clusterName, nodegroupName)
+	if err != nil {
+		return err
+	}
+	if currentVersion != *latestReleaseVersion {
+		return fmt.Errorf("ERROR: Update failed. Expected version %s, got %s", *latestReleaseVersion, currentVersion)
+	}
+
 	return nil
 }
 
-func (c Client) getClusterVersion(ctx context.Context, clusterName *string) (string, error) {
+func (c Client) GetClusterVersion(ctx context.Context, clusterName *string) (string, error) {
 	clusterInfo, err := c.eks.DescribeCluster(ctx, &eks.DescribeClusterInput{
 		Name: clusterName,
 	})
@@ -143,7 +172,7 @@ func (c Client) UpdateAddon(ctx context.Context, clusterName *string, addonName
 		return fmt.Errorf("ERROR: Unable to describe addon "+*addonName+" in the cluster: %w", err)
 	}
 
-	k8sVersion, err := c.getClusterVersion(ctx, clusterName)
+	k8sVersion, err := c.GetClusterVersion(ctx, clusterName)
 	if err != nil {
 		return err
 	}

pkg/aws/ssm/ssm.go

@@ -0,0 +1,39 @@
+package ssm
+
+import (
+	"context"
+
+	"github.com/armory-io/eks-auto-updater/pkg/aws/eks"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/ssm"
+)
+
+type Interface interface {
+	GetLatestAMIReleaseVersion(ctx context.Context, version, clusterName *string) (string, error)
+}
+
+type Client struct {
+	ssm       *ssm.Client
+	eksClient eks.Interface
+}
+
+func NewFromConfig(cfg aws.Config, eksClient eks.Interface) (Interface, error) {
+	c := &Client{}
+
+	c.ssm = ssm.NewFromConfig(cfg)
+	c.eksClient = eksClient
+
+	return c, nil
+}
+
+func (c Client) GetLatestAMIReleaseVersion(ctx context.Context, version, clusterName *string) (string, error) {
+	parameter, err := c.ssm.GetParameter(ctx, &ssm.GetParameterInput{
+		Name: aws.String("/aws/service/eks/optimized-ami/" + *version + "/amazon-linux-2/recommended/release_version"),
+	})
+	if err != nil {
+		return "", err
+	}
+
+	return *parameter.Parameter.Value, nil
+}