NO-JIRA: cleanup iamRoles as part of nightly test cleanup (#4382)

* NO-JIRA: cleanup iam_roles as part of nightly test cleanup

* remove reference to log, use fmt

* fix panic in cleanup code

* code review feedback

* code review feedback

hack/aws-acceptance-test-cleanup/main.go

@@ -81,6 +81,11 @@ func realMain(ctx context.Context) error {
 		return err
 	}
 
+	// Find IAM roles and delete
+	if err := cleanupIAMRoles(ctx, iamClient); err != nil {
+		return err
+	}
+
 	// Find OIDC providers to delete.
 	oidcProvidersOutput, err := iamClient.ListOpenIDConnectProvidersWithContext(ctx, &iam.ListOpenIDConnectProvidersInput{})
 	if err != nil {
@@ -759,6 +764,118 @@ func destroyBackoff(ctx context.Context, resourceKind string, resourceID string,
 	}, backoff.WithContext(expoBackoff, ctx))
 }
 
+func cleanupIAMRoles(ctx context.Context, iamClient *iam.IAM) error {
+	var rolesToDelete []*iam.Role
+	rolesToDelete, err := filterIAMRolesWithPrefix(ctx, iamClient, "consul-k8s-")
+	if err != nil {
+		return fmt.Errorf("failed to list roles: %w", err)
+	}
+
+	if len(rolesToDelete) == 0 {
+		fmt.Println("Found no iamRoles to clean up")
+		return nil
+	} else {
+		fmt.Printf("Found %d IAM roles to clean up\n", len(rolesToDelete))
+	}
+
+	// Delete filtered roles
+	for _, role := range rolesToDelete {
+		roleName := aws.StringValue(role.RoleName)
+		err := detachRolePolicies(iamClient, role.RoleName)
+		if err != nil {
+			fmt.Printf("Failed to detach policies for role %s: %v", roleName, err)
+			continue
+		}
+
+		_, err = iamClient.DeleteRole(&iam.DeleteRoleInput{
+			RoleName: role.RoleName,
+		})
+		if err != nil {
+			fmt.Printf("Failed to delete role %s: %v", roleName, err)
+		} else {
+			fmt.Printf("Deleted role: %s", roleName)
+		}
+	}
+
+	return nil
+}
+
+// filterIAMRolesWithPrefix is a callback function used with ListRolesPages.
+// It filters roles based on specified prefix and appends matching roles to rolesToDelete.
+//
+// Parameters:
+// - page: A single page of IAM roles returned by the AWS API
+// - lastPage: A boolean indicating whether this is the last page of results
+// - rolesToDelete: A pointer to the slice where matching roles are accumulated
+// - rolePrefix: The prefix to filter roles by
+func filterIAMRolesWithPrefix(ctx context.Context, iamClient *iam.IAM, prefix string) ([]*iam.Role, error) {
+	var roles []*iam.Role
+
+	err := iamClient.ListRolesPagesWithContext(ctx, &iam.ListRolesInput{},
+		func(page *iam.ListRolesOutput, lastPage bool) bool {
+			for _, role := range page.Roles {
+				name := aws.StringValue(role.RoleName)
+				if strings.HasPrefix(name, prefix) {
+					roles = append(roles, role)
+				}
+			}
+
+			return true
+		})
+	if err != nil {
+		return nil, err
+	}
+
+	return roles, nil
+}
+
+func detachRolePolicies(iamClient *iam.IAM, roleName *string) error {
+	if roleName == nil {
+		return fmt.Errorf("roleName is nil")
+	}
+
+	err := iamClient.ListAttachedRolePoliciesPages(&iam.ListAttachedRolePoliciesInput{
+		RoleName: roleName,
+	}, func(page *iam.ListAttachedRolePoliciesOutput, lastPage bool) bool {
+		err := detachPoliciesFromRole(iamClient, roleName, page)
+		if err != nil {
+			fmt.Printf("Error detaching policies from role %s: %v", *roleName, err)
+		}
+		return !lastPage
+	})
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// detachPoliciesFromRole detaches all policies from a given role.
+//
+// Parameters:
+// - iamClient: The IAM service client
+// - roleName: Pointer to the name of the role
+// - page: A single page of attached policies returned by the AWS API
+func detachPoliciesFromRole(iamClient *iam.IAM, roleName *string, page *iam.ListAttachedRolePoliciesOutput) error {
+	for _, policy := range page.AttachedPolicies {
+		if policy.PolicyArn == nil {
+			fmt.Printf("Warning: PolicyArn is nil for a policy attached to: %s", *roleName)
+			continue
+		}
+
+		_, err := iamClient.DetachRolePolicy(&iam.DetachRolePolicyInput{
+			RoleName:  roleName,
+			PolicyArn: policy.PolicyArn,
+		})
+		if err != nil {
+			fmt.Printf("Failed to detach policy %s from role %s: %v\n", aws.StringValue(policy.PolicyArn), *roleName, err)
+		}
+	}
+
+	return nil
+}
+
 func cleanupPersistentVolumes(ctx context.Context, ec2Client *ec2.EC2) error {
 	var nextToken *string
 	var toDeleteVolumes []*ec2.Volume