Added unit tests and reduced redundant code (#183)

* Added NDBClient interface and sendRequest method to make code DRY

* Added NDBClient interface and sendRequest method to make code DRY

* Changed ndb_api functions to use ndb_client interface

* Added clone unit tests

* Added uts for db and dbserver.go

* Removed the unused test directory

* Added helper UTs

* Added some util tests

* Added a coverage target to makefile

* Addressed comments and fixed conflicts

* Removed double pointers

* added ut coverage

* small changes

* bumped up go version and removed samples for make build

* from 1.22 to 1.22.0

* bumped to 1.22.0

* changed e2e tests to create log directory if it does not exist and print error message if it fails before log creation

* controller-gen bump up

* bump down to 1.21.7

---------

Co-authored-by: Mazin S <[email protected]>

.github/workflows/build-dev.yaml

@@ -18,7 +18,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: "^1.20"
+          go-version: "1.21.7"
 
       - name: Test build
         run: make test build

.github/workflows/release.yaml

@@ -16,7 +16,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: "^1.20"
+          go-version: "1.21.7"
 
       - name: Install tools
         uses: redhat-actions/openshift-tools-installer@v1

.github/workflows/synopsys-schedule.yaml

@@ -14,7 +14,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: "^1.20"
+          go-version: "1.21.7"
 
       - name: Build Project
         run: make build

.github/workflows/synopsys.yaml

@@ -16,7 +16,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: "^1.20"
+          go-version: "1.21.7"
 
       - name: Build Project
         run: make build

.github/workflows/trivy-scan.yaml

@@ -25,7 +25,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: "^1.20"
+          go-version: "1.21.7"
 
       - name: Build Project
         run: make build

Dockerfile

@@ -1,5 +1,5 @@
 # Build the manager binary
-FROM golang:1.20 as builder
+FROM golang:1.21.7 as builder
 
 WORKDIR /workspace
 # Copy the Go Modules manifests

Makefile

@@ -114,6 +114,14 @@ vet: ## Run go vet against code.
 test: manifests generate fmt vet envtest ## Run tests.
 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test `go list ./... | grep -v "/automation"` -coverprofile cover.out
 
+.PHONY: test-coverage
+test-coverage: manifests generate fmt vet envtest ## Run unit tests creating the output to report coverage
+	- rm -rf *.out  # Remove all coverage files if exists
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test `go list ./... | grep -v -E "/automation|/sandbox|/controllers"` -coverprofile=cover.out.tmp
+	cat cover.out.tmp | grep -v "zz_generated.deepcopy.go" > cover.out
+	go tool cover -func cover.out
+	rm cover.out.tmp
+
 ##@ Build
 
 .PHONY: build

automation/tests/cloning/mongo-si_test/mongo_si_test.go

@@ -41,6 +41,7 @@ func (suite *MongoCloningSingleInstanceTestSuite) SetupSuite() {
 	// Setup logger and context
 	logger, err := util.SetupLogger(fmt.Sprintf("%s/mongo-cloning-si_test.log", automation.CLONING_LOG_PATH), "mongo-cloning-si: ")
 	if err != nil {
+		fmt.Println(err)
 		suite.T().FailNow()
 	}
 	ctx = util.SetupContext(context.Background(), logger)

automation/tests/cloning/mssql-si_test/mssql_si_test.go

@@ -41,6 +41,7 @@ func (suite *MSSQLCloningSingleInstanceTestSuite) SetupSuite() {
 	// Setup logger and context
 	logger, err := util.SetupLogger(fmt.Sprintf("%s/mssql-cloning-si_test.log", automation.CLONING_LOG_PATH), "mssql-cloning-si: ")
 	if err != nil {
+		fmt.Println(err)
 		suite.T().FailNow()
 	}
 	ctx = util.SetupContext(context.Background(), logger)

automation/tests/cloning/mysql-si_test/mysql_si_test.go

@@ -41,6 +41,7 @@ func (suite *MySQLCloningSingleInstanceTestSuite) SetupSuite() {
 	// Setup logger and context
 	logger, err := util.SetupLogger(fmt.Sprintf("%s/mysql-cloning-si_test.log", automation.CLONING_LOG_PATH), "mysql-cloning-si: ")
 	if err != nil {
+		fmt.Println(err)
 		suite.T().FailNow()
 	}
 	ctx = util.SetupContext(context.Background(), logger)

automation/tests/cloning/pg-si_test/pg_si_test.go

@@ -41,6 +41,7 @@ func (suite *PostgresCloningSingleInstanceTestSuite) SetupSuite() {
 	// Setup logger and context
 	logger, err := util.SetupLogger(fmt.Sprintf("%s/pg-cloning-si_test.log", automation.CLONING_LOG_PATH), "pg-cloning-si: ")
 	if err != nil {
+		fmt.Println(err)
 		suite.T().FailNow()
 	}
 	ctx = util.SetupContext(context.Background(), logger)

automation/tests/provisioning/mongo-si_test/mongo-si_test.go

@@ -41,6 +41,7 @@ func (suite *MongoProvisioningSingleInstanceTestSuite) SetupSuite() {
 	// Setup logger and context
 	logger, err := util.SetupLogger(fmt.Sprintf("%s/mongo-provisioning-si_test.log", automation.PROVISIONING_LOG_PATH), "mongo-provisioning-si: ")
 	if err != nil {
+		fmt.Println(err)
 		suite.T().FailNow()
 	}
 	ctx = util.SetupContext(context.Background(), logger)
@@ -175,7 +176,7 @@ func (suite *MongoProvisioningSingleInstanceTestSuite) TestTimeMachineSuccess()
 		logger.Println("Timemachine response retrieved.")
 	}
 
-	err = util.CheckTmInfo(suite.ctx, suite.setupTypes.Database, &tm)
+	err = util.CheckTmInfo(suite.ctx, suite.setupTypes.Database, tm)
 	if err != nil {
 		logger.Printf("Error: TestTimeMachineSuccess() failed! %v", err)
 		assert.FailNow("Error: TestTimeMachineSuccess() failed! %v", err)

automation/tests/provisioning/mssql-si_test/mssql-si_test.go

@@ -40,6 +40,7 @@ func (suite *MSSQLProvisioningSingleInstanceTestSuite) SetupSuite() {
 	// Setup logger and context
 	logger, err := util.SetupLogger(fmt.Sprintf("%s/mssql-provisioning-si_test.log", automation.PROVISIONING_LOG_PATH), "mssql-provisioning-si: ")
 	if err != nil {
+		fmt.Println(err)
 		suite.T().FailNow()
 	}
 	ctx = util.SetupContext(context.Background(), logger)
@@ -174,7 +175,7 @@ func (suite *MSSQLProvisioningSingleInstanceTestSuite) TestTimeMachineSuccess()
 		logger.Println("Timemachine response retrieved.")
 	}
 
-	err = util.CheckTmInfo(suite.ctx, suite.setupTypes.Database, &tm)
+	err = util.CheckTmInfo(suite.ctx, suite.setupTypes.Database, tm)
 	if err != nil {
 		logger.Printf("Error: TestTimeMachineSuccess() failed! %v", err)
 		assert.FailNow("Error: TestTimeMachineSuccess() failed! %v", err)

automation/tests/provisioning/mysql-si_test/mysql-si_test.go

@@ -41,6 +41,7 @@ func (suite *MySQLProvisioningSingleInstanceTestSuite) SetupSuite() {
 	// Setup logger and context
 	logger, err := util.SetupLogger(fmt.Sprintf("%s/mysql-provisioning-si_test.log", automation.PROVISIONING_LOG_PATH), "mysql-provisioning-si: ")
 	if err != nil {
+		fmt.Println(err)
 		suite.T().FailNow()
 	}
 	ctx = util.SetupContext(context.Background(), logger)
@@ -175,7 +176,7 @@ func (suite *MySQLProvisioningSingleInstanceTestSuite) TestTimeMachineSuccess()
 		logger.Println("Timemachine response retrieved.")
 	}
 
-	err = util.CheckTmInfo(suite.ctx, suite.setupTypes.Database, &tm)
+	err = util.CheckTmInfo(suite.ctx, suite.setupTypes.Database, tm)
 	if err != nil {
 		logger.Printf("Error: TestTimeMachineSuccess() failed! %v", err)
 		assert.FailNow("Error: TestTimeMachineSuccess() failed! %v", err)

automation/tests/provisioning/pg-si_test/pg-si_test.go

@@ -41,6 +41,7 @@ func (suite *PostgresProvisioningSingleInstanceTestSuite) SetupSuite() {
 	// Setup logger and context
 	logger, err := util.SetupLogger(fmt.Sprintf("%s/pg-provisioning-si_test.log", automation.PROVISIONING_LOG_PATH), "pg-provisioning-si: ")
 	if err != nil {
+		fmt.Println(err)
 		suite.T().FailNow()
 	}
 	ctx = util.SetupContext(context.Background(), logger)
@@ -175,7 +176,7 @@ func (suite *PostgresProvisioningSingleInstanceTestSuite) TestTimeMachineSuccess
 		logger.Println("Timemachine response retrieved.")
 	}
 
-	err = util.CheckTmInfo(suite.ctx, suite.setupTypes.Database, &tm)
+	err = util.CheckTmInfo(suite.ctx, suite.setupTypes.Database, tm)
 	if err != nil {
 		logger.Printf("Error: TestTimeMachineSuccess() failed! %v", err)
 		assert.FailNow("Error: TestTimeMachineSuccess() failed! %v", err)

automation/util/cloning_helpers.go

@@ -61,7 +61,7 @@ func getDatabaseName(ctx context.Context, database *ndbv1alpha1.Database) (name
 }
 
 // Gets snapshot id from TimeMachineGetSnapshotsResponse
-func getSnapshotId(ctx context.Context, response ndb_api.TimeMachineGetSnapshotsResponse, nxClusterId string) (string, error) {
+func getSnapshotId(ctx context.Context, response *ndb_api.TimeMachineGetSnapshotsResponse, nxClusterId string) (string, error) {
 	logger := GetLogger(ctx)
 	logger.Println("getSnapshotId() starting...")
 

automation/util/cloning_test_suite_manager.go

@@ -32,7 +32,7 @@ func (cm *CloningTestSuiteManager) TearDown(ctx context.Context, st *SetupTypes,
 	return
 }
 
-func (cm *CloningTestSuiteManager) GetDatabaseOrCloneResponse(ctx context.Context, st *SetupTypes, clientset *kubernetes.Clientset, v1alpha1ClientSet *clientsetv1alpha1.V1alpha1Client) (databaseResponse ndb_api.DatabaseResponse, err error) {
+func (cm *CloningTestSuiteManager) GetDatabaseOrCloneResponse(ctx context.Context, st *SetupTypes, clientset *kubernetes.Clientset, v1alpha1ClientSet *clientsetv1alpha1.V1alpha1Client) (databaseResponse *ndb_api.DatabaseResponse, err error) {
 	logger := GetLogger(ctx)
 	logger.Println("CloneTestSuiteManager.GetDatabaseResponse() starting...")
 
@@ -55,6 +55,6 @@ func (cm *CloningTestSuiteManager) GetAppResponse(ctx context.Context, st *Setup
 }
 
 // EMPTY STUB
-func (dm *CloningTestSuiteManager) GetTimemachineResponseByDatabaseId(ctx context.Context, st *SetupTypes, clientset *kubernetes.Clientset, v1alpha1ClientSet *clientsetv1alpha1.V1alpha1Client) (timemachineResponse ndb_api.TimeMachineResponse, err error) {
-	return ndb_api.TimeMachineResponse{}, nil
+func (dm *CloningTestSuiteManager) GetTimemachineResponseByDatabaseId(ctx context.Context, st *SetupTypes, clientset *kubernetes.Clientset, v1alpha1ClientSet *clientsetv1alpha1.V1alpha1Client) (timemachineResponse *ndb_api.TimeMachineResponse, err error) {
+	return nil, nil
 }

automation/util/interfaces.go

@@ -41,7 +41,7 @@ type TestSuiteManager interface {
 		ctx context.Context,
 		st *SetupTypes,
 		clientset *kubernetes.Clientset,
-		v1alpha1ClientSet *clientsetv1alpha1.V1alpha1Client) (databaseResponse ndb_api.DatabaseResponse, err error)
+		v1alpha1ClientSet *clientsetv1alpha1.V1alpha1Client) (databaseResponse *ndb_api.DatabaseResponse, err error)
 	GetAppResponse(
 		ctx context.Context,
 		st *SetupTypes,
@@ -51,7 +51,7 @@ type TestSuiteManager interface {
 		ctx context.Context,
 		st *SetupTypes,
 		clientset *kubernetes.Clientset,
-		v1alpha1ClientSet *clientsetv1alpha1.V1alpha1Client) (timemachineResponse ndb_api.TimeMachineResponse, err error)
+		v1alpha1ClientSet *clientsetv1alpha1.V1alpha1Client) (timemachineResponse *ndb_api.TimeMachineResponse, err error)
 }
 
 type CloningTestSuiteManager struct{}

automation/util/provisioning_test_suite_manager.go

@@ -36,7 +36,7 @@ func (pm *ProvisioningTestSuiteManager) TearDown(ctx context.Context, st *SetupT
 	return
 }
 
-func (pm *ProvisioningTestSuiteManager) GetDatabaseOrCloneResponse(ctx context.Context, st *SetupTypes, clientset *kubernetes.Clientset, v1alpha1ClientSet *clientsetv1alpha1.V1alpha1Client) (databaseResponse ndb_api.DatabaseResponse, err error) {
+func (pm *ProvisioningTestSuiteManager) GetDatabaseOrCloneResponse(ctx context.Context, st *SetupTypes, clientset *kubernetes.Clientset, v1alpha1ClientSet *clientsetv1alpha1.V1alpha1Client) (databaseResponse *ndb_api.DatabaseResponse, err error) {
 	logger := GetLogger(ctx)
 	logger.Println("DatabaseTestSuiteManager.GetDatabaseResponse() starting...")
 
@@ -59,23 +59,23 @@ func (pm *ProvisioningTestSuiteManager) GetAppResponse(ctx context.Context, st *
 }
 
 // Tests TM Response
-func (pm *ProvisioningTestSuiteManager) GetTimemachineResponseByDatabaseId(ctx context.Context, st *SetupTypes, clientset *kubernetes.Clientset, v1alpha1ClientSet *clientsetv1alpha1.V1alpha1Client) (timemachineResponse ndb_api.TimeMachineResponse, err error) {
+func (pm *ProvisioningTestSuiteManager) GetTimemachineResponseByDatabaseId(ctx context.Context, st *SetupTypes, clientset *kubernetes.Clientset, v1alpha1ClientSet *clientsetv1alpha1.V1alpha1Client) (timemachineResponse *ndb_api.TimeMachineResponse, err error) {
 	logger := GetLogger(ctx)
 	logger.Println("GetTimemachineResponse() starting...")
 	errBaseMsg := "Error: GetTimemachineResponse() ended"
 
 	// Get NDBServer CR
 	ndbServer, err := v1alpha1ClientSet.NDBServers(st.NdbServer.Namespace).Get(st.NdbServer.Name, metav1.GetOptions{})
 	if err != nil {
-		return ndb_api.TimeMachineResponse{}, fmt.Errorf("%s! Could not fetch ndbServer '%s' CR! %s\n", errBaseMsg, ndbServer.Name, err)
+		return nil, fmt.Errorf("%s! Could not fetch ndbServer '%s' CR! %s\n", errBaseMsg, ndbServer.Name, err)
 	} else {
 		logger.Printf("Retrieved ndbServer '%s' CR from v1alpha1ClientSet", ndbServer.Name)
 	}
 
 	// Get Database CR
 	database, err := v1alpha1ClientSet.Databases(st.Database.Namespace).Get(st.Database.Name, metav1.GetOptions{})
 	if err != nil {
-		return ndb_api.TimeMachineResponse{}, fmt.Errorf("%s! Could not fetch database '%s' CR! %s\n", errBaseMsg, database.Name, err)
+		return nil, fmt.Errorf("%s! Could not fetch database '%s' CR! %s\n", errBaseMsg, database.Name, err)
 	} else {
 		logger.Printf("Retrieved database '%s' CR from v1alpha1ClientSet", database.Name)
 	}
@@ -85,20 +85,20 @@ func (pm *ProvisioningTestSuiteManager) GetTimemachineResponseByDatabaseId(ctx c
 	secret, err := clientset.CoreV1().Secrets(database.Namespace).Get(context.TODO(), ndb_secret_name, metav1.GetOptions{})
 	username, password := string(secret.Data[common.SECRET_DATA_KEY_USERNAME]), string(secret.Data[common.SECRET_DATA_KEY_PASSWORD])
 	if err != nil || username == "" || password == "" {
-		return ndb_api.TimeMachineResponse{}, fmt.Errorf("%s! Could not fetch data from secret! %s\n", errBaseMsg, err)
+		return nil, fmt.Errorf("%s! Could not fetch data from secret! %s\n", errBaseMsg, err)
 	}
 
 	// Create ndbClient and getting databaseResponse so we can get timemachine id
 	ndbClient := ndb_client.NewNDBClient(username, password, ndbServer.Spec.Server, "", true)
 	databaseResponse, err := ndb_api.GetDatabaseById(context.TODO(), ndbClient, database.Status.Id)
 	if err != nil {
-		return ndb_api.TimeMachineResponse{}, fmt.Errorf("%s! Database response from ndb_api failed! %s\n", errBaseMsg, err)
+		return nil, fmt.Errorf("%s! Database response from ndb_api failed! %s\n", errBaseMsg, err)
 	}
 
 	// Get timemachine
 	timemachineResponse, err = ndb_api.GetTimeMachineById(context.TODO(), ndbClient, databaseResponse.TimeMachineId)
 	if err != nil {
-		return ndb_api.TimeMachineResponse{}, fmt.Errorf("%s! time machine response from ndb_api failed! %s\n", errBaseMsg, err)
+		return nil, fmt.Errorf("%s! time machine response from ndb_api failed! %s\n", errBaseMsg, err)
 	}
 
 	logger.Printf("Timemachine response.status: %s.\n", timemachineResponse.Status)

automation/util/setup.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"path/filepath"
 	"reflect"
 	"time"
 
@@ -15,7 +16,6 @@ import (
 	automation "github.com/nutanix-cloud-native/ndb-operator/automation"
 	clientsetv1alpha1 "github.com/nutanix-cloud-native/ndb-operator/automation/clientset/v1alpha1"
 	corev1 "k8s.io/api/core/v1"
-	v1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
@@ -40,11 +40,20 @@ func SetupLogger(path string, rootName string) (*log.Logger, error) {
 		_ = os.Remove(path)
 	}
 
+	// Get the directory of the path
+	dir := filepath.Dir(path)
+
+	// Create the directory and all parent directories if they do not exist
+	if err := os.MkdirAll(dir, 0777); err != nil {
+		return nil, err
+	}
+
 	// Creates the file
 	file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
 	if err != nil {
 		return nil, err
 	}
+
 
 	// Links the logger to the file and returns the logger
 	return log.New(file, rootName, log.Ldate|log.Ltime|log.Lshortfile), nil
@@ -67,7 +76,7 @@ func CheckRequiredEnv(ctx context.Context) (err error) {
 	// Loading env variables
 	err = godotenv.Load("../../.env")
 	if err != nil {
-		return fmt.Errorf("Error: loadEnv() ended! %s", err)
+		return fmt.Errorf("error: loadEnv() ended! %s", err)
 	} else {
 		logger.Print("Loaded .env file!")
 	}
@@ -87,7 +96,7 @@ func CheckRequiredEnv(ctx context.Context) (err error) {
 		}
 	}
 	if len(missingRequiredEnvs) != 0 {
-		return fmt.Errorf("Error: CheckRequiredEnv() ended! Missing the following required env variables: %s", missingRequiredEnvs)
+		return fmt.Errorf("error: loadEnv() ended! Missing the following required env variables: %s", missingRequiredEnvs)
 	} else {
 		logger.Print("Found no missing required env variables!")
 	}
@@ -113,7 +122,7 @@ func SetupKubeconfig(ctx context.Context) (config *rest.Config, err error) {
 	}
 
 	if err != nil {
-		return nil, fmt.Errorf("Error: SetupKubeconfig() ended! %s", err)
+		return nil, fmt.Errorf("error: SetupKubeconfig() ended! %s", err)
 	}
 
 	logger.Println("SetupKubeconfig() ended!")
@@ -131,13 +140,13 @@ func SetupSchemeAndClientSet(ctx context.Context, config *rest.Config) (v1alpha1
 
 	v1alpha1ClientSet, err = clientsetv1alpha1.NewForConfig(config)
 	if err != nil {
-		return nil, nil, fmt.Errorf("Error: SetupSchemeAndClientSet() ended! %s", err)
+		return nil, nil, fmt.Errorf("error: SetupSchemeAndClientSet() ended! %s", err)
 	}
 	logger.Printf("Created v1alpha1Client.")
 
 	clientset, err = kubernetes.NewForConfig(config)
 	if err != nil {
-		return nil, nil, fmt.Errorf("Error: SetupSchemeAndClientSet() ended! %s", err)
+		return nil, nil, fmt.Errorf("error: SetupSchemeAndClientSet() ended! %s", err)
 	}
 	logger.Printf("Created clientset.")
 
@@ -171,23 +180,23 @@ func SetupTypeTemplates(ctx context.Context) (setupTypes *SetupTypes, err error)
 	}
 
 	// Create ndbSecret template automation.NDB_SECRET_PATH
-	ndbSecret := &v1.Secret{}
+	ndbSecret := &corev1.Secret{}
 	if err := CreateTypeFromPath(ndbSecret, automation.NDB_SECRET_PATH); err != nil {
 		errMsg += fmt.Sprintf("NdbSecret with path %s failed! %v. ", automation.NDB_SECRET_PATH, err)
 	} else {
 		logMsg += fmt.Sprintf("NdbSecret with path %s created. ", automation.NDB_SECRET_PATH)
 	}
 
 	// Create dbSecret template from automation.DB_SECRET_PATH
-	dbSecret := &v1.Secret{}
+	dbSecret := &corev1.Secret{}
 	if err := CreateTypeFromPath(dbSecret, automation.DB_SECRET_PATH); err != nil {
 		errMsg += fmt.Sprintf("DbSecret with path %s failed! %v. ", automation.DB_SECRET_PATH, err)
 	} else {
 		logMsg += fmt.Sprintf("DbSecret with path %s created. ", automation.DB_SECRET_PATH)
 	}
 
 	// Create appPod template from automation.APP_POD_PATH
-	appPod := &v1.Pod{}
+	appPod := &corev1.Pod{}
 	if err := CreateTypeFromPath(appPod, automation.APP_POD_PATH); err != nil {
 		errMsg += fmt.Sprintf("App Pod with path %s failed! %v. ", automation.APP_POD_PATH, err)
 	} else {