Removing BasicAuth requirement for Connections (#210)

Fixes #206

ChangeLog:
 - Adding better testing patterns allowing easy ENV based switch from
   basic to token based auth.
 - Adding a matrix for token tests

.github/workflows/go.yml

@@ -11,23 +11,22 @@ jobs:
   test:
     strategy:
       matrix:
-        go: [ 1.21.0 ]
-        grafana: [ 8.5.22, 9.4.3, 10.1.4 ]
+        go: [ {version: 1.21.0, token: 1}, {version: 1.21.0, token: 0}]
+        grafana: [ 10.1.4 ]
 
     env:
       GRAFANA_INTEGRATION: 1
+      TEST_TOKEN_CONFIG: "${{ matrix.go.token }}"
 
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
-          go-version: ${{ matrix.go }}
-
+          go-version: ${{ matrix.go.version }}
       - name: Verify go version
         run: go version
-
       - uses: actions/cache@v3
         with:
           path: |
@@ -36,14 +35,17 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-go-
 
+      - name: Validate ENV Value
+        shell: bash
+        run: | 
+          echo "token IS $TEST_TOKEN_CONFIG"
       - name: Calc coverage
-        if: "${{ matrix.go == '1.21.0' && matrix.grafana == '10.1.4' }}"
+        if: "${{ matrix.go.version == '1.21.0' && matrix.grafana == '10.1.4' && matrix.go.token == '0' }}"
         run: |
           go test -v -covermode=atomic -coverprofile=coverage.out ./...
       - name: Convert coverage.out to coverage.lcov
-        if: "${{ matrix.go == '1.21.0' && matrix.grafana == '10.1.4' }}"
+        if: "${{ matrix.go.version == '1.21.0' && matrix.grafana == '10.1.4' && matrix.go.token == '0' }}"
         uses: jandelgado/[email protected]
-      - name: Test
-        if: "${{ matrix.grafana != '10.1.4' }}"
+      - name: Test 
+        if: "${{ matrix.go.token == '1' }}"
         run: go test -v ./...
-

.gitignore

@@ -8,3 +8,4 @@ prod/
 public
 resources
 .idea
+config/token*.yml

Taskfile.yml

@@ -110,6 +110,15 @@ tasks:
         - go test -v ./... -cover
       env:
         GRAFANA_INTEGRATION: "1"
+        TEST_TOKEN_CONFIG: "0"
+  test_tokens:
+      desc: "test Token Based Only"
+      cmds:
+        - go test -v ./... -cover
+      env:
+        GRAFANA_INTEGRATION: "1"
+        TEST_TOKEN_CONFIG: "1"
+
   vuln_check:
       desc: "Vulnerability check"
       cmds:

config/testing.yml

@@ -1,16 +1,10 @@
 ## Configuration intended to be used with integration tests
-context_name: qa
+context_name: testing
 
 contexts:
   testing:
     output_path: test/data
     connections:
-#      exclude_filters:
-#        - field: "name"
-#          regex: "DEV-*|-Dev-*"
-#        - field: "type"
-#          regex: "elasticsearch"
-#          inclusive: true
       credential_rules:
         - rules:
             - field: "name"

go.mod

@@ -31,6 +31,7 @@ require (
 	github.com/carlmjohnson/requests v0.23.4
 	github.com/esnet/grafana-swagger-api-golang v0.0.0-20230904013855-9a47b55d30d3
 	github.com/go-openapi/runtime v0.26.0
+	github.com/google/uuid v1.3.1
 	github.com/jedib0t/go-pretty/v6 v6.4.8
 	github.com/ory/dockertest/v3 v3.10.0
 	github.com/samber/lo v1.38.1
@@ -101,7 +102,6 @@ require (
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/s2a-go v0.1.7 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
-	github.com/google/uuid v1.3.1 // indirect
 	github.com/google/wire v0.5.0 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.1 // indirect
 	github.com/googleapis/gax-go/v2 v2.12.0 // indirect

internal/config/config_test.go

@@ -35,13 +35,15 @@ func DuplicateConfig(t *testing.T) string {
 }
 
 func TestSetup(t *testing.T) {
+	os.Setenv("GDG_CONTEXT_NAME", "qa")
 	//clear all ENV values
 	for _, key := range os.Environ() {
 		if strings.Contains(key, "GDG_") {
 			os.Unsetenv(key)
 		}
 	}
 
+	os.Setenv("GDG_CONTEXT_NAME", "qa")
 	config.InitConfig("testing.yml", "")
 	conf := config.Config().ViperConfig()
 	log.Info(conf.ConfigFileUsed())

internal/service/common_test.go

@@ -4,6 +4,7 @@ import (
 	"github.com/esnet/gdg/internal/config"
 	"github.com/gosimple/slug"
 	"github.com/stretchr/testify/assert"
+	"os"
 	"testing"
 )
 
@@ -18,6 +19,8 @@ func TestSlug(t *testing.T) {
 }
 
 func TestUserPath(t *testing.T) {
+	err := os.Setenv("GDG_CONTEXT_NAME", "qa")
+	assert.Nil(t, err)
 	config.InitConfig("testing.yml", "'")
 	path := buildResourceFolder("", config.UserResource)
 	assert.Equal(t, "qa/users/", path)

internal/service/connections.go

@@ -166,7 +166,7 @@ func (s *DashNGoImpl) UploadConnections(filter filters.Filter) []string {
 
 			if creds != nil {
 				user := creds.User
-				var secureData map[string]string = make(map[string]string)
+				var secureData = make(map[string]string)
 				newDS.BasicAuthUser = user
 				secureData["basicAuthPassword"] = creds.Password
 				newDS.SecureJSONData = secureData
@@ -178,7 +178,7 @@ func (s *DashNGoImpl) UploadConnections(filter filters.Filter) []string {
 				if existingDS.Name == newDS.Name {
 					deleteParam := datasources.NewDeleteDataSourceByIDParams()
 					deleteParam.ID = fmt.Sprintf("%d", existingDS.ID)
-					if _, err := s.client.Datasources.DeleteDataSourceByID(deleteParam, s.getBasicAuth()); err != nil {
+					if _, err := s.client.Datasources.DeleteDataSourceByID(deleteParam, s.getAuth()); err != nil {
 						log.Errorf("error on deleting datasource %s with %s", newDS.Name, err)
 					}
 					break

internal/service/user.go

@@ -192,6 +192,9 @@ func (s *DashNGoImpl) UploadUsers(filter filters.Filter) []models.UserProfileDTO
 
 // ListUsers list all grafana users
 func (s *DashNGoImpl) ListUsers(filter filters.Filter) []*models.UserSearchHitDTO {
+	if !s.grafanaConf.IsBasicAuth() {
+		log.Fatal("User listing requires basic auth to be configured.  Token based listing is not supported")
+	}
 	var filteredUsers []*models.UserSearchHitDTO
 	params := users.NewSearchUsersParams()
 	params.Page = gapi.ToPtr(int64(1))

test/cloud_integration_test.go

@@ -14,7 +14,6 @@ func TestCloudDataSourceCRUD(t *testing.T) {
 		t.Skip("skipping integration test")
 	}
 
-	os.Setenv("GDG_CONTEXT_NAME", "testing")
 	apiClient, _ := initTest(t, nil)
 
 	//Wipe all data from grafana

test/common_test.go

@@ -5,8 +5,10 @@ import (
 	"fmt"
 	"github.com/esnet/gdg/internal/config"
 	"github.com/esnet/gdg/internal/service"
+	"github.com/google/uuid"
 	"github.com/ory/dockertest/v3"
 	log "github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v3"
 	"net"
 	"os"
 	"sync"
@@ -115,12 +117,53 @@ func TestMain(m *testing.M) {
 }
 
 func initTest(t *testing.T, cfgName *string) (service.GrafanaService, *viper.Viper) {
+	apiClient, v := createSimpleClient(t, cfgName)
+
+	if os.Getenv("TEST_TOKEN_CONFIG") != "1" {
+		return apiClient, v
+	}
+
+	testData, _ := os.ReadFile(v.ConfigFileUsed())
+	data := map[string]interface{}{}
+	err := yaml.Unmarshal(testData, &data)
+	assert.Nil(t, err)
+
+	apiClient.DeleteAllTokens() //Remove any old data
+	tokenName, _ := uuid.NewUUID()
+	newKey, err := apiClient.CreateAPIKey(tokenName.String(), "admin", 0)
+	assert.Nil(t, err)
+
+	wrapper := map[string]*config.GrafanaConfig{}
+	_ = wrapper
+
+	level1 := data["contexts"].(map[string]interface{})
+	level2 := level1["testing"].(map[string]interface{})
+	level2["token"] = newKey.Key
+	level2["user_name"] = ""
+	level2["password"] = ""
+
+	updatedCfg, err := yaml.Marshal(data)
+	assert.Nil(t, err)
+	tokenCfg, err := os.CreateTemp("config", "token*.yml")
+	assert.Nil(t, err, "Unable to create token configuration file")
+	newCfg := tokenCfg.Name()
+	err = os.WriteFile(newCfg, updatedCfg, 0644)
+	assert.Nil(t, err)
+
+	return createSimpleClient(t, &newCfg)
+
+}
+
+func createSimpleClient(t *testing.T, cfgName *string) (service.GrafanaService, *viper.Viper) {
 	if cfgName == nil {
 		cfgName = new(string)
 		*cfgName = "testing.yml"
 	}
+
 	actualPort := grafanaResource.GetPort(fmt.Sprintf("%s/tcp", "3000"))
-	os.Setenv("GDG_CONTEXTS__TESTING__URL", fmt.Sprintf("http://localhost:%s", actualPort))
+	err := os.Setenv("GDG_CONTEXTS__TESTING__URL", fmt.Sprintf("http://localhost:%s", actualPort))
+
+	assert.Nil(t, err)
 
 	config.InitConfig(*cfgName, "'")
 	conf := config.Config().ViperConfig()

test/connections_integration_test.go

@@ -14,6 +14,7 @@ func TestConnectionsCRUD(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping integration test")
 	}
+
 	apiClient, _ := initTest(t, nil)
 	filtersEntity := service.NewConnectionFilter("")
 	log.Info("Exporting all connections")

test/dashboard_integration_test.go

@@ -5,8 +5,6 @@ import (
 	"github.com/esnet/gdg/internal/service"
 	"github.com/esnet/gdg/internal/service/filters"
 	"github.com/esnet/grafana-swagger-api-golang/goclient/models"
-	"gopkg.in/yaml.v3"
-	"os"
 	"strings"
 	"testing"
 
@@ -19,77 +17,6 @@ import (
 // - Add single dashboard test -d <>
 // - Add Folder dashboard test -f <>
 
-func TestDashboardAPIKeyCRUD(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test")
-	}
-
-	//TODO: convert to a token based pattern
-	apiClient, _ := initTest(t, nil)
-	testData, _ := os.ReadFile("config/testing.yml")
-	data := map[string]interface{}{}
-	err := yaml.Unmarshal(testData, &data)
-	assert.Nil(t, err)
-
-	apiClient.DeleteAllTokens() //Remove any old data
-	newKey, err := apiClient.CreateAPIKey("testing", "admin", 0)
-	assert.Nil(t, err)
-
-	wrapper := map[string]*config.GrafanaConfig{}
-	_ = wrapper
-
-	level1 := data["contexts"].(map[string]interface{})
-	level2 := level1["testing"].(map[string]interface{})
-	level2["token"] = newKey.Key
-	level2["user_name"] = ""
-	level2["password"] = ""
-
-	updatedCfg, err := yaml.Marshal(data)
-	assert.Nil(t, err)
-	newCfg := "config/tokenTest.yml"
-	err = os.WriteFile(newCfg, updatedCfg, 0644)
-	assert.Nil(t, err)
-
-	apiClient, _ = initTest(t, &newCfg)
-
-	filtersEntity := service.NewDashboardFilter("", "", "")
-	log.Info("Exporting all dashboards")
-	apiClient.UploadDashboards(filtersEntity)
-	log.Info("Listing all dashboards")
-	boards := apiClient.ListDashboards(filtersEntity)
-	log.Infof("Imported %d dashboards", len(boards))
-	ignoredSkipped := true
-	var generalBoard *models.Hit
-	var otherBoard *models.Hit
-	for ndx, board := range boards {
-		log.Infof(board.Slug)
-		if board.Slug == "latency-patterns" {
-			ignoredSkipped = false
-		}
-		if board.Slug == "individual-flows" {
-			generalBoard = boards[ndx]
-		}
-		if board.Slug == "flow-information" {
-			otherBoard = boards[ndx]
-		}
-	}
-	assert.NotNil(t, otherBoard)
-	assert.NotNil(t, generalBoard)
-	assert.True(t, ignoredSkipped)
-	validateGeneralBoard(t, generalBoard)
-	validateOtherBoard(t, otherBoard)
-	//Import Dashboards
-	log.Info("Importing Dashboards")
-	list := apiClient.DownloadDashboards(filtersEntity)
-	assert.Equal(t, len(list), len(boards))
-	log.Info("Deleting Dashboards")
-	deleteList := apiClient.DeleteAllDashboards(filtersEntity)
-	assert.Equal(t, len(deleteList), len(boards))
-	log.Info("List Dashboards again")
-	boards = apiClient.ListDashboards(filtersEntity)
-	assert.Equal(t, len(boards), 0)
-}
-
 func TestDashboardCRUD(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping integration test")

test/organizations_integration_test.go

@@ -4,6 +4,7 @@ import (
 	"github.com/esnet/gdg/internal/service"
 	"github.com/esnet/grafana-swagger-api-golang/goclient/models"
 	"golang.org/x/exp/slices"
+	"os"
 	"sort"
 	"testing"
 
@@ -14,6 +15,9 @@ func TestOrgsCrud(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping integration test")
 	}
+	if os.Getenv("TEST_TOKEN_CONFIG") == "1" {
+		t.Skip("Skipping Token configuration, Organization CRUD requires Basic Auth")
+	}
 	apiClient, _ := initTest(t, nil)
 	orgs := apiClient.ListOrganizations()
 	assert.Equal(t, len(orgs), 1)
@@ -31,6 +35,9 @@ func TestOrgUserMembership(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping integration test")
 	}
+	if os.Getenv("TEST_TOKEN_CONFIG") == "1" {
+		t.Skip("Skipping Token configuration, Organization CRUD requires Basic Auth")
+	}
 	apiClient, _ := initTest(t, nil)
 	//Create Orgs in case they aren't already present.
 	apiClient.UploadOrganizations()

test/team_integration_test.go

@@ -4,6 +4,7 @@ import (
 	"github.com/esnet/gdg/internal/service"
 	"github.com/esnet/grafana-swagger-api-golang/goclient/models"
 	"golang.org/x/exp/maps"
+	"os"
 	"testing"
 
 	log "github.com/sirupsen/logrus"
@@ -14,7 +15,9 @@ func TestTeamCRUD(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping integration test")
 	}
-
+	if os.Getenv("TEST_TOKEN_CONFIG") == "1" {
+		t.Skip("Skipping Token configuration, Team and User CRUD requires Basic Auth")
+	}
 	filter := service.NewTeamFilter("")
 	apiClient, _ := initTest(t, nil)
 	log.Info("Exporting current user list")