From 09e5db53b95ee3fa9be5a68c5dd76d2bbf00ab19 Mon Sep 17 00:00:00 2001 From: Samir F Date: Mon, 18 Mar 2024 11:20:07 -0400 Subject: [PATCH] Cleaning up gdg-template (#267) * Updating documentation * Cleaning up gdg-template ChangeLog: - Refactoring GDG template tool - Refactoring GDG CLI tool * Additional documentation prepping for the v0.6.0 release. --- Taskfile.yml | 82 ++++----- cli/version.go | 8 +- cmd/gdg-generate/cli/config.go | 31 ++++ cmd/gdg-generate/cli/root.go | 60 ++++++ cmd/gdg-generate/cli/templates.go | 70 +++++++ cmd/gdg-generate/cli/version.go | 19 ++ cmd/gdg-generate/main.go | 67 +------ go.mod | 4 +- go.sum | 3 +- internal/templating/templating.go | 16 +- internal/version/version.go | 9 + website/config/_default/params.toml | 6 +- website/content/docs/developer/_index.md | 4 + .../content/docs/developer/contributing.md | 54 ++++++ website/content/docs/developer/developer.md | 40 ++++ website/content/docs/gdg/configuration.md | 172 ++++++++++++------ website/content/docs/gdg/developer.md | 40 ---- website/content/docs/gdg/faq.md | 36 ++++ website/content/docs/gdg/getting_started.md | 20 +- website/content/docs/gdg/installation.md | 12 +- website/content/docs/reference/_index.md | 21 --- website/content/docs/reference/example.md | 25 --- website/content/docs/releases/gdg_0.6.md | 59 ++++-- website/content/docs/templating/_index.md | 4 +- .../content/docs/templating/description.md | 59 +++--- website/content/docs/tutorials/_index.md | 2 +- website/content/docs/usage_guide/_index.md | 4 + .../docs/{gdg => usage_guide}/backup_guide.md | 43 ++--- .../{gdg => usage_guide}/enterprise_guide.md | 2 +- .../docs/{gdg => usage_guide}/tools_guide.md | 0 website/content/privacy.md | 14 -- website/hugo_stats.json | 27 ++- 32 files changed, 629 insertions(+), 384 deletions(-) create mode 100644 cmd/gdg-generate/cli/config.go create mode 100644 cmd/gdg-generate/cli/root.go create mode 100644 cmd/gdg-generate/cli/templates.go create mode 100644 cmd/gdg-generate/cli/version.go create mode 100644 website/content/docs/developer/_index.md create mode 100644 website/content/docs/developer/contributing.md create mode 100644 website/content/docs/developer/developer.md delete mode 100644 website/content/docs/gdg/developer.md create mode 100644 website/content/docs/gdg/faq.md delete mode 100644 website/content/docs/reference/_index.md delete mode 100644 website/content/docs/reference/example.md create mode 100644 website/content/docs/usage_guide/_index.md rename website/content/docs/{gdg => usage_guide}/backup_guide.md (88%) rename website/content/docs/{gdg => usage_guide}/enterprise_guide.md (99%) rename website/content/docs/{gdg => usage_guide}/tools_guide.md (100%) delete mode 100644 website/content/privacy.md diff --git a/Taskfile.yml b/Taskfile.yml index fa4d05e5..02368af2 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -2,14 +2,14 @@ version: "3" vars: BIN_NAME: gdg - VERSION : { sh: grep "const Version " internal/version/version.go | sed -E 's/.*"(.+)"$$/\1/' } + VERSION: { sh: grep "const Version " internal/version/version.go | sed -E 's/.*"(.+)"$$/\1/' } GIT_COMMIT: { sh: git rev-parse HEAD } GIT_DIRTY: { sh: test -n "`git status --porcelain`" && echo "+CHANGES" || true } BUILD_DATE: { sh: date '+%Y-%m-%d-%H:%M:%S' } IMAGE_NAME: "esnet/gdg" LD_FLAGS: "-s -w -X github.com/esnet/gdg/internal/version.GitCommit={{ .GIT_COMMIT}}{{ .GIT_DIRTY}} -X github.com/esnet/gdg/internal/version.BuildDate={{ .BUILD_DATE }} " -dotenv: ['.env'] +dotenv: [ '.env' ] tasks: default: @@ -23,7 +23,7 @@ tasks: - go install github.com/securego/gosec/v2/cmd/gosec@master - go install golang.org/x/vuln/cmd/govulncheck@latest - go install github.com/vektra/mockery/v2@v2.42.0 - + - go install github.com/goreleaser/goreleaser@v1.24.0 security: desc: "Run security scan" cmds: @@ -31,54 +31,54 @@ tasks: lint: desc: "Lint project, skipping test files." cmds: - - golangci-lint run --timeout=30m --skip-dirs "(^|/)test($|/)" --skip-files "_test.go" ./... + - golangci-lint run --timeout=30m --skip-dirs "(^|/)test($|/)" --skip-files "_test.go" ./... spellcheck: desc: "Check Spelling across code" cmds: - - misspell . | grep -v "website" + - misspell . | grep -v "website" lint_tests: desc: "Lint project, including test files." cmds: - - golangci-lint run ./... + - golangci-lint run ./... authors: desc: "Building GDG" cmds: - echo "Authors\n=======\n" > AUTHORS.md - "git log --raw | grep \"^Author: \" | sort | uniq | cut -d ' ' -f2 | sed 's/^/- /' >> AUTHORS.md" silent: false - mocks: + mocks: desc: "Re-generate Mocks" cmds: - rm -fr internal/service/mocks - - mockery + - mockery linux: desc: "Build linux binary" cmds: - - env GOOS='linux' GOARCH='amd64' go build -ldflags "{{ .LD_FLAGS }}" -o bin/{{ .BIN_NAME }}_linux cmd/gdg/main.go + - env GOOS='linux' GOARCH='amd64' go build -ldflags "{{ .LD_FLAGS }}" -o bin/{{ .BIN_NAME }}_linux cmd/gdg/main.go build_all: - desc: "Buiding All binaries" + desc: "Buiding All binaries" cmds: - task: build - task: build_generate build: - desc: "Buiding {{ .BIN_NAME }} {{ .VERSION }}" + desc: "Buiding {{ .BIN_NAME }} {{ .VERSION }}" cmds: - echo "GOPATH=${GOPATH}" - - go build -ldflags "{{ .LD_FLAGS }}" -o bin/{{ .BIN_NAME }} cmd/gdg/main.go + - go build -ldflags "{{ .LD_FLAGS }}" -o bin/{{ .BIN_NAME }} cmd/gdg/main.go build_generate: - desc: "Buiding {{ .BIN_NAME }}-generate {{ .VERSION }}" + desc: "Buiding {{ .BIN_NAME }}-generate {{ .VERSION }}" cmds: - echo "GOPATH=${GOPATH}" - - go build -ldflags "{{ .LD_FLAGS }}" -o bin/{{ .BIN_NAME }}-generate cmd/gdg-generate/main.go - install: - desc: "installing {{ .BIN_NAME }} {{ .VERSION }}" - cmds: + - go build -ldflags "{{ .LD_FLAGS }}" -o bin/{{ .BIN_NAME }}-generate cmd/gdg-generate/main.go + install: + desc: "installing {{ .BIN_NAME }} {{ .VERSION }}" + cmds: - echo "GOPATH=${GOPATH}" - go install -ldflags "{{ .LD_FLAGS}}" cmd/gdg/main.go silent: false - push: + push: desc: "Pushing docker image to registry: latest {{ .VERSION }} {{ .GIT_COMMIT }}" - deps: [tag] + deps: [ tag ] cmds: - docker push $(IMAGE_NAME):{{ .GIT_COMMIT }} - docker push $(IMAGE_NAME):{{ .VERSION }} @@ -86,37 +86,37 @@ tasks: clean: desc: "clean up data" cmds: - - "test ! -e bin/{{ .BIN_NAME }} || rm bin/{{ .BIN_NAME }}" + - "test ! -e bin/{{ .BIN_NAME }} || rm bin/{{ .BIN_NAME }}" - "rm -fr dist/" - release-snapshot: - deps: [clean] + release-snapshot: + deps: [ clean ] desc: "Release Snapshot" cmds: - goreleaser build --snapshot - release: - deps: [clean] + release: + deps: [ clean ] desc: "TEST Release of GDG, no validation, no publish" cmds: - goreleaser release --skip=publish,validate test: - desc: "test check" - cmds: - - go test -v -coverpkg=./... -covermode=atomic -coverprofile=coverage.out ./... - - go tool cover -html=coverage.out - env: - GRAFANA_INTEGRATION: "1" - TEST_TOKEN_CONFIG: "0" + desc: "test check" + cmds: + - go test -v -coverpkg=./... -covermode=atomic -coverprofile=coverage.out ./... + - go tool cover -html=coverage.out + env: + GRAFANA_INTEGRATION: "1" + TEST_TOKEN_CONFIG: "0" test_tokens: - desc: "test Token Based Only" - cmds: - - go test -v -coverpkg=./... -covermode=atomic -coverprofile=coverage.out ./... - - go tool cover -html=coverage.out - env: - GRAFANA_INTEGRATION: "1" - TEST_TOKEN_CONFIG: "1" + desc: "test Token Based Only" + cmds: + - go test -v -coverpkg=./... -covermode=atomic -coverprofile=coverage.out ./... + - go tool cover -html=coverage.out + env: + GRAFANA_INTEGRATION: "1" + TEST_TOKEN_CONFIG: "1" vuln_check: - desc: "Vulnerability check" - cmds: - - govulncheck ./... + desc: "Vulnerability check" + cmds: + - govulncheck ./... diff --git a/cli/version.go b/cli/version.go index f059ccda..0412d140 100644 --- a/cli/version.go +++ b/cli/version.go @@ -2,23 +2,17 @@ package cli import ( "context" - "fmt" "github.com/bep/simplecobra" "github.com/esnet/gdg/cli/support" "github.com/esnet/gdg/internal/version" "github.com/spf13/cobra" - "log/slog" ) func newVersionCmd() simplecobra.Commander { return &support.SimpleCommand{ NameP: "version", RunFunc: func(ctx context.Context, cd *simplecobra.Commandeer, r *support.RootCommand, args []string) error { - slog.Info(fmt.Sprintf("Build Date: %s", version.BuildDate)) - slog.Info(fmt.Sprintf("Git Commit: %s", version.GitCommit)) - slog.Info(fmt.Sprintf("Version: %s", version.Version)) - slog.Info(fmt.Sprintf("Go Version: %s", version.GoVersion)) - slog.Info(fmt.Sprintf("OS / Arch: %s", version.OsArch)) + version.PrintVersionInfo() return nil }, WithCFunc: func(cmd *cobra.Command, r *support.RootCommand) { diff --git a/cmd/gdg-generate/cli/config.go b/cmd/gdg-generate/cli/config.go new file mode 100644 index 00000000..5658cc15 --- /dev/null +++ b/cmd/gdg-generate/cli/config.go @@ -0,0 +1,31 @@ +package cli + +import ( + "fmt" + "github.com/esnet/gdg/internal/config" + "github.com/spf13/cobra" + "gopkg.in/yaml.v3" + "log" + "log/slog" +) + +var showConfigCmd = &cobra.Command{ + Use: "config", + Short: "Show current templates configuration", + Long: `Show current templates configuration`, + Aliases: []string{"cfg"}, + Run: func(cmd *cobra.Command, args []string) { + data, err := yaml.Marshal(config.Config().GetTemplateConfig()) + if err != nil { + log.Fatalf("unable to load template configuration: %v", err) + } + slog.Info("Configuration", + slog.String("template-config", tplCfgFile), + slog.String("gdg-config", cfgFile)) + fmt.Println(string(data)) + }, +} + +func init() { + rootCmd.AddCommand(showConfigCmd) +} diff --git a/cmd/gdg-generate/cli/root.go b/cmd/gdg-generate/cli/root.go new file mode 100644 index 00000000..207719ab --- /dev/null +++ b/cmd/gdg-generate/cli/root.go @@ -0,0 +1,60 @@ +package cli + +import ( + assets "github.com/esnet/gdg/config" + "github.com/esnet/gdg/internal/config" + appconfig "github.com/esnet/gdg/internal/log" + "github.com/esnet/gdg/internal/templating" + "github.com/spf13/cobra" + "log" + "log/slog" + "os" +) + +var ( + cfgFile string + tplCfgFile string + template templating.Templating + rootCmd = &cobra.Command{ + Use: "gdg-generate", + Short: "Generates dashboard templates for use with GDG given a valid configuration", + Long: `Generates dashboard templates for use with GDG given a valid configuration`, + } +) + +func init() { + cobra.OnInitialize(initConfig) + rootCmd.CompletionOptions.DisableDefaultCmd = true + + rootCmd.PersistentFlags().StringP("config", "c", "", "config file (default: config/importer.yml)") + rootCmd.PersistentFlags().StringP("template-config", "", "", "GDG Template configuration file override. (default: config/templates.yml)") +} + +func initConfig() { + var err error + cfgFile, err = rootCmd.Flags().GetString("config") + if err != nil { + log.Fatal("unable to get config file") + } + tplCfgFile, err = rootCmd.Flags().GetString("template-config") + if err != nil { + log.Fatal("unable to get template config file") + } + + defaultConfiguration, err := assets.GetFile("importer-example.yml") + if err != nil { + slog.Warn("unable to load default configuration, no fallback") + } + + config.InitGdgConfig(cfgFile, defaultConfiguration) + config.InitTemplateConfig(tplCfgFile) + cfg := config.Config() + appconfig.InitializeAppLogger(os.Stdout, os.Stderr, cfg.IsDebug()) + template = templating.NewTemplate() +} + +func Execute() { + if err := rootCmd.Execute(); err != nil { + log.Fatal(err.Error()) + } +} diff --git a/cmd/gdg-generate/cli/templates.go b/cmd/gdg-generate/cli/templates.go new file mode 100644 index 00000000..1d08c700 --- /dev/null +++ b/cmd/gdg-generate/cli/templates.go @@ -0,0 +1,70 @@ +package cli + +import ( + "fmt" + "github.com/jedib0t/go-pretty/v6/table" + "github.com/spf13/cobra" + "log" + "log/slog" + "os" +) + +var tplCmd = &cobra.Command{ + Use: "template", + Aliases: []string{"tpl", "templates"}, + Short: "Templating Utilities", + Long: `Templating Utilities`, +} + +var listTemplatesCmd = &cobra.Command{ + Use: "list", + Short: "List current templates", + Long: `List current templates`, + Run: func(cmd *cobra.Command, args []string) { + templates := template.ListTemplates() + slog.Info("Available templates for current configuration", + slog.String("template-config", tplCfgFile), + slog.String("gdg-config", cfgFile)) + for ndx, t := range templates { + slog.Info(fmt.Sprintf("%d: %s", ndx+1, t)) + } + }, +} + +var generateTemplatesCmd = &cobra.Command{ + Use: "generate", + Aliases: []string{}, + Short: "Generate current templates", + Long: `Generate current templates`, + Run: func(cmd *cobra.Command, args []string) { + templateFilter, _ := cmd.Flags().GetString("template") + payload, err := template.Generate(templateFilter) + if err != nil { + log.Fatal("Failed to generate templates", slog.Any("err", err)) + } + + tableObj := table.NewWriter() + tableObj.SetOutputMirror(os.Stdout) + tableObj.SetStyle(table.StyleLight) + + tableObj.AppendHeader(table.Row{"Template Name", "Output"}) + count := 0 + for key, val := range payload { + count += len(val) + for _, file := range val { + tableObj.AppendRow(table.Row{key, file}) + } + } + slog.Info("Generate dashboards for the given templates", + slog.Any("template-count", len(payload)), + slog.Any("dashboard-count", count)) + tableObj.Render() + }, +} + +func init() { + rootCmd.AddCommand(tplCmd) + tplCmd.AddCommand(listTemplatesCmd) + tplCmd.AddCommand(generateTemplatesCmd) + generateTemplatesCmd.PersistentFlags().StringP("template", "t", "", "Specify template name, optional. Default is to operate on all configured templates that are found.") +} diff --git a/cmd/gdg-generate/cli/version.go b/cmd/gdg-generate/cli/version.go new file mode 100644 index 00000000..b11b0340 --- /dev/null +++ b/cmd/gdg-generate/cli/version.go @@ -0,0 +1,19 @@ +package cli + +import ( + "github.com/esnet/gdg/internal/version" + "github.com/spf13/cobra" +) + +var versionCmd = &cobra.Command{ + Use: "version", + Short: "Print the version number of gdg-generate", + Long: `Print the version number of gdg-generate`, + Run: func(cmd *cobra.Command, args []string) { + version.PrintVersionInfo() + }, +} + +func init() { + rootCmd.AddCommand(versionCmd) +} diff --git a/cmd/gdg-generate/main.go b/cmd/gdg-generate/main.go index 653cf28c..99dba1dd 100644 --- a/cmd/gdg-generate/main.go +++ b/cmd/gdg-generate/main.go @@ -1,72 +1,9 @@ package main import ( - "fmt" - assets "github.com/esnet/gdg/config" - "github.com/esnet/gdg/internal/config" - appconfig "github.com/esnet/gdg/internal/log" - "github.com/esnet/gdg/internal/templating" - "github.com/jedib0t/go-pretty/v6/table" - flag "github.com/spf13/pflag" - "gopkg.in/yaml.v3" - "log" - "log/slog" - "os" + "github.com/esnet/gdg/cmd/gdg-generate/cli" ) func main() { - //Using pflag over corba for now, as this should be a simple enough CLI tool - var cfgName = flag.StringP("config", "c", "importer.yml", "GDG Configuration file override.") - var tmpCfgName = flag.StringP("ct", "", "templates.yml", "GDG Template configuration file override.") - var showTemplateCfg = flag.BoolP("show-config", "", false, "Will display the current template configuration") - var listTemplates = flag.BoolP("list-templates", "", false, "List all current templates") - var templateName = flag.StringP("template", "t", "", "Specify template name, optional. Default is to operate on all configured templates that are found.") - flag.Parse() - defaultConfiguration, err := assets.GetFile("importer-example.yml") - if err != nil { - slog.Warn("unable to load default configuration, no fallback") - } - - config.InitGdgConfig(*cfgName, defaultConfiguration) - config.InitTemplateConfig(*tmpCfgName) - cfg := config.Config() - appconfig.InitializeAppLogger(os.Stdout, os.Stderr, cfg.IsDebug()) - - if *showTemplateCfg { - data, err := yaml.Marshal(cfg.GetTemplateConfig()) - if err != nil { - log.Fatalf("unable to load template configuration: %v", err) - } - slog.Info(fmt.Sprintf("Configuration\n%s", string(data))) - return - } - slog.Info("Context is set to: ", slog.String("context", cfg.GetGDGConfig().ContextName)) - template := templating.NewTemplate() - - if *listTemplates { - templates := template.ListTemplates() - for ndx, t := range templates { - slog.Info(fmt.Sprintf("%d: %s", ndx+1, t)) - } - - return - } - - payload, err := template.Generate(*templateName) - if err != nil { - log.Fatal("Failed to generate templates", slog.Any("err", err)) - } - - tableObj := table.NewWriter() - tableObj.SetOutputMirror(os.Stdout) - tableObj.SetStyle(table.StyleLight) - - tableObj.AppendHeader(table.Row{"Template Name", "Output"}) - for key, val := range payload { - for _, file := range val { - tableObj.AppendRow(table.Row{key, file}) - } - } - - tableObj.Render() + cli.Execute() } diff --git a/go.mod b/go.mod index dc05d655..32bf5c52 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,6 @@ require ( github.com/samber/lo v1.39.0 github.com/sethvargo/go-password v0.2.0 github.com/spf13/cobra v1.8.0 - github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.18.2 github.com/stretchr/testify v1.8.4 github.com/testcontainers/testcontainers-go v0.26.0 @@ -49,7 +48,7 @@ require ( github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect - github.com/BurntSushi/toml v0.3.1 // indirect + github.com/BurntSushi/toml v1.3.2 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver/v3 v3.2.1 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect @@ -160,6 +159,7 @@ require ( github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/objx v0.5.1 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/tidwall/match v1.1.1 // indirect diff --git a/go.sum b/go.sum index 002404d4..b30c3a00 100644 --- a/go.sum +++ b/go.sum @@ -33,8 +33,9 @@ github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+X github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 h1:DzHpqpoJVaCgOUdVHxE8QB52S6NiVdDQvGlny1qvPqA= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= +github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= diff --git a/internal/templating/templating.go b/internal/templating/templating.go index f542ef06..0f16549a 100644 --- a/internal/templating/templating.go +++ b/internal/templating/templating.go @@ -54,15 +54,14 @@ func (t *templateImpl) ListTemplates() []string { func (t *templateImpl) Generate(templateName string) (map[string][]string, error) { result := make(map[string][]string) //Remove extension if included - templateName = strings.ReplaceAll(templateName, ".go.tmpl", "") cfg := config.Config() var entities []config.TemplateDashboards - entities = cfg.GetTemplateConfig().Entities.Dashboards - if templateName != "" { - entity, ok := cfg.GetTemplateConfig().GetTemplate(templateName) - if ok { - entities = append(entities, *entity) - } + templateName = strings.ReplaceAll(templateName, ".go.tmpl", "") + templateEntity, ok := cfg.GetTemplateConfig().GetTemplate(templateName) + if ok { + entities = append(entities, *templateEntity) + } else { + entities = cfg.GetTemplateConfig().Entities.Dashboards } for _, entity := range entities { result[entity.TemplateName] = make([]string, 0) @@ -93,7 +92,8 @@ func (t *templateImpl) Generate(templateName string) (map[string][]string, error //Merge two maps. tmpl, err := template.New("").Funcs(fns).Parse(string(templateData)) if err != nil { - slog.Error("unable to parse template") + slog.Error("unable to parse template", slog.Any("err", err)) + continue } //Create new file. diff --git a/internal/version/version.go b/internal/version/version.go index f20a5af2..6f31bf00 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -2,6 +2,7 @@ package version import ( "fmt" + "log/slog" "runtime" ) @@ -19,3 +20,11 @@ var GoVersion = runtime.Version() // OsArch returns the os and arch used to build the binary var OsArch = fmt.Sprintf("%s %s", runtime.GOOS, runtime.GOARCH) + +func PrintVersionInfo() { + slog.Info(fmt.Sprintf("Build Date: %s", BuildDate)) + slog.Info(fmt.Sprintf("Git Commit: %s", GitCommit)) + slog.Info(fmt.Sprintf("Version: %s", Version)) + slog.Info(fmt.Sprintf("Go Version: %s", GoVersion)) + slog.Info(fmt.Sprintf("OS / Arch: %s", OsArch)) +} diff --git a/website/config/_default/params.toml b/website/config/_default/params.toml index 76c50f1f..78c4f345 100644 --- a/website/config/_default/params.toml +++ b/website/config/_default/params.toml @@ -20,9 +20,9 @@ mainSections = ["docs"] containerBreakpoint = "lg" # "", "sm", "md", "lg" (default), "xl", "xxl", or "fluid" ## Button - navBarButton = true # false (default) or true - navBarButtonUrl = "/docs/prologue/introduction/" - navBarButtonText = "Get started" + navBarButton = false # false (default) or true + navBarButtonUrl = "/gdg/docs/gdg/installation/" + navBarButtonText = "Getting started" # FlexSearch flexSearch = true # true (default) or false diff --git a/website/content/docs/developer/_index.md b/website/content/docs/developer/_index.md new file mode 100644 index 00000000..0fb3dfb6 --- /dev/null +++ b/website/content/docs/developer/_index.md @@ -0,0 +1,4 @@ +--- +title: "Developer Guide" +weight: 6 +--- diff --git a/website/content/docs/developer/contributing.md b/website/content/docs/developer/contributing.md new file mode 100644 index 00000000..2ad4266b --- /dev/null +++ b/website/content/docs/developer/contributing.md @@ -0,0 +1,54 @@ +--- +title: "Contributing" +weight: 2 +--- + +First of all, gdg contains two binaries. `gdg` which manages grafana entities and provide some tooling facilities +and `gdg-generate` a tool to help generate dashboards from templates to allow for some more flexible management. +gdg-generate is fairly new, but the same quality of PRs would be nice to have. + +## GDG + +There two types of features that get added to gdg, a `tool` or `backup` feature. + +### Backup Feature + +A backup feature is some entity that you want to add to GDG that it will track. + +For example, if you want to add support for managing a playlist. + +Typically, we want to be able to: + +1. List all entities in the current namespace (or Org) +2. Download all entities +3. Upload all entities + +In order to add a new feature you will need to: + +0. Create an issue to track this work and explain the feature being added/requested. +1. Create a new CLI subcommand for playlist. under the backup command. +2. Extend the service to be able to list/download/upload etc the entities. +3. Write a unit test for the given entities. If need be add some seed data under test/data/ +4. Update the documentation accordingly to reflect the new changes. All docs live under + the [website/content](https://github.com/esnet/gdg/tree/master/website/content/docs). All files are in markdown. If + you wish you can load the website locally by running: `npm install && hugo serve` + +#### Testing + +There are multiple types of tests. + +1. `Integration tests`. The most common one, an instance of grafana will be available and will connect to it to create/update/delete entities. +2. Unit tests can be created for any unit of work, and will always be executed. +3. CLI tooling tests. use `task mocks` to generate mocks for your tests. The purpose of this test is to validate the CLI parsing behavior rather than the service functionality. Ensure that filtering works, stdout is redirected to the test and can validate the output matches the expectations. + +### Tools Feature + +This area is more about managing grafana entities. The tools would provide way for creating service accounts, listing tokens, adding user to orgs, etc. + +Testing such features can be a bit more difficult, but we can see enough data to validate the behavior that's always great and makes for a much more stable feature long term. + +### Enterprise features + +{{< callout context="caution" title="Caution" icon="alert-triangle" >}} +There are a few enterprise features that GDG supports, but unfortunately as there is no enterprise version we can access in CICD testing is very limited. +{{< /callout >}} diff --git a/website/content/docs/developer/developer.md b/website/content/docs/developer/developer.md new file mode 100644 index 00000000..c3eae49f --- /dev/null +++ b/website/content/docs/developer/developer.md @@ -0,0 +1,40 @@ +--- +title: "Developer Guide" +weight: 1 +--- + +## Dependencies + +Development requirements. + - As gdg is written in [go](https://go.dev/), a current compiler is required. + - task management tool [go-task](https://github.com/go-task/task). [Installation](https://taskfile.dev/installation/) + - [Docker](https://www.docker.com/products/docker-desktop/) for running tests + +Install the remaining dependencies via: `task install_tools` + +## Building/Running gdg + +Running it then should be as simple as: + +```bash +$ task build_all +$ ./bin/gdg ## main binary +$ ./bin/gdg-generate ## Dashboard Templating engine +``` + +Requires [task](https://github.com/go-task/task.git) to be installed locally + +## Running Tests + +BasicAuth: + `task test` +Token: + `task test_tokens` + +## Making a release + +You can validate that goreleaser work appropriately via + +`task release-snapshot` or `task release` + +The actual release will be done once a tag has been created via the CICD pipeline, artifacts will be generated and a website update will be published. diff --git a/website/content/docs/gdg/configuration.md b/website/content/docs/gdg/configuration.md index d1958813..a0fe86d9 100644 --- a/website/content/docs/gdg/configuration.md +++ b/website/content/docs/gdg/configuration.md @@ -2,30 +2,30 @@ title: "Configuration" weight: 14 --- -## Getting started -This project requires Go to be installed. On OS X with Homebrew you can just run `brew install go`. +## Configuration +make a copy of [config/importer-example.yml](https://github.com/esnet/gdg/blob/master/config/importer-example.yml) and +name it `config/importer.yml` or simply run `gdg tools ctx new ` which will walk you through setting up a new +context to use with your grafana installation. +## Authentication +### Authentication Token - -make a copy of [config/importer-example.yml](https://github.com/esnet/gdg/blob/master/config/importer-example.yml) and name it `config/importer.yml` You'll need GRAFANA ADMINISTRATIVE privileges to proceed. - - -### Authentication - -#### Authentication Token - -You can use an Authentication Token / API Key to authenticate with the Grafana API, which can be generated in your Grafana Web UI => Configuration => API Keys. You can then use it in your configuration file (eg. `importer.yml`). +You can use an Authentication Token / API Key to authenticate with the Grafana API, which can be generated in your +Grafana Web UI => Configuration => API Keys. You can then use it in your configuration file (eg. `importer.yml`). {{< callout context="caution" title="Caution" icon="alert-triangle" >}} -gdg is currently using viper to read in the config. Since viper makes all keys lowercase, we also have the same limitation. Camel case will be read in but be aware that a section named fooBar == Foobar == foobar etc. +gdg is currently using viper to read in the config. Since viper makes all keys lowercase, we also have the same +limitation. Camel case will be read in but be aware that a section named fooBar == Foobar == foobar etc. {{< /callout >}} -#### Service Accounts +### Service Accounts -Service Accounts are supported and interchangeable for tokens. If you wish to use a service account, simply put the token value from the service account for `token:`. Please make sure you've granted the account the needed permissions for the operations you are trying to perform. +Service Accounts are supported and interchangeable for tokens. If you wish to use a service account, simply put the +token value from the service account for `token:`. Please make sure you've granted the account the needed permissions +for the operations you are trying to perform. ```yaml context_name: main @@ -59,9 +59,11 @@ global: ignore_ssl_errors: false ``` -#### Username / Password +### Username / Password + +You can also use username/password credentials of an admin user to authenticate with the Grafana API. You can specify +them in your configuration file (eg. `importer.yml`). -You can also use username/password credentials of an admin user to authenticate with the Grafana API. You can specify them in your configuration file (eg. `importer.yml`). ``` context_name: main @@ -78,19 +80,24 @@ global: debug: true ignore_ssl_errors: false ``` +## Cloud Configuration + +Several S3 compatible cloud providers are supported. Please see this [section](https://software.es.net/gdg/docs/gdg/cloud_configuration/) for more detailed instructions. -### Connection +## Connection -#### Connection Credentials +### Connection Credentials + +#### Current Behavior (Version +0.5.2) -##### Current Behavior (Version +0.5.2) If the connection has BasicAuth enabled, then we'll attempt to set the auth with the following rules. We will try to find a match given the rules specified: - - field: matches any valid gjson path and retrieves it's value. ie. `jsonData.maxConcurrentShardRequests` and validates it against a golang supported [Regex](https://github.com/google/re2/wiki/Syntax). - - It goes down the list of rules and returns the auth for the first matching one. The rules should be listed from more specific to more broad. The default rule ideally should be at the end. - +- `field`: matches any valid json path and retrieves its value. ie. `jsonData.maxConcurrentShardRequests` and validates + it against a golang supported [Regex](https://github.com/google/re2/wiki/Syntax). +- It goes down the list of rules and returns the auth for the first matching one. The rules should be listed from more + specific to more broad. The default rule ideally should be at the end. ```yaml testing: @@ -112,19 +119,20 @@ We will try to find a match given the rules specified: - Other ``` -the secure_data will read the file from {output_path}/secure/. It will then use that + +the secure_data will read the file from {output_path}/secure/. It will then use that information to construct the datasource. Default setting if you use basic auth is shown below. ```json { - "basicAuthPassword": "password", - "user": "user" + "basicAuthPassword": "password", + "user": "user" } ``` -##### Version v0.4.2-v0.5.1 +#### Version v0.4.2-v0.5.1 {{< details "Legacy Behavior " >}} Preview behavior did not support the use of a secure/secureData.json pattern, instead an auth: codeblock was used. @@ -154,11 +162,9 @@ Example can be seen below: ``` - - {{< /details >}} -##### Version Prior to v0.4.2 +#### Version Prior to v0.4.2 {{< details "Legacy Behavior " >}} @@ -191,20 +197,23 @@ An example of a configuration can be seen below - Other ``` + {{< /details >}} -#### Connection Filters +### Connection Filters #### Current Behavior (+v0.4.2) -You can filter based on any field and have it be an exclusive (default) or inclusive (ie Only allow values that match) to be listed/imported etc. +You can filter based on any field and have it be an exclusive (default) or inclusive (ie Only allow values that match) +to be listed/imported etc. Pattern matching is the same as the Credentials mapping. - - field represents any valid JSON Path - - regex: any valid [regex](https://github.com/google/re2/wiki/Syntax) supported by golang +- field represents any valid JSON Path +- regex: any valid [regex](https://github.com/google/re2/wiki/Syntax) supported by golang -The example below will exclude any connections named "Google Sheets". It will also only include connections with the type elasticsearch or mysql +The example below will exclude any connections named "Google Sheets". It will also only include connections with the +type elasticsearch or mysql ```yaml contexts: @@ -220,8 +229,10 @@ contexts: ``` #### Legacy Behavior + {{< details "Legacy Behavior " >}} -This feature allows you to exclude connection by name or include them by type. Please note that the logic switches based on the data type. +This feature allows you to exclude connection by name or include them by type. Please note that the logic switches based +on the data type. **name filter:** @@ -247,49 +258,94 @@ datasources: The snippet above will ONLY import connections for elasticsearch +{{< /details >}} +{{< callout context="note" title="Note" icon="info-circle" >}} +If you configure both, an auth `Token` and `BasicAuth`, then the Token is given priority. +Watched folders under grafana is a white list of folders that are being managed by gdg. By default, only "General" is managed. -{{< /details >}} +{{< /callout >}} -#### Notes +## Organization -If you configure both, Auth Token and Username/Password, then the Token is given priority. -Watched folders under grafana is a white list of folders that are being managed by the tool. By default only "General" is managed. +The organization is set for a given context via the `orgnization_name`. If the org is not set, gdg will fallback on the default value that grafana starts out with `Main Org.` -env.output defines where the files will be saved and imported from. +Additionally, if there is an organization specific behavior, it can be added to a context by adding the following config to your config: -### Global Flags +```yaml + watched_folders_override: + - organization_name: "Special Org" + folders: + - General + - SpecialFolder +``` -`globals.debug` when set will print a more verbose output (Development In Progress) -`globals.ignore_ssl_errors` when set will disregard any SSL errors and proceed as expected +In this case, watched_folder is ignored in favor of the newly provided list. -### Environment Overrides -If you wish to override certain value via the environment, like credentials and such you can do so. +## Users -The pattern for GDG's is as follows: `GDG_SECTION__SECTION__keyname` +Users can be imported/exported but the behavior is a bit limited. We cannot retrieve the credentials of the given user. If the users are uploaded, then any user uploaded will have a new password set. The default behavior is set password to the sha256 of the login.json -For example if I want to set the context name to a different value I can use: +Example: ```sh -GDG_CONTEXT_NAME="testing" gdg tools ctx show ## Which will override the value from the context file. -GDG_CONTEXTS__TESTING__URL="www.google.com" Will override the URL with the one provided. - ``` +echo -n admin.json | openssl sha256 +> SHA2-256(stdin)= f172318957c89be30c2c54abcebb778a86246bbad2325d7133c4dc605319f72b +``` +As this can be a security risk if an intruder knows the algorithm, an option to generate random passwords is also available. This can be configured for any `context` -**NOTE:** Complex data type are not supported, so if the value is an array it can't be currently set. +```yaml + user: + random_password: true + min_length: 8 + max_length: 20 +``` +The downside, is naturally this is a one time operation. Once the password is set it once again can no longer be retrieved. The only time the password is printed is after the successful upload of all users. +## Global Flags +These are flags that apply across all contexts. They are top level configuration and are used to drive gdg's application +behavior. -### Building/Running the app +Here are the currently supported flags you may configure. -Running it then should be as simple as: +```yaml +global: + debug: true + ignore_ssl_errors: false ##when set to true will ignore invalid SSL errors + retry_count: 3 ## Will retry any failed API request up to 3 times. + retry_delay: 5s ## will wait for specified duration before trying again. -```bash -$ task build_all -$ ./bin/gdg ## main binary -$ ./bin/gdg-generate ## Dashboard Templating engine ``` -Requires [task](https://github.com/go-task/task.git) to be installed locally +- `debug`: when set will print a more verbose output (Development In Progress). Setting the env flag of DEBUG=1 will + also generate verbose output for all HTTP calls. +- `ignore_ssl_errors`: when set will disregard any SSL errors and proceed as expected +- `retry_count`: will try N number of times before giving up +- `retry_delay`: a duration to wait before trying again. Be careful with this setting, if it's too short, the retry + won't matter, if too long the app will be very slow. + +## Environment Overrides + +{{< callout context="caution" title="Caution" icon="alert-triangle" >}} +Complex data type is not supported. If the value is an array it can't be currently set, via ENV overrides. + +{{< /callout >}} + +If you wish to override certain value via the environment, like credentials and such you can do so. + +The pattern for GDG's is as follows: `GDG_SECTION__SECTION__keyname` + +For example if I want to set the context name to a different value I can use: + +```sh +GDG_CONTEXT_NAME="testing" gdg tools ctx show ## Which will override the value from the context file. +GDG_CONTEXTS__TESTING__URL="www.google.com" Will override the URL with the one provided. + ``` + + + + diff --git a/website/content/docs/gdg/developer.md b/website/content/docs/gdg/developer.md deleted file mode 100644 index 5dc40c9a..00000000 --- a/website/content/docs/gdg/developer.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -title: "Developer Guide" -weight: 19 ---- -## Running Tests -1. Bring up a grafana instance locally with default credentials of admin/admin. `docker-compose up -d grafana` -2. Once the instance is up simply run `go test ./...` or `make test` - -## Making a release - -Install goreleaser. - -```sh -brew install goreleaser/tap/goreleaser -brew reinstall goreleaser` -``` - -Alternatively if you have a more recent version of Go. - -```sh -go install github.com/goreleaser/goreleaser@latest -``` - -export your GITHUB_TOKEN. - -```sh -export GITHUB_TOKEN="secret" -``` - -git tag v0.1.0 -goreleaser release - - -NOTE: CI/CD pipeline should do all this automatically. `make release-snapshot` is used to test the release build process. Once a build is tagged all artifacts should be built automatically and attached to the github release page. - -NOTE: mac binary are not signed so will likely complain. - - - - diff --git a/website/content/docs/gdg/faq.md b/website/content/docs/gdg/faq.md new file mode 100644 index 00000000..db703cf8 --- /dev/null +++ b/website/content/docs/gdg/faq.md @@ -0,0 +1,36 @@ +--- +title: "Frequently Asked Questions" +weight: 1 +--- + +### Can I use GDG to backup grafana? + +yes and no. GDG is not a full backup solution. If you are migrating from one server to another, doing disaster recovery gdg is not the right tool for the job. Grafana already has some really nice [guides](https://grafana.com/docs/grafana/latest/administration/back-up-grafana/) for how to do that. + +GDG is used to backup and recreate specific entities. Usually it is used to allow for a consistent way to move say a dashboard, connections, alerts etc from one installation of grafana to the next. + +The typical use case can be to manage release cycles. Example: + +Create dashboard on dev, test everything out, pull into on a feature branch, deploy to staging environment. Validate everything looks good and promote it to production. + +### Why does GDG not list all my dashboards? + +By default, GDG works on a select list of `watched` or monitored folders. If none are specified, it will limit itself to only operating on `General`. + +The folders that it DOES monitor, it is assumed that gdg has full control over. Meaning, anything in those folders is under its pervue, so it may delete all connections and replace them with the ones in its exports. As far as it knows, nobody else should be writing to it. The upload operation implies that you want to sync the export data with the data found in the backup local, cloud, or otherwise. + +PS. if you want to list/import etc all dashboards, you can set the following config for your context. + +```yaml + filter_override: + ignore_dashboard_filters: false # When set to true all Watched filtered folders will be ignored and ALL folders will be acted on + +``` + +### I need feature X, can you please add that in? + +Maybe? If there's enough cycles, it could benefit others, and the feature makes sense I'd be happy to. It is also an OSS project, so contributions are always appreciated and welcome. See [contributing](https://software.es.net/gdg/docs/developer/contributing/) for more info. + +### I need help, where do I go? + +There is a "Discussion" area on github where you can start a [conversation](https://github.com/esnet/gdg/discussions) and ask questions. If you think this is a bug in GDG itself, then please file an issue [here](https://github.com/esnet/gdg/issues). There is a small slack on channel titled `#gdg` in the grafana slack, you're free to join [here](https://slack.grafana.com/) diff --git a/website/content/docs/gdg/getting_started.md b/website/content/docs/gdg/getting_started.md index a5bcd9ce..806fc2e4 100644 --- a/website/content/docs/gdg/getting_started.md +++ b/website/content/docs/gdg/getting_started.md @@ -19,7 +19,8 @@ When creating a new context, you will be asked for authorization type, your defa ### Import / Download Dashboards Minimal configuration (eg. the `importer.yml` file) that you need to download your dashboards from your Grafana endpoint: -``` + +```yaml context_name: all contexts: @@ -63,7 +64,7 @@ After executing `./bin/gdg dash import` you can find the dashboards of the `Infr ### Export / Upload Dashboards Minimal configuration (eg. the `importer.yml` file) that you need to upload your dashboards from your Grafana endpoint: -``` +```yaml context_name: all contexts: @@ -86,14 +87,15 @@ You need to adjust three parts in the configuration in order to function: - API Key OR Username / Passoword for Admin user. See [authentication](configuration.md) section if you need more information. - Uploaded Folders: The `watched` field defines folders which will be considered for manipulation. The dashboards should be stored as JSON files in the `$OUTPUT_PATH/dashboards/$GRAFANA_FOLDER_NAME` directory. Where `$OUTPUT_PATH` is the path defined in the `dashboard_output` configuration property and `$GRAFANA_FOLDER_NAME` the name of the folder to which the dashboards will be uploaded. In case of the above configuration file, the dashboards should be stored locally in the `dashboards/dashboards/Example` and `dashboards/dashboards/Infrastructure/` directories. -``` +```sh ├── bin | └── gdg └── exports - └── dashboards - ├── Example - | └── cluster-scaling.json - └── Infrastructure - └── aws-ecs.json + └── org_main-org + | └── dashboards + | └─ Example + | | └── cluster-scaling.json + | └─ Infrastructure + | └── aws-ecs.json ``` -You can execute `./bin/gdg dash export` to upload the local dashboards to your Grafana. Afterwards, you can try running `./bin/gdg dash list` in order to confirm that your dashboards were uploaded successfully. +You can execute `./bin/gdg backup dash export` to upload the local dashboards to your Grafana. Afterwards, you can try running `./bin/gdg dash list` in order to confirm that your dashboards were uploaded successfully. diff --git a/website/content/docs/gdg/installation.md b/website/content/docs/gdg/installation.md index 19bbc267..1dad59c2 100644 --- a/website/content/docs/gdg/installation.md +++ b/website/content/docs/gdg/installation.md @@ -17,8 +17,8 @@ The following packages are currently supported: Install from package involves downloading the appropriate package from the [release](https://github.com/esnet/gdg/releases) and installing it as you usually do on your favorite Distro. ```sh -rpm -Uvh ./gdg_0.5.3_amd64.rpm -dpkg -i ./gdg_0.5.3_amd64.deb +rpm -Uvh ./gdg_0.6.0_amd64.rpm +dpkg -i ./gdg_0.6.0_amd64.deb ``` ### Homebrew Installation @@ -43,7 +43,7 @@ The docker tags are released started with 0.3.1. Each release will generate a m You can see the available images [here](https://github.com/esnet/gdg/pkgs/container/gdg) ```sh -docker pull ghcr.io/esnet/gdg:0.5.3 +docker pull ghcr.io/esnet/gdg:0.6.0 ``` NOTE: ghcr.io/esnet/gdg:0.3 will also point to 0.3.1 until 0.3.2 is released after which it'll point to 0.3.2 @@ -54,7 +54,7 @@ Example compose. version: '3.7' services: gdg: - image: ghcr.io/esnet/gdg:0.5.3 + image: ghcr.io/esnet/gdg:0.6.0 command: "--help" ## Add additional parameters here # command: "ds export" ## Pass any cmd on here. volumes: @@ -80,7 +80,7 @@ If you have go install you may run the following command to install gdg. Keep i **gdg** ```sh go install github.com/esnet/gdg/cmd/gdg@latest #for latest -go install github.com/esnet/gdg/cmd/gdg@v0.5.3 #for a specific version +go install github.com/esnet/gdg/cmd/gdg@v0.6.0 #for a specific version ``` You can verify the version by running `gdg version`. @@ -88,7 +88,7 @@ You can verify the version by running `gdg version`. **gdg-generate** ```sh go install github.com/esnet/gdg/cmd/gdg-generate@latest #for latest -go install github.com/esnet/gdg/cmd/gdg-generate@v0.5.3 #for a specific version +go install github.com/esnet/gdg/cmd/gdg-generate@v0.6.0 #for a specific version ``` --- diff --git a/website/content/docs/reference/_index.md b/website/content/docs/reference/_index.md deleted file mode 100644 index a6ccc22f..00000000 --- a/website/content/docs/reference/_index.md +++ /dev/null @@ -1,21 +0,0 @@ ---- -title: "Reference" -description: "" -summary: "" -date: 2023-09-07T16:12:37+02:00 -lastmod: 2023-09-07T16:12:37+02:00 -draft: true -menu: - docs: - parent: "" - identifier: "reference-22e9ba8aefa7ef9891199cf8db3a08cd" -weight: 900 -toc: true -sidebar: - collapsed: true -seo: - title: "" # custom title (optional) - description: "" # custom description (recommended) - canonical: "" # custom canonical URL (optional) - noindex: false # false (default) or true ---- diff --git a/website/content/docs/reference/example.md b/website/content/docs/reference/example.md deleted file mode 100644 index cb35d270..00000000 --- a/website/content/docs/reference/example.md +++ /dev/null @@ -1,25 +0,0 @@ ---- -title: "Example Reference" -description: "Reference pages are ideal for outlining how things work in terse and clear terms." -summary: "" -date: 2023-09-07T16:13:18+02:00 -lastmod: 2023-09-07T16:13:18+02:00 -draft: true -menu: - docs: - parent: "" - identifier: "example-ee51430687e728ba6e68dea3359133ad" -weight: 910 -toc: true -seo: - title: "" # custom title (optional) - description: "" # custom description (recommended) - canonical: "" # custom canonical URL (optional) - noindex: false # false (default) or true ---- - -Reference pages are ideal for outlining how things work in terse and clear terms. Less concerned with telling a story or addressing a specific use case, they should give a comprehensive outline of what your documenting. - -## Further reading - -- Read [about reference](https://diataxis.fr/reference/) in the Diátaxis framework diff --git a/website/content/docs/releases/gdg_0.6.md b/website/content/docs/releases/gdg_0.6.md index b24f5412..98fb840d 100644 --- a/website/content/docs/releases/gdg_0.6.md +++ b/website/content/docs/releases/gdg_0.6.md @@ -3,36 +3,61 @@ title: "version v0.6" description: "Release Notes for v0.6" date: 2023-09-01T00:00:00 draft: false -images: [] +images: [ ] weight: 197 toc: true --- -## Release Notes for v0.6.0 +## Release Notes for v0.6.0 + **Release Date: 03/11/2024** ### Breaking Changes - - This is a major release, so once again doing some cleanups and introducing some breaking changes. Please use version v0.5.2 if you wish to maintain backward compatibility. Previous version organized downloads by org_id. Example an export for dashboards would be stored in `export/org_1/dashboards`. The org ID is inconsistent and something that cannot be guaranteed across deployment of grafana. Instead, we've switched over to using a slug of the OrgName. The default dashboard backup will go in: `export/org_main-org/dashboards`. - - - This is mentioned below, but `datasources` config key was deprecated and replaced with `connections`. Previous version would warn about the change, that functionality has been removed. Please Update to using connections, if you haven't already done so. - - - The import/export dashboards keyword provided confusion. It has been phased out bit by bit, but all references to it should now be fully removed. Import is now 'download', and export is now 'upload'. - - `organization_i` is deprecated in the importer config in favor of `organization_name`. - - +- This is a major release, so once again doing some cleanups and introducing some breaking changes. Please use version + v0.5.2 if you wish to maintain backward compatibility. Previous version organized downloads by org_id. Example an + export for dashboards would be stored in `export/org_1/dashboards`. The org ID is inconsistent and something that + cannot be guaranteed across installations of grafana. Instead, we've switched over to using a slug of the OrgName. The + default dashboard backup will go in: `export/org_main-org/dashboards`. + + {{< callout context="caution" title="Caution" icon="alert-triangle" >}} + If you renamed the default organization to something else besides `Main Org.`, be sure to set `organization_name` in your config, otherwise nothing will work. + {{< /callout >}} +- The import/export dashboards keyword provided confusion. It has been phased out bit by bit. In version 0.6 all references have now been removed. Import has been renamed to `download`, and export is now `upload`. +- `organization_id` is deprecated in the importer config in favor of `organization_name`. + +### New Features + +- Retry logic. You can now set a number of `retry_count` and `retry_delay` in the configuration that retries failed API calls +- User upload now has the ability to generate random passwords. (Please be aware that those values can't be recreated) +- JSON output has been added. This can be somewhat unstructured but is available to the user. `--output json` with the default rendering being table format. +- Linter support! (Beta) Grafana official linter has been added to GDG `tools dashboard lint`. +- `gdg-generate` CLI behavior has been updated to better mimic what `gdg` is already doing. Restructuring into subcommands. +- Org Preferences can now be retrieved when listing Orgs. `--with-preferences` + {{< callout context="caution" title="Caution" icon="alert-triangle" >}} +This is a heavy call currently till this [issue](https://github.com/grafana/grafana/issues/84309) is resolved. Use with caution if you have many Organizations in your grafana instance. + {{< /callout >}} ### Changes -- [#192](https://github.com/esnet/gdg/issues/192) Dropping support for various entities. import/export no longer supported. Removed warning for datasources (Deprecated config). Removed AlertNotification as it's been deprecated from grafana for a while now. -- [#254](https://github.com/esnet/gdg/issues/254) [#258](https://github.com/esnet/gdg/issues/258) org_id usage has been deprecated. Switching mainly using orgName / SlugName to allow for a more consistent experience between grafana installations. Change affects gdg and gdg-generate -- User import now has support for random password generator. Only printed upon import. -- [#259](https://github.com/esnet/gdg/issues/259) Adding support for Org Properties. Allowing a user to update a given orgs properties. Data also added to org Listing. -- [#251](https://github.com/esnet/gdg/issues/251) Adding a dashboard linter tool. The official grafana is recommended, but GDG will provide similar functionality. +- [#192](https://github.com/esnet/gdg/issues/192) Dropping support for various entities. import/export no longer + supported. Removed warning for datasources (Deprecated config). Removed AlertNotification as it's been deprecated from + grafana for a while now. +- [#254](https://github.com/esnet/gdg/issues/254) [#258](https://github.com/esnet/gdg/issues/258) org_id usage has been + deprecated. Switching mainly using orgName / SlugName to allow for a more consistent experience between grafana + installations. Change affects gdg and gdg-generate +- User import now has support for random password generator. Only printed upon import. +- [#259](https://github.com/esnet/gdg/issues/259) Adding support for Org Properties. Allowing a user to update a given + orgs properties. Data also added to org Listing. +- [#251](https://github.com/esnet/gdg/issues/251) Adding a dashboard linter tool. The official grafana is recommended, + but GDG will provide similar functionality. ### Bug Fixes +- [#253](https://github.com/esnet/gdg/issues/253) In order to manage orgs, the grafana admin that is configured needs to be a part of all organizations. This ticket adds a sanity check to ensure that the configured Grafana Admin is part of all known organizations. It then programmatically adds the user (if user confirms and the feature is supported), otherwise gdg will list all orgs that the user needs to be added. - ### Developer Changes - - Upgraded to go 1.22 - - Updated documentation instructions relating to install + +- Upgraded to go 1.22 +- Updated documentation instructions relating to install +- Website theme upgraded to latest version diff --git a/website/content/docs/templating/_index.md b/website/content/docs/templating/_index.md index 99b5e8f5..b7fa43cd 100644 --- a/website/content/docs/templating/_index.md +++ b/website/content/docs/templating/_index.md @@ -1,4 +1,4 @@ --- -title: "Templating Docs" -weight: 3 +title: "GDG Generate Docs (templating)" +weight: 4 --- diff --git a/website/content/docs/templating/description.md b/website/content/docs/templating/description.md index 88bbd1ee..2eb6eb91 100644 --- a/website/content/docs/templating/description.md +++ b/website/content/docs/templating/description.md @@ -1,12 +1,12 @@ --- -title: "Templating Tool" +title: "Usage Guide" weight: 1 --- GDG has now introduced a new supporting tool that works in conjunction with GDG. It is currently dependent on the GDG configuration since it will operate on the currently selected context. You can confirm what the current context is by -running `gdg ctx show` +running `gdg tools ctx show` For example, my current output is as follows: @@ -24,29 +24,20 @@ context_name: - General - Other connections: - exclude_filters: - - { } credential_rules: - rules: - field: name regex: misc - field: url - auth: - user: user - password: password + secure_data: "misc.json" - rules: - field: url regex: .*esproxy2* - auth: - user: admin - password: secret + secure_data: "proxy.json" - rules: - field: name regex: .* - auth: - user: user - password: password - datasources: { } + secure_data: "default.json" filter_override: ignore_dashboard_filters: false output_path: test/data @@ -55,7 +46,7 @@ context_name: Most of the config isn't that interesting, except the output_path will be used to determine where the newly generated dashboards will be. Make sure you have a valid configuration before continuing. -## What does gdg-generate do? +### What does gdg-generate do? There are use cases where an almost identical dashboard is needed except we need to replace certain parts of it. @@ -64,6 +55,8 @@ logo, or footer. All of these are difficult to control from grafana itself and e great user experience. This allows you to configure and generate a new dashboard with any set of variables and dictionaries that you will seed to the tool. +### Configuration + The configuration that drives this application is `templates.yml`. You can see an example below. ```yaml @@ -73,7 +66,7 @@ entities: output: ## The section below defines one or multiple destination and the associated configuration ## that goes with it. - folder: "General" ## Name of the new folder where the template will be created - org_id: 2 ## Organization ID, will be converted to a name in a future version + organization_name: "Main Org." dashboard_name: "" ## Optional, defaults to template_name.json template_data: ## Template Data the dictionary of datasets that can be used in the template, # it's basically your 'seed data'. Everything is contains is absolutely arbitrary @@ -101,7 +94,8 @@ Additionally, there a few functions exposed and available to you that allows you | QuotedStringJoin | {{ .lightsources \| QuotedStringJoin }} | [sun,moon,lightbulb] | "sun","moon","lightbulb" | ``` -There is also a large collection of functions that have been imported from [sprig](https://masterminds.github.io/sprig/) and are available for use. +There is also a large collection of functions that have been imported from [sprig](https://masterminds.github.io/sprig/) +and are available for use. ### Example Templating Snippets @@ -112,7 +106,8 @@ Data Injection "annotations": { "list": [ { - "$$hashKey": "{{ .title | lower | ToSlug}}", // Inserting data and piping it to two different functions. In this case, ToLower is redundant, but it serves as a chained example. + "$$hashKey": "{{ .title | lower | ToSlug}}", + // Inserting data and piping it to two different functions. In this case, ToLower is redundant, but it serves as a chained example. "builtIn": 1, "datasource": "Grafana", "enable": true, @@ -131,33 +126,41 @@ Iterating and conditionals. ```json { "link_text": [ - {{if .enabledlight}} // conditional to check if to insert or not - {{ range $v := .lightsources}} // Iterating through list - {{ $v }} // Inserting value - {{ end }} + {{ if .enabledlight }} + // conditional to check if to insert or not + {{ range $v: = .lightsources }} + // Iterating through list + {{ $v }} + // Inserting value + {{ end }} {{ end }} ] } ``` + Inserting a comma delimited list ```json "link_url": [ - "{{ .lightsources | join "," }}", - "/grafana/d/000000003/bandwidth-dashboard", - "/grafana/d/xk26IFhmk/flow-data", + "{{ .lightsources | join ", " }}", + "/grafana/d/000000003/bandwidth-dashboard", + "/grafana/d/xk26IFhmk/flow-data", ] ``` + ### Usage As part of the installation you will have access to gdg-generate. -NOTE: -c and -ct are optional parameters as is -t if you're relying on the defaults. --t will filter the config and only process the template you've specified. + +{{< callout note >}}--config, --template-config, and -t are optional parameters. gdg-generate will fallback on defaults if +none are specified. If -t is not provided, all templates will be processed + {{< /callout >}} ```sh -gdg-generate -c config/importer.yml --ct config/template.yaml -t template_example +gdg-generate --config config/importer.yml --template-config config/template.yaml template generate -t template_example ``` + Example output: ```sh diff --git a/website/content/docs/tutorials/_index.md b/website/content/docs/tutorials/_index.md index 25b273bb..a6d552c4 100644 --- a/website/content/docs/tutorials/_index.md +++ b/website/content/docs/tutorials/_index.md @@ -1,4 +1,4 @@ --- title: "Tutorials" -weight: 4 +weight: 5 --- diff --git a/website/content/docs/usage_guide/_index.md b/website/content/docs/usage_guide/_index.md new file mode 100644 index 00000000..49739516 --- /dev/null +++ b/website/content/docs/usage_guide/_index.md @@ -0,0 +1,4 @@ +--- +title: "Usage Guide" +weight: 3 +--- diff --git a/website/content/docs/gdg/backup_guide.md b/website/content/docs/usage_guide/backup_guide.md similarity index 88% rename from website/content/docs/gdg/backup_guide.md rename to website/content/docs/usage_guide/backup_guide.md index 3c2e550c..d960137a 100644 --- a/website/content/docs/gdg/backup_guide.md +++ b/website/content/docs/usage_guide/backup_guide.md @@ -1,35 +1,20 @@ --- -title: "Backup Commands Guide" +title: "Backup Guide" weight: 16 --- Every namespace supporting CRUD operations has the functions: list, download, upload, clear operating on only the monitored folders. - -### Alert Notifications (DEPRECATED) - -This will stop working soon both as a concept in grafana and something that GDG will support. - -Allows you to manage alertnotifications (an) if you have any setup - -```sh -./bin/gdg backup an list -- Lists all alert notifications -./bin/gdg backup an download -- retrieve and save all alertnotifications from grafana -./bin/gdg backup an upload -- writes all local alert notifications to grafana -./bin/gdg backup an clear -- Deletes all alert notifications -``` - ### Connections -#### (was: DataSources) -Note: Starting with 0.4.6 this was renamed to connections. +{{< callout note >}} Starting with v0.4.6 "Datasources" was renamed to connections. {{< /callout >}} -Connections credentials are keyed by the name of the DataSource. See see [config example](https://github.com/esnet/gdg/blob/master/conf/importer-example.yml). If the connection JSON doesn't have auth enabled, the credentials are ignored. If Credentials are missing, we'll fall back on default credentials if any exist. The password is set as a value for basicAuthPassword in the API payload. +Connections credentials are keyed by the name of the DataSource. See [config example](https://github.com/esnet/gdg/blob/master/config/importer-example.yml). If the connection JSON doesn't have auth enabled, the credentials are ignored. If Credentials are missing, we'll fall back on default credentials if any exist. The password is set as a value for basicAuthPassword in the API payload. Datasources are imported or exported from _organization_ specified in configuration file otherwise current organization user is used. -All commands can use `connection` or `c` to manage datasources. (Legacy options of `datasource` and `ds` are also supported) +All commands can use `connection` or `c` to manage datasources. ```sh ./bin/gdg backup c list -- Lists all current connections @@ -120,7 +105,7 @@ The listing includes the folder name, followed by several lines with "PERMISSION Library elements are components that can be shared among multiple dashboards. Folder matching will still be applied, so any folders not monitored will be ignored unless explicitly specified. If wildcard flag is enabled, all elements will be acted on irrelevant of folder location -All commands can use `libraryelements` aliased to `library` and `lib` for laziness purposes. +All commands can use `libraryelements` aliased to `library` and `lib` for laziness purposes. A more extensive tutorial is available [here](https://software.es.net/gdg/docs/tutorials/library_elements/) ```sh ./bin/gdg backup lib list -- Lists all library components @@ -133,10 +118,17 @@ All commands can use `libraryelements` aliased to `library` and `lib` for lazine ### Organizations -#### Auth: Requires Grafana Admin (Tokens not supported, Org Admins don't have access) + +{{< callout context="danger" title="Danger" icon="alert-octagon" >}} +Auth: Requires Grafana Admin + + - Tokens/service account tokens are tied to a specific org and are therefore not supported. + - Organization Admins don't have access to list all Orgs, therefore are also not supported. + + {{< /callout >}} + Command can use `organizations` or `org` to manage organizations. -NOTE: this only manages top level of the orgs structure. It's mainly useful to maintain consistency. ```sh ./bin/gdg backup org list -- Lists all organizations @@ -144,12 +136,9 @@ NOTE: this only manages top level of the orgs structure. It's mainly useful to m ./bin/gdg backup org download -- Download Orgs to grafana ``` -### Teams - -{{< callout context="caution" title="Caution" icon="alert-triangle" >}} -Admin team members are unable to be exported back. Currently all members except the server admin will be exported as regular members -{{< /callout >}} +A tutorial on working with [organizations](https://software.es.net/gdg/docs/tutorials/organization-and-authentication/) is available. +### Teams {{< callout context="caution" title="Caution" icon="alert-triangle" >}} Users need to be created before team export can succeed diff --git a/website/content/docs/gdg/enterprise_guide.md b/website/content/docs/usage_guide/enterprise_guide.md similarity index 99% rename from website/content/docs/gdg/enterprise_guide.md rename to website/content/docs/usage_guide/enterprise_guide.md index e792dd23..6eb6f1f2 100644 --- a/website/content/docs/gdg/enterprise_guide.md +++ b/website/content/docs/usage_guide/enterprise_guide.md @@ -1,5 +1,5 @@ --- -title: "Enterprise User Guide" +title: "Enterprise Guide" weight: 18 --- The features listed below are for the enterprise edition of Grafana only. They will not work on the OSS version. diff --git a/website/content/docs/gdg/tools_guide.md b/website/content/docs/usage_guide/tools_guide.md similarity index 100% rename from website/content/docs/gdg/tools_guide.md rename to website/content/docs/usage_guide/tools_guide.md diff --git a/website/content/privacy.md b/website/content/privacy.md deleted file mode 100644 index 5d589539..00000000 --- a/website/content/privacy.md +++ /dev/null @@ -1,14 +0,0 @@ ---- -title: "Privacy Policy" -description: "" -summary: "" -date: 2023-09-07T17:19:07+02:00 -lastmod: 2023-09-07T17:19:07+02:00 -draft: false -type: "legal" -seo: - title: "" # custom title (optional) - description: "" # custom description (recommended) - canonical: "" # custom canonical URL (optional) - noindex: false # false (default) or true ---- diff --git a/website/hugo_stats.json b/website/hugo_stats.json index cc26b50b..26478796 100644 --- a/website/hugo_stats.json +++ b/website/hugo_stats.json @@ -56,11 +56,10 @@ "classes": [ "DocSearch-Label", "active", + "alert-octagon", "alert-triangle", "anchor", - "blog-header", "btn", - "btn-block", "btn-close", "btn-cta", "btn-lg", @@ -70,7 +69,9 @@ "callout-body", "callout-caution", "callout-content", + "callout-danger", "callout-icon", + "callout-note", "callout-title", "card", "card-body", @@ -138,6 +139,7 @@ "icon-tabler-search", "icon-tabler-sun", "icon-tabler-x", + "info-circle", "is-terminal", "justify-content-between", "justify-content-center", @@ -171,7 +173,6 @@ "ms-3", "ms-auto", "ms-lg-2", - "mt-2", "mt-3", "mt-4", "mt-n3", @@ -194,7 +195,6 @@ "offcanvas-start", "offcanvas-title", "order-3", - "order-lg-3", "order-lg-4", "p-0", "p-2", @@ -204,11 +204,9 @@ "pb-2", "pb-3", "pe-4", - "privacy", "ps-3", "pt-4", "px-0", - "px-4", "query-no-results", "rounded-pill", "row", @@ -248,19 +246,22 @@ ], "ids": [ "TableOfContents", - "alert-notifications-deprecated", "api-sdk-changes", + "auth", "auth--requires-grafana-admin-tokens-not-supported-org-admins-dont-have-access", "authentication", "authentication-management", "authentication-token", "available-functions", + "backup-feature", "breaking-changes", "bug-fixes", "bug-fixes-1", "bug-fixes-2", + "buildingrunning-gdg", "buildingrunning-the-app", "buttonColorMode", + "can-i-use-gdg-to-backup-grafana", "changes", "changes-1", "changes-2", @@ -284,12 +285,14 @@ "datamodel-changes", "deleting-elements", "deleting-related-dashboard", + "dependencies", "devel", "developer-changes", "developer-changes-1", "docker-usage", "doks-docs-nav", "download-orgs-dashboards", + "enterprise-features", "environment-overrides", "example-templating-snippets", "export--upload-dashboards", @@ -297,10 +300,14 @@ "folder-permissions", "folders", "folders-update", + "gdg", "general", "getting-started", "global-flags", + "heading", "homebrew-installation", + "i-need-feature-x-can-you-please-add-that-in", + "i-need-help-where-do-i-go", "import--download-dashboards", "import-components", "importing-dashboards", @@ -324,6 +331,7 @@ "offcanvasNavMainLabel", "offcanvasNavSection", "offcanvasNavSectionLabel", + "organization", "organization-workflow", "organizations", "organizations-preferences", @@ -354,16 +362,19 @@ "switching-organizations", "teams", "technical-debt", + "testing", "toc", "token-management", + "tools-feature", "usage", "username--password", "users", "version", "version-prior-to-v042", "version-v042-v051", - "was-datasources", "what-does-gdg-generate-do", + "what-is-the-meaning-of-life", + "why-does-gdg-not-list-all-my-dashboards", "wild-card-flag" ] }