Add support for vendor path setting in atmos.yaml (#737)

* feat: vendor path setting in yaml support

* feat: refactoring vendor config path

* general refactoring process vendor

* vendor yaml path update

* sync master

* blank line yaml

* feat: set vendor path

* vendor improvements checking

* vendor refactor remove redundant code

* implement env vars for vendor file

* implement env vars flag for vendor file

* update docs vendor path

* improve description

* added demo base vendor files general refactoring

* formating changes

* feat: added doublestar support

* address commit requests

* deduplicate imports feature

* general fixes and vendoring folder formating clean up

* general fixes and vendoring folder formating clean up

* fixes for iteration over vendor files

* put back deduplication logic

* handle no files case

* Update atmos.yaml

* update pkg yaml

---------

Co-authored-by: Andriy Knysh <[email protected]>
Co-authored-by: Erik Osterman (CEO @ Cloud Posse) <[email protected]>

atmos.yaml

@@ -15,7 +15,17 @@
 # are independent settings (supporting both absolute and relative paths).
 # If 'base_path' is provided, 'components.terraform.base_path', 'components.helmfile.base_path', 'stacks.base_path' and 'workflows.base_path'
 # are considered paths relative to 'base_path'.
-base_path: "./examples/quick-start-advanced"
+base_path: "./"
+
+vendor:
+  # Path to vendor configuration file or directory containing vendor files
+  # Supports both absolute and relative paths
+  # If a directory is specified, all .yaml files in the directory will be processed in lexicographical order
+  # Can also be set using 'ATMOS_VENDOR_BASE_PATH' ENV var, or '--vendor-base-path' command-line argument
+  # Examples:
+  #   base_path: "./vendor.yaml"          # Single file
+  #   base_path: "./vendor.d/"            # Directory containing multiple .yaml files
+  base_path: "./vendor.yaml"
 
 components:
   terraform:

examples/demo-vendoring/atmos.yaml

@@ -16,6 +16,16 @@ stacks:
     - "**/_defaults.yaml"
   name_pattern: "{stage}"
 
+vendor:  
+  # Single file
+  base_path: "./vendor.yaml"
+
+  # Directory with multiple files
+  #base_path: "./vendor"
+
+  # Absolute path
+  #base_path: "vendor.d/vendor1.yaml"
+
 logs:
   file: "/dev/stderr"
   level: Info

examples/demo-vendoring/stacks/catalog/myapp.yaml

@@ -0,0 +1,9 @@
+components:
+  terraform:
+    myapp:
+      vars:
+        location: Los Angeles
+        lang: en
+        format: ''
+        options: '0'
+        units: m

examples/demo-vendoring/stacks/deploy/dev.yaml

@@ -0,0 +1,12 @@
+vars:
+  stage: dev
+
+import:
+  - catalog/myapp
+
+components:
+  terraform:
+    myapp:
+      vars:
+        location: Stockholm
+        lang: se

examples/demo-vendoring/stacks/deploy/prod.yaml

@@ -0,0 +1,12 @@
+vars:
+  stage: prod
+
+import:
+  - catalog/myapp
+
+components:
+  terraform:
+    myapp:
+      vars:
+        location: Los Angeles
+        lang: en

examples/demo-vendoring/stacks/deploy/staging.yaml

@@ -0,0 +1,12 @@
+vars:
+  stage: staging
+
+import:
+  - catalog/myapp
+
+components:
+  terraform:
+    myapp:
+      vars:
+        location: Los Angeles
+        lang: en

examples/demo-vendoring/vendor.d/vendor1.yaml

@@ -0,0 +1,16 @@
+apiVersion: atmos/v1
+kind: AtmosVendorConfig
+metadata:
+  name: vendor-test-1
+  description: |
+    Demo vendor configuration showcasing the structure and usage of Atmos vendor configs.
+    This example demonstrates component sourcing from GitHub repositories with versioning.
+spec:
+  sources:
+    - component: "ipinfo"
+      source: "github.com/cloudposse/atmos.git//examples/demo-library/{{ .Component }}?ref={{.Version}}"
+      version: "main"
+      targets:
+        - "components/terraform/{{ .Component }}/{{.Version}}"
+      tags:
+        - demo

examples/demo-vendoring/vendor/vendor1.yaml

@@ -0,0 +1,16 @@
+apiVersion: atmos/v1
+kind: AtmosVendorConfig
+metadata:
+  name: vendor-test-1
+  description: |
+    Demo vendor configuration showcasing the structure and usage of Atmos vendor configs.
+    This example demonstrates component sourcing from GitHub repositories with versioning.
+spec:
+  sources:
+    - component: "ipinfo"
+      source: "github.com/cloudposse/atmos.git//examples/demo-library/{{ .Component }}?ref={{.Version}}"
+      version: "main"
+      targets:
+        - "components/terraform/{{ .Component }}/{{.Version}}"
+      tags:
+        - demo

examples/demo-vendoring/vendor/vendor2.yaml

@@ -0,0 +1,16 @@
+apiVersion: atmos/v1
+kind: AtmosVendorConfig
+metadata:
+  name: vendor-test-2
+  description: |
+    Demo vendor configuration showcasing the structure and usage of Atmos vendor configs.
+    This example demonstrates component sourcing from GitHub repositories with versioning.
+spec:
+  sources:
+    - component: "weather"
+      source: "github.com/cloudposse/atmos.git//examples/demo-library/{{ .Component }}?ref={{.Version}}"
+      version: "main"
+      targets:
+        - "components/terraform/{{ .Component }}/{{.Version}}"
+      tags:
+        - demo

internal/exec/utils.go

@@ -31,6 +31,7 @@ var (
 		cfg.CliConfigDirFlag,
 		cfg.StackDirFlag,
 		cfg.BasePathFlag,
+		cfg.VendorBasePathFlag,
 		cfg.GlobalOptionsFlag,
 		cfg.DeployRunInitFlag,
 		cfg.InitRunReconfigure,
@@ -759,6 +760,19 @@ func processArgsAndFlags(componentType string, inputArgsAndFlags []string) (sche
 			info.BasePath = stacksDirFlagParts[1]
 		}
 
+		if arg == cfg.VendorBasePathFlag {
+			if len(inputArgsAndFlags) <= (i + 1) {
+				return info, fmt.Errorf("invalid flag: %s", arg)
+			}
+			info.VendorBasePath = inputArgsAndFlags[i+1]
+		} else if strings.HasPrefix(arg+"=", cfg.VendorBasePathFlag) {
+			var vendorBasePathFlagParts = strings.Split(arg, "=")
+			if len(vendorBasePathFlagParts) != 2 {
+				return info, fmt.Errorf("invalid flag: %s", arg)
+			}
+			info.VendorBasePath = vendorBasePathFlagParts[1]
+		}
+
 		if arg == cfg.DeployRunInitFlag {
 			if len(inputArgsAndFlags) <= (i + 1) {
 				return info, fmt.Errorf("invalid flag: %s", arg)

internal/exec/vendor_utils.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
@@ -14,7 +15,9 @@ import (
 	cp "github.com/otiai10/copy"
 	"github.com/samber/lo"
 	"github.com/spf13/cobra"
+	"gopkg.in/yaml.v3"
 
+	"github.com/bmatcuk/doublestar/v4"
 	cfg "github.com/cloudposse/atmos/pkg/config"
 	"github.com/cloudposse/atmos/pkg/schema"
 	u "github.com/cloudposse/atmos/pkg/utils"
@@ -80,7 +83,7 @@ func ExecuteVendorPullCommand(cmd *cobra.Command, args []string) error {
 
 	// Check `vendor.yaml`
 	vendorConfig, vendorConfigExists, foundVendorConfigFile, err := ReadAndProcessVendorConfigFile(cliConfig, cfg.AtmosVendorConfigFileName)
-	if vendorConfigExists && err != nil {
+	if err != nil {
 		return err
 	}
 
@@ -120,48 +123,111 @@ func ExecuteVendorPullCommand(cmd *cobra.Command, args []string) error {
 }
 
 // ReadAndProcessVendorConfigFile reads and processes the Atmos vendoring config file `vendor.yaml`
-func ReadAndProcessVendorConfigFile(cliConfig schema.CliConfiguration, vendorConfigFile string) (
-	schema.AtmosVendorConfig,
-	bool,
-	string,
-	error,
-) {
+func ReadAndProcessVendorConfigFile(
+	cliConfig schema.CliConfiguration,
+	vendorConfigFile string,
+) (schema.AtmosVendorConfig, bool, string, error) {
 	var vendorConfig schema.AtmosVendorConfig
-	vendorConfigFileExists := true
 
-	// Check if the vendoring manifest file exists
-	foundVendorConfigFile, fileExists := u.SearchConfigFile(vendorConfigFile)
+	// Initialize empty sources slice
+	vendorConfig.Spec.Sources = []schema.AtmosVendorSource{}
 
-	if !fileExists {
-		// Look for the vendoring manifest in the directory pointed to by the `base_path` setting in the `atmos.yaml`
-		pathToVendorConfig := path.Join(cliConfig.BasePath, vendorConfigFile)
+	var vendorConfigFileExists bool
+	var foundVendorConfigFile string
 
-		if !u.FileExists(pathToVendorConfig) {
-			vendorConfigFileExists = false
-			return vendorConfig, vendorConfigFileExists, "", fmt.Errorf("vendor config file '%s' does not exist", pathToVendorConfig)
+	// Check if vendor config is specified in atmos.yaml
+	if cliConfig.Vendor.BasePath != "" {
+		if !filepath.IsAbs(cliConfig.Vendor.BasePath) {
+			foundVendorConfigFile = filepath.Join(cliConfig.BasePath, cliConfig.Vendor.BasePath)
+		} else {
+			foundVendorConfigFile = cliConfig.Vendor.BasePath
 		}
+	} else {
+		// Path is not defined in atmos.yaml, proceed with existing logic
+		var fileExists bool
+		foundVendorConfigFile, fileExists = u.SearchConfigFile(vendorConfigFile)
 
-		foundVendorConfigFile = pathToVendorConfig
+		if !fileExists {
+			// Look for the vendoring manifest in the directory pointed to by the `base_path` setting in `atmos.yaml`
+			pathToVendorConfig := path.Join(cliConfig.BasePath, vendorConfigFile)
+
+			if !u.FileExists(pathToVendorConfig) {
+				vendorConfigFileExists = false
+				return vendorConfig, vendorConfigFileExists, "", fmt.Errorf("vendor config file or directory '%s' does not exist", pathToVendorConfig)
+			}
+
+			foundVendorConfigFile = pathToVendorConfig
+		}
 	}
 
-	vendorConfigFileContent, err := os.ReadFile(foundVendorConfigFile)
+	// Check if it's a directory
+	fileInfo, err := os.Stat(foundVendorConfigFile)
 	if err != nil {
-		return vendorConfig, vendorConfigFileExists, "", err
+		return vendorConfig, false, "", err
 	}
 
-	vendorConfig, err = u.UnmarshalYAML[schema.AtmosVendorConfig](string(vendorConfigFileContent))
-	if err != nil {
-		return vendorConfig, vendorConfigFileExists, "", err
+	var configFiles []string
+	if fileInfo.IsDir() {
+		matches, err := doublestar.Glob(os.DirFS(foundVendorConfigFile), "*.{yaml,yml}")
+		if err != nil {
+			return vendorConfig, false, "", err
+		}
+		for _, match := range matches {
+			configFiles = append(configFiles, filepath.Join(foundVendorConfigFile, match))
+		}
+		sort.Strings(configFiles)
+		if len(configFiles) == 0 {
+			return vendorConfig, false, "", fmt.Errorf("no YAML configuration files found in directory '%s'", foundVendorConfigFile)
+		}
+	} else {
+		configFiles = []string{foundVendorConfigFile}
 	}
 
-	if vendorConfig.Kind != "AtmosVendorConfig" {
-		return vendorConfig, vendorConfigFileExists, "",
-			fmt.Errorf("invalid 'kind: %s' in the vendor config file '%s'. Supported kinds: 'AtmosVendorConfig'",
-				vendorConfig.Kind,
-				foundVendorConfigFile,
-			)
+	// Process all config files
+	var mergedSources []schema.AtmosVendorSource
+	var mergedImports []string
+	sourceMap := make(map[string]bool) // Track unique sources by component name
+	importMap := make(map[string]bool) // Track unique imports
+
+	for _, configFile := range configFiles {
+		var currentConfig schema.AtmosVendorConfig
+		yamlFile, err := os.ReadFile(configFile)
+		if err != nil {
+			return vendorConfig, false, "", err
+		}
+
+		err = yaml.Unmarshal(yamlFile, &currentConfig)
+		if err != nil {
+			return vendorConfig, false, "", err
+		}
+
+		// Merge sources, checking for duplicates
+		for _, source := range currentConfig.Spec.Sources {
+			if source.Component != "" {
+				if sourceMap[source.Component] {
+					return vendorConfig, false, "", fmt.Errorf("duplicate component '%s' found in config file '%s'",
+						source.Component, configFile)
+				}
+				sourceMap[source.Component] = true
+			}
+			mergedSources = append(mergedSources, source)
+		}
+
+		// Merge imports, checking for duplicates
+		for _, imp := range currentConfig.Spec.Imports {
+			if importMap[imp] {
+				continue // Skip duplicate imports
+			}
+			importMap[imp] = true
+			mergedImports = append(mergedImports, imp)
+		}
 	}
 
+	// Create final merged config
+	vendorConfig.Spec.Sources = mergedSources
+	vendorConfig.Spec.Imports = mergedImports
+	vendorConfigFileExists = true
+
 	return vendorConfig, vendorConfigFileExists, foundVendorConfigFile, nil
 }
 
@@ -531,7 +597,7 @@ func processVendorImports(
 			return nil, nil, err
 		}
 
-		for i, _ := range vendorConfig.Spec.Sources {
+		for i := range vendorConfig.Spec.Sources {
 			vendorConfig.Spec.Sources[i].File = imp
 		}
 

pkg/config/const.go

@@ -18,6 +18,7 @@ const (
 	CliConfigDirFlag            = "--config-dir"
 	StackDirFlag                = "--stacks-dir"
 	BasePathFlag                = "--base-path"
+	VendorBasePathFlag          = "--vendor-base-path"
 	WorkflowDirFlag             = "--workflows-dir"
 	KubeConfigConfigFlag        = "--kubeconfig-path"
 	JsonSchemaDirFlag           = "--schemas-jsonschema-dir"

pkg/config/utils.go

@@ -163,6 +163,12 @@ func processEnvVars(cliConfig *schema.CliConfiguration) error {
 		cliConfig.BasePath = basePath
 	}
 
+	vendorBasePath := os.Getenv("ATMOS_VENDOR_BASE_PATH")
+	if len(vendorBasePath) > 0 {
+		u.LogTrace(*cliConfig, fmt.Sprintf("Found ENV var ATMOS_VENDOR_BASE_PATH=%s", vendorBasePath))
+		cliConfig.Vendor.BasePath = vendorBasePath
+	}
+
 	stacksBasePath := os.Getenv("ATMOS_STACKS_BASE_PATH")
 	if len(stacksBasePath) > 0 {
 		u.LogTrace(*cliConfig, fmt.Sprintf("Found ENV var ATMOS_STACKS_BASE_PATH=%s", stacksBasePath))

pkg/schema/schema.go

@@ -15,6 +15,7 @@ type CliConfiguration struct {
 	Schemas                       Schemas        `yaml:"schemas,omitempty" json:"schemas,omitempty" mapstructure:"schemas"`
 	Templates                     Templates      `yaml:"templates,omitempty" json:"templates,omitempty" mapstructure:"templates"`
 	Settings                      CliSettings    `yaml:"settings,omitempty" json:"settings,omitempty" mapstructure:"settings"`
+	Vendor                        Vendor         `yaml:"vendor,omitempty" json:"vendor,omitempty" mapstructure:"vendor"`
 	Initialized                   bool           `yaml:"initialized" json:"initialized" mapstructure:"initialized"`
 	StacksBaseAbsolutePath        string         `yaml:"stacksBaseAbsolutePath,omitempty" json:"stacksBaseAbsolutePath,omitempty" mapstructure:"stacksBaseAbsolutePath"`
 	IncludeStackAbsolutePaths     []string       `yaml:"includeStackAbsolutePaths,omitempty" json:"includeStackAbsolutePaths,omitempty" mapstructure:"includeStackAbsolutePaths"`
@@ -135,6 +136,7 @@ type ArgsAndFlagsInfo struct {
 	StacksDir                 string
 	WorkflowsDir              string
 	BasePath                  string
+	VendorBasePath            string
 	DeployRunInit             string
 	InitRunReconfigure        string
 	AutoGenerateBackendFile   string
@@ -181,6 +183,7 @@ type ConfigAndStacksInfo struct {
 	AdditionalArgsAndFlags        []string
 	GlobalOptions                 []string
 	BasePath                      string
+	VendorBasePathFlag            string
 	TerraformCommand              string
 	TerraformDir                  string
 	HelmfileCommand               string
@@ -552,3 +555,9 @@ type AtmosVendorConfig struct {
 	Metadata   AtmosVendorMetadata
 	Spec       AtmosVendorSpec `yaml:"spec" json:"spec" mapstructure:"spec"`
 }
+
+type Vendor struct {
+	// Path to vendor configuration file or directory containing vendor files
+	// If a directory is specified, all .yaml files in the directory will be processed in lexicographical order
+	BasePath string `yaml:"base_path" json:"base_path" mapstructure:"base_path"`
+}