diff --git a/pkg/fleet/env/env.go b/pkg/fleet/env/env.go index 30f6b5be30ec9..1f09e466d1125 100644 --- a/pkg/fleet/env/env.go +++ b/pkg/fleet/env/env.go @@ -9,6 +9,7 @@ package env import ( "fmt" "os" + "slices" "strings" "github.com/DataDog/datadog-agent/pkg/config" @@ -23,6 +24,7 @@ const ( envRegistryAuth = "DD_INSTALLER_REGISTRY_AUTH" envDefaultPackageVersion = "DD_INSTALLER_DEFAULT_PKG_VERSION" envDefaultPackageInstall = "DD_INSTALLER_DEFAULT_PKG_INSTALL" + envApmLibraries = "DD_APM_INSTRUMENTATION_LIBRARIES" ) var defaultEnv = Env{ @@ -43,6 +45,21 @@ var defaultEnv = Env{ }, } +// ApmLibLanguage is a language defined in DD_APM_INSTRUMENTATION_LIBRARIES env var +type ApmLibLanguage string + +// ApmLibVersion is the version of the library defined in DD_APM_INSTRUMENTATION_LIBRARIES env var +type ApmLibVersion string + +// AsVersionTag returns the version tag associated with the version of the library defined in DD_APM_INSTRUMENTATION_LIBRARIES +// if the value is empty we return latest +func (v ApmLibVersion) AsVersionTag() string { + if v == "" { + return "latest" + } + return string(v) + "-1" +} + // Env contains the configuration for the installer. type Env struct { APIKey string @@ -57,6 +74,8 @@ type Env struct { DefaultPackagesInstallOverride map[string]bool DefaultPackagesVersionOverride map[string]string + ApmLibraries map[ApmLibLanguage]ApmLibVersion + InstallScript InstallScriptEnv } @@ -75,6 +94,8 @@ func FromEnv() *Env { DefaultPackagesInstallOverride: overridesByNameFromEnv(envDefaultPackageInstall, func(s string) bool { return s == "true" }), DefaultPackagesVersionOverride: overridesByNameFromEnv(envDefaultPackageVersion, func(s string) string { return s }), + ApmLibraries: parseApmLibrariesEnv(), + InstallScript: installScriptEnvFromEnv(), } } @@ -108,6 +129,18 @@ func (e *Env) ToEnv() []string { if e.RegistryAuthOverride != "" { env = append(env, envRegistryAuth+"="+e.RegistryAuthOverride) } + if len(e.ApmLibraries) > 0 { + libraries := []string{} + for l, v := range e.ApmLibraries { + l := string(l) + if v != "" { + l = l + ":" + string(v) + } + libraries = append(libraries, l) + } + slices.Sort(libraries) + env = append(env, envApmLibraries+"="+strings.Join(libraries, ",")) + } env = append(env, overridesByNameToEnv(envRegistryURL, e.RegistryOverrideByImage)...) env = append(env, overridesByNameToEnv(envRegistryAuth, e.RegistryAuthOverrideByImage)...) env = append(env, overridesByNameToEnv(envDefaultPackageInstall, e.DefaultPackagesInstallOverride)...) @@ -115,6 +148,19 @@ func (e *Env) ToEnv() []string { return env } +func parseApmLibrariesEnv() map[ApmLibLanguage]ApmLibVersion { + apmLibraries := os.Getenv(envApmLibraries) + apmLibrariesVersion := map[ApmLibLanguage]ApmLibVersion{} + if apmLibraries == "" { + return apmLibrariesVersion + } + for _, library := range strings.Split(apmLibraries, ",") { + libraryName, libraryVersion, _ := strings.Cut(library, ":") + apmLibrariesVersion[ApmLibLanguage(libraryName)] = ApmLibVersion(libraryVersion) + } + return apmLibrariesVersion +} + func overridesByNameFromEnv[T any](envPrefix string, convert func(string) T) map[string]T { env := os.Environ() overridesByPackage := map[string]T{} diff --git a/pkg/fleet/env/env_test.go b/pkg/fleet/env/env_test.go index c58877f4df48c..b957b1caee416 100644 --- a/pkg/fleet/env/env_test.go +++ b/pkg/fleet/env/env_test.go @@ -29,6 +29,7 @@ func TestFromEnv(t *testing.T) { RegistryAuthOverrideByImage: map[string]string{}, DefaultPackagesInstallOverride: map[string]bool{}, DefaultPackagesVersionOverride: map[string]string{}, + ApmLibraries: map[ApmLibLanguage]ApmLibVersion{}, InstallScript: InstallScriptEnv{ APMInstrumentationEnabled: APMInstrumentationNotSet, }, @@ -50,6 +51,7 @@ func TestFromEnv(t *testing.T) { envDefaultPackageInstall + "_ANOTHER_PACKAGE": "false", envDefaultPackageVersion + "_PACKAGE": "1.2.3", envDefaultPackageVersion + "_ANOTHER_PACKAGE": "4.5.6", + envApmLibraries: "java,dotnet:latest,ruby:1.2", envApmInstrumentationEnabled: "all", }, expected: &Env{ @@ -74,6 +76,11 @@ func TestFromEnv(t *testing.T) { "package": "1.2.3", "another-package": "4.5.6", }, + ApmLibraries: map[ApmLibLanguage]ApmLibVersion{ + "java": "", + "dotnet": "latest", + "ruby": "1.2", + }, InstallScript: InstallScriptEnv{ APMInstrumentationEnabled: APMInstrumentationEnabledAll, }, @@ -88,7 +95,7 @@ func TestFromEnv(t *testing.T) { defer os.Unsetenv(key) } result := FromEnv() - assert.Equal(t, tt.expected, result) + assert.Equal(t, tt.expected, result, "failed %v", tt.name) }) } } @@ -128,6 +135,11 @@ func TestToEnv(t *testing.T) { "package": "1.2.3", "another-package": "4.5.6", }, + ApmLibraries: map[ApmLibLanguage]ApmLibVersion{ + "java": "", + "dotnet": "latest", + "ruby": "1.2", + }, }, expected: []string{ "DD_API_KEY=123456", @@ -135,6 +147,7 @@ func TestToEnv(t *testing.T) { "DD_REMOTE_UPDATES=true", "DD_INSTALLER_REGISTRY_URL=registry.example.com", "DD_INSTALLER_REGISTRY_AUTH=auth", + "DD_APM_INSTRUMENTATION_LIBRARIES=dotnet:latest,java,ruby:1.2", "DD_INSTALLER_REGISTRY_URL_IMAGE=another.registry.example.com", "DD_INSTALLER_REGISTRY_URL_ANOTHER_IMAGE=yet.another.registry.example.com", "DD_INSTALLER_REGISTRY_AUTH_IMAGE=another.auth", diff --git a/pkg/fleet/installer/default_packages.go b/pkg/fleet/installer/default_packages.go index 114e77dabb492..ff3c3e8f10092 100644 --- a/pkg/fleet/installer/default_packages.go +++ b/pkg/fleet/installer/default_packages.go @@ -7,62 +7,94 @@ package installer import ( "slices" + "strings" "github.com/DataDog/datadog-agent/pkg/fleet/env" "github.com/DataDog/datadog-agent/pkg/fleet/internal/oci" ) -type defaultPackage struct { - name string +// Package represents a package known to the installer +type Package struct { + Name string released bool releasedBySite []string releasedWithRemoteUpdates bool - condition func(*env.Env) bool + condition func(Package, *env.Env) bool } -var defaultPackagesList = []defaultPackage{ - {name: "datadog-apm-inject", released: false, condition: apmInjectEnabled}, - {name: "datadog-apm-library-java", released: false, condition: apmInjectEnabled}, - {name: "datadog-apm-library-ruby", released: false, condition: apmInjectEnabled}, - {name: "datadog-apm-library-js", released: false, condition: apmInjectEnabled}, - {name: "datadog-apm-library-dotnet", released: false, condition: apmInjectEnabled}, - {name: "datadog-apm-library-python", released: false, condition: apmInjectEnabled}, - {name: "datadog-agent", released: false, releasedWithRemoteUpdates: true}, +// PackagesList lists all known packages. Not all of them are installable +var PackagesList = []Package{ + {Name: "datadog-apm-inject", released: false, condition: apmInjectEnabled}, + {Name: "datadog-apm-library-java", released: false, condition: apmLanguageEnabled}, + {Name: "datadog-apm-library-ruby", released: false, condition: apmLanguageEnabled}, + {Name: "datadog-apm-library-js", released: false, condition: apmLanguageEnabled}, + {Name: "datadog-apm-library-dotnet", released: false, condition: apmLanguageEnabled}, + {Name: "datadog-apm-library-python", released: false, condition: apmLanguageEnabled}, + {Name: "datadog-agent", released: false, releasedWithRemoteUpdates: true}, } // DefaultPackages resolves the default packages URLs to install based on the environment. func DefaultPackages(env *env.Env) []string { - return defaultPackages(env, defaultPackagesList) + return defaultPackages(env, PackagesList) } -func defaultPackages(env *env.Env, defaultPackages []defaultPackage) []string { +func defaultPackages(env *env.Env, defaultPackages []Package) []string { var packages []string for _, p := range defaultPackages { released := p.released || slices.Contains(p.releasedBySite, env.Site) || (p.releasedWithRemoteUpdates && env.RemoteUpdates) - installOverride, isOverridden := env.DefaultPackagesInstallOverride[p.name] - condition := p.condition == nil || p.condition(env) + installOverride, isOverridden := env.DefaultPackagesInstallOverride[p.Name] + condition := p.condition == nil || p.condition(p, env) shouldInstall := released && condition if isOverridden { shouldInstall = installOverride } + if !shouldInstall { + continue + } + + version := "latest" + + // Respect pinned version of APM packages if we don't define any overwrite + if apmLibVersion, ok := env.ApmLibraries[packageToLanguage(p.Name)]; ok { + version = apmLibVersion.AsVersionTag() + // TODO(paullgdc): Emit a warning here if APM packages are not pinned to at least a major + } - if shouldInstall { - version := "latest" - if v, ok := env.DefaultPackagesVersionOverride[p.name]; ok { - version = v - } - url := oci.PackageURL(env, p.name, version) - packages = append(packages, url) + if v, ok := env.DefaultPackagesVersionOverride[p.Name]; ok { + version = v } + url := oci.PackageURL(env, p.Name, version) + packages = append(packages, url) } return packages } -func apmInjectEnabled(e *env.Env) bool { +func apmInjectEnabled(_ Package, e *env.Env) bool { switch e.InstallScript.APMInstrumentationEnabled { case env.APMInstrumentationEnabledAll, env.APMInstrumentationEnabledDocker, env.APMInstrumentationEnabledHost: return true } return false } + +func apmLanguageEnabled(p Package, e *env.Env) bool { + if !apmInjectEnabled(p, e) { + return false + } + if _, ok := e.ApmLibraries[packageToLanguage(p.Name)]; ok { + return true + } + if _, ok := e.ApmLibraries["all"]; ok { + return true + } + return false +} + +func packageToLanguage(packageName string) env.ApmLibLanguage { + lang, found := strings.CutPrefix(packageName, "datadog-apm-library-") + if !found { + return "" + } + return env.ApmLibLanguage(lang) +} diff --git a/pkg/fleet/installer/default_packages_test.go b/pkg/fleet/installer/default_packages_test.go index d631446c61d5c..77c32696cbc37 100644 --- a/pkg/fleet/installer/default_packages_test.go +++ b/pkg/fleet/installer/default_packages_test.go @@ -40,7 +40,7 @@ func TestDefaultPackages(t *testing.T) { } type testCase struct { name string - packages []defaultPackage + packages []Package env *env.Env expected []pkg } @@ -48,70 +48,133 @@ func TestDefaultPackages(t *testing.T) { tests := []testCase{ { name: "No packages", - packages: []defaultPackage{}, + packages: []Package{}, env: &env.Env{}, expected: nil, }, { name: "Package not released", - packages: []defaultPackage{{name: "datadog-agent", released: false}}, + packages: []Package{{Name: "datadog-agent", released: false}}, env: &env.Env{}, expected: nil, }, { name: "Package released", - packages: []defaultPackage{{name: "datadog-agent", released: true}}, + packages: []Package{{Name: "datadog-agent", released: true}}, env: &env.Env{}, expected: []pkg{{n: "datadog-agent", v: "latest"}}, }, { name: "Package released with remote updates", - packages: []defaultPackage{{name: "datadog-agent", released: false, releasedWithRemoteUpdates: true}}, + packages: []Package{{Name: "datadog-agent", released: false, releasedWithRemoteUpdates: true}}, env: &env.Env{RemoteUpdates: true}, expected: []pkg{{n: "datadog-agent", v: "latest"}}, }, { name: "Package released to another site", - packages: []defaultPackage{{name: "datadog-agent", releasedBySite: []string{"datadoghq.eu"}}}, + packages: []Package{{Name: "datadog-agent", releasedBySite: []string{"datadoghq.eu"}}}, env: &env.Env{Site: "datadoghq.com"}, expected: nil, }, { name: "Package released to the right site", - packages: []defaultPackage{{name: "datadog-agent", releasedBySite: []string{"datadoghq.eu"}}, {name: "datadog-package-2", releasedBySite: []string{"datadoghq.com"}}}, + packages: []Package{{Name: "datadog-agent", releasedBySite: []string{"datadoghq.eu"}}, {Name: "datadog-package-2", releasedBySite: []string{"datadoghq.com"}}}, env: &env.Env{Site: "datadoghq.eu"}, expected: []pkg{{n: "datadog-agent", v: "latest"}}, }, { name: "Package not released but forced install", - packages: []defaultPackage{{name: "datadog-agent", released: false}}, + packages: []Package{{Name: "datadog-agent", released: false}}, env: &env.Env{DefaultPackagesInstallOverride: map[string]bool{"datadog-agent": true}}, expected: []pkg{{n: "datadog-agent", v: "latest"}}, }, { name: "Package released but condition not met", - packages: []defaultPackage{{name: "datadog-agent", released: true, condition: func(e *env.Env) bool { return false }}}, + packages: []Package{{Name: "datadog-agent", released: true, condition: func(Package, *env.Env) bool { return false }}}, env: &env.Env{}, expected: nil, }, { name: "Package forced to install and version override", - packages: []defaultPackage{{name: "datadog-agent", released: false}}, + packages: []Package{{Name: "datadog-agent", released: false}}, env: &env.Env{DefaultPackagesInstallOverride: map[string]bool{"datadog-agent": true}, DefaultPackagesVersionOverride: map[string]string{"datadog-agent": "1.2.3"}}, expected: []pkg{{n: "datadog-agent", v: "1.2.3"}}, }, { name: "APM inject before agent", - packages: []defaultPackage{{name: "datadog-apm-inject", released: true}, {name: "datadog-agent", released: true}}, + packages: []Package{{Name: "datadog-apm-inject", released: true}, {Name: "datadog-agent", released: true}}, env: &env.Env{}, expected: []pkg{{n: "datadog-apm-inject", v: "latest"}, {n: "datadog-agent", v: "latest"}}, }, { name: "Package released but forced not to install", - packages: []defaultPackage{{name: "datadog-agent", released: true}}, + packages: []Package{{Name: "datadog-agent", released: true}}, env: &env.Env{DefaultPackagesInstallOverride: map[string]bool{"datadog-agent": false}}, expected: nil, }, + { + name: "Package is a language with a pinned version", + packages: []Package{ + {Name: "datadog-apm-library-java", released: true, condition: apmLanguageEnabled}, + {Name: "datadog-apm-library-ruby", released: true, condition: apmLanguageEnabled}, + {Name: "datadog-apm-library-js", released: true, condition: apmLanguageEnabled}, + }, + env: &env.Env{ + ApmLibraries: map[env.ApmLibLanguage]env.ApmLibVersion{ + "java": "1.0", + "ruby": "", + }, + InstallScript: env.InstallScriptEnv{ + APMInstrumentationEnabled: "all", + }, + }, + expected: []pkg{{n: "datadog-apm-library-java", v: "1.0-1"}, {n: "datadog-apm-library-ruby", v: "latest"}}, + }, + { + name: "Package is a language with a pinned version", + packages: []Package{ + {Name: "datadog-apm-library-java", released: true, condition: apmLanguageEnabled}, + {Name: "datadog-apm-library-ruby", released: true, condition: apmLanguageEnabled}, + {Name: "datadog-apm-library-js", released: true, condition: apmLanguageEnabled}, + }, + env: &env.Env{ + ApmLibraries: map[env.ApmLibLanguage]env.ApmLibVersion{ + "all": "", + }, + InstallScript: env.InstallScriptEnv{ + APMInstrumentationEnabled: "all", + }, + }, + expected: []pkg{ + {n: "datadog-apm-library-java", v: "latest"}, + {n: "datadog-apm-library-ruby", v: "latest"}, + {n: "datadog-apm-library-js", v: "latest"}, + }, + }, + { + name: "Override ignore package pin", + packages: []Package{ + {Name: "datadog-apm-library-java", released: false, condition: apmLanguageEnabled}, + {Name: "datadog-apm-library-ruby", released: false, condition: apmLanguageEnabled}, + {Name: "datadog-apm-library-js", released: false, condition: apmLanguageEnabled}, + }, + env: &env.Env{ + ApmLibraries: map[env.ApmLibLanguage]env.ApmLibVersion{ + "java": "1.2", + }, + InstallScript: env.InstallScriptEnv{ + APMInstrumentationEnabled: "all", + }, + DefaultPackagesInstallOverride: map[string]bool{ + "datadog-apm-library-java": true, + "datadog-apm-library-ruby": true, + }, + }, + expected: []pkg{ + {n: "datadog-apm-library-java", v: "1.2-1"}, + {n: "datadog-apm-library-ruby", v: "latest"}, + }, + }, } for _, tt := range tests { diff --git a/pkg/fleet/installer/installer.go b/pkg/fleet/installer/installer.go index dd153ec391de7..e2b9c44f40e79 100644 --- a/pkg/fleet/installer/installer.go +++ b/pkg/fleet/installer/installer.go @@ -13,6 +13,7 @@ import ( "net/http" "os" "path/filepath" + "strings" "sync" "time" @@ -102,6 +103,18 @@ func (i *installerImpl) States() (map[string]repository.State, error) { // IsInstalled checks if a package is installed. func (i *installerImpl) IsInstalled(_ context.Context, pkg string) (bool, error) { + // The install script passes the package name as either - or = + // depending on the platform so we strip the version prefix by looking for the "real" package name + hasMatch := false + for _, p := range PackagesList { + if strings.HasPrefix(pkg, p.Name) { + if hasMatch { + return false, fmt.Errorf("the package %v matches multiple known packages", pkg) + } + pkg = p.Name + hasMatch = true + } + } hasPackage, err := i.db.HasPackage(pkg) if err != nil { return false, fmt.Errorf("could not list packages: %w", err) diff --git a/test/new-e2e/tests/installer/host/host.go b/test/new-e2e/tests/installer/host/host.go index 254df7e725695..23994316bf4f3 100644 --- a/test/new-e2e/tests/installer/host/host.go +++ b/test/new-e2e/tests/installer/host/host.go @@ -161,6 +161,12 @@ func (h *Host) AssertPackageInstalledByInstaller(pkgs ...string) { } } +// AssertPackageVersion checks if a package is installed with the correct version +func (h *Host) AssertPackageVersion(pkg string, version string) { + state := h.State() + state.AssertDirExists(filepath.Join("/opt/datadog-packages/", pkg, version), 0755, "root", "root") +} + // AssertPackageInstalledByPackageManager checks if a package is installed by the package manager on the host. func (h *Host) AssertPackageInstalledByPackageManager(pkgs ...string) { for _, pkg := range pkgs { diff --git a/test/new-e2e/tests/installer/package_apm_inject_test.go b/test/new-e2e/tests/installer/package_apm_inject_test.go index 4e37710fbc09c..2e4359d3ae398 100644 --- a/test/new-e2e/tests/installer/package_apm_inject_test.go +++ b/test/new-e2e/tests/installer/package_apm_inject_test.go @@ -295,6 +295,28 @@ func (s *packageApmInjectSuite) TestInstrument() { s.assertDockerdInstrumented(injectOCIPath) } +func (s *packageApmInjectSuite) TestPackagePinning() { + // Deb install using today's defaults + err := s.RunInstallScriptWithError( + "DD_APM_INSTRUMENTATION_ENABLED=all", + "DD_APM_INSTRUMENTATION_LIBRARIES=python:2.8.2-dev,dotnet", + envForceInstall("datadog-apm-inject"), + envForceInstall("datadog-apm-library-python"), + envForceInstall("datadog-apm-library-dotnet"), + envForceInstall("datadog-agent"), + ) + defer s.Purge() + defer s.purgeInjectorDebInstall() + assert.NoError(s.T(), err) + + s.assertLDPreloadInstrumented(injectOCIPath) + s.assertSocketPath("/var/run/datadog-installer/apm.socket") + s.assertDockerdInstrumented(injectOCIPath) + + s.host.AssertPackageInstalledByInstaller("datadog-apm-library-python", "datadog-apm-library-dotnet") + s.host.AssertPackageVersion("datadog-apm-library-python", "2.8.2-dev") +} + func (s *packageApmInjectSuite) TestUninstrument() { s.host.InstallDocker() s.RunInstallScriptWithError(