Merge remote-tracking branch 'origin/main' into romain.marcadier/zerolog

# Conflicts:
#	internal/jobserver/pkgs/resolve.go

.github/workflows/validate.yml

@@ -267,7 +267,7 @@ jobs:
           - macos
           - ubuntu
           - windows
-        go-version: [oldstable, stable]
+        go-version: [oldstable, stable, '~1.24.0-rc.1']
         build-mode: [DRIVER]
         include:
           # Alternate build modes (only on ubuntu, latest go; to save CI time)

_integration-tests/go.mod

@@ -7,6 +7,7 @@ replace github.com/DataDog/orchestrion => ../
 require (
 	cloud.google.com/go/pubsub v1.45.1
 	github.com/99designs/gqlgen v0.17.57
+	github.com/DataDog/go-libddwaf/v3 v3.5.1
 	github.com/DataDog/orchestrion v1.0.1
 	github.com/IBM/sarama v1.43.3
 	github.com/Shopify/sarama v1.38.1
@@ -87,7 +88,6 @@ require (
 	github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 // indirect
 	github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 // indirect
 	github.com/DataDog/datadog-go/v5 v5.5.0 // indirect
-	github.com/DataDog/go-libddwaf/v3 v3.5.1 // indirect
 	github.com/DataDog/go-runtime-metrics-internal v0.0.3 // indirect
 	github.com/DataDog/go-sqllexer v0.0.17 // indirect
 	github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect

_integration-tests/tests/os/lfi.go

@@ -18,6 +18,7 @@ import (
 
 	"datadoghq.dev/orchestrion/_integration-tests/utils"
 	"datadoghq.dev/orchestrion/_integration-tests/validator/trace"
+	waf "github.com/DataDog/go-libddwaf/v3"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"gopkg.in/DataDog/dd-trace-go.v1/appsec/events"
@@ -33,6 +34,9 @@ func (tc *TestCase) Setup(t *testing.T) {
 	if runtime.GOOS == "windows" {
 		t.Skip("appsec does not support Windows")
 	}
+	if ok, err := waf.Health(); !ok {
+		t.Skip("WAF is not available:", err)
+	}
 
 	t.Setenv("DD_APPSEC_RULES", "../testdata/rasp-only-rules.json")
 	t.Setenv("DD_APPSEC_ENABLED", "true")

internal/cmd/server.go

@@ -14,10 +14,12 @@ import (
 	"time"
 
 	"github.com/DataDog/orchestrion/internal/filelock"
+	"github.com/DataDog/orchestrion/internal/goflags"
 	"github.com/DataDog/orchestrion/internal/jobserver"
 	"github.com/DataDog/orchestrion/internal/jobserver/client"
 	"github.com/fsnotify/fsnotify"
 	"github.com/rs/zerolog"
+	"gopkg.in/yaml.v3"
 
 	"github.com/urfave/cli/v2"
 )
@@ -33,18 +35,31 @@ var Server = &cli.Command{
 		},
 		&cli.IntFlag{
 			Name:        "port",
-			Usage:       "Choose a port to listen on",
+			Usage:       "Choose a port to listen on.",
 			Value:       -1,
 			DefaultText: "random",
 		},
 		&cli.DurationFlag{
 			Name:  "inactivity-timeout",
-			Usage: "Automatically shut down after a period without any connected client",
+			Usage: "Automatically shut down after a period without any connected client.",
 			Value: time.Minute,
 		},
 		&cli.BoolFlag{
 			Name:  "nats-logging",
-			Usage: "Enable NATS server logging",
+			Usage: "Enable NATS server logging.",
+		},
+		&cli.StringFlag{
+			Name:        "build-flags",
+			Usage:       "Specify the 'go build' flags to use when resolving packages. This is specified as a YAML array and must start with a valid go subcommand (e.g, 'build').",
+			DefaultText: "Looked up the process hierarchy",
+			Action: func(_ *cli.Context, val string) error {
+				var args []string
+				if err := yaml.Unmarshal([]byte(val), &args); err != nil {
+					return cli.Exit(fmt.Errorf("invalid -build-flags value: %w", err), 2)
+				}
+				goflags.SetFlags(".", args)
+				return nil
+			},
 		},
 	},
 	Hidden: true,

internal/jobserver/pkgs/resolve.go

@@ -160,8 +160,10 @@ func (s *service) resolve(ctx context.Context, req *ResolveRequest) (ResolveResp
 				Mode:
 				// We need the export file (the whole point of the resolution)
 				packages.NeedExportFile |
-					// We want to also resolve transitive dependencies, so we need Deps & Imports
-					packages.NeedDeps | packages.NeedImports |
+					// We want to also resolve transitive dependencies, so we need Deps & Imports. We also
+					// need CompiledGoFiles in order to see imports possibly added by the toolchain (cgo,
+					// cover, etc...)
+					packages.NeedCompiledGoFiles | packages.NeedDeps | packages.NeedImports |
 					// Finally, we need the resolved package import path
 					packages.NeedName,
 				Dir:        req.Dir,
@@ -175,6 +177,9 @@ func (s *service) resolve(ctx context.Context, req *ResolveRequest) (ResolveResp
 			log.Error().Str("pattern", req.Pattern).Err(err).Msg("pkgs.Resolve failed")
 			return nil, err
 		}
+		if len(pkgs) == 0 {
+			return nil, fmt.Errorf("no packages returned for pattern: %q", req.Pattern)
+		}
 
 		resp := make(ResolveResponse)
 		var errs error

internal/toolexec/aspect/linkdeps/linkdeps.go

@@ -48,7 +48,7 @@ func FromImportConfig(importcfg *importcfg.ImportConfig) (LinkDeps, error) {
 			return LinkDeps{}, fmt.Errorf("reading %s from %s=%s: %w", Filename, importPath, archivePath, err)
 		}
 
-		for _, dep := range ld.Dependencies() {
+		for dep := range ld.deps {
 			if _, satisfied := importcfg.PackageFile[dep]; satisfied {
 				// This transitive link-time dependency is already satisfied at
 				// compile-time, so we don't need to carry it over.

internal/toolexec/aspect/oncompile.go

@@ -184,33 +184,39 @@ func (w Weaver) OnCompile(ctx context.Context, cmd *proxy.CompileCommand) (err e
 		log.Debug().Stringer("kind", kind).Str("import-path", depImportPath).Msg("Recording synthetic " + linkdeps.Filename + " dependency")
 		linkDeps.Add(depImportPath)
 
-		if kind == typed.ImportStatement {
-			// Imported packages need to be provided in the compilation's importcfg file
-			deps, err := resolvePackageFiles(ctx, depImportPath, cmd.WorkDir)
+		if kind != typed.ImportStatement {
+			// We cannot attempt to resolve link-time dependencies (relocation targets), as these are
+			// typically used to avoid creating dependency cycles. Corrollary to this, the `link.deps`
+			// file will not contain transitive closures for these packages, so we need to resolve these
+			// at link-time.
+			continue
+		}
+
+		// Imported packages need to be provided in the compilation's importcfg file
+		deps, err := resolvePackageFiles(ctx, depImportPath, cmd.WorkDir)
+		if err != nil {
+			return fmt.Errorf("resolving woven dependency on %s: %w", depImportPath, err)
+		}
+		for dep, archive := range deps {
+			deps, err := linkdeps.FromArchive(archive)
 			if err != nil {
-				return fmt.Errorf("resolving woven dependency on %s: %w", depImportPath, err)
+				return fmt.Errorf("reading %s from %s[%s]: %w", linkdeps.Filename, dep, archive, err)
 			}
-			for dep, archive := range deps {
-				deps, err := linkdeps.FromArchive(archive)
-				if err != nil {
-					return fmt.Errorf("reading %s from %s[%s]: %w", linkdeps.Filename, dep, archive, err)
-				}
-				log.Debug().Str("import-path", dep).Msg("Processing " + linkdeps.Filename + " dependencies")
-				for _, tDep := range deps.Dependencies() {
-					if _, found := imports.PackageFile[tDep]; !found {
-						log.Debug().Str("import-path", dep).Str("transitive", tDep).Str("inherited-from", depImportPath).Msg("Copying transitive " + linkdeps.Filename + " dependency")
-						linkDeps.Add(tDep)
-					}
+			log.Debug().Str("import-path", dep).Msg("Processing " + linkdeps.Filename + " dependencies")
+			for _, tDep := range deps.Dependencies() {
+				if _, found := imports.PackageFile[tDep]; !found {
+					log.Debug().Str("import-path", dep).Str("transitive", tDep).Str("inherited-from", depImportPath).Msg("Copying transitive " + linkdeps.Filename + " dependency")
+					linkDeps.Add(tDep)
 				}
+			}
 
-				if _, ok := imports.PackageFile[dep]; ok {
-					// Already part of natural dependencies, nothing to do...
-					continue
-				}
-				log.Debug().Str("import-path", dep).Str("inherited-from", depImportPath).Str("archive", archive).Msg("Recording transitive dependency")
-				imports.PackageFile[dep] = archive
-				regUpdated = true
+			if _, ok := imports.PackageFile[dep]; ok {
+				// Already part of natural dependencies, nothing to do...
+				continue
 			}
+			log.Debug().Str("import-path", dep).Str("inherited-from", depImportPath).Str("archive", archive).Msg("Recording transitive dependency")
+			imports.PackageFile[dep] = archive
+			regUpdated = true
 		}
 	}
 

internal/toolexec/aspect/onlink.go

@@ -39,6 +39,7 @@ func (Weaver) OnLink(ctx context.Context, cmd *proxy.LinkCommand) error {
 				continue
 			}
 
+			log.Tracef("Resolving %s dependency on %q...\n", linkdeps.Filename, depPath)
 			deps, err := resolvePackageFiles(ctx, depPath, cmd.WorkDir)
 			if err != nil {
 				return fmt.Errorf("resolving %q: %w", depPath, err)

internal/toolexec/aspect/resolve.go

@@ -45,10 +45,18 @@ func resolvePackageFiles(ctx context.Context, importPath string, workDir string)
 	}
 
 	// Check for missing archives...
+	var found bool
 	for ip, arch := range archives {
 		if arch == "" {
 			return nil, fmt.Errorf("failed to resolve archive for %q", ip)
 		}
+		if ip == importPath {
+			found = true
+		}
+	}
+
+	if !found {
+		return nil, fmt.Errorf("resolution did not include requested package %q", importPath)
 	}
 
 	return archives, nil