Skip to content

Commit

Permalink
Merge pull request #2471 from tonistiigi/v0.14.1-picks
Browse files Browse the repository at this point in the history 
[v0.14] cherry picks for v0.14.1
  • Loading branch information
@tonistiigi
tonistiigi authored May 22, 2024
2 parents 171fcbe + a3b1fae commit 59582a8
Show file tree
Hide file tree
Showing 20 changed files with 331 additions and 179 deletions.
76 changes: 41 additions & 35 deletions build/build.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -541,7 +541,7 @@ func BuildWithResultHandler(ctx context.Context, nodes []builder.Node, opt map[s
}

if pushNames != "" {
progress.Write(pw, fmt.Sprintf("merging manifest list %s", pushNames), func() error {
err := progress.Write(pw, fmt.Sprintf("merging manifest list %s", pushNames), func() error {
descs := make([]specs.Descriptor, 0, len(res))

for _, r := range res {
Expand Down Expand Up @@ -637,6 +637,9 @@ func BuildWithResultHandler(ctx context.Context, nodes []builder.Node, opt map[s
}
return nil
})
if err != nil {
return err
}
}
return nil
})
Expand Down Expand Up @@ -781,11 +784,11 @@ func calculateChildTargets(reqs map[string][]*reqForNode, opt map[string]Options
}

func waitContextDeps(ctx context.Context, index int, results *waitmap.Map, so *client.SolveOpt) error {
m := map[string]string{}
m := map[string][]string{}
for k, v := range so.FrontendAttrs {
if strings.HasPrefix(k, "context:") && strings.HasPrefix(v, "target:") {
target := resultKey(index, strings.TrimPrefix(v, "target:"))
m[target] = k
m[target] = append(m[target], k)
}
}
if len(m) == 0 {
Expand All @@ -800,7 +803,7 @@ func waitContextDeps(ctx context.Context, index int, results *waitmap.Map, so *c
return err
}

for k, v := range m {
for k, contexts := range m {
r, ok := res[k]
if !ok {
continue
Expand All @@ -815,51 +818,54 @@ func waitContextDeps(ctx context.Context, index int, results *waitmap.Map, so *c
if so.FrontendInputs == nil {
so.FrontendInputs = map[string]llb.State{}
}
if len(rr.Refs) > 0 {
for platform, r := range rr.Refs {
st, err := r.ToState()

for _, v := range contexts {
if len(rr.Refs) > 0 {
for platform, r := range rr.Refs {
st, err := r.ToState()
if err != nil {
return err
}
so.FrontendInputs[k+"::"+platform] = st
so.FrontendAttrs[v+"::"+platform] = "input:" + k + "::" + platform
metadata := make(map[string][]byte)
if dt, ok := rr.Metadata[exptypes.ExporterImageConfigKey+"/"+platform]; ok {
metadata[exptypes.ExporterImageConfigKey] = dt
}
if dt, ok := rr.Metadata["containerimage.buildinfo/"+platform]; ok {
metadata["containerimage.buildinfo"] = dt
}
if len(metadata) > 0 {
dt, err := json.Marshal(metadata)
if err != nil {
return err
}
so.FrontendAttrs["input-metadata:"+k+"::"+platform] = string(dt)
}
}
delete(so.FrontendAttrs, v)
}
if rr.Ref != nil {
st, err := rr.Ref.ToState()
if err != nil {
return err
}
so.FrontendInputs[k+"::"+platform] = st
so.FrontendAttrs[v+"::"+platform] = "input:" + k + "::" + platform
so.FrontendInputs[k] = st
so.FrontendAttrs[v] = "input:" + k
metadata := make(map[string][]byte)
if dt, ok := rr.Metadata[exptypes.ExporterImageConfigKey+"/"+platform]; ok {
if dt, ok := rr.Metadata[exptypes.ExporterImageConfigKey]; ok {
metadata[exptypes.ExporterImageConfigKey] = dt
}
if dt, ok := rr.Metadata["containerimage.buildinfo/"+platform]; ok {
if dt, ok := rr.Metadata["containerimage.buildinfo"]; ok {
metadata["containerimage.buildinfo"] = dt
}
if len(metadata) > 0 {
dt, err := json.Marshal(metadata)
if err != nil {
return err
}
so.FrontendAttrs["input-metadata:"+k+"::"+platform] = string(dt)
}
}
delete(so.FrontendAttrs, v)
}
if rr.Ref != nil {
st, err := rr.Ref.ToState()
if err != nil {
return err
}
so.FrontendInputs[k] = st
so.FrontendAttrs[v] = "input:" + k
metadata := make(map[string][]byte)
if dt, ok := rr.Metadata[exptypes.ExporterImageConfigKey]; ok {
metadata[exptypes.ExporterImageConfigKey] = dt
}
if dt, ok := rr.Metadata["containerimage.buildinfo"]; ok {
metadata["containerimage.buildinfo"] = dt
}
if len(metadata) > 0 {
dt, err := json.Marshal(metadata)
if err != nil {
return err
so.FrontendAttrs["input-metadata:"+k] = string(dt)
}
so.FrontendAttrs["input-metadata:"+k] = string(dt)
}
}
}
Expand Down
56 changes: 48 additions & 8 deletions build/driver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package build
import (
"context"
"fmt"
"sync"

"github.com/containerd/containerd/platforms"
"github.com/docker/buildx/builder"
Expand Down Expand Up @@ -46,10 +47,22 @@ func (dp resolvedNode) BuildOpts(ctx context.Context) (gateway.BuildOpts, error)

type matchMaker func(specs.Platform) platforms.MatchComparer

type cachedGroup[T any] struct {
g flightcontrol.Group[T]
cache map[int]T
cacheMu sync.Mutex
}

func newCachedGroup[T any]() cachedGroup[T] {
return cachedGroup[T]{
cache: map[int]T{},
}
}

type nodeResolver struct {
nodes []builder.Node
clients flightcontrol.Group[*client.Client]
opt flightcontrol.Group[gateway.BuildOpts]
nodes []builder.Node
clients cachedGroup[*client.Client]
buildOpts cachedGroup[gateway.BuildOpts]
}

func resolveDrivers(ctx context.Context, nodes []builder.Node, opt map[string]Options, pw progress.Writer) (map[string][]*resolvedNode, error) {
Expand All @@ -63,7 +76,9 @@ func resolveDrivers(ctx context.Context, nodes []builder.Node, opt map[string]Op

func newDriverResolver(nodes []builder.Node) *nodeResolver {
r := &nodeResolver{
nodes: nodes,
nodes: nodes,
clients: newCachedGroup[*client.Client](),
buildOpts: newCachedGroup[gateway.BuildOpts](),
}
return r
}
Expand Down Expand Up @@ -179,6 +194,7 @@ func (r *nodeResolver) resolve(ctx context.Context, ps []specs.Platform, pw prog
resolver: r,
driverIndex: 0,
})
nodeIdxs = append(nodeIdxs, 0)
} else {
for i, idx := range nodeIdxs {
node := &resolvedNode{
Expand Down Expand Up @@ -237,11 +253,24 @@ func (r *nodeResolver) boot(ctx context.Context, idxs []int, pw progress.Writer)
for i, idx := range idxs {
i, idx := i, idx
eg.Go(func() error {
c, err := r.clients.Do(ctx, fmt.Sprint(idx), func(ctx context.Context) (*client.Client, error) {
c, err := r.clients.g.Do(ctx, fmt.Sprint(idx), func(ctx context.Context) (*client.Client, error) {
if r.nodes[idx].Driver == nil {
return nil, nil
}
return driver.Boot(ctx, baseCtx, r.nodes[idx].Driver, pw)
r.clients.cacheMu.Lock()
c, ok := r.clients.cache[idx]
r.clients.cacheMu.Unlock()
if ok {
return c, nil
}
c, err := driver.Boot(ctx, baseCtx, r.nodes[idx].Driver, pw)
if err != nil {
return nil, err
}
r.clients.cacheMu.Lock()
r.clients.cache[idx] = c
r.clients.cacheMu.Unlock()
return c, nil
})
if err != nil {
return err
Expand Down Expand Up @@ -272,14 +301,25 @@ func (r *nodeResolver) opts(ctx context.Context, idxs []int, pw progress.Writer)
continue
}
eg.Go(func() error {
opt, err := r.opt.Do(ctx, fmt.Sprint(idx), func(ctx context.Context) (gateway.BuildOpts, error) {
opt := gateway.BuildOpts{}
opt, err := r.buildOpts.g.Do(ctx, fmt.Sprint(idx), func(ctx context.Context) (gateway.BuildOpts, error) {
r.buildOpts.cacheMu.Lock()
opt, ok := r.buildOpts.cache[idx]
r.buildOpts.cacheMu.Unlock()
if ok {
return opt, nil
}
_, err := c.Build(ctx, client.SolveOpt{
Internal: true,
}, "buildx", func(ctx context.Context, c gateway.Client) (*gateway.Result, error) {
opt = c.BuildOpts()
return nil, nil
}, nil)
if err != nil {
return gateway.BuildOpts{}, err
}
r.buildOpts.cacheMu.Lock()
r.buildOpts.cache[idx] = opt
r.buildOpts.cacheMu.Unlock()
return opt, err
})
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion build/opt.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ func toSolveOpt(ctx context.Context, node builder.Node, multiDriver bool, opt Op
case 1:
// valid
case 0:
if !noDefaultLoad() {
if !noDefaultLoad() && opt.PrintFunc == nil {
if nodeDriver.IsMobyDriver() {
// backwards compat for docker driver only:
// this ensures the build results in a docker image.
Expand Down
19 changes: 19 additions & 0 deletions cmd/buildx/main.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package main

import (
"context"
"fmt"
"os"

Expand All @@ -15,6 +16,7 @@ import (
cliflags "github.com/docker/cli/cli/flags"
"github.com/moby/buildkit/solver/errdefs"
"github.com/moby/buildkit/util/stack"
"go.opentelemetry.io/otel"

//nolint:staticcheck // vendored dependencies may still use this
"github.com/containerd/containerd/pkg/seed"
Expand All @@ -38,10 +40,27 @@ func runStandalone(cmd *command.DockerCli) error {
if err := cmd.Initialize(cliflags.NewClientOptions()); err != nil {
return err
}
defer flushMetrics(cmd)

rootCmd := commands.NewRootCmd(os.Args[0], false, cmd)
return rootCmd.Execute()
}

// flushMetrics will manually flush metrics from the configured
// meter provider. This is needed when running in standalone mode
// because the meter provider is initialized by the cli library,
// but the mechanism for forcing it to report is not presently
// exposed and not invoked when run in standalone mode.
// There are plans to fix that in the next release, but this is
// needed temporarily until the API for this is more thorough.
func flushMetrics(cmd *command.DockerCli) {
if mp, ok := cmd.MeterProvider().(command.MeterProvider); ok {
if err := mp.ForceFlush(context.Background()); err != nil {
otel.Handle(err)
}
}
}

func runPlugin(cmd *command.DockerCli) error {
rootCmd := commands.NewRootCmd("buildx", true, cmd)
return plugin.RunPlugin(cmd, rootCmd, manager.Metadata{
Expand Down
55 changes: 27 additions & 28 deletions commands/build.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,27 +122,26 @@ func (o *buildOptions) toControllerOptions() (*controllerapi.BuildOptions, error
}

opts := controllerapi.BuildOptions{
Allow: o.allow,
Annotations: o.annotations,
BuildArgs: buildArgs,
CgroupParent: o.cgroupParent,
ContextPath: o.contextPath,
DockerfileName: o.dockerfileName,
ExtraHosts: o.extraHosts,
Labels: labels,
NetworkMode: o.networkMode,
NoCacheFilter: o.noCacheFilter,
Platforms: o.platforms,
ShmSize: int64(o.shmSize),
Tags: o.tags,
Target: o.target,
Ulimits: dockerUlimitToControllerUlimit(o.ulimits),
Builder: o.builder,
NoCache: o.noCache,
Pull: o.pull,
ExportPush: o.exportPush,
ExportLoad: o.exportLoad,
WithProvenanceResponse: len(o.metadataFile) > 0,
Allow: o.allow,
Annotations: o.annotations,
BuildArgs: buildArgs,
CgroupParent: o.cgroupParent,
ContextPath: o.contextPath,
DockerfileName: o.dockerfileName,
ExtraHosts: o.extraHosts,
Labels: labels,
NetworkMode: o.networkMode,
NoCacheFilter: o.noCacheFilter,
Platforms: o.platforms,
ShmSize: int64(o.shmSize),
Tags: o.tags,
Target: o.target,
Ulimits: dockerUlimitToControllerUlimit(o.ulimits),
Builder: o.builder,
NoCache: o.noCache,
Pull: o.pull,
ExportPush: o.exportPush,
ExportLoad: o.exportLoad,
}

// TODO: extract env var parsing to a method easily usable by library consumers
Expand Down Expand Up @@ -207,6 +206,8 @@ func (o *buildOptions) toControllerOptions() (*controllerapi.BuildOptions, error
return nil, err
}

opts.WithProvenanceResponse = opts.PrintFunc == nil && len(o.metadataFile) > 0

return &opts, nil
}

Expand Down Expand Up @@ -268,8 +269,7 @@ func (o *buildOptionsHash) String() string {
}

func runBuild(ctx context.Context, dockerCli command.Cli, options buildOptions) (err error) {
mp := dockerCli.MeterProvider(ctx)
defer metricutil.Shutdown(ctx, mp)
mp := dockerCli.MeterProvider()

ctx, end, err := tracing.TraceCurrentCommand(ctx, "build")
if err != nil {
Expand Down Expand Up @@ -365,15 +365,14 @@ func runBuild(ctx context.Context, dockerCli command.Cli, options buildOptions)
return errors.Wrap(err, "writing image ID file")
}
}
if options.metadataFile != "" {
if err := writeMetadataFile(options.metadataFile, decodeExporterResponse(resp.ExporterResponse)); err != nil {
return err
}
}
if opts.PrintFunc != nil {
if err := printResult(opts.PrintFunc, resp.ExporterResponse); err != nil {
return err
}
} else if options.metadataFile != "" {
if err := writeMetadataFile(options.metadataFile, decodeExporterResponse(resp.ExporterResponse)); err != nil {
return err
}
}
return nil
}
Expand Down
Loading

0 comments on commit 59582a8

Please sign in to comment.