Skip to content

Commit

Permalink
bake: various fixes for composable attributes
Browse files Browse the repository at this point in the history 
This changes how the composable attributes are implemented and provides
various fixes to the first iteration.

Cache-from and cache-to now no longer print sensitive values that are
automatically added. These automatically added attributes are added when
the protobuf is created rather than at the time of parsing so they will
no longer be printed. If they are part of the original configuration
file, they will still be printed.

Empty strings will now be skipped. This was the original behavior and
composable attributes removed this functionality accidentally. This
functionality is now restored.

This also expands the available syntax that works with each of the
composable attributes. It is now possible to interleave the csv syntax
with the object syntax without any problems. The canonical form is still
the object syntax and variables are resolved according to that syntax.

Signed-off-by: Jonathan A. Sternberg <[email protected]>
  • Loading branch information
@jsternberg
jsternberg committed Dec 9, 2024
1 parent d4eca07 commit 4e32acc
Show file tree
Hide file tree
Showing 20 changed files with 899 additions and 311 deletions.
103 changes: 48 additions & 55 deletions bake/bake.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -698,30 +698,30 @@ type Target struct {
// Inherits is the only field that cannot be overridden with --set
Inherits []string `json:"inherits,omitempty" hcl:"inherits,optional" cty:"inherits"`

Annotations []string `json:"annotations,omitempty" hcl:"annotations,optional" cty:"annotations"`
Attest []string `json:"attest,omitempty" hcl:"attest,optional" cty:"attest"`
Context *string `json:"context,omitempty" hcl:"context,optional" cty:"context"`
Contexts map[string]string `json:"contexts,omitempty" hcl:"contexts,optional" cty:"contexts"`
Dockerfile *string `json:"dockerfile,omitempty" hcl:"dockerfile,optional" cty:"dockerfile"`
DockerfileInline *string `json:"dockerfile-inline,omitempty" hcl:"dockerfile-inline,optional" cty:"dockerfile-inline"`
Args map[string]*string `json:"args,omitempty" hcl:"args,optional" cty:"args"`
Labels map[string]*string `json:"labels,omitempty" hcl:"labels,optional" cty:"labels"`
Tags []string `json:"tags,omitempty" hcl:"tags,optional" cty:"tags"`
CacheFrom []*buildflags.CacheOptionsEntry `json:"cache-from,omitempty" hcl:"cache-from,optional" cty:"cache-from"`
CacheTo []*buildflags.CacheOptionsEntry `json:"cache-to,omitempty" hcl:"cache-to,optional" cty:"cache-to"`
Target *string `json:"target,omitempty" hcl:"target,optional" cty:"target"`
Secrets []*buildflags.Secret `json:"secret,omitempty" hcl:"secret,optional" cty:"secret"`
SSH []*buildflags.SSH `json:"ssh,omitempty" hcl:"ssh,optional" cty:"ssh"`
Platforms []string `json:"platforms,omitempty" hcl:"platforms,optional" cty:"platforms"`
Outputs []*buildflags.ExportEntry `json:"output,omitempty" hcl:"output,optional" cty:"output"`
Pull *bool `json:"pull,omitempty" hcl:"pull,optional" cty:"pull"`
NoCache *bool `json:"no-cache,omitempty" hcl:"no-cache,optional" cty:"no-cache"`
NetworkMode *string `json:"network,omitempty" hcl:"network,optional" cty:"network"`
NoCacheFilter []string `json:"no-cache-filter,omitempty" hcl:"no-cache-filter,optional" cty:"no-cache-filter"`
ShmSize *string `json:"shm-size,omitempty" hcl:"shm-size,optional"`
Ulimits []string `json:"ulimits,omitempty" hcl:"ulimits,optional"`
Call *string `json:"call,omitempty" hcl:"call,optional" cty:"call"`
Entitlements []string `json:"entitlements,omitempty" hcl:"entitlements,optional" cty:"entitlements"`
Annotations []string `json:"annotations,omitempty" hcl:"annotations,optional" cty:"annotations"`
Attest []string `json:"attest,omitempty" hcl:"attest,optional" cty:"attest"`
Context *string `json:"context,omitempty" hcl:"context,optional" cty:"context"`
Contexts map[string]string `json:"contexts,omitempty" hcl:"contexts,optional" cty:"contexts"`
Dockerfile *string `json:"dockerfile,omitempty" hcl:"dockerfile,optional" cty:"dockerfile"`
DockerfileInline *string `json:"dockerfile-inline,omitempty" hcl:"dockerfile-inline,optional" cty:"dockerfile-inline"`
Args map[string]*string `json:"args,omitempty" hcl:"args,optional" cty:"args"`
Labels map[string]*string `json:"labels,omitempty" hcl:"labels,optional" cty:"labels"`
Tags []string `json:"tags,omitempty" hcl:"tags,optional" cty:"tags"`
CacheFrom buildflags.CacheOptions `json:"cache-from,omitempty" hcl:"cache-from,optional" cty:"cache-from"`
CacheTo buildflags.CacheOptions `json:"cache-to,omitempty" hcl:"cache-to,optional" cty:"cache-to"`
Target *string `json:"target,omitempty" hcl:"target,optional" cty:"target"`
Secrets buildflags.Secrets `json:"secret,omitempty" hcl:"secret,optional" cty:"secret"`
SSH buildflags.SSHKeys `json:"ssh,omitempty" hcl:"ssh,optional" cty:"ssh"`
Platforms []string `json:"platforms,omitempty" hcl:"platforms,optional" cty:"platforms"`
Outputs buildflags.Exports `json:"output,omitempty" hcl:"output,optional" cty:"output"`
Pull *bool `json:"pull,omitempty" hcl:"pull,optional" cty:"pull"`
NoCache *bool `json:"no-cache,omitempty" hcl:"no-cache,optional" cty:"no-cache"`
NetworkMode *string `json:"network,omitempty" hcl:"network,optional" cty:"network"`
NoCacheFilter []string `json:"no-cache-filter,omitempty" hcl:"no-cache-filter,optional" cty:"no-cache-filter"`
ShmSize *string `json:"shm-size,omitempty" hcl:"shm-size,optional"`
Ulimits []string `json:"ulimits,omitempty" hcl:"ulimits,optional"`
Call *string `json:"call,omitempty" hcl:"call,optional" cty:"call"`
Entitlements []string `json:"entitlements,omitempty" hcl:"entitlements,optional" cty:"entitlements"`
// IMPORTANT: if you add more fields here, do not forget to update newOverrides/AddOverrides and docs/bake-reference.md.

// linked is a private field to mark a target used as a linked one
Expand All @@ -739,12 +739,12 @@ func (t *Target) normalize() {
t.Annotations = removeDupesStr(t.Annotations)
t.Attest = removeAttestDupes(t.Attest)
t.Tags = removeDupesStr(t.Tags)
t.Secrets = removeDupes(t.Secrets)
t.Secrets = t.Secrets.Normalize()
t.SSH = removeDupes(t.SSH)
t.Platforms = removeDupesStr(t.Platforms)
t.CacheFrom = removeDupes(t.CacheFrom)
t.CacheTo = removeDupes(t.CacheTo)
t.Outputs = removeDupes(t.Outputs)
t.CacheFrom = t.CacheFrom.Normalize()
t.CacheTo = t.CacheTo.Normalize()
t.Outputs = t.Outputs.Normalize()
t.NoCacheFilter = removeDupesStr(t.NoCacheFilter)
t.Ulimits = removeDupesStr(t.Ulimits)

Expand Down Expand Up @@ -815,16 +815,16 @@ func (t *Target) Merge(t2 *Target) {
t.Attest = removeAttestDupes(t.Attest)
}
if t2.Secrets != nil { // merge
t.Secrets = append(t.Secrets, t2.Secrets...)
t.Secrets = t.Secrets.Merge(t2.Secrets)
}
if t2.SSH != nil { // merge
t.SSH = append(t.SSH, t2.SSH...)
t.SSH = t.SSH.Merge(t2.SSH)
}
if t2.Platforms != nil { // no merge
t.Platforms = t2.Platforms
}
if t2.CacheFrom != nil { // merge
t.CacheFrom = append(t.CacheFrom, t2.CacheFrom...)
t.CacheFrom = t.CacheFrom.Merge(t2.CacheFrom)
}
if t2.CacheTo != nil { // no merge
t.CacheTo = t2.CacheTo
Expand Down Expand Up @@ -1333,13 +1333,8 @@ func toBuildOpt(t *Target, inp *Input) (*build.Options, error) {
}
bo.Platforms = platforms

secrets := make([]*controllerapi.Secret, len(t.Secrets))
for i, s := range t.Secrets {
secrets[i] = s.ToPB()
}
bo.SecretSpecs = secrets

secretAttachment, err := controllerapi.CreateSecrets(secrets)
bo.SecretSpecs = t.Secrets.ToPB()
secretAttachment, err := controllerapi.CreateSecrets(bo.SecretSpecs)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -1372,24 +1367,14 @@ func toBuildOpt(t *Target, inp *Input) (*build.Options, error) {
}
}

cacheImports := make([]*controllerapi.CacheOptionsEntry, len(t.CacheFrom))
for i, ci := range t.CacheFrom {
cacheImports[i] = ci.ToPB()
}
bo.CacheFrom = controllerapi.CreateCaches(cacheImports)

cacheExports := make([]*controllerapi.CacheOptionsEntry, len(t.CacheTo))
for i, ce := range t.CacheTo {
cacheExports[i] = ce.ToPB()
if t.CacheFrom != nil {
bo.CacheFrom = controllerapi.CreateCaches(t.CacheFrom.ToPB())
}
bo.CacheTo = controllerapi.CreateCaches(cacheExports)

outputs := make([]*controllerapi.ExportEntry, len(t.Outputs))
for i, output := range t.Outputs {
outputs[i] = output.ToPB()
if t.CacheTo != nil {
bo.CacheTo = controllerapi.CreateCaches(t.CacheTo.ToPB())
}

bo.Exports, bo.ExportsLocalPathsTemporary, err = controllerapi.CreateExports(outputs)
bo.Exports, bo.ExportsLocalPathsTemporary, err = controllerapi.CreateExports(t.Outputs.ToPB())
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -1616,6 +1601,10 @@ type arrValue[B any] interface {
func parseArrValue[T any, PT arrValue[T]](s []string) ([]*T, error) {
outputs := make([]*T, 0, len(s))
for _, text := range s {
if text == "" {
continue
}

output := new(T)
if err := PT(output).UnmarshalText([]byte(text)); err != nil {
return nil, err
Expand All @@ -1625,9 +1614,13 @@ func parseArrValue[T any, PT arrValue[T]](s []string) ([]*T, error) {
return outputs, nil
}

func parseCacheArrValues(s []string) ([]*buildflags.CacheOptionsEntry, error) {
outs := make([]*buildflags.CacheOptionsEntry, 0, len(s))
func parseCacheArrValues(s []string) (buildflags.CacheOptions, error) {
var outs buildflags.CacheOptions
for _, in := range s {
if in == "" {
continue
}

if !strings.Contains(in, "=") {
// This is ref only format. Each field in the CSV is its own entry.
fields, err := csvvalue.Fields(in, nil)
Expand Down
20 changes: 20 additions & 0 deletions bake/bake_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2019,6 +2019,26 @@ target "app" {
})
}

// https://github.com/docker/buildx/pull/428
// https://github.com/docker/buildx/issues/2822
func TestEmptyAttribute(t *testing.T) {
fp := File{
Name: "docker-bake.hcl",
Data: []byte(`
target "app" {
output = [""]
}
`),
}

ctx := context.TODO()

m, _, err := ReadTargets(ctx, []File{fp}, []string{"app"}, nil, nil, &EntitlementConf{})
require.Equal(t, 1, len(m))
require.Len(t, m["app"].Outputs, 0)
require.NoError(t, err)
}

func stringify[V fmt.Stringer](values []V) []string {
s := make([]string, len(values))
for i, v := range values {
Expand Down
10 changes: 5 additions & 5 deletions bake/compose.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -353,28 +353,28 @@ func (t *Target) composeExtTarget(exts map[string]interface{}) error {
if err != nil {
return err
}
t.CacheFrom = removeDupes(append(t.CacheFrom, cacheFrom...))
t.CacheFrom = t.CacheFrom.Merge(cacheFrom)
}
if len(xb.CacheTo) > 0 {
cacheTo, err := parseCacheArrValues(xb.CacheTo)
if err != nil {
return err
}
t.CacheTo = removeDupes(append(t.CacheTo, cacheTo...))
t.CacheTo = t.CacheTo.Merge(cacheTo)
}
if len(xb.Secrets) > 0 {
secrets, err := parseArrValue[buildflags.Secret](xb.Secrets)
if err != nil {
return err
}
t.Secrets = removeDupes(append(t.Secrets, secrets...))
t.Secrets = t.Secrets.Merge(secrets)
}
if len(xb.SSH) > 0 {
ssh, err := parseArrValue[buildflags.SSH](xb.SSH)
if err != nil {
return err
}
t.SSH = removeDupes(append(t.SSH, ssh...))
t.SSH = t.SSH.Merge(ssh)
slices.SortFunc(t.SSH, func(a, b *buildflags.SSH) int {
return a.Less(b)
})
Expand All @@ -387,7 +387,7 @@ func (t *Target) composeExtTarget(exts map[string]interface{}) error {
if err != nil {
return err
}
t.Outputs = removeDupes(append(t.Outputs, outputs...))
t.Outputs = t.Outputs.Merge(outputs)
}
if xb.Pull != nil {
t.Pull = xb.Pull
Expand Down
12 changes: 7 additions & 5 deletions bake/hcl_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -606,7 +606,7 @@ func TestHCLAttrsCapsuleType(t *testing.T) {
target "app" {
cache-from = [
{ type = "registry", ref = "user/app:cache" },
{ type = "local", src = "path/to/cache" },
"type=local,src=path/to/cache",
]
cache-to = [
Expand All @@ -615,6 +615,7 @@ func TestHCLAttrsCapsuleType(t *testing.T) {
output = [
{ type = "oci", dest = "../out.tar" },
"type=local,dest=../out",
]
secret = [
Expand All @@ -633,7 +634,7 @@ func TestHCLAttrsCapsuleType(t *testing.T) {
require.NoError(t, err)

require.Equal(t, 1, len(c.Targets))
require.Equal(t, []string{"type=oci,dest=../out.tar"}, stringify(c.Targets[0].Outputs))
require.Equal(t, []string{"type=local,dest=../out", "type=oci,dest=../out.tar"}, stringify(c.Targets[0].Outputs))
require.Equal(t, []string{"type=local,src=path/to/cache", "user/app:cache"}, stringify(c.Targets[0].CacheFrom))
require.Equal(t, []string{"type=local,dest=path/to/cache"}, stringify(c.Targets[0].CacheTo))
require.Equal(t, []string{"id=mysecret,src=/local/secret", "id=mysecret2,env=TOKEN"}, stringify(c.Targets[0].Secrets))
Expand All @@ -649,13 +650,14 @@ func TestHCLAttrsCapsuleTypeVars(t *testing.T) {
target "app" {
cache-from = [
{ type = "registry", ref = "user/app:cache" },
{ type = "local", src = "path/to/cache" },
"type=local,src=path/to/cache",
]
cache-to = [ target.app.cache-from[0] ]
output = [
{ type = "oci", dest = "../out.tar" },
"type=local,dest=../out",
]
secret = [
Expand All @@ -674,7 +676,7 @@ func TestHCLAttrsCapsuleTypeVars(t *testing.T) {
output = [ "type=oci,dest=../${foo}.tar" ]
secret = [
{ id = target.app.output[0].type, src = "/local/secret" },
{ id = target.app.output[0].type, src = "/${target.app.cache-from[1].type}/secret" },
]
}
`)
Expand All @@ -696,7 +698,7 @@ func TestHCLAttrsCapsuleTypeVars(t *testing.T) {
}

app := findTarget(t, "app")
require.Equal(t, []string{"type=oci,dest=../out.tar"}, stringify(app.Outputs))
require.Equal(t, []string{"type=local,dest=../out", "type=oci,dest=../out.tar"}, stringify(app.Outputs))
require.Equal(t, []string{"type=local,src=path/to/cache", "user/app:cache"}, stringify(app.CacheFrom))
require.Equal(t, []string{"user/app:cache"}, stringify(app.CacheTo))
require.Equal(t, []string{"id=mysecret,src=/local/secret"}, stringify(app.Secrets))
Expand Down
Loading

0 comments on commit 4e32acc

Please sign in to comment.