Skip to content

Commit

Permalink
bake: cache-from/cache-to options no longer print sensitive values
Browse files Browse the repository at this point in the history 
This refactors how the cache-from/cache-to composable attributes work so
they no longer print sensitive values that are automatically added.

This also expands the available syntax that works with the cache
options. It is now possible to interleave the csv syntax with the object
syntax without any problems. The canonical form is still the object
syntax and variables are resolved according to that syntax.

`cache-from` and `cache-to` now correctly ignore empty string inputs so
these can be used with variables.

Signed-off-by: Jonathan A. Sternberg <[email protected]>
  • Loading branch information
@jsternberg
jsternberg committed Dec 9, 2024
1 parent d4eca07 commit 59fa21a
Show file tree
Hide file tree
Showing 10 changed files with 432 additions and 128 deletions.
75 changes: 37 additions & 38 deletions bake/bake.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -698,30 +698,30 @@ type Target struct {
// Inherits is the only field that cannot be overridden with --set
Inherits []string `json:"inherits,omitempty" hcl:"inherits,optional" cty:"inherits"`

Annotations []string `json:"annotations,omitempty" hcl:"annotations,optional" cty:"annotations"`
Attest []string `json:"attest,omitempty" hcl:"attest,optional" cty:"attest"`
Context *string `json:"context,omitempty" hcl:"context,optional" cty:"context"`
Contexts map[string]string `json:"contexts,omitempty" hcl:"contexts,optional" cty:"contexts"`
Dockerfile *string `json:"dockerfile,omitempty" hcl:"dockerfile,optional" cty:"dockerfile"`
DockerfileInline *string `json:"dockerfile-inline,omitempty" hcl:"dockerfile-inline,optional" cty:"dockerfile-inline"`
Args map[string]*string `json:"args,omitempty" hcl:"args,optional" cty:"args"`
Labels map[string]*string `json:"labels,omitempty" hcl:"labels,optional" cty:"labels"`
Tags []string `json:"tags,omitempty" hcl:"tags,optional" cty:"tags"`
CacheFrom []*buildflags.CacheOptionsEntry `json:"cache-from,omitempty" hcl:"cache-from,optional" cty:"cache-from"`
CacheTo []*buildflags.CacheOptionsEntry `json:"cache-to,omitempty" hcl:"cache-to,optional" cty:"cache-to"`
Target *string `json:"target,omitempty" hcl:"target,optional" cty:"target"`
Secrets []*buildflags.Secret `json:"secret,omitempty" hcl:"secret,optional" cty:"secret"`
SSH []*buildflags.SSH `json:"ssh,omitempty" hcl:"ssh,optional" cty:"ssh"`
Platforms []string `json:"platforms,omitempty" hcl:"platforms,optional" cty:"platforms"`
Outputs []*buildflags.ExportEntry `json:"output,omitempty" hcl:"output,optional" cty:"output"`
Pull *bool `json:"pull,omitempty" hcl:"pull,optional" cty:"pull"`
NoCache *bool `json:"no-cache,omitempty" hcl:"no-cache,optional" cty:"no-cache"`
NetworkMode *string `json:"network,omitempty" hcl:"network,optional" cty:"network"`
NoCacheFilter []string `json:"no-cache-filter,omitempty" hcl:"no-cache-filter,optional" cty:"no-cache-filter"`
ShmSize *string `json:"shm-size,omitempty" hcl:"shm-size,optional"`
Ulimits []string `json:"ulimits,omitempty" hcl:"ulimits,optional"`
Call *string `json:"call,omitempty" hcl:"call,optional" cty:"call"`
Entitlements []string `json:"entitlements,omitempty" hcl:"entitlements,optional" cty:"entitlements"`
Annotations []string `json:"annotations,omitempty" hcl:"annotations,optional" cty:"annotations"`
Attest []string `json:"attest,omitempty" hcl:"attest,optional" cty:"attest"`
Context *string `json:"context,omitempty" hcl:"context,optional" cty:"context"`
Contexts map[string]string `json:"contexts,omitempty" hcl:"contexts,optional" cty:"contexts"`
Dockerfile *string `json:"dockerfile,omitempty" hcl:"dockerfile,optional" cty:"dockerfile"`
DockerfileInline *string `json:"dockerfile-inline,omitempty" hcl:"dockerfile-inline,optional" cty:"dockerfile-inline"`
Args map[string]*string `json:"args,omitempty" hcl:"args,optional" cty:"args"`
Labels map[string]*string `json:"labels,omitempty" hcl:"labels,optional" cty:"labels"`
Tags []string `json:"tags,omitempty" hcl:"tags,optional" cty:"tags"`
CacheFrom buildflags.CacheOptions `json:"cache-from,omitempty" hcl:"cache-from,optional" cty:"cache-from"`
CacheTo buildflags.CacheOptions `json:"cache-to,omitempty" hcl:"cache-to,optional" cty:"cache-to"`
Target *string `json:"target,omitempty" hcl:"target,optional" cty:"target"`
Secrets []*buildflags.Secret `json:"secret,omitempty" hcl:"secret,optional" cty:"secret"`
SSH []*buildflags.SSH `json:"ssh,omitempty" hcl:"ssh,optional" cty:"ssh"`
Platforms []string `json:"platforms,omitempty" hcl:"platforms,optional" cty:"platforms"`
Outputs []*buildflags.ExportEntry `json:"output,omitempty" hcl:"output,optional" cty:"output"`
Pull *bool `json:"pull,omitempty" hcl:"pull,optional" cty:"pull"`
NoCache *bool `json:"no-cache,omitempty" hcl:"no-cache,optional" cty:"no-cache"`
NetworkMode *string `json:"network,omitempty" hcl:"network,optional" cty:"network"`
NoCacheFilter []string `json:"no-cache-filter,omitempty" hcl:"no-cache-filter,optional" cty:"no-cache-filter"`
ShmSize *string `json:"shm-size,omitempty" hcl:"shm-size,optional"`
Ulimits []string `json:"ulimits,omitempty" hcl:"ulimits,optional"`
Call *string `json:"call,omitempty" hcl:"call,optional" cty:"call"`
Entitlements []string `json:"entitlements,omitempty" hcl:"entitlements,optional" cty:"entitlements"`
// IMPORTANT: if you add more fields here, do not forget to update newOverrides/AddOverrides and docs/bake-reference.md.

// linked is a private field to mark a target used as a linked one
Expand All @@ -742,8 +742,8 @@ func (t *Target) normalize() {
t.Secrets = removeDupes(t.Secrets)
t.SSH = removeDupes(t.SSH)
t.Platforms = removeDupesStr(t.Platforms)
t.CacheFrom = removeDupes(t.CacheFrom)
t.CacheTo = removeDupes(t.CacheTo)
t.CacheFrom = t.CacheFrom.Normalize()
t.CacheTo = t.CacheTo.Normalize()
t.Outputs = removeDupes(t.Outputs)
t.NoCacheFilter = removeDupesStr(t.NoCacheFilter)
t.Ulimits = removeDupesStr(t.Ulimits)
Expand Down Expand Up @@ -824,7 +824,7 @@ func (t *Target) Merge(t2 *Target) {
t.Platforms = t2.Platforms
}
if t2.CacheFrom != nil { // merge
t.CacheFrom = append(t.CacheFrom, t2.CacheFrom...)
t.CacheFrom = t.CacheFrom.Merge(t2.CacheFrom)
}
if t2.CacheTo != nil { // no merge
t.CacheTo = t2.CacheTo
Expand Down Expand Up @@ -1372,17 +1372,12 @@ func toBuildOpt(t *Target, inp *Input) (*build.Options, error) {
}
}

cacheImports := make([]*controllerapi.CacheOptionsEntry, len(t.CacheFrom))
for i, ci := range t.CacheFrom {
cacheImports[i] = ci.ToPB()
if t.CacheFrom != nil {
bo.CacheFrom = controllerapi.CreateCaches(t.CacheFrom.ToPB())
}
bo.CacheFrom = controllerapi.CreateCaches(cacheImports)

cacheExports := make([]*controllerapi.CacheOptionsEntry, len(t.CacheTo))
for i, ce := range t.CacheTo {
cacheExports[i] = ce.ToPB()
if t.CacheTo != nil {
bo.CacheTo = controllerapi.CreateCaches(t.CacheTo.ToPB())
}
bo.CacheTo = controllerapi.CreateCaches(cacheExports)

outputs := make([]*controllerapi.ExportEntry, len(t.Outputs))
for i, output := range t.Outputs {
Expand Down Expand Up @@ -1625,9 +1620,13 @@ func parseArrValue[T any, PT arrValue[T]](s []string) ([]*T, error) {
return outputs, nil
}

func parseCacheArrValues(s []string) ([]*buildflags.CacheOptionsEntry, error) {
outs := make([]*buildflags.CacheOptionsEntry, 0, len(s))
func parseCacheArrValues(s []string) (buildflags.CacheOptions, error) {
var outs buildflags.CacheOptions
for _, in := range s {
if in == "" {
continue
}

if !strings.Contains(in, "=") {
// This is ref only format. Each field in the CSV is its own entry.
fields, err := csvvalue.Fields(in, nil)
Expand Down
4 changes: 2 additions & 2 deletions bake/compose.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -353,14 +353,14 @@ func (t *Target) composeExtTarget(exts map[string]interface{}) error {
if err != nil {
return err
}
t.CacheFrom = removeDupes(append(t.CacheFrom, cacheFrom...))
t.CacheFrom = t.CacheFrom.Merge(cacheFrom)
}
if len(xb.CacheTo) > 0 {
cacheTo, err := parseCacheArrValues(xb.CacheTo)
if err != nil {
return err
}
t.CacheTo = removeDupes(append(t.CacheTo, cacheTo...))
t.CacheTo = t.CacheTo.Merge(cacheTo)
}
if len(xb.Secrets) > 0 {
secrets, err := parseArrValue[buildflags.Secret](xb.Secrets)
Expand Down
6 changes: 3 additions & 3 deletions bake/hcl_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -606,7 +606,7 @@ func TestHCLAttrsCapsuleType(t *testing.T) {
target "app" {
cache-from = [
{ type = "registry", ref = "user/app:cache" },
{ type = "local", src = "path/to/cache" },
"type=local,src=path/to/cache",
]
cache-to = [
Expand Down Expand Up @@ -649,7 +649,7 @@ func TestHCLAttrsCapsuleTypeVars(t *testing.T) {
target "app" {
cache-from = [
{ type = "registry", ref = "user/app:cache" },
{ type = "local", src = "path/to/cache" },
"type=local,src=path/to/cache",
]
cache-to = [ target.app.cache-from[0] ]
Expand All @@ -674,7 +674,7 @@ func TestHCLAttrsCapsuleTypeVars(t *testing.T) {
output = [ "type=oci,dest=../${foo}.tar" ]
secret = [
{ id = target.app.output[0].type, src = "/local/secret" },
{ id = target.app.output[0].type, src = "/${target.app.cache-from[1].type}/secret" },
]
}
`)
Expand Down
97 changes: 61 additions & 36 deletions bake/hclparser/type_implied_ext.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,75 +10,100 @@ import (
"github.com/zclconf/go-cty/cty/gocty"
)

type CapsuleValue interface {
// FromCtyValue will initialize this value using a cty.Value.
FromCtyValue(in cty.Value, path cty.Path) error

type ToCtyValueConverter interface {
// ToCtyValue will convert this capsule value into a native
// cty.Value. This should not return a capsule type.
ToCtyValue() cty.Value
}

type FromCtyValueConverter interface {
// FromCtyValue will initialize this value using a cty.Value.
FromCtyValue(in cty.Value, path cty.Path) error
}

type extensionType int

const (
nativeTypeExtension extensionType = iota
unwrapCapsuleValueExtension extensionType = iota
)

func impliedTypeExt(rt reflect.Type, _ cty.Path) (cty.Type, error) {
if rt.AssignableTo(capsuleValueType) {
if rt.Kind() != reflect.Pointer {
rt = reflect.PointerTo(rt)
}

if isCapsuleType(rt) {
return capsuleValueCapsuleType(rt), nil
}
return cty.NilType, errdefs.ErrNotImplemented
}

var (
capsuleValueType = reflect.TypeFor[CapsuleValue]()
capsuleValueTypes sync.Map
)
func isCapsuleType(rt reflect.Type) bool {
fromCtyValueType := reflect.TypeFor[FromCtyValueConverter]()
toCtyValueType := reflect.TypeFor[ToCtyValueConverter]()
return rt.Implements(fromCtyValueType) && rt.Implements(toCtyValueType)
}

var capsuleValueTypes sync.Map

func capsuleValueCapsuleType(rt reflect.Type) cty.Type {
if val, loaded := capsuleValueTypes.Load(rt); loaded {
if rt.Kind() != reflect.Pointer {
panic("capsule value must be a pointer")
}

elem := rt.Elem()
if val, loaded := capsuleValueTypes.Load(elem); loaded {
return val.(cty.Type)
}

// First time used.
ety := cty.CapsuleWithOps(rt.Name(), rt.Elem(), &cty.CapsuleOps{
toCtyValueType := reflect.TypeFor[ToCtyValueConverter]()

// First time used. Initialize new capsule ops.
ops := &cty.CapsuleOps{
ConversionTo: func(_ cty.Type) func(cty.Value, cty.Path) (any, error) {
return func(in cty.Value, p cty.Path) (any, error) {
rv := reflect.New(rt.Elem()).Interface()
if err := rv.(CapsuleValue).FromCtyValue(in, p); err != nil {
rv := reflect.New(elem).Interface()
if err := rv.(FromCtyValueConverter).FromCtyValue(in, p); err != nil {
return nil, err
}
return rv, nil
}
},
ConversionFrom: func(want cty.Type) func(any, cty.Path) (cty.Value, error) {
return func(in any, _ cty.Path) (cty.Value, error) {
v := in.(CapsuleValue).ToCtyValue()
rv := reflect.ValueOf(in).Convert(toCtyValueType)
v := rv.Interface().(ToCtyValueConverter).ToCtyValue()
return convert.Convert(v, want)
}
},
ExtensionData: func(key any) any {
switch key {
case nativeTypeExtension:
zero := reflect.Zero(rt).Interface()
return zero.(CapsuleValue).ToCtyValue().Type()
default:
return nil
case unwrapCapsuleValueExtension:
zero := reflect.Zero(elem).Interface()
if conv, ok := zero.(ToCtyValueConverter); ok {
return conv.ToCtyValue().Type()
}

zero = reflect.Zero(rt).Interface()
if conv, ok := zero.(ToCtyValueConverter); ok {
return conv.ToCtyValue().Type()
}
}
return nil
},
})
}

// Attempt to store the new type. Use whichever was loaded first in the case of a race condition.
val, _ := capsuleValueTypes.LoadOrStore(rt, ety)
// Attempt to store the new type. Use whichever was loaded first in the case
// of a race condition.
ety := cty.CapsuleWithOps(elem.Name(), elem, ops)
val, _ := capsuleValueTypes.LoadOrStore(elem, ety)
return val.(cty.Type)
}

// ToNativeValue will convert a value to only native cty types which will
// remove capsule types if possible.
func ToNativeValue(in cty.Value) cty.Value {
want := toNativeType(in.Type())
// UnwrapCtyValue will unwrap capsule type values into their native cty value
// equivalents if possible.
func UnwrapCtyValue(in cty.Value) cty.Value {
want := toCtyValueType(in.Type())
if in.Type().Equals(want) {
return in
} else if out, err := convert.Convert(in, want); err == nil {
Expand All @@ -87,25 +112,25 @@ func ToNativeValue(in cty.Value) cty.Value {
return cty.NullVal(want)
}

func toNativeType(in cty.Type) cty.Type {
func toCtyValueType(in cty.Type) cty.Type {
if et := in.MapElementType(); et != nil {
return cty.Map(toNativeType(*et))
return cty.Map(toCtyValueType(*et))
}

if et := in.SetElementType(); et != nil {
return cty.Set(toNativeType(*et))
return cty.Set(toCtyValueType(*et))
}

if et := in.ListElementType(); et != nil {
return cty.List(toNativeType(*et))
return cty.List(toCtyValueType(*et))
}

if in.IsObjectType() {
var optional []string
inAttrTypes := in.AttributeTypes()
outAttrTypes := make(map[string]cty.Type, len(inAttrTypes))
for name, typ := range inAttrTypes {
outAttrTypes[name] = toNativeType(typ)
outAttrTypes[name] = toCtyValueType(typ)
if in.AttributeOptional(name) {
optional = append(optional, name)
}
Expand All @@ -117,13 +142,13 @@ func toNativeType(in cty.Type) cty.Type {
inTypes := in.TupleElementTypes()
outTypes := make([]cty.Type, len(inTypes))
for i, typ := range inTypes {
outTypes[i] = toNativeType(typ)
outTypes[i] = toCtyValueType(typ)
}
return cty.Tuple(outTypes)
}

if in.IsCapsuleType() {
if out := in.CapsuleExtensionData(nativeTypeExtension); out != nil {
if out := in.CapsuleExtensionData(unwrapCapsuleValueExtension); out != nil {
return out.(cty.Type)
}
return cty.DynamicPseudoType
Expand All @@ -137,5 +162,5 @@ func ToCtyValue(val any, ty cty.Type) (cty.Value, error) {
if err != nil {
return out, err
}
return ToNativeValue(out), nil
return UnwrapCtyValue(out), nil
}
Loading

0 comments on commit 59fa21a

Please sign in to comment.