feat: use bindings in checks (#502)

* feat: use bindings in checks Signed-off-by: Charles-Edouard Brétéché <[email protected]> * fix unit tests Signed-off-by: Charles-Edouard Brétéché <[email protected]> * go mod Signed-off-by: Charles-Edouard Brétéché <[email protected]> * release notes Signed-off-by: Charles-Edouard Brétéché <[email protected]> * release notes Signed-off-by: Charles-Edouard Brétéché <[email protected]> --------- Signed-off-by: Charles-Edouard Brétéché <[email protected]>
kyverno · Nov 25, 2023 · c581b40 · c581b40
1 parent 283f027
commit c581b40
Show file tree

Hide file tree

Showing 9 changed files with 51 additions and 47 deletions.
diff --git a/.release-notes/main.md b/.release-notes/main.md
@@ -18,6 +18,7 @@ Release notes for `TODO`.
 
 - Object reference in `delete` is not under the `ref` field
 - Check in `apply` and `create` operations was renamed to `expect` and is now an array of `Expectation`s (a combination of a match and a check)
+- Additional data passed to `check`s are now done using bindings (`$error`, `$stdout`, `$stderr`, etc...)
 
 ## :dizzy: New features :dizzy:
 
@@ -28,6 +29,7 @@ Release notes for `TODO`.
 
 - Fixed a kuttl migration failure in case of unsupported file name
 - Fixed a potential invalid name when migrating a kuttl test step
+- Fixed `check` set to `null` in kuttl migration command
 - Fixed a manifest discovery issue where manifests could be loaded in the wrong order
 - Fixed a manifest discovery issue where error manifests where not discovered correctly
 

diff --git a/go.mod b/go.mod
@@ -6,6 +6,7 @@ require (
 	github.com/dustinkirkland/golang-petname v0.0.0-20231002161417-6a283f1aaaf2
 	github.com/fatih/color v1.16.0
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
+	github.com/jmespath-community/go-jmespath v1.1.2-0.20231004164315-78945398586a
 	github.com/kudobuilder/kuttl v0.15.0
 	github.com/kyverno/kyverno v1.5.0-rc1.0.20231030172702-fb530626ba01
 	github.com/kyverno/kyverno-json v0.0.2-0.20231122195501-65ed240d6bf9
@@ -61,7 +62,6 @@ require (
 	github.com/imdario/mergo v0.3.16 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jinzhu/copier v0.4.0 // indirect
-	github.com/jmespath-community/go-jmespath v1.1.2-0.20231004164315-78945398586a // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/jtolds/gls v4.20.0+incompatible // indirect

diff --git a/pkg/runner/operations/apply/apply.go b/pkg/runner/operations/apply/apply.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 
+	"github.com/jmespath-community/go-jmespath/pkg/binding"
 	"github.com/kyverno/chainsaw/pkg/apis/v1alpha1"
 	"github.com/kyverno/chainsaw/pkg/client"
 	"github.com/kyverno/chainsaw/pkg/runner/cleanup"
@@ -98,12 +99,11 @@ func (o *operation) createResource(ctx context.Context) error {
 }
 
 func (o *operation) handleCheck(ctx context.Context, err error) error {
-	actual := map[string]interface{}{
-		"error":    nil,
-		"resource": o.obj,
-	}
-	if err != nil {
-		actual["error"] = err.Error()
+	bindings := binding.NewBindings()
+	if err == nil {
+		bindings = bindings.Register("$error", binding.NewBinding(nil))
+	} else {
+		bindings = bindings.Register("$error", binding.NewBinding(err.Error()))
 	}
 	// TODO refactor into a check package
 	matched := false
@@ -120,7 +120,7 @@ func (o *operation) handleCheck(ctx context.Context, err error) error {
 			}
 		}
 		matched = true
-		errs, validationErr := assert.Validate(ctx, expectation.Check.Value, actual, nil)
+		errs, validationErr := assert.Validate(ctx, expectation.Check.Value, o.obj, bindings)
 		if validationErr != nil {
 			return validationErr
 		}

diff --git a/pkg/runner/operations/apply/apply_test.go b/pkg/runner/operations/apply/apply_test.go
@@ -170,11 +170,11 @@ func Test_apply(t *testing.T) {
 			expect: []v1alpha1.Expectation{{
 				Check: v1alpha1.Check{
 					Value: map[string]interface{}{
-						"(error != null)": true,
+						"($error != null)": true,
 					},
 				},
 			}},
-			expectedErr: errors.New("(error != null): Invalid value: false: Expected value: true"),
+			expectedErr: errors.New("($error != null): Invalid value: false: Expected value: true"),
 		},
 		{
 			name:   "Unexpected create success when should fail",
@@ -190,11 +190,11 @@ func Test_apply(t *testing.T) {
 			expect: []v1alpha1.Expectation{{
 				Check: v1alpha1.Check{
 					Value: map[string]interface{}{
-						"(error != null)": true,
+						"($error != null)": true,
 					},
 				},
 			}},
-			expectedErr: errors.New("(error != null): Invalid value: false: Expected value: true"),
+			expectedErr: errors.New("($error != null): Invalid value: false: Expected value: true"),
 		},
 		{
 			name:   "Expected patch failure",
@@ -211,7 +211,7 @@ func Test_apply(t *testing.T) {
 			expect: []v1alpha1.Expectation{{
 				Check: v1alpha1.Check{
 					Value: map[string]interface{}{
-						"error": "expected patch failure",
+						"($error)": "expected patch failure",
 					},
 				},
 			}},
@@ -231,7 +231,7 @@ func Test_apply(t *testing.T) {
 			expect: []v1alpha1.Expectation{{
 				Check: v1alpha1.Check{
 					Value: map[string]interface{}{
-						"error": "expected create failure",
+						"($error)": "expected create failure",
 					},
 				},
 			}},

diff --git a/pkg/runner/operations/command/command.go b/pkg/runner/operations/command/command.go
@@ -6,6 +6,7 @@ import (
 	"os"
 	"os/exec"
 
+	"github.com/jmespath-community/go-jmespath/pkg/binding"
 	"github.com/kyverno/chainsaw/pkg/apis/v1alpha1"
 	"github.com/kyverno/chainsaw/pkg/runner/logging"
 	"github.com/kyverno/chainsaw/pkg/runner/operations"
@@ -63,15 +64,15 @@ func (o *operation) Exec(ctx context.Context) (_err error) {
 	if o.command.Check == nil || o.command.Check.Value == nil {
 		return cmdErr
 	} else {
-		actual := map[string]interface{}{
-			"error":  nil,
-			"stdout": output.Out(),
-			"stderr": output.Err(),
-		}
-		if cmdErr != nil {
-			actual["error"] = cmdErr.Error()
+		bindings := binding.NewBindings()
+		if err == nil {
+			bindings = bindings.Register("$error", binding.NewBinding(nil))
+		} else {
+			bindings = bindings.Register("$error", binding.NewBinding(err.Error()))
 		}
-		errs, err := assert.Validate(ctx, o.command.Check.Value, actual, nil)
+		bindings = bindings.Register("$stdout", binding.NewBinding(output.Out()))
+		bindings = bindings.Register("$stderr", binding.NewBinding(output.Err()))
+		errs, err := assert.Validate(ctx, o.command.Check.Value, nil, bindings)
 		if err != nil {
 			return err
 		}

diff --git a/pkg/runner/operations/create/create.go b/pkg/runner/operations/create/create.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 
+	"github.com/jmespath-community/go-jmespath/pkg/binding"
 	"github.com/kyverno/chainsaw/pkg/apis/v1alpha1"
 	"github.com/kyverno/chainsaw/pkg/client"
 	"github.com/kyverno/chainsaw/pkg/runner/cleanup"
@@ -85,12 +86,11 @@ func (o *operation) create_Resource(ctx context.Context) error {
 }
 
 func (o *operation) handleCheck(ctx context.Context, err error) error {
-	actual := map[string]interface{}{
-		"error":    nil,
-		"resource": o.obj,
-	}
-	if err != nil {
-		actual["error"] = err.Error()
+	bindings := binding.NewBindings()
+	if err == nil {
+		bindings = bindings.Register("$error", binding.NewBinding(nil))
+	} else {
+		bindings = bindings.Register("$error", binding.NewBinding(err.Error()))
 	}
 	// TODO refactor into a check package
 	matched := false
@@ -107,7 +107,7 @@ func (o *operation) handleCheck(ctx context.Context, err error) error {
 			}
 		}
 		matched = true
-		errs, validationErr := assert.Validate(ctx, expectation.Check.Value, actual, nil)
+		errs, validationErr := assert.Validate(ctx, expectation.Check.Value, o.obj, bindings)
 		if validationErr != nil {
 			return validationErr
 		}

diff --git a/pkg/runner/operations/create/create_test.go b/pkg/runner/operations/create/create_test.go
@@ -133,7 +133,7 @@ func Test_create(t *testing.T) {
 			expect: []v1alpha1.Expectation{{
 				Check: v1alpha1.Check{
 					Value: map[string]interface{}{
-						"error": "some arbitrary error",
+						"($error)": "some arbitrary error",
 					},
 				},
 			}},
@@ -167,11 +167,11 @@ func Test_create(t *testing.T) {
 			expect: []v1alpha1.Expectation{{
 				Check: v1alpha1.Check{
 					Value: map[string]interface{}{
-						"(error != null)": true,
+						"($error != null)": true,
 					},
 				},
 			}},
-			expectedErr: errors.New("(error != null): Invalid value: false: Expected value: true"),
+			expectedErr: errors.New("($error != null): Invalid value: false: Expected value: true"),
 		},
 	}
 	for _, tt := range tests {

diff --git a/pkg/runner/operations/delete/delete.go b/pkg/runner/operations/delete/delete.go
@@ -3,6 +3,7 @@ package delete
 import (
 	"context"
 
+	"github.com/jmespath-community/go-jmespath/pkg/binding"
 	"github.com/kyverno/chainsaw/pkg/apis/v1alpha1"
 	"github.com/kyverno/chainsaw/pkg/client"
 	"github.com/kyverno/chainsaw/pkg/runner/logging"
@@ -100,14 +101,13 @@ func (o *operation) handleCheck(ctx context.Context, candidate *unstructured.Uns
 	if o.check == nil || o.check.Value == nil {
 		return err
 	}
-	actual := map[string]interface{}{
-		"error":    nil,
-		"resource": candidate.Object,
+	bindings := binding.NewBindings()
+	if err == nil {
+		bindings = bindings.Register("$error", binding.NewBinding(nil))
+	} else {
+		bindings = bindings.Register("$error", binding.NewBinding(err.Error()))
 	}
-	if err != nil {
-		actual["error"] = err.Error()
-	}
-	errs, validationErr := assert.Validate(ctx, o.check.Value, actual, nil)
+	errs, validationErr := assert.Validate(ctx, o.check.Value, candidate, bindings)
 	if validationErr != nil {
 		return validationErr
 	}

diff --git a/pkg/runner/operations/script/script.go b/pkg/runner/operations/script/script.go
@@ -6,6 +6,7 @@ import (
 	"os"
 	"os/exec"
 
+	"github.com/jmespath-community/go-jmespath/pkg/binding"
 	"github.com/kyverno/chainsaw/pkg/apis/v1alpha1"
 	"github.com/kyverno/chainsaw/pkg/runner/logging"
 	"github.com/kyverno/chainsaw/pkg/runner/operations"
@@ -62,15 +63,15 @@ func (o *operation) Exec(ctx context.Context) (_err error) {
 	if o.script.Check == nil || o.script.Check.Value == nil {
 		return cmdErr
 	} else {
-		actual := map[string]interface{}{
-			"error":  nil,
-			"stdout": output.Out(),
-			"stderr": output.Err(),
-		}
-		if cmdErr != nil {
-			actual["error"] = cmdErr.Error()
+		bindings := binding.NewBindings()
+		if err == nil {
+			bindings = bindings.Register("$error", binding.NewBinding(nil))
+		} else {
+			bindings = bindings.Register("$error", binding.NewBinding(err.Error()))
 		}
-		errs, err := assert.Validate(ctx, o.script.Check.Value, actual, nil)
+		bindings = bindings.Register("$stdout", binding.NewBinding(output.Out()))
+		bindings = bindings.Register("$stderr", binding.NewBinding(output.Err()))
+		errs, err := assert.Validate(ctx, o.script.Check.Value, nil, bindings)
 		if err != nil {
 			return err
 		}