Skip to content

Commit

Permalink
feat: add metrics (#19)
Browse files Browse the repository at this point in the history
  • Loading branch information
yusank authored Nov 30, 2022
1 parent 98370d1 commit 7f5a8c5
Show file tree
Hide file tree
Showing 5 changed files with 184 additions and 15 deletions.
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ require (
github.com/evanphx/json-patch v4.12.0+incompatible
github.com/golang/mock v1.5.0
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.11.0
github.com/stretchr/testify v1.7.0
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b
gomodules.xyz/jsonpatch/v2 v2.2.0
Expand Down Expand Up @@ -51,7 +52,6 @@ require (
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v1.11.0 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.28.0 // indirect
github.com/prometheus/procfs v0.6.0 // indirect
Expand Down
2 changes: 2 additions & 0 deletions utils/dynamiclister/dynamiclister.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"k8s.io/klog/v2"

"github.com/k-cloud-labs/pkg/utils/informermanager"
"github.com/k-cloud-labs/pkg/utils/metrics"
)

// DynamicResourceLister define a cached dynamic resource lister
Expand Down Expand Up @@ -118,6 +119,7 @@ func (d *dynamicResourceListerImpl) GVKToResourceLister(gvk schema.GroupVersionK
// add to cache
go func() {
if err := d.RegisterNewResource(true, gvk); err != nil {
metrics.SyncResourceError(gvk)
klog.ErrorS(err, "RegisterNewResource got error", "gvk", gvk.String())
}
}()
Expand Down
139 changes: 139 additions & 0 deletions utils/metrics/metrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
package metrics

import (
"fmt"

"github.com/prometheus/client_golang/prometheus"
"k8s.io/apimachinery/pkg/runtime/schema"
"sigs.k8s.io/controller-runtime/pkg/metrics"
)

type ErrorType string

const (
ErrorTypeUnknown ErrorType = "unknown"
ErrorTypeCueExecute ErrorType = "cue_execute_error"
ErrTypePrepareCueParams ErrorType = "prepare_cue_params_error"

SubSystemName = "kcloudlabs"
)

var (
policyTotalNumber = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Subsystem: SubSystemName,
Name: "policy_total_number",
Help: "Total number of added policies",
},
[]string{"policy_type"},
)
overridePolicyMatchedCount = prometheus.NewCounterVec(
prometheus.CounterOpts{
Subsystem: SubSystemName,
Name: "override_policy_matched_count",
Help: "The number of resources changes matched with override policies",
},
[]string{"name", "resource_type"},
)

overridePolicyOverrideCount = prometheus.NewCounterVec(
prometheus.CounterOpts{
Subsystem: SubSystemName,
Name: "override_policy_override_count",
Help: "The number of resources changes override by override policies",
},
[]string{"name", "resource_type"},
)

validatePolicyMatchedCount = prometheus.NewCounterVec(
prometheus.CounterOpts{
Subsystem: SubSystemName,
Name: "validate_policy_matched_count",
Help: "The number of resources changes matched with validate policies",
},
[]string{"name", "resource_type"},
)

validatePolicyRejectCount = prometheus.NewCounterVec(
prometheus.CounterOpts{
Subsystem: SubSystemName,
Name: "validate_policy_reject_count",
Help: "The number of resources changes rejected by validate policies",
},
[]string{"name", "resource_type"},
)

policyErrorCount = prometheus.NewCounterVec(
prometheus.CounterOpts{
Subsystem: SubSystemName,
Name: "policy_error_count",
Help: "Count of error when policy engine handle policy",
},
[]string{"name", "resource_type", "error_type"},
)

resourceSyncErrorCount = prometheus.NewCounterVec(
prometheus.CounterOpts{
Subsystem: SubSystemName,
Name: "resource_sync_error_count",
Help: "Count of error when sync resource to cache",
},
[]string{"resource_type"},
)
)

func init() {
metrics.Registry.MustRegister(
policyTotalNumber,
overridePolicyMatchedCount,
overridePolicyOverrideCount,
validatePolicyMatchedCount,
validatePolicyRejectCount,
policyErrorCount,
resourceSyncErrorCount,
)
}

func IncrPolicy(policyType string) {
policyTotalNumber.WithLabelValues(policyType).Inc()
}

func DecPolicy(policyType string) {
policyTotalNumber.WithLabelValues(policyType).Dec()
}

func SetPolicyNumber(policyType string, n int64) {
policyTotalNumber.WithLabelValues(policyType).Set(float64(n))
}

func OverridePolicyMatched(policyName string, resourceGVK schema.GroupVersionKind) {
overridePolicyMatchedCount.WithLabelValues(policyName,
fmt.Sprintf("%s/%s/%s", resourceGVK.Group, resourceGVK.Version, resourceGVK.Kind)).Inc()
}

func OverridePolicyOverride(policyName string, resourceGVK schema.GroupVersionKind) {
overridePolicyOverrideCount.WithLabelValues(policyName,
fmt.Sprintf("%s/%s/%s", resourceGVK.Group, resourceGVK.Version, resourceGVK.Kind)).Inc()
}

func ValidatePolicyMatched(policyName string, resourceGVK schema.GroupVersionKind) {
validatePolicyMatchedCount.WithLabelValues(policyName,
fmt.Sprintf("%s/%s/%s", resourceGVK.Group, resourceGVK.Version, resourceGVK.Kind)).Inc()
}

func ValidatePolicyReject(policyName string, resourceGVK schema.GroupVersionKind) {
validatePolicyRejectCount.WithLabelValues(policyName,
fmt.Sprintf("%s/%s/%s", resourceGVK.Group, resourceGVK.Version, resourceGVK.Kind)).Inc()
}

func PolicyGotError(policyName string, resourceGVK schema.GroupVersionKind, errorType ErrorType) {
policyErrorCount.WithLabelValues(policyName,
fmt.Sprintf("%s/%s/%s", resourceGVK.Group, resourceGVK.Version, resourceGVK.Kind),
string(errorType)).Inc()
}

func SyncResourceError(resourceGVK schema.GroupVersionKind) {
resourceSyncErrorCount.WithLabelValues(
fmt.Sprintf("%s/%s/%s", resourceGVK.Group, resourceGVK.Version, resourceGVK.Kind),
).Inc()
}
49 changes: 35 additions & 14 deletions utils/overridemanager/overridemanager.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"github.com/k-cloud-labs/pkg/utils"
"github.com/k-cloud-labs/pkg/utils/cue"
"github.com/k-cloud-labs/pkg/utils/dynamiclister"
"github.com/k-cloud-labs/pkg/utils/metrics"
"github.com/k-cloud-labs/pkg/utils/util"
)

Expand Down Expand Up @@ -118,7 +119,8 @@ func (o *overrideManagerImpl) applyClusterOverridePolicies(rawObj, oldObj *unstr

appliedOverrides := &AppliedOverrides{}
for _, p := range matchingPolicyOverriders {
if err := o.applyPolicyOverriders(rawObj, oldObj, p.overriders); err != nil {
metrics.OverridePolicyMatched(p.name, rawObj.GroupVersionKind())
if err := o.applyPolicyOverriders(rawObj, oldObj, p); err != nil {
klog.ErrorS(err, "Failed to apply cluster overriders.", "clusteroverridepolicy", p.name, "resource", klog.KObj(rawObj), "operation", operation)
return nil, err
}
Expand Down Expand Up @@ -154,7 +156,8 @@ func (o *overrideManagerImpl) applyOverridePolicies(rawObj, oldObj *unstructured

appliedOverriders := &AppliedOverrides{}
for _, p := range matchingPolicyOverriders {
if err := o.applyPolicyOverriders(rawObj, oldObj, p.overriders); err != nil {
metrics.OverridePolicyMatched(p.namespace+"/"+p.name, rawObj.GroupVersionKind())
if err := o.applyPolicyOverriders(rawObj, oldObj, p); err != nil {
klog.ErrorS(err, "Failed to apply overriders.",
"overridepolicy", fmt.Sprintf("%s/%s", p.namespace, p.name), "resource", klog.KObj(rawObj), "operation", operation)
return nil, fmt.Errorf("appling policy(%v/%v) err=%v", p.namespace, p.name, err)
Expand Down Expand Up @@ -202,44 +205,62 @@ func (o *overrideManagerImpl) getOverridersFromOverridePolicies(policies []Gener
}

// applyPolicyOverriders applies OverridePolicy/ClusterOverridePolicy overriders to target object
func (o *overrideManagerImpl) applyPolicyOverriders(rawObj, oldObj *unstructured.Unstructured, overriders policyv1alpha1.Overriders) error {
if overriders.Template != nil && overriders.RenderedCue != "" {
p, err := cue.BuildCueParamsViaOverridePolicy(o.dynamicLister, rawObj, overriders.Template)
func (o *overrideManagerImpl) applyPolicyOverriders(rawObj, oldObj *unstructured.Unstructured, p policyOverriders) error {
policyName := p.name
if p.namespace != "" {
policyName = p.namespace + "/" + p.name
}
if p.overriders.Template != nil && p.overriders.RenderedCue != "" {
cp, err := cue.BuildCueParamsViaOverridePolicy(o.dynamicLister, rawObj, p.overriders.Template)
if err != nil {
metrics.PolicyGotError(policyName, rawObj.GroupVersionKind(), metrics.ErrTypePrepareCueParams)
return fmt.Errorf("BuildCueParamsViaOverridePolicy error=%w", err)
}
p.Object = rawObj
p.OldObject = oldObj
if p.OldObject == nil {
p.OldObject = &unstructured.Unstructured{Object: map[string]interface{}{}}
cp.Object = rawObj
cp.OldObject = oldObj
if cp.OldObject == nil {
cp.OldObject = &unstructured.Unstructured{Object: map[string]interface{}{}}
}
params := []cue.Parameter{
{
Object: p,
Object: cp,
Name: utils.DataParameterName,
},
}

patches, err := executeCueV2(overriders.RenderedCue, params)
patches, err := executeCueV2(p.overriders.RenderedCue, params)
if err != nil {
metrics.PolicyGotError(policyName, rawObj.GroupVersionKind(), metrics.ErrorTypeCueExecute)
return err
}

if len(patches) > 0 {
metrics.OverridePolicyOverride(policyName, rawObj.GroupVersionKind())
}

if err := applyJSONPatch(rawObj, patches); err != nil {
return err
}
}
if overriders.Cue != "" {
patches, err := executeCue(rawObj, overriders.Cue)
if p.overriders.Cue != "" {
patches, err := executeCue(rawObj, p.overriders.Cue)
if err != nil {
metrics.PolicyGotError(policyName, rawObj.GroupVersionKind(), metrics.ErrorTypeCueExecute)
return err
}
if patches != nil && len(*patches) > 0 {
metrics.OverridePolicyOverride(policyName, rawObj.GroupVersionKind())
}
if err := applyJSONPatch(rawObj, *patches); err != nil {
return err
}
}

return applyJSONPatch(rawObj, parseJSONPatchesByPlaintext(overriders.Plaintext))
if len(p.overriders.Plaintext) > 0 {
metrics.OverridePolicyOverride(policyName, rawObj.GroupVersionKind())
}

return applyJSONPatch(rawObj, parseJSONPatchesByPlaintext(p.overriders.Plaintext))
}

// applyJSONPatch applies the override on to the given unstructured object.
Expand Down
7 changes: 7 additions & 0 deletions utils/validatemanager/validatemanager.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"github.com/k-cloud-labs/pkg/utils"
"github.com/k-cloud-labs/pkg/utils/cue"
"github.com/k-cloud-labs/pkg/utils/dynamiclister"
"github.com/k-cloud-labs/pkg/utils/metrics"
"github.com/k-cloud-labs/pkg/utils/util"
)

Expand Down Expand Up @@ -81,6 +82,7 @@ func (m *validateManagerImpl) applyValidatePolicy(cvp *policyv1alpha1.ClusterVal
return &ValidateResult{Valid: true}, nil
}

metrics.ValidatePolicyMatched(cvp.Name, rawObj.GroupVersionKind())
klog.V(4).InfoS("resource matched a validate policy", "operation", operation, "policy", cvp.GroupVersionKind(),
"resource", fmt.Sprintf("%v/%v/%v", rawObj.GroupVersionKind(), rawObj.GetNamespace(), rawObj.GetName()))
for _, rule := range cvp.Spec.ValidateRules {
Expand Down Expand Up @@ -109,6 +111,7 @@ func (m *validateManagerImpl) applyValidatePolicy(cvp *policyv1alpha1.ClusterVal
klog.V(2).InfoS("Applied validate policy.",
"validatepolicy", cvp.Name, "resource", klog.KObj(rawObj), "operation", operation)
if !result.Valid {
metrics.ValidatePolicyReject(cvp.Name, rawObj.GroupVersionKind())
return result, nil
}
}
Expand All @@ -117,13 +120,15 @@ func (m *validateManagerImpl) applyValidatePolicy(cvp *policyv1alpha1.ClusterVal
if rule.Cue != "" {
result, err := executeCue(rawObj, oldObj, rule.Cue)
if err != nil {
metrics.PolicyGotError(rawObj.GetName(), rawObj.GroupVersionKind(), metrics.ErrorTypeCueExecute)
klog.ErrorS(err, "Failed to apply validate policy.",
"validatepolicy", cvp.Name, "resource", klog.KObj(rawObj), "operation", operation)
return nil, err
}
klog.V(2).InfoS("Applied validate policy.",
"validatepolicy", cvp.Name, "resource", klog.KObj(rawObj), "operation", operation)
if !result.Valid {
metrics.ValidatePolicyReject(cvp.Name, rawObj.GroupVersionKind())
return result, nil
}
}
Expand All @@ -146,6 +151,7 @@ func (m *validateManagerImpl) executeTemplate(params *cue.CueParams, rule *polic

extraParams, err := cue.BuildCueParamsViaValidatePolicy(m.dynamicClient, params.Object, rule.Template)
if err != nil {
metrics.PolicyGotError(cvpName, params.Object.GroupVersionKind(), metrics.ErrTypePrepareCueParams)
klog.ErrorS(err, "Failed to build validate policy params.",
"validatepolicy", cvpName, "resource", klog.KObj(params.Object))
return nil, err
Expand All @@ -169,6 +175,7 @@ func (m *validateManagerImpl) executeTemplate(params *cue.CueParams, rule *polic
},
})
if err != nil {
metrics.PolicyGotError(cvpName, params.Object.GroupVersionKind(), metrics.ErrorTypeCueExecute)
return nil, err
}

Expand Down

0 comments on commit 7f5a8c5

Please sign in to comment.