8614 regex replication filters

src/pkg/reg/filter/artifact.go

@@ -19,6 +19,8 @@ import (
 
 	"github.com/goharbor/harbor/src/pkg/reg/model"
 	"github.com/goharbor/harbor/src/pkg/reg/util"
+
+	"regexp"
 )
 
 // DoFilterArtifacts filter the artifacts according to the filters
@@ -35,6 +37,10 @@ func BuildArtifactFilters(filters []*model.Filter) (ArtifactFilters, error) {
 	var fs ArtifactFilters
 	for _, filter := range filters {
 		var f ArtifactFilter
+
+		// following block builds ArtifactFilters based on the type of filter
+		// filter types are the ones shown on UI, like Tag, Label
+		// it builds for filters using double star and regex
 		switch filter.Type {
 		case model.FilterTypeLabel:
 			f = &artifactLabelFilter{
@@ -46,6 +52,18 @@ func BuildArtifactFilters(filters []*model.Filter) (ArtifactFilters, error) {
 				pattern:    filter.Value.(string),
 				decoration: filter.Decoration,
 			}
+
+		case model.FilterTypeTagRegex:
+			f = &artifactTagFilterRegex{
+				pattern:    filter.Value.(string),
+				decoration: filter.Decoration,
+			}
+
+		case model.FilterTypeLabelRegex:
+			f = &artifactLabelFilterRegex{
+				labels:     filter.Value.([]string),
+				decoration: filter.Decoration,
+			}
 		}
 		if f != nil {
 			fs = append(fs, f)
@@ -227,3 +245,131 @@ func (a *artifactTagFilter) Filter(artifacts []*model.Artifact) ([]*model.Artifa
 	}
 	return result, nil
 }
+
+type artifactTagFilterRegex struct {
+	// regex pattern
+	pattern string
+	// "matches", "excludes"
+	decoration string
+}
+
+func (a *artifactTagFilterRegex) Filter(artifacts []*model.Artifact) ([]*model.Artifact, error) {
+	if len(a.pattern) == 0 {
+		return artifacts, nil
+	}
+
+	// Compiling regex & checking if its valid
+	filterRegexPattern, err := regexp.Compile(a.pattern)
+	if err != nil {
+		return nil, err
+	}
+
+	var result []*model.Artifact
+	for _, artifact := range artifacts {
+		// for individual artifact, use its own tags to match, reserve the matched tags.
+		// for accessory artifact, use the parent tags to match,
+		var tagsForMatching []string
+		if artifact.IsAcc {
+			tagsForMatching = append(tagsForMatching, artifact.ParentTags...)
+		} else {
+			tagsForMatching = append(tagsForMatching, artifact.Tags...)
+		}
+
+		// untagged artifact
+		if len(tagsForMatching) == 0 {
+			// filter matching using regex
+			match := filterRegexPattern.MatchString("")
+
+			if a.decoration == model.Excludes {
+				if !match {
+					result = append(result, artifact)
+				}
+			} else {
+				if match {
+					result = append(result, artifact)
+				}
+			}
+			continue
+		}
+
+		// tagged artifact
+		var tags []string
+		for _, tag := range tagsForMatching {
+			// filter matching using regex
+			match := filterRegexPattern.MatchString(tag)
+
+			if a.decoration == model.Excludes {
+				if !match {
+					tags = append(tags, tag)
+				}
+			} else {
+				if match {
+					tags = append(tags, tag)
+				}
+			}
+		}
+		if len(tags) == 0 {
+			continue
+		}
+		// copy a new artifact here to avoid changing the original one
+		if artifact.IsAcc {
+			result = append(result, &model.Artifact{
+				Type:   artifact.Type,
+				Digest: artifact.Digest,
+				Labels: artifact.Labels,
+				Tags:   artifact.Tags, // use its own tags to replicate
+			})
+		} else {
+			result = append(result, &model.Artifact{
+				Type:   artifact.Type,
+				Digest: artifact.Digest,
+				Labels: artifact.Labels,
+				Tags:   tags, // only replicate the matched tags
+			})
+		}
+	}
+	return result, nil
+}
+
+type artifactLabelFilterRegex struct {
+	labels []string
+	// "matches", "excludes"
+	decoration string
+}
+
+func (a *artifactLabelFilterRegex) Filter(artifacts []*model.Artifact) ([]*model.Artifact, error) {
+	if len(a.labels) == 0 {
+		return artifacts, nil
+	}
+	var result []*model.Artifact
+	for _, artifact := range artifacts {
+		match := true
+	outer:
+		for _, label := range a.labels {
+			filterRegexPattern, err := regexp.Compile(label)
+			if err != nil {
+				return nil, err
+			}
+
+			for _, lbl := range artifact.Labels {
+				exists := filterRegexPattern.MatchString(lbl)
+
+				if !exists {
+					match = false
+					break outer
+				}
+			}
+		}
+		// add the artifact to the result list if it contains all labels defined for the filter
+		if a.decoration == model.Excludes {
+			if !match {
+				result = append(result, artifact)
+			}
+		} else {
+			if match {
+				result = append(result, artifact)
+			}
+		}
+	}
+	return result, nil
+}

src/pkg/reg/filter/resource_test.go

@@ -1,13 +1,96 @@
 package filter
 
 import (
+	"fmt"
 	"testing"
 
 	"github.com/stretchr/testify/require"
 
 	"github.com/goharbor/harbor/src/pkg/reg/model"
 )
 
+func TestArtifactTagRegexFilters(t *testing.T) {
+	var artifacts = []*model.Artifact{
+		{
+			Type:   model.ResourceTypeArtifact,
+			Digest: "aaaaa",
+			Tags: []string{
+				"test1",
+				"test2",
+				"harbor1",
+				"1.8.0",
+			},
+		},
+		{
+			Type:   model.ResourceTypeArtifact,
+			Digest: "bbbbb",
+			Tags: []string{
+				"test3",
+				"harbor2",
+			},
+		},
+		{
+			Type:   model.ResourceTypeArtifact,
+			Digest: "ccccc",
+			Tags: []string{
+				"harbor3",
+				"1.9.01",
+			},
+		},
+		{
+			Type:   model.ResourceTypeArtifact,
+			Digest: "ddddd",
+		},
+	}
+
+	var filters = []*model.Filter{
+
+		// the following pattern is from doublestar testing
+		// it works but number of filtered artifacts is mismatched
+		// so the test validating below will fail
+
+		{
+			Type:  model.FilterTypeTagRegex,
+			Value: `^((\d\d?).(\d\d?).(\d\d?(-stable)?))$`,
+		},
+	}
+
+	artFilters, err := BuildArtifactFilters(filters)
+	require.Nil(t, err)
+
+	arts, err := artFilters.Filter(artifacts)
+	require.Nil(t, err)
+	require.Equal(t, 2, len(arts))
+	require.EqualValues(t, "aaaaa", arts[0].Digest)
+	fmt.Println(arts[0].Tags)
+	require.EqualValues(t, []string{"1.8.0"}, arts[0].Tags)
+	require.EqualValues(t, "ccccc", arts[1].Digest)
+	require.EqualValues(t, []string{"1.9.01"}, arts[1].Tags)
+
+	filters = []*model.Filter{
+		{
+			Type:       model.FilterTypeTagRegex,
+			Value:      "test*",
+			Decoration: model.Excludes,
+		},
+	}
+
+	artFilters, err = BuildArtifactFilters(filters)
+	require.Nil(t, err)
+
+	arts, err = artFilters.Filter(artifacts)
+	require.Nil(t, err)
+	require.Equal(t, 4, len(arts))
+	require.EqualValues(t, "aaaaa", arts[0].Digest)
+	require.EqualValues(t, []string{"harbor1", "1.8.0"}, arts[0].Tags)
+	require.EqualValues(t, "bbbbb", arts[1].Digest)
+	require.EqualValues(t, []string{"harbor2"}, arts[1].Tags)
+	require.EqualValues(t, "ccccc", arts[2].Digest)
+	require.EqualValues(t, []string{"harbor3", "1.9.01"}, arts[2].Tags)
+	require.EqualValues(t, "ddddd", arts[3].Digest)
+	require.Nil(t, arts[3].Tags)
+}
+
 func TestArtifactTagFilters(t *testing.T) {
 	var artifacts = []*model.Artifact{
 		{
@@ -161,3 +244,83 @@ func TestArtifactLabelFilters(t *testing.T) {
 	require.EqualValues(t, "ddddd", arts[1].Digest)
 	require.Nil(t, arts[1].Labels)
 }
+
+func TestArtifactLabelFiltersRegex(t *testing.T) {
+	var artifacts = []*model.Artifact{
+		{
+			Type:   model.ResourceTypeArtifact,
+			Digest: "aaaaa",
+			Tags: []string{
+				"test1",
+				"test2",
+				"harbor1",
+			},
+			Labels: []string{
+				"label1",
+			},
+		},
+		{
+			Type:   model.ResourceTypeArtifact,
+			Digest: "bbbbb",
+			Tags: []string{
+				"test3",
+				"harbor2",
+			},
+			Labels: []string{
+				"label1",
+				"label2",
+			},
+		},
+		{
+			Type:   model.ResourceTypeArtifact,
+			Digest: "ccccc",
+			Tags: []string{
+				"harbor3",
+			},
+			Labels: []string{
+				"label3",
+			},
+		},
+		{
+			Type:   model.ResourceTypeArtifact,
+			Digest: "ddddd",
+		},
+	}
+
+	var filters = []*model.Filter{
+		{
+			Type:  model.FilterTypeLabelRegex,
+			Value: []string{`label*`},
+		},
+	}
+
+	artFilters, err := BuildArtifactFilters(filters)
+	require.Nil(t, err)
+
+	arts, err := artFilters.Filter(artifacts)
+
+	require.Nil(t, err)
+	require.Equal(t, 4, len(arts))
+	require.EqualValues(t, "aaaaa", arts[0].Digest)
+	require.EqualValues(t, []string{"label1"}, arts[0].Labels)
+	require.EqualValues(t, "bbbbb", arts[1].Digest)
+	require.EqualValues(t, []string{"label1", "label2"}, arts[1].Labels)
+
+	filters = []*model.Filter{
+		{
+			Type:       model.FilterTypeLabelRegex,
+			Value:      []string{`label1`},
+			Decoration: model.Excludes,
+		},
+	}
+
+	artFilters, err = BuildArtifactFilters(filters)
+	require.Nil(t, err)
+
+	arts, err = artFilters.Filter(artifacts)
+	require.Nil(t, err)
+	require.Equal(t, 2, len(arts))
+	require.EqualValues(t, "bbbbb", arts[0].Digest)
+	require.EqualValues(t, []string{"label1", "label2"}, arts[0].Labels)
+	require.EqualValues(t, "ccccc", arts[1].Digest)
+}

src/pkg/reg/model/policy.go

@@ -18,10 +18,12 @@ import "github.com/goharbor/harbor/src/lib/errors"
 
 // const definition
 const (
-	FilterTypeResource = "resource"
-	FilterTypeName     = "name"
-	FilterTypeTag      = "tag"
-	FilterTypeLabel    = "label"
+	FilterTypeResource   = "resource"
+	FilterTypeName       = "name"
+	FilterTypeTag        = "tag"
+	FilterTypeLabel      = "label"
+	FilterTypeLabelRegex = "labelRegex"
+	FilterTypeTagRegex   = "tagRegex"
 
 	TriggerTypeManual     = "manual"
 	TriggerTypeScheduled  = "scheduled"
@@ -42,7 +44,7 @@ type Filter struct {
 
 func (f *Filter) Validate() error {
 	switch f.Type {
-	case FilterTypeResource, FilterTypeName, FilterTypeTag:
+	case FilterTypeResource, FilterTypeName, FilterTypeTag, FilterTypeTagRegex:
 		value, ok := f.Value.(string)
 		if !ok {
 			return errors.New(nil).WithCode(errors.BadRequestCode).
@@ -61,7 +63,7 @@ func (f *Filter) Validate() error {
 					WithMessage("only tag and label filter support decoration")
 			}
 		}
-	case FilterTypeLabel:
+	case FilterTypeLabel, FilterTypeLabelRegex:
 		labels, ok := f.Value.([]interface{})
 		if !ok {
 			return errors.New(nil).WithCode(errors.BadRequestCode).