Merge pull request #1021 from threefoldtech/development_proxy_multi_s…

…tatus_filtering

allow filtering nodes by list of statuses

grid-cli/internal/filters/filters.go

@@ -69,10 +69,9 @@ func buildGenericFilter(mrus, srus, hrus, ips *uint64, farmIDs []uint64, domain
 	if hrus != nil {
 		freeHRUs = convertGBToBytes(*hrus)
 	}
-	status := "up"
 	rented := false
 	return types.NodeFilter{
-		Status:  &status,
+		Status:  []string{"up"},
 		FreeMRU: freeMRUs,
 		FreeSRU: freeSRUs,
 		FreeHRU: freeHRUs,

grid-client/deployer/node_filter.go

@@ -176,8 +176,7 @@ func getNodes(ctx context.Context, tfPlugin TFPluginClient, options types.NodeFi
 }
 
 var (
-	trueVal  = true
-	statusUp = "up"
+	trueVal = true
 )
 
 // GetPublicNode return public node ID
@@ -192,7 +191,7 @@ func GetPublicNode(ctx context.Context, tfPlugin TFPluginClient, preferredNodes
 		tfPlugin,
 		types.NodeFilter{
 			IPv4:   &trueVal,
-			Status: &statusUp,
+			Status: []string{"up"},
 		},
 		nil,
 		nil,

grid-client/deployer/node_filter_test.go

@@ -83,12 +83,11 @@ func ExampleFilterNodes() {
 	}
 
 	trueVal := true
-	statusUp := "up"
 	freeMRU := uint64(2048)
 	freeSRU := uint64(2048)
 
 	filter := types.NodeFilter{
-		Status:  &statusUp,
+		Status:  []string{"up"},
 		IPv4:    &trueVal,
 		FreeMRU: &freeMRU,
 		FreeSRU: &freeSRU,

grid-client/integration_tests/node_filter.go

@@ -6,7 +6,6 @@ import (
 )
 
 var (
-	statusUp         = "up"
 	minRootfs uint64 = 2
 	minMemory uint64 = 2
 	minCPU           = 2
@@ -91,7 +90,7 @@ func generateNodeFilter(opts ...nodeFilterOpts) types.NodeFilter {
 
 	nodeFilter := types.NodeFilter{
 		FarmIDs: []uint64{1}, // freefarm is used in tests
-		Status:  &statusUp,
+		Status:  []string{"up"},
 		FreeSRU: convertGBToBytes(cfg.freeSRU + minRootfs),
 	}
 

grid-proxy/CHANGELOG.md

@@ -12,6 +12,13 @@ Check `/version` on any instance to know the version.
 
 ## Releases
 
+### v0.15.4
+
+---
+
+- `feat` consider nodes with extra_fee as dedicated
+- `feat` allow filtering nodes by list of statuses
+
 ### v0.15.3
 
 ---

grid-proxy/charts/gridproxy/Chart.yaml

@@ -20,6 +20,6 @@ version: 1.0.0
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
-appVersion: 0.15.3
+appVersion: 0.15.4
 
 # make sure to update the changelog with the changes in this release

grid-proxy/internal/explorer/db/postgres.go

@@ -125,7 +125,7 @@ func (d *PostgresDatabase) GetStats(ctx context.Context, filter types.StatsFilte
 
 	condition := "TRUE"
 	if filter.Status != nil {
-		condition = nodestatus.DecideNodeStatusCondition(*filter.Status)
+		condition = nodestatus.DecideNodeStatusCondition(filter.Status)
 	}
 
 	if res := d.gormDB.WithContext(ctx).
@@ -351,7 +351,7 @@ func (d *PostgresDatabase) farmTableQuery(ctx context.Context, filter types.Farm
 	if filter.NodeAvailableFor != nil || filter.NodeFreeHRU != nil ||
 		filter.NodeCertified != nil || filter.NodeFreeMRU != nil ||
 		filter.NodeFreeSRU != nil || filter.NodeHasGPU != nil ||
-		filter.NodeRentedBy != nil || filter.NodeStatus != nil ||
+		filter.NodeRentedBy != nil || (filter.NodeStatus != nil && len(filter.NodeStatus) != 0) ||
 		filter.NodeTotalCRU != nil || filter.Country != nil ||
 		filter.Region != nil {
 		q.Joins(`RIGHT JOIN (?) AS resources_cache on resources_cache.farm_id = farm.farm_id`, nodeQuery).
@@ -407,8 +407,8 @@ func (d *PostgresDatabase) GetFarms(ctx context.Context, filter types.FarmFilter
 		nodeQuery = nodeQuery.Where("LOWER(resources_cache.region) = LOWER(?)", *filter.Region)
 	}
 
-	if filter.NodeStatus != nil {
-		condition := nodestatus.DecideNodeStatusCondition(*filter.NodeStatus)
+	if filter.NodeStatus != nil && len(filter.NodeStatus) != 0 {
+		condition := nodestatus.DecideNodeStatusCondition(filter.NodeStatus)
 		nodeQuery = nodeQuery.Where(condition)
 	}
 
@@ -532,7 +532,7 @@ func (d *PostgresDatabase) GetNodes(ctx context.Context, filter types.NodeFilter
 
 	condition := "TRUE"
 	if filter.Status != nil {
-		condition = nodestatus.DecideNodeStatusCondition(*filter.Status)
+		condition = nodestatus.DecideNodeStatusCondition(filter.Status)
 	}
 
 	q = q.Where(condition)

grid-proxy/internal/explorer/param_parser_test.go

@@ -12,14 +12,12 @@ import (
 func TestDecodesNodeFilterAndLimitParams(t *testing.T) {
 	r := &http.Request{
 		URL: &url.URL{
-			RawQuery: "farm_ids=1,2&status=up&size=10",
+			RawQuery: "farm_ids=1,2&status=up,standby&size=10",
 		},
 	}
-	ids := []uint64{1, 2}
-	status := "up"
 	expectedFilter := types.NodeFilter{
-		FarmIDs: ids,
-		Status:  &status,
+		FarmIDs: []uint64{1, 2},
+		Status:  []string{"up", "standby"},
 	}
 	expectedLimit := types.Limit{
 		Size: 10,
@@ -40,9 +38,8 @@ func TestEmptyParam(t *testing.T) {
 			RawQuery: "free_mru=&status=up&size=10",
 		},
 	}
-	status := "up"
 	expectedFilter := types.NodeFilter{
-		Status: &status,
+		Status: []string{"up"},
 	}
 	expectedLimit := types.Limit{
 		Size: 10,

grid-proxy/internal/indexer/utils.go

@@ -10,9 +10,8 @@ import (
 )
 
 func queryUpNodes(ctx context.Context, database db.Database, nodeTwinIdChan chan uint32) {
-	status := "up"
 	filter := types.NodeFilter{
-		Status: &status,
+		Status: []string{"up"},
 	}
 	limit := types.Limit{Size: 100, Page: 1}
 	hasNext := true

grid-proxy/pkg/client/grid_client_test.go

@@ -32,15 +32,14 @@ var (
 )
 
 func nodesFilterValues() (types.NodeFilter, types.Limit, string) {
-	Up := "up"
 	Egypt := "Egypt"
 	Mansoura := "Mansoura"
 	Freefarm := "Freefarm"
 	trueVal := true
 	falseVal := false
 	ints := []uint64{0, 1, 2, 3, 4, 5, 6}
 	f := types.NodeFilter{
-		Status:       &Up,
+		Status:       []string{"up"},
 		FreeMRU:      &ints[1],
 		FreeHRU:      &ints[2],
 		FreeSRU:      &ints[3],
@@ -344,12 +343,11 @@ func testSuccess(t *testing.T, f ProxyFunc) {
 }
 
 func TestPrepareURL(t *testing.T) {
-	status := "st"
 	freeMRU := uint64(10)
 	farmIDs := []uint64{1, 2, 3}
 	dedicated := true
 	filter := types.NodeFilter{
-		Status:    &status,
+		Status:    []string{"st"},
 		FreeMRU:   &freeMRU,
 		FarmIDs:   farmIDs,
 		Dedicated: &dedicated,

grid-proxy/pkg/nodestatus/nodestatus.go

@@ -24,7 +24,7 @@ var (
 )
 
 // return the condition to be used in the SQL query to get the nodes with the given status.
-func DecideNodeStatusCondition(status string) string {
+func DecideNodeStatusCondition(statuses []string) string {
 	nodeUpInterval := time.Now().Unix() - int64(nodeUpStateFactor)*int64(nodeUpReportInterval.Seconds())
 	nodeStandbyInterval := time.Now().Unix() - int64(nodeStandbyStateFactor)*int64(nodeStandbyReportInterval.Seconds())
 
@@ -33,14 +33,18 @@ func DecideNodeStatusCondition(status string) string {
 	inStandbyInterval := fmt.Sprintf("node.updated_at >= %d", nodeStandbyInterval)
 	outStandbyInterval := fmt.Sprintf("node.updated_at < %d", nodeStandbyInterval)
 
-	condition := "TRUE"
+	condition := ""
+	conditions := map[string]string{
+		"up":      fmt.Sprintf(`%s AND (%s OR (%s))`, inUpInterval, nilPower, poweredOn),
+		"down":    fmt.Sprintf(`(%s AND (%s OR (%s))) OR %s`, outUpInterval, nilPower, poweredOn, outStandbyInterval),
+		"standby": fmt.Sprintf(`((%s) OR (%s) OR (%s)) AND %s`, poweredOff, poweringOff, poweringOn, inStandbyInterval),
+	}
 
-	if status == "up" {
-		condition = fmt.Sprintf(`%s AND (%s OR (%s))`, inUpInterval, nilPower, poweredOn)
-	} else if status == "down" {
-		condition = fmt.Sprintf(`(%s AND (%s OR (%s))) OR %s`, outUpInterval, nilPower, poweredOn, outStandbyInterval)
-	} else if status == "standby" {
-		condition = fmt.Sprintf(`((%s) OR (%s) OR (%s)) AND %s`, poweredOff, poweringOff, poweringOn, inStandbyInterval)
+	for idx, status := range statuses {
+		if idx != 0 && idx < len(statuses) {
+			condition += " OR "
+		}
+		condition += "(" + conditions[status] + ")"
 	}
 
 	return condition

grid-proxy/pkg/types/farms.go

@@ -23,25 +23,25 @@ type PublicIP struct {
 
 // FarmFilter farm filters
 type FarmFilter struct {
-	FreeIPs           *uint64 `schema:"free_ips,omitempty"`
-	TotalIPs          *uint64 `schema:"total_ips,omitempty"`
-	StellarAddress    *string `schema:"stellar_address,omitempty"`
-	PricingPolicyID   *uint64 `schema:"pricing_policy_id,omitempty"`
-	FarmID            *uint64 `schema:"farm_id,omitempty"`
-	TwinID            *uint64 `schema:"twin_id,omitempty"`
-	Name              *string `schema:"name,omitempty"`
-	NameContains      *string `schema:"name_contains,omitempty"`
-	CertificationType *string `schema:"certification_type,omitempty"`
-	Dedicated         *bool   `schema:"dedicated,omitempty"`
-	NodeFreeMRU       *uint64 `schema:"node_free_mru,omitempty"`
-	NodeFreeHRU       *uint64 `schema:"node_free_hru,omitempty"`
-	NodeFreeSRU       *uint64 `schema:"node_free_sru,omitempty"`
-	NodeTotalCRU      *uint64 `schema:"node_total_cru,omitempty"`
-	NodeStatus        *string `schema:"node_status,omitempty"`
-	NodeRentedBy      *uint64 `schema:"node_rented_by,omitempty"`
-	NodeAvailableFor  *uint64 `schema:"node_available_for,omitempty"`
-	NodeHasGPU        *bool   `schema:"node_has_gpu,omitempty"`
-	NodeCertified     *bool   `schema:"node_certified,omitempty"`
-	Country           *string `schema:"country,omitempty"`
-	Region            *string `schema:"region,omitempty"`
+	FreeIPs           *uint64  `schema:"free_ips,omitempty"`
+	TotalIPs          *uint64  `schema:"total_ips,omitempty"`
+	StellarAddress    *string  `schema:"stellar_address,omitempty"`
+	PricingPolicyID   *uint64  `schema:"pricing_policy_id,omitempty"`
+	FarmID            *uint64  `schema:"farm_id,omitempty"`
+	TwinID            *uint64  `schema:"twin_id,omitempty"`
+	Name              *string  `schema:"name,omitempty"`
+	NameContains      *string  `schema:"name_contains,omitempty"`
+	CertificationType *string  `schema:"certification_type,omitempty"`
+	Dedicated         *bool    `schema:"dedicated,omitempty"`
+	NodeFreeMRU       *uint64  `schema:"node_free_mru,omitempty"`
+	NodeFreeHRU       *uint64  `schema:"node_free_hru,omitempty"`
+	NodeFreeSRU       *uint64  `schema:"node_free_sru,omitempty"`
+	NodeTotalCRU      *uint64  `schema:"node_total_cru,omitempty"`
+	NodeStatus        []string `schema:"node_status,omitempty"`
+	NodeRentedBy      *uint64  `schema:"node_rented_by,omitempty"`
+	NodeAvailableFor  *uint64  `schema:"node_available_for,omitempty"`
+	NodeHasGPU        *bool    `schema:"node_has_gpu,omitempty"`
+	NodeCertified     *bool    `schema:"node_certified,omitempty"`
+	Country           *string  `schema:"country,omitempty"`
+	Region            *string  `schema:"region,omitempty"`
 }

grid-proxy/pkg/types/nodes.go

@@ -113,7 +113,7 @@ type Capacity struct {
 
 // NodeFilter node filters
 type NodeFilter struct {
-	Status            *string  `schema:"status,omitempty"`
+	Status            []string `schema:"status,omitempty"`
 	FreeMRU           *uint64  `schema:"free_mru,omitempty"`
 	FreeHRU           *uint64  `schema:"free_hru,omitempty"`
 	FreeSRU           *uint64  `schema:"free_sru,omitempty"`

grid-proxy/pkg/types/stats.go

@@ -22,7 +22,7 @@ type Stats struct {
 
 // StatsFilter statistics filters
 type StatsFilter struct {
-	Status *string `schema:"status,omitempty"`
+	Status []string `schema:"status,omitempty"`
 }
 
 // NodeStatisticsResources resources returned on node statistics

grid-proxy/tests/queries/counters_test.go

@@ -16,15 +16,16 @@ var statuses = []string{"up", "standby", "down"}
 
 var statsFilterRandomValues = map[string]func() interface{}{
 	"Status": func() interface{} {
-		return &statuses[rand.Intn(3)]
+		randomLen := rand.Intn(len(statuses))
+		return getRandomSliceFrom(statuses, randomLen)
 	},
 }
 
 func TestStats(t *testing.T) {
 	t.Parallel()
 	t.Run("stats up test", func(t *testing.T) {
 		f := proxytypes.StatsFilter{
-			Status: &STATUS_UP,
+			Status: []string{STATUS_UP},
 		}
 
 		want, err := mockClient.Stats(context.Background(), f)

grid-proxy/tests/queries/farm_test.go

@@ -114,8 +114,8 @@ var farmFilterRandomValueGenerator = map[string]func(agg FarmsAggregate) interfa
 		return &cru
 	},
 	"NodeStatus": func(agg FarmsAggregate) interface{} {
-		nodeStatuses := []string{"up", "down", "standby"}
-		return &nodeStatuses[rand.Intn(len(nodeStatuses))]
+		randomLen := rand.Intn(len(statuses))
+		return getRandomSliceFrom(statuses, randomLen)
 	},
 	"NodeAvailableFor": func(agg FarmsAggregate) interface{} {
 		return &agg.rentersTwinIDs[rand.Intn(len(agg.rentersTwinIDs))]

grid-proxy/tests/queries/mock_client/counters.go

@@ -2,6 +2,7 @@ package mock
 
 import (
 	"context"
+	"slices"
 
 	"github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/nodestatus"
 	"github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/types"
@@ -23,7 +24,8 @@ func (g *GridProxyMockClient) Stats(ctx context.Context, filter types.StatsFilte
 			State:  node.Power.State,
 			Target: node.Power.Target,
 		}
-		if filter.Status == nil || *filter.Status == nodestatus.DecideNodeStatus(nodePower, int64(node.UpdatedAt)) {
+		st := nodestatus.DecideNodeStatus(nodePower, int64(node.UpdatedAt))
+		if filter.Status == nil || len(filter.Status) == 0 || slices.Contains(filter.Status, st) {
 			res.Nodes++
 			distribution[node.Country] += 1
 			res.TotalCRU += int64(g.data.NodeTotalResources[node.NodeID].CRU)

grid-proxy/tests/queries/mock_client/farms.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/nodestatus"
 	"github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/types"
+	"golang.org/x/exp/slices"
 )
 
 // Farms returns farms with the given filters and pagination parameters
@@ -112,7 +113,7 @@ func (f *Farm) satisfies(filter types.FarmFilter, data *DBData) bool {
 	if filter.NodeAvailableFor != nil || filter.NodeCertified != nil ||
 		filter.NodeFreeHRU != nil || filter.NodeFreeMRU != nil ||
 		filter.NodeFreeSRU != nil || filter.NodeHasGPU != nil ||
-		filter.NodeRentedBy != nil || filter.NodeStatus != nil ||
+		filter.NodeRentedBy != nil || (filter.NodeStatus != nil && len(filter.NodeStatus) != 0) ||
 		filter.Country != nil || filter.Region != nil ||
 		filter.NodeTotalCRU != nil {
 		if !f.satisfyFarmNodesFilter(data, filter) {
@@ -166,7 +167,8 @@ func (f *Farm) satisfyFarmNodesFilter(data *DBData, filter types.FarmFilter) boo
 			State:  node.Power.State,
 			Target: node.Power.Target,
 		}
-		if filter.NodeStatus != nil && *filter.NodeStatus != nodestatus.DecideNodeStatus(nodePower, int64(node.UpdatedAt)) {
+		nodeStatus := nodestatus.DecideNodeStatus(nodePower, int64(node.UpdatedAt))
+		if filter.NodeStatus != nil && len(filter.NodeStatus) != 0 && !slices.Contains(filter.NodeStatus, nodeStatus) {
 			continue
 		}
 

grid-proxy/tests/queries/mock_client/nodes.go

@@ -294,7 +294,8 @@ func (n *Node) satisfies(f types.NodeFilter, data *DBData) bool {
 	used := data.NodeUsedResources[n.NodeID]
 	free := CalcFreeResources(total, used)
 
-	if f.Status != nil && *f.Status != nodestatus.DecideNodeStatus(nodePower, int64(n.UpdatedAt)) {
+	nodeStatus := nodestatus.DecideNodeStatus(nodePower, int64(n.UpdatedAt))
+	if f.Status != nil && len(f.Status) != 0 && !slices.Contains(f.Status, nodeStatus) {
 		return false
 	}