From 0c0246a1b86c1dfc5804383b372d816f219f38bc Mon Sep 17 00:00:00 2001 From: Omar Abdulaziz Date: Wed, 13 Nov 2024 09:43:22 +0200 Subject: [PATCH 1/2] init location indexer: - remove the region column and triggers from the cache table - add the indexer with the schema changes and generator - make the removeDuplicates function generic to be used with all indexers --- grid-proxy/cmds/proxy_server/main.go | 17 +++++ .../internal/explorer/db/indexer_calls.go | 8 +++ grid-proxy/internal/explorer/db/postgres.go | 14 +++- grid-proxy/internal/explorer/db/setup.sql | 27 +------- grid-proxy/internal/explorer/db/types.go | 1 + grid-proxy/internal/indexer/health.go | 19 ++--- grid-proxy/internal/indexer/location.go | 54 +++++++++++++++ grid-proxy/internal/indexer/utils.go | 12 ++++ grid-proxy/internal/indexer/utils_test.go | 52 ++++++++++++++ grid-proxy/pkg/types/indexer.go | 10 +++ grid-proxy/tests/queries/mock_client/farms.go | 2 +- .../tests/queries/mock_client/loader.go | 69 ++++++++----------- grid-proxy/tests/queries/mock_client/nodes.go | 2 +- grid-proxy/tools/db/crafter/generator.go | 56 ++++++--------- grid-proxy/tools/db/crafter/types.go | 11 --- grid-proxy/tools/db/generate.go | 4 +- grid-proxy/tools/db/schema.sql | 39 ++++------- 17 files changed, 236 insertions(+), 161 deletions(-) create mode 100644 grid-proxy/internal/indexer/location.go create mode 100644 grid-proxy/internal/indexer/utils_test.go diff --git a/grid-proxy/cmds/proxy_server/main.go b/grid-proxy/cmds/proxy_server/main.go index c37bdd2a3..954b818fd 100644 --- a/grid-proxy/cmds/proxy_server/main.go +++ b/grid-proxy/cmds/proxy_server/main.go @@ -72,6 +72,8 @@ type flags struct { workloadsIndexerIntervalMins uint featuresIndexerNumWorkers uint featuresIndexerIntervalMins uint + locationIndexerNumWorkers uint + locationIndexerIntervalMins uint } func main() { @@ -111,6 +113,9 @@ func main() { flag.UintVar(&f.workloadsIndexerNumWorkers, "workloads-indexer-workers", 10, "number of workers checking on node workloads number") flag.UintVar(&f.featuresIndexerIntervalMins, "features-indexer-interval", 60*24, "node features check interval in min") flag.UintVar(&f.featuresIndexerNumWorkers, "features-indexer-workers", 10, "number of workers checking on node supported features") + flag.UintVar(&f.locationIndexerIntervalMins, "location-indexer-interval", 60*6, "node location check interval in min") + flag.UintVar(&f.locationIndexerNumWorkers, "location-indexer-workers", 100, "number of workers checking on node location") + flag.Parse() // shows version and exit @@ -153,6 +158,8 @@ func main() { indexerIntervals := make(map[string]uint) if !f.noIndexer { startIndexers(ctx, f, &db, rpcRmbClient) + + // for the health endpoint indexerIntervals["gpu"] = f.gpuIndexerIntervalMins indexerIntervals["health"] = f.healthIndexerIntervalMins indexerIntervals["dmi"] = f.dmiIndexerIntervalMins @@ -160,6 +167,7 @@ func main() { indexerIntervals["ipv6"] = f.ipv6IndexerIntervalMins indexerIntervals["speed"] = f.speedIndexerIntervalMins indexerIntervals["features"] = f.featuresIndexerIntervalMins + indexerIntervals["location"] = f.locationIndexerIntervalMins } else { log.Info().Msg("Indexers did not start") } @@ -238,6 +246,15 @@ func startIndexers(ctx context.Context, f flags, db db.Database, rpcRmbClient *p f.featuresIndexerNumWorkers, ) featIdx.Start(ctx) + + locationIdx := indexer.NewIndexer[types.NodeLocation]( + indexer.NewLocationWork(f.locationIndexerIntervalMins), + "location", + db, + rpcRmbClient, + f.locationIndexerNumWorkers, + ) + locationIdx.Start(ctx) } func app(s *http.Server, f flags) error { diff --git a/grid-proxy/internal/explorer/db/indexer_calls.go b/grid-proxy/internal/explorer/db/indexer_calls.go index 4d406986b..464f8f88f 100644 --- a/grid-proxy/internal/explorer/db/indexer_calls.go +++ b/grid-proxy/internal/explorer/db/indexer_calls.go @@ -84,3 +84,11 @@ func (p *PostgresDatabase) UpsertNodeFeatures(ctx context.Context, features []ty } return p.gormDB.WithContext(ctx).Table("node_features").Clauses(conflictClause).Create(&features).Error } + +func (p *PostgresDatabase) UpsertNodeLocation(ctx context.Context, locations []types.NodeLocation) error { + conflictClause := clause.OnConflict{ + Columns: []clause.Column{{Name: "country"}}, + DoUpdates: clause.AssignmentColumns([]string{"continent", "updated_at"}), + } + return p.gormDB.WithContext(ctx).Table("node_location").Clauses(conflictClause).Create(&locations).Error +} diff --git a/grid-proxy/internal/explorer/db/postgres.go b/grid-proxy/internal/explorer/db/postgres.go index 8e9e4d28f..433d69428 100644 --- a/grid-proxy/internal/explorer/db/postgres.go +++ b/grid-proxy/internal/explorer/db/postgres.go @@ -136,6 +136,9 @@ func (d *PostgresDatabase) GetLastUpsertsTimestamp() (types.IndexersState, error if res := d.gormDB.Table("node_features").Select("updated_at").Where("updated_at IS NOT NULL").Order("updated_at DESC").Limit(1).Scan(&report.Features.UpdatedAt); res.Error != nil { return report, errors.Wrap(res.Error, "couldn't get features last updated_at") } + if res := d.gormDB.Table("node_location").Select("updated_at").Where("updated_at IS NOT NULL").Order("updated_at DESC").Limit(1).Scan(&report.Features.UpdatedAt); res.Error != nil { + return report, errors.Wrap(res.Error, "couldn't get features last updated_at") + } return report, nil } @@ -148,6 +151,7 @@ func (d *PostgresDatabase) Initialize() error { &types.HasIpv6{}, &types.NodesWorkloads{}, &types.NodeFeatures{}, + &types.NodeLocation{}, ); err != nil { return errors.Wrap(err, "failed to migrate indexer tables") } @@ -387,6 +391,7 @@ func (d *PostgresDatabase) nodeTableQuery(ctx context.Context, filter types.Node LEFT JOIN health_report ON node.twin_id = health_report.node_twin_id LEFT JOIN node_ipv6 ON node.twin_id = node_ipv6.node_twin_id LEFT JOIN node_features ON node.twin_id = node_features.node_twin_id + LEFT JOIN node_location ON node.country = node_location.country `) if filter.IsGpuFilterRequested() { @@ -440,7 +445,10 @@ func (d *PostgresDatabase) farmTableQuery(ctx context.Context, filter types.Farm func (d *PostgresDatabase) GetFarms(ctx context.Context, filter types.FarmFilter, limit types.Limit) ([]Farm, uint, error) { nodeQuery := d.gormDB.Table("resources_cache"). Select("resources_cache.farm_id", "renter", "resources_cache.extra_fee"). - Joins("LEFT JOIN node ON node.node_id = resources_cache.node_id"). + Joins(` + LEFT JOIN node ON node.node_id = resources_cache.node_id + LEFT JOIN node_location ON node_location.country = resources_cache.country + `). Group(`resources_cache.farm_id, renter, resources_cache.extra_fee`) if filter.NodeFreeMRU != nil { @@ -469,7 +477,7 @@ func (d *PostgresDatabase) GetFarms(ctx context.Context, filter types.FarmFilter } if filter.Region != nil { - nodeQuery = nodeQuery.Where("LOWER(resources_cache.region) = LOWER(?)", *filter.Region) + nodeQuery = nodeQuery.Where("LOWER(node_location.continent) = LOWER(?)", *filter.Region) } if len(filter.NodeStatus) != 0 { @@ -673,7 +681,7 @@ func (d *PostgresDatabase) GetNodes(ctx context.Context, filter types.NodeFilter q = q.Where("node.city ILIKE '%' || ? || '%'", *filter.CityContains) } if filter.Region != nil { - q = q.Where("LOWER(resources_cache.region) = LOWER(?)", *filter.Region) + q = q.Where("LOWER(node_location.continent) = LOWER(?)", *filter.Region) } if filter.NodeID != nil { q = q.Where("node.node_id = ?", *filter.NodeID) diff --git a/grid-proxy/internal/explorer/db/setup.sql b/grid-proxy/internal/explorer/db/setup.sql index 746074dc8..df116899d 100644 --- a/grid-proxy/internal/explorer/db/setup.sql +++ b/grid-proxy/internal/explorer/db/setup.sql @@ -119,7 +119,6 @@ SELECT rent_contract.contract_id as rent_contract_id, count(node_contract.contract_id) as node_contracts_count, node.country as country, - country.region as region, COALESCE(dmi.bios, '{}') as bios, COALESCE(dmi.baseboard, '{}') as baseboard, COALESCE(dmi.processor, '[]') as processor, @@ -136,7 +135,6 @@ FROM node LEFT JOIN contract_resources ON node_contract.resources_used_id = contract_resources.id LEFT JOIN node_resources_total AS node_resources_total ON node_resources_total.node_id = node.id LEFT JOIN rent_contract on node.node_id = rent_contract.node_id AND rent_contract.state IN ('Created', 'GracePeriod') - LEFT JOIN country ON LOWER(node.country) = LOWER(country.name) LEFT JOIN speed ON node.twin_id = speed.node_twin_id LEFT JOIN dmi ON node.twin_id = dmi.node_twin_id LEFT JOIN farm ON farm.farm_id = node.farm_id @@ -163,7 +161,6 @@ GROUP BY COALESCE(node_gpu_agg.gpus, '[]'), COALESCE(node_gpu_agg.gpu_count, 0), node.country, - country.region, COALESCE(dmi.bios, '{}'), COALESCE(dmi.baseboard, '{}'), COALESCE(dmi.processor, '[]'), @@ -172,8 +169,7 @@ GROUP BY COALESCE(speed.download, 0), node.certification, node.extra_fee, - farm.pricing_policy_id, - country.region; + farm.pricing_policy_id; DROP TABLE IF EXISTS resources_cache; CREATE TABLE IF NOT EXISTS resources_cache( @@ -194,7 +190,6 @@ CREATE TABLE IF NOT EXISTS resources_cache( rent_contract_id INTEGER, node_contracts_count INTEGER NOT NULL, country TEXT, - region TEXT, bios jsonb, baseboard jsonb, processor jsonb, @@ -283,27 +278,11 @@ CREATE INDEX IF NOT EXISTS idx_public_config_node_id ON public_config USING gin( /* Node Trigger: - Insert node record > Insert new resources_cache record - - Update node country > update resources_cache country/region */ CREATE OR REPLACE FUNCTION reflect_node_changes() RETURNS TRIGGER AS $$ BEGIN - IF (TG_OP = 'UPDATE') THEN - BEGIN - UPDATE resources_cache - SET - country = NEW.country, - region = ( - SELECT region FROM country WHERE LOWER(country.name) = LOWER(NEW.country) - ) - WHERE - resources_cache.node_id = NEW.node_id; - EXCEPTION - WHEN OTHERS THEN - RAISE NOTICE 'Error updating resources_cache: %', SQLERRM; - END; - - ELSIF (TG_OP = 'INSERT') THEN + IF (TG_OP = 'INSERT') THEN BEGIN INSERT INTO resources_cache SELECT * @@ -328,7 +307,7 @@ END; $$ LANGUAGE plpgsql; CREATE OR REPLACE TRIGGER tg_node - AFTER INSERT OR DELETE OR UPDATE OF country + AFTER INSERT OR DELETE ON node FOR EACH ROW EXECUTE PROCEDURE reflect_node_changes(); diff --git a/grid-proxy/internal/explorer/db/types.go b/grid-proxy/internal/explorer/db/types.go index f571bd615..bc0d4c587 100644 --- a/grid-proxy/internal/explorer/db/types.go +++ b/grid-proxy/internal/explorer/db/types.go @@ -42,6 +42,7 @@ type Database interface { UpsertNodeIpv6Report(ctx context.Context, ips []types.HasIpv6) error UpsertNodeWorkloads(ctx context.Context, workloads []types.NodesWorkloads) error UpsertNodeFeatures(ctx context.Context, features []types.NodeFeatures) error + UpsertNodeLocation(ctx context.Context, locations []types.NodeLocation) error } type ContractBilling types.ContractBilling diff --git a/grid-proxy/internal/indexer/health.go b/grid-proxy/internal/indexer/health.go index 20b416d67..05f52eedd 100644 --- a/grid-proxy/internal/indexer/health.go +++ b/grid-proxy/internal/indexer/health.go @@ -40,8 +40,10 @@ func (w *HealthWork) Get(ctx context.Context, rmb *peer.RpcClient, twinId uint32 func (w *HealthWork) Upsert(ctx context.Context, db db.Database, batch []types.HealthReport) error { // to prevent having multiple data for the same twin from different finders - batch = removeDuplicates(batch) - return db.UpsertNodeHealth(ctx, batch) + unique := removeDuplicates(batch, func(n types.HealthReport) uint32 { + return n.NodeTwinId + }) + return db.UpsertNodeHealth(ctx, unique) } func getHealthReport(response diagnostics.Diagnostics, twinId uint32) types.HealthReport { @@ -53,16 +55,3 @@ func getHealthReport(response diagnostics.Diagnostics, twinId uint32) types.Heal return report } - -func removeDuplicates(reports []types.HealthReport) []types.HealthReport { - seen := make(map[uint32]bool) - result := []types.HealthReport{} - for _, report := range reports { - if _, ok := seen[report.NodeTwinId]; !ok { - seen[report.NodeTwinId] = true - result = append(result, report) - } - } - - return result -} diff --git a/grid-proxy/internal/indexer/location.go b/grid-proxy/internal/indexer/location.go new file mode 100644 index 000000000..67ec67c8e --- /dev/null +++ b/grid-proxy/internal/indexer/location.go @@ -0,0 +1,54 @@ +package indexer + +import ( + "context" + "time" + + "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/internal/explorer/db" + "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/types" + "github.com/threefoldtech/tfgrid-sdk-go/rmb-sdk-go/peer" + "github.com/threefoldtech/zos/pkg/geoip" +) + +const locationCmd = "zos.location.get" + +var _ Work[types.NodeLocation] = (*LocationWork)(nil) + +type LocationWork struct { + finders map[string]time.Duration +} + +func NewLocationWork(interval uint) *LocationWork { + return &LocationWork{ + finders: map[string]time.Duration{ + "up": time.Duration(interval) * time.Minute, + }, + } +} + +func (w *LocationWork) Finders() map[string]time.Duration { + return w.finders +} + +func (w *LocationWork) Get(ctx context.Context, rmb *peer.RpcClient, id uint32) ([]types.NodeLocation, error) { + var loc geoip.Location + if err := callNode(ctx, rmb, locationCmd, nil, id, &loc); err != nil { + return []types.NodeLocation{}, nil + } + + return []types.NodeLocation{ + { + Country: loc.Country, + Continent: loc.Continent, + UpdatedAt: time.Now().Unix(), + }, + }, nil +} + +func (w *LocationWork) Upsert(ctx context.Context, db db.Database, batch []types.NodeLocation) error { + unique := removeDuplicates(batch, func(n types.NodeLocation) string { + return n.Country + }) + + return db.UpsertNodeLocation(ctx, unique) +} diff --git a/grid-proxy/internal/indexer/utils.go b/grid-proxy/internal/indexer/utils.go index 5035b91fe..b1bc00a32 100644 --- a/grid-proxy/internal/indexer/utils.go +++ b/grid-proxy/internal/indexer/utils.go @@ -50,3 +50,15 @@ func callNode(ctx context.Context, rmbClient *peer.RpcClient, cmd string, payloa return rmbClient.Call(subCtx, twinId, cmd, payload, result) } + +func removeDuplicates[T any, K comparable](items []T, keyFunc func(T) K) (result []T) { + seen := make(map[K]bool) + for _, item := range items { + key := keyFunc(item) + if _, ok := seen[key]; !ok { + seen[key] = true + result = append(result, item) + } + } + return +} diff --git a/grid-proxy/internal/indexer/utils_test.go b/grid-proxy/internal/indexer/utils_test.go new file mode 100644 index 000000000..ae5aea747 --- /dev/null +++ b/grid-proxy/internal/indexer/utils_test.go @@ -0,0 +1,52 @@ +package indexer + +import ( + "reflect" + "testing" + + "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/types" +) + +func TestRemoveDuplicates(t *testing.T) { + t.Run("remove duplicate countries", func(t *testing.T) { + locations := []types.NodeLocation{ + {Country: "Egypt", Continent: "Africa"}, + {Country: "Egypt", Continent: "Africa"}, + {Country: "Belgium", Continent: "Europe"}, + } + + uniqueLocations := []types.NodeLocation{ + {Country: "Egypt", Continent: "Africa"}, + {Country: "Belgium", Continent: "Europe"}, + } + + gotLocations := removeDuplicates(locations, func(n types.NodeLocation) string { + return n.Country + }) + + if !reflect.DeepEqual(uniqueLocations, gotLocations) { + t.Errorf("expected %v, but got %v", uniqueLocations, gotLocations) + } + }) + + t.Run("remove duplicate health reports", func(t *testing.T) { + healthReports := []types.HealthReport{ + {NodeTwinId: 1, Healthy: true}, + {NodeTwinId: 1, Healthy: true}, + {NodeTwinId: 2, Healthy: true}, + } + + uniqueReports := []types.HealthReport{ + {NodeTwinId: 1, Healthy: true}, + {NodeTwinId: 2, Healthy: true}, + } + + gotReports := removeDuplicates(healthReports, func(h types.HealthReport) uint32 { + return h.NodeTwinId + }) + + if !reflect.DeepEqual(gotReports, uniqueReports) { + t.Errorf("expected %v, but got %v", uniqueReports, gotReports) + } + }) +} diff --git a/grid-proxy/pkg/types/indexer.go b/grid-proxy/pkg/types/indexer.go index c7eeb27de..b358d1236 100644 --- a/grid-proxy/pkg/types/indexer.go +++ b/grid-proxy/pkg/types/indexer.go @@ -107,3 +107,13 @@ type NodeFeatures struct { func (NodeFeatures) TableName() string { return "node_features" } + +type NodeLocation struct { + Country string `json:"country" gorm:"unique;not null"` + Continent string `json:"continent"` + UpdatedAt int64 `json:"updated_at"` +} + +func (NodeLocation) TableName() string { + return "node_location" +} diff --git a/grid-proxy/tests/queries/mock_client/farms.go b/grid-proxy/tests/queries/mock_client/farms.go index b6b7e8a64..5170e555a 100644 --- a/grid-proxy/tests/queries/mock_client/farms.go +++ b/grid-proxy/tests/queries/mock_client/farms.go @@ -175,7 +175,7 @@ func (f *Farm) satisfyFarmNodesFilter(data *DBData, filter types.FarmFilter) boo continue } - if filter.Region != nil && !strings.EqualFold(*filter.Region, data.Regions[strings.ToLower(node.Country)]) { + if filter.Region != nil && !strings.EqualFold(*filter.Region, data.Regions[node.Country]) { continue } diff --git a/grid-proxy/tests/queries/mock_client/loader.go b/grid-proxy/tests/queries/mock_client/loader.go index b980b3458..adb8100a0 100644 --- a/grid-proxy/tests/queries/mock_client/loader.go +++ b/grid-proxy/tests/queries/mock_client/loader.go @@ -4,7 +4,6 @@ import ( "database/sql" "encoding/json" "math" - "strings" "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/types" "github.com/threefoldtech/zos/pkg/gridtypes" @@ -450,44 +449,6 @@ func loadContractBillingReports(db *sql.DB, data *DBData) error { return nil } -func loadCountries(db *sql.DB, data *DBData) error { - rows, err := db.Query(` - SELECT - COALESCE(id, ''), - COALESCE(country_id, 0), - COALESCE(code, ''), - COALESCE(name, ''), - COALESCE(region, ''), - COALESCE(subregion, ''), - COALESCE(lat, ''), - COALESCE(long, '') - FROM - country; - `) - if err != nil { - return err - } - - for rows.Next() { - var country Country - if err := rows.Scan( - &country.ID, - &country.CountryID, - &country.Code, - &country.Name, - &country.Region, - &country.Subregion, - &country.Lat, - &country.Long, - ); err != nil { - return err - } - data.Regions[strings.ToLower(country.Name)] = country.Region - } - - return nil -} - func loadLocations(db *sql.DB, data *DBData) error { rows, err := db.Query(` SELECT @@ -735,6 +696,30 @@ func loadPricingPolicies(db *sql.DB, data *DBData) error { return nil } +func loadNodeLocation(db *sql.DB, data *DBData) error { + rows, err := db.Query(` + SELECT + COALESCE(continent, ''), + COALESCE(country, '') + FROM + node_location;`) + if err != nil { + return err + } + for rows.Next() { + var loc types.NodeLocation + if err := rows.Scan( + &loc.Continent, + &loc.Country, + ); err != nil { + return err + } + data.Regions[loc.Country] = loc.Continent + } + + return nil +} + func parseUnit(unitString string) Unit { var unit Unit _ = json.Unmarshal([]byte(unitString), &unit) @@ -812,15 +797,15 @@ func Load(db *sql.DB, gormDB *gorm.DB) (DBData, error) { if err := loadNodeGPUs(db, &data); err != nil { return data, err } - if err := loadCountries(db, &data); err != nil { - return data, err - } if err := loadLocations(db, &data); err != nil { return data, err } if err := loadHealthReports(db, &data); err != nil { return data, err } + if err := loadNodeLocation(db, &data); err != nil { + return data, err + } if err := loadNodeIpv6(db, &data); err != nil { return data, err } diff --git a/grid-proxy/tests/queries/mock_client/nodes.go b/grid-proxy/tests/queries/mock_client/nodes.go index 93f74bf7b..0f225e3ff 100644 --- a/grid-proxy/tests/queries/mock_client/nodes.go +++ b/grid-proxy/tests/queries/mock_client/nodes.go @@ -364,7 +364,7 @@ func (n *Node) satisfies(f types.NodeFilter, data *DBData) bool { return false } - if f.Region != nil && !strings.EqualFold(*f.Region, data.Regions[strings.ToLower(n.Country)]) { + if f.Region != nil && !strings.EqualFold(*f.Region, data.Regions[n.Country]) { return false } diff --git a/grid-proxy/tools/db/crafter/generator.go b/grid-proxy/tools/db/crafter/generator.go index 2c51f32c1..1b1725851 100644 --- a/grid-proxy/tools/db/crafter/generator.go +++ b/grid-proxy/tools/db/crafter/generator.go @@ -806,41 +806,6 @@ func (c *Crafter) GenerateNodeGPUs() error { return nil } -func (c *Crafter) GenerateCountries() error { - var countriesValues []string - - // depends on nodeStart to not duplicate the value of country.id - start := c.NodeStart - - index := start - for countryName, region := range regions { - index++ - country := country{ - id: fmt.Sprintf("country-%d", index), - country_id: uint64(index), - name: countryName, - code: countriesCodes[countryName], - region: region, - subregion: "unknown", - lat: fmt.Sprintf("%d", 0), - long: fmt.Sprintf("%d", 0), - } - - countryTuple, err := objectToTupleString(country) - if err != nil { - return fmt.Errorf("failed to convert country object to tuple string: %w", err) - } - countriesValues = append(countriesValues, countryTuple) - } - - if err := c.insertTuples(country{}, countriesValues); err != nil { - return fmt.Errorf("failed to insert country: %w", err) - } - fmt.Println("countries generated") - - return nil -} - func (c *Crafter) GenerateSpeedReports() error { start := c.NodeStart end := c.NodeStart + c.NodeCount @@ -1015,7 +980,26 @@ func (c *Crafter) GenerateNodeFeatures() error { if err := c.gormDB.Create(reports).Error; err != nil { return fmt.Errorf("failed to insert node features reports: %w", err) } - fmt.Println("node features number reports generated") + fmt.Println("node features reports generated") + + return nil +} + +func (c *Crafter) GenerateLocations() error { + var reports []types.NodeLocation + for country, region := range regions { + report := types.NodeLocation{ + Country: country, + Continent: region, + UpdatedAt: time.Now().Unix(), + } + reports = append(reports, report) + } + + if err := c.gormDB.Create(reports).Error; err != nil { + return fmt.Errorf("failed to insert node location reports: %w", err) + } + fmt.Println("node location reports generated") return nil } diff --git a/grid-proxy/tools/db/crafter/types.go b/grid-proxy/tools/db/crafter/types.go index 9a086d647..ef0f6914c 100644 --- a/grid-proxy/tools/db/crafter/types.go +++ b/grid-proxy/tools/db/crafter/types.go @@ -237,14 +237,3 @@ type node_gpu struct { device string contract int } - -type country struct { - id string - country_id uint64 - code string - name string - region string - subregion string - lat string - long string -} diff --git a/grid-proxy/tools/db/generate.go b/grid-proxy/tools/db/generate.go index 2385b4e21..3f74d6316 100644 --- a/grid-proxy/tools/db/generate.go +++ b/grid-proxy/tools/db/generate.go @@ -113,8 +113,8 @@ func generateData(db *sql.DB, gormDB *gorm.DB, seed int) error { return fmt.Errorf("failed to generate node gpus: %w", err) } - if err := generator.GenerateCountries(); err != nil { - return fmt.Errorf("failed to generate countries: %w", err) + if err := generator.GenerateLocations(); err != nil { + return fmt.Errorf("failed to generate locations: %w", err) } if err := generator.GenerateSpeedReports(); err != nil { diff --git a/grid-proxy/tools/db/schema.sql b/grid-proxy/tools/db/schema.sql index 66c38325a..e8f50fcd1 100644 --- a/grid-proxy/tools/db/schema.sql +++ b/grid-proxy/tools/db/schema.sql @@ -88,24 +88,6 @@ CREATE TABLE public.contract_resources ( ALTER TABLE public.contract_resources OWNER TO postgres; --- --- Name: country; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.country ( - id character varying NOT NULL, - country_id integer NOT NULL, - code text NOT NULL, - name text NOT NULL, - region text NOT NULL, - subregion text NOT NULL, - lat text, - long text -); - - -ALTER TABLE public.country OWNER TO postgres; - -- -- Name: entity; Type: TABLE; Schema: public; Owner: postgres -- @@ -778,14 +760,6 @@ ALTER TABLE ONLY public.entity_proof ADD CONSTRAINT "PK_b55dee5f461106682013d0beef8" PRIMARY KEY (id); --- --- Name: country PK_bf6e37c231c4f4ea56dcd887269; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.country - ADD CONSTRAINT "PK_bf6e37c231c4f4ea56dcd887269" PRIMARY KEY (id); - - -- -- Name: nru_consumption PK_ca7956fb8fcdb7198737387d9a8; Type: CONSTRAINT; Schema: public; Owner: postgres -- @@ -1124,3 +1098,16 @@ CREATE TABLE IF NOT EXISTS public.node_features ( ALTER TABLE public.node_features OWNER TO postgres; +-- +-- Name: node_location; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE IF NOT EXISTS public.node_location ( + country text NOT NULL, + continent text, + updated_at bigint +); + +ALTER TABLE public.node_location + OWNER TO postgres; + From 3a340ac52d6a3b9bb38fed596608948809bda876 Mon Sep 17 00:00:00 2001 From: Omar Abdulaziz Date: Wed, 13 Nov 2024 09:49:42 +0200 Subject: [PATCH 2/2] remove unused country codes --- grid-proxy/tools/db/crafter/test_values.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/grid-proxy/tools/db/crafter/test_values.go b/grid-proxy/tools/db/crafter/test_values.go index 767efd1c6..a47c05ed5 100644 --- a/grid-proxy/tools/db/crafter/test_values.go +++ b/grid-proxy/tools/db/crafter/test_values.go @@ -10,12 +10,6 @@ var ( "Egypt": "Africa", "United Kingdom": "Europe", } - countriesCodes = map[string]string{ - "Belgium": "BG", - "United States": "US", - "Egypt": "EG", - "United Kingdom": "UK", - } cities = map[string][]string{ "Belgium": {"Brussels", "Antwerp", "Ghent", "Charleroi"}, "United States": {"New York", "Chicago", "Los Angeles", "San Francisco"},