implement grid-compose

go.work

@@ -1,4 +1,4 @@
-go 1.21
+go 1.21.0
 
 use (
 	./activation-service
@@ -12,4 +12,5 @@ use (
 	./tfrobot
 	./tools/relay-cache-warmer
 	./user-contracts-mon
+	./grid-compose
 )

grid-compose/.gitignore

@@ -0,0 +1,2 @@
+bin/*
+.pre-commit-config.yaml

grid-compose/Makefile

@@ -0,0 +1,6 @@
+BIN_PATH = bin/main
+
+.PHONY: build
+
+build:
+	@go build -o $(BIN_PATH) ./main.go

grid-compose/README.md

@@ -0,0 +1,22 @@
+# Grid-Compose
+
+is a tool for running multi-vm applications on TFGrid defined using a Yaml formatted file.
+
+## Usage
+
+`REQUIRED` EnvVars:
+
+- `MNEMONIC`: your secret words
+- `NETWORK`: one of (dev, qa, test, main)
+
+```bash
+grid-compose [OPTIONS] [COMMAND]
+
+OPTIONS:
+    -f path to yaml file, default is ./grid-compose.yaml
+
+COMMANDS:
+    - version: shows the project version
+    - up:      deploy the app
+    - down:    cancel all deployments
+```

grid-compose/cmd/down.go

@@ -0,0 +1,28 @@
+package cmd
+
+import (
+	"github.com/rs/zerolog/log"
+	"github.com/spf13/cobra"
+	"github.com/threefoldtech/tfgrid-sdk-go/grid-compose/internal"
+)
+
+var downCmd = &cobra.Command{
+	Use:   "down",
+	Short: "cancel your project on the grid",
+	Run: func(cmd *cobra.Command, args []string) {
+		if err := down(); err != nil {
+			log.Fatal().Err(err).Send()
+		}
+	},
+}
+
+func down() error {
+	for key := range app.Specs.Services {
+		projectName := internal.GetProjectName(key, app.Client.TwinID)
+		log.Info().Str("projectName", projectName).Msg("canceling deployments")
+		if err := app.Client.CancelByProjectName(projectName); err != nil {
+			return err
+		}
+	}
+	return nil
+}

grid-compose/cmd/ps.go

@@ -0,0 +1,123 @@
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/rs/zerolog/log"
+	"github.com/spf13/cobra"
+	"github.com/threefoldtech/tfgrid-sdk-go/grid-client/deployer"
+	"github.com/threefoldtech/tfgrid-sdk-go/grid-client/workloads"
+	"github.com/threefoldtech/tfgrid-sdk-go/grid-compose/internal"
+	"github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/types"
+)
+
+var psCmd = &cobra.Command{
+	Use:   "ps",
+	Short: "list containers",
+	Run: func(cmd *cobra.Command, args []string) {
+		if err := ps(cmd.Context()); err != nil {
+			log.Fatal().Err(err).Send()
+		}
+	},
+}
+
+func ps(ctx context.Context) error {
+	twindId := uint64(app.Client.TwinID)
+	filters := types.ContractFilter{
+		TwinID: &twindId,
+	}
+	limits := types.Limit{
+		Size: 100,
+	}
+	cache := make(map[string]bool, 0)
+
+	contracts, _, err := app.Client.GridProxyClient.Contracts(ctx, filters, limits)
+
+	if err != nil {
+		return err
+	}
+
+	for _, contract := range contracts {
+		if contract.Type != "node" || contract.State == "Deleted" {
+			continue
+		}
+
+		details, err := workloads.ParseDeploymentData(contract.Details.(types.NodeContractDetails).DeploymentData)
+		if err != nil {
+			return err
+		}
+
+		if strings.Split(details.ProjectName, "/")[0] != "compose" || cache[details.ProjectName] {
+			continue
+		}
+
+		res, err := GetVM(ctx, app.Client, details.Name)
+		if err != nil {
+			return err
+		}
+
+		s, err := json.MarshalIndent(res, "", "\t")
+		if err != nil {
+			log.Fatal().Err(err).Send()
+		}
+
+		log.Info().Msg("vm:\n" + string(s))
+		cache[details.ProjectName] = true
+	}
+
+	return nil
+}
+
+// GetVM gets a vm with its project name
+func GetVM(ctx context.Context, t deployer.TFPluginClient, name string) (workloads.Deployment, error) {
+	name, _ = strings.CutSuffix(name, "net")
+	projectName := internal.GetProjectName(name, app.Client.TwinID)
+
+	// try to get contracts with the old project name format "<name>"
+	contracts, err := t.ContractsGetter.ListContractsOfProjectName(projectName, true)
+	if err != nil {
+		return workloads.Deployment{}, err
+	}
+
+	if len(contracts.NodeContracts) == 0 {
+		// if could not find any contracts try to get contracts with the new project name format "vm/<name>"
+		projectName = fmt.Sprintf("vm/%s", name)
+		contracts, err = t.ContractsGetter.ListContractsOfProjectName(projectName, true)
+		if err != nil {
+			return workloads.Deployment{}, err
+		}
+
+		if len(contracts.NodeContracts) == 0 {
+			return workloads.Deployment{}, fmt.Errorf("couldn't find any contracts with name %s", name)
+		}
+	}
+
+	var nodeID uint32
+
+	for _, contract := range contracts.NodeContracts {
+		contractID, err := strconv.ParseUint(contract.ContractID, 10, 64)
+		if err != nil {
+			return workloads.Deployment{}, err
+		}
+
+		nodeID = contract.NodeID
+		checkIfExistAndAppend(t, nodeID, contractID)
+
+	}
+
+	return t.State.LoadDeploymentFromGrid(ctx, nodeID, name)
+}
+
+func checkIfExistAndAppend(t deployer.TFPluginClient, node uint32, contractID uint64) {
+	for _, n := range t.State.CurrentNodeDeployments[node] {
+		if n == contractID {
+			return
+		}
+	}
+
+	t.State.CurrentNodeDeployments[node] = append(t.State.CurrentNodeDeployments[node], contractID)
+}

grid-compose/cmd/root.go

@@ -0,0 +1,94 @@
+package cmd
+
+import (
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
+	"github.com/spf13/cobra"
+	"github.com/threefoldtech/tfgrid-sdk-go/grid-client/deployer"
+	"github.com/threefoldtech/tfgrid-sdk-go/grid-compose/pkg/types"
+	"gopkg.in/yaml.v3"
+)
+
+var (
+	app        App
+	configFile string
+	network    string
+	mnemonic   string
+)
+
+var rootCmd = &cobra.Command{
+	Use:   "grid-compose",
+	Short: "Grid-Compose is a tool for running multi-vm applications on TFGrid defined using a Yaml formatted file.",
+	PersistentPreRun: func(cmd *cobra.Command, args []string) {
+		var err error
+		app, err = NewApp(network, mnemonic, configFile)
+		if err != nil {
+			log.Fatal().Err(err).Send()
+		}
+	},
+}
+
+func init() {
+	network = os.Getenv("NETWORK")
+	mnemonic = os.Getenv("MNEMONIC")
+	rootCmd.PersistentFlags().StringVarP(&configFile, "file", "f", "./grid-compose.yaml", "the grid-compose configuration file")
+
+	log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
+
+	rootCmd.AddCommand(versionCmd)
+	rootCmd.AddCommand(upCmd)
+	rootCmd.AddCommand(psCmd)
+	rootCmd.AddCommand(downCmd)
+}
+
+type App struct {
+	Client deployer.TFPluginClient
+	Specs  types.Specs
+}
+
+func NewApp(net, mne, filePath string) (App, error) {
+	specs, err := LoadSpecsFromFile(filePath)
+	if err != nil {
+		return App{}, fmt.Errorf("failed to load specs from file: %w", err)
+	}
+
+	client, err := deployer.NewTFPluginClient(mne, deployer.WithNetwork(net))
+	if err != nil {
+		return App{}, fmt.Errorf("failed to load grid client: %w", err)
+	}
+
+	return App{
+		Specs:  specs,
+		Client: client,
+	}, nil
+}
+
+func LoadSpecsFromFile(filePath string) (types.Specs, error) {
+	file, err := os.Open(filePath)
+	if err != nil {
+		return types.Specs{}, fmt.Errorf("failed to open file: %w", err)
+	}
+	defer file.Close()
+
+	content, err := io.ReadAll(file)
+	if err != nil {
+		return types.Specs{}, fmt.Errorf("failed to read file: %w", err)
+	}
+
+	var specs types.Specs
+	if err := yaml.Unmarshal(content, &specs); err != nil {
+		return types.Specs{}, fmt.Errorf("failed to parse file: %w", err)
+	}
+
+	return specs, nil
+}
+
+func Execute() {
+	if err := rootCmd.Execute(); err != nil {
+		log.Fatal().Err(err).Send()
+	}
+}

grid-compose/cmd/up.go

@@ -0,0 +1,95 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"net"
+
+	"github.com/rs/zerolog/log"
+	"github.com/spf13/cobra"
+	"github.com/threefoldtech/tfgrid-sdk-go/grid-client/workloads"
+	"github.com/threefoldtech/tfgrid-sdk-go/grid-compose/internal"
+	"github.com/threefoldtech/zos/pkg/gridtypes"
+)
+
+var upCmd = &cobra.Command{
+	Use:   "up",
+	Short: "deploy application on the grid",
+	Run: func(cmd *cobra.Command, args []string) {
+		if err := up(cmd.Context()); err != nil {
+			log.Fatal().Err(err).Send()
+		}
+	},
+}
+
+func up(ctx context.Context) error {
+	results := make(map[string]workloads.VM)
+	for key, val := range app.Specs.Services {
+		projectName := internal.GetProjectName(key, app.Client.TwinID)
+
+		networkName := key + "net"
+		log.Info().Str("projectName", projectName).Str("workloadName", networkName).Msg("deploying network")
+		net := workloads.ZNet{
+			Name:  networkName,
+			Nodes: []uint32{uint32(val.NodeID)},
+			IPRange: gridtypes.NewIPNet(net.IPNet{
+				IP:   net.IPv4(10, 20, 0, 0),
+				Mask: net.CIDRMask(16, 32),
+			}),
+			SolutionType: projectName,
+		}
+		if err := app.Client.NetworkDeployer.Deploy(ctx, &net); err != nil {
+			return err
+		}
+
+		log.Info().Str("projectName", projectName).Str("workloadName", key).Msg("deploying vm")
+		vm := workloads.VM{
+			Name:        key,
+			Flist:       val.Flist,
+			Entrypoint:  val.Entrypoint,
+			EnvVars:     val.Environment,
+			CPU:         int(val.Resources.CPU),
+			Memory:      int(val.Resources.Memory),
+			NetworkName: net.Name,
+		}
+
+		for _, net := range val.Networks {
+			switch net {
+			case "publicIp4":
+				vm.PublicIP = true
+			case "publicIp6":
+				vm.PublicIP6 = true
+			case "yggdrasil":
+				vm.Planetary = true
+			case "mycelium":
+				seed, err := internal.GetRandomMyceliumIPSeed()
+				if err != nil {
+					return fmt.Errorf("failed to get mycelium seed: %w", err)
+				}
+				vm.MyceliumIPSeed = seed
+			}
+		}
+
+		dl := workloads.NewDeployment(vm.Name, uint32(val.NodeID), projectName, nil, net.Name, nil, nil, []workloads.VM{vm}, nil)
+		if err := app.Client.DeploymentDeployer.Deploy(ctx, &dl); err != nil {
+			log.Error().Err(err).Msg("reverting deployed network")
+			if err := app.Client.NetworkDeployer.Cancel(ctx, &net); err != nil {
+				return err
+			}
+			return err
+		}
+
+		res, err := app.Client.State.LoadVMFromGrid(ctx, uint32(val.NodeID), vm.Name, dl.Name)
+		if err != nil {
+			return err
+		}
+
+		results[vm.Name] = res
+	}
+
+	for key, val := range results {
+		fmt.Printf("%s vm addresses:\n", key)
+		fmt.Println(internal.GetVmAddresses(val))
+	}
+	return nil
+}