Add restic wrapper library

glide.yaml

@@ -57,8 +57,8 @@ import:
   version: v1.4.0
 - package: github.com/pkg/errors
   version: v0.8.0
-- package: github.com/prometheus/client_golang/prometheus
-  version: master
+- package: github.com/prometheus/client_golang
+  version: v0.9.2
 - package: github.com/russross/blackfriday
   version: v1.5.2
 - package: github.com/spf13/afero

pkg/restic/backup.go

@@ -0,0 +1,65 @@
+package restic
+
+import "time"
+
+func (w *ResticWrapper) RunBackup(backupOption *BackupOptions) (*BackupOutput, error) {
+	// Start clock to measure total session duration
+	startTime := time.Now()
+
+	// Initialize restic repository if it does not exist
+	_, err := w.initRepositoryIfAbsent()
+	if err != nil {
+		return nil, err
+	}
+
+	backupOutput := &BackupOutput{}
+
+	// Backup all target directories
+	for _, dir := range backupOption.BackupDirs {
+		out, err := w.backup(dir, nil)
+		if err != nil {
+			return nil, err
+		}
+		// Extract information from the output of backup command
+		err = backupOutput.extractBackupInfo(out, dir)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// Check repository integrity
+	out, err := w.check()
+	if err != nil {
+		return nil, err
+	}
+	// Extract information from output of "check" command
+	backupOutput.extractCheckInfo(out)
+
+	// Cleanup old snapshot according to retention policy
+	out, err = w.cleanup(backupOption.Cleanup)
+	if err != nil {
+		return nil, err
+	}
+	// Extract information from output of cleanup command
+	err = backupOutput.extractCleanupInfo(out)
+	if err != nil {
+		return nil, err
+	}
+
+	// Read repository statics after cleanup
+	out, err = w.stats()
+	if err != nil {
+		return nil, err
+	}
+	// Extract information from output of "stats" command
+	err = backupOutput.extractStatsInfo(out)
+	if err != nil {
+		return nil, err
+	}
+
+	// Backup complete. Read current time and calculate total session duration.
+	endTime := time.Now()
+	backupOutput.SessionDuration = endTime.Sub(startTime).String()
+
+	return backupOutput, nil
+}

pkg/restic/commands.go

@@ -0,0 +1,178 @@
+package restic
+
+import (
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/appscode/go/log"
+	"github.com/pkg/errors"
+)
+
+const (
+	Exe = "/bin/restic"
+)
+
+type Snapshot struct {
+	ID       string    `json:"id"`
+	Time     time.Time `json:"time"`
+	Tree     string    `json:"tree"`
+	Paths    []string  `json:"paths"`
+	Hostname string    `json:"hostname"`
+	Username string    `json:"username"`
+	UID      int       `json:"uid"`
+	Gid      int       `json:"gid"`
+	Tags     []string  `json:"tags"`
+}
+
+func (w *ResticWrapper) listSnapshots(snapshotIDs []string) ([]Snapshot, error) {
+	result := make([]Snapshot, 0)
+	args := w.appendCacheDirFlag([]interface{}{"snapshots", "--json", "--quiet", "--no-lock"})
+	args = w.appendCaCertFlag(args)
+	for _, id := range snapshotIDs {
+		args = append(args, id)
+	}
+
+	err := w.sh.Command(Exe, args...).UnmarshalJSON(&result)
+	return result, err
+}
+
+func (w *ResticWrapper) deleteSnapshots(snapshotIDs []string) ([]byte, error) {
+	args := w.appendCacheDirFlag([]interface{}{"forget", "--quiet", "--prune"})
+	args = w.appendCaCertFlag(args)
+	for _, id := range snapshotIDs {
+		args = append(args, id)
+	}
+
+	return w.run(Exe, args)
+}
+
+func (w *ResticWrapper) initRepositoryIfAbsent() ([]byte, error) {
+	log.Infoln("Ensuring restic repository in the backend")
+	args := w.appendCacheDirFlag([]interface{}{"snapshots", "--json"})
+	args = w.appendCaCertFlag(args)
+	if _, err := w.run(Exe, args); err != nil {
+		args = w.appendCacheDirFlag([]interface{}{"init"})
+		args = w.appendCaCertFlag(args)
+
+		return w.run(Exe, args)
+	}
+	return nil, nil
+}
+
+func (w *ResticWrapper) backup(path string, tags []string) ([]byte, error) {
+	log.Infoln("Backing up target data")
+	args := []interface{}{"backup", path}
+	if w.config.Hostname != "" {
+		args = append(args, "--host")
+		args = append(args, w.config.Hostname)
+	}
+	// add tags if any
+	for _, tag := range tags {
+		args = append(args, "--tag")
+		args = append(args, tag)
+	}
+	args = w.appendCacheDirFlag(args)
+	args = w.appendCaCertFlag(args)
+
+	return w.run(Exe, args)
+}
+
+func (w *ResticWrapper) cleanup(cleanupOptions CleanupOptions) ([]byte, error) {
+	log.Infoln("Cleaning old snapshots according to retention policy")
+
+	args := []interface{}{"forget"}
+
+	policy := cleanupOptions.RetentionPolicyName
+	if !strings.HasPrefix(policy, "--") {
+		policy = "--" + policy
+	}
+
+	args = append(args, policy)
+	args = append(args, cleanupOptions.RetentionValue)
+
+	if cleanupOptions.Prune {
+		args = append(args, "--prune")
+	}
+
+	if cleanupOptions.DryRun {
+		args = append(args, "--dry-run")
+	}
+
+	if len(args) > 1 {
+		args = w.appendCacheDirFlag(args)
+		args = w.appendCaCertFlag(args)
+
+		return w.run(Exe, args)
+	}
+	return nil, nil
+}
+
+func (w *ResticWrapper) restore(path, host, snapshotID string) ([]byte, error) {
+	log.Infoln("Restoring backed up data")
+	args := []interface{}{"restore"}
+	if snapshotID != "" {
+		args = append(args, snapshotID)
+	} else {
+		args = append(args, "latest")
+	}
+	args = append(args, "--path")
+	args = append(args, path) // source-path specified in restic fileGroup
+	args = append(args, "--host")
+	args = append(args, host)
+
+	// Remove last part from the path.
+	// https://github.com/appscode/stash/issues/392
+	args = append(args, "--target")
+	args = append(args, filepath.Dir(path))
+
+	args = w.appendCacheDirFlag(args)
+	args = w.appendCaCertFlag(args)
+
+	return w.run(Exe, args)
+}
+
+func (w *ResticWrapper) check() ([]byte, error) {
+	log.Infoln("Checking integrity of repository")
+	args := w.appendCacheDirFlag([]interface{}{"check"})
+	args = w.appendCaCertFlag(args)
+
+	return w.run(Exe, args)
+}
+
+func (w *ResticWrapper) stats() ([]byte, error) {
+	log.Infoln("Reading repository status")
+	args := w.appendCacheDirFlag([]interface{}{"stats"})
+	args = append(args, "--mode=raw-data", "--quiet")
+	args = w.appendCaCertFlag(args)
+
+	return w.run(Exe, args)
+}
+
+func (w *ResticWrapper) appendCacheDirFlag(args []interface{}) []interface{} {
+	if w.config.EnableCache {
+		cacheDir := filepath.Join(w.config.ScratchDir, "restic-cache")
+		return append(args, "--cache-dir", cacheDir)
+	}
+	return append(args, "--no-cache")
+}
+
+func (w *ResticWrapper) appendCaCertFlag(args []interface{}) []interface{} {
+	if w.config.CacertFile != "" {
+		return append(args, "--cacert", w.config.CacertFile)
+	}
+	return args
+}
+
+func (w *ResticWrapper) run(cmd string, args []interface{}) ([]byte, error) {
+	out, err := w.sh.Command(cmd, args...).Output()
+	if err != nil {
+		log.Errorf("Error running command '%s %s' output:\n%s", cmd, args, string(out))
+		parts := strings.Split(strings.TrimSuffix(string(out), "\n"), "\n")
+		if len(parts) > 1 {
+			parts = parts[len(parts)-1:]
+			return nil, errors.New(parts[0])
+		}
+	}
+	return out, err
+}

pkg/restic/config.go

@@ -0,0 +1,70 @@
+package restic
+
+import (
+	v1beta1_api "github.com/appscode/stash/apis/stash/v1beta1"
+	shell "github.com/codeskyblue/go-sh"
+)
+
+type ResticWrapper struct {
+	sh     *shell.Session
+	config SetupOptions
+}
+
+type BackupOptions struct {
+	BackupDirs []string
+	Cleanup    CleanupOptions
+}
+
+type RestoreOptions struct {
+	Rules []v1beta1_api.Rule
+}
+
+type SetupOptions struct {
+	Provider    string
+	Bucket      string
+	Endpoint    string
+	Path        string
+	SecretDir   string
+	CacertFile  string
+	ScratchDir  string
+	EnableCache bool
+	Hostname    string
+}
+
+type CleanupOptions struct {
+	RetentionPolicyName string
+	RetentionValue      string
+	Prune               bool
+	DryRun              bool
+}
+
+type RetentionPolicy struct {
+	Policy string
+	Value  string
+	Prune  bool
+	DryRun bool
+}
+
+type MetricsOptions struct {
+	Enabled        bool
+	PushgatewayURL string
+	MetricFileDir  string
+	Labels         []string
+	JobName        string
+}
+
+func NewResticWrapper(options SetupOptions) (*ResticWrapper, error) {
+	wrapper := &ResticWrapper{
+		sh:     shell.NewSession(),
+		config: options,
+	}
+	wrapper.sh.SetDir(wrapper.config.ScratchDir)
+	wrapper.sh.ShowCMD = true
+
+	// Setup restic environments
+	err := wrapper.setupEnv()
+	if err != nil {
+		return nil, err
+	}
+	return wrapper, nil
+}