feat(remote): support include git remote (#1652)

go.mod

@@ -9,6 +9,8 @@ require (
 	github.com/davecgh/go-spew v1.1.1
 	github.com/dominikbraun/graph v0.23.0
 	github.com/fatih/color v1.17.0
+	github.com/go-git/go-billy/v5 v5.5.0
+	github.com/go-git/go-git/v5 v5.12.0
 	github.com/go-task/slim-sprig/v3 v3.0.0
 	github.com/go-task/template v0.1.0
 	github.com/joho/godotenv v1.5.1
@@ -19,6 +21,7 @@ require (
 	github.com/sajari/fuzzy v1.0.0
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.9.0
+	github.com/whilp/git-urls v1.0.0
 	github.com/zeebo/xxh3 v1.0.2
 	golang.org/x/sync v0.8.0
 	golang.org/x/term v0.24.0
@@ -27,13 +30,31 @@ require (
 )
 
 require (
+	dario.cat/mergo v1.0.0 // indirect
+	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/ProtonMail/go-crypto v1.0.0 // indirect
+	github.com/cloudflare/circl v1.3.7 // indirect
+	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
 	github.com/dlclark/regexp2 v1.11.0 // indirect
-	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
+	github.com/emirpasic/gods v1.18.1 // indirect
+	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
+	github.com/kevinburke/ssh_config v1.2.0 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/muesli/cancelreader v0.2.2 // indirect
+	github.com/pjbgf/sha1cd v0.3.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
+	github.com/skeema/knownhosts v1.2.2 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
+	github.com/xanzy/ssh-agent v0.3.3 // indirect
+	golang.org/x/crypto v0.23.0 // indirect
+	golang.org/x/mod v0.17.0 // indirect
+	golang.org/x/net v0.25.0 // indirect
 	golang.org/x/sys v0.25.0 // indirect
-	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
+	golang.org/x/tools v0.21.0 // indirect
+	gopkg.in/warnings.v0 v0.1.2 // indirect
 )

taskfile/node.go

@@ -7,6 +7,8 @@ import (
 	"strings"
 	"time"
 
+	giturls "github.com/whilp/git-urls"
+
 	"github.com/go-task/task/v3/errors"
 	"github.com/go-task/task/v3/internal/experiments"
 	"github.com/go-task/task/v3/internal/logger"
@@ -48,24 +50,39 @@ func NewNode(
 ) (Node, error) {
 	var node Node
 	var err error
-	switch getScheme(entrypoint) {
+	scheme, err := getScheme(entrypoint)
+	if err != nil {
+		return nil, err
+	}
+	switch scheme {
+	case "git":
+		node, err = NewGitNode(entrypoint, dir, insecure, opts...)
 	case "http", "https":
 		node, err = NewHTTPNode(l, entrypoint, dir, insecure, timeout, opts...)
 	default:
-		// If no other scheme matches, we assume it's a file
 		node, err = NewFileNode(l, entrypoint, dir, opts...)
+
 	}
+
 	if node.Remote() && !experiments.RemoteTaskfiles.Enabled {
 		return nil, errors.New("task: Remote taskfiles are not enabled. You can read more about this experiment and how to enable it at https://taskfile.dev/experiments/remote-taskfiles")
 	}
 	return node, err
 }
 
-func getScheme(uri string) string {
+func getScheme(uri string) (string, error) {
+	u, err := giturls.Parse(uri)
+	if u == nil {
+		return "", err
+	}
+	if strings.HasSuffix(strings.Split(u.Path, "//")[0], ".git") && (u.Scheme == "git" || u.Scheme == "ssh" || u.Scheme == "https" || u.Scheme == "http") {
+		return "git", nil
+	}
+
 	if i := strings.Index(uri, "://"); i != -1 {
-		return uri[:i]
+		return uri[:i], nil
 	}
-	return ""
+	return "", nil
 }
 
 func getDefaultDir(entrypoint, dir string) string {

taskfile/node_file.go

@@ -81,6 +81,9 @@ func (node *FileNode) ResolveEntrypoint(entrypoint string) (string, error) {
 	if strings.Contains(entrypoint, "://") {
 		return entrypoint, nil
 	}
+	if strings.HasPrefix(entrypoint, "git") {
+		return entrypoint, nil
+	}
 
 	path, err := execext.Expand(entrypoint)
 	if err != nil {

taskfile/node_git.go

@@ -0,0 +1,126 @@
+package taskfile
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/url"
+	"path/filepath"
+	"strings"
+
+	"github.com/go-git/go-billy/v5/memfs"
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/go-git/go-git/v5/storage/memory"
+	giturls "github.com/whilp/git-urls"
+
+	"github.com/go-task/task/v3/errors"
+	"github.com/go-task/task/v3/internal/execext"
+	"github.com/go-task/task/v3/internal/filepathext"
+)
+
+// An GitNode is a node that reads a Taskfile from a remote location via Git.
+type GitNode struct {
+	*BaseNode
+	URL    *url.URL
+	rawUrl string
+	ref    string
+	path   string
+}
+
+func NewGitNode(
+	entrypoint string,
+	dir string,
+	insecure bool,
+	opts ...NodeOption,
+) (*GitNode, error) {
+	base := NewBaseNode(dir, opts...)
+	u, err := giturls.Parse(entrypoint)
+	if err != nil {
+		return nil, err
+	}
+
+	basePath, path := func() (string, string) {
+		x := strings.Split(u.Path, "//")
+		return x[0], x[1]
+	}()
+	ref := u.Query().Get("ref")
+
+	rawUrl := u.String()
+
+	u.RawQuery = ""
+	u.Path = basePath
+
+	if u.Scheme == "http" && !insecure {
+		return nil, &errors.TaskfileNotSecureError{URI: entrypoint}
+	}
+	return &GitNode{
+		BaseNode: base,
+		URL:      u,
+		rawUrl:   rawUrl,
+		ref:      ref,
+		path:     path,
+	}, nil
+}
+
+func (node *GitNode) Location() string {
+	return node.rawUrl
+}
+
+func (node *GitNode) Remote() bool {
+	return true
+}
+
+func (node *GitNode) Read(_ context.Context) ([]byte, error) {
+	fs := memfs.New()
+	storer := memory.NewStorage()
+	_, err := git.Clone(storer, fs, &git.CloneOptions{
+		URL:           node.URL.String(),
+		ReferenceName: plumbing.ReferenceName(node.ref),
+		SingleBranch:  true,
+		Depth:         1,
+	})
+	if err != nil {
+		return nil, err
+	}
+	file, err := fs.Open(node.path)
+	if err != nil {
+		return nil, err
+	}
+	// Read the entire response body
+	b, err := io.ReadAll(file)
+	if err != nil {
+		return nil, err
+	}
+
+	return b, nil
+}
+
+func (node *GitNode) ResolveEntrypoint(entrypoint string) (string, error) {
+	dir, _ := filepath.Split(node.path)
+	resolvedEntrypoint := fmt.Sprintf("%s//%s", node.URL, filepath.Join(dir, entrypoint))
+	if node.ref != "" {
+		return fmt.Sprintf("%s?ref=%s", resolvedEntrypoint, node.ref), nil
+	}
+	return resolvedEntrypoint, nil
+}
+
+func (node *GitNode) ResolveDir(dir string) (string, error) {
+	path, err := execext.Expand(dir)
+	if err != nil {
+		return "", err
+	}
+
+	if filepathext.IsAbs(path) {
+		return path, nil
+	}
+
+	// NOTE: Uses the directory of the entrypoint (Taskfile), not the current working directory
+	// This means that files are included relative to one another
+	entrypointDir := filepath.Dir(node.Dir())
+	return filepathext.SmartJoin(entrypointDir, path), nil
+}
+
+func (node *GitNode) FilenameAndLastDir() (string, string) {
+	return filepath.Base(node.path), filepath.Base(filepath.Dir(node.path))
+}

taskfile/node_git_test.go

@@ -0,0 +1,75 @@
+package taskfile
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGitNode_ssh(t *testing.T) {
+	node, err := NewGitNode("[email protected]:foo/bar.git//Taskfile.yml?ref=main", "", false)
+	assert.NoError(t, err)
+	assert.Equal(t, "main", node.ref)
+	assert.Equal(t, "Taskfile.yml", node.path)
+	assert.Equal(t, "ssh://[email protected]/foo/bar.git//Taskfile.yml?ref=main", node.rawUrl)
+	assert.Equal(t, "ssh://[email protected]/foo/bar.git", node.URL.String())
+	entrypoint, err := node.ResolveEntrypoint("common.yml")
+	assert.NoError(t, err)
+	assert.Equal(t, "ssh://[email protected]/foo/bar.git//common.yml?ref=main", entrypoint)
+}
+
+func TestGitNode_sshWithDir(t *testing.T) {
+	node, err := NewGitNode("[email protected]:foo/bar.git//directory/Taskfile.yml?ref=main", "", false)
+	assert.NoError(t, err)
+	assert.Equal(t, "main", node.ref)
+	assert.Equal(t, "directory/Taskfile.yml", node.path)
+	assert.Equal(t, "ssh://[email protected]/foo/bar.git//directory/Taskfile.yml?ref=main", node.rawUrl)
+	assert.Equal(t, "ssh://[email protected]/foo/bar.git", node.URL.String())
+	entrypoint, err := node.ResolveEntrypoint("common.yml")
+	assert.NoError(t, err)
+	assert.Equal(t, "ssh://[email protected]/foo/bar.git//directory/common.yml?ref=main", entrypoint)
+}
+
+func TestGitNode_https(t *testing.T) {
+	node, err := NewGitNode("https://github.com/foo/bar.git//Taskfile.yml?ref=main", "", false)
+	assert.NoError(t, err)
+	assert.Equal(t, "main", node.ref)
+	assert.Equal(t, "Taskfile.yml", node.path)
+	assert.Equal(t, "https://github.com/foo/bar.git//Taskfile.yml?ref=main", node.rawUrl)
+	assert.Equal(t, "https://github.com/foo/bar.git", node.URL.String())
+	entrypoint, err := node.ResolveEntrypoint("common.yml")
+	assert.NoError(t, err)
+	assert.Equal(t, "https://github.com/foo/bar.git//common.yml?ref=main", entrypoint)
+}
+
+func TestGitNode_httpsWithDir(t *testing.T) {
+	node, err := NewGitNode("https://github.com/foo/bar.git//directory/Taskfile.yml?ref=main", "", false)
+	assert.NoError(t, err)
+	assert.Equal(t, "main", node.ref)
+	assert.Equal(t, "directory/Taskfile.yml", node.path)
+	assert.Equal(t, "https://github.com/foo/bar.git//directory/Taskfile.yml?ref=main", node.rawUrl)
+	assert.Equal(t, "https://github.com/foo/bar.git", node.URL.String())
+	entrypoint, err := node.ResolveEntrypoint("common.yml")
+	assert.NoError(t, err)
+	assert.Equal(t, "https://github.com/foo/bar.git//directory/common.yml?ref=main", entrypoint)
+}
+
+func TestGitNode_FilenameAndDir(t *testing.T) {
+	node, err := NewGitNode("https://github.com/foo/bar.git//directory/Taskfile.yml?ref=main", "", false)
+	assert.NoError(t, err)
+	filename, dir := node.FilenameAndLastDir()
+	assert.Equal(t, "Taskfile.yml", filename)
+	assert.Equal(t, "directory", dir)
+
+	node, err = NewGitNode("https://github.com/foo/bar.git//Taskfile.yml?ref=main", "", false)
+	assert.NoError(t, err)
+	filename, dir = node.FilenameAndLastDir()
+	assert.Equal(t, "Taskfile.yml", filename)
+	assert.Equal(t, ".", dir)
+
+	node, err = NewGitNode("https://github.com/foo/bar.git//multiple/directory/Taskfile.yml?ref=main", "", false)
+	assert.NoError(t, err)
+	filename, dir = node.FilenameAndLastDir()
+	assert.Equal(t, "Taskfile.yml", filename)
+	assert.Equal(t, "directory", dir)
+}

taskfile/node_http.go

@@ -74,7 +74,6 @@ func (node *HTTPNode) Read(ctx context.Context) ([]byte, error) {
 		return nil, errors.TaskfileFetchFailedError{URI: node.URL.String()}
 	}
 	defer resp.Body.Close()
-
 	if resp.StatusCode != http.StatusOK {
 		return nil, errors.TaskfileFetchFailedError{
 			URI:            node.URL.String(),

taskfile/node_test.go

@@ -0,0 +1,22 @@
+package taskfile
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestScheme(t *testing.T) {
+	scheme, err := getScheme("https://github.com/foo/bar.git")
+	assert.NoError(t, err)
+	assert.Equal(t, "git", scheme)
+	scheme, err = getScheme("https://github.com/foo/bar.git?ref=v1//taskfile/common.yml")
+	assert.NoError(t, err)
+	assert.Equal(t, "git", scheme)
+	scheme, err = getScheme("[email protected]:foo/bar.git?ref=main//Taskfile.yml")
+	assert.NoError(t, err)
+	assert.Equal(t, "git", scheme)
+	scheme, err = getScheme("https://github.com/foo/common.yml")
+	assert.NoError(t, err)
+	assert.Equal(t, "https", scheme)
+}

website/docs/experiments/remote_taskfiles.mdx

@@ -62,6 +62,16 @@ includes:
 `TOKEN=my-token task my-remote-namespace:hello` will be resolved by Task to
 `https://[email protected]/my-org/my-repo/main/Taskfile.yml`
 
+## Git nodes
+
+You can also include a Taskfile from a Git node. We currently support ssh-style and http / https addresses like `[email protected]/foo/bar.git//Taskfiles.yml?ref=v1` and `https://example.com/foo/bar.git//Taskfiles.yml?ref=v1`.
+
+You need to follow this pattern : `<baseUrl>.git//<path>?ref=<ref>`.
+The `ref` parameter, optional, can be a branch name or a tag, if not provided it'll pick up the default branch.
+The `path` is the path to the Taskfile in the repository.
+
+If you want to use the SSH protocol, you need to make sure that your ssh-agent has your private ssh keys added so that they can be used during authentication.
+
 ## Security
 
 Running commands from sources that you do not control is always a potential