Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(remote): support include git remote #1652

Merged
merged 11 commits into from
Sep 24, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 23 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ require (
github.com/davecgh/go-spew v1.1.1
github.com/dominikbraun/graph v0.23.0
github.com/fatih/color v1.17.0
github.com/go-git/go-billy/v5 v5.5.0
github.com/go-git/go-git/v5 v5.12.0
github.com/go-task/slim-sprig/v3 v3.0.0
github.com/go-task/template v0.1.0
github.com/joho/godotenv v1.5.1
Expand All @@ -19,6 +21,7 @@ require (
github.com/sajari/fuzzy v1.0.0
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.9.0
github.com/whilp/git-urls v1.0.0
github.com/zeebo/xxh3 v1.0.2
golang.org/x/sync v0.8.0
golang.org/x/term v0.24.0
Expand All @@ -27,13 +30,31 @@ require (
)

require (
dario.cat/mergo v1.0.0 // indirect
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/ProtonMail/go-crypto v1.0.0 // indirect
github.com/cloudflare/circl v1.3.7 // indirect
github.com/cyphar/filepath-securejoin v0.2.4 // indirect
github.com/dlclark/regexp2 v1.11.0 // indirect
github.com/klauspost/cpuid/v2 v2.0.9 // indirect
github.com/emirpasic/gods v1.18.1 // indirect
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/kevinburke/ssh_config v1.2.0 // indirect
github.com/klauspost/cpuid/v2 v2.2.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/muesli/cancelreader v0.2.2 // indirect
github.com/pjbgf/sha1cd v0.3.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
github.com/skeema/knownhosts v1.2.2 // indirect
github.com/stretchr/objx v0.5.2 // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
golang.org/x/crypto v0.23.0 // indirect
golang.org/x/mod v0.17.0 // indirect
golang.org/x/net v0.25.0 // indirect
golang.org/x/sys v0.25.0 // indirect
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
golang.org/x/tools v0.21.0 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
)
129 changes: 125 additions & 4 deletions go.sum

Large diffs are not rendered by default.

27 changes: 22 additions & 5 deletions taskfile/node.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import (
"strings"
"time"

giturls "github.com/whilp/git-urls"

"github.com/go-task/task/v3/errors"
"github.com/go-task/task/v3/internal/experiments"
"github.com/go-task/task/v3/internal/logger"
Expand Down Expand Up @@ -48,24 +50,39 @@ func NewNode(
) (Node, error) {
var node Node
var err error
switch getScheme(entrypoint) {
scheme, err := getScheme(entrypoint)
if err != nil {
return nil, err
}
switch scheme {
case "git":
node, err = NewGitNode(entrypoint, dir, insecure, opts...)
case "http", "https":
node, err = NewHTTPNode(l, entrypoint, dir, insecure, timeout, opts...)
default:
// If no other scheme matches, we assume it's a file
node, err = NewFileNode(l, entrypoint, dir, opts...)

}

if node.Remote() && !experiments.RemoteTaskfiles.Enabled {
return nil, errors.New("task: Remote taskfiles are not enabled. You can read more about this experiment and how to enable it at https://taskfile.dev/experiments/remote-taskfiles")
}
return node, err
}

func getScheme(uri string) string {
func getScheme(uri string) (string, error) {
u, err := giturls.Parse(uri)
if u == nil {
return "", err
}
if strings.HasSuffix(strings.Split(u.Path, "//")[0], ".git") && (u.Scheme == "git" || u.Scheme == "ssh" || u.Scheme == "https" || u.Scheme == "http") {
return "git", nil
}

if i := strings.Index(uri, "://"); i != -1 {
return uri[:i]
return uri[:i], nil
}
return ""
return "", nil
}

func getDefaultDir(entrypoint, dir string) string {
Expand Down
3 changes: 3 additions & 0 deletions taskfile/node_file.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,9 @@ func (node *FileNode) ResolveEntrypoint(entrypoint string) (string, error) {
if strings.Contains(entrypoint, "://") {
return entrypoint, nil
}
if strings.HasPrefix(entrypoint, "git") {
return entrypoint, nil
}

path, err := execext.Expand(entrypoint)
if err != nil {
Expand Down
126 changes: 126 additions & 0 deletions taskfile/node_git.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
package taskfile

import (
"context"
"fmt"
"io"
"net/url"
"path/filepath"
"strings"

"github.com/go-git/go-billy/v5/memfs"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/storage/memory"
giturls "github.com/whilp/git-urls"

"github.com/go-task/task/v3/errors"
"github.com/go-task/task/v3/internal/execext"
"github.com/go-task/task/v3/internal/filepathext"
)

// An GitNode is a node that reads a Taskfile from a remote location via Git.
type GitNode struct {
*BaseNode
URL *url.URL
rawUrl string
ref string
path string
}

func NewGitNode(
entrypoint string,
dir string,
insecure bool,
opts ...NodeOption,
) (*GitNode, error) {
base := NewBaseNode(dir, opts...)
u, err := giturls.Parse(entrypoint)
if err != nil {
return nil, err
}

basePath, path := func() (string, string) {
x := strings.Split(u.Path, "//")
return x[0], x[1]
}()
ref := u.Query().Get("ref")

rawUrl := u.String()

u.RawQuery = ""
u.Path = basePath

if u.Scheme == "http" && !insecure {
return nil, &errors.TaskfileNotSecureError{URI: entrypoint}
}
return &GitNode{
BaseNode: base,
URL: u,
rawUrl: rawUrl,
ref: ref,
path: path,
}, nil
}

func (node *GitNode) Location() string {
return node.rawUrl
}

func (node *GitNode) Remote() bool {
return true
}

func (node *GitNode) Read(_ context.Context) ([]byte, error) {
fs := memfs.New()
storer := memory.NewStorage()
_, err := git.Clone(storer, fs, &git.CloneOptions{
URL: node.URL.String(),
ReferenceName: plumbing.ReferenceName(node.ref),
SingleBranch: true,
Depth: 1,
})
if err != nil {
return nil, err
}
file, err := fs.Open(node.path)
if err != nil {
return nil, err
}
// Read the entire response body
b, err := io.ReadAll(file)
if err != nil {
return nil, err
}

return b, nil
}

func (node *GitNode) ResolveEntrypoint(entrypoint string) (string, error) {
dir, _ := filepath.Split(node.path)
resolvedEntrypoint := fmt.Sprintf("%s//%s", node.URL, filepath.Join(dir, entrypoint))
if node.ref != "" {
return fmt.Sprintf("%s?ref=%s", resolvedEntrypoint, node.ref), nil
}
return resolvedEntrypoint, nil
}

func (node *GitNode) ResolveDir(dir string) (string, error) {
path, err := execext.Expand(dir)
if err != nil {
return "", err
}

if filepathext.IsAbs(path) {
return path, nil
}

// NOTE: Uses the directory of the entrypoint (Taskfile), not the current working directory
// This means that files are included relative to one another
entrypointDir := filepath.Dir(node.Dir())
return filepathext.SmartJoin(entrypointDir, path), nil
}

func (node *GitNode) FilenameAndLastDir() (string, string) {
return filepath.Base(node.path), filepath.Base(filepath.Dir(node.path))
}
75 changes: 75 additions & 0 deletions taskfile/node_git_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
package taskfile

import (
"testing"

"github.com/stretchr/testify/assert"
)

func TestGitNode_ssh(t *testing.T) {
node, err := NewGitNode("[email protected]:foo/bar.git//Taskfile.yml?ref=main", "", false)
assert.NoError(t, err)
assert.Equal(t, "main", node.ref)
assert.Equal(t, "Taskfile.yml", node.path)
assert.Equal(t, "ssh://[email protected]/foo/bar.git//Taskfile.yml?ref=main", node.rawUrl)
assert.Equal(t, "ssh://[email protected]/foo/bar.git", node.URL.String())
entrypoint, err := node.ResolveEntrypoint("common.yml")
assert.NoError(t, err)
assert.Equal(t, "ssh://[email protected]/foo/bar.git//common.yml?ref=main", entrypoint)
}

func TestGitNode_sshWithDir(t *testing.T) {
node, err := NewGitNode("[email protected]:foo/bar.git//directory/Taskfile.yml?ref=main", "", false)
assert.NoError(t, err)
assert.Equal(t, "main", node.ref)
assert.Equal(t, "directory/Taskfile.yml", node.path)
assert.Equal(t, "ssh://[email protected]/foo/bar.git//directory/Taskfile.yml?ref=main", node.rawUrl)
assert.Equal(t, "ssh://[email protected]/foo/bar.git", node.URL.String())
entrypoint, err := node.ResolveEntrypoint("common.yml")
assert.NoError(t, err)
assert.Equal(t, "ssh://[email protected]/foo/bar.git//directory/common.yml?ref=main", entrypoint)
}

func TestGitNode_https(t *testing.T) {
node, err := NewGitNode("https://github.com/foo/bar.git//Taskfile.yml?ref=main", "", false)
assert.NoError(t, err)
assert.Equal(t, "main", node.ref)
assert.Equal(t, "Taskfile.yml", node.path)
assert.Equal(t, "https://github.com/foo/bar.git//Taskfile.yml?ref=main", node.rawUrl)
assert.Equal(t, "https://github.com/foo/bar.git", node.URL.String())
entrypoint, err := node.ResolveEntrypoint("common.yml")
assert.NoError(t, err)
assert.Equal(t, "https://github.com/foo/bar.git//common.yml?ref=main", entrypoint)
}

func TestGitNode_httpsWithDir(t *testing.T) {
node, err := NewGitNode("https://github.com/foo/bar.git//directory/Taskfile.yml?ref=main", "", false)
assert.NoError(t, err)
assert.Equal(t, "main", node.ref)
assert.Equal(t, "directory/Taskfile.yml", node.path)
assert.Equal(t, "https://github.com/foo/bar.git//directory/Taskfile.yml?ref=main", node.rawUrl)
assert.Equal(t, "https://github.com/foo/bar.git", node.URL.String())
entrypoint, err := node.ResolveEntrypoint("common.yml")
assert.NoError(t, err)
assert.Equal(t, "https://github.com/foo/bar.git//directory/common.yml?ref=main", entrypoint)
}

func TestGitNode_FilenameAndDir(t *testing.T) {
node, err := NewGitNode("https://github.com/foo/bar.git//directory/Taskfile.yml?ref=main", "", false)
assert.NoError(t, err)
filename, dir := node.FilenameAndLastDir()
assert.Equal(t, "Taskfile.yml", filename)
assert.Equal(t, "directory", dir)

node, err = NewGitNode("https://github.com/foo/bar.git//Taskfile.yml?ref=main", "", false)
assert.NoError(t, err)
filename, dir = node.FilenameAndLastDir()
assert.Equal(t, "Taskfile.yml", filename)
assert.Equal(t, ".", dir)

node, err = NewGitNode("https://github.com/foo/bar.git//multiple/directory/Taskfile.yml?ref=main", "", false)
assert.NoError(t, err)
filename, dir = node.FilenameAndLastDir()
assert.Equal(t, "Taskfile.yml", filename)
assert.Equal(t, "directory", dir)
}
1 change: 0 additions & 1 deletion taskfile/node_http.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,6 @@ func (node *HTTPNode) Read(ctx context.Context) ([]byte, error) {
return nil, errors.TaskfileFetchFailedError{URI: node.URL.String()}
}
defer resp.Body.Close()

if resp.StatusCode != http.StatusOK {
return nil, errors.TaskfileFetchFailedError{
URI: node.URL.String(),
Expand Down
22 changes: 22 additions & 0 deletions taskfile/node_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
package taskfile

import (
"testing"

"github.com/stretchr/testify/assert"
)

func TestScheme(t *testing.T) {
scheme, err := getScheme("https://github.com/foo/bar.git")
assert.NoError(t, err)
assert.Equal(t, "git", scheme)
scheme, err = getScheme("https://github.com/foo/bar.git?ref=v1//taskfile/common.yml")
assert.NoError(t, err)
assert.Equal(t, "git", scheme)
scheme, err = getScheme("[email protected]:foo/bar.git?ref=main//Taskfile.yml")
assert.NoError(t, err)
assert.Equal(t, "git", scheme)
scheme, err = getScheme("https://github.com/foo/common.yml")
assert.NoError(t, err)
assert.Equal(t, "https", scheme)
}
10 changes: 10 additions & 0 deletions website/docs/experiments/remote_taskfiles.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,16 @@ includes:
`TOKEN=my-token task my-remote-namespace:hello` will be resolved by Task to
`https://[email protected]/my-org/my-repo/main/Taskfile.yml`

## Git nodes

You can also include a Taskfile from a Git node. We currently support ssh-style and http / https addresses like `[email protected]/foo/bar.git//Taskfiles.yml?ref=v1` and `https://example.com/foo/bar.git//Taskfiles.yml?ref=v1`.

You need to follow this pattern : `<baseUrl>.git//<path>?ref=<ref>`.
The `ref` parameter, optional, can be a branch name or a tag, if not provided it'll pick up the default branch.
The `path` is the path to the Taskfile in the repository.

If you want to use the SSH protocol, you need to make sure that your ssh-agent has your private ssh keys added so that they can be used during authentication.

## Security

Running commands from sources that you do not control is always a potential
Expand Down
Loading