Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Support multiple providers for Remote Taskfiles and support directories #1774

Closed
wants to merge 13 commits into from
Closed
32 changes: 31 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ require (
github.com/fatih/color v1.17.0
github.com/go-task/slim-sprig/v3 v3.0.0
github.com/go-task/template v0.1.0
github.com/hashicorp/go-getter v1.7.5
github.com/joho/godotenv v1.5.1
github.com/mattn/go-zglob v0.0.6
github.com/mitchellh/hashstructure/v2 v2.0.2
Expand All @@ -27,13 +28,42 @@ require (
)

require (
cloud.google.com/go v0.104.0 // indirect
cloud.google.com/go/compute v1.10.0 // indirect
cloud.google.com/go/iam v0.5.0 // indirect
cloud.google.com/go/storage v1.27.0 // indirect
github.com/aws/aws-sdk-go v1.44.122 // indirect
github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
github.com/dlclark/regexp2 v1.11.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect
github.com/googleapis/gax-go/v2 v2.6.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-safetemp v1.0.0 // indirect
github.com/hashicorp/go-version v1.6.0 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/klauspost/compress v1.15.11 // indirect
github.com/klauspost/cpuid/v2 v2.0.9 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/go-testing-interface v1.14.1 // indirect
github.com/muesli/cancelreader v0.2.2 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/stretchr/objx v0.5.2 // indirect
github.com/ulikunitz/xz v0.5.10 // indirect
go.opencensus.io v0.23.0 // indirect
golang.org/x/net v0.1.0 // indirect
golang.org/x/oauth2 v0.1.0 // indirect
golang.org/x/sys v0.25.0 // indirect
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
golang.org/x/text v0.4.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/api v0.100.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20221025140454-527a21cfbd71 // indirect
google.golang.org/grpc v1.50.1 // indirect
google.golang.org/protobuf v1.28.1 // indirect
)
871 changes: 871 additions & 0 deletions go.sum

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion internal/compiler/compiler.go
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ func (c *Compiler) getSpecialVars(t *ast.Task, call *ast.Call) (map[string]strin
"ROOT_TASKFILE": filepathext.SmartJoin(c.Dir, c.Entrypoint),
"ROOT_DIR": c.Dir,
"TASKFILE": t.Location.Taskfile,
"TASKFILE_DIR": filepath.Dir(t.Location.Taskfile),
"TASKFILE_DIR": t.Location.TaskfileDir,
"USER_WORKING_DIR": c.UserWorkingDir,
"TASK_VERSION": version.GetVersion(),
}, nil
Expand Down
228 changes: 228 additions & 0 deletions task_test.go
Original file line number Diff line number Diff line change
@@ -1,17 +1,23 @@
package task_test

import (
"archive/zip"
"bytes"
"context"
"fmt"
"io"
"math/rand"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"regexp"
"runtime"
"strings"
"sync"
"testing"
"time"

"github.com/Masterminds/semver/v3"
"github.com/stretchr/testify/assert"
Expand All @@ -21,9 +27,12 @@ import (
"github.com/go-task/task/v3/errors"
"github.com/go-task/task/v3/internal/experiments"
"github.com/go-task/task/v3/internal/filepathext"
"github.com/go-task/task/v3/internal/logger"
"github.com/go-task/task/v3/taskfile/ast"
)

var random = rand.New(rand.NewSource(time.Now().UnixNano()))

func init() {
_ = os.Setenv("NO_COLOR", "1")
}
Expand Down Expand Up @@ -1042,6 +1051,225 @@ func TestIncludesMultiLevel(t *testing.T) {
tt.Run(t)
}

func TestIncludesRemote(t *testing.T) {
dir := "testdata/includes_remote"

os.RemoveAll(filepath.Join(dir, ".task"))

srv := httptest.NewServer(http.FileServer(http.Dir(dir)))
defer srv.Close()

createZipFileOfDir(t, filepath.Join(dir, "tasks-root.zip"), dir)
createZipFileOfDir(t, filepath.Join(dir, "tasks-first.zip"), filepath.Join(dir, "first"))

tcs := []struct {
rootTaskfile string
firstRemote string
secondRemote string
extraTasks []string
}{
//
// NOTE: When adding content for tests that use `getGitRemoteURL`,
// you must commit the test data for the tests to be able to find it.
//
// These tests will not see data in the working tree because they clone
// this repo.
//
{
// Ensure non-remote includes still work
firstRemote: "./first/Taskfile.yml",
secondRemote: "./second/Taskfile.yml",
},
{
firstRemote: srv.URL + "/first/Taskfile.yml",
secondRemote: srv.URL + "/first/second/Taskfile.yml",
},
{
firstRemote: srv.URL + "/first/Taskfile.yml",
secondRemote: "./second/Taskfile.yml",
},
{
firstRemote: getGitRemoteURL(t, dir+"/first"),
secondRemote: getGitRemoteURL(t, dir+"/first/second"),
},
{
firstRemote: srv.URL + "/first/Taskfile.yml",
secondRemote: getGitRemoteURL(t, dir+"/first/second"),
},
{
firstRemote: getGitRemoteURL(t, dir+"/first"),
secondRemote: srv.URL + "/first/second/Taskfile.yml",
},
{
firstRemote: getGitRemoteURL(t, dir+"/first"),
secondRemote: "./second/Taskfile.yml",
extraTasks: []string{
"first:check-if-neighbor-file-exists",
"first:second:check-if-neighbor-file-exists",
},
},
{
firstRemote: getGitRemoteURL(t, dir+"/first") + "&taskfile=Taskfile2.yml",
extraTasks: []string{
"first:first-taskfile2-task",
},
},
{
firstRemote: getGitRemoteURL(t, dir+"/first") + "&taskfile=second/Taskfile2.yml",
extraTasks: []string{
"first:second-taskfile2-task",
},
},
{
firstRemote: srv.URL + "/tasks-first.zip",
secondRemote: "./second/Taskfile.yml",
extraTasks: []string{
"first:check-if-neighbor-file-exists",
"first:second:check-if-neighbor-file-exists",
},
},
{
rootTaskfile: srv.URL + "/Taskfile.yml",
firstRemote: "./first/Taskfile.yml",
secondRemote: "./second/Taskfile.yml",
},
{
rootTaskfile: getGitRemoteURL(t, dir),
firstRemote: "./first/Taskfile.yml",
secondRemote: "./second/Taskfile.yml",
extraTasks: []string{
"first:check-if-neighbor-file-exists",
"first:second:check-if-neighbor-file-exists",
},
},
{
rootTaskfile: srv.URL + "/tasks-root.zip",
firstRemote: "./first/Taskfile.yml",
secondRemote: "./second/Taskfile.yml",
extraTasks: []string{
"first:check-if-neighbor-file-exists",
"first:second:check-if-neighbor-file-exists",
},
},
}

tasks := []string{
"first:write-file",
"first:second:write-file",
}

for i, tc := range tcs {
t.Run(fmt.Sprint(i), func(t *testing.T) {
t.Setenv("FIRST_REMOTE_URL", tc.firstRemote)
t.Setenv("SECOND_REMOTE_URL", tc.secondRemote)

executors := []struct {
name string
executor *task.Executor
}{
{
name: "online, always download",
executor: &task.Executor{
Dir: dir,
Entrypoint: tc.rootTaskfile,
Timeout: time.Minute,
Insecure: true,
Verbose: true,

// Without caching
AssumeYes: true,
Download: true,
Offline: false,
},
},
{
name: "offline, use-cache",
executor: &task.Executor{
Dir: dir,
Entrypoint: tc.rootTaskfile,
Timeout: time.Minute,
Insecure: true,
Verbose: true,

// With caching
AssumeYes: false,
Download: false,
Offline: true,
},
},
}

for j, e := range executors {
t.Run(fmt.Sprint(j), func(t *testing.T) {
var buff SyncBuffer
defer func() { t.Log("\noutput:\n", buff.buf.String()) }()

e.executor.Stderr = &buff
e.executor.Stdout = &buff
e.executor.Logger = &logger.Logger{Stderr: &buff, Stdout: &buff, Verbose: true}

require.NoError(t, e.executor.Setup())

for k, task := range tasks {
t.Run(task, func(t *testing.T) {
expectedContent := fmt.Sprint(random.Int63())
t.Setenv("CONTENT", expectedContent)

outputFile := fmt.Sprintf("%d.%d.txt", i, k)
t.Setenv("OUTPUT_FILE", outputFile)

path := filepath.Join(dir, outputFile)
require.NoError(t, os.RemoveAll(path))

require.NoError(t, e.executor.Run(context.Background(), &ast.Call{Task: task}))

actualContent, err := os.ReadFile(path)
require.NoError(t, err)
assert.Equal(t, expectedContent, strings.TrimSpace(string(actualContent)))
})
}

for _, task := range tc.extraTasks {
t.Run(task, func(t *testing.T) {
require.NoError(t, e.executor.Run(context.Background(), &ast.Call{Task: task}))
})
}
})
}
})
}
}

func createZipFileOfDir(t *testing.T, zipFilePath string, dir string) {
f, err := os.OpenFile(zipFilePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o644)
require.NoError(t, err)
defer f.Close()

w := zip.NewWriter(f)
err = w.AddFS(os.DirFS(dir))
require.NoError(t, err)
w.Close()
}

func getGitRemoteURL(t *testing.T, path string) string {
repoRoot, err := exec.Command("git", "rev-parse", "--show-toplevel").Output()
require.NoError(t, err)

// This is to support Github Workflows on PRs where we are in a detached HEAD state.
branch := os.Getenv("GITHUB_REF")
if branch == "" {
b, err := exec.Command("git", "branch", "--show-current").Output()
require.NoError(t, err)
branch = string(b)
}

return fmt.Sprintf("git::file://%s//%s?ref=%s&depth=1",
strings.TrimSpace(string(repoRoot)),
path,
strings.TrimSpace(string(branch)),
)
}

func TestIncludeCycle(t *testing.T) {
const dir = "testdata/includes_cycle"

Expand Down
14 changes: 8 additions & 6 deletions taskfile/ast/location.go
Original file line number Diff line number Diff line change
@@ -1,18 +1,20 @@
package ast

type Location struct {
Line int
Column int
Taskfile string
Line int
Column int
Taskfile string
TaskfileDir string
}

func (l *Location) DeepCopy() *Location {
if l == nil {
return nil
}
return &Location{
Line: l.Line,
Column: l.Column,
Taskfile: l.Taskfile,
Line: l.Line,
Column: l.Column,
Taskfile: l.Taskfile,
TaskfileDir: l.TaskfileDir,
}
}
Loading