From 22ce67c5e50e81ac63902615e54745c0247f19d8 Mon Sep 17 00:00:00 2001 From: Pete Davison Date: Tue, 12 Sep 2023 16:42:54 -0500 Subject: [PATCH] feat: remote taskfiles (HTTP) (#1152) * feat: remote taskfiles over http * feat: allow insecure connections when --insecure flag is provided * feat: better error handling for fetch errors * fix: ensure cache directory always exists * fix: setup logger before everything else * feat: put remote taskfiles behind an experiment * feat: --download and --offline flags for remote taskfiles * feat: node.Read accepts a context * feat: experiment docs * chore: changelog * chore: remove unused optional param from Node interface * chore: tidy up and generalise NewNode function * fix: use sha256 in remote checksum * feat: --download by itself will not run a task * feat: custom error if remote taskfiles experiment is not enabled * refactor: BaseNode functional options and simplified FileNode * fix: use hex encoding for checksum instead of b64 --- CHANGELOG.md | 8 +- args/args.go | 12 +- args/args_test.go | 36 ++---- cmd/task/task.go | 24 ++++ docs/docs/experiments/remote_taskfiles.md | 81 +++++++++++++ errors/errors.go | 14 +++ errors/errors_taskfile.go | 73 +++++++++++- internal/experiments/experiments.go | 21 +++- internal/logger/logger.go | 17 +++ setup.go | 30 +++-- task.go | 3 + task_test.go | 6 + taskfile/included_taskfile.go | 6 + taskfile/read/cache.go | 58 +++++++++ taskfile/read/node.go | 35 ++++-- taskfile/read/node_base.go | 43 +++++-- taskfile/read/node_file.go | 56 ++++----- taskfile/read/node_http.go | 67 +++++++++++ taskfile/read/taskfile.go | 139 ++++++++++++++++++++-- 19 files changed, 610 insertions(+), 119 deletions(-) create mode 100644 docs/docs/experiments/remote_taskfiles.md create mode 100644 taskfile/read/cache.go create mode 100644 taskfile/read/node_http.go diff --git a/CHANGELOG.md b/CHANGELOG.md index e0f0e71025..2a6b56b25a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,10 @@ ## Unreleased -- Prep work for remote Taskfiles (#1316 by @pd93). +- Prep work for Remote Taskfiles (#1316 by @pd93). +- Added the + [Remote Taskfiles experiment](https://taskfile.dev/experiments/remote-taskfiles) + as a draft (#1152, #1317 by @pd93). ## v3.29.1 - 2023-08-26 @@ -42,7 +45,8 @@ - Bug fixes were made to the [npm installation method](https://taskfile.dev/installation/#npm). (#1190, by @sounisi5011). -- Added the [gentle force experiment](https://taskfile.dev/experiments) as a +- Added the + [gentle force experiment](https://taskfile.dev/experiments/gentle-force) as a draft (#1200, #1216 by @pd93). - Added an `--experiments` flag to allow you to see which experiments are enabled (#1242 by @pd93). diff --git a/args/args.go b/args/args.go index 801ff88bb0..921b723a23 100644 --- a/args/args.go +++ b/args/args.go @@ -8,7 +8,7 @@ import ( // ParseV3 parses command line argument: tasks and global variables func ParseV3(args ...string) ([]taskfile.Call, *taskfile.Vars) { - var calls []taskfile.Call + calls := []taskfile.Call{} globals := &taskfile.Vars{} for _, arg := range args { @@ -21,16 +21,12 @@ func ParseV3(args ...string) ([]taskfile.Call, *taskfile.Vars) { globals.Set(name, taskfile.Var{Static: value}) } - if len(calls) == 0 { - calls = append(calls, taskfile.Call{Task: "default", Direct: true}) - } - return calls, globals } // ParseV2 parses command line argument: tasks and vars of each task func ParseV2(args ...string) ([]taskfile.Call, *taskfile.Vars) { - var calls []taskfile.Call + calls := []taskfile.Call{} globals := &taskfile.Vars{} for _, arg := range args { @@ -51,10 +47,6 @@ func ParseV2(args ...string) ([]taskfile.Call, *taskfile.Vars) { } } - if len(calls) == 0 { - calls = append(calls, taskfile.Call{Task: "default", Direct: true}) - } - return calls, globals } diff --git a/args/args_test.go b/args/args_test.go index 3b74c7ddba..5cea4b0296 100644 --- a/args/args_test.go +++ b/args/args_test.go @@ -73,22 +73,16 @@ func TestArgsV3(t *testing.T) { }, }, { - Args: nil, - ExpectedCalls: []taskfile.Call{ - {Task: "default", Direct: true}, - }, + Args: nil, + ExpectedCalls: []taskfile.Call{}, }, { - Args: []string{}, - ExpectedCalls: []taskfile.Call{ - {Task: "default", Direct: true}, - }, + Args: []string{}, + ExpectedCalls: []taskfile.Call{}, }, { - Args: []string{"FOO=bar", "BAR=baz"}, - ExpectedCalls: []taskfile.Call{ - {Task: "default", Direct: true}, - }, + Args: []string{"FOO=bar", "BAR=baz"}, + ExpectedCalls: []taskfile.Call{}, ExpectedGlobals: &taskfile.Vars{ OrderedMap: orderedmap.FromMapWithOrder( map[string]taskfile.Var{ @@ -191,22 +185,16 @@ func TestArgsV2(t *testing.T) { }, }, { - Args: nil, - ExpectedCalls: []taskfile.Call{ - {Task: "default", Direct: true}, - }, + Args: nil, + ExpectedCalls: []taskfile.Call{}, }, { - Args: []string{}, - ExpectedCalls: []taskfile.Call{ - {Task: "default", Direct: true}, - }, + Args: []string{}, + ExpectedCalls: []taskfile.Call{}, }, { - Args: []string{"FOO=bar", "BAR=baz"}, - ExpectedCalls: []taskfile.Call{ - {Task: "default", Direct: true}, - }, + Args: []string{"FOO=bar", "BAR=baz"}, + ExpectedCalls: []taskfile.Call{}, ExpectedGlobals: &taskfile.Vars{ OrderedMap: orderedmap.FromMapWithOrder( map[string]taskfile.Var{ diff --git a/cmd/task/task.go b/cmd/task/task.go index dc70c6c59e..b424bfacd7 100644 --- a/cmd/task/task.go +++ b/cmd/task/task.go @@ -53,6 +53,7 @@ var flags struct { listJson bool taskSort string status bool + insecure bool force bool forceAll bool watch bool @@ -71,6 +72,8 @@ var flags struct { interval time.Duration global bool experiments bool + download bool + offline bool } func main() { @@ -112,6 +115,7 @@ func run() error { pflag.BoolVarP(&flags.listJson, "json", "j", false, "Formats task list as JSON.") pflag.StringVar(&flags.taskSort, "sort", "", "Changes the order of the tasks when listed. [default|alphanumeric|none].") pflag.BoolVar(&flags.status, "status", false, "Exits with non-zero exit code if any of the given tasks is not up-to-date.") + pflag.BoolVar(&flags.insecure, "insecure", false, "Forces Task to download Taskfiles over insecure connections.") pflag.BoolVarP(&flags.watch, "watch", "w", false, "Enables watch of the given task.") pflag.BoolVarP(&flags.verbose, "verbose", "v", false, "Enables verbose mode.") pflag.BoolVarP(&flags.silent, "silent", "s", false, "Disables echoing.") @@ -140,6 +144,12 @@ func run() error { pflag.BoolVarP(&flags.forceAll, "force", "f", false, "Forces execution even when the task is up-to-date.") } + // Remote Taskfiles experiment will adds the "download" and "offline" flags + if experiments.RemoteTaskfiles { + pflag.BoolVar(&flags.download, "download", false, "Downloads a cached version of a remote Taskfile.") + pflag.BoolVar(&flags.offline, "offline", false, "Forces Task to only use local or cached Taskfiles.") + } + pflag.Parse() if flags.version { @@ -173,6 +183,10 @@ func run() error { return nil } + if flags.download && flags.offline { + return errors.New("task: You can't set both --download and --offline flags") + } + if flags.global && flags.dir != "" { log.Fatal("task: You can't set both --global and --dir") return nil @@ -216,6 +230,9 @@ func run() error { e := task.Executor{ Force: flags.force, ForceAll: flags.forceAll, + Insecure: flags.insecure, + Download: flags.download, + Offline: flags.offline, Watch: flags.watch, Verbose: flags.verbose, Silent: flags.silent, @@ -278,6 +295,13 @@ func run() error { calls, globals = args.ParseV2(tasksAndVars...) } + // If there are no calls, run the default task instead + // Unless the download flag is specified, in which case we want to download + // the Taskfile and do nothing else + if len(calls) == 0 && !flags.download { + calls = append(calls, taskfile.Call{Task: "default", Direct: true}) + } + globals.Set("CLI_ARGS", taskfile.Var{Static: cliArgs}) e.Taskfile.Vars.Merge(globals) diff --git a/docs/docs/experiments/remote_taskfiles.md b/docs/docs/experiments/remote_taskfiles.md new file mode 100644 index 0000000000..fb8ba12696 --- /dev/null +++ b/docs/docs/experiments/remote_taskfiles.md @@ -0,0 +1,81 @@ +--- +slug: /experiments/remote-taskfiles/ +--- + +# Remote Taskfiles + +- Issue: [#1317][remote-taskfiles-experiment] +- Environment variable: `TASK_X_REMOTE_TASKFILES=1` + +This experiment allows you to specify a remote Taskfile URL when including a +Taskfile. For example: + +```yaml +version: '3' + +include: + my-remote-namespace: https://raw.githubusercontent.com/my-org/my-repo/main/Taskfile.yml +``` + +This works exactly the same way that including a local file does. Any tasks in +the remote Taskfile will be available to run from your main Taskfile via the +namespace `my-remote-namespace`. For example, if the remote file contains the +following: + +```yaml +version: '3' + +tasks: + hello: + silent: true + cmds: + - echo "Hello from the remote Taskfile!" +``` + +and you run `task my-remote-namespace:hello`, it will print the text: "Hello +from the remote Taskfile!" to your console. + +## Security + +Running commands from sources that you do not control is always a potential +security risk. For this reason, we have added some checks when using remote +Taskfiles: + +1. When running a task from a remote Taskfile for the first time, Task will + print a warning to the console asking you to check that you are sure that you + trust the source of the Taskfile. If you do not accept the prompt, then Task + will exit with code `104` (not trusted) and nothing will run. If you accept + the prompt, the remote Taskfile will run and further calls to the remote + Taskfile will not prompt you again. +2. Whenever you run a remote Taskfile, Task will create and store a checksum of + the file that you are running. If the checksum changes, then Task will print + another warning to the console to inform you that the contents of the remote + file has changed. If you do not accept the prompt, then Task will exit with + code `104` (not trusted) and nothing will run. If you accept the prompt, the + checksum will be updated and the remote Taskfile will run. + +Task currently supports both `http` and `https` URLs. However, the `http` +requests will not execute by default unless you run the task with the +`--insecure` flag. This is to protect you from accidentally running a remote +Taskfile that is hosted on and unencrypted connection. Sources that are not +protected by TLS are vulnerable to [man-in-the-middle +attacks][man-in-the-middle-attacks] and should be avoided unless you know what +you are doing. + +## Caching & Running Offline + +If for whatever reason, you don't have access to the internet, but you still +need to be able to run your tasks, you are able to use the `--download` flag to +store a cached copy of the remote Taskfile. + + + +If Task detects that you have a local copy of the remote Taskfile, it will use +your local copy instead of downloading the remote file. You can force Task to +work offline by using the `--offline` flag. This will prevent Task from making +any calls to remote sources. + + +[remote-taskfiles-experiment]: https://github.com/go-task/task/issues/1317 +[man-in-the-middle-attacks]: https://en.wikipedia.org/wiki/Man-in-the-middle_attack + diff --git a/errors/errors.go b/errors/errors.go index 78dd72fbdf..694e0c5fd3 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -13,6 +13,10 @@ const ( CodeTaskfileNotFound int = iota + 100 CodeTaskfileAlreadyExists CodeTaskfileInvalid + CodeTaskfileFetchFailed + CodeTaskfileNotTrusted + CodeTaskfileNotSecure + CodeTaskfileCacheNotFound ) // Task related exit codes @@ -40,3 +44,13 @@ type TaskError interface { func New(text string) error { return errors.New(text) } + +// Is wraps the standard errors.Is function so that we don't need to alias that package. +func Is(err, target error) bool { + return errors.Is(err, target) +} + +// As wraps the standard errors.As function so that we don't need to alias that package. +func As(err error, target any) bool { + return errors.As(err, target) +} diff --git a/errors/errors_taskfile.go b/errors/errors_taskfile.go index 3c942977bd..3860153907 100644 --- a/errors/errors_taskfile.go +++ b/errors/errors_taskfile.go @@ -2,6 +2,7 @@ package errors import ( "fmt" + "net/http" ) // TaskfileNotFoundError is returned when no appropriate Taskfile is found when @@ -16,7 +17,7 @@ func (err TaskfileNotFoundError) Error() string { if err.Walk { walkText = " (or any of the parent directories)" } - return fmt.Sprintf(`task: No Taskfile found at "%s"%s`, err.URI, walkText) + return fmt.Sprintf(`task: No Taskfile found at %q%s`, err.URI, walkText) } func (err TaskfileNotFoundError) Code() int { @@ -49,3 +50,73 @@ func (err TaskfileInvalidError) Error() string { func (err TaskfileInvalidError) Code() int { return CodeTaskfileInvalid } + +// TaskfileFetchFailedError is returned when no appropriate Taskfile is found when +// searching the filesystem. +type TaskfileFetchFailedError struct { + URI string + HTTPStatusCode int +} + +func (err TaskfileFetchFailedError) Error() string { + var statusText string + if err.HTTPStatusCode != 0 { + statusText = fmt.Sprintf(" with status code %d (%s)", err.HTTPStatusCode, http.StatusText(err.HTTPStatusCode)) + } + return fmt.Sprintf(`task: Download of %q failed%s`, err.URI, statusText) +} + +func (err TaskfileFetchFailedError) Code() int { + return CodeTaskfileFetchFailed +} + +// TaskfileNotTrustedError is returned when the user does not accept the trust +// prompt when downloading a remote Taskfile. +type TaskfileNotTrustedError struct { + URI string +} + +func (err *TaskfileNotTrustedError) Error() string { + return fmt.Sprintf( + `task: Taskfile %q not trusted by user`, + err.URI, + ) +} + +func (err *TaskfileNotTrustedError) Code() int { + return CodeTaskfileNotTrusted +} + +// TaskfileNotSecureError is returned when the user attempts to download a +// remote Taskfile over an insecure connection. +type TaskfileNotSecureError struct { + URI string +} + +func (err *TaskfileNotSecureError) Error() string { + return fmt.Sprintf( + `task: Taskfile %q cannot be downloaded over an insecure connection. You can override this by using the --insecure flag`, + err.URI, + ) +} + +func (err *TaskfileNotSecureError) Code() int { + return CodeTaskfileNotSecure +} + +// TaskfileCacheNotFound is returned when the user attempts to use an offline +// (cached) Taskfile but the files does not exist in the cache. +type TaskfileCacheNotFound struct { + URI string +} + +func (err *TaskfileCacheNotFound) Error() string { + return fmt.Sprintf( + `task: Taskfile %q was not found in the cache. Remove the --offline flag to use a remote copy or download it using the --download flag`, + err.URI, + ) +} + +func (err *TaskfileCacheNotFound) Code() int { + return CodeTaskfileCacheNotFound +} diff --git a/internal/experiments/experiments.go b/internal/experiments/experiments.go index b444a2af10..4a1e0ab5e0 100644 --- a/internal/experiments/experiments.go +++ b/internal/experiments/experiments.go @@ -2,6 +2,7 @@ package experiments import ( "fmt" + "io" "os" "strings" "text/tabwriter" @@ -13,11 +14,16 @@ import ( const envPrefix = "TASK_X_" -var GentleForce bool +// A list of experiments. +var ( + GentleForce bool + RemoteTaskfiles bool +) func init() { readDotEnv() GentleForce = parseEnv("GENTLE_FORCE") + RemoteTaskfiles = parseEnv("REMOTE_TASKFILES") } func parseEnv(xName string) bool { @@ -35,10 +41,15 @@ func readDotEnv() { } } -func List(l *logger.Logger) error { - w := tabwriter.NewWriter(os.Stdout, 0, 8, 6, ' ', 0) +func printExperiment(w io.Writer, l *logger.Logger, name string, value bool) { l.FOutf(w, logger.Yellow, "* ") - l.FOutf(w, logger.Green, "GENTLE_FORCE") - l.FOutf(w, logger.Default, ": \t%t\n", GentleForce) + l.FOutf(w, logger.Green, name) + l.FOutf(w, logger.Default, ": \t%t\n", value) +} + +func List(l *logger.Logger) error { + w := tabwriter.NewWriter(os.Stdout, 0, 8, 0, ' ', 0) + printExperiment(w, l, "GENTLE_FORCE", GentleForce) + printExperiment(w, l, "REMOTE_TASKFILES", RemoteTaskfiles) return w.Flush() } diff --git a/internal/logger/logger.go b/internal/logger/logger.go index 8eeac5a17a..ab031a46c4 100644 --- a/internal/logger/logger.go +++ b/internal/logger/logger.go @@ -1,11 +1,14 @@ package logger import ( + "bufio" "io" "os" "strconv" + "strings" "github.com/fatih/color" + "golang.org/x/exp/slices" ) type ( @@ -104,3 +107,17 @@ func (l *Logger) VerboseErrf(color Color, s string, args ...any) { l.Errf(color, s, args...) } } + +func (l *Logger) Prompt(color Color, s string, defaultValue string, continueValues ...string) (bool, error) { + if len(continueValues) == 0 { + return false, nil + } + l.Outf(color, "%s [%s/%s]\n", s, strings.ToLower(continueValues[0]), strings.ToUpper(defaultValue)) + reader := bufio.NewReader(os.Stdin) + input, err := reader.ReadString('\n') + if err != nil { + return false, err + } + input = strings.TrimSpace(strings.ToLower(input)) + return slices.Contains(continueValues, input), nil +} diff --git a/setup.go b/setup.go index 2388039507..726e256731 100644 --- a/setup.go +++ b/setup.go @@ -23,21 +23,18 @@ import ( ) func (e *Executor) Setup() error { + e.setupLogger() if err := e.setCurrentDir(); err != nil { return err } - - if err := e.readTaskfile(); err != nil { + if err := e.setupTempDir(); err != nil { return err } - - e.setupFuzzyModel() - - if err := e.setupTempDir(); err != nil { + if err := e.readTaskfile(); err != nil { return err } + e.setupFuzzyModel() e.setupStdFiles() - e.setupLogger() if err := e.setupOutput(); err != nil { return err } @@ -75,15 +72,22 @@ func (e *Executor) setCurrentDir() error { } func (e *Executor) readTaskfile() error { - var err error - e.Taskfile, err = read.Taskfile(&read.FileNode{ - Dir: e.Dir, - Entrypoint: e.Entrypoint, - }) + uri := filepath.Join(e.Dir, e.Entrypoint) + node, err := read.NewNode(uri, e.Insecure) + if err != nil { + return err + } + e.Taskfile, err = read.Taskfile( + node, + e.Insecure, + e.Download, + e.Offline, + e.TempDir, + e.Logger, + ) if err != nil { return err } - e.Dir = filepath.Dir(e.Taskfile.Location) return nil } diff --git a/task.go b/task.go index dd540a41dc..b02eceb303 100644 --- a/task.go +++ b/task.go @@ -51,6 +51,9 @@ type Executor struct { Entrypoint string Force bool ForceAll bool + Insecure bool + Download bool + Offline bool Watch bool Verbose bool Silent bool diff --git a/task_test.go b/task_test.go index a589629c96..6bdc078d28 100644 --- a/task_test.go +++ b/task_test.go @@ -676,6 +676,7 @@ func TestPromptInSummary(t *testing.T) { t.Run(test.name, func(t *testing.T) { var inBuff bytes.Buffer var outBuff bytes.Buffer + var errBuff bytes.Buffer inBuff.Write([]byte(test.input)) @@ -683,6 +684,7 @@ func TestPromptInSummary(t *testing.T) { Dir: dir, Stdin: &inBuff, Stdout: &outBuff, + Stderr: &errBuff, AssumesTerm: true, } require.NoError(t, e.Setup()) @@ -702,6 +704,7 @@ func TestPromptWithIndirectTask(t *testing.T) { const dir = "testdata/prompt" var inBuff bytes.Buffer var outBuff bytes.Buffer + var errBuff bytes.Buffer inBuff.Write([]byte("y\n")) @@ -709,6 +712,7 @@ func TestPromptWithIndirectTask(t *testing.T) { Dir: dir, Stdin: &inBuff, Stdout: &outBuff, + Stderr: &errBuff, AssumesTerm: true, } require.NoError(t, e.Setup()) @@ -732,6 +736,7 @@ func TestPromptAssumeYes(t *testing.T) { t.Run(test.name, func(t *testing.T) { var inBuff bytes.Buffer var outBuff bytes.Buffer + var errBuff bytes.Buffer // always cancel the prompt so we can require.Error inBuff.Write([]byte("\n")) @@ -740,6 +745,7 @@ func TestPromptAssumeYes(t *testing.T) { Dir: dir, Stdin: &inBuff, Stdout: &outBuff, + Stderr: &errBuff, AssumeYes: test.assumeYes, } require.NoError(t, e.Setup()) diff --git a/taskfile/included_taskfile.go b/taskfile/included_taskfile.go index c409052e20..3c204fc803 100644 --- a/taskfile/included_taskfile.go +++ b/taskfile/included_taskfile.go @@ -3,6 +3,7 @@ package taskfile import ( "fmt" "path/filepath" + "strings" "github.com/go-task/task/v3/internal/execext" "github.com/go-task/task/v3/internal/filepathext" @@ -148,6 +149,11 @@ func (it *IncludedTaskfile) FullDirPath() (string, error) { } func (it *IncludedTaskfile) resolvePath(path string) (string, error) { + // If the file is remote, we don't need to resolve the path + if strings.Contains(it.Taskfile, "://") { + return path, nil + } + path, err := execext.Expand(path) if err != nil { return "", err diff --git a/taskfile/read/cache.go b/taskfile/read/cache.go new file mode 100644 index 0000000000..3cc16a17ba --- /dev/null +++ b/taskfile/read/cache.go @@ -0,0 +1,58 @@ +package read + +import ( + "crypto/sha256" + "fmt" + "os" + "path/filepath" + "strings" +) + +type Cache struct { + dir string +} + +func NewCache(dir string) (*Cache, error) { + dir = filepath.Join(dir, "remote") + if err := os.MkdirAll(dir, 0o755); err != nil { + return nil, err + } + return &Cache{ + dir: dir, + }, nil +} + +func checksum(b []byte) string { + h := sha256.New() + h.Write(b) + return fmt.Sprintf("%x", h.Sum(nil)) +} + +func (c *Cache) write(node Node, b []byte) error { + return os.WriteFile(c.cacheFilePath(node), b, 0o644) +} + +func (c *Cache) read(node Node) ([]byte, error) { + return os.ReadFile(c.cacheFilePath(node)) +} + +func (c *Cache) writeChecksum(node Node, checksum string) error { + return os.WriteFile(c.checksumFilePath(node), []byte(checksum), 0o644) +} + +func (c *Cache) readChecksum(node Node) string { + b, _ := os.ReadFile(c.checksumFilePath(node)) + return string(b) +} + +func (c *Cache) key(node Node) string { + return strings.TrimRight(checksum([]byte(node.Location())), "=") +} + +func (c *Cache) cacheFilePath(node Node) string { + return filepath.Join(c.dir, fmt.Sprintf("%s.yaml", c.key(node))) +} + +func (c *Cache) checksumFilePath(node Node) string { + return filepath.Join(c.dir, fmt.Sprintf("%s.checksum", c.key(node))) +} diff --git a/taskfile/read/node.go b/taskfile/read/node.go index c1b828516d..f9cff2c286 100644 --- a/taskfile/read/node.go +++ b/taskfile/read/node.go @@ -1,30 +1,39 @@ package read import ( + "context" "strings" - "github.com/go-task/task/v3/taskfile" + "github.com/go-task/task/v3/errors" + "github.com/go-task/task/v3/internal/experiments" ) type Node interface { - Read() (*taskfile.Taskfile, error) + Read(ctx context.Context) ([]byte, error) Parent() Node - Optional() bool Location() string + Optional() bool + Remote() bool } -func NewNodeFromIncludedTaskfile(parent Node, includedTaskfile taskfile.IncludedTaskfile) (Node, error) { - switch getScheme(includedTaskfile.Taskfile) { - // TODO: Add support for other schemes. - // If no other scheme matches, we assume it's a file. - // This also allows users to explicitly set a file:// scheme. +func NewNode( + uri string, + insecure bool, + opts ...NodeOption, +) (Node, error) { + var node Node + var err error + switch getScheme(uri) { + case "http", "https": + node, err = NewHTTPNode(uri, insecure, opts...) default: - path, err := includedTaskfile.FullTaskfilePath() - if err != nil { - return nil, err - } - return NewFileNode(parent, path, includedTaskfile.Optional) + // If no other scheme matches, we assume it's a file + node, err = NewFileNode(uri, opts...) + } + if node.Remote() && !experiments.RemoteTaskfiles { + return nil, errors.New("task: Remote taskfiles are not enabled. You can read more about this experiment and how to enable it at https://taskfile.dev/experiments/remote-taskfiles") } + return node, err } func getScheme(uri string) string { diff --git a/taskfile/read/node_base.go b/taskfile/read/node_base.go index 5a1a5d64f5..ea1088222e 100644 --- a/taskfile/read/node_base.go +++ b/taskfile/read/node_base.go @@ -1,18 +1,47 @@ package read -// BaseNode is a generic node that implements the Parent() and Optional() -// methods of the NodeReader interface. It does not implement the Read() method -// and it designed to be embedded in other node types so that this boilerplate -// code does not need to be repeated. -type BaseNode struct { - parent Node - optional bool +type ( + NodeOption func(*BaseNode) + // BaseNode is a generic node that implements the Parent() and Optional() + // methods of the NodeReader interface. It does not implement the Read() method + // and it designed to be embedded in other node types so that this boilerplate + // code does not need to be repeated. + BaseNode struct { + parent Node + optional bool + } +) + +func NewBaseNode(opts ...NodeOption) *BaseNode { + node := &BaseNode{ + parent: nil, + optional: false, + } + + // Apply options + for _, opt := range opts { + opt(node) + } + + return node +} + +func WithParent(parent Node) NodeOption { + return func(node *BaseNode) { + node.parent = parent + } } func (node *BaseNode) Parent() Node { return node.parent } +func WithOptional(optional bool) NodeOption { + return func(node *BaseNode) { + node.optional = optional + } +} + func (node *BaseNode) Optional() bool { return node.optional } diff --git a/taskfile/read/node_file.go b/taskfile/read/node_file.go index f4246c571d..5cf25dd72f 100644 --- a/taskfile/read/node_file.go +++ b/taskfile/read/node_file.go @@ -1,34 +1,36 @@ package read import ( + "context" + "io" "os" "path/filepath" - "gopkg.in/yaml.v3" - - "github.com/go-task/task/v3/errors" "github.com/go-task/task/v3/internal/filepathext" - "github.com/go-task/task/v3/taskfile" ) // A FileNode is a node that reads a taskfile from the local filesystem. type FileNode struct { - BaseNode + *BaseNode Dir string Entrypoint string } -func NewFileNode(parent Node, path string, optional bool) (*FileNode, error) { - path, err := exists(path) +func NewFileNode(uri string, opts ...NodeOption) (*FileNode, error) { + base := NewBaseNode(opts...) + if uri == "" { + d, err := os.Getwd() + if err != nil { + return nil, err + } + uri = d + } + path, err := existsWalk(uri) if err != nil { return nil, err } - return &FileNode{ - BaseNode: BaseNode{ - parent: parent, - optional: optional, - }, + BaseNode: base, Dir: filepath.Dir(path), Entrypoint: filepath.Base(path), }, nil @@ -38,33 +40,15 @@ func (node *FileNode) Location() string { return filepathext.SmartJoin(node.Dir, node.Entrypoint) } -func (node *FileNode) Read() (*taskfile.Taskfile, error) { - if node.Dir == "" { - d, err := os.Getwd() - if err != nil { - return nil, err - } - node.Dir = d - } - - path, err := existsWalk(node.Location()) - if err != nil { - return nil, err - } - node.Dir = filepath.Dir(path) - node.Entrypoint = filepath.Base(path) +func (node *FileNode) Remote() bool { + return false +} - f, err := os.Open(path) +func (node *FileNode) Read(ctx context.Context) ([]byte, error) { + f, err := os.Open(node.Location()) if err != nil { return nil, err } defer f.Close() - - var t taskfile.Taskfile - if err := yaml.NewDecoder(f).Decode(&t); err != nil { - return nil, &errors.TaskfileInvalidError{URI: filepathext.TryAbsToRel(path), Err: err} - } - - t.Location = path - return &t, nil + return io.ReadAll(f) } diff --git a/taskfile/read/node_http.go b/taskfile/read/node_http.go new file mode 100644 index 0000000000..e27ff2a8db --- /dev/null +++ b/taskfile/read/node_http.go @@ -0,0 +1,67 @@ +package read + +import ( + "context" + "io" + "net/http" + "net/url" + + "github.com/go-task/task/v3/errors" +) + +// An HTTPNode is a node that reads a Taskfile from a remote location via HTTP. +type HTTPNode struct { + *BaseNode + URL *url.URL +} + +func NewHTTPNode(uri string, insecure bool, opts ...NodeOption) (*HTTPNode, error) { + base := NewBaseNode(opts...) + url, err := url.Parse(uri) + if err != nil { + return nil, err + } + if url.Scheme == "http" && !insecure { + return nil, &errors.TaskfileNotSecureError{URI: uri} + } + return &HTTPNode{ + BaseNode: base, + URL: url, + }, nil +} + +func (node *HTTPNode) Location() string { + return node.URL.String() +} + +func (node *HTTPNode) Remote() bool { + return true +} + +func (node *HTTPNode) Read(ctx context.Context) ([]byte, error) { + req, err := http.NewRequest("GET", node.URL.String(), nil) + if err != nil { + return nil, errors.TaskfileFetchFailedError{URI: node.URL.String()} + } + + resp, err := http.DefaultClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, errors.TaskfileFetchFailedError{URI: node.URL.String()} + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, errors.TaskfileFetchFailedError{ + URI: node.URL.String(), + HTTPStatusCode: resp.StatusCode, + } + } + + // Read the entire response body + b, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + return b, nil +} diff --git a/taskfile/read/taskfile.go b/taskfile/read/taskfile.go index efd48fd2fa..d952b5cf7e 100644 --- a/taskfile/read/taskfile.go +++ b/taskfile/read/taskfile.go @@ -1,13 +1,17 @@ package read import ( + "context" "fmt" "os" "path/filepath" "runtime" + "gopkg.in/yaml.v3" + "github.com/go-task/task/v3/errors" "github.com/go-task/task/v3/internal/filepathext" + "github.com/go-task/task/v3/internal/logger" "github.com/go-task/task/v3/internal/sysinfo" "github.com/go-task/task/v3/internal/templater" "github.com/go-task/task/v3/taskfile" @@ -29,13 +33,112 @@ var ( } ) +func readTaskfile( + node Node, + download, + offline bool, + tempDir string, + l *logger.Logger, +) (*taskfile.Taskfile, error) { + var b []byte + + cache, err := NewCache(tempDir) + if err != nil { + return nil, err + } + + // If the file is remote, check if we have a cached copy + // If we're told to download, skip the cache + if node.Remote() && !download { + if b, err = cache.read(node); !errors.Is(err, os.ErrNotExist) && err != nil { + return nil, err + } + + if b != nil { + l.VerboseOutf(logger.Magenta, "task: [%s] Fetched cached copy\n", node.Location()) + } + } + + // If the file is remote, we found nothing in the cache and we're not + // allowed to download it then we can't do anything. + if node.Remote() && b == nil && offline { + if b == nil && offline { + return nil, &errors.TaskfileCacheNotFound{URI: node.Location()} + } + } + + // If we still don't have a copy, get the file in the usual way + if b == nil { + b, err = node.Read(context.Background()) + if err != nil { + return nil, err + } + + // If the node was remote, we need to check the checksum + if node.Remote() { + l.VerboseOutf(logger.Magenta, "task: [%s] Fetched remote copy\n", node.Location()) + + // Get the checksums + checksum := checksum(b) + cachedChecksum := cache.readChecksum(node) + + // If the checksum doesn't exist, prompt the user to continue + if cachedChecksum == "" { + if cont, err := l.Prompt(logger.Yellow, fmt.Sprintf("The task you are attempting to run depends on the remote Taskfile at %q.\n--- Make sure you trust the source of this Taskfile before continuing ---\nContinue?", node.Location()), "n", "y", "yes"); err != nil { + return nil, err + } else if !cont { + return nil, &errors.TaskfileNotTrustedError{URI: node.Location()} + } + } else if checksum != cachedChecksum { + // If there is a cached hash, but it doesn't match the expected hash, prompt the user to continue + if cont, err := l.Prompt(logger.Yellow, fmt.Sprintf("The Taskfile at %q has changed since you last used it!\n--- Make sure you trust the source of this Taskfile before continuing ---\nContinue?", node.Location()), "n", "y", "yes"); err != nil { + return nil, err + } else if !cont { + return nil, &errors.TaskfileNotTrustedError{URI: node.Location()} + } + } + + // If the hash has changed (or is new), store it in the cache + if checksum != cachedChecksum { + if err := cache.writeChecksum(node, checksum); err != nil { + return nil, err + } + } + } + } + + // If the file is remote and we need to cache it + if node.Remote() && download { + l.VerboseOutf(logger.Magenta, "task: [%s] Caching downloaded file\n", node.Location()) + // Cache the file for later + if err = cache.write(node, b); err != nil { + return nil, err + } + } + + var t taskfile.Taskfile + if err := yaml.Unmarshal(b, &t); err != nil { + return nil, &errors.TaskfileInvalidError{URI: filepathext.TryAbsToRel(node.Location()), Err: err} + } + t.Location = node.Location() + + return &t, nil +} + // Taskfile reads a Taskfile for a given directory // Uses current dir when dir is left empty. Uses Taskfile.yml // or Taskfile.yaml when entrypoint is left empty -func Taskfile(node Node) (*taskfile.Taskfile, error) { +func Taskfile( + node Node, + insecure bool, + download bool, + offline bool, + tempDir string, + l *logger.Logger, +) (*taskfile.Taskfile, error) { var _taskfile func(Node) (*taskfile.Taskfile, error) _taskfile = func(node Node) (*taskfile.Taskfile, error) { - t, err := node.Read() + t, err := readTaskfile(node, download, offline, tempDir, l) if err != nil { return nil, err } @@ -70,7 +173,15 @@ func Taskfile(node Node) (*taskfile.Taskfile, error) { } } - includeReaderNode, err := NewNodeFromIncludedTaskfile(node, includedTask) + uri, err := includedTask.FullTaskfilePath() + if err != nil { + return err + } + + includeReaderNode, err := NewNode(uri, insecure, + WithParent(node), + WithOptional(includedTask.Optional), + ) if err != nil { if includedTask.Optional { return nil @@ -149,17 +260,19 @@ func Taskfile(node Node) (*taskfile.Taskfile, error) { path := filepathext.SmartJoin(node.Dir, fmt.Sprintf("Taskfile_%s.yml", runtime.GOOS)) if _, err = os.Stat(path); err == nil { osNode := &FileNode{ - BaseNode: BaseNode{ - parent: node, - optional: false, - }, + BaseNode: NewBaseNode(WithParent(node)), Entrypoint: path, Dir: node.Dir, } - osTaskfile, err := osNode.Read() + b, err := osNode.Read(context.Background()) if err != nil { return nil, err } + var osTaskfile *taskfile.Taskfile + if err := yaml.Unmarshal(b, &osTaskfile); err != nil { + return nil, &errors.TaskfileInvalidError{URI: filepathext.TryAbsToRel(node.Location()), Err: err} + } + t.Location = node.Location() if err = taskfile.Merge(t, osTaskfile, nil); err != nil { return nil, err } @@ -183,6 +296,11 @@ func Taskfile(node Node) (*taskfile.Taskfile, error) { return _taskfile(node) } +// exists will check if a file at the given path exists. If it does, it will +// return the path to it. If it does not, it will search the search for any +// files at the given path with any of the default Taskfile files names. If any +// of these match a file, the first matching path will be returned. If no files +// are found, an error will be returned. func exists(path string) (string, error) { fi, err := os.Stat(path) if err != nil { @@ -202,6 +320,11 @@ func exists(path string) (string, error) { return "", errors.TaskfileNotFoundError{URI: path, Walk: false} } +// existsWalk will check if a file at the given path exists by calling the +// exists function. If a file is not found, it will walk up the directory tree +// calling the exists function until it finds a file or reaches the root +// directory. On supported operating systems, it will also check if the user ID +// of the directory changes and abort if it does. func existsWalk(path string) (string, error) { origPath := path owner, err := sysinfo.Owner(path)