From 79ec7c248c83f48302c9646ff41d1ace312584e4 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Wed, 15 Nov 2023 02:50:53 -0800 Subject: [PATCH] feat: check and cache the task definition, variables, status This adds a new checker for Task definitions, status, and any other variable that affect a task. The implementation hashes the task structure and writes it to `.task/definition`. In later runs, this is checked to make sure the definition is up-to-date. If the definition has changed (e.g. one of the variables has changed), the task will run again. Fixes #548 Fixes #455 --- go.mod | 1 + go.sum | 2 + internal/fingerprint/checker.go | 7 +++ internal/fingerprint/definition.go | 85 ++++++++++++++++++++++++++++++ internal/fingerprint/task.go | 73 ++++++++++++++++++------- taskfile/var.go | 8 +++ 6 files changed, 158 insertions(+), 18 deletions(-) create mode 100644 internal/fingerprint/definition.go diff --git a/go.mod b/go.mod index df17e82673..1665f34f4b 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mitchellh/hashstructure v1.1.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/stretchr/objx v0.5.0 // indirect golang.org/x/sys v0.14.0 // indirect diff --git a/go.sum b/go.sum index 72f19e8246..ea609caca1 100644 --- a/go.sum +++ b/go.sum @@ -23,6 +23,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-zglob v0.0.4 h1:LQi2iOm0/fGgu80AioIJ/1j9w9Oh+9DZ39J4VAGzHQM= github.com/mattn/go-zglob v0.0.4/go.mod h1:MxxjyoXXnMxfIpxTK2GAkw1w8glPsQILx3N5wrKakiY= +github.com/mitchellh/hashstructure v1.1.0 h1:P6P1hdjqAAknpY/M1CGipelZgp+4y9ja9kmUZPXP+H0= +github.com/mitchellh/hashstructure v1.1.0/go.mod h1:xUDAozZz0Wmdiufv0uyhnHkUTN6/6d8ulp4AwfLKrmA= github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4= github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/internal/fingerprint/checker.go b/internal/fingerprint/checker.go index 4cdc7e8da2..171768adae 100644 --- a/internal/fingerprint/checker.go +++ b/internal/fingerprint/checker.go @@ -18,3 +18,10 @@ type SourcesCheckable interface { OnError(t *taskfile.Task) error Kind() string } + +// DefinitionCheckable defines any type that checks the definition of a task. +type DefinitionCheckable interface { + HashDefinition(t *taskfile.Task) (*string, error) + IsUpToDate(maybeDefinitionPath *string) (bool, error) + Cleanup(definitionPath *string) error +} diff --git a/internal/fingerprint/definition.go b/internal/fingerprint/definition.go new file mode 100644 index 0000000000..6a6fd40518 --- /dev/null +++ b/internal/fingerprint/definition.go @@ -0,0 +1,85 @@ +package fingerprint + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/go-task/task/v3/internal/logger" + "github.com/go-task/task/v3/taskfile" + "github.com/mitchellh/hashstructure/v2" +) + +// DefinitionChecker checks if the task definition and any of its variables/environment variables change. +type DefinitionChecker struct { + tempDir string + dry bool + logger *logger.Logger +} + +func NewDefinitionChecker(tempDir string, dry bool, logger *logger.Logger) *DefinitionChecker { + return &DefinitionChecker{ + tempDir: tempDir, + dry: dry, + logger: logger, + } +} + +// IsUpToDate returns true if the task definition is up-to-date. +// As the second return value, it returns the path to the task definition file if it exists. +// This file should be cleaned up if the task is not up-to-date by other checkers. +func (checker *DefinitionChecker) IsUpToDate(maybeDefinitionPath *string) (bool, error) { + if maybeDefinitionPath == nil { + return false, fmt.Errorf("task: task definition path is nil") + } + definitionPath := *maybeDefinitionPath + + // check if the file exists + _, err := os.Stat(definitionPath) + if err == nil { + checker.logger.VerboseOutf(logger.Magenta, "task: task definition is up-to-date: %s\n", definitionPath) + // file exists, the task definition is up to + return true, nil + } + + // task is not up-to-date as the file does not exist + // create the hash file if not in dry mode + if !checker.dry { + // create the file + if err := os.MkdirAll(filepath.Dir(definitionPath), 0o755); err != nil { + return false, err + } + _, err = os.Create(definitionPath) + if err != nil { + return false, err + } + checker.logger.VerboseOutf(logger.Yellow, "task: task definition was written as: %s\n", definitionPath) + } + return false, nil +} + +func (checker *DefinitionChecker) HashDefinition(t *taskfile.Task) (*string, error) { + // hash the task + hash, err := hashstructure.Hash(t, hashstructure.FormatV2, nil) + if err != nil { + // failed to hash the task. Consider the task as not up-to-date + return nil, err + } + + // the path to the task definition file with the hash in the filename + hashPath := filepath.Join(checker.tempDir, "definition", normalizeFilename(fmt.Sprintf("%s-%d", t.Name(), hash))) + return &hashPath, nil +} + +// Cleanup removes the task definition file if it exists. +func (checker *DefinitionChecker) Cleanup(definitionPath *string) error { + if definitionPath != nil { + // if the file exists, remove it + if _, err := os.Stat(*definitionPath); err == nil { + if err := os.Remove(*definitionPath); err != nil { + return err + } + } + } + return nil +} diff --git a/internal/fingerprint/task.go b/internal/fingerprint/task.go index 6156c3860b..8c0e75ea63 100644 --- a/internal/fingerprint/task.go +++ b/internal/fingerprint/task.go @@ -10,12 +10,13 @@ import ( type ( CheckerOption func(*CheckerConfig) CheckerConfig struct { - method string - dry bool - tempDir string - logger *logger.Logger - statusChecker StatusCheckable - sourcesChecker SourcesCheckable + method string + dry bool + tempDir string + logger *logger.Logger + definitionChecker DefinitionCheckable + statusChecker StatusCheckable + sourcesChecker SourcesCheckable } ) @@ -66,12 +67,13 @@ func IsTaskUpToDate( // Default config config := &CheckerConfig{ - method: "none", - tempDir: "", - dry: false, - logger: nil, - statusChecker: nil, - sourcesChecker: nil, + method: "none", + tempDir: "", + dry: false, + logger: nil, + definitionChecker: nil, + statusChecker: nil, + sourcesChecker: nil, } // Apply functional options @@ -92,9 +94,34 @@ func IsTaskUpToDate( } } + // if no definition checker was given, set up the default one + if config.definitionChecker == nil { + config.definitionChecker = NewDefinitionChecker(config.tempDir, config.dry, config.logger) + } + statusIsSet := len(t.Status) != 0 sourcesIsSet := len(t.Sources) != 0 + // hash the task definition + maybeDefinitionPath, err := config.definitionChecker.HashDefinition(t) + if err != nil { + return false, err + } + + // if the status or sources are set, check if the definition is up-to-date + // TODO: allow caching based on the task definition even if status or sources are not set + if sourcesIsSet || statusIsSet { + // if other conditions are , check if the definition is up-to-date + isDefinitionUpToDate, err := config.definitionChecker.IsUpToDate(maybeDefinitionPath) + if err != nil { + return false, err + } + // defintion is not up-to-date, early return + if !isDefinitionUpToDate { + return false, nil + } + } + // If status is set, check if it is up-to-date if statusIsSet { statusUpToDate, err = config.statusChecker.IsUpToDate(ctx, t) @@ -111,22 +138,32 @@ func IsTaskUpToDate( } } + // If no status or sources are set, the task should always run + // i.e. it is never considered "up-to-date" + var isUpToDate = false + // If both status and sources are set, the task is up-to-date if both are up-to-date if statusIsSet && sourcesIsSet { - return statusUpToDate && sourcesUpToDate, nil + isUpToDate = statusUpToDate && sourcesUpToDate } // If only status is set, the task is up-to-date if the status is up-to-date if statusIsSet { - return statusUpToDate, nil + isUpToDate = statusUpToDate } // If only sources is set, the task is up-to-date if the sources are up-to-date if sourcesIsSet { - return sourcesUpToDate, nil + isUpToDate = sourcesUpToDate } - // If no status or sources are set, the task should always run - // i.e. it is never considered "up-to-date" - return false, nil + if !isUpToDate { + // if the task is not up-to-date for any reason, remove the definition file from previous runs if it exists + err = config.definitionChecker.Cleanup(maybeDefinitionPath) + if err != nil { + return false, err + } + } + + return isUpToDate, nil } diff --git a/taskfile/var.go b/taskfile/var.go index ba1c2d2627..156bd47cf0 100644 --- a/taskfile/var.go +++ b/taskfile/var.go @@ -6,6 +6,7 @@ import ( "gopkg.in/yaml.v3" "github.com/go-task/task/v3/internal/orderedmap" + "github.com/mitchellh/hashstructure/v2" ) // Vars is a string[string] variables map. @@ -69,6 +70,13 @@ func (vs *Vars) DeepCopy() *Vars { } } +func (vs *Vars) Hash() (uint64, error) { + if vs == nil { + return hashstructure.Hash([]Var{}, hashstructure.FormatV2, nil) + } + return hashstructure.Hash(vs.Values(), hashstructure.FormatV2, nil) +} + // Var represents either a static or dynamic variable. type Var struct { Static string