Skip to content

Commit

Permalink
feat: check and cache the task definition, variables, status
Browse files Browse the repository at this point in the history
This adds a new checker for Task definitions, status, and any other variable that affect a task.

The implementation hashes the task structure and writes it to `.task/definition`. In later runs, this is checked to make sure the definition is up-to-date. If the definition has changed (e.g. one of the variables has changed), the task will run again.

Fixes go-task#548
Fixes go-task#455
  • Loading branch information
aminya committed Nov 16, 2023
1 parent 834babe commit 79ec7c2
Show file tree
Hide file tree
Showing 6 changed files with 158 additions and 18 deletions.
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ require (
github.com/klauspost/cpuid/v2 v2.0.9 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mitchellh/hashstructure v1.1.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/stretchr/objx v0.5.0 // indirect
golang.org/x/sys v0.14.0 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-zglob v0.0.4 h1:LQi2iOm0/fGgu80AioIJ/1j9w9Oh+9DZ39J4VAGzHQM=
github.com/mattn/go-zglob v0.0.4/go.mod h1:MxxjyoXXnMxfIpxTK2GAkw1w8glPsQILx3N5wrKakiY=
github.com/mitchellh/hashstructure v1.1.0 h1:P6P1hdjqAAknpY/M1CGipelZgp+4y9ja9kmUZPXP+H0=
github.com/mitchellh/hashstructure v1.1.0/go.mod h1:xUDAozZz0Wmdiufv0uyhnHkUTN6/6d8ulp4AwfLKrmA=
github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4=
github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
Expand Down
7 changes: 7 additions & 0 deletions internal/fingerprint/checker.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,3 +18,10 @@ type SourcesCheckable interface {
OnError(t *taskfile.Task) error
Kind() string
}

// DefinitionCheckable defines any type that checks the definition of a task.
type DefinitionCheckable interface {
HashDefinition(t *taskfile.Task) (*string, error)
IsUpToDate(maybeDefinitionPath *string) (bool, error)
Cleanup(definitionPath *string) error
}
85 changes: 85 additions & 0 deletions internal/fingerprint/definition.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
package fingerprint

import (
"fmt"
"os"
"path/filepath"

"github.com/go-task/task/v3/internal/logger"
"github.com/go-task/task/v3/taskfile"
"github.com/mitchellh/hashstructure/v2"
)

// DefinitionChecker checks if the task definition and any of its variables/environment variables change.
type DefinitionChecker struct {
tempDir string
dry bool
logger *logger.Logger
}

func NewDefinitionChecker(tempDir string, dry bool, logger *logger.Logger) *DefinitionChecker {
return &DefinitionChecker{
tempDir: tempDir,
dry: dry,
logger: logger,
}
}

// IsUpToDate returns true if the task definition is up-to-date.
// As the second return value, it returns the path to the task definition file if it exists.
// This file should be cleaned up if the task is not up-to-date by other checkers.
func (checker *DefinitionChecker) IsUpToDate(maybeDefinitionPath *string) (bool, error) {
if maybeDefinitionPath == nil {
return false, fmt.Errorf("task: task definition path is nil")
}
definitionPath := *maybeDefinitionPath

// check if the file exists
_, err := os.Stat(definitionPath)
if err == nil {
checker.logger.VerboseOutf(logger.Magenta, "task: task definition is up-to-date: %s\n", definitionPath)
// file exists, the task definition is up to
return true, nil
}

// task is not up-to-date as the file does not exist
// create the hash file if not in dry mode
if !checker.dry {
// create the file
if err := os.MkdirAll(filepath.Dir(definitionPath), 0o755); err != nil {
return false, err
}
_, err = os.Create(definitionPath)
if err != nil {
return false, err
}
checker.logger.VerboseOutf(logger.Yellow, "task: task definition was written as: %s\n", definitionPath)
}
return false, nil
}

func (checker *DefinitionChecker) HashDefinition(t *taskfile.Task) (*string, error) {
// hash the task
hash, err := hashstructure.Hash(t, hashstructure.FormatV2, nil)
if err != nil {
// failed to hash the task. Consider the task as not up-to-date
return nil, err
}

// the path to the task definition file with the hash in the filename
hashPath := filepath.Join(checker.tempDir, "definition", normalizeFilename(fmt.Sprintf("%s-%d", t.Name(), hash)))
return &hashPath, nil
}

// Cleanup removes the task definition file if it exists.
func (checker *DefinitionChecker) Cleanup(definitionPath *string) error {
if definitionPath != nil {
// if the file exists, remove it
if _, err := os.Stat(*definitionPath); err == nil {
if err := os.Remove(*definitionPath); err != nil {
return err
}
}
}
return nil
}
73 changes: 55 additions & 18 deletions internal/fingerprint/task.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,13 @@ import (
type (
CheckerOption func(*CheckerConfig)
CheckerConfig struct {
method string
dry bool
tempDir string
logger *logger.Logger
statusChecker StatusCheckable
sourcesChecker SourcesCheckable
method string
dry bool
tempDir string
logger *logger.Logger
definitionChecker DefinitionCheckable
statusChecker StatusCheckable
sourcesChecker SourcesCheckable
}
)

Expand Down Expand Up @@ -66,12 +67,13 @@ func IsTaskUpToDate(

// Default config
config := &CheckerConfig{
method: "none",
tempDir: "",
dry: false,
logger: nil,
statusChecker: nil,
sourcesChecker: nil,
method: "none",
tempDir: "",
dry: false,
logger: nil,
definitionChecker: nil,
statusChecker: nil,
sourcesChecker: nil,
}

// Apply functional options
Expand All @@ -92,9 +94,34 @@ func IsTaskUpToDate(
}
}

// if no definition checker was given, set up the default one
if config.definitionChecker == nil {
config.definitionChecker = NewDefinitionChecker(config.tempDir, config.dry, config.logger)
}

statusIsSet := len(t.Status) != 0
sourcesIsSet := len(t.Sources) != 0

// hash the task definition
maybeDefinitionPath, err := config.definitionChecker.HashDefinition(t)
if err != nil {
return false, err
}

// if the status or sources are set, check if the definition is up-to-date
// TODO: allow caching based on the task definition even if status or sources are not set
if sourcesIsSet || statusIsSet {
// if other conditions are , check if the definition is up-to-date
isDefinitionUpToDate, err := config.definitionChecker.IsUpToDate(maybeDefinitionPath)
if err != nil {
return false, err
}
// defintion is not up-to-date, early return
if !isDefinitionUpToDate {
return false, nil
}
}

// If status is set, check if it is up-to-date
if statusIsSet {
statusUpToDate, err = config.statusChecker.IsUpToDate(ctx, t)
Expand All @@ -111,22 +138,32 @@ func IsTaskUpToDate(
}
}

// If no status or sources are set, the task should always run
// i.e. it is never considered "up-to-date"
var isUpToDate = false

// If both status and sources are set, the task is up-to-date if both are up-to-date
if statusIsSet && sourcesIsSet {
return statusUpToDate && sourcesUpToDate, nil
isUpToDate = statusUpToDate && sourcesUpToDate
}

// If only status is set, the task is up-to-date if the status is up-to-date
if statusIsSet {
return statusUpToDate, nil
isUpToDate = statusUpToDate
}

// If only sources is set, the task is up-to-date if the sources are up-to-date
if sourcesIsSet {
return sourcesUpToDate, nil
isUpToDate = sourcesUpToDate
}

// If no status or sources are set, the task should always run
// i.e. it is never considered "up-to-date"
return false, nil
if !isUpToDate {
// if the task is not up-to-date for any reason, remove the definition file from previous runs if it exists
err = config.definitionChecker.Cleanup(maybeDefinitionPath)
if err != nil {
return false, err
}
}

return isUpToDate, nil
}
8 changes: 8 additions & 0 deletions taskfile/var.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"gopkg.in/yaml.v3"

"github.com/go-task/task/v3/internal/orderedmap"
"github.com/mitchellh/hashstructure/v2"
)

// Vars is a string[string] variables map.
Expand Down Expand Up @@ -69,6 +70,13 @@ func (vs *Vars) DeepCopy() *Vars {
}
}

func (vs *Vars) Hash() (uint64, error) {
if vs == nil {
return hashstructure.Hash([]Var{}, hashstructure.FormatV2, nil)
}
return hashstructure.Hash(vs.Values(), hashstructure.FormatV2, nil)
}

// Var represents either a static or dynamic variable.
type Var struct {
Static string
Expand Down

0 comments on commit 79ec7c2

Please sign in to comment.