Skip to content

Commit

Permalink
Implement checksum validation for generates (alternative design)
Browse files Browse the repository at this point in the history
Avoid writing any state in IsUpToDate, instead return the current source
state and also check after the task is run that the sources checksum has not
been modified
  • Loading branch information
vanackere committed Sep 27, 2024
1 parent ab2dbba commit 580492b
Show file tree
Hide file tree
Showing 9 changed files with 110 additions and 104 deletions.
2 changes: 1 addition & 1 deletion help.go
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ func (e *Executor) ToEditorOutput(tasks []*ast.Task, noStatus bool) (*editors.Ta
if tasks[i].Method != "" {
method = tasks[i].Method
}
upToDate, err := fingerprint.IsTaskUpToDate(context.Background(), tasks[i],
upToDate, _, err := fingerprint.IsTaskUpToDate(context.Background(), tasks[i],
fingerprint.WithMethod(method),
fingerprint.WithTempDir(e.TempDir.Fingerprint),
fingerprint.WithDry(e.Dry),
Expand Down
4 changes: 2 additions & 2 deletions internal/fingerprint/checker.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ type StatusCheckable interface {

// SourcesCheckable defines any type that can check if the sources of a task are up-to-date.
type SourcesCheckable interface {
SetUpToDate(t *ast.Task) error
IsUpToDate(t *ast.Task) (bool, error)
SetUpToDate(t *ast.Task, sourceState string) error
IsUpToDate(t *ast.Task) (upToDate bool, sourceState string, err error)
Value(t *ast.Task) (any, error)
OnError(t *ast.Task) error
Kind() string
Expand Down
44 changes: 18 additions & 26 deletions internal/fingerprint/sources_checksum.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,9 @@ func NewChecksumChecker(tempDir string, dry bool) *ChecksumChecker {
}
}

func (checker *ChecksumChecker) IsUpToDate(t *ast.Task) (bool, error) {
func (checker *ChecksumChecker) IsUpToDate(t *ast.Task) (bool, string, error) {
if len(t.Sources) == 0 && len(t.Generates) == 0 {
return false, nil
return false, "", nil
}

checksumFile := checker.checksumFilePath(t)
Expand All @@ -41,27 +41,18 @@ func (checker *ChecksumChecker) IsUpToDate(t *ast.Task) (bool, error) {

newSourcesHash, err := checker.checksum(t, t.Sources)
if err != nil {
return false, err
return false, "", err
}

newGeneratesHash, err := checker.checksum(t, t.Generates)
if err != nil {
return false, err
return false, "", err
}

if !checker.dry && oldSourcesHash != newSourcesHash {
// make sure current sources hash are saved to file before executing the task,
// the proper "generated" hash will be saved after the task is executed
_ = os.MkdirAll(filepathext.SmartJoin(checker.tempDir, "checksum"), 0o755)
if err = os.WriteFile(checksumFile, []byte(newSourcesHash+"\n"+"_"), 0o644); err != nil {
return false, err
}
}

return oldSourcesHash == newSourcesHash && oldGeneratesdHash == newGeneratesHash, nil
return oldSourcesHash == newSourcesHash && oldGeneratesdHash == newGeneratesHash, newSourcesHash, nil
}

func (checker *ChecksumChecker) SetUpToDate(t *ast.Task) error {
func (checker *ChecksumChecker) SetUpToDate(t *ast.Task, sourceHash string) error {
if len(t.Sources) == 0 && len(t.Generates) == 0 {
return nil
}
Expand All @@ -70,27 +61,28 @@ func (checker *ChecksumChecker) SetUpToDate(t *ast.Task) error {
return nil
}

checksumFile := checker.checksumFilePath(t)

data, _ := os.ReadFile(checksumFile)
oldHashes := strings.TrimSpace(string(data))
oldSourcesHash, oldGeneratesdHash, _ := strings.Cut(oldHashes, "\n")

newSourcesHash, err := checker.checksum(t, t.Sources)
if err != nil {
return err
}

checksumFile := checker.checksumFilePath(t)

if sourceHash != "" && newSourcesHash != sourceHash {
// sources have changed since the task was executed, remove the checksum file
// since the next execution will have a different checksum
os.Remove(checksumFile)
return nil
}

newGeneratesHash, err := checker.checksum(t, t.Generates)
if err != nil {
return err
}

if oldSourcesHash != newSourcesHash || oldGeneratesdHash != newGeneratesHash {
_ = os.MkdirAll(filepathext.SmartJoin(checker.tempDir, "checksum"), 0o755)
if err = os.WriteFile(checksumFile, []byte(oldSourcesHash+"\n"+newGeneratesHash+"\n"), 0o644); err != nil {
return err
}
_ = os.MkdirAll(filepathext.SmartJoin(checker.tempDir, "checksum"), 0o755)
if err = os.WriteFile(checksumFile, []byte(newSourcesHash+"\n"+newGeneratesHash+"\n"), 0o644); err != nil {
return err
}

return nil
Expand Down
6 changes: 3 additions & 3 deletions internal/fingerprint/sources_none.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,11 @@ import "github.com/go-task/task/v3/taskfile/ast"
// It will always report that the task is not up-to-date.
type NoneChecker struct{}

func (NoneChecker) IsUpToDate(t *ast.Task) (bool, error) {
return false, nil
func (NoneChecker) IsUpToDate(t *ast.Task) (bool, string, error) {
return false, "", nil
}

func (NoneChecker) SetUpToDate(t *ast.Task) error {
func (NoneChecker) SetUpToDate(t *ast.Task, sourceState string) error {
return nil
}

Expand Down
22 changes: 11 additions & 11 deletions internal/fingerprint/sources_timestamp.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,18 +23,18 @@ func NewTimestampChecker(tempDir string, dry bool) *TimestampChecker {
}

// IsUpToDate implements the Checker interface
func (checker *TimestampChecker) IsUpToDate(t *ast.Task) (bool, error) {
func (checker *TimestampChecker) IsUpToDate(t *ast.Task) (bool, string, error) {
if len(t.Sources) == 0 {
return false, nil
return false, "", nil
}

sources, err := Globs(t.Dir, t.Sources)
if err != nil {
return false, nil
return false, "", nil
}
generates, err := Globs(t.Dir, t.Generates)
if err != nil {
return false, nil
return false, "", nil
}

timestampFile := checker.timestampFilePath(t)
Expand All @@ -48,11 +48,11 @@ func (checker *TimestampChecker) IsUpToDate(t *ast.Task) (bool, error) {
// Create the timestamp file for the next execution when the file does not exist.
if !checker.dry {
if err := os.MkdirAll(filepath.Dir(timestampFile), 0o755); err != nil {
return false, err
return false, "", err
}
f, err := os.Create(timestampFile)
if err != nil {
return false, err
return false, "", err
}
f.Close()
}
Expand All @@ -65,26 +65,26 @@ func (checker *TimestampChecker) IsUpToDate(t *ast.Task) (bool, error) {
// Get the max time of the generates.
generateMaxTime, err := getMaxTime(generates...)
if err != nil || generateMaxTime.IsZero() {
return false, nil
return false, "", nil
}

// Check if any of the source files is newer than the max time of the generates.
shouldUpdate, err := anyFileNewerThan(sources, generateMaxTime)
if err != nil {
return false, nil
return false, "", nil
}

// Modify the metadata of the file to the the current time.
if !checker.dry {
if err := os.Chtimes(timestampFile, taskTime, taskTime); err != nil {
return false, err
return false, "", err
}
}

return !shouldUpdate, nil
return !shouldUpdate, "", nil
}

func (checker *TimestampChecker) SetUpToDate(t *ast.Task) error {
func (checker *TimestampChecker) SetUpToDate(t *ast.Task, sourceState string) error {
return nil // TODO: implement
}

Expand Down
22 changes: 12 additions & 10 deletions internal/fingerprint/task.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ func IsTaskUpToDate(
ctx context.Context,
t *ast.Task,
opts ...CheckerOption,
) (bool, error) {
) (bool, string, error) {
var statusUpToDate bool
var sourcesUpToDate bool
var err error
Expand Down Expand Up @@ -88,7 +88,7 @@ func IsTaskUpToDate(
if config.sourcesChecker == nil {
config.sourcesChecker, err = NewSourcesChecker(config.method, config.tempDir, config.dry)
if err != nil {
return false, err
return false, "", err
}
}

Expand All @@ -99,41 +99,43 @@ func IsTaskUpToDate(
if statusIsSet {
statusUpToDate, err = config.statusChecker.IsUpToDate(ctx, t)
if err != nil {
return false, err
return false, "", err
}
}

sourcesState := ""
// If sources is set, check if they are up-to-date
if sourcesIsSet {
sourcesUpToDate, err = config.sourcesChecker.IsUpToDate(t)
sourcesUpToDate, sourcesState, err = config.sourcesChecker.IsUpToDate(t)
if err != nil {
return false, err
return false, "", err
}
}

// If both status and sources are set, the task is up-to-date if both are up-to-date
if statusIsSet && sourcesIsSet {
return statusUpToDate && sourcesUpToDate, nil
return statusUpToDate && sourcesUpToDate, sourcesState, nil
}

// If only status is set, the task is up-to-date if the status is up-to-date
if statusIsSet {
return statusUpToDate, nil
return statusUpToDate, sourcesState, nil
}

// If only sources is set, the task is up-to-date if the sources are up-to-date
if sourcesIsSet {
return sourcesUpToDate, nil
return sourcesUpToDate, sourcesState, nil
}

// If no status or sources are set, the task should always run
// i.e. it is never considered "up-to-date"
return false, nil
return false, sourcesState, nil
}

func SetTaskUpToDate(
ctx context.Context,
t *ast.Task,
sourceState string,
opts ...CheckerOption,
) error {
var err error
Expand Down Expand Up @@ -166,5 +168,5 @@ func SetTaskUpToDate(
}
}

return config.sourcesChecker.SetUpToDate(t)
return config.sourcesChecker.SetUpToDate(t, sourceState)
}
Loading

0 comments on commit 580492b

Please sign in to comment.