Skip to content

Commit

Permalink
Implement checksum validation for generates
Browse files Browse the repository at this point in the history
Follows the same algorithms than for sources, should be much more
understandable & reliable
  • Loading branch information
vanackere committed Sep 27, 2024
1 parent e367f10 commit ab2dbba
Show file tree
Hide file tree
Showing 7 changed files with 163 additions and 27 deletions.
1 change: 1 addition & 0 deletions internal/fingerprint/checker.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ type StatusCheckable interface {

// SourcesCheckable defines any type that can check if the sources of a task are up-to-date.
type SourcesCheckable interface {
SetUpToDate(t *ast.Task) error
IsUpToDate(t *ast.Task) (bool, error)
Value(t *ast.Task) (any, error)
OnError(t *ast.Task) error
Expand Down
81 changes: 55 additions & 26 deletions internal/fingerprint/sources_checksum.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,51 +29,80 @@ func NewChecksumChecker(tempDir string, dry bool) *ChecksumChecker {
}

func (checker *ChecksumChecker) IsUpToDate(t *ast.Task) (bool, error) {
if len(t.Sources) == 0 {
if len(t.Sources) == 0 && len(t.Generates) == 0 {
return false, nil
}

checksumFile := checker.checksumFilePath(t)

data, _ := os.ReadFile(checksumFile)
oldHash := strings.TrimSpace(string(data))
oldHashes := strings.TrimSpace(string(data))
oldSourcesHash, oldGeneratesdHash, _ := strings.Cut(oldHashes, "\n")

newHash, err := checker.checksum(t)
newSourcesHash, err := checker.checksum(t, t.Sources)
if err != nil {
return false, nil
return false, err
}

if !checker.dry && oldHash != newHash {
newGeneratesHash, err := checker.checksum(t, t.Generates)
if err != nil {
return false, err
}

if !checker.dry && oldSourcesHash != newSourcesHash {
// make sure current sources hash are saved to file before executing the task,
// the proper "generated" hash will be saved after the task is executed
_ = os.MkdirAll(filepathext.SmartJoin(checker.tempDir, "checksum"), 0o755)
if err = os.WriteFile(checksumFile, []byte(newHash+"\n"), 0o644); err != nil {
if err = os.WriteFile(checksumFile, []byte(newSourcesHash+"\n"+"_"), 0o644); err != nil {
return false, err
}
}

if len(t.Generates) > 0 {
// For each specified 'generates' field, check whether the files actually exist
for _, g := range t.Generates {
if g.Negate {
continue
}
generates, err := Glob(t.Dir, g.Glob)
if os.IsNotExist(err) {
return false, nil
}
if err != nil {
return false, err
}
if len(generates) == 0 {
return false, nil
}
return oldSourcesHash == newSourcesHash && oldGeneratesdHash == newGeneratesHash, nil
}

func (checker *ChecksumChecker) SetUpToDate(t *ast.Task) error {
if len(t.Sources) == 0 && len(t.Generates) == 0 {
return nil
}

if checker.dry {
return nil
}

checksumFile := checker.checksumFilePath(t)

data, _ := os.ReadFile(checksumFile)
oldHashes := strings.TrimSpace(string(data))
oldSourcesHash, oldGeneratesdHash, _ := strings.Cut(oldHashes, "\n")

newSourcesHash, err := checker.checksum(t, t.Sources)
if err != nil {
return err
}

newGeneratesHash, err := checker.checksum(t, t.Generates)
if err != nil {
return err
}

if oldSourcesHash != newSourcesHash || oldGeneratesdHash != newGeneratesHash {
_ = os.MkdirAll(filepathext.SmartJoin(checker.tempDir, "checksum"), 0o755)
if err = os.WriteFile(checksumFile, []byte(oldSourcesHash+"\n"+newGeneratesHash+"\n"), 0o644); err != nil {
return err
}
}

return oldHash == newHash, nil
return nil
}

func (checker *ChecksumChecker) Value(t *ast.Task) (any, error) {
return checker.checksum(t)
c1, err := checker.checksum(t, t.Sources)
if err != nil {
return c1, err
}
c2, err := checker.checksum(t, t.Generates)
return c1 + "\n" + c2, err
}

func (checker *ChecksumChecker) OnError(t *ast.Task) error {
Expand All @@ -87,8 +116,8 @@ func (*ChecksumChecker) Kind() string {
return "checksum"
}

func (c *ChecksumChecker) checksum(t *ast.Task) (string, error) {
sources, err := Globs(t.Dir, t.Sources)
func (c *ChecksumChecker) checksum(t *ast.Task, globs []*ast.Glob) (string, error) {
sources, err := Globs(t.Dir, globs)
if err != nil {
return "", err
}
Expand Down
4 changes: 4 additions & 0 deletions internal/fingerprint/sources_none.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,10 @@ func (NoneChecker) IsUpToDate(t *ast.Task) (bool, error) {
return false, nil
}

func (NoneChecker) SetUpToDate(t *ast.Task) error {
return nil
}

func (NoneChecker) Value(t *ast.Task) (any, error) {
return "", nil
}
Expand Down
4 changes: 4 additions & 0 deletions internal/fingerprint/sources_timestamp.go
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,10 @@ func (checker *TimestampChecker) IsUpToDate(t *ast.Task) (bool, error) {
return !shouldUpdate, nil
}

func (checker *TimestampChecker) SetUpToDate(t *ast.Task) error {
return nil // TODO: implement
}

func (checker *TimestampChecker) Kind() string {
return "timestamp"
}
Expand Down
40 changes: 39 additions & 1 deletion internal/fingerprint/task.go
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ func IsTaskUpToDate(
}

statusIsSet := len(t.Status) != 0
sourcesIsSet := len(t.Sources) != 0
sourcesIsSet := len(t.Sources) != 0 || len(t.Generates) != 0

// If status is set, check if it is up-to-date
if statusIsSet {
Expand Down Expand Up @@ -130,3 +130,41 @@ func IsTaskUpToDate(
// i.e. it is never considered "up-to-date"
return false, nil
}

func SetTaskUpToDate(
ctx context.Context,
t *ast.Task,
opts ...CheckerOption,
) error {
var err error

// Default config
config := &CheckerConfig{
method: "none",
tempDir: "",
dry: false,
logger: nil,
statusChecker: nil,
sourcesChecker: nil,
}

// Apply functional options
for _, opt := range opts {
opt(config)
}

// If no status checker was given, set up the default one
if config.statusChecker == nil {
config.statusChecker = NewStatusChecker(config.logger)
}

// If no sources checker was given, set up the default one
if config.sourcesChecker == nil {
config.sourcesChecker, err = NewSourcesChecker(config.method, config.tempDir, config.dry)
if err != nil {
return err
}
}

return config.sourcesChecker.SetUpToDate(t)
}
46 changes: 46 additions & 0 deletions internal/mocks/sources_checkable.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

14 changes: 14 additions & 0 deletions task.go
Original file line number Diff line number Diff line change
Expand Up @@ -278,6 +278,20 @@ func (e *Executor) RunTask(ctx context.Context, call *ast.Call) error {
return &errors.TaskRunError{TaskName: t.Task, Err: err}
}
}
if !skipFingerprinting {
// Get the fingerprinting method to use
method := e.Taskfile.Method
if t.Method != "" {
method = t.Method
}
e.Logger.VerboseErrf(logger.Magenta, "task: %q setting task up to date\n", call.Task)
fingerprint.SetTaskUpToDate(ctx, t,
fingerprint.WithMethod(method),
fingerprint.WithTempDir(e.TempDir.Fingerprint),
fingerprint.WithDry(e.Dry),
fingerprint.WithLogger(e.Logger),
)
}
e.Logger.VerboseErrf(logger.Magenta, "task: %q finished\n", call.Task)
return nil
})
Expand Down

0 comments on commit ab2dbba

Please sign in to comment.