feat: support negative globs

Co-authored-by: Andrey Nering <[email protected]>

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## Unreleased
 
+- Added ability to exclude some files from `sources:` by using `exclude:` (#225,
+  #1324 by @pd93 and @andreynering).
 - The
   [Remote Taskfiles experiment](https://taskfile.dev/experiments/remote-taskfiles)
   now prefers remote files over cached ones by default (#1317, #1345 by @pd93).

docs/docs/usage.md

@@ -642,11 +642,27 @@ tasks:
       - public/bundle.css
 ```
 
-`sources` and `generates` can be files or file patterns. When given, Task will
+`sources` and `generates` can be files or glob patterns. When given, Task will
 compare the checksum of the source files to determine if it's necessary to run
 the task. If not, it will just print a message like `Task "js" is up to date`.
 
-If you prefer this check to be made by the modification timestamp of the files,
+`exclude:` can also be used to exclude files from fingerprinting.
+Sources are evaluated in order, so `exclude:` must come after the positive
+glob it is negating.
+
+```yaml
+version: '3'
+
+tasks:
+  css:
+    sources:
+      - mysources/**/*.css
+      - exclude: mysources/ignoreme.css
+    generates:
+      - public/bundle.css
+```
+
+If you prefer these check to be made by the modification timestamp of the files,
 instead of its checksum (content), just set the `method` property to
 `timestamp`.
 
@@ -1001,9 +1017,9 @@ This works for all types of variables.
 
 ## Looping over values
 
-As of v3.28.0, Task allows you to loop over certain values and execute a
-command for each. There are a number of ways to do this depending on the type
-of value you want to loop over.
+As of v3.28.0, Task allows you to loop over certain values and execute a command
+for each. There are a number of ways to do this depending on the type of value
+you want to loop over.
 
 ### Looping over a static list
 
@@ -1043,9 +1059,8 @@ match that glob.
 
 Source paths will always be returned as paths relative to the task directory. If
 you need to convert this to an absolute path, you can use the built-in
-`joinPath` function.
-There are some [special variables](/api/#special-variables) that you may find
-useful for this.
+`joinPath` function. There are some [special variables](/api/#special-variables)
+that you may find useful for this.
 
 ```yaml
 version: '3'

docs/static/schema.json

@@ -88,14 +88,14 @@
             "description": "A list of sources to check before running this task. Relevant for `checksum` and `timestamp` methods. Can be file paths or star globs.",
             "type": "array",
             "items": {
-              "type": "string"
+              "$ref": "#/definitions/3/glob"
             }
           },
           "generates": {
             "description": "A list of files meant to be generated by this task. Relevant for `timestamp` method. Can be file paths or star globs.",
             "type": "array",
             "items": {
-              "type": "string"
+              "$ref": "#/definitions/3/glob"
             }
           },
           "status": {
@@ -446,6 +446,25 @@
           }
         }
       },
+      "glob": {
+        "anyOf": [
+          {
+            "type": "string"
+          },
+          {
+            "$ref": "#/definitions/3/glob_obj"
+          }
+        ]
+      },
+      "glob_obj": {
+        "type": "object",
+        "properties": {
+          "exclude": {
+            "description": "File or glob patter to exclude from the list",
+            "type": "string"
+          }
+        }
+      },
       "run": {
         "type": "string",
         "enum": ["always", "once", "when_changed"]

internal/fingerprint/glob.go

@@ -8,16 +8,25 @@ import (
 
 	"github.com/go-task/task/v3/internal/execext"
 	"github.com/go-task/task/v3/internal/filepathext"
+	"github.com/go-task/task/v3/taskfile"
 )
 
-func Globs(dir string, globs []string) ([]string, error) {
-	files := make([]string, 0)
+func Globs(dir string, globs []*taskfile.Glob) ([]string, error) {
+	fileMap := make(map[string]bool)
 	for _, g := range globs {
-		f, err := Glob(dir, g)
+		matches, err := Glob(dir, g.Glob)
 		if err != nil {
 			continue
 		}
-		files = append(files, f...)
+		for _, match := range matches {
+			fileMap[match] = !g.Negate
+		}
+	}
+	files := make([]string, 0)
+	for file, includePath := range fileMap {
+		if includePath {
+			files = append(files, file)
+		}
 	}
 	sort.Strings(files)
 	return files, nil

internal/fingerprint/sources_checksum.go

@@ -53,7 +53,10 @@ func (checker *ChecksumChecker) IsUpToDate(t *taskfile.Task) (bool, error) {
 	if len(t.Generates) > 0 {
 		// For each specified 'generates' field, check whether the files actually exist
 		for _, g := range t.Generates {
-			generates, err := Glob(t.Dir, g)
+			if g.Negate {
+				continue
+			}
+			generates, err := Glob(t.Dir, g.Glob)
 			if os.IsNotExist(err) {
 				return false, nil
 			}

internal/fingerprint/task_test.go

@@ -47,7 +47,7 @@ func TestIsTaskUpToDate(t *testing.T) {
 			name: "expect TRUE when no status is defined and sources are up-to-date",
 			task: &taskfile.Task{
 				Status:  nil,
-				Sources: []string{"sources"},
+				Sources: []*taskfile.Glob{{Glob: "sources"}},
 			},
 			setupMockStatusChecker: nil,
 			setupMockSourcesChecker: func(m *mocks.SourcesCheckable) {
@@ -59,7 +59,7 @@ func TestIsTaskUpToDate(t *testing.T) {
 			name: "expect FALSE when no status is defined and sources are NOT up-to-date",
 			task: &taskfile.Task{
 				Status:  nil,
-				Sources: []string{"sources"},
+				Sources: []*taskfile.Glob{{Glob: "sources"}},
 			},
 			setupMockStatusChecker: nil,
 			setupMockSourcesChecker: func(m *mocks.SourcesCheckable) {
@@ -83,7 +83,7 @@ func TestIsTaskUpToDate(t *testing.T) {
 			name: "expect TRUE when status and sources are up-to-date",
 			task: &taskfile.Task{
 				Status:  []string{"status"},
-				Sources: []string{"sources"},
+				Sources: []*taskfile.Glob{{Glob: "sources"}},
 			},
 			setupMockStatusChecker: func(m *mocks.StatusCheckable) {
 				m.EXPECT().IsUpToDate(mock.Anything, mock.Anything).Return(true, nil)
@@ -97,7 +97,7 @@ func TestIsTaskUpToDate(t *testing.T) {
 			name: "expect FALSE when status is up-to-date, but sources are NOT up-to-date",
 			task: &taskfile.Task{
 				Status:  []string{"status"},
-				Sources: []string{"sources"},
+				Sources: []*taskfile.Glob{{Glob: "sources"}},
 			},
 			setupMockStatusChecker: func(m *mocks.StatusCheckable) {
 				m.EXPECT().IsUpToDate(mock.Anything, mock.Anything).Return(true, nil)
@@ -123,7 +123,7 @@ func TestIsTaskUpToDate(t *testing.T) {
 			name: "expect FALSE when status is NOT up-to-date, but sources are up-to-date",
 			task: &taskfile.Task{
 				Status:  []string{"status"},
-				Sources: []string{"sources"},
+				Sources: []*taskfile.Glob{{Glob: "sources"}},
 			},
 			setupMockStatusChecker: func(m *mocks.StatusCheckable) {
 				m.EXPECT().IsUpToDate(mock.Anything, mock.Anything).Return(false, nil)
@@ -137,7 +137,7 @@ func TestIsTaskUpToDate(t *testing.T) {
 			name: "expect FALSE when status and sources are NOT up-to-date",
 			task: &taskfile.Task{
 				Status:  []string{"status"},
-				Sources: []string{"sources"},
+				Sources: []*taskfile.Glob{{Glob: "sources"}},
 			},
 			setupMockStatusChecker: func(m *mocks.StatusCheckable) {
 				m.EXPECT().IsUpToDate(mock.Anything, mock.Anything).Return(false, nil)

internal/templater/templater.go

@@ -80,6 +80,21 @@ func (r *Templater) ReplaceSlice(strs []string) []string {
 	return new
 }
 
+func (r *Templater) ReplaceGlobs(globs []*taskfile.Glob) []*taskfile.Glob {
+	if r.err != nil || len(globs) == 0 {
+		return nil
+	}
+
+	new := make([]*taskfile.Glob, len(globs))
+	for i, g := range globs {
+		new[i] = &taskfile.Glob{
+			Glob:   r.Replace(g.Glob),
+			Negate: g.Negate,
+		}
+	}
+	return new
+}
+
 func (r *Templater) ReplaceVars(vars *taskfile.Vars) *taskfile.Vars {
 	return r.replaceVars(vars, nil)
 }

task_test.go

@@ -1891,27 +1891,47 @@ func TestEvaluateSymlinksInPaths(t *testing.T) {
 		Stderr: &buff,
 		Silent: false,
 	}
-	require.NoError(t, e.Setup())
-	err := e.Run(context.Background(), taskfile.Call{Task: "default"})
-	require.NoError(t, err)
-	assert.NotEqual(t, `task: Task "default" is up to date`, strings.TrimSpace(buff.String()))
-	buff.Reset()
-	err = e.Run(context.Background(), taskfile.Call{Task: "test-sym"})
-	require.NoError(t, err)
-	assert.NotEqual(t, `task: Task "test-sym" is up to date`, strings.TrimSpace(buff.String()))
-	buff.Reset()
-	err = e.Run(context.Background(), taskfile.Call{Task: "default"})
-	require.NoError(t, err)
-	assert.NotEqual(t, `task: Task "default" is up to date`, strings.TrimSpace(buff.String()))
-	buff.Reset()
-	err = e.Run(context.Background(), taskfile.Call{Task: "default"})
-	require.NoError(t, err)
-	assert.Equal(t, `task: Task "default" is up to date`, strings.TrimSpace(buff.String()))
-	buff.Reset()
-	err = e.Run(context.Background(), taskfile.Call{Task: "reset"})
-	require.NoError(t, err)
-	buff.Reset()
-	err = os.RemoveAll(dir + "/.task")
+	tests := []struct {
+		name     string
+		task     string
+		expected string
+	}{
+		{
+			name:     "default (1)",
+			task:     "default",
+			expected: "task: [default] echo \"some job\"\nsome job",
+		},
+		{
+			name:     "test-sym (1)",
+			task:     "test-sym",
+			expected: "task: [test-sym] echo \"shared file source changed\" > src/shared/b",
+		},
+		{
+			name:     "default (2)",
+			task:     "default",
+			expected: "task: [default] echo \"some job\"\nsome job",
+		},
+		{
+			name:     "default (3)",
+			task:     "default",
+			expected: `task: Task "default" is up to date`,
+		},
+		{
+			name:     "reset",
+			task:     "reset",
+			expected: "task: [reset] echo \"shared file source\" > src/shared/b\ntask: [reset] echo \"file source\" > src/a",
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			require.NoError(t, e.Setup())
+			err := e.Run(context.Background(), taskfile.Call{Task: test.task})
+			require.NoError(t, err)
+			assert.Equal(t, test.expected, strings.TrimSpace(buff.String()))
+			buff.Reset()
+		})
+	}
+	err := os.RemoveAll(dir + "/.task")
 	require.NoError(t, err)
 }
 

taskfile/glob.go

@@ -0,0 +1,32 @@
+package taskfile
+
+import (
+	"fmt"
+
+	"gopkg.in/yaml.v3"
+)
+
+type Glob struct {
+	Glob   string
+	Negate bool
+}
+
+func (g *Glob) UnmarshalYAML(node *yaml.Node) error {
+	switch node.Kind {
+	case yaml.ScalarNode:
+		g.Glob = node.Value
+		return nil
+	case yaml.MappingNode:
+		var glob struct {
+			Exclude string
+		}
+		if err := node.Decode(&glob); err != nil {
+			return err
+		}
+		g.Glob = glob.Exclude
+		g.Negate = true
+		return nil
+	default:
+		return fmt.Errorf("yaml: line %d: cannot unmarshal %s into task", node.Line, node.ShortTag())
+	}
+}

taskfile/task.go

@@ -19,8 +19,8 @@ type Task struct {
 	Summary              string
 	Requires             *Requires
 	Aliases              []string
-	Sources              []string
-	Generates            []string
+	Sources              []*Glob
+	Generates            []*Glob
 	Status               []string
 	Preconditions        []*Precondition
 	Dir                  string
@@ -83,8 +83,8 @@ func (t *Task) UnmarshalYAML(node *yaml.Node) error {
 			Prompt        string
 			Summary       string
 			Aliases       []string
-			Sources       []string
-			Generates     []string
+			Sources       []*Glob
+			Generates     []*Glob
 			Status        []string
 			Preconditions []*Precondition
 			Dir           string

testdata/checksum/Taskfile.yml

@@ -6,7 +6,9 @@ tasks:
       - cp ./source.txt ./generated.txt
     sources:
       - ./**/glob-with-inexistent-file.txt
-      - ./source.txt
+      - ./*.txt
+      - exclude: ./ignore_me.txt
+      - exclude: ./generated.txt
     generates:
       - ./generated.txt
     method: checksum

testdata/checksum/ignore_me.txt

@@ -0,0 +1 @@
+plz ignore me

variables.go

@@ -50,8 +50,8 @@ func (e *Executor) compiledTask(call taskfile.Call, evaluateShVars bool) (*taskf
 		Prompt:               r.Replace(origTask.Prompt),
 		Summary:              r.Replace(origTask.Summary),
 		Aliases:              origTask.Aliases,
-		Sources:              r.ReplaceSlice(origTask.Sources),
-		Generates:            r.ReplaceSlice(origTask.Generates),
+		Sources:              r.ReplaceGlobs(origTask.Sources),
+		Generates:            r.ReplaceGlobs(origTask.Generates),
 		Dir:                  r.Replace(origTask.Dir),
 		Set:                  origTask.Set,
 		Shopt:                origTask.Shopt,

watch.go

@@ -142,7 +142,12 @@ func (e *Executor) registerWatchedFiles(w *watcher.Watcher, calls ...taskfile.Ca
 			}
 		}
 
-		for _, s := range task.Sources {
+		globs, err := fingerprint.Globs(task.Dir, task.Sources)
+		if err != nil {
+			return err
+		}
+
+		for _, s := range globs {
 			files, err := fingerprint.Glob(task.Dir, s)
 			if err != nil {
 				return fmt.Errorf("task: %s: %w", s, err)