Skip to content

Commit

Permalink
Checksum: Use 128kb buffer to optimize file reads
Browse files Browse the repository at this point in the history
This seems to perform well against a micro-benchmark reading the Linux kernel, while using an amount of memory that should be acceptable on almost all systems.
  • Loading branch information
ReillyBrogan committed Sep 8, 2023
1 parent f7aa4fb commit d07a2cb
Showing 1 changed file with 3 additions and 2 deletions.
5 changes: 3 additions & 2 deletions internal/fingerprint/sources_checksum.go
Original file line number Diff line number Diff line change
Expand Up @@ -91,16 +91,17 @@ func (c *ChecksumChecker) checksum(t *taskfile.Task) (string, error) {
}

h := xxh3.New()
buf := make([]byte, 128 * 1024)
for _, f := range sources {
// also sum the filename, so checksum changes for renaming a file
if _, err := io.Copy(h, strings.NewReader(filepath.Base(f))); err != nil {
if _, err := io.CopyBuffer(h, strings.NewReader(filepath.Base(f)), buf); err != nil {
return "", err
}
f, err := os.Open(f)
if err != nil {
return "", err
}
if _, err = io.Copy(h, f); err != nil {
if _, err = io.CopyBuffer(h, f, buf); err != nil {
return "", err
}
f.Close()
Expand Down

0 comments on commit d07a2cb

Please sign in to comment.