-
Notifications
You must be signed in to change notification settings - Fork 445
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Fixed potential deadlocks when deploying Apple configuration profiles (…
…#24777) (#24797) Cherry pick for #24771 Fixing deadlocks found in loadtest: https://docs.google.com/document/d/1-Q6qFTd7CDm-lh7MVRgpNlNNJijk6JZ4KO49R1fp80U/edit?tab=t.0 - added retries to statements prone to deadlocks # Checklist for submitter - [x] Changes file added for user-visible changes in `changes/`, `orbit/changes/` or `ee/fleetd-chrome/changes`. - [x] Manual QA for all new/changed functionality (cherry picked from commit 1e5da18)
- Loading branch information
Showing
10 changed files
with
142 additions
and
103 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| Fixed potential deadlocks when deploying Apple configuration profiles. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,93 @@ | ||
| package common_mysql | ||
|
|
||
| import ( | ||
| "context" | ||
| "database/sql" | ||
| "errors" | ||
| "time" | ||
|
|
||
| "github.com/VividCortex/mysqlerr" | ||
| "github.com/cenkalti/backoff/v4" | ||
| "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" | ||
| "github.com/go-kit/log" | ||
| "github.com/go-sql-driver/mysql" | ||
| "github.com/jmoiron/sqlx" | ||
| ) | ||
|
|
||
| var DoRetryErr = errors.New("fleet datastore retry") | ||
|
|
||
| type TxFn func(tx sqlx.ExtContext) error | ||
|
|
||
| // WithRetryTxx provides a common way to commit/rollback a txFn wrapped in a retry with exponential backoff | ||
| func WithRetryTxx(ctx context.Context, db *sqlx.DB, fn TxFn, logger log.Logger) error { | ||
| operation := func() error { | ||
| tx, err := db.BeginTxx(ctx, nil) | ||
| if err != nil { | ||
| return ctxerr.Wrap(ctx, err, "create transaction") | ||
| } | ||
|
|
||
| defer func() { | ||
| if p := recover(); p != nil { | ||
| if err := tx.Rollback(); err != nil { | ||
| logger.Log("err", err, "msg", "error encountered during transaction panic rollback") | ||
| } | ||
| panic(p) | ||
| } | ||
| }() | ||
|
|
||
| if err := fn(tx); err != nil { | ||
| rbErr := tx.Rollback() | ||
| if rbErr != nil && rbErr != sql.ErrTxDone { | ||
| // Consider rollback errors to be non-retryable | ||
| return backoff.Permanent(ctxerr.Wrapf(ctx, err, "got err '%s' rolling back after err", rbErr.Error())) | ||
| } | ||
|
|
||
| if retryableError(err) { | ||
| return err | ||
| } | ||
|
|
||
| // Consider any other errors to be non-retryable | ||
| return backoff.Permanent(err) | ||
| } | ||
|
|
||
| if err := tx.Commit(); err != nil { | ||
| err = ctxerr.Wrap(ctx, err, "commit transaction") | ||
|
|
||
| if retryableError(err) { | ||
| return err | ||
| } | ||
|
|
||
| return backoff.Permanent(err) | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| expBo := backoff.NewExponentialBackOff() | ||
| // MySQL innodb_lock_wait_timeout default is 50 seconds, so transaction can be waiting for a lock for several seconds. | ||
| // Setting a higher MaxElapsedTime to increase probability that transaction will be retried. | ||
| // This will reduce the number of retryable 'Deadlock found' errors. However, with a loaded DB, we will still see | ||
| // 'Context cancelled' errors when the server drops long-lasting connections. | ||
| expBo.MaxElapsedTime = 1 * time.Minute | ||
| bo := backoff.WithMaxRetries(expBo, 5) | ||
| return backoff.Retry(operation, bo) | ||
| } | ||
|
|
||
| // retryableError determines whether a MySQL error can be retried. By default | ||
| // errors are considered non-retryable. Only errors that we know have a | ||
| // possibility of succeeding on a retry should return true in this function. | ||
| func retryableError(err error) bool { | ||
| base := ctxerr.Cause(err) | ||
| if b, ok := base.(*mysql.MySQLError); ok { | ||
| switch b.Number { | ||
| // Consider lock related errors to be retryable | ||
| case mysqlerr.ER_LOCK_DEADLOCK, mysqlerr.ER_LOCK_WAIT_TIMEOUT: | ||
| return true | ||
| } | ||
| } | ||
| if errors.Is(err, DoRetryErr) { | ||
| return true | ||
| } | ||
|
|
||
| return false | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters