-
Notifications
You must be signed in to change notification settings - Fork 45
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #414 from ivcosla/feature/apps-retry-on-network-error
Feature/apps retry on network error
- Loading branch information
Showing
59 changed files
with
11,408 additions
and
10 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,103 @@ | ||
| package netutil | ||
|
|
||
| import ( | ||
| "errors" | ||
| "time" | ||
|
|
||
| "github.com/prometheus/common/log" | ||
| ) | ||
|
|
||
| // Package errors | ||
| var ( | ||
| ErrMaximumRetriesReached = errors.New("maximum retries attempted without success") | ||
| ) | ||
|
|
||
| // RetryFunc is a function used as argument of (*Retrier).Do(), which will retry on error unless it is whitelisted | ||
| type RetryFunc func() error | ||
|
|
||
| // Retrier holds a configuration for how retries should be performed | ||
| type Retrier struct { | ||
| exponentialBackoff time.Duration // multiplied on every retry by a exponentialFactor | ||
| exponentialFactor uint32 // multiplier for the backoff duration that is applied on every retry | ||
| times uint32 // number of times that the given function is going to be retried until success, if 0 it will be retried forever until success | ||
| errWhitelist map[error]struct{} | ||
| } | ||
|
|
||
| // NewRetrier returns a retrier that is ready to call Do() method | ||
| func NewRetrier(exponentialBackoff time.Duration, times, factor uint32) *Retrier { | ||
| return &Retrier{ | ||
| exponentialBackoff: exponentialBackoff, | ||
| times: times, | ||
| exponentialFactor: factor, | ||
| errWhitelist: make(map[error]struct{}), | ||
| } | ||
| } | ||
|
|
||
| // WithErrWhitelist sets a list of errors into the retrier, if the RetryFunc provided to Do() fails with one of them it will return inmediatelly with such error. Calling | ||
| // this function is not thread-safe, and is advised to only use it when initializing the Retrier | ||
| func (r *Retrier) WithErrWhitelist(errors ...error) *Retrier { | ||
| m := make(map[error]struct{}) | ||
| for _, err := range errors { | ||
| m[err] = struct{}{} | ||
| } | ||
|
|
||
| r.errWhitelist = m | ||
| return r | ||
| } | ||
|
|
||
| // Do takes a RetryFunc and attempts to execute it, if it fails with an error it will be retried a maximum of given times with an exponentialBackoff, until it returns | ||
| // nil or an error that is whitelisted | ||
| func (r Retrier) Do(f RetryFunc) error { | ||
| if r.times == 0 { | ||
| return r.retryUntilSuccess(f) | ||
| } | ||
|
|
||
| return r.retryNTimes(f) | ||
| } | ||
|
|
||
| func (r Retrier) retryNTimes(f RetryFunc) error { | ||
| currentBackoff := r.exponentialBackoff | ||
|
|
||
| for i := uint32(0); i < r.times; i++ { | ||
| err := f() | ||
| if err != nil { | ||
| if r.isWhitelisted(err) { | ||
| return err | ||
| } | ||
|
|
||
| log.Warn(err) | ||
| currentBackoff = currentBackoff * time.Duration(r.exponentialFactor) | ||
| time.Sleep(currentBackoff) | ||
| continue | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| return ErrMaximumRetriesReached | ||
| } | ||
|
|
||
| func (r Retrier) retryUntilSuccess(f RetryFunc) error { | ||
| currentBackoff := r.exponentialBackoff | ||
|
|
||
| for { | ||
| err := f() | ||
| if err != nil { | ||
| if r.isWhitelisted(err) { | ||
| return err | ||
| } | ||
|
|
||
| log.Warn(err) | ||
| currentBackoff = currentBackoff * time.Duration(r.exponentialFactor) | ||
| time.Sleep(currentBackoff) | ||
| continue | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
| } | ||
|
|
||
| func (r Retrier) isWhitelisted(err error) bool { | ||
| _, ok := r.errWhitelist[err] | ||
| return ok | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,61 @@ | ||
| package netutil | ||
|
|
||
| import ( | ||
| "errors" | ||
| "testing" | ||
| "time" | ||
|
|
||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| func TestRetrier_Do(t *testing.T) { | ||
| r := NewRetrier(time.Millisecond*100, 3, 2) | ||
| c := 0 | ||
| threshold := 2 | ||
| f := func() error { | ||
| c++ | ||
| if c >= threshold { | ||
| return nil | ||
| } | ||
|
|
||
| return errors.New("foo") | ||
| } | ||
|
|
||
| t.Run("should retry", func(t *testing.T) { | ||
| c = 0 | ||
|
|
||
| err := r.Do(f) | ||
| require.NoError(t, err) | ||
| }) | ||
|
|
||
| t.Run("if retry reaches max number of times should error", func(t *testing.T) { | ||
| c = 0 | ||
| threshold = 4 | ||
| defer func() { | ||
| threshold = 2 | ||
| }() | ||
|
|
||
| err := r.Do(f) | ||
| require.Error(t, err) | ||
| }) | ||
|
|
||
| t.Run("should return whitelisted errors if any instead of retry", func(t *testing.T) { | ||
| bar := errors.New("bar") | ||
| wR := NewRetrier(50*time.Millisecond, 1, 2).WithErrWhitelist(bar) | ||
| barF := func() error { | ||
| return bar | ||
| } | ||
|
|
||
| err := wR.Do(barF) | ||
| require.EqualError(t, err, bar.Error()) | ||
| }) | ||
|
|
||
| t.Run("if times is 0, should retry until success", func(t *testing.T) { | ||
| c = 0 | ||
| loopR := NewRetrier(50*time.Millisecond, 0, 1) | ||
| err := loopR.Do(f) | ||
| require.NoError(t, err) | ||
|
|
||
| require.Equal(t, threshold, c) | ||
| }) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
Oops, something went wrong.