backoff: new package.
This commit is contained in:
parent
dd98356479
commit
9d1e3ab2f0
31
LICENSE
31
LICENSE
|
|
@ -11,3 +11,34 @@ ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
|||
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
|
||||
|
||||
=======================================================================
|
||||
The backoff package (written during my time at Cloudflare) is released
|
||||
under the following license:
|
||||
|
||||
Copyright (c) 2016 CloudFlare Inc.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions
|
||||
are met:
|
||||
|
||||
Redistributions of source code must retain the above copyright notice,
|
||||
this list of conditions and the following disclaimer.
|
||||
|
||||
Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
|
||||
TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
||||
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
|
||||
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
|
||||
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ Contents:
|
|||
|
||||
ahash/ Provides hashes from string algorithm specifiers.
|
||||
assert/ Error handling, assertion-style.
|
||||
backoff/ Implementation of an intelligent backoff strategy.
|
||||
cmd/
|
||||
atping/ Automated TCP ping, meant for putting in cronjobs.
|
||||
certchain/ Display the certificate chain from a
|
||||
|
|
|
|||
|
|
@ -0,0 +1,24 @@
|
|||
Copyright (c) 2016 CloudFlare Inc.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions
|
||||
are met:
|
||||
|
||||
Redistributions of source code must retain the above copyright notice,
|
||||
this list of conditions and the following disclaimer.
|
||||
|
||||
Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
|
||||
TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
||||
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
|
||||
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
|
||||
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
|
@ -0,0 +1,83 @@
|
|||
# backoff
|
||||
## Go implementation of "Exponential Backoff And Jitter"
|
||||
|
||||
This package implements the backoff strategy described in the AWS
|
||||
Architecture Blog article
|
||||
["Exponential Backoff And Jitter"](http://www.awsarchitectureblog.com/2015/03/backoff.html). Essentially,
|
||||
the backoff has an interval `time.Duration`; the *n<sup>th</sup>* call
|
||||
to backoff will return an a `time.Duration` that is *2 <sup>n</sup> *
|
||||
interval*. If jitter is enabled (which is the default behaviour), the
|
||||
duration is a random value between 0 and *2 <sup>n</sup> * interval*.
|
||||
The backoff is configured with a maximum duration that will not be
|
||||
exceeded; e.g., by default, the longest duration returned is
|
||||
`backoff.DefaultMaxDuration`.
|
||||
|
||||
## Usage
|
||||
|
||||
A `Backoff` is initialised with a call to `New`. Using zero values
|
||||
causes it to use `DefaultMaxDuration` and `DefaultInterval` as the
|
||||
maximum duration and interval.
|
||||
|
||||
```
|
||||
package something
|
||||
|
||||
import "github.com/cloudflare/backoff"
|
||||
|
||||
func retryable() {
|
||||
b := backoff.New(0, 0)
|
||||
for {
|
||||
err := someOperation()
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
|
||||
log.Printf("error in someOperation: %v", err)
|
||||
<-time.After(b.Duration())
|
||||
}
|
||||
|
||||
log.Printf("succeeded after %d tries", b.Tries()+1)
|
||||
b.Reset()
|
||||
}
|
||||
```
|
||||
|
||||
It can also be used to rate limit code that should retry infinitely, but which does not
|
||||
use `Backoff` itself.
|
||||
|
||||
```
|
||||
package something
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/cloudflare/backoff"
|
||||
)
|
||||
|
||||
func retryable() {
|
||||
b := backoff.New(0, 0)
|
||||
b.SetDecay(30 * time.Second)
|
||||
|
||||
for {
|
||||
// b will reset if someOperation returns later than
|
||||
// the last call to b.Duration() + 30s.
|
||||
err := someOperation()
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
|
||||
log.Printf("error in someOperation: %v", err)
|
||||
<-time.After(b.Duration())
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Tunables
|
||||
|
||||
* `NewWithoutJitter` creates a Backoff that doesn't use jitter.
|
||||
|
||||
The default behaviour is controlled by two variables:
|
||||
|
||||
* `DefaultInterval` sets the base interval for backoffs created with
|
||||
the zero `time.Duration` value in the `Interval` field.
|
||||
* `DefaultMaxDuration` sets the maximum duration for backoffs created
|
||||
with the zero `time.Duration` value in the `MaxDuration` field.
|
||||
|
||||
|
|
@ -0,0 +1,197 @@
|
|||
// Package backoff contains an implementation of an intelligent backoff
|
||||
// strategy. It is based on the approach in the AWS architecture blog
|
||||
// article titled "Exponential Backoff And Jitter", which is found at
|
||||
// http://www.awsarchitectureblog.com/2015/03/backoff.html.
|
||||
//
|
||||
// Essentially, the backoff has an interval `time.Duration`; the nth
|
||||
// call to backoff will return a `time.Duration` that is 2^n *
|
||||
// interval. If jitter is enabled (which is the default behaviour),
|
||||
// the duration is a random value between 0 and 2^n * interval. The
|
||||
// backoff is configured with a maximum duration that will not be
|
||||
// exceeded.
|
||||
//
|
||||
// The `New` function will attempt to use the system's cryptographic
|
||||
// random number generator to seed a Go math/rand random number
|
||||
// source. If this fails, the package will panic on startup.
|
||||
package backoff
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/binary"
|
||||
"io"
|
||||
"math"
|
||||
mrand "math/rand"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
var prngMu sync.Mutex
|
||||
var prng *mrand.Rand
|
||||
|
||||
// DefaultInterval is used when a Backoff is initialised with a
|
||||
// zero-value Interval.
|
||||
var DefaultInterval = 5 * time.Minute
|
||||
|
||||
// DefaultMaxDuration is maximum amount of time that the backoff will
|
||||
// delay for.
|
||||
var DefaultMaxDuration = 6 * time.Hour
|
||||
|
||||
// A Backoff contains the information needed to intelligently backoff
|
||||
// and retry operations using an exponential backoff algorithm. It should
|
||||
// be initialised with a call to `New`.
|
||||
//
|
||||
// Only use a Backoff from a single goroutine, it is not safe for concurrent
|
||||
// access.
|
||||
type Backoff struct {
|
||||
// maxDuration is the largest possible duration that can be
|
||||
// returned from a call to Duration.
|
||||
maxDuration time.Duration
|
||||
|
||||
// interval controls the time step for backing off.
|
||||
interval time.Duration
|
||||
|
||||
// noJitter controls whether to use the "Full Jitter"
|
||||
// improvement to attempt to smooth out spikes in a high
|
||||
// contention scenario. If noJitter is set to true, no
|
||||
// jitter will be introduced.
|
||||
noJitter bool
|
||||
|
||||
// decay controls the decay of n. If it is non-zero, n is
|
||||
// reset if more than the last backoff + decay has elapsed since
|
||||
// the last try.
|
||||
decay time.Duration
|
||||
|
||||
n uint64
|
||||
lastTry time.Time
|
||||
}
|
||||
|
||||
// New creates a new backoff with the specified max duration and
|
||||
// interval. Zero values may be used to use the default values.
|
||||
//
|
||||
// Panics if either max or interval is negative.
|
||||
func New(max time.Duration, interval time.Duration) *Backoff {
|
||||
if max < 0 || interval < 0 {
|
||||
panic("backoff: max or interval is negative")
|
||||
}
|
||||
|
||||
b := &Backoff{
|
||||
maxDuration: max,
|
||||
interval: interval,
|
||||
}
|
||||
b.setup()
|
||||
return b
|
||||
}
|
||||
|
||||
// NewWithoutJitter works similarly to New, except that the created
|
||||
// Backoff will not use jitter.
|
||||
func NewWithoutJitter(max time.Duration, interval time.Duration) *Backoff {
|
||||
b := New(max, interval)
|
||||
b.noJitter = true
|
||||
return b
|
||||
}
|
||||
|
||||
func init() {
|
||||
var buf [8]byte
|
||||
var n int64
|
||||
|
||||
_, err := io.ReadFull(rand.Reader, buf[:])
|
||||
if err != nil {
|
||||
panic(err.Error())
|
||||
}
|
||||
|
||||
n = int64(binary.LittleEndian.Uint64(buf[:]))
|
||||
|
||||
src := mrand.NewSource(n)
|
||||
prng = mrand.New(src)
|
||||
}
|
||||
|
||||
func (b *Backoff) setup() {
|
||||
if b.interval == 0 {
|
||||
b.interval = DefaultInterval
|
||||
}
|
||||
|
||||
if b.maxDuration == 0 {
|
||||
b.maxDuration = DefaultMaxDuration
|
||||
}
|
||||
}
|
||||
|
||||
// Duration returns a time.Duration appropriate for the backoff,
|
||||
// incrementing the attempt counter.
|
||||
func (b *Backoff) Duration() time.Duration {
|
||||
b.setup()
|
||||
|
||||
b.decayN()
|
||||
|
||||
t := b.duration(b.n)
|
||||
|
||||
if b.n < math.MaxUint64 {
|
||||
b.n++
|
||||
}
|
||||
|
||||
if !b.noJitter {
|
||||
prngMu.Lock()
|
||||
t = time.Duration(prng.Int63n(int64(t)))
|
||||
prngMu.Unlock()
|
||||
}
|
||||
|
||||
return t
|
||||
}
|
||||
|
||||
// requires b to be locked.
|
||||
func (b *Backoff) duration(n uint64) (t time.Duration) {
|
||||
// Saturate pow
|
||||
pow := time.Duration(math.MaxInt64)
|
||||
if n < 63 {
|
||||
pow = 1 << n
|
||||
}
|
||||
|
||||
t = b.interval * pow
|
||||
if t/pow != b.interval || t > b.maxDuration {
|
||||
t = b.maxDuration
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Reset resets the attempt counter of a backoff.
|
||||
//
|
||||
// It should be called when the rate-limited action succeeds.
|
||||
func (b *Backoff) Reset() {
|
||||
b.lastTry = time.Time{}
|
||||
b.n = 0
|
||||
}
|
||||
|
||||
// SetDecay sets the duration after which the try counter will be reset.
|
||||
// Panics if decay is smaller than 0.
|
||||
//
|
||||
// The decay only kicks in if at least the last backoff + decay has elapsed
|
||||
// since the last try.
|
||||
func (b *Backoff) SetDecay(decay time.Duration) {
|
||||
if decay < 0 {
|
||||
panic("backoff: decay < 0")
|
||||
}
|
||||
|
||||
b.decay = decay
|
||||
}
|
||||
|
||||
// requires b to be locked
|
||||
func (b *Backoff) decayN() {
|
||||
if b.decay == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
if b.lastTry.IsZero() {
|
||||
b.lastTry = time.Now()
|
||||
return
|
||||
}
|
||||
|
||||
lastDuration := b.duration(b.n - 1)
|
||||
decayed := time.Since(b.lastTry) > lastDuration+b.decay
|
||||
b.lastTry = time.Now()
|
||||
|
||||
if !decayed {
|
||||
return
|
||||
}
|
||||
|
||||
b.n = 0
|
||||
}
|
||||
|
|
@ -0,0 +1,175 @@
|
|||
package backoff
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// If given New with 0's and no jitter, ensure that certain invariants are met:
|
||||
//
|
||||
// - the default max duration and interval should be used
|
||||
// - noJitter should be true
|
||||
// - the RNG should not be initialised
|
||||
// - the first duration should be equal to the default interval
|
||||
func TestDefaults(t *testing.T) {
|
||||
b := NewWithoutJitter(0, 0)
|
||||
|
||||
if b.maxDuration != DefaultMaxDuration {
|
||||
t.Fatalf("expected new backoff to use the default max duration (%s), but have %s", DefaultMaxDuration, b.maxDuration)
|
||||
}
|
||||
|
||||
if b.interval != DefaultInterval {
|
||||
t.Fatalf("exepcted new backoff to use the default interval (%s), but have %s", DefaultInterval, b.interval)
|
||||
}
|
||||
|
||||
if b.noJitter != true {
|
||||
t.Fatal("backoff should have been initialised without jitter")
|
||||
}
|
||||
|
||||
dur := b.Duration()
|
||||
if dur != DefaultInterval {
|
||||
t.Fatalf("expected first duration to be %s, have %s", DefaultInterval, dur)
|
||||
}
|
||||
}
|
||||
|
||||
// Given a zero-value initialised Backoff, it should be transparently
|
||||
// setup.
|
||||
func TestSetup(t *testing.T) {
|
||||
b := new(Backoff)
|
||||
dur := b.Duration()
|
||||
if dur < 0 || dur > (5*time.Minute) {
|
||||
t.Fatalf("want duration between 0 and 5 minutes, have %s", dur)
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure that tries incremenets as expected.
|
||||
func TestTries(t *testing.T) {
|
||||
b := NewWithoutJitter(5, 1)
|
||||
|
||||
for i := uint64(0); i < 3; i++ {
|
||||
if b.n != i {
|
||||
t.Fatalf("want tries=%d, have tries=%d", i, b.n)
|
||||
}
|
||||
|
||||
pow := 1 << i
|
||||
expected := time.Duration(pow)
|
||||
dur := b.Duration()
|
||||
if dur != expected {
|
||||
t.Fatalf("want duration=%d, have duration=%d at i=%d", expected, dur, i)
|
||||
}
|
||||
}
|
||||
|
||||
for i := uint(3); i < 5; i++ {
|
||||
dur := b.Duration()
|
||||
if dur != 5 {
|
||||
t.Fatalf("want duration=5, have %d at i=%d", dur, i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure that a call to Reset will actually reset the Backoff.
|
||||
func TestReset(t *testing.T) {
|
||||
const iter = 10
|
||||
b := New(1000, 1)
|
||||
for i := 0; i < iter; i++ {
|
||||
_ = b.Duration()
|
||||
}
|
||||
|
||||
if b.n != iter {
|
||||
t.Fatalf("expected tries=%d, have tries=%d", iter, b.n)
|
||||
}
|
||||
|
||||
b.Reset()
|
||||
if b.n != 0 {
|
||||
t.Fatalf("expected tries=0 after reset, have tries=%d", b.n)
|
||||
}
|
||||
}
|
||||
|
||||
const decay = 5 * time.Millisecond
|
||||
const max = 10 * time.Millisecond
|
||||
const interval = time.Millisecond
|
||||
|
||||
func TestDecay(t *testing.T) {
|
||||
const iter = 10
|
||||
|
||||
b := NewWithoutJitter(max, 1)
|
||||
b.SetDecay(decay)
|
||||
|
||||
var backoff time.Duration
|
||||
for i := 0; i < iter; i++ {
|
||||
backoff = b.Duration()
|
||||
}
|
||||
|
||||
if b.n != iter {
|
||||
t.Fatalf("expected tries=%d, have tries=%d", iter, b.n)
|
||||
}
|
||||
|
||||
// Don't decay below backoff
|
||||
b.lastTry = time.Now().Add(-backoff + 1)
|
||||
backoff = b.Duration()
|
||||
if b.n != iter+1 {
|
||||
t.Fatalf("expected tries=%d, have tries=%d", iter+1, b.n)
|
||||
}
|
||||
|
||||
// Reset after backoff + decay
|
||||
b.lastTry = time.Now().Add(-backoff - decay)
|
||||
b.Duration()
|
||||
if b.n != 1 {
|
||||
t.Fatalf("expected tries=%d, have tries=%d", 1, b.n)
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure that decay works even if the retry counter is saturated.
|
||||
func TestDecaySaturation(t *testing.T) {
|
||||
b := NewWithoutJitter(1<<2, 1)
|
||||
b.SetDecay(decay)
|
||||
|
||||
var duration time.Duration
|
||||
for i := 0; i <= 2; i++ {
|
||||
duration = b.Duration()
|
||||
}
|
||||
|
||||
if duration != 1<<2 {
|
||||
t.Fatalf("expected duration=%v, have duration=%v", 1<<2, duration)
|
||||
}
|
||||
|
||||
b.lastTry = time.Now().Add(-duration - decay)
|
||||
b.n = math.MaxUint64
|
||||
|
||||
duration = b.Duration()
|
||||
if duration != 1 {
|
||||
t.Errorf("expected duration=%v, have duration=%v", 1, duration)
|
||||
}
|
||||
}
|
||||
|
||||
func ExampleBackoff_SetDecay() {
|
||||
b := NewWithoutJitter(max, interval)
|
||||
b.SetDecay(decay)
|
||||
|
||||
// try 0
|
||||
fmt.Println(b.Duration())
|
||||
|
||||
// try 1
|
||||
fmt.Println(b.Duration())
|
||||
|
||||
// try 2
|
||||
duration := b.Duration()
|
||||
fmt.Println(duration)
|
||||
|
||||
// try 3, below decay
|
||||
time.Sleep(duration)
|
||||
duration = b.Duration()
|
||||
fmt.Println(duration)
|
||||
|
||||
// try 4, resets
|
||||
time.Sleep(duration + decay)
|
||||
fmt.Println(b.Duration())
|
||||
|
||||
// Output: 1ms
|
||||
// 2ms
|
||||
// 4ms
|
||||
// 8ms
|
||||
// 1ms
|
||||
}
|
||||
Loading…
Reference in New Issue