kyle
/
goutils
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

backoff: new package.

This commit is contained in:
Kyle Isom 2023-05-26 17:28:06 +00:00
parent dd98356479
commit 9d1e3ab2f0
6 changed files with 511 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
LICENSE
View File

@ -11,3 +11,34 @@ ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
=======================================================================
The backoff package (written during my time at Cloudflare) is released
under the following license:
Copyright (c) 2016 CloudFlare Inc.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

1
README.md
View File

@ -12,6 +12,7 @@ Contents:
ahash/ Provides hashes from string algorithm specifiers. ahash/ Provides hashes from string algorithm specifiers.
assert/ Error handling, assertion-style. assert/ Error handling, assertion-style.
backoff/ Implementation of an intelligent backoff strategy.
cmd/ cmd/
atping/ Automated TCP ping, meant for putting in cronjobs. atping/ Automated TCP ping, meant for putting in cronjobs.
certchain/ Display the certificate chain from a certchain/ Display the certificate chain from a

24
backoff/LICENSE Normal file
View File

@ -0,0 +1,24 @@
Copyright (c) 2016 CloudFlare Inc.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

83
backoff/README.md Normal file
View File

@ -0,0 +1,83 @@
# backoff
## Go implementation of "Exponential Backoff And Jitter"
This package implements the backoff strategy described in the AWS
Architecture Blog article
["Exponential Backoff And Jitter"](http://www.awsarchitectureblog.com/2015/03/backoff.html). Essentially,
the backoff has an interval `time.Duration`; the *n<sup>th</sup>* call
to backoff will return an a `time.Duration` that is *2 <sup>n</sup> *
interval*. If jitter is enabled (which is the default behaviour), the
duration is a random value between 0 and *2 <sup>n</sup> * interval*.
The backoff is configured with a maximum duration that will not be
exceeded; e.g., by default, the longest duration returned is
`backoff.DefaultMaxDuration`.
## Usage
A `Backoff` is initialised with a call to `New`. Using zero values
causes it to use `DefaultMaxDuration` and `DefaultInterval` as the
maximum duration and interval.
```
package something
import "github.com/cloudflare/backoff"
func retryable() {
b := backoff.New(0, 0)
for {
err := someOperation()
if err == nil {
break
}
log.Printf("error in someOperation: %v", err)
<-time.After(b.Duration())
}
log.Printf("succeeded after %d tries", b.Tries()+1)
b.Reset()
}
```
It can also be used to rate limit code that should retry infinitely, but which does not
use `Backoff` itself.
```
package something
import (
"time"
"github.com/cloudflare/backoff"
)
func retryable() {
b := backoff.New(0, 0)
b.SetDecay(30 * time.Second)
for {
// b will reset if someOperation returns later than
// the last call to b.Duration() + 30s.
err := someOperation()
if err == nil {
break
}
log.Printf("error in someOperation: %v", err)
<-time.After(b.Duration())
}
}
```
## Tunables
* `NewWithoutJitter` creates a Backoff that doesn't use jitter.
The default behaviour is controlled by two variables:
* `DefaultInterval` sets the base interval for backoffs created with
the zero `time.Duration` value in the `Interval` field.
* `DefaultMaxDuration` sets the maximum duration for backoffs created
with the zero `time.Duration` value in the `MaxDuration` field.

197
backoff/backoff.go Normal file
View File

@ -0,0 +1,197 @@
// Package backoff contains an implementation of an intelligent backoff
// strategy. It is based on the approach in the AWS architecture blog
// article titled "Exponential Backoff And Jitter", which is found at
// http://www.awsarchitectureblog.com/2015/03/backoff.html.
//
// Essentially, the backoff has an interval `time.Duration`; the nth
// call to backoff will return a `time.Duration` that is 2^n *
// interval. If jitter is enabled (which is the default behaviour),
// the duration is a random value between 0 and 2^n * interval. The
// backoff is configured with a maximum duration that will not be
// exceeded.
//
// The `New` function will attempt to use the system's cryptographic
// random number generator to seed a Go math/rand random number
// source. If this fails, the package will panic on startup.
package backoff
import (
"crypto/rand"
"encoding/binary"
"io"
"math"
mrand "math/rand"
"sync"
"time"
)
var prngMu sync.Mutex
var prng *mrand.Rand
// DefaultInterval is used when a Backoff is initialised with a
// zero-value Interval.
var DefaultInterval = 5 * time.Minute
// DefaultMaxDuration is maximum amount of time that the backoff will
// delay for.
var DefaultMaxDuration = 6 * time.Hour
// A Backoff contains the information needed to intelligently backoff
// and retry operations using an exponential backoff algorithm. It should
// be initialised with a call to `New`.
//
// Only use a Backoff from a single goroutine, it is not safe for concurrent
// access.
type Backoff struct {
// maxDuration is the largest possible duration that can be
// returned from a call to Duration.
maxDuration time.Duration
// interval controls the time step for backing off.
interval time.Duration
// noJitter controls whether to use the "Full Jitter"
// improvement to attempt to smooth out spikes in a high
// contention scenario. If noJitter is set to true, no
// jitter will be introduced.
noJitter bool
// decay controls the decay of n. If it is non-zero, n is
// reset if more than the last backoff + decay has elapsed since
// the last try.
decay time.Duration
n uint64
lastTry time.Time
}
// New creates a new backoff with the specified max duration and
// interval. Zero values may be used to use the default values.
//
// Panics if either max or interval is negative.
func New(max time.Duration, interval time.Duration) *Backoff {
if max < 0 || interval < 0 {
panic("backoff: max or interval is negative")
}
b := &Backoff{
maxDuration: max,
interval: interval,
}
b.setup()
return b
}
// NewWithoutJitter works similarly to New, except that the created
// Backoff will not use jitter.
func NewWithoutJitter(max time.Duration, interval time.Duration) *Backoff {
b := New(max, interval)
b.noJitter = true
return b
}
func init() {
var buf [8]byte
var n int64
_, err := io.ReadFull(rand.Reader, buf[:])
if err != nil {
panic(err.Error())
}
n = int64(binary.LittleEndian.Uint64(buf[:]))
src := mrand.NewSource(n)
prng = mrand.New(src)
}
func (b *Backoff) setup() {
if b.interval == 0 {
b.interval = DefaultInterval
}
if b.maxDuration == 0 {
b.maxDuration = DefaultMaxDuration
}
}
// Duration returns a time.Duration appropriate for the backoff,
// incrementing the attempt counter.
func (b *Backoff) Duration() time.Duration {
b.setup()
b.decayN()
t := b.duration(b.n)
if b.n < math.MaxUint64 {
b.n++
}
if !b.noJitter {
prngMu.Lock()
t = time.Duration(prng.Int63n(int64(t)))
prngMu.Unlock()
}
return t
}
// requires b to be locked.
func (b *Backoff) duration(n uint64) (t time.Duration) {
// Saturate pow
pow := time.Duration(math.MaxInt64)
if n < 63 {
pow = 1 << n
}
t = b.interval * pow
if t/pow != b.interval || t > b.maxDuration {
t = b.maxDuration
}
return
}
// Reset resets the attempt counter of a backoff.
//
// It should be called when the rate-limited action succeeds.
func (b *Backoff) Reset() {
b.lastTry = time.Time{}
b.n = 0
}
// SetDecay sets the duration after which the try counter will be reset.
// Panics if decay is smaller than 0.
//
// The decay only kicks in if at least the last backoff + decay has elapsed
// since the last try.
func (b *Backoff) SetDecay(decay time.Duration) {
if decay < 0 {
panic("backoff: decay < 0")
}
b.decay = decay
}
// requires b to be locked
func (b *Backoff) decayN() {
if b.decay == 0 {
return
}
if b.lastTry.IsZero() {
b.lastTry = time.Now()
return
}
lastDuration := b.duration(b.n - 1)
decayed := time.Since(b.lastTry) > lastDuration+b.decay
b.lastTry = time.Now()
if !decayed {
return
}
b.n = 0
}

175
backoff/backoff_test.go Normal file
View File

@ -0,0 +1,175 @@
package backoff
import (
"fmt"
"math"
"testing"
"time"
)
// If given New with 0's and no jitter, ensure that certain invariants are met:
//
// - the default max duration and interval should be used
// - noJitter should be true
// - the RNG should not be initialised
// - the first duration should be equal to the default interval
func TestDefaults(t *testing.T) {
b := NewWithoutJitter(0, 0)
if b.maxDuration != DefaultMaxDuration {
t.Fatalf("expected new backoff to use the default max duration (%s), but have %s", DefaultMaxDuration, b.maxDuration)
}
if b.interval != DefaultInterval {
t.Fatalf("exepcted new backoff to use the default interval (%s), but have %s", DefaultInterval, b.interval)
}
if b.noJitter != true {
t.Fatal("backoff should have been initialised without jitter")
}
dur := b.Duration()
if dur != DefaultInterval {
t.Fatalf("expected first duration to be %s, have %s", DefaultInterval, dur)
}
}
// Given a zero-value initialised Backoff, it should be transparently
// setup.
func TestSetup(t *testing.T) {
b := new(Backoff)
dur := b.Duration()
if dur < 0 || dur > (5*time.Minute) {
t.Fatalf("want duration between 0 and 5 minutes, have %s", dur)
}
}
// Ensure that tries incremenets as expected.
func TestTries(t *testing.T) {
b := NewWithoutJitter(5, 1)
for i := uint64(0); i < 3; i++ {
if b.n != i {
t.Fatalf("want tries=%d, have tries=%d", i, b.n)
}
pow := 1 << i
expected := time.Duration(pow)
dur := b.Duration()
if dur != expected {
t.Fatalf("want duration=%d, have duration=%d at i=%d", expected, dur, i)
}
}
for i := uint(3); i < 5; i++ {
dur := b.Duration()
if dur != 5 {
t.Fatalf("want duration=5, have %d at i=%d", dur, i)
}
}
}
// Ensure that a call to Reset will actually reset the Backoff.
func TestReset(t *testing.T) {
const iter = 10
b := New(1000, 1)
for i := 0; i < iter; i++ {
_ = b.Duration()
}
if b.n != iter {
t.Fatalf("expected tries=%d, have tries=%d", iter, b.n)
}
b.Reset()
if b.n != 0 {
t.Fatalf("expected tries=0 after reset, have tries=%d", b.n)
}
}
const decay = 5 * time.Millisecond
const max = 10 * time.Millisecond
const interval = time.Millisecond
func TestDecay(t *testing.T) {
const iter = 10
b := NewWithoutJitter(max, 1)
b.SetDecay(decay)
var backoff time.Duration
for i := 0; i < iter; i++ {
backoff = b.Duration()
}
if b.n != iter {
t.Fatalf("expected tries=%d, have tries=%d", iter, b.n)
}
// Don't decay below backoff
b.lastTry = time.Now().Add(-backoff + 1)
backoff = b.Duration()
if b.n != iter+1 {
t.Fatalf("expected tries=%d, have tries=%d", iter+1, b.n)
}
// Reset after backoff + decay
b.lastTry = time.Now().Add(-backoff - decay)
b.Duration()
if b.n != 1 {
t.Fatalf("expected tries=%d, have tries=%d", 1, b.n)
}
}
// Ensure that decay works even if the retry counter is saturated.
func TestDecaySaturation(t *testing.T) {
b := NewWithoutJitter(1<<2, 1)
b.SetDecay(decay)
var duration time.Duration
for i := 0; i <= 2; i++ {
duration = b.Duration()
}
if duration != 1<<2 {
t.Fatalf("expected duration=%v, have duration=%v", 1<<2, duration)
}
b.lastTry = time.Now().Add(-duration - decay)
b.n = math.MaxUint64
duration = b.Duration()
if duration != 1 {
t.Errorf("expected duration=%v, have duration=%v", 1, duration)
}
}
func ExampleBackoff_SetDecay() {
b := NewWithoutJitter(max, interval)
b.SetDecay(decay)
// try 0
fmt.Println(b.Duration())
// try 1
fmt.Println(b.Duration())
// try 2
duration := b.Duration()
fmt.Println(duration)
// try 3, below decay
time.Sleep(duration)
duration = b.Duration()
fmt.Println(duration)
// try 4, resets
time.Sleep(duration + decay)
fmt.Println(b.Duration())
// Output: 1ms
// 2ms
// 4ms
// 8ms
// 1ms
}