add aws_expo to support Full Jitter

backoff.py

@@ -171,7 +171,7 @@ def emit(self, record):
 logger.setLevel(logging.ERROR)
 
 
-def expo(base=2, max_value=None):
+def expo(init_value=1, base=2, max_value=None):
     """Generator for exponential decay.
 
     Args:
@@ -182,7 +182,7 @@ def expo(base=2, max_value=None):
     """
     n = 0
     while True:
-        a = base ** n
+        a = init_value * base ** n
         if max_value is None or a < max_value:
             yield a
             n += 1
@@ -218,10 +218,22 @@ def constant(interval=1):
         yield interval
 
 
+def random_jitter(value):
+    return value+random.random()
+
+
+def full_jitter(value):
+    return random.uniform(0, value)
+
+
+def equal_jitter(value):
+    return (value/2.0) + (random.uniform(0, value/2.0))
+
+
 def on_predicate(wait_gen,
                  predicate=operator.not_,
                  max_tries=None,
-                 jitter=random.random,
+                 jitter=full_jitter,
                  on_success=None,
                  on_backoff=None,
                  on_giveup=None,
@@ -239,11 +251,10 @@ def on_predicate(wait_gen,
             up. In the case of failure, the result of the last attempt
             will be returned.  The default value of None means their
             is no limit to the number of tries.
-        jitter: Callable returning an offset in seconds to add to the
-            value yielded by wait_gen. When used with the default
-            random function, this staggers wait times a random number
-            of milliseconds to help spread out load in the case that
-            there are multiple simultaneous retries occuring.
+        jitter: Callable returning an offset to the value yielded by wait_gen.
+            This staggers wait times a random number of milliseconds to help
+            spread out load in the case that there are multiple simultaneous
+            retries occuring.
         on_success: Callable (or iterable of callables) with a unary
             signature to be called in the event of success. The
             parameter is a dict containing details about the invocation.
@@ -281,7 +292,16 @@ def retry(*args, **kwargs):
                                   'value': ret})
                         break
 
-                    seconds = next(wait) + jitter()
+                    value = next(wait)
+                    try:
+                        if jitter is not None:
+                            seconds = jitter(value)
+                        else:
+                            seconds = value
+                    except TypeError:
+                        # support deprecated nullary jitter function signature
+                        # which returns a delta rather than a jittered value
+                        seconds = value + jitter()
 
                     for hdlr in backoff_hdlrs:
                         hdlr({'target': target,
@@ -313,7 +333,7 @@ def retry(*args, **kwargs):
 def on_exception(wait_gen,
                  exception,
                  max_tries=None,
-                 jitter=random.random,
+                 jitter=full_jitter,
                  on_success=None,
                  on_backoff=None,
                  on_giveup=None,
@@ -329,11 +349,10 @@ def on_exception(wait_gen,
             up. Once exhausted, the exception will be allowed to escape.
             The default value of None means their is no limit to the
             number of tries.
-        jitter: Callable returning an offset in seconds to add to the
-            value yielded by wait_gen. When used with the default
-            random function, this staggers wait times a random number
-            of milliseconds to help spread out load in the case that
-            there are multiple simultaneous retries occuring.
+        jitter: Callable returning an offset to the value yielded by wait_gen.
+            This staggers wait times a random number of milliseconds to help
+            spread out load in the case that there are multiple simultaneous
+            retries occuring.
         on_success: Callable (or iterable of callables) with a unary
             signature to be called in the event of success. The
             parameter is a dict containing details about the invocation.
@@ -371,7 +390,16 @@ def retry(*args, **kwargs):
                                   'tries': tries})
                         raise
 
-                    seconds = next(wait) + jitter()
+                    value = next(wait)
+                    try:
+                        if jitter is not None:
+                            seconds = jitter(value)
+                        else:
+                            seconds = value
+                    except TypeError:
+                        # support deprecated nullary jitter function signature
+                        # which returns a delta rather than a jittered value
+                        seconds = value + jitter()
 
                     for hdlr in backoff_hdlrs:
                         hdlr({'target': target,

backoff_tests.py

@@ -4,6 +4,23 @@
 import collections
 import functools
 import pytest
+import random
+
+
+def test_full_jitter():
+    for input in range(100):
+        for i in range(100):
+            jitter = backoff.full_jitter(input)
+        assert jitter >= 0
+        assert jitter <= input
+
+
+def test_equal_jitter():
+    for input in range(100):
+        for i in range(100):
+            jitter = backoff.equal_jitter(input)
+            assert jitter >= input/2.0
+            assert jitter <= input
 
 
 def test_expo():
@@ -18,6 +35,18 @@ def test_expo_base3():
         assert 3 ** i == next(gen)
 
 
+def test_expo_init3():
+    gen = backoff.expo(init_value=3)
+    for i in range(9):
+        assert 3 * 2 ** i == next(gen)
+
+
+def test_expo_base3_init5():
+    gen = backoff.expo(base=3, init_value=5)
+    for i in range(9):
+        assert 5 * 3 ** i == next(gen)
+
+
 def test_expo_max_value():
     gen = backoff.expo(max_value=2 ** 4)
     expected = [1, 2, 4, 8, 16, 16, 16]
@@ -63,7 +92,7 @@ def return_true(log, n):
 def test_on_predicate_max_tries(monkeypatch):
     monkeypatch.setattr('time.sleep', lambda x: None)
 
-    @backoff.on_predicate(backoff.expo, max_tries=3)
+    @backoff.on_predicate(backoff.expo, jitter=None, max_tries=3)
     def return_true(log, n):
         val = (len(log) == n)
         log.append(val)
@@ -115,7 +144,7 @@ def keyerror_valueerror_then_true(log):
 def test_on_exception_max_tries(monkeypatch):
     monkeypatch.setattr('time.sleep', lambda x: None)
 
-    @backoff.on_exception(backoff.expo, KeyError, max_tries=3)
+    @backoff.on_exception(backoff.expo, KeyError, jitter=None, max_tries=3)
     def keyerror_then_true(log, n, foo=None):
         if len(log) == n:
             return True
@@ -167,6 +196,97 @@ def _save_target(f):
     return f
 
 
+def test_on_exception_success_random_jitter(monkeypatch):
+    monkeypatch.setattr('time.sleep', lambda x: None)
+
+    log, log_success, log_backoff, log_giveup = _log_hdlrs()
+
+    @backoff.on_exception(backoff.expo,
+                          Exception,
+                          on_success=log_success,
+                          on_backoff=log_backoff,
+                          on_giveup=log_giveup,
+                          jitter=backoff.random_jitter,
+                          init_value=0.5)
+    @_save_target
+    def succeeder(*args, **kwargs):
+        # succeed after we've backed off twice
+        if len(log['backoff']) < 2:
+            raise ValueError("catch me")
+
+    succeeder(1, 2, 3, foo=1, bar=2)
+
+    # we try 3 times, backing off twice before succeeding
+    assert len(log['success']) == 1
+    assert len(log['backoff']) == 2
+    assert len(log['giveup']) == 0
+
+    for i in range(2):
+        details = log['backoff'][i]
+        assert details['wait'] >= 0.5 * 2 ** i
+
+
+def test_on_exception_success_full_jitter(monkeypatch):
+    monkeypatch.setattr('time.sleep', lambda x: None)
+
+    log, log_success, log_backoff, log_giveup = _log_hdlrs()
+
+    @backoff.on_exception(backoff.expo,
+                          Exception,
+                          on_success=log_success,
+                          on_backoff=log_backoff,
+                          on_giveup=log_giveup,
+                          jitter=backoff.full_jitter,
+                          init_value=0.5)
+    @_save_target
+    def succeeder(*args, **kwargs):
+        # succeed after we've backed off twice
+        if len(log['backoff']) < 2:
+            raise ValueError("catch me")
+
+    succeeder(1, 2, 3, foo=1, bar=2)
+
+    # we try 3 times, backing off twice before succeeding
+    assert len(log['success']) == 1
+    assert len(log['backoff']) == 2
+    assert len(log['giveup']) == 0
+
+    for i in range(2):
+        details = log['backoff'][i]
+        assert details['wait'] <= 0.5 * 2 ** i
+
+
+def test_on_exception_success_equal_jitter(monkeypatch):
+    monkeypatch.setattr('time.sleep', lambda x: None)
+
+    log, log_success, log_backoff, log_giveup = _log_hdlrs()
+
+    @backoff.on_exception(backoff.expo,
+                          Exception,
+                          on_success=log_success,
+                          on_backoff=log_backoff,
+                          on_giveup=log_giveup,
+                          jitter=backoff.equal_jitter,
+                          init_value=0.5)
+    @_save_target
+    def succeeder(*args, **kwargs):
+        # succeed after we've backed off twice
+        if len(log['backoff']) < 2:
+            raise ValueError("catch me")
+
+    succeeder(1, 2, 3, foo=1, bar=2)
+
+    # we try 3 times, backing off twice before succeeding
+    assert len(log['success']) == 1
+    assert len(log['backoff']) == 2
+    assert len(log['giveup']) == 0
+
+    for i in range(2):
+        details = log['backoff'][i]
+        assert details['wait'] >= (0.5 * 2 ** i) / 2.0
+        assert details['wait'] <= 0.5 * 2 ** i
+
+
 def test_on_exception_success():
     log, log_success, log_backoff, log_giveup = _log_hdlrs()
 
@@ -329,3 +449,89 @@ def emptiness(*args, **kwargs):
                            'target': emptiness._target,
                            'tries': 3,
                            'value': None}
+
+
+# To maintain backward compatibility,
+# on_predicate should support 0-argument jitter function.
+def test_on_exception_success_0_arg_jitter(monkeypatch):
+    monkeypatch.setattr('time.sleep', lambda x: None)
+    monkeypatch.setattr('random.random', lambda: 0)
+
+    log, log_success, log_backoff, log_giveup = _log_hdlrs()
+
+    @backoff.on_exception(backoff.constant,
+                          Exception,
+                          on_success=log_success,
+                          on_backoff=log_backoff,
+                          on_giveup=log_giveup,
+                          jitter=random.random,
+                          interval=0)
+    @_save_target
+    def succeeder(*args, **kwargs):
+        # succeed after we've backed off twice
+        if len(log['backoff']) < 2:
+            raise ValueError("catch me")
+
+    succeeder(1, 2, 3, foo=1, bar=2)
+
+    # we try 3 times, backing off twice before succeeding
+    assert len(log['success']) == 1
+    assert len(log['backoff']) == 2
+    assert len(log['giveup']) == 0
+
+    for i in range(2):
+        details = log['backoff'][i]
+        assert details == {'args': (1, 2, 3),
+                           'kwargs': {'foo': 1, 'bar': 2},
+                           'target': succeeder._target,
+                           'tries': i + 1,
+                           'wait': 0}
+
+    details = log['success'][0]
+    assert details == {'args': (1, 2, 3),
+                       'kwargs': {'foo': 1, 'bar': 2},
+                       'target': succeeder._target,
+                       'tries': 3}
+
+
+# To maintain backward compatibility,
+# on_predicate should support 0-argument jitter function.
+def test_on_predicate_success_0_arg_jitter(monkeypatch):
+    monkeypatch.setattr('time.sleep', lambda x: None)
+    monkeypatch.setattr('random.random', lambda: 0)
+
+    log, log_success, log_backoff, log_giveup = _log_hdlrs()
+
+    @backoff.on_predicate(backoff.constant,
+                          on_success=log_success,
+                          on_backoff=log_backoff,
+                          on_giveup=log_giveup,
+                          jitter=random.random,
+                          interval=0)
+    @_save_target
+    def success(*args, **kwargs):
+        # succeed after we've backed off twice
+        return len(log['backoff']) == 2
+
+    success(1, 2, 3, foo=1, bar=2)
+
+    # we try 3 times, backing off twice before succeeding
+    assert len(log['success']) == 1
+    assert len(log['backoff']) == 2
+    assert len(log['giveup']) == 0
+
+    for i in range(2):
+        details = log['backoff'][i]
+        assert details == {'args': (1, 2, 3),
+                           'kwargs': {'foo': 1, 'bar': 2},
+                           'target': success._target,
+                           'tries': i + 1,
+                           'value': False,
+                           'wait': 0}
+
+    details = log['success'][0]
+    assert details == {'args': (1, 2, 3),
+                       'kwargs': {'foo': 1, 'bar': 2},
+                       'target': success._target,
+                       'tries': 3,
+                       'value': True}