diff --git a/backoff.py b/backoff.py index b189fe2..280b0ad 100644 --- a/backoff.py +++ b/backoff.py @@ -171,7 +171,7 @@ def emit(self, record): logger.setLevel(logging.ERROR) -def expo(base=2, max_value=None): +def expo(init_value=1, base=2, max_value=None): """Generator for exponential decay. Args: @@ -182,7 +182,7 @@ def expo(base=2, max_value=None): """ n = 0 while True: - a = base ** n + a = init_value * base ** n if max_value is None or a < max_value: yield a n += 1 @@ -218,10 +218,22 @@ def constant(interval=1): yield interval +def random_jitter(value): + return value+random.random() + + +def full_jitter(value): + return random.uniform(0, value) + + +def equal_jitter(value): + return (value/2.0) + (random.uniform(0, value/2.0)) + + def on_predicate(wait_gen, predicate=operator.not_, max_tries=None, - jitter=random.random, + jitter=full_jitter, on_success=None, on_backoff=None, on_giveup=None, @@ -239,11 +251,10 @@ def on_predicate(wait_gen, up. In the case of failure, the result of the last attempt will be returned. The default value of None means their is no limit to the number of tries. - jitter: Callable returning an offset in seconds to add to the - value yielded by wait_gen. When used with the default - random function, this staggers wait times a random number - of milliseconds to help spread out load in the case that - there are multiple simultaneous retries occuring. + jitter: Callable returning an offset to the value yielded by wait_gen. + This staggers wait times a random number of milliseconds to help + spread out load in the case that there are multiple simultaneous + retries occuring. on_success: Callable (or iterable of callables) with a unary signature to be called in the event of success. The parameter is a dict containing details about the invocation. @@ -281,7 +292,16 @@ def retry(*args, **kwargs): 'value': ret}) break - seconds = next(wait) + jitter() + value = next(wait) + try: + if jitter is not None: + seconds = jitter(value) + else: + seconds = value + except TypeError: + # support deprecated nullary jitter function signature + # which returns a delta rather than a jittered value + seconds = value + jitter() for hdlr in backoff_hdlrs: hdlr({'target': target, @@ -313,7 +333,7 @@ def retry(*args, **kwargs): def on_exception(wait_gen, exception, max_tries=None, - jitter=random.random, + jitter=full_jitter, on_success=None, on_backoff=None, on_giveup=None, @@ -329,11 +349,10 @@ def on_exception(wait_gen, up. Once exhausted, the exception will be allowed to escape. The default value of None means their is no limit to the number of tries. - jitter: Callable returning an offset in seconds to add to the - value yielded by wait_gen. When used with the default - random function, this staggers wait times a random number - of milliseconds to help spread out load in the case that - there are multiple simultaneous retries occuring. + jitter: Callable returning an offset to the value yielded by wait_gen. + This staggers wait times a random number of milliseconds to help + spread out load in the case that there are multiple simultaneous + retries occuring. on_success: Callable (or iterable of callables) with a unary signature to be called in the event of success. The parameter is a dict containing details about the invocation. @@ -371,7 +390,16 @@ def retry(*args, **kwargs): 'tries': tries}) raise - seconds = next(wait) + jitter() + value = next(wait) + try: + if jitter is not None: + seconds = jitter(value) + else: + seconds = value + except TypeError: + # support deprecated nullary jitter function signature + # which returns a delta rather than a jittered value + seconds = value + jitter() for hdlr in backoff_hdlrs: hdlr({'target': target, diff --git a/backoff_tests.py b/backoff_tests.py index 8fa94e3..e8b546e 100644 --- a/backoff_tests.py +++ b/backoff_tests.py @@ -4,6 +4,23 @@ import collections import functools import pytest +import random + + +def test_full_jitter(): + for input in range(100): + for i in range(100): + jitter = backoff.full_jitter(input) + assert jitter >= 0 + assert jitter <= input + + +def test_equal_jitter(): + for input in range(100): + for i in range(100): + jitter = backoff.equal_jitter(input) + assert jitter >= input/2.0 + assert jitter <= input def test_expo(): @@ -18,6 +35,18 @@ def test_expo_base3(): assert 3 ** i == next(gen) +def test_expo_init3(): + gen = backoff.expo(init_value=3) + for i in range(9): + assert 3 * 2 ** i == next(gen) + + +def test_expo_base3_init5(): + gen = backoff.expo(base=3, init_value=5) + for i in range(9): + assert 5 * 3 ** i == next(gen) + + def test_expo_max_value(): gen = backoff.expo(max_value=2 ** 4) expected = [1, 2, 4, 8, 16, 16, 16] @@ -63,7 +92,7 @@ def return_true(log, n): def test_on_predicate_max_tries(monkeypatch): monkeypatch.setattr('time.sleep', lambda x: None) - @backoff.on_predicate(backoff.expo, max_tries=3) + @backoff.on_predicate(backoff.expo, jitter=None, max_tries=3) def return_true(log, n): val = (len(log) == n) log.append(val) @@ -115,7 +144,7 @@ def keyerror_valueerror_then_true(log): def test_on_exception_max_tries(monkeypatch): monkeypatch.setattr('time.sleep', lambda x: None) - @backoff.on_exception(backoff.expo, KeyError, max_tries=3) + @backoff.on_exception(backoff.expo, KeyError, jitter=None, max_tries=3) def keyerror_then_true(log, n, foo=None): if len(log) == n: return True @@ -167,6 +196,97 @@ def _save_target(f): return f +def test_on_exception_success_random_jitter(monkeypatch): + monkeypatch.setattr('time.sleep', lambda x: None) + + log, log_success, log_backoff, log_giveup = _log_hdlrs() + + @backoff.on_exception(backoff.expo, + Exception, + on_success=log_success, + on_backoff=log_backoff, + on_giveup=log_giveup, + jitter=backoff.random_jitter, + init_value=0.5) + @_save_target + def succeeder(*args, **kwargs): + # succeed after we've backed off twice + if len(log['backoff']) < 2: + raise ValueError("catch me") + + succeeder(1, 2, 3, foo=1, bar=2) + + # we try 3 times, backing off twice before succeeding + assert len(log['success']) == 1 + assert len(log['backoff']) == 2 + assert len(log['giveup']) == 0 + + for i in range(2): + details = log['backoff'][i] + assert details['wait'] >= 0.5 * 2 ** i + + +def test_on_exception_success_full_jitter(monkeypatch): + monkeypatch.setattr('time.sleep', lambda x: None) + + log, log_success, log_backoff, log_giveup = _log_hdlrs() + + @backoff.on_exception(backoff.expo, + Exception, + on_success=log_success, + on_backoff=log_backoff, + on_giveup=log_giveup, + jitter=backoff.full_jitter, + init_value=0.5) + @_save_target + def succeeder(*args, **kwargs): + # succeed after we've backed off twice + if len(log['backoff']) < 2: + raise ValueError("catch me") + + succeeder(1, 2, 3, foo=1, bar=2) + + # we try 3 times, backing off twice before succeeding + assert len(log['success']) == 1 + assert len(log['backoff']) == 2 + assert len(log['giveup']) == 0 + + for i in range(2): + details = log['backoff'][i] + assert details['wait'] <= 0.5 * 2 ** i + + +def test_on_exception_success_equal_jitter(monkeypatch): + monkeypatch.setattr('time.sleep', lambda x: None) + + log, log_success, log_backoff, log_giveup = _log_hdlrs() + + @backoff.on_exception(backoff.expo, + Exception, + on_success=log_success, + on_backoff=log_backoff, + on_giveup=log_giveup, + jitter=backoff.equal_jitter, + init_value=0.5) + @_save_target + def succeeder(*args, **kwargs): + # succeed after we've backed off twice + if len(log['backoff']) < 2: + raise ValueError("catch me") + + succeeder(1, 2, 3, foo=1, bar=2) + + # we try 3 times, backing off twice before succeeding + assert len(log['success']) == 1 + assert len(log['backoff']) == 2 + assert len(log['giveup']) == 0 + + for i in range(2): + details = log['backoff'][i] + assert details['wait'] >= (0.5 * 2 ** i) / 2.0 + assert details['wait'] <= 0.5 * 2 ** i + + def test_on_exception_success(): log, log_success, log_backoff, log_giveup = _log_hdlrs() @@ -329,3 +449,89 @@ def emptiness(*args, **kwargs): 'target': emptiness._target, 'tries': 3, 'value': None} + + +# To maintain backward compatibility, +# on_predicate should support 0-argument jitter function. +def test_on_exception_success_0_arg_jitter(monkeypatch): + monkeypatch.setattr('time.sleep', lambda x: None) + monkeypatch.setattr('random.random', lambda: 0) + + log, log_success, log_backoff, log_giveup = _log_hdlrs() + + @backoff.on_exception(backoff.constant, + Exception, + on_success=log_success, + on_backoff=log_backoff, + on_giveup=log_giveup, + jitter=random.random, + interval=0) + @_save_target + def succeeder(*args, **kwargs): + # succeed after we've backed off twice + if len(log['backoff']) < 2: + raise ValueError("catch me") + + succeeder(1, 2, 3, foo=1, bar=2) + + # we try 3 times, backing off twice before succeeding + assert len(log['success']) == 1 + assert len(log['backoff']) == 2 + assert len(log['giveup']) == 0 + + for i in range(2): + details = log['backoff'][i] + assert details == {'args': (1, 2, 3), + 'kwargs': {'foo': 1, 'bar': 2}, + 'target': succeeder._target, + 'tries': i + 1, + 'wait': 0} + + details = log['success'][0] + assert details == {'args': (1, 2, 3), + 'kwargs': {'foo': 1, 'bar': 2}, + 'target': succeeder._target, + 'tries': 3} + + +# To maintain backward compatibility, +# on_predicate should support 0-argument jitter function. +def test_on_predicate_success_0_arg_jitter(monkeypatch): + monkeypatch.setattr('time.sleep', lambda x: None) + monkeypatch.setattr('random.random', lambda: 0) + + log, log_success, log_backoff, log_giveup = _log_hdlrs() + + @backoff.on_predicate(backoff.constant, + on_success=log_success, + on_backoff=log_backoff, + on_giveup=log_giveup, + jitter=random.random, + interval=0) + @_save_target + def success(*args, **kwargs): + # succeed after we've backed off twice + return len(log['backoff']) == 2 + + success(1, 2, 3, foo=1, bar=2) + + # we try 3 times, backing off twice before succeeding + assert len(log['success']) == 1 + assert len(log['backoff']) == 2 + assert len(log['giveup']) == 0 + + for i in range(2): + details = log['backoff'][i] + assert details == {'args': (1, 2, 3), + 'kwargs': {'foo': 1, 'bar': 2}, + 'target': success._target, + 'tries': i + 1, + 'value': False, + 'wait': 0} + + details = log['success'][0] + assert details == {'args': (1, 2, 3), + 'kwargs': {'foo': 1, 'bar': 2}, + 'target': success._target, + 'tries': 3, + 'value': True}