Rm some redundant clean procedures, fix tests

Author: vshlapakov

tests/hubstorage/conftest.py

@@ -71,6 +71,11 @@ def pytest_configure(config):
         my_vcr.before_record_request = lambda request: None
 
 
+def is_using_real_services(request):
+    return (request.config.option.update_cassettes or
+            request.config.option.ignore_cassettes)
+
+
 @pytest.fixture(scope='session')
 def hsclient():
     return HubstorageClient(auth=TEST_AUTH, endpoint=TEST_ENDPOINT)
@@ -87,23 +92,20 @@ def hsspiderid(hsproject):
     return str(hsproject.ids.spider(TEST_SPIDER_NAME, create=1))
 
 
-@my_vcr.use_cassette()
 @pytest.fixture(scope='session')
-def hscollection(hsproject):
+def hscollection(hsproject, request):
     collection = get_test_collection(hsproject)
-    clean_collection(collection)
+    if is_using_real_services(request):
+        clean_collection(collection)
     yield collection
-    clean_collection(collection)
 
 
-@my_vcr.use_cassette()
 @pytest.fixture(autouse=True, scope='session')
-def setup_session(hsclient, hsproject, hscollection):
-    set_testbotgroup(hsproject)
-    remove_all_jobs(hsproject)
+def setup_session(hsclient, hsproject, hscollection, request):
+    if is_using_real_services(request):
+        set_testbotgroup(hsproject)
+        remove_all_jobs(hsproject)
     yield
-    remove_all_jobs(hsproject)
-    unset_testbotgroup(hsproject)
     hsclient.close()
 
 
@@ -116,10 +118,7 @@ def setup_vcrpy(request, hsproject):
         request.function.__module__.split('.')[-1],
         request.function.__name__
     )
-    # we should clean jobs only when working with real services
-    # doesn't make sense to clean jobs when working with cassettes
-    if (request.config.option.update_cassettes or
-            request.config.option.ignore_cassettes):
+    if is_using_real_services(request):
         remove_all_jobs(hsproject)
     with my_vcr.use_cassette(cassette_name):
         yield
@@ -145,14 +144,11 @@ def remove_all_jobs(hsproject):
             del hsproject.settings[k]
     hsproject.settings.save()
 
-    # Cleanup JobQ
-    jobq = hsproject.jobq
-    for queuename in ('pending', 'running', 'finished'):
-        info = {'summary': [None]}  # poor-guy do...while
-        while info['summary']:
-            info = jobq.summary(queuename)
-            for summary in info['summary']:
-                _remove_job(hsproject, summary['key'])
+    # Cleanup JobQ: run 2 times to ensure we covered all jobs
+    for queuename in ('pending', 'running', 'finished')*2:
+        info = hsproject.jobq.summary(queuename)
+        for summary in info['summary']:
+            _remove_job(hsproject, summary['key'])
 
 
 def _remove_job(hsproject, jobkey):

tests/hubstorage/test_project.py

@@ -12,7 +12,7 @@
 from .conftest import TEST_PROJECT_ID, TEST_SPIDER_NAME
 from .conftest import hsspiderid
 from .conftest import start_job
-from .conftest import set_testbotgroup
+from .conftest import set_testbotgroup, unset_testbotgroup
 from .testutil import failing_downloader
 
 
@@ -157,14 +157,14 @@ def test_broad(hsproject, hsspiderid):
 
 
 @pytest.fixture
-def restore_botgroup(hsproject):
+def unset_botgroup(hsproject):
+    unset_testbotgroup(hsproject)
     yield
     set_testbotgroup(hsproject)
 
 
-def test_settings(hsproject, restore_botgroup):
+def test_settings(hsproject, unset_botgroup):
     settings = dict(hsproject.settings)
-    settings.pop('botgroups', None)  # ignore testsuite botgroups
     assert settings == {}
     # use some fixed timestamp to represent current time
     hsproject.settings['created'] = created = 1476803148638

tests/hubstorage/test_retry.py

@@ -4,6 +4,7 @@
 import json
 import re
 
+from mock import patch
 import responses
 from requests import HTTPError, ConnectionError
 from scrapinghub import HubstorageClient
@@ -18,13 +19,25 @@
 DELETE = responses.DELETE
 
 
+@patch.multiple('scrapinghub.HubstorageClient',
+                RETRY_DEFAULT_JITTER_MS=1,
+                RETRY_DEFAULT_EXPONENTIAL_BACKOFF_MS=1)
 def hsclient_with_retries(max_retries=3, max_retry_time=1):
     return HubstorageClient(
         auth=TEST_AUTH, endpoint=TEST_ENDPOINT,
         max_retries=max_retries, max_retry_time=max_retry_time,
     )
 
 
+@patch('scrapinghub.hubstorage.client.Retrying')
+def test_hsclient_with_retries_no_wait(retrying_class):
+    hsclient_with_retries()
+    assert retrying_class.call_count == 1
+    call = retrying_class.call_args_list[0]
+    assert call[1]['wait_jitter_max'] == 1
+    assert int(call[1]['wait_exponential_multiplier']) == 33
+
+
 def mock_api(method=GET, callback=None, url_match='/.*'):
     """
     Mock an API URL using the responses library.