Improve fixture for collection/frontier

Author: vshlapakov

tests/hubstorage/conftest.py

@@ -30,37 +30,46 @@ def hsproject(hsclient):
     return hsclient.get_project(TEST_PROJECT_ID)
 
 
+#@my_vcr.use_cassette()
 @pytest.fixture(scope='session')
 def hsspiderid(hsproject):
     return str(hsproject.ids.spider(TEST_SPIDER_NAME, create=1))
 
 
-@pytest.fixture(scope='session')
-def hscollection(hsproject):
-    return hsproject.collections.new_store(TEST_COLLECTION_NAME)
-
 
-@my_vcr.use_cassette()
+#@my_vcr.use_cassette()
 @pytest.fixture(autouse=True, scope='session')
 def setup_session(hsclient, hsproject, hscollection):
-    _set_testbotgroup(hsproject)
+    set_testbotgroup(hsproject)
     yield
-    _unset_testbotgroup(hsproject)
+    remove_all_jobs(hsproject)
+    unset_testbotgroup(hsproject)
     hsclient.close()
 
 
 @pytest.fixture(autouse=True)
-def setup_vcrpy_per_test(request, hsproject, hscollection):
+def setup_vcrpy_per_test(request, hsproject):
     # generates names like "test_module/test_function.yaml"
     # vcrpy creates the cassetes automatically under VCR_CASSETES_DIR
     cassette_name = '{}/{}.yaml'.format(
         request.function.__module__.split('.')[-1],
         request.function.__name__
     )
-    with my_vcr.use_cassette(cassette_name):
-        clean_environment(hsproject, hscollection)
-        yield
-        clean_environment(hsproject, hscollection)
+    #with my_vcr.use_cassette(cassette_name):
+    remove_all_jobs(hsproject)
+    yield
+
+
+#@my_vcr.use_cassette()
+@pytest.fixture(scope='session')
+def hscollection(hsproject):
+    collection = hsproject.collections.new_store(TEST_COLLECTION_NAME)
+    for item in collection.iter_values():
+        collection.delete(item['_key'])
+    yield collection
+    # drop all items in test collection
+    for item in collection.iter_values():
+        collection.delete(item['_key'])
 
 # ----------------------------------------------------------------------------
 
@@ -76,17 +85,7 @@ def start_job(hsproject, **startparams):
 # Clean environment section
 
 
-def clean_environment(hsproject, hscollection):
-    _remove_all_jobs(hsproject)
-    # drop all items in test collection
-    for item in hscollection.iter_values():
-        hscollection.delete(item['_key'])
-    # delete frontier slot
-    frontier = hsproject.frontier
-    frontier.delete_slot(TEST_FRONTIER_NAME, TEST_FRONTIER_SLOT)
-
-
-def _remove_all_jobs(hsproject):
+def remove_all_jobs(hsproject):
     for k in list(hsproject.settings.keys()):
         if k != 'botgroups':
             del hsproject.settings[k]
@@ -113,15 +112,15 @@ def _remove_job(hsproject, jobkey):
 # Botgroups helpers section
 
 
-def _set_testbotgroup(hsproject):
+def set_testbotgroup(hsproject):
     hsproject.settings.apipost(jl={'botgroups': [TEST_BOTGROUP]})
     # Additional step to populate JobQ's botgroups table
     url = urlpathjoin(TEST_ENDPOINT, 'botgroups', TEST_BOTGROUP, 'max_running')
     requests.post(url, auth=hsproject.auth, data='null')
     hsproject.settings.expire()
 
 
-def _unset_testbotgroup(hsproject):
+def unset_testbotgroup(hsproject):
     hsproject.settings.apidelete('botgroups')
     hsproject.settings.expire()
     # Additional step to delete botgroups in JobQ

tests/hubstorage/test_frontier.py

@@ -1,9 +1,16 @@
 """
 Test Frontier
 """
+import pytest
 from .conftest import TEST_FRONTIER_NAME, TEST_FRONTIER_SLOT
 
 
+@pytest.fixture(autouse=True)
+def delete_frontier_slot(hsproject):
+    frontier = hsproject.frontier
+    frontier.delete_slot(TEST_FRONTIER_NAME, TEST_FRONTIER_SLOT)
+
+
 def _get_urls(batch):
     return [r[0] for r in batch['requests']]
 

tests/hubstorage/test_project.py

@@ -1,10 +1,8 @@
 """
 Test Project
 """
-import json
-from random import randint, random
-
 import six
+import json
 import pytest
 from six.moves import range
 from requests.exceptions import HTTPError
@@ -14,6 +12,7 @@
 from .conftest import TEST_PROJECT_ID, TEST_SPIDER_NAME
 from .conftest import hsspiderid
 from .conftest import start_job
+from .conftest import set_testbotgroup
 from .testutil import failing_downloader
 
 
@@ -157,7 +156,13 @@ def test_broad(hsproject, hsspiderid):
     job.purged()
 
 
-def test_settings(hsproject):
+@pytest.fixture
+def restore_botgroup(hsproject):
+    yield
+    set_testbotgroup(hsproject)
+
+
+def test_settings(hsproject, restore_botgroup):
     settings = dict(hsproject.settings)
     settings.pop('botgroups', None)  # ignore testsuite botgroups
     assert settings == {}