getting rid of legacy logging system

Author: sibiryakov

docs/source/topics/frontera-settings.rst

@@ -131,16 +131,6 @@ Delay between calls to backend for new batches in Scrapy scheduler, when queue s
 without hitting backend on every request. Increase it if calls to your backend is taking too long, and decrease
 if you need a fast spider bootstrap from seeds.
 
-
-.. setting:: EVENT_LOGGER
-
-EVENT_LOGGER
-------------
-
-Default: ``'frontera.logger.events.EventLogManager'``
-
-The EventLoggerManager class to be used by the Frontier.
-
 .. setting:: KAFKA_GET_TIMEOUT
 
 KAFKA_GET_TIMEOUT
@@ -150,15 +140,6 @@ Default: ``5.0``
 
 Time process should block until requested amount of data will be received from message bus.
 
-.. setting:: LOGGER
-
-LOGGER
-------
-
-Default: ``'frontera.logger.FrontierLogger'``
-
-The Logger class to be used by the Frontier.
-
 .. setting:: MAX_NEXT_REQUESTS
 
 MAX_NEXT_REQUESTS
@@ -638,7 +619,6 @@ Logging default settings
 
 Values::
 
-    LOGGER = 'frontera.logger.FrontierLogger'
     LOGGING_ENABLED = True
 
     LOGGING_EVENTS_ENABLED = False
@@ -667,5 +647,3 @@ Values::
         "frontera.logger.handlers.COLOR_CONSOLE_DEBUGGING",
     ]
 
-    EVENT_LOG_MANAGER = 'frontera.logger.events.EventLogManager'
-

frontera/contrib/backends/hbase.py

@@ -17,6 +17,7 @@
 from frontera.core.models import Request
 from frontera.contrib.backends.partitioners import Crc32NamePartitioner
 from frontera.utils.misc import chunks
+import logging
 
 
 _pack_functions = {
@@ -60,11 +61,11 @@ class HBaseQueue(Queue):
 
     GET_RETRIES = 3
 
-    def __init__(self, connection, partitions, logger, table_name, drop=False):
+    def __init__(self, connection, partitions, table_name, drop=False):
         self.connection = connection
         self.partitions = [i for i in range(0, partitions)]
         self.partitioner = Crc32NamePartitioner(self.partitions)
-        self.logger = logger
+        self.logger = logging.getLogger("hbase.queue")
         self.table_name = table_name
 
         tables = set(self.connection.tables())
@@ -88,7 +89,7 @@ def schedule(self, batch):
                 if 'domain' not in request.meta:
                     _, hostname, _, _, _, _ = parse_domain_from_url_fast(request.url)
                     if not hostname:
-                        self.logger.error("Can't get hostname for URL %s, fingerprint %s" % (request.url, fprint))
+                        self.logger.error("Can't get hostname for URL %s, fingerprint %s", request.url, fprint)
                     request.meta['domain'] = {'name': hostname}
                 to_schedule.append((score, fprint, request.meta['domain'], request.url))
         self._schedule(to_schedule)
@@ -183,7 +184,8 @@ def get_next_requests(self, max_n_requests, partition_id, **kwargs):
         while tries < self.GET_RETRIES:
             tries += 1
             limit *= 5.5 if tries > 1 else 1.0
-            self.logger.debug("Try %d, limit %d, last attempt: requests %d, hosts %d" % (tries, limit, count, len(queue.keys())))
+            self.logger.debug("Try %d, limit %d, last attempt: requests %d, hosts %d",
+                              tries, limit, count, len(queue.keys()))
             meta_map.clear()
             queue.clear()
             count = 0
@@ -213,7 +215,7 @@ def get_next_requests(self, max_n_requests, partition_id, **kwargs):
                 continue
             break
 
-        self.logger.debug("Finished: tries %d, hosts %d, requests %d" % (tries, len(queue.keys()), count))
+        self.logger.debug("Finished: tries %d, hosts %d, requests %d", tries, len(queue.keys()), count)
 
         # For every fingerprint collect it's row keys and return all fingerprints from them
         fprint_map = {}
@@ -241,7 +243,7 @@ def get_next_requests(self, max_n_requests, partition_id, **kwargs):
         with table.batch(transaction=True) as b:
             for rk in trash_can:
                 b.delete(rk)
-        self.logger.debug("%d row keys removed" % (len(trash_can)))
+        self.logger.debug("%d row keys removed", len(trash_can))
         return results
 
     def count(self):
@@ -250,10 +252,10 @@ def count(self):
 
 class HBaseState(States):
 
-    def __init__(self, connection, table_name, logger, cache_size_limit):
+    def __init__(self, connection, table_name, cache_size_limit):
         self.connection = connection
         self._table_name = table_name
-        self.logger = logger
+        self.logger = logging.getLogger("hbase.states")
         self._state_cache = {}
         self._cache_size_limit = cache_size_limit
 
@@ -373,7 +375,7 @@ class HBaseBackend(DistributedBackend):
 
     def __init__(self, manager):
         self.manager = manager
-        self.logger = manager.logger.backend
+        self.logger = logging.getLogger("hbase.backend")
         settings = manager.settings
         port = settings.get('HBASE_THRIFT_PORT')
         hosts = settings.get('HBASE_THRIFT_HOST')
@@ -400,7 +402,7 @@ def __init__(self, manager):
     def strategy_worker(cls, manager):
         o = cls(manager)
         settings = manager.settings
-        o._states = HBaseState(o.connection, settings.get('HBASE_METADATA_TABLE'), o.manager.logger.backend,
+        o._states = HBaseState(o.connection, settings.get('HBASE_METADATA_TABLE'),
                                settings.get('HBASE_STATE_CACHE_SIZE_LIMIT'))
         return o
 
@@ -409,7 +411,7 @@ def db_worker(cls, manager):
         o = cls(manager)
         settings = manager.settings
         drop_all_tables = settings.get('HBASE_DROP_ALL_TABLES')
-        o._queue = HBaseQueue(o.connection, o.queue_partitions, o.manager.logger.backend,
+        o._queue = HBaseQueue(o.connection, o.queue_partitions,
                               settings.get('HBASE_QUEUE_TABLE'), drop=drop_all_tables)
         o._metadata = HBaseMetadata(o.connection, settings.get('HBASE_METADATA_TABLE'), drop_all_tables,
                                     settings.get('HBASE_USE_SNAPPY'), settings.get('HBASE_BATCH_SIZE'),
@@ -461,5 +463,5 @@ def get_next_requests(self, max_next_requests, **kwargs):
             results = self.queue.get_next_requests(max_next_requests, partition_id, min_requests=64,
                                                    min_hosts=24, max_requests_per_host=128)
             next_pages.extend(results)
-            self.logger.debug("Got %d requests for partition id %d" % (len(results), partition_id))
+            self.logger.debug("Got %d requests for partition id %d", len(results), partition_id)
         return next_pages

frontera/contrib/backends/remote/messagebus.py

@@ -3,12 +3,12 @@
 from frontera.core import OverusedBuffer
 from codecs.msgpack import Encoder, Decoder
 from frontera.utils.misc import load_object
+import logging
 
 
 class MessageBusBackend(Backend):
     def __init__(self, manager):
-        self._manager = manager
-        settings = self._manager.settings
+        settings = manager.settings
         messagebus = load_object(settings.get('MESSAGE_BUS'))
         self.mb = messagebus(settings)
         store_content = settings.get('STORE_CONTENT')
@@ -21,6 +21,7 @@ def __init__(self, manager):
         self._get_timeout = float(settings.get('KAFKA_GET_TIMEOUT'))
         self._buffer = OverusedBuffer(self._get_next_requests,
                                       manager.logger.manager.debug)
+        self._logger = logging.getLogger("messagebus-backend")
 
     @classmethod
     def from_manager(clas, manager):
@@ -47,8 +48,7 @@ def _get_next_requests(self, max_n_requests, **kwargs):
             try:
                 request = self._decoder.decode_request(encoded)
             except Exception, exc:
-                self._manager.logger.backend.warning("Could not decode message: {0}, error {1}".format(encoded,
-                                                                                                       str(exc)))
+                self._logger.warning("Could not decode message: {0}, error {1}".format(encoded, str(exc)))
             else:
                 requests.append(request)
         self.spider_log_producer.send('0123456789abcdef0123456789abcdef012345678',

frontera/contrib/messagebus/kafkabus.py

@@ -12,7 +12,7 @@
 from logging import getLogger
 from time import sleep
 
-logger = getLogger("kafkabus")
+logger = getLogger("messagebus.kafka")
 
 
 class Consumer(BaseStreamConsumer):

frontera/contrib/messagebus/zeromq/__init__.py

@@ -143,7 +143,7 @@ def send(self, key, *messages):
         self.counters[0] = counter
 
 
-class ScorinLogStream(BaseScoringLogStream):
+class ScoringLogStream(BaseScoringLogStream):
     def __init__(self, messagebus):
         self.context = messagebus.context
         self.in_location = messagebus.socket_config.sw_out()
@@ -215,7 +215,7 @@ def spider_log(self):
         return SpiderLogStream(self)
 
     def scoring_log(self):
-        return ScorinLogStream(self)
+        return ScoringLogStream(self)
 
     def spider_feed(self):
         return SpiderFeedStream(self)