made suggested changes

Author: voith

frontera/contrib/backends/remote/codecs/json.py

@@ -9,33 +9,33 @@
 from w3lib.util import to_unicode, to_bytes
 
 
-def _convert(obj):
+def _encode_recursively(obj):
     """
-    recursively convert an object to unicode and store its type
+    recursively encodes an object to unicode and store its type
     """
     if isinstance(obj, bytes):
         return 'bytes', to_unicode(obj)
     elif isinstance(obj, dict):
-        return 'dict', [(_convert(k), _convert(v)) for k, v in six.iteritems(obj)]
+        return 'dict', [(_encode_recursively(k), _encode_recursively(v)) for k, v in six.iteritems(obj)]
     elif isinstance(obj, (list, tuple)):
-        return type(obj).__name__, [_convert(item) for item in obj]
+        return type(obj).__name__, [_encode_recursively(item) for item in obj]
     return 'other', obj
 
 
-def _reconvert(obj):
+def _decode_recursively(obj):
     """
-    reconvert an object changed by method `convert` back to
+    decode an object changed by method `_encode_recursively` back to
     its original form
     """
     obj_type = obj[0]
     obj_value = obj[1]
     if obj_type == 'bytes':
         return to_bytes(obj_value)
     elif obj_type == 'dict':
-        return dict([(_reconvert(k), _reconvert(v)) for k, v in obj_value])
+        return dict([(_decode_recursively(k), _decode_recursively(v)) for k, v in obj_value])
     elif obj_type in ['list', 'tuple']:
         _type = list if obj_type == 'list' else tuple
-        return _type([_reconvert(item) for item in obj_value])
+        return _type([_decode_recursively(item) for item in obj_value])
     return obj_value
 
 
@@ -76,8 +76,8 @@ def __init__(self, request_model, *a, **kw):
         super(Encoder, self).__init__(request_model, *a, **kw)
 
     def encode(self, obj):
-        converted = _convert(obj)
-        return super(Encoder, self).encode(converted)
+        encoded = _encode_recursively(obj)
+        return super(Encoder, self).encode(encoded)
 
     def encode_add_seeds(self, seeds):
         return self.encode({
@@ -151,7 +151,7 @@ def _request_from_object(self, obj):
                                    meta=obj['meta'])
 
     def decode(self, message):
-        message = _reconvert(super(Decoder, self).decode(message))
+        message = _decode_recursively(super(Decoder, self).decode(message))
         if message['type'] == 'links_extracted':
             request = self._request_from_object(message['r'])
             links = [self._request_from_object(link) for link in message['links']]
@@ -177,7 +177,7 @@ def decode(self, message):
         return TypeError('Unknown message type')
 
     def decode_request(self, message):
-        obj = _reconvert(super(Decoder, self).decode(message))
+        obj = _decode_recursively(super(Decoder, self).decode(message))
         return self._request_model(url=obj['url'],
                                    method=obj['method'],
                                    headers=obj['headers'],

frontera/contrib/backends/remote/codecs/msgpack.py

@@ -6,11 +6,29 @@
 from msgpack import packb, unpackb
 
 from frontera.core.codec import BaseDecoder, BaseEncoder
+import six
 from w3lib.util import to_native_str
 
 
 def _prepare_request_message(request):
-    return [request.url, request.method, request.headers, request.cookies, request.meta]
+    def serialize(obj):
+        """Recursively walk object's hierarchy."""
+        if isinstance(obj, (bool, six.integer_types, float, six.binary_type, six.text_type)):
+            return obj
+        elif isinstance(obj, dict):
+            obj = obj.copy()
+            for key in obj:
+                obj[key] = serialize(obj[key])
+            return obj
+        elif isinstance(obj, list):
+            return [serialize(item) for item in obj]
+        elif isinstance(obj, tuple):
+            return tuple(serialize([item for item in obj]))
+        elif hasattr(obj, '__dict__'):
+            return serialize(obj.__dict__)
+        else:
+            return None
+    return [request.url, request.method, request.headers, request.cookies, serialize(request.meta)]
 
 
 def _prepare_response_message(response, send_body):
@@ -22,29 +40,29 @@ def __init__(self, request_model, *a, **kw):
         self.send_body = True if 'send_body' in kw and kw['send_body'] else False
 
     def encode_add_seeds(self, seeds):
-        return packb([b'as', [_prepare_request_message(seed) for seed in seeds]], use_bin_type=True, encoding="utf-8")
+        return packb([b'as', [_prepare_request_message(seed) for seed in seeds]], use_bin_type=True)
 
     def encode_page_crawled(self, response):
-        return packb([b'pc', _prepare_response_message(response, self.send_body)], use_bin_type=True, encoding="utf-8")
+        return packb([b'pc', _prepare_response_message(response, self.send_body)], use_bin_type=True)
 
     def encode_links_extracted(self, request, links):
         return packb([b'le', _prepare_request_message(request), [_prepare_request_message(link) for link in links]],
-                     use_bin_type=True, encoding="utf-8")
+                     use_bin_type=True)
 
     def encode_request_error(self, request, error):
-        return packb([b're', _prepare_request_message(request), str(error)], use_bin_type=True, encoding="utf-8")
+        return packb([b're', _prepare_request_message(request), str(error)], use_bin_type=True)
 
     def encode_request(self, request):
-        return packb(_prepare_request_message(request), use_bin_type=True, encoding="utf-8")
+        return packb(_prepare_request_message(request), use_bin_type=True)
 
     def encode_update_score(self, request, score, schedule):
-        return packb([b'us', _prepare_request_message(request), score, schedule], use_bin_type=True, encoding="utf-8")
+        return packb([b'us', _prepare_request_message(request), score, schedule], use_bin_type=True)
 
     def encode_new_job_id(self, job_id):
-        return packb([b'njid', int(job_id)], use_bin_type=True, encoding="utf-8")
+        return packb([b'njid', int(job_id)], use_bin_type=True)
 
     def encode_offset(self, partition_id, offset):
-        return packb([b'of', int(partition_id), int(offset)], use_bin_type=True, encoding="utf-8")
+        return packb([b'of', int(partition_id), int(offset)], use_bin_type=True)
 
 
 class Decoder(BaseDecoder):
@@ -68,7 +86,7 @@ def _request_from_object(self, obj):
                                    meta=obj[4])
 
     def decode(self, buffer):
-        obj = unpackb(buffer, encoding="utf-8")
+        obj = unpackb(buffer, encoding='utf-8')
         if obj[0] == b'pc':
             return ('page_crawled',
                     self._response_from_object(obj[1]))
@@ -89,4 +107,4 @@ def decode(self, buffer):
         return TypeError('Unknown message type')
 
     def decode_request(self, buffer):
-        return self._request_from_object(unpackb(buffer, encoding="utf-8"))
+        return self._request_from_object(unpackb(buffer, encoding='utf-8'))

tests/test_codecs.py

@@ -5,12 +5,38 @@
 import unittest
 from frontera.contrib.backends.remote.codecs.json import (Encoder as JsonEncoder,
                                                           Decoder as JsonDecoder,
-                                                          _convert, _reconvert)
+                                                          _encode_recursively, _decode_recursively)
 from frontera.contrib.backends.remote.codecs.msgpack import Encoder as MsgPackEncoder, Decoder as MsgPackDecoder
 from frontera.core.models import Request, Response
 import pytest
 
 
+def _compare_dicts(dict1, dict2):
+    """
+     Compares two dicts
+    :return: True if both dicts are equal else False
+    """
+    if dict1 == None or dict2 == None:
+        return False
+
+    if type(dict1) is not dict or type(dict2) is not dict:
+        return False
+
+    shared_keys = set(dict2.keys()) & set(dict2.keys())
+
+    if not (len(shared_keys) == len(dict1.keys()) and len(shared_keys) == len(dict2.keys())):
+        return False
+
+    dicts_are_equal = True
+    for key in dict1.keys():
+        if type(dict1[key]) is dict:
+            dicts_are_equal = dicts_are_equal and _compare_dicts(dict1[key], dict2[key])
+        else:
+            dicts_are_equal = dicts_are_equal and (dict1[key] == dict2[key]) and (type(dict1[key]) == type(dict2[key]))
+
+    return dicts_are_equal
+
+
 @pytest.mark.parametrize(
     ('encoder', 'decoder'), [
         (MsgPackEncoder, MsgPackDecoder),
@@ -19,8 +45,8 @@
 )
 def test_codec(encoder, decoder):
     def check_request(req1, req2):
-        assert req1.url == req2.url and req1.meta == req2.meta and req1.headers == req2.headers \
-            and req1.method == req2.method
+        assert req1.url == req2.url and _compare_dicts(req1.meta, req2.meta) == True and \
+               _compare_dicts(req1.headers, req2.headers) == True and req1.method == req2.method
 
     enc = encoder(Request, send_body=True)
     dec = decoder(Request, Response)
@@ -87,12 +113,12 @@ def check_request(req1, req2):
     check_request(o, req)
 
 
-class TestConvertReconvertJson(unittest.TestCase):
+class TestEncodeDecodeJson(unittest.TestCase):
     """
-    Test for testing methods `_convert` and `_reconvert` used in json codec
+    Test for testing methods `_encode_recursively` and `_decode_recursively` used in json codec
     """
 
-    def test_convert_encode_decode_reconvert(self):
+    def test_encode_decode_json_recursively(self):
         _int = 1
         _bytes = b'bytes'
         _unicode = u'unicode'
@@ -112,11 +138,11 @@ def test_convert_encode_decode_reconvert(self):
         encoder = json.JSONEncoder()
         decoder = json.JSONDecoder()
         for original_msg in msgs:
-            converted_msg = _convert(original_msg)
-            encoded_msg = encoder.encode(converted_msg)
-            decoded_msg = decoder.decode(encoded_msg)
-            reconverted_msg = _reconvert(decoded_msg)
-            if isinstance(reconverted_msg, dict):
-                self.assertDictEqual(reconverted_msg, original_msg)
-            elif isinstance(reconverted_msg, (list, tuple)):
-                self.assertSequenceEqual(reconverted_msg, original_msg)
+            encoded_msg_1 = _encode_recursively(original_msg)
+            encoded_msg_2 = encoder.encode(encoded_msg_1)
+            decoded_msg_2 = decoder.decode(encoded_msg_2)
+            decoded_msg_1 = _decode_recursively(decoded_msg_2)
+            if isinstance(decoded_msg_1, dict):
+                self.assertDictEqual(decoded_msg_1, original_msg)
+            elif isinstance(decoded_msg_1, (list, tuple)):
+                self.assertSequenceEqual(decoded_msg_1, original_msg)