test: add Python 3 support

Python 2 support is EOL [1] and we should support to run our python
tests using Python 3.x. test-run used as test-runner for memcached
already supports Python 3, see [2].

Patch adds support of running tests with Python 3 without breaking
compatibility with Python 2. Testing on CI switched to Python 3.

1. https://www.python.org/doc/sunset-python-2/
2. tarantool/test-run#20

Closes #82

Author: ligurio

.github/workflows/testing.yaml

@@ -33,10 +33,10 @@ jobs:
 
       - run: cmake .
 
-      - name: Setup Python 2 for tests
+      - name: Setup Python 3 for tests
         uses: actions/setup-python@v2
         with:
-          python-version: 2.7
+          python-version: 3.8
 
       - name: Install test requirements
         run: pip install -r test-run/requirements.txt

README.md

@@ -14,10 +14,11 @@ Memcached protocol 'wrapper' for Tarantool.
 
  * Tarantol 1.6.8+ with header files (tarantool && tarantool-dev packages).
  * Cyrus SASL library (with header files)
- * Python >= 2.7, <3 with next packages (for testing only):
+ * Python >= 2.7 with next packages (for testing only):
    - PyYAML
    - msgpack-python
    - six==1.9.0
+   - gevent==1.1b5
 
 ### Installation
 
@@ -149,7 +150,7 @@ Usual rules for memcached are applicable for this plugin:
 	 import bmemcached
 	 client = bmemcached.Client(('127.0.0.1:11211', ), 'testuser', 'testpasswd')
 	 client.set('key', 'value')
-	 print client.get('key')
+	 print(client.get('key'))
 	 ```
 
 For custom configuration file path, please, use `SASL_CONF_PATH` environment variable.

test-run

@@ -10,14 +10,16 @@
 from internal.memcached_connection import MemcachedBinaryConnection
 from internal.memcached_connection import STATUS, COMMANDS
 
+from internal.compat import string_types
+
 mc = MemcachedBinaryConnection("127.0.0.1", iproto.py_con.port)
 
 def iequal(left, right, level = 1):
     if (left != right):
         tb = traceback.extract_stack()[-(level + 1)]
-        print "Error on line %s:%d: %s not equal %s" % (tb[0], tb[1],
-                repr(left), repr(right))
-        if (isinstance(left, basestring)):
+        print("Error on line %s:%d: %s not equal %s" % (tb[0], tb[1],
+                repr(left), repr(right)))
+        if (isinstance(left, string_types)):
             if (len(left) != len(right)):
                 print("length is different")
         return False
@@ -39,7 +41,7 @@ def check(key, flags, val, level = 0):
     val  = "x" * i
     mc.setq(key, val, flags=82, nosend=True)
     mc.setq("alt_%s" % key, "blah", flags=82, nosend=True)
-    data = "".join(mc.commands)
+    data = b"".join(mc.commands)
     mc.commands = []
 
     if (len(data) > 2024):

test/binary/binary-boundary.test.py

@@ -15,15 +15,15 @@
 def iequal(left, right, level = 1):
     if (left != right):
         tb = traceback.extract_stack()[-(level + 1)]
-        print "Error on line %s:%d: %s not equal %s" % (tb[0], tb[1],
-                repr(left), repr(right))
+        print("Error on line %s:%d: %s not equal %s" % (tb[0], tb[1],
+                repr(left), repr(right)))
         return False
     return True
 
 def issert(stmt, level = 1):
     if not bool(stmt):
         tb = traceback.extract_stack()[-(level + 1)]
-        print "Error on line %s:%d: result is False" % (tb[0], tb[1])
+        print("Error on line %s:%d: result is False" % (tb[0], tb[1]))
         return False
     return True
 
@@ -41,15 +41,15 @@ def check(key, flags, val, level = 0):
 
 stat = mc.stat("reset")
 
-for i in xrange(10000):
+for i in range(10000):
     mc.set('key-%d' % i, 'value-%d' % i, expire=1)
 
 stat = mc.stat()
-while int(stat.get('evictions', '0')) < 10000:
+while int(stat.get(b'evictions', '0')) < 10000:
     time.sleep(0.01)
     stat = mc.stat()
 
-issert('evictions' in stat)
-iequal(int(mc.stat().get('evictions', 0)), 10000)
+issert(b'evictions' in stat)
+iequal(int(mc.stat().get(b'evictions', 0)), 10000)
 
 sys.path = saved_path

test/binary/binary-expire.test.py

@@ -14,24 +14,24 @@
 def iequal(left, right, level = 1):
     if (left != right):
         tb = traceback.extract_stack()[-(level + 1)]
-        print "Error on line %s:%d: %s not equal %s" % (tb[0], tb[1],
-                repr(left), repr(right))
+        print("Error on line %s:%d: %s not equal %s" % (tb[0], tb[1],
+                repr(left), repr(right)))
         return False
     return True
 
 def inequal(left, right, level = 0):
     if (left == right):
         tb = traceback.extract_stack()[-(level + 1)]
-        print "Error on line %s:%d: %s equal %s" % (tb[0], tb[1],
-                repr(left), repr(right))
+        print("Error on line %s:%d: %s equal %s" % (tb[0], tb[1],
+                repr(left), repr(right)))
         return False
     return True
 
 def issert(stmt, level = 0):
     if bool(stmt):
         tb = traceback.extract_stack()[-(level + 1)]
-        print "Error on line %s:%d: is False" % (tb[0], tb[1],
-                repr(left), repr(right))
+        print("Error on line %s:%d: is False" % (tb[0], tb[1],
+                repr(left), repr(right)))
         return False
     return True
 

test/binary/binary-toobig.test.py

@@ -14,24 +14,24 @@
 def iequal(left, right, level = 1):
     if (left != right):
         tb = traceback.extract_stack()[-(level + 1)]
-        print "Error on line %s:%d: %s not equal %s" % (tb[0], tb[1],
-                repr(left), repr(right))
+        print("Error on line %s:%d: %s not equal %s" % (tb[0], tb[1],
+                repr(left), repr(right)))
         return False
     return True
 
 def inequal(left, right, level = 0):
     if (left == right):
         tb = traceback.extract_stack()[-(level + 1)]
-        print "Error on line %s:%d: %s equal %s" % (tb[0], tb[1],
-                repr(left), repr(right))
+        print("Error on line %s:%d: %s equal %s" % (tb[0], tb[1],
+                repr(left), repr(right)))
         return False
     return True
 
 def issert(stmt, level = 0):
     if bool(stmt):
         tb = traceback.extract_stack()[-(level + 1)]
-        print "Error on line %s:%d: is False" % (tb[0], tb[1],
-                repr(left), repr(right))
+        print("Error on line %s:%d: is False" % (tb[0], tb[1],
+                repr(left), repr(right)))
         return False
     return True
 
@@ -251,7 +251,7 @@ def check_empty_response(con):
 empty("y")
 
 stat2 = mc.stat()
-iequal(int(stat1['cmd_flush']) + 1, int(stat2['cmd_flush']))
+iequal(int(stat1[b'cmd_flush']) + 1, int(stat2[b'cmd_flush']))
 
 print("""#----------------------------# diagnostics append #---------------------------#""")
 key, value, suffix = "appendkey", "some value", " more"

test/binary/binary.test.py

@@ -2,4 +2,4 @@
 core = tarantool
 script = binary.lua
 description = memcached binary tests
-disabled = binary-expire.test.py binary-boundary.test.py
+disabled = binary-boundary.test.py

test/binary/suite.ini

@@ -12,6 +12,8 @@
 from internal.memcached_connection import MemcachedBinaryConnection
 from internal.memcached_connection import STATUS, COMMANDS
 
+from internal.compat import bytes_to_str
+
 mc = MemcachedBinaryConnection("127.0.0.1", iproto.py_con.port)
 mc.flush()
 
@@ -20,6 +22,7 @@
 
 task = Popen(cmd, stdout=PIPE, stderr=STDOUT)
 
-print task.communicate()[0]
+testcase = task.communicate()[0]
+print(bytes_to_str(testcase))
 
 sys.path = saved_path

test/capable/capable-binary.test.py

@@ -12,6 +12,8 @@
 from internal.memcached_connection import MemcachedBinaryConnection
 from internal.memcached_connection import STATUS, COMMANDS
 
+from internal.compat import bytes_to_str
+
 mc = MemcachedBinaryConnection("127.0.0.1", iproto.py_con.port)
 mc.flush()
 
@@ -20,6 +22,7 @@
 
 task = Popen(cmd, stdout=PIPE, stderr=STDOUT)
 
-print task.communicate()[0]
+testcase = task.communicate()[0]
+print(bytes_to_str(testcase))
 
 sys.path = saved_path

test/capable/capable-text.test.py

@@ -0,0 +1,55 @@
+import sys
+
+# Added for compatibility with Python 3.
+# Python 2 provided the class basestring, from which both str and unicode
+# derived. In Python 3, the concept of basestring makes no sense: text is only
+# represented by str.
+try:
+  basestring
+except NameError:
+  basestring = str
+
+# Useful for very coarse version differentiation.
+PY3 = sys.version_info[0] == 3
+PY2 = sys.version_info[0] == 2
+
+if PY3:
+    string_types = str,
+    integer_types = int,
+else:
+    string_types = basestring,      # noqa: F821
+    integer_types = (int, long)     # noqa: F821
+
+def assert_bytes(b):
+    """ Ensure given value is <bytes>.
+    """
+    if type(b) != bytes:
+        raise ValueError('Internal error: expected {}, got {}: {}'.format(
+            str(bytes), str(type(b)), repr(b)))
+
+def assert_str(s):
+    """ Ensure given value is <str>.
+    """
+    if type(s) != str:
+        raise ValueError('Internal error: expected {}, got {}: {}'.format(
+            str(str), str(type(s)), repr(s)))
+
+def bytes_to_str(b):
+    """ Convert <bytes> to <str>.
+
+        No-op on Python 2.
+    """
+    assert_bytes(b)
+    if PY2:
+        return b
+    return b.decode('utf-8')
+
+def str_to_bytes(s):
+    """ Convert <str> to <bytes>.
+
+        No-op on Python 2.
+    """
+    assert_str(s)
+    if PY2:
+        return s
+    return s.encode('utf-8')

test/internal/compat.py

@@ -4,6 +4,7 @@
 from struct import Struct
 
 from lib.tarantool_connection import TarantoolConnection
+from internal.compat import bytes_to_str, str_to_bytes
 
 HEADER        = '!BBHBBHLLQ'
 HEADER_STRUCT = Struct(HEADER)
@@ -165,7 +166,7 @@ def __init__(self, host, port):
         self.connect()
 
     def send(self):
-        commands = ''.join(self.commands)
+        commands = b''.join(self.commands)
         self.socket.sendall(commands)
         self.commands = []
 
@@ -202,8 +203,8 @@ def _read_and_parse_response(self):
         assert(magic == MAGIC['response'])
         if status == STATUS['OK']:
             assert(ext_len == ext_lenv)
-            assert(((key_lenv > 0 or key_lenv is None) and key_len > 0) or key_len == 0)
-            assert(((val_lenv > 0 or val_lenv is None) and val_len > 0) or val_len == 0)
+            assert(((key_lenv is None or key_lenv > 0) and key_len > 0) or key_len == 0)
+            assert(((val_lenv is None or val_lenv > 0) and val_len > 0) or val_len == 0)
         else:
             assert(val_len > 0)
 
@@ -238,7 +239,10 @@ def _read_and_parse_response(self):
             if is_indecrq(op):
                 val = INDECR_STRUCT.unpack_from(val)[0]
         if val is not None:
-            retval['val'] = val
+            if isinstance(val, bytes):
+                retval['val'] = bytes_to_str(val)
+            else:
+                retval['val'] = val
 
         return retval
 
@@ -268,9 +272,9 @@ def append_query(self, cmd, args):
         retval = [
             struct.pack(MAGIC['request'], op, key_len, ext_len, dtype, 0,
                         tot_len, opaque, cas, *extra),
-            key, val
+            str_to_bytes(key), str_to_bytes(val)
         ]
-        cmd = ''.join(retval)
+        cmd = b''.join(retval)
         self.commands.append(cmd)
 
     @binary_decorator
@@ -559,9 +563,9 @@ def execute_no_reconnect(self, commands, silent = False):
 
     def send(self, commands, silent = False):
         self.commands = commands
-        self.socket.sendall(commands)
+        self.socket.sendall(str_to_bytes(commands))
         if not silent:
-            print "<<" + '-' * 50
+            print("<<" + '-' * 50)
             sys.stdout.write(self.commands.strip() + '\n')
             #sys.stdout.write(self.commands)
 
@@ -595,7 +599,7 @@ def recv(self, silent = False):
                 self.reply_unknown(cmd)
 
         if not silent:
-            print ">>" + '-'*50
+            print(">>" + '-'*50)
             sys.stdout.write(self.reply.strip() + '\n')
             #sys.stdout.write(self.reply)
 
@@ -643,7 +647,7 @@ def reply_retrieval(self, cmd):
                 break
             else:
                 # unknown
-                print "error: unknown line: '%s'" % key
+                print("error: unknown line: '%s'" % key)
                 self.reply += "error: unknown line: '%s'" % key
                 break
 
@@ -699,7 +703,7 @@ def read_line(self):
             index = buf.find(MEMCACHED_SEPARATOR)
             if index > 0:
                 break
-            data = self.socket.recv(1048576)
+            data = bytes_to_str(self.socket.recv(1048576))
             if not data:
                 return None
             buf += data