diff --git a/README.rst b/README.rst index 3d265da0..a4586250 100644 --- a/README.rst +++ b/README.rst @@ -81,6 +81,25 @@ NOTE This driver is synchronous, so connection mustn't be shared between threads/processes. +Run tests +^^^^^^^^^ + +On Linux: + +.. code-block:: console + $ python setup.py test + +On Windows: + +* Setup a Linux machine with installed tarantool (called ``remote`` later). +* (on ``remote``) Copy ``unit/suites/lib/tarantool_python_ci.lua`` to + ``/etc/tarantool/instances.available``. +* (on ``remote``) Run ``tarantoolctl start tarantool_python_ci``. +* Set the following environment variables: + * ``REMOTE_TARANTOOL_HOST=...``, + * ``REMOTE_TARANTOOL_CONSOLE_PORT=3302``. +* Run ``python setup.py test``. + .. _`Tarantool`: .. _`Tarantool Database`: .. _`Tarantool Homepage`: http://tarantool.org diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 00000000..e0485939 --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,23 @@ +environment: + matrix: + - PYTHON: "C:\\Python27" + - PYTHON: "C:\\Python27-x64" + - PYTHON: "C:\\Python34" + - PYTHON: "C:\\Python34-x64" + - PYTHON: "C:\\Python35" + - PYTHON: "C:\\Python35-x64" + - PYTHON: "C:\\Python36" + - PYTHON: "C:\\Python36-x64" + - PYTHON: "C:\\Python37" + - PYTHON: "C:\\Python37-x64" + +install: + # install runtime dependencies + - "%PYTHON%\\python.exe -m pip install -r requirements.txt" + # install testing dependencies + - "%PYTHON%\\python.exe -m pip install pyyaml" + +build: off + +test_script: + - "%PYTHON%\\python.exe setup.py test" diff --git a/unit/suites/lib/remote_tarantool_server.py b/unit/suites/lib/remote_tarantool_server.py new file mode 100644 index 00000000..fa983f9b --- /dev/null +++ b/unit/suites/lib/remote_tarantool_server.py @@ -0,0 +1,94 @@ +from __future__ import print_function + +import sys +import os +import random +import string +import time + +from .tarantool_admin import TarantoolAdmin + + +# a time during which try to acquire a lock +AWAIT_TIME = 60 # seconds + +# on which port bind a socket for binary protocol +BINARY_PORT = 3301 + + +def get_random_string(): + return ''.join(random.choice(string.ascii_lowercase) for _ in range(16)) + + +class RemoteTarantoolServer(object): + def __init__(self): + self.host = os.environ['REMOTE_TARANTOOL_HOST'] + + self.args = {} + self.args['primary'] = BINARY_PORT + self.args['admin'] = os.environ['REMOTE_TARANTOOL_CONSOLE_PORT'] + + assert(self.args['primary'] != self.args['admin']) + + # a name to using for a lock + self.whoami = get_random_string() + + self.admin = TarantoolAdmin(self.host, self.args['admin']) + self.lock_is_acquired = False + + # emulate stopped server + self.acquire_lock() + self.admin.execute('box.cfg{listen = box.NULL}') + + def acquire_lock(self): + deadline = time.time() + AWAIT_TIME + while True: + res = self.admin.execute('return acquire_lock("%s")' % self.whoami) + ok = res[0] + err = res[1] if not ok else None + if ok: + break + if time.time() > deadline: + raise RuntimeError('can not acquire "%s" lock: %s' % ( + self.whoami, str(err))) + print('waiting to acquire "%s" lock' % self.whoami, + file=sys.stderr) + time.sleep(1) + self.lock_is_acquired = True + + def touch_lock(self): + assert(self.lock_is_acquired) + res = self.admin.execute('return touch_lock("%s")' % self.whoami) + ok = res[0] + err = res[1] if not ok else None + if not ok: + raise RuntimeError('can not update "%s" lock: %s' % ( + self.whoami, str(err))) + + def release_lock(self): + res = self.admin.execute('return release_lock("%s")' % self.whoami) + ok = res[0] + err = res[1] if not ok else None + if not ok: + raise RuntimeError('can not release "%s" lock: %s' % ( + self.whoami, str(err))) + self.lock_is_acquired = False + + def start(self): + if not self.lock_is_acquired: + self.acquire_lock() + self.admin.execute('box.cfg{listen = "0.0.0.0:%s"}' % + self.args['primary']) + + def stop(self): + self.admin.execute('box.cfg{listen = box.NULL}') + self.release_lock() + + def is_started(self): + return self.lock_is_acquired + + def clean(self): + pass + + def __del__(self): + self.admin.disconnect() diff --git a/unit/suites/lib/tarantool_admin.py b/unit/suites/lib/tarantool_admin.py new file mode 100644 index 00000000..2d478a8b --- /dev/null +++ b/unit/suites/lib/tarantool_admin.py @@ -0,0 +1,64 @@ +import socket +import yaml + + +class TarantoolAdmin(object): + def __init__(self, host, port): + self.host = host + self.port = port + self.is_connected = False + self.socket = None + + def connect(self): + self.socket = socket.create_connection((self.host, self.port)) + self.socket.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) + self.is_connected = True + self.socket.recv(256) # skip greeting + + def disconnect(self): + if self.is_connected: + self.socket.close() + self.socket = None + self.is_connected = False + + def reconnect(self): + self.disconnect() + self.connect() + + def __enter__(self): + self.connect() + return self + + def __exit__(self, type, value, tb): + self.disconnect() + + def __call__(self, command): + return self.execute(command) + + def execute(self, command): + if not command: + return + + if not self.is_connected: + self.connect() + + cmd = (command.replace('\n', ' ') + '\n').encode() + try: + self.socket.sendall(cmd) + except socket.error: + # reconnect and try again + self.reconnect() + self.socket.sendall(cmd) + + bufsiz = 4096 + res = "" + + while True: + buf = self.socket.recv(bufsiz) + if not buf: + break + res = res + buf.decode() + if (res.rfind("\n...\n") >= 0 or res.rfind("\r\n...\r\n") >= 0): + break + + return yaml.load(res) diff --git a/unit/suites/lib/tarantool_python_ci.lua b/unit/suites/lib/tarantool_python_ci.lua new file mode 100644 index 00000000..2305c0d7 --- /dev/null +++ b/unit/suites/lib/tarantool_python_ci.lua @@ -0,0 +1,347 @@ +#!/usr/bin/env tarantool + +local console = require('console') +local clock = require('clock') +local log = require('log') + +local CONSOLE_PORT = 3302 + +box.cfg({}) +console.listen(CONSOLE_PORT) + +-- forward declarations +local clean +local init + +-- {{{ locking + +local LOCK_LIFETIME = 30 -- seconds + +local locked_by = nil +local locked_at = 0 -- unix time, seconds + +local function clean_lock() + locked_by = nil + locked_at = 0 + clean() + init() +end + +local function set_lock(who) + locked_by = who + locked_at = clock.monotonic() +end + +local function clean_dead_lock() + if locked_by ~= nil and clock.monotonic() - locked_at > LOCK_LIFETIME then + log.info(('removed dead "%s" lock'):format(tostring(locked_by))) + clean_lock() + end +end + +local function is_locked_by(who) + return locked_by == who +end + +local function is_locked() + return locked_by ~= nil +end + +local function acquire_lock(who) + assert(type(who) == 'string') + clean_dead_lock() + if is_locked_by(who) then + -- update lock time + set_lock(who) + log.info(('updated "%s" lock'):format(who)) + return true + end + if is_locked() then + local err = 'locked by ' .. tostring(locked_by) + log.info(('can not update "%s" lock: %s'):format(who, err)) + return false, err + end + set_lock(who) + log.info(('set "%s" lock'):format(who)) + return true +end + +local function touch_lock(who) + assert(type(who) == 'string') + clean_dead_lock() + if is_locked_by(who) then + -- update lock time + set_lock(who) + log.info(('updated "%s" lock'):format(who)) + return true + end + if is_locked() then + local err = 'locked by ' .. tostring(locked_by) + log.info(('can not update "%s" lock: %s'):format(who, err)) + return false, err + end + local err = 'is not locked' + log.info(('can not update "%s" lock: %s'):format(who, err)) + return false, err +end + +local function release_lock(who) + assert(type(who) == 'string') + if is_locked_by(who) then + clean_lock() + log.info(('released "%s" lock'):format(who)) + return true + end + clean_dead_lock() + if is_locked() then + local err = 'locked by ' .. tostring(locked_by) + log.info(('can not release "%s" lock: %s'):format(who, err)) + return false, err + end + local err = 'is not locked' + log.info(('can not release "%s" lock: %s'):format(who, err)) + return false, err +end + +-- }}} + +-- {{{ init + +init = function() + _G.acquire_lock = acquire_lock + _G.touch_lock = touch_lock + _G.release_lock = release_lock +end + +-- }}} + +-- {{{ clean + +-- Copy of cleanup_cluster() from test_run.lua. +local function cleanup_cluster() + local cluster = box.space._cluster:select() + for _, tuple in pairs(cluster) do + if tuple[1] ~= box.info.id then + box.space._cluster:delete(tuple[1]) + end + end +end + +-- Copy of clean() from pretest_clean.lua from test-run. +clean = function() + local _SPACE_NAME = 3 + + box.space._space:pairs():map(function(tuple) + local name = tuple[_SPACE_NAME] + return name + end):filter(function(name) + -- skip internal spaces + local first_char = string.sub(name, 1, 1) + return first_char ~= '_' + end):each(function(name) + box.space[name]:drop() + end) + + local _USER_TYPE = 4 + local _USER_NAME = 3 + + local allowed_users = { + guest = true, + admin = true, + } + box.space._user:pairs():filter(function(tuple) + local tuple_type = tuple[_USER_TYPE] + return tuple_type == 'user' + end):map(function(tuple) + local name = tuple[_USER_NAME] + return name + end):filter(function(name) + return not allowed_users[name] + end):each(function(name) + box.schema.user.drop(name) + end) + + local allowed_roles = { + public = true, + replication = true, + super = true, + } + box.space._user:pairs():filter(function(tuple) + local tuple_type = tuple[_USER_TYPE] + return tuple_type == 'role' + end):map(function(tuple) + local name = tuple[_USER_NAME] + return name + end):filter(function(name) + return not allowed_roles[name] + end):each(function(name) + box.schema.role.drop(name) + end) + + local _FUNC_NAME = 3 + local allowed_funcs = { + ['box.schema.user.info'] = true, + } + box.space._func:pairs():map(function(tuple) + local name = tuple[_FUNC_NAME] + return name + end):filter(function(name) + return not allowed_funcs[name] + end):each(function(name) + box.schema.func.drop(name) + end) + + cleanup_cluster() + + local cleanup_list = function(list, allowed) + for k, _ in pairs(list) do + if not allowed[k] then + list[k] = nil + end + end + end + + local allowed_globals = { + -- modules + bit = true, + coroutine = true, + debug = true, + io = true, + jit = true, + math = true, + os = true, + package = true, + string = true, + table = true, + utf8 = true, + -- variables + _G = true, + _VERSION = true, + arg = true, + -- functions + assert = true, + collectgarbage = true, + dofile = true, + error = true, + gcinfo = true, + getfenv = true, + getmetatable = true, + ipairs = true, + load = true, + loadfile = true, + loadstring = true, + module = true, + next = true, + pairs = true, + pcall = true, + print = true, + rawequal = true, + rawget = true, + rawset = true, + require = true, + select = true, + setfenv = true, + setmetatable = true, + tonumber = true, + tonumber64 = true, + tostring = true, + type = true, + unpack = true, + xpcall = true, + -- tarantool + _TARANTOOL = true, + box = true, + dostring = true, + help = true, + newproxy = true, + role_check_grant_revoke_of_sys_priv = true, + tutorial = true, + update_format = true, + } + cleanup_list(_G, allowed_globals) + + local allowed_packages = { + ['_G'] = true, + bit = true, + box = true, + ['box.backup'] = true, + ['box.internal'] = true, + ['box.internal.sequence'] = true, + ['box.internal.session'] = true, + ['box.internal.space'] = true, + buffer = true, + clock = true, + console = true, + coroutine = true, + crypto = true, + csv = true, + debug = true, + digest = true, + errno = true, + ffi = true, + fiber = true, + fio = true, + fun = true, + help = true, + ['help.en_US'] = true, + ['http.client'] = true, + iconv = true, + ['internal.argparse'] = true, + ['internal.trigger'] = true, + io = true, + jit = true, + ['jit.bc'] = true, + ['jit.bcsave'] = true, + ['jit.dis_x64'] = true, + ['jit.dis_x86'] = true, + ['jit.dump'] = true, + ['jit.opt'] = true, + ['jit.p'] = true, + ['jit.profile'] = true, + ['jit.util'] = true, + ['jit.v'] = true, + ['jit.vmdef'] = true, + ['jit.zone'] = true, + json = true, + log = true, + math = true, + msgpack = true, + msgpackffi = true, + ['net.box'] = true, + ['net.box.lib'] = true, + os = true, + package = true, + pickle = true, + pwd = true, + socket = true, + strict = true, + string = true, + table = true, + ['table.clear'] = true, + ['table.new'] = true, + tap = true, + tarantool = true, + title = true, + uri = true, + utf8 = true, + uuid = true, + xlog = true, + yaml = true, + } + cleanup_list(package.loaded, allowed_packages) + + local user_count = box.space._user:count() + assert(user_count == 4 or user_count == 5, + 'box.space._user:count() should be 4 (1.10) or 5 (2.0)') + assert(box.space._func:count() == 1, + 'box.space._func:count() should be only one') + assert(box.space._cluster:count() == 1, + 'box.space._cluster:count() should be only one') + + box.cfg({listen = box.NULL}) +end + +-- }}} + +clean() +init() diff --git a/unit/suites/lib/tarantool_server.py b/unit/suites/lib/tarantool_server.py index 24091822..291914db 100644 --- a/unit/suites/lib/tarantool_server.py +++ b/unit/suites/lib/tarantool_server.py @@ -8,11 +8,12 @@ import socket import tempfile -import yaml import time import shutil import subprocess +from .tarantool_admin import TarantoolAdmin + def check_port(port, rais=True): try: sock = socket.create_connection(("localhost", port)) @@ -36,76 +37,6 @@ class RunnerException(object): pass -class TarantoolAdmin(object): - def __init__(self, host, port): - self.host = host - self.port = port - self.is_connected = False - self.socket = None - - def connect(self): - self.socket = socket.create_connection((self.host, self.port)) - self.socket.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) - self.is_connected = True - self.socket.recv(256) # skip greating - - def disconnect(self): - if self.is_connected: - self.socket.close() - self.socket = None - self.is_connected = False - - def reconnect(self): - self.disconnect() - self.connect() - - def opt_reconnect(self): - """ On a socket which was disconnected, recv of 0 bytes immediately - returns with no data. On a socket which is alive, it returns EAGAIN. - Make use of this property and detect whether or not the socket is - dead. Reconnect a dead socket, do nothing if the socket is good.""" - try: - if self.socket is None or self.socket.recv(1, socket.MSG_DONTWAIT|socket.MSG_PEEK) == '': - self.reconnect() - except socket.error as e: - if e.errno == errno.EAGAIN: - pass - else: - self.reconnect() - - def execute(self, command): - self.opt_reconnect() - return self.execute_no_reconnect(command) - - def __enter__(self): - self.connect() - return self - - def __exit__(self, type, value, tb): - self.disconnect() - - def __call__(self, command): - return self.execute(' '.join(command.split('\n'))) - - def execute_no_reconnect(self, command): - if not command: - return - cmd = command.replace('\n', ' ') + '\n' - self.socket.sendall(cmd.encode()) - - bufsiz = 4096 - res = "" - - while True: - buf = self.socket.recv(bufsiz) - if not buf: - break - res = res + buf.decode() - if (res.rfind("\n...\n") >= 0 or res.rfind("\r\n...\r\n") >= 0): - break - - return yaml.load(res) - class TarantoolServer(object): default_tarantool = { "bin": "tarantool", @@ -183,14 +114,22 @@ def log_des(self): self._log_des.close() delattr(self, '_log_des') + def __new__(cls): + if os.name == 'nt': + from .remote_tarantool_server import RemoteTarantoolServer + return RemoteTarantoolServer() + return super(TarantoolServer, cls).__new__(cls) + def __init__(self): os.popen('ulimit -c unlimited') + self.host = 'localhost' self.args = {} self.args['primary'] = find_port() self.args['admin'] = find_port(self.args['primary'] + 1) self._admin = self.args['admin'] self.vardir = tempfile.mkdtemp(prefix='var_', dir=os.getcwd()) self.find_exe() + self.process = None def find_exe(self): if 'TARANTOOL_BOX_PATH' in os.environ: @@ -272,3 +211,10 @@ def __del__(self): self.stop() self.clean() + def touch_lock(self): + # A stub method to be compatible with + # RemoteTarantoolServer. + pass + + def is_started(self): + return self.process is not None diff --git a/unit/suites/test_dml.py b/unit/suites/test_dml.py index 2cf31bd3..8df0530e 100644 --- a/unit/suites/test_dml.py +++ b/unit/suites/test_dml.py @@ -13,7 +13,7 @@ def setUpClass(self): self.srv = TarantoolServer() self.srv.script = 'unit/suites/box.lua' self.srv.start() - self.con = tarantool.Connection('localhost', self.srv.args['primary']) + self.con = tarantool.Connection(self.srv.host, self.srv.args['primary']) self.adm = self.srv.admin self.space_created = self.adm("box.schema.create_space('space_1')") self.adm(""" @@ -39,6 +39,11 @@ def setUpClass(self): self.adm("fiber = require('fiber')") self.adm("uuid = require('uuid')") + def setUp(self): + # prevent a remote tarantool from clean our session + if self.srv.is_started(): + self.srv.touch_lock() + def test_00_00_authenticate(self): self.assertIsNone(self.srv.admin(""" box.schema.user.create('test', { password = 'test' }) @@ -55,6 +60,9 @@ def test_00_01_space_created(self): def test_00_02_fill_space(self): # Fill space with values for i in range(1, 500): + if i % 10 == 0: + # prevent a remote tarantool from clean our session + self.srv.touch_lock() self.assertEqual( self.con.insert('space_1', [i, i%5, 'tuple_'+str(i)])[0], [i, i%5, 'tuple_'+str(i)] @@ -153,7 +161,7 @@ def test_06_update(self): [[2, 2, 'tuplalal_3']]) def test_07_call_16(self): - con = tarantool.Connection('localhost', self.srv.args['primary'], call_16 = True) + con = tarantool.Connection(self.srv.host, self.srv.args['primary'], call_16 = True) con.authenticate('test', 'test') self.assertSequenceEqual(con.call('json.decode', '[123, 234, 345]'), [[123, 234, 345]]) self.assertSequenceEqual(con.call('json.decode', ['[123, 234, 345]']), [[123, 234, 345]]) @@ -179,7 +187,7 @@ def test_07_call_16(self): self.assertSequenceEqual(con.call('box.tuple.new', 'fld_1'), [['fld_1']]) def test_07_call_17(self): - con = tarantool.Connection('localhost', self.srv.args['primary']) + con = tarantool.Connection(self.srv.host, self.srv.args['primary']) con.authenticate('test', 'test') self.assertSequenceEqual(con.call('json.decode', '[123, 234, 345]'), [[123, 234, 345]]) self.assertSequenceEqual(con.call('json.decode', ['[123, 234, 345]']), [[123, 234, 345]]) diff --git a/unit/suites/test_schema.py b/unit/suites/test_schema.py index d5ef0882..c4be111d 100755 --- a/unit/suites/test_schema.py +++ b/unit/suites/test_schema.py @@ -12,9 +12,14 @@ def setUpClass(self): self.srv = TarantoolServer() self.srv.script = 'unit/suites/box.lua' self.srv.start() - self.con = tarantool.Connection('localhost', self.srv.args['primary']) + self.con = tarantool.Connection(self.srv.host, self.srv.args['primary']) self.sch = self.con.schema + def setUp(self): + # prevent a remote tarantool from clean our session + if self.srv.is_started(): + self.srv.touch_lock() + def test_00_authenticate(self): self.assertIsNone(self.srv.admin("box.schema.user.create('test', { password = 'test' })")) self.assertIsNone(self.srv.admin("box.schema.user.grant('test', 'read,write', 'space', '_space')"))