Run performance test for metrics

DifferentialOrange · DifferentialOrange · commit 06b96ce985bb · 2021-12-15T14:46:46.000+03:00
Before this patch, performance tests ran together with unit and integration with `--coverage` flag. Coverage analysis cropped the result of performance tests to 10-15 times. For metrics integration it resulted in timeout errors and drop of performance which is not reproduces with coverage disabled. Moreover, before this patch log capture was disabled and performance tests did not displayed any results after run. After this patch, `make -C build coverage` will run lightweight version of performance test. `make -C build performance` will run real performance tests. This path also adds new cases to performance test to compare module performance with or without statistics, statistic wrappers and compare different metrics drivers. Closes #233, follows up #224
diff --git a/.github/workflows/test_on_push.yaml b/.github/workflows/test_on_push.yaml
@@ -16,7 +16,11 @@ jobs:
         tarantool-version: ["1.10.6", "1.10", "2.2", "2.3", "2.4", "2.5", "2.6", "2.7", "2.8"]
         metrics-version: [""]
         remove-merger: [false]
+        perf-test: [false]
         include:
+          - tarantool-version: "1.10"
+            metrics-version: "0.12.0"
+            perf-test: true
           - tarantool-version: "2.7"
             remove-merger: true
           - tarantool-version: "2.8"
@@ -26,6 +30,7 @@ jobs:
           - tarantool-version: "2.8"
             coveralls: true
             metrics-version: "0.12.0"
+            perf-test: true
       fail-fast: false
     runs-on: [ubuntu-latest]
     steps:
@@ -72,6 +77,10 @@ jobs:
       - name: Run tests and code coverage analysis
         run: make -C build coverage
 
+      - name: Run performance tests
+        run: make -C build performance
+        if: ${{ matrix.perf-test }}
+
       - name: Send code coverage to coveralls.io
         run: make -C build coveralls
         if: ${{ matrix.coveralls }}
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -36,6 +36,14 @@ add_custom_target(luatest
   COMMENT "Run regression tests"
 )
 
+set(PERFORMANCE_TESTS_SUBDIR "test/performance")
+
+add_custom_target(performance
+  COMMAND PERF_MODE_ON=true ${LUATEST} -v -c ${PERFORMANCE_TESTS_SUBDIR}
+  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+  COMMENT "Run performance tests"
+)
+
 add_custom_target(coverage
   COMMAND ${LUACOV} ${PROJECT_SOURCE_DIR} && grep -A999 '^Summary' ${CODE_COVERAGE_REPORT}
   DEPENDS ${CODE_COVERAGE_STATS}
diff --git a/crud/stats/metrics_registry.lua b/crud/stats/metrics_registry.lua
@@ -28,6 +28,7 @@ local metric_name = {
 
 local LATENCY_QUANTILE = 0.99
 
+-- Increasing tolerance threshold affects performance.
 local DEFAULT_QUANTILES = {
     [LATENCY_QUANTILE] = 1e-3,
 }
diff --git a/test/performance/select_perf_test.lua b/test/performance/select_perf_test.lua
@@ -9,7 +9,7 @@ local g = t.group('perf')
 
 local helpers = require('test.helper')
 
-g.before_all = function()
+g.before_all(function(g)
     g.cluster = helpers.Cluster:new({
         datadir = fio.tempdir(),
         server_command = helpers.entrypoint('srv_select'),
@@ -53,21 +53,32 @@ g.before_all = function()
         },
     })
     g.cluster:start()
-end
 
-g.after_all = function()
+    g.router = g.cluster:server('router').net_box
+
+    helpers.prepare_simple_functions(g.router)
+    g.router:eval(" crud = require('crud') ")
+
+    -- Run real perf tests only with flag, otherwise run short version
+    -- to test compatibility as part of unit/integration test run.
+    g.perf_mode_on = os.getenv('PERF_MODE_ON')
+end)
+
+g.before_each(function(g)
+    helpers.truncate_space_on_cluster(g.cluster, 'customers')
+end)
+
+g.after_all(function(g)
     g.cluster:stop()
     fio.rmtree(g.cluster.datadir)
-end
-
-g.before_each(function() end)
+end)
 
-local function insert_customers(conn, id, count, timeout, report)
+local function insert_customers(conn, id, count, timeout, report, call_func)
     local customer = {id, box.NULL, 'David', 'Smith', 33, 'Los Angeles'}
     local start = fiber.clock()
 
     while (fiber.clock() - start) < timeout do
-        local ok, res, err = pcall(conn.call, conn, [[package.loaded.crud.insert]], {'customers', customer})
+        local ok, res, err = pcall(conn.call, conn, call_func, {'customers', customer})
         if not ok then
             log.error('Insert error: %s', res)
             table.insert(report.errors, res)
@@ -82,11 +93,123 @@ local function insert_customers(conn, id, count, timeout, report)
     end
 end
 
-local function select_customers(conn, id, timeout, report)
+local insert_cases = {
+    without_stats_wrapper = {
+        prepare = function(g)
+            g.router:eval("plain_insert = require('crud.insert').tuple")
+        end,
+        call = "plain_insert",
+    },
+    with_stats_disabled = {
+        prepare = function(g)
+            g.cluster:server('router').net_box:call("package.loaded.crud.disable_stats")
+        end,
+        call = "package.loaded.crud.insert",
+    },
+    with_local_stats = {
+        prepare = function(g)
+            g.cluster:server('router').net_box:call("package.loaded.crud.enable_stats", {{ driver = 'local' }})
+        end,
+        call = "package.loaded.crud.insert",
+    },
+    with_metrics_stats = {
+        prepare = function(g)
+            local router = g.cluster:server('router').net_box
+            local is_metrics_supported = router:eval([[
+                return require('crud.stats.metrics_registry').is_supported()
+            ]])
+            t.skip_if(is_metrics_supported == false, 'Metrics registry is unsupported')
+            router:call("package.loaded.crud.enable_stats", {{ driver = 'metrics' }})
+        end,
+        call = "package.loaded.crud.insert",
+    },
+}
+
+for name, case in pairs(insert_cases) do
+    local test_name = ('test_insert_%s'):format(name)
+
+    if case.prepare ~= nil then
+        g.before_test(test_name, case.prepare)
+    end
+
+    g[test_name] = function(g)
+        local timeout, fiber_count, connection_count
+        if g.perf_mode_on then
+            timeout = 30
+            fiber_count = 600
+            connection_count = 10
+        else
+            timeout = 2
+            fiber_count = 10
+            connection_count = 2
+        end
+
+        local connections = {}
+
+        local server = g.cluster.main_server
+        for _ = 1, connection_count do
+            local c = net_box:connect(server.net_box_uri, server.net_box_credentials)
+            assert(c)
+            table.insert(connections, c)
+        end
+
+        local fibers = {}
+        local report = {errors = {}, count = 0}
+        for id = 1, fiber_count do
+            local conn_id = id % connection_count + 1
+            local conn = connections[conn_id]
+            local f = fiber.new(insert_customers, conn, id, fiber_count, timeout, report, case.call)
+            f:set_joinable(true)
+            table.insert(fibers, f)
+        end
+
+        for i = 1, fiber_count do
+            fibers[i]:join()
+        end
+
+        log.error('\n%s: requests %d, rps %d, errors %d',
+            test_name, report.count, report.count / timeout, #report.errors)
+    end
+end
+
+
+local select_cases = {
+    without_stats_wrapper = {
+        prepare = function(g)
+            g.router:eval("plain_select = require('crud.select').call")
+        end,
+        call = "plain_select",
+    },
+    with_stats_disabled = {
+        prepare = function(g)
+            g.router:call("package.loaded.crud.disable_stats")
+        end,
+        call = "package.loaded.crud.select",
+    },
+    with_local_stats = {
+        prepare = function(g)
+           g.router:call("package.loaded.crud.enable_stats", {{ driver = 'local' }})
+        end,
+        call = "package.loaded.crud.select",
+    },
+    with_metrics_stats = {
+        prepare = function(g)
+            local router = g.cluster:server('router').net_box
+            local is_metrics_supported = router:eval([[
+                return require('crud.stats.metrics_registry').is_supported()
+            ]])
+            t.skip_if(is_metrics_supported == false, 'Metrics registry is unsupported')
+            router:call("package.loaded.crud.enable_stats", {{ driver = 'metrics' }})
+        end,
+        call = "package.loaded.crud.select",
+    },
+}
+
+local function select_customers(conn, id, timeout, report, call_func)
     local start = fiber.clock()
     local ok, err = pcall(function()
         while (fiber.clock() - start) < timeout do
-            local _, err = conn:call([[package.loaded.crud.select]], {'customers', {{'>', 'id', id}}, {first = 10}})
+            local _, err = conn:call(call_func, {'customers', {{'>', 'id', id}}, {first = 10}})
             if err ~= nil then
                 errors.wrap(err)
                 log.error(err)
@@ -102,66 +225,50 @@ local function select_customers(conn, id, timeout, report)
     end
 end
 
-g.test_insert = function()
-    local timeout = 30
-    local fiber_count = 600
-    local connection_count = 10
-    local connections = {}
-
-    local server = g.cluster.main_server
-    server.net_box:eval([[require('crud')]])
-    for _ = 1, connection_count do
-        local c = net_box:connect(server.net_box_uri, server.net_box_credentials)
-        assert(c)
-        table.insert(connections, c)
-    end
+for name, case in pairs(select_cases) do
+    local test_name = ('test_select_%s'):format(name)
 
-    local fibers = {}
-    local report = {errors = {}, count = 0}
-    for id = 1, fiber_count do
-        local conn_id = id % connection_count + 1
-        local conn = connections[conn_id]
-        local f = fiber.new(insert_customers, conn, id, fiber_count, timeout, report)
-        f:set_joinable(true)
-        table.insert(fibers, f)
+    if case.prepare ~= nil then
+        g.before_test(test_name, case.prepare)
     end
 
-    for i = 1, fiber_count do
-        fibers[i]:join()
-    end
+    g[test_name] = function(g)
+        local timeout, fiber_count, connection_count
+        if g.perf_mode_on then
+            timeout = 30
+            fiber_count = 200
+            connection_count = 10
+        else
+            timeout = 2
+            fiber_count = 5
+            connection_count = 2
+        end
 
-    log.error('\nINSERT: requests %d, rps %d, errors %d',
-        report.count, report.count / timeout, #report.errors)
-end
+        local connections = {}
 
-g.test_select = function()
-    local timeout = 30
-    local fiber_count = 200
-    local connection_count = 10
-    local connections = {}
-
-    local server = g.cluster.main_server
-    server.net_box:eval([[require('crud')]])
-    for _ = 1, connection_count do
-        local c = net_box:connect(server.net_box_uri, server.net_box_credentials)
-        assert(c)
-        table.insert(connections, c)
-    end
+        local server = g.cluster.main_server
 
-    local fibers = {}
-    local report = {errors = {}, count = 0}
-    for id = 1, fiber_count do
-        local conn_id = id % connection_count + 1
-        local conn = connections[conn_id]
-        local f = fiber.new(select_customers, conn, id, timeout, report)
-        f:set_joinable(true)
-        table.insert(fibers, f)
-    end
+        for _ = 1, connection_count do
+            local c = net_box:connect(server.net_box_uri, server.net_box_credentials)
+            assert(c)
+            table.insert(connections, c)
+        end
 
-    for i = 1, fiber_count do
-        fibers[i]:join()
-    end
+        local fibers = {}
+        local report = {errors = {}, count = 0}
+        for id = 1, fiber_count do
+            local conn_id = id % connection_count + 1
+            local conn = connections[conn_id]
+            local f = fiber.new(select_customers, conn, id, timeout, report, case.call)
+            f:set_joinable(true)
+            table.insert(fibers, f)
+        end
+
+        for i = 1, fiber_count do
+            fibers[i]:join()
+        end
 
-    log.error('\nSELECT: requests %d, rps %d, errors %d',
-        report.count, report.count / timeout, #report.errors)
+        log.error('\n%s: requests %d, rps %d, errors %d',
+            test_name, report.count, report.count / timeout, #report.errors)
+    end
 end

Original file line number	Diff line number	Diff line change
`@@ -28,6 +28,7 @@ local metric_name = {`
`28`	`28`
`29`	`29`	`local LATENCY_QUANTILE = 0.99`
`30`	`30`
	`31`	`+-- Increasing tolerance threshold affects performance.`
`31`	`32`	`local DEFAULT_QUANTILES = {`
`32`	`33`	`[LATENCY_QUANTILE] = 1e-3,`
`33`	`34`	`}`