stats: integrate with metrics rock

If `metrics` [1] found, you can use metrics collectors to store
statistics. `metrics >= 0.10.0` is required to use metrics driver.
(`metrics >= 0.9.0` is required to use summary quantiles with
age buckets. `metrics >= 0.5.0, < 0.9.0` is unsupported
due to quantile overflow bug [2]. `metrics == 0.9.0` has bug that do
not permits to create summary collector without quantiles [3].
In fact, user may use `metrics >= 0.5.0`, `metrics != 0.9.0`
if he wants to use metrics without quantiles, and `metrics >= 0.9.0`
if he wants to use metrics with quantiles. But this is confusing,
so let's use a single restriction for both cases.)

The metrics are part of global registry and can be exported together
(e.g. to Prometheus) with default tools without any additional
configuration. Disabling stats destroys the collectors.

Metrics collectors are used by default if supported. To explicitly set
driver, call `crud.cfg{ stats = true, stats_driver = driver }`
('local' or 'metrics'). To enable quantiles, call
```
crud.cfg{
    stats = true,
    stats_driver = 'metrics',
    stats_quantiles = true,
}
```
With quantiles, `latency` statistics are changed to 0.99 quantile
of request execution time (with aging). Quantiles computations increases
performance overhead up to 10% when used in statistics.

Add CI matrix to run tests with `metrics` installed. To get full
coverage on coveralls, #248 must be resolved.

1. https://github.com/tarantool/metrics
2. tarantool/metrics#235
3. tarantool/metrics#262

Closes #224

Author: DifferentialOrange

.github/workflows/test_on_push.yaml

@@ -13,13 +13,21 @@ jobs:
       matrix:
         # We need 1.10.6 here to check that module works with
         # old Tarantool versions that don't have "tuple-keydef"/"tuple-merger" support.
-        tarantool-version: ["1.10.6", "1.10", "2.2", "2.3", "2.4", "2.5", "2.6", "2.7"]
+        tarantool-version: ["1.10.6", "1.10", "2.2", "2.3", "2.4", "2.5", "2.6", "2.7", "2.8"]
+        metrics-version: [""]
         remove-merger: [false]
         include:
+          - tarantool-version: "1.10"
+            metrics-version: "0.12.0"
           - tarantool-version: "2.7"
             remove-merger: true
+          - tarantool-version: "2.8"
+            metrics-version: "0.1.8"
+          - tarantool-version: "2.8"
+            metrics-version: "0.10.0"
           - tarantool-version: "2.8"
             coveralls: true
+            metrics-version: "0.12.0"
       fail-fast: false
     runs-on: [ubuntu-latest]
     steps:
@@ -47,6 +55,10 @@ jobs:
           tarantool --version
           ./deps.sh
 
+      - name: Install metrics
+        if: matrix.metrics-version != ''
+        run: tarantoolctl rocks install metrics ${{ matrix.metrics-version }}
+
       - name: Remove external merger if needed
         if: ${{ matrix.remove-merger }}
         run: rm .rocks/lib/tarantool/tuple/merger.so
@@ -71,6 +83,7 @@ jobs:
     strategy:
       matrix:
         bundle_version: [ "1.10.11-0-gf0b0e7ecf-r422", "2.7.3-0-gdddf926c3-r422" ]
+        metrics-version: ["", "0.12.0"]
       fail-fast: false
     runs-on: [ ubuntu-latest ]
     steps:
@@ -86,6 +99,10 @@ jobs:
           tarantool --version
           ./deps.sh
 
+      - name: Install metrics
+        if: matrix.metrics-version != ''
+        run: tarantoolctl rocks install metrics ${{ matrix.metrics-version }}
+
       # This server starts and listen on 8084 port that is used for tests
       - name: Stop Mono server
         run: sudo kill -9 $(sudo lsof -t -i tcp:8084) || true

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Added
 * Statistics for CRUD operations on router (#224).
+* Integrate CRUD statistics with [`metrics`](https://github.com/tarantool/metrics) (#224).
 
 ### Changed
 

README.md

@@ -606,11 +606,28 @@ crud.cfg{ stats = false }
 crud.reset_stats()
 ```
 
+If [`metrics`](https://github.com/tarantool/metrics) `0.10.0` or greater
+found, metrics collectors will be used by default to store statistics
+instead of local collectors. Quantiles in metrics summary collections
+are disabled by default. You can manually choose driver and enable quantiles.
+```lua
+-- Use metrics collectors. (Default if metrics found).
+crud.cfg{ stats = true, stats_driver = 'metrics' }
+
+-- Use metrics collectors with 0.99 quantiles.
+crud.cfg{ stats = true, stats_driver = 'metrics', stats_quantiles = true }
+
+-- Use simple local collectors.
+crud.cfg{ stats = true, stats_driver = 'local' }
+```
+
 You can use `crud.cfg` to check current stats state.
 ```lua
 crud.cfg
 ---
-- stats: true
+- stats_quantiles: true
+  stats: true
+  stats_driver: local
 ...
 ```
 
@@ -660,9 +677,39 @@ Possible statistics operation labels are
 Each operation section contains of different collectors
 for success calls and error (both error throw and `nil, err`)
 returns. `count` is total requests count since instance start
-or stats restart. `latency` is average time of requests execution,
+or stats restart. `latency` is 0.99 quantile of request execution
+time if `metrics` driver used and quantiles enabled,
+otherwise `latency` is total average.
 `time` is the total time of requests execution.
 
+In [`metrics`](https://www.tarantool.io/en/doc/latest/book/monitoring/)
+registry statistics are stored as `tnt_crud_stats` metrics
+with `operation`, `status` and `name` labels.
+```
+metrics:collect()
+---
+- - label_pairs:
+      status: ok
+      operation: insert
+      name: customers
+    value: 221411
+    metric_name: tnt_crud_stats_count
+  - label_pairs:
+      status: ok
+      operation: insert
+      name: customers
+    value: 10.49834896344692
+    metric_name: tnt_crud_stats_sum
+  - label_pairs:
+      status: ok
+      operation: insert
+      name: customers
+      quantile: 0.99
+    value: 0.00023606420935973
+    metric_name: tnt_crud_stats
+...
+```
+
 `select` section additionally contains `details` collectors.
 ```lua
 crud.stats('my_space').select.details
@@ -679,6 +726,10 @@ looked up on storages while collecting responses for calls (including
 scrolls for multibatch requests). Details data is updated as part of
 the request process, so you may get new details before `select`/`pairs`
 call is finished and observed with count, latency and time collectors.
+In [`metrics`](https://www.tarantool.io/en/doc/latest/book/monitoring/)
+registry they are stored as `tnt_crud_map_reduces`,
+`tnt_crud_tuples_fetched` and `tnt_crud_tuples_lookup` metrics
+with `{ operation = 'select', name = space_name }` labels.
 
 Since `pairs` request behavior differs from any other crud request, its
 statistics collection also has specific behavior. Statistics (`select`
@@ -690,7 +741,10 @@ collector.
 
 Statistics are preserved between package reloads. Statistics are preserved
 between [Tarantool Cartridge role reloads](https://www.tarantool.io/en/doc/latest/book/cartridge/cartridge_api/modules/cartridge.roles/#reload)
-if you use CRUD Cartridge roles.
+if you use CRUD Cartridge roles. Beware that metrics 0.12.0 and below do not
+support preserving stats between role reload
+(see [tarantool/metrics#334](https://github.com/tarantool/metrics/issues/334)),
+thus this feature will be unsupported for `metrics` driver.
 
 ## Cartridge roles
 

crud/cfg.lua

@@ -16,11 +16,49 @@ local function set_defaults_if_empty(cfg)
         cfg.stats = false
     end
 
+    if cfg.stats_driver == nil then
+        cfg.stats_driver = stats.get_default_driver()
+    end
+
+    if cfg.stats_quantiles == nil then
+        cfg.stats_quantiles = false
+    end
+
     return cfg
 end
 
 local cfg = set_defaults_if_empty(stash.get(stash.name.cfg))
 
+local function configure_stats(cfg, opts)
+    if  (opts.stats == nil)
+    and (opts.stats_driver == nil)
+    and (opts.stats_quantiles == nil) then
+        return
+    end
+
+    if opts.stats == nil then
+        opts.stats = cfg.stats
+    end
+
+    if opts.stats_driver == nil then
+        opts.stats_driver = cfg.stats_driver
+    end
+
+    if opts.stats_quantiles == nil then
+        opts.stats_quantiles = cfg.stats_quantiles
+    end
+
+    if opts.stats == true then
+        stats.enable{ driver = opts.stats_driver, quantiles = opts.stats_quantiles }
+    else
+        stats.disable()
+    end
+
+    rawset(cfg, 'stats', opts.stats)
+    rawset(cfg, 'stats_driver', opts.stats_driver)
+    rawset(cfg, 'stats_quantiles', opts.stats_quantiles)
+end
+
 --- Configure CRUD module.
 --
 -- @local
@@ -32,23 +70,33 @@ local cfg = set_defaults_if_empty(stash.get(stash.name.cfg))
 -- @bool[opt] opts.stats
 --  Enable or disable statistics collect.
 --  Statistics are observed only on router instances.
+
+-- @string[opt] opts.stats_driver
+--  `'local'` or `'metrics'`.
+--  If `'local'`, stores statistics in local registry (some Lua tables)
+--  and computes latency as overall average. `'metrics'` requires
+--  `metrics >= 0.10.0` installed and stores statistics in
+--  global metrics registry (integrated with exporters).
+--  `'metrics'` driver supports computing latency as 0.99 quantile with aging.
+--  If `'metrics'` driver is available, it is used by default,
+--  otherwise `'local'` is used.
+--
+-- @bool[opt] opts.stats_quantiles
+--  Enable or disable statistics quantiles (only for metrics driver).
+--  Quantiles computations increases performance overhead up to 10%.
 --
 -- @return Copy of configuration table.
 --
 local function __call(self, opts)
-    checks('table', { stats = '?boolean' })
+    checks('table', {
+        stats = '?boolean',
+        stats_driver = '?string',
+        stats_quantiles = '?boolean'
+    })
 
-    opts = opts or {}
+    opts = table.deepcopy(opts) or {}
 
-    if opts.stats ~= nil then
-        if opts.stats == true then
-            stats.enable()
-        else
-            stats.disable()
-        end
-
-        rawset(cfg, 'stats', opts.stats)
-    end
+    configure_stats(cfg, opts)
 
     return self
 end

crud/common/stash.lua

@@ -16,10 +16,14 @@ local stash = {}
 -- @tfield string stats_local_registry
 --  Stash for local metrics registry.
 --
+-- @tfield string stats_metrics_registry
+--  Stash for metrics rocks statistics registry.
+--
 stash.name = {
     cfg = '__crud_cfg',
     stats_internal = '__crud_stats_internal',
-    stats_local_registry = '__crud_stats_local_registry'
+    stats_local_registry = '__crud_stats_local_registry',
+    stats_metrics_registry = '__crud_stats_metrics_registry'
 }
 
 --- Setup Tarantool Cartridge reload.

crud/stats/local_registry.lua

@@ -2,13 +2,16 @@
 -- @module crud.stats.local_registry
 --
 
+local errors = require('errors')
+
 local dev_checks = require('crud.common.dev_checks')
 local stash = require('crud.common.stash')
 local op_module = require('crud.stats.operation')
 local registry_common = require('crud.stats.registry_common')
 
 local registry = {}
 local internal = stash.get(stash.name.stats_local_registry)
+local StatsLocalError = errors.new_class('StatsLocalError', {capture_stack = false})
 
 --- Initialize local metrics registry.
 --
@@ -17,9 +20,19 @@ local internal = stash.get(stash.name.stats_local_registry)
 --
 -- @function init
 --
--- @treturn boolean Returns true.
+-- @tab opts
 --
-function registry.init()
+-- @bool opts.quantiles
+--  Quantiles is not supported for local, only `false` is valid.
+--
+-- @treturn boolean Returns `true`.
+--
+function registry.init(opts)
+    dev_checks({ quantiles = 'boolean' })
+
+    StatsLocalError:assert(opts.quantiles == false,
+        "Quantiles are not supported for 'local' statistics registry")
+
     internal.registry = {}
     internal.registry.spaces = {}