api: support schema introspection

This patch add `crud.schema` introspection handle, which allows
to inspect which spaces are available and what format do they have.
The feature covers two use cases:
- when application users want to know what data they can manipulate,
- when crud integration tools want to know data format
  for pre-processing (for example, CRUD HTTP API).

Author: DifferentialOrange

CHANGELOG.md

@@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## Unreleased
 
+### Added
+* Space schema introspection API `crud.schema` (#380).
+
 ### Changed
 * `deps.sh` installs the `vshard` instead of the `cartridge` by default (#364).
   You could to specify an environment variable `CARTIRDGE_VERSION` to install

README.md

@@ -1711,6 +1711,82 @@ end
 rv:close()
 ```
 
+### Schema
+
+`crud` routers provide API to introspect spaces schema.
+
+```lua
+local schema, err = crud.update(space_name, opts)
+```
+
+where:
+
+* `space_name` (`?string`) - name of the space (if `nil`, provides info for all spaces)
+* `opts`:
+  * `timeout` (`?number`) - `vshard.call` timeout and vshard master
+    discovery timeout (in seconds), default value is 2
+  * `vshard_router` (`?string|table`) - Cartridge vshard group name or
+    vshard router instance. Set this parameter if your space is not
+    a part of the default vshard cluster
+
+Returns space schema (or spaces schema map), error.
+
+Beware that schema info is not exactly the same as underlying storage spaces schema.
+The reason is that `crud` generates `bucket_id`, if it isn't provided,
+so this field is actually nullable for a `crud` user. We also do not expose
+`bucket_id` index info since it's a vshard utility and do not related
+to application logic.
+
+**Example:**
+
+```lua
+crud.schema('customers')
+---
+- format:
+    - name: id
+      type: unsigned
+    - name: bucket_id
+      type: unsigned
+      is_nullable: true
+    - name: name
+      type: string
+    - name: age
+      type: number
+  indexes:
+    0:
+      unique: true
+      parts:
+      - fieldno: 1
+        type: unsigned
+        exclude_null: false
+        is_nullable: false
+      id: 0
+      type: TREE
+      name: primary_index
+    2:
+      unique: false
+      parts:
+      - fieldno: 4
+        type: number
+        exclude_null: false
+        is_nullable: false
+      id: 2
+      type: TREE
+      name: age
+...
+```
+
+```lua
+crud.schema()
+---
+- customers:
+    format: ...
+    indexes: ...
+  shops:
+    format: ...
+    indexes: ...
+```
+
 ## Cartridge roles
 
 `cartridge.roles.crud-storage` is a Tarantool Cartridge role that depends on the

crud.lua

@@ -21,6 +21,7 @@ local sharding_metadata = require('crud.common.sharding.sharding_metadata')
 local utils = require('crud.common.utils')
 local stats = require('crud.stats')
 local readview = require('crud.readview')
+local schema = require('crud.schema')
 
 local crud = {}
 
@@ -152,6 +153,10 @@ crud.storage_info = utils.storage_info
 -- @function readview
 crud.readview = readview.new
 
+-- @refer schema.call
+-- @function schema
+crud.schema = schema.call
+
 --- Initializes crud on node
 --
 -- Exports all functions that are used for calls

crud/common/schema.lua

@@ -113,11 +113,7 @@ function schema.wrap_func_reload(vshard_router, func, ...)
     return res, err
 end
 
-local function get_space_schema_hash(space)
-    if space == nil then
-        return ''
-    end
-
+schema.get_normalized_space_schema = function(space)
     local indexes_info = {}
     for i = 0, table.maxn(space.index) do
         local index = space.index[i]
@@ -133,12 +129,19 @@ local function get_space_schema_hash(space)
         end
     end
 
-    local space_info = {
+    return {
         format = space:format(),
         indexes = indexes_info,
     }
+end
+
+local function get_space_schema_hash(space)
+    if space == nil then
+        return ''
+    end
 
-    return digest.murmur(msgpack.encode(space_info))
+    local sch = schema.get_normalized_space_schema(space)
+    return digest.murmur(msgpack.encode(sch))
 end
 
 function schema.filter_obj_fields(obj, field_names)

crud/common/utils.lua

@@ -110,7 +110,7 @@ local function get_replicaset_by_replica_uuid(replicasets, uuid)
     return nil
 end
 
-function utils.get_space(space_name, vshard_router, timeout, replica_uuid)
+function utils.get_spaces(vshard_router, timeout, replica_uuid)
     local replicasets, replicaset
     timeout = timeout or const.DEFAULT_VSHARD_CALL_TIMEOUT
     local deadline = fiber.clock() + timeout
@@ -160,9 +160,17 @@ function utils.get_space(space_name, vshard_router, timeout, replica_uuid)
         return nil, GetSpaceError:new(error_msg)
     end
 
-    local space = replicaset.master.conn.space[space_name]
+    return replicaset.master.conn.space, nil, replicaset.master.conn.schema_version
+end
+
+function utils.get_space(space_name, vshard_router, timeout, replica_uuid)
+    local spaces, err, schema_version = utils.get_spaces(vshard_router, timeout, replica_uuid)
+
+    if spaces == nil then
+        return nil, err
+    end
 
-    return space, nil, replicaset.master.conn.schema_version
+    return spaces[space_name], err, schema_version
 end
 
 function utils.get_space_format(space_name, vshard_router)

crud/schema.lua

@@ -0,0 +1,112 @@
+local checks = require('checks')
+local errors = require('errors')
+
+local SchemaError = errors.new_class('SchemaError', {capture_stack = false})
+
+local schema_module = require('crud.common.schema')
+local utils = require('crud.common.utils')
+
+local schema = {}
+
+local system_spaces = {
+    -- https://github.com/tarantool/tarantool/blob/3240201a2f5bac3bddf8a74015db9b351954e0b5/src/box/schema_def.h#L77-L127
+    ['_vinyl_deferred_delete'] = true,
+    ['_schema'] = true,
+    ['_collation'] = true,
+    ['_vcollation'] = true,
+    ['_space'] = true,
+    ['_vspace'] = true,
+    ['_sequence'] = true,
+    ['_sequence_data'] = true,
+    ['_vsequence'] = true,
+    ['_index'] = true,
+    ['_vindex'] = true,
+    ['_func'] = true,
+    ['_vfunc'] = true,
+    ['_user'] = true,
+    ['_vuser'] = true,
+    ['_priv'] = true,
+    ['_vpriv'] = true,
+    ['_cluster'] = true,
+    ['_trigger'] = true,
+    ['_truncate'] = true,
+    ['_space_sequence'] = true,
+    ['_vspace_sequence'] = true,
+    ['_fk_constraint'] = true,
+    ['_ck_constraint'] = true,
+    ['_func_index'] = true,
+    ['_session_settings'] = true,
+    -- https://github.com/tarantool/vshard/blob/b3c27b32637863e9a03503e641bb7c8c69779a00/vshard/storage/init.lua#L752
+    ['_bucket'] = true,
+    -- https://github.com/tarantool/ddl/blob/b55d0ff7409f32e4d527e2d25444d883bce4163b/test/set_sharding_metadata_test.lua#L92-L98
+    ['_ddl_sharding_key'] = true,
+    ['_ddl_sharding_func'] = true,
+}
+
+local function get_crud_schema(space)
+    local sch = schema_module.get_normalized_space_schema(space)
+
+    -- bucket_id is not nullable for a storage, yet
+    -- it is optional for a crud user.
+    for _, v in ipairs(sch.format) do
+        if v.name == 'bucket_id' then
+            v.is_nullable = true
+        end
+    end
+
+    for id, v in pairs(sch.indexes) do
+        -- There is no reason for a user to know about
+        -- bucket_id index.
+        if v.name == 'bucket_id' then
+            sch.indexes[id] = nil
+        end
+    end
+
+    return sch
+end
+
+schema.call = function(space_name, opts)
+    checks('?string', {
+        vshard_router = '?string|table',
+        timeout = '?number',
+    })
+
+    opts = opts or {}
+
+    local vshard_router, err = utils.get_vshard_router_instance(opts.vshard_router)
+    if err ~= nil then
+        return nil, SchemaError:new(err)
+    end
+
+    local _, err = schema_module.reload_schema(vshard_router)
+    if err ~= nil then
+        return nil, SchemaError:new(err)
+    end
+
+    local spaces, err = utils.get_spaces(vshard_router, opts.timeout)
+    if err ~= nil then
+        return nil, SchemaError:new(err)
+    end
+
+    if space_name ~= nil then
+        local space = spaces[space_name]
+        if space == nil then
+            return nil, SchemaError:new("Space %q doesn't exist", space_name)
+        end
+        return get_crud_schema(space)
+    else
+        local resp = {}
+
+        for name, space in pairs(spaces) do
+            -- Can be indexed by space id and space name,
+            -- so we need to be careful with duplicates.
+            if type(name) == 'string' and system_spaces[name] == nil then
+                resp[name] = get_crud_schema(space)
+            end
+        end
+
+        return resp
+    end
+end
+
+return schema

test/entrypoint/srv_schema/cartridge_init.lua

@@ -0,0 +1,39 @@
+#!/usr/bin/env tarantool
+
+require('strict').on()
+_G.is_initialized = function() return false end
+
+local log = require('log')
+local errors = require('errors')
+local cartridge = require('cartridge')
+
+if package.setsearchroot ~= nil then
+    package.setsearchroot()
+else
+    package.path = package.path .. debug.sourcedir() .. "/?.lua;"
+end
+
+package.preload['customers-storage'] = function()
+    return {
+        role_name = 'customers-storage',
+        init = require('storage_init')
+    }
+end
+
+local ok, err = errors.pcall('CartridgeCfgError', cartridge.cfg, {
+    advertise_uri = 'localhost:3301',
+    http_port = 8081,
+    bucket_count = 3000,
+    roles = {
+        'cartridge.roles.crud-router',
+        'cartridge.roles.crud-storage',
+        'customers-storage',
+    }}
+)
+
+if not ok then
+    log.error('%s', err)
+    os.exit(1)
+end
+
+_G.is_initialized = cartridge.is_healthy

test/entrypoint/srv_schema/storage_init.lua

@@ -0,0 +1,53 @@
+return function()
+    if box.info.ro == true then
+        return
+    end
+
+    local engine = os.getenv('ENGINE') or 'memtx'
+
+    local customers_space = box.schema.space.create('customers', {
+        format = {
+            {name = 'id', type = 'unsigned'},
+            {name = 'bucket_id', type = 'unsigned'},
+            {name = 'name', type = 'string'},
+            {name = 'age', type = 'number'},
+        },
+        if_not_exists = true,
+        engine = engine,
+    })
+    customers_space:create_index('id', {
+        parts = { {field = 'id'} },
+        if_not_exists = true,
+    })
+    customers_space:create_index('bucket_id', {
+        parts = { {field = 'bucket_id'} },
+        unique = false,
+        if_not_exists = true,
+    })
+
+    local shops_space = box.schema.space.create('shops', {
+        format = {
+            {name = 'registry_id', type = 'unsigned'},
+            {name = 'bucket_id', type = 'unsigned'},
+            {name = 'name', type = 'string'},
+            {name = 'address', type = 'string'},
+            {name = 'owner', type = 'string', is_nullable = true},
+        },
+        if_not_exists = true,
+        engine = engine,
+    })
+    shops_space:create_index('registry', {
+        parts = { {field = 'registry_id'} },
+        if_not_exists = true,
+    })
+    shops_space:create_index('bucket_id', {
+        parts = { {field = 'bucket_id'} },
+        unique = false,
+        if_not_exists = true,
+    })
+    shops_space:create_index('address', {
+        parts = { {field = 'address'} },
+        unique = true,
+        if_not_exists = true,
+    })
+end