Skip to content

Commit e2331a6

Browse files
ligurioligurio
committed
Use sharding keys to calculate bucket id (WIP)
CRUD allows to automatically calculate `bucket_id` based on primary key or one can specify `bucket_id` explicitly [1]. However it is often required to calculate `bucket_id` using sharding keys created by DDL schema. DDL module exposes space with sharding keys as a part of public API [2], so everyone is allowed to set and get sharding keys there without adding DDL module to dependencies. Patch allows to calculate `bucket_id` value automatically when sharding keys used in a sharded space. 1. #46 2. https://github.com/tarantool/ddl#api 3. #46 (comment) Closes #166
1 parent 275e312 commit e2331a6

File tree

10 files changed

+297
-10
lines changed

10 files changed

+297
-10
lines changed

‎CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
1616

1717
### Added
1818

19+
* Support calculating `bucket_id` based on `ddl.sharding_key`.
1920
* Added jsonpath indexes support for queries
2021
* `tuple-merger` module updated to 0.0.2
2122

‎crud/delete.lua

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,8 @@ end
9494
--
9595
-- @tparam ?number opts.bucket_id
9696
-- Bucket ID
97-
-- (by default, it's vshard.router.bucket_id_strcrc32 of primary key)
97+
-- (by default, it's vshard.router.bucket_id_strcrc32 of primary key
98+
-- or secondary key)
9899
--
99100
-- @return[1] object
100101
-- @treturn[2] nil

‎crud/get.lua

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,8 @@ end
104104
--
105105
-- @tparam ?number opts.bucket_id
106106
-- Bucket ID
107-
-- (by default, it's vshard.router.bucket_id_strcrc32 of primary key)
107+
-- (by default, it's vshard.router.bucket_id_strcrc32 of primary
108+
-- or secondary key)
108109
--
109110
-- @tparam ?boolean opts.prefer_replica
110111
-- Call on replica if it's possible

‎crud/insert.lua

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,8 @@ end
107107
--
108108
-- @tparam ?number opts.bucket_id
109109
-- Bucket ID
110-
-- (by default, it's vshard.router.bucket_id_strcrc32 of primary key)
110+
-- (by default, it's vshard.router.bucket_id_strcrc32 of primary
111+
-- or secondary key)
111112
--
112113
-- @return[1] tuple
113114
-- @treturn[2] nil

‎crud/replace.lua

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,8 @@ end
111111
--
112112
-- @tparam ?number opts.bucket_id
113113
-- Bucket ID
114-
-- (by default, it's vshard.router.bucket_id_strcrc32 of primary key)
114+
-- (by default, it's vshard.router.bucket_id_strcrc32 of primary or
115+
-- secondary key)
115116
--
116117
-- @return[1] object
117118
-- @treturn[2] nil

‎crud/select/plan.lua

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ local errors = require('errors')
33
local compare_conditions = require('crud.compare.conditions')
44
local utils = require('crud.common.utils')
55
local dev_checks = require('crud.common.dev_checks')
6+
-- ddl module is optional
7+
local _, ddl = pcall(require, 'ddl')
68

79
local compat = require('crud.common.compat')
810
local has_keydef = compat.exists('tuple.keydef', 'key_def')
@@ -16,6 +18,7 @@ local select_plan = {}
1618

1719
local IndexTypeError = errors.new_class('IndexTypeError', {capture_stack = false})
1820
local FilterFieldsError = errors.new_class('FilterFieldsError', {capture_stack = false})
21+
local DDLError = errors.new_class('DDLError', {capture_stack = false})
1922

2023
local function index_is_allowed(index)
2124
return index.type == 'TREE'
@@ -226,12 +229,24 @@ function select_plan.new(space, conditions, opts)
226229
end
227230
end
228231

229-
local sharding_index = primary_index -- XXX: only sharding by primary key is supported
232+
local sharding_index = primary_index
230233

231-
-- get sharding key value
234+
-- getting sharding_key specified in DDL schema
232235
local sharding_key
233-
if scan_value ~= nil and (scan_iter == box.index.EQ or scan_iter == box.index.REQ) then
234-
sharding_key = extract_sharding_key_from_scan_value(scan_value, scan_index, sharding_index)
236+
if ddl ~= nil then
237+
local schema, err = ddl.get_schema(space_name)
238+
if schema == nil then
239+
return nil, DDLError:new('Failed to obtain a DDL schema')
240+
end
241+
sharding_key = schema.sharding_key
242+
end
243+
244+
-- getting sharding_key used in primary_index
245+
if sharding_key == nil then
246+
local sharding_index = primary_index
247+
if scan_value ~= nil and (scan_iter == box.index.EQ or scan_iter == box.index.REQ) then
248+
sharding_key = extract_sharding_key_from_scan_value(scan_value, scan_index, sharding_index)
249+
end
235250
end
236251

237252
if sharding_key ~= nil and opts.force_map_call ~= true then

‎crud/update.lua

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,8 @@ end
134134
--
135135
-- @tparam ?number opts.bucket_id
136136
-- Bucket ID
137-
-- (by default, it's vshard.router.bucket_id_strcrc32 of primary key)
137+
-- (by default, it's vshard.router.bucket_id_strcrc32 of primary or
138+
-- secondary key)
138139
--
139140
-- @return[1] object
140141
-- @treturn[2] nil

‎crud/upsert.lua

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,8 @@ end
116116
--
117117
-- @tparam ?number opts.bucket_id
118118
-- Bucket ID
119-
-- (by default, it's vshard.router.bucket_id_strcrc32 of primary key)
119+
-- (by default, it's vshard.router.bucket_id_strcrc32 of primary or
120+
-- secondary key)
120121
--
121122
-- @return[1] tuple
122123
-- @treturn[2] nil

‎test/entrypoint/srv_ddl.lua

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
#!/usr/bin/env tarantool
2+
3+
require('strict').on()
4+
_G.is_initialized = function() return false end
5+
6+
local log = require('log')
7+
local errors = require('errors')
8+
local cartridge = require('cartridge')
9+
local ddl = require('ddl')
10+
11+
package.preload['customers-storage'] = function()
12+
return {
13+
role_name = 'customers-storage',
14+
init = function()
15+
local engine = os.getenv('ENGINE') or 'memtx'
16+
local schema_customers = {
17+
is_local = true,
18+
engine = engine,
19+
temporary = false,
20+
sharding_key = {
21+
'name',
22+
'age',
23+
},
24+
format = {
25+
{name = 'id', type = 'unsigned', is_nullable = false},
26+
{name = 'bucket_id', type = 'unsigned', is_nullable = false},
27+
{name = 'name', type = 'string', is_nullable = false},
28+
{name = 'age', type = 'number', is_nullable = false},
29+
},
30+
indexes = {
31+
{
32+
name = 'primary',
33+
type = 'TREE',
34+
unique = true,
35+
parts = {
36+
{path = 'id', type = 'unsigned', is_nullable = false},
37+
},
38+
},
39+
{
40+
name = 'bucket_id',
41+
type = 'TREE',
42+
unique = false,
43+
parts = {
44+
{path = 'bucket_id', type = 'unsigned', is_nullable = false},
45+
},
46+
},
47+
}
48+
}
49+
local schema = {
50+
spaces = {
51+
['customers'] = schema_customers,
52+
},
53+
}
54+
if not box.cfg.read_only then
55+
local ok, err = ddl.set_schema(schema)
56+
if not ok then
57+
error(err)
58+
end
59+
end
60+
end,
61+
}
62+
end
63+
64+
local ok, err = errors.pcall('CartridgeCfgError', cartridge.cfg, {
65+
advertise_uri = 'localhost:3301',
66+
http_port = 8081,
67+
bucket_count = 3000,
68+
roles = {
69+
'customers-storage',
70+
'cartridge.roles.crud-router',
71+
'cartridge.roles.crud-storage',
72+
},
73+
})
74+
75+
if not ok then
76+
log.error('%s', err)
77+
os.exit(1)
78+
end
79+
80+
_G.is_initialized = cartridge.is_healthy

‎test/integration/ddl_sharding_key_test.lua

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
#!/usr/bin/env tarantool
2+
3+
local fio = require('fio')
4+
local t = require('luatest')
5+
local crud = require('crud')
6+
7+
local helpers = require('test.helper')
8+
9+
local ok, ddl = pcall(require, 'ddl')
10+
if not ok then
11+
t.skip('Please, install ddl rock to run tests')
12+
end
13+
14+
local pgroup = helpers.pgroup.new('ddl_sharding_key', {
15+
engine = {'memtx', 'vinyl'},
16+
})
17+
18+
pgroup:set_before_all(function(g)
19+
g.cluster = helpers.Cluster:new({
20+
datadir = fio.tempdir(),
21+
server_command = helpers.entrypoint('srv_ddl'),
22+
use_vshard = true,
23+
replicasets = helpers.get_test_replicasets(),
24+
env = {
25+
['ENGINE'] = g.params.engine,
26+
},
27+
})
28+
g.cluster:start()
29+
local result, err = g.cluster.main_server.net_box:eval([[
30+
local ddl = require('ddl')
31+
32+
local ok, err = ddl.get_schema()
33+
return ok, err
34+
]])
35+
t.assert_equals(type(result), 'table')
36+
t.assert_equals(err, nil)
37+
end)
38+
39+
pgroup:set_after_all(function(g) helpers.stop_cluster(g.cluster) end)
40+
41+
pgroup:set_before_each(function(g)
42+
helpers.truncate_space_on_cluster(g.cluster, 'customers')
43+
end)
44+
45+
pgroup:add('test_key', function(g)
46+
local result, err = g.cluster.main_server.net_box:call('crud.insert_object',
47+
{'customers', {id = 33, name = 'Mayakovsky', age = 36}})
48+
t.assert_equals(err, nil)
49+
end)
50+
51+
pgroup:add('test_select', function(g)
52+
t.skip('not implemented')
53+
end)
54+
55+
pgroup:add('test_insert_get', function(g)
56+
-- insert
57+
local result, err = g.cluster.main_server.net_box:call('crud.insert',
58+
{'customers', {2, box.NULL, 'Ivan', 20}})
59+
60+
t.assert_equals(err, nil)
61+
t.assert_equals(result.metadata, {
62+
{name = 'id', type = 'unsigned', is_nullable = false},
63+
{name = 'bucket_id', type = 'unsigned', is_nullable = false},
64+
{name = 'name', type = 'string', is_nullable = false},
65+
{name = 'age', type = 'number', is_nullable = false},
66+
})
67+
t.assert_equals(result.rows, {{2, 401, 'Ivan', 20}})
68+
69+
-- get
70+
local result, err = g.cluster.main_server.net_box:call('crud.get', {'customers', 2})
71+
72+
t.assert_equals(err, nil)
73+
t.assert(result ~= nil)
74+
t.assert_equals(result.rows, {{2, 401, 'Ivan', 20}})
75+
end)
76+
77+
pgroup:add('test_update', function(g)
78+
-- insert tuple
79+
local result, err = g.cluster.main_server.net_box:call(
80+
'crud.insert_object', {'customers', {id = 22, name = 'Leo', age = 72}})
81+
82+
t.assert_equals(err, nil)
83+
t.assert_equals(result.metadata, {
84+
{name = 'id', type = 'unsigned', is_nullable = false},
85+
{name = 'bucket_id', type = 'unsigned', is_nullable = false},
86+
{name = 'name', type = 'string', is_nullable = false},
87+
{name = 'age', type = 'number', is_nullable = false},
88+
})
89+
local objects = crud.unflatten_rows(result.rows, result.metadata)
90+
t.assert_equals(objects, {{id = 22, name = 'Leo', age = 72, bucket_id = 655}})
91+
92+
-- update age and name fields
93+
local result, err = g.cluster.main_server.net_box:call('crud.update', {'customers', 22, {
94+
{'+', 'age', 10},
95+
{'=', 'name', 'Leo Tolstoy'},
96+
}})
97+
98+
t.assert_equals(err, nil)
99+
local objects = crud.unflatten_rows(result.rows, result.metadata)
100+
t.assert_equals(objects, {{id = 22, name = 'Leo Tolstoy', age = 82, bucket_id = 655}})
101+
102+
-- get
103+
local result, err = g.cluster.main_server.net_box:call('crud.get', {'customers', 22})
104+
105+
t.assert_equals(err, nil)
106+
local objects = crud.unflatten_rows(result.rows, result.metadata)
107+
t.assert_equals(objects, {{id = 22, name = 'Leo Tolstoy', age = 82, bucket_id = 655}})
108+
end)
109+
110+
pgroup:add('test_delete', function(g)
111+
-- insert tuple
112+
local result, err = g.cluster.main_server.net_box:call(
113+
'crud.insert_object', {'customers', {id = 33, name = 'Mayakovsky', age = 36}})
114+
115+
t.assert_equals(err, nil)
116+
t.assert_equals(result.metadata, {
117+
{name = 'id', type = 'unsigned', is_nullable = false},
118+
{name = 'bucket_id', type = 'unsigned', is_nullable = false},
119+
{name = 'name', type = 'string', is_nullable = false},
120+
{name = 'age', type = 'number', is_nullable = false},
121+
})
122+
local objects = crud.unflatten_rows(result.rows, result.metadata)
123+
t.assert_equals(objects, {{id = 33, name = 'Mayakovsky', age = 36, bucket_id = 907}})
124+
125+
-- delete
126+
local result, err = g.cluster.main_server.net_box:call('crud.delete', {'customers', 33})
127+
128+
t.assert_equals(err, nil)
129+
if g.params.engine == 'memtx' then
130+
local objects = crud.unflatten_rows(result.rows, result.metadata)
131+
t.assert_equals(objects, {{id = 33, name = 'Mayakovsky', age = 36, bucket_id = 907}})
132+
else
133+
t.assert_equals(#result.rows, 0)
134+
end
135+
136+
-- get
137+
local result, err = g.cluster.main_server.net_box:call('crud.get', {'customers', 33})
138+
139+
t.assert_equals(err, nil)
140+
t.assert_equals(#result.rows, 0)
141+
end)
142+
143+
pgroup:add('test_replace', function(g)
144+
local result, err = g.cluster.main_server.net_box:call(
145+
'crud.replace', {'customers', {45, box.NULL, 'John Fedor', 99}})
146+
147+
t.assert_equals(err, nil)
148+
t.assert_equals(result.metadata, {
149+
{name = 'id', type = 'unsigned', is_nullable = false},
150+
{name = 'bucket_id', type = 'unsigned', is_nullable = false},
151+
{name = 'name', type = 'string', is_nullable = false},
152+
{name = 'age', type = 'number', is_nullable = false},
153+
})
154+
t.assert_equals(result.rows, {{45, 392, 'John Fedor', 99}})
155+
156+
-- replace again
157+
local result, err = g.cluster.main_server.net_box:call(
158+
'crud.replace', {'customers', {45, box.NULL, 'John Fedor', 100}})
159+
160+
t.assert_equals(err, nil)
161+
t.assert_equals(result.rows, {{45, 392, 'John Fedor', 100}})
162+
end)
163+
164+
pgroup:add('test_upsert', function(g)
165+
-- upsert tuple first time
166+
local result, err = g.cluster.main_server.net_box:call('crud.upsert',
167+
{'customers', {67, box.NULL, 'Saltykov-Shchedrin', 63}, {
168+
{'=', 'name', 'Mikhail Saltykov-Shchedrin'},
169+
}})
170+
171+
t.assert_equals(#result.rows, 0)
172+
t.assert_equals(result.metadata, {
173+
{name = 'id', type = 'unsigned', is_nullable = false},
174+
{name = 'bucket_id', type = 'unsigned', is_nullable = false},
175+
{name = 'name', type = 'string', is_nullable = false},
176+
{name = 'age', type = 'number', is_nullable = false},
177+
})
178+
t.assert_equals(err, nil)
179+
180+
-- get
181+
local result, err = g.cluster.main_server.net_box:call('crud.get', {'customers', 67})
182+
183+
t.assert_equals(err, nil)
184+
t.assert_equals(result.rows, {{67, 1143, 'Saltykov-Shchedrin', 63}})
185+
end)

0 commit comments

Comments
 (0)