Skip to content

Commit d772851

Browse files
committed
Jsonpath
1 parent 9caf4b3 commit d772851

15 files changed

+657
-132
lines changed

CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
3131
* Functions ``stop()`` for the roles ``crud-storage`` and ``crud-router``.
3232
* Option flag `force_map_call` for `select()`/`pairs()`
3333
to disable the `bucket_id` computation from primary key.
34+
* Added support for jsonpath for select.
3435

3536
## [0.6.0] - 2021-03-29
3637

crud/common/utils.lua

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,13 @@ local function determine_enabled_features()
181181

182182
-- since Tarantool 2.4
183183
enabled_tarantool_features.uuids = major >= 2 and (minor > 4 or minor == 4 and patch >= 1)
184+
185+
-- since Tarantool 1.10
186+
enabled_tarantool_features.jsonpath_filters = major >= 2 or (major >= 1 and minor >= 10)
187+
188+
-- since Tarantool 2.6.3 / 2.7.2 / 2.8.1
189+
enabled_tarantool_features.jsonpath_indexes = major >= 2 and ((minor >= 8 and patch >= 1) or
190+
(minor >= 6 and patch >= 3) or (minor >= 7 and patch >= 2))
184191
end
185192

186193
function utils.tarantool_supports_fieldpaths()
@@ -199,6 +206,22 @@ function utils.tarantool_supports_uuids()
199206
return enabled_tarantool_features.uuids
200207
end
201208

209+
function utils.tarantool_supports_jsonpath_filters()
210+
if enabled_tarantool_features.jsonpath_filters == nil then
211+
determine_enabled_features()
212+
end
213+
214+
return enabled_tarantool_features.jsonpath_filters
215+
end
216+
217+
function utils.tarantool_supports_jsonpath_indexes()
218+
if enabled_tarantool_features.jsonpath_indexes == nil then
219+
determine_enabled_features()
220+
end
221+
222+
return enabled_tarantool_features.jsonpath_indexes
223+
end
224+
202225
local function add_nullable_fields_recursive(operations, operations_map, space_format, tuple, id)
203226
if id < 2 or tuple[id - 1] ~= box.NULL then
204227
return operations

crud/compare/conditions.lua

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ for func_name, operator in pairs(cond_operators_by_func_names) do
6363
return new_condition({
6464
operator = operator,
6565
operand = operand,
66-
values = values
66+
values = values,
6767
})
6868
end
6969
end

crud/select/compat/select.lua

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -34,12 +34,6 @@ local function build_select_iterator(space_name, user_conditions, opts)
3434
return nil, SelectError:new("batch_size should be > 0")
3535
end
3636

37-
-- check conditions
38-
local conditions, err = compare_conditions.parse(user_conditions)
39-
if err ~= nil then
40-
return nil, SelectError:new("Failed to parse conditions: %s", err)
41-
end
42-
4337
local replicasets, err = vshard.router.routeall()
4438
if err ~= nil then
4539
return nil, SelectError:new("Failed to get all replicasets: %s", err)
@@ -51,6 +45,12 @@ local function build_select_iterator(space_name, user_conditions, opts)
5145
end
5246
local space_format = space:format()
5347

48+
-- check conditions
49+
local conditions, err = select_conditions.parse(user_conditions, space_format)
50+
if err ~= nil then
51+
return nil, SelectError:new("Failed to parse conditions: %s", err)
52+
end
53+
5454
-- plan select
5555
local plan, err = select_plan.new(space, conditions, {
5656
first = opts.first,

crud/select/compat/select_old.lua

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -86,12 +86,6 @@ local function build_select_iterator(space_name, user_conditions, opts)
8686

8787
local batch_size = opts.batch_size or common.DEFAULT_BATCH_SIZE
8888

89-
-- check conditions
90-
local conditions, err = compare_conditions.parse(user_conditions)
91-
if err ~= nil then
92-
return nil, SelectError:new("Failed to parse conditions: %s", err)
93-
end
94-
9589
local replicasets, err = vshard.router.routeall()
9690
if err ~= nil then
9791
return nil, SelectError:new("Failed to get all replicasets: %s", err)
@@ -103,6 +97,12 @@ local function build_select_iterator(space_name, user_conditions, opts)
10397
end
10498
local space_format = space:format()
10599

100+
-- check conditions
101+
local conditions, err = select_conditions.parse(user_conditions, space_format)
102+
if err ~= nil then
103+
return nil, SelectError:new("Failed to parse conditions: %s", err)
104+
end
105+
106106
-- plan select
107107
local plan, err = select_plan.new(space, conditions, {
108108
first = opts.first,

crud/select/filters.lua

Lines changed: 66 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,10 @@
1-
local json = require('json')
21
local errors = require('errors')
32

43
local utils = require('crud.common.utils')
54
local dev_checks = require('crud.common.dev_checks')
65
local collations = require('crud.common.collations')
76
local compare_conditions = require('crud.compare.conditions')
87

9-
local ParseConditionsError = errors.new_class('ParseConditionsError', {capture_stack = false})
108
local GenFiltersError = errors.new_class('GenFiltersError', {capture_stack = false})
119

1210
local filters = {}
@@ -97,31 +95,42 @@ local function parse(space, conditions, opts)
9795
for i, condition in ipairs(conditions) do
9896
if i ~= opts.scan_condition_num then
9997
-- Index check (including one and multicolumn)
100-
local fieldnos
101-
local fields_types
98+
local fields
99+
local fields_types = {}
102100
local values_opts
103101

104102
local index = space_indexes[condition.operand]
105103

106104
if index ~= nil then
107-
fieldnos = get_index_fieldnos(index)
105+
fields = get_index_fieldnos(index)
108106
fields_types = get_index_fields_types(index)
109107
values_opts = get_values_opts(index)
110-
elseif fieldnos_by_names[condition.operand] ~= nil then
111-
local fiendno = fieldnos_by_names[condition.operand]
112-
fieldnos = {fiendno}
113-
local field_format = space_format[fiendno]
114-
fields_types = {field_format.type}
115-
local is_nullable = field_format.is_nullable == true
108+
else
109+
local fieldno = fieldnos_by_names[condition.operand]
110+
111+
if fieldno ~= nil then
112+
fields = {fieldno}
113+
else
114+
-- We assume this is jsonpath, so it is
115+
-- not in fieldnos_by_name map.
116+
fields = {condition.operand}
117+
end
118+
119+
local field_format = space_format[fieldno]
120+
local is_nullable
121+
122+
if field_format ~= nil then
123+
fields_types = {field_format.type}
124+
is_nullable = field_format.is_nullable == true
125+
end
126+
116127
values_opts = {
117128
{is_nullable = is_nullable, collation = nil},
118129
}
119-
else
120-
return nil, ParseConditionsError('No field or index is found for condition %s', json.encode(condition))
121130
end
122131

123132
table.insert(filter_conditions, {
124-
fieldnos = fieldnos,
133+
fields = fields,
125134
operator = condition.operator,
126135
values = condition.values,
127136
types = fields_types,
@@ -156,12 +165,30 @@ end
156165
local PARSE_ARGS_TEMPLATE = 'local tuple = ...'
157166
local LIB_FUNC_HEADER_TEMPLATE = 'function M.%s(%s)'
158167

168+
local function format_path(path)
169+
local path_type = type(path)
170+
if path_type == 'number' then
171+
return tostring(path)
172+
elseif path_type == 'string' then
173+
return ('%q'):format(path)
174+
end
175+
176+
assert(false, ('Unexpected format: %s'):format(path_type))
177+
end
178+
159179
local function concat_conditions(conditions, operator)
160180
return '(' .. table.concat(conditions, (' %s '):format(operator)) .. ')'
161181
end
162182

163-
local function get_field_variable_name(fieldno)
164-
return string.format('field_%s', fieldno)
183+
local function get_field_variable_name(field)
184+
local field_type = type(field)
185+
if field_type == 'number' then
186+
field = tostring(field)
187+
elseif field_type == 'string' then
188+
field = string.gsub(field, '([().^$%[%]%+%-%*%?%%\'"])', '_')
189+
end
190+
191+
return string.format('field_%s', field)
165192
end
166193

167194
local function get_eq_func_name(id)
@@ -173,38 +200,39 @@ local function get_cmp_func_name(id)
173200
end
174201

175202
local function gen_tuple_fields_def_code(filter_conditions)
176-
-- get field numbers
177-
local fieldnos_added = {}
178-
local fieldnos = {}
203+
-- get field names
204+
local fields_added = {}
205+
local fields = {}
179206

180207
for _, cond in ipairs(filter_conditions) do
181208
for i = 1, #cond.values do
182-
local fieldno = cond.fieldnos[i]
183-
if not fieldnos_added[fieldno] then
184-
table.insert(fieldnos, fieldno)
185-
fieldnos_added[fieldno] = true
209+
local field = cond.fields[i]
210+
211+
if not fields_added[field] then
212+
table.insert(fields, field)
213+
fields_added[field] = true
186214
end
187215
end
188216
end
189217

190218
-- gen definitions for all used fields
191219
local fields_def_parts = {}
192220

193-
for _, fieldno in ipairs(fieldnos) do
221+
for _, field in ipairs(fields) do
194222
table.insert(fields_def_parts, string.format(
195223
'local %s = tuple[%s]',
196-
get_field_variable_name(fieldno), fieldno
224+
get_field_variable_name(field), format_path(field)
197225
))
198226
end
199227

200228
return table.concat(fields_def_parts, '\n')
201229
end
202230

203-
local function format_comp_with_value(fieldno, func_name, value)
231+
local function format_comp_with_value(field, func_name, value)
204232
return string.format(
205233
'%s(%s, %s)',
206234
func_name,
207-
get_field_variable_name(fieldno),
235+
get_field_variable_name(field),
208236
format_value(value)
209237
)
210238
end
@@ -238,7 +266,7 @@ local function format_eq(cond)
238266
local values_opts = cond.values_opts or {}
239267

240268
for j = 1, #cond.values do
241-
local fieldno = cond.fieldnos[j]
269+
local field = cond.fields[j]
242270
local value = cond.values[j]
243271
local value_type = cond.types[j]
244272
local value_opts = values_opts[j] or {}
@@ -254,7 +282,7 @@ local function format_eq(cond)
254282
func_name = 'eq_uuid'
255283
end
256284

257-
table.insert(cond_strings, format_comp_with_value(fieldno, func_name, value))
285+
table.insert(cond_strings, format_comp_with_value(field, func_name, value))
258286
end
259287

260288
return cond_strings
@@ -265,7 +293,7 @@ local function format_lt(cond)
265293
local values_opts = cond.values_opts or {}
266294

267295
for j = 1, #cond.values do
268-
local fieldno = cond.fieldnos[j]
296+
local field = cond.fields[j]
269297
local value = cond.values[j]
270298
local value_type = cond.types[j]
271299
local value_opts = values_opts[j] or {}
@@ -279,9 +307,10 @@ local function format_lt(cond)
279307
elseif value_type == 'uuid' then
280308
func_name = 'lt_uuid'
281309
end
310+
282311
func_name = add_strict_postfix(func_name, value_opts)
283312

284-
table.insert(cond_strings, format_comp_with_value(fieldno, func_name, value))
313+
table.insert(cond_strings, format_comp_with_value(field, func_name, value))
285314
end
286315

287316
return cond_strings
@@ -366,10 +395,10 @@ local function gen_cmp_array_func_code(operator, func_name, cond, func_args_code
366395
return table.concat(func_code_lines, '\n')
367396
end
368397

369-
local function function_args_by_fieldnos(fieldnos)
398+
local function function_args_by_field(fields)
370399
local arg_names = {}
371-
for _, fieldno in ipairs(fieldnos) do
372-
table.insert(arg_names, get_field_variable_name(fieldno))
400+
for _, field in ipairs(fields) do
401+
table.insert(arg_names, get_field_variable_name(field))
373402
end
374403
return table.concat(arg_names, ', ')
375404
end
@@ -408,8 +437,8 @@ local function gen_filter_code(filter_conditions)
408437
table.insert(filter_code_parts, '')
409438

410439
for i, cond in ipairs(filter_conditions) do
411-
local args_fieldnos = { unpack(cond.fieldnos, 1, #cond.values) }
412-
local func_args_code = function_args_by_fieldnos(args_fieldnos)
440+
local args_fields = { unpack(cond.fields, 1, #cond.values) }
441+
local func_args_code = function_args_by_field(args_fields)
413442

414443
local library_func_name, library_func_code = gen_library_func(i, cond, func_args_code)
415444
table.insert(library_funcs_code_parts, library_func_code)
@@ -623,6 +652,7 @@ function filters.gen_func(space, conditions, opts)
623652
scan_condition_num = opts.scan_condition_num,
624653
tarantool_iter = opts.tarantool_iter,
625654
})
655+
626656
if err ~= nil then
627657
return nil, GenFiltersError:new("Failed to generate filters for specified conditions: %s", err)
628658
end

crud/select/plan.lua

Lines changed: 0 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,7 @@ local dev_checks = require('crud.common.dev_checks')
66

77
local select_plan = {}
88

9-
local SelectPlanError = errors.new_class('SelectPlanError', {capture_stack = false})
109
local IndexTypeError = errors.new_class('IndexTypeError', {capture_stack = false})
11-
local ValidateConditionsError = errors.new_class('ValidateConditionsError', {capture_stack = false})
1210
local FilterFieldsError = errors.new_class('FilterFieldsError', {capture_stack = false})
1311

1412
local function index_is_allowed(index)
@@ -42,34 +40,6 @@ local function get_index_for_condition(space_indexes, space_format, condition)
4240
end
4341
end
4442

45-
local function validate_conditions(conditions, space_indexes, space_format)
46-
local field_names = {}
47-
for _, field_format in ipairs(space_format) do
48-
field_names[field_format.name] = true
49-
end
50-
51-
local index_names = {}
52-
53-
-- If we use # (not table.maxn), we may lose indexes, when user drop some indexes.
54-
-- E.g: we have table with indexes id {1, 2, 3, nil, nil, 6}.
55-
-- If we use #{1, 2, 3, nil, nil, 6} (== 3) we will lose index with id = 6.
56-
-- See details: https://github.com/tarantool/crud/issues/103
57-
for i = 0, table.maxn(space_indexes) do
58-
local index = space_indexes[i]
59-
if index ~= nil then
60-
index_names[index.name] = true
61-
end
62-
end
63-
64-
for _, condition in ipairs(conditions) do
65-
if index_names[condition.operand] == nil and field_names[condition.operand] == nil then
66-
return false, ValidateConditionsError:new("No field or index %q found", condition.operand)
67-
end
68-
end
69-
70-
return true
71-
end
72-
7343
local function extract_sharding_key_from_scan_value(scan_value, scan_index, sharding_index)
7444
if #scan_value < #sharding_index.parts then
7545
return nil
@@ -181,11 +151,6 @@ function select_plan.new(space, conditions, opts)
181151
local space_indexes = space.index
182152
local space_format = space:format()
183153

184-
local ok, err = validate_conditions(conditions, space_indexes, space_format)
185-
if not ok then
186-
return nil, SelectPlanError:new('Passed bad conditions: %s', err)
187-
end
188-
189154
if conditions == nil then -- also cdata<NULL>
190155
conditions = {}
191156
end

0 commit comments

Comments
 (0)