diff --git a/.changeset/sharp-phones-travel.md b/.changeset/sharp-phones-travel.md new file mode 100644 index 000000000..36eb89164 --- /dev/null +++ b/.changeset/sharp-phones-travel.md @@ -0,0 +1,6 @@ +--- +"@hyperdx/common-utils": patch +"@hyperdx/app": patch +--- + +fix: Fix sidebar when selecting JSON property diff --git a/packages/app/src/hooks/useRowWhere.tsx b/packages/app/src/hooks/useRowWhere.tsx index 0c354f04a..62d438c80 100644 --- a/packages/app/src/hooks/useRowWhere.tsx +++ b/packages/app/src/hooks/useRowWhere.tsx @@ -66,7 +66,7 @@ export function processRowToWhereClause( // Currently we can't distinguish null or 'null' if (value === 'null') { - return SqlString.format(`isNull(??)`, [column]); + return SqlString.format(`isNull(??)`, [valueExpr]); } if (value.length > 1000 || column.length > 1000) { console.warn('Search value/object key too large.'); diff --git a/packages/common-utils/src/__tests__/clickhouse.test.ts b/packages/common-utils/src/__tests__/clickhouse.test.ts index 7873e9f7e..6136ffaec 100644 --- a/packages/common-utils/src/__tests__/clickhouse.test.ts +++ b/packages/common-utils/src/__tests__/clickhouse.test.ts @@ -200,6 +200,27 @@ describe('chSqlToAliasMap - alias unit test', () => { }; expect(res).toEqual(aliasMap); }); + + it('Alias, with JSON expressions', () => { + const chSqlInput: ChSql = { + sql: "SELECT Timestamp as ts,ResourceAttributes.service.name as service,toStartOfDay(LogAttributes.start.`time`) as start_time,Body,TimestampTime,ServiceName,TimestampTime FROM {HYPERDX_PARAM_1544803905:Identifier}.{HYPERDX_PARAM_129845054:Identifier} WHERE (TimestampTime >= fromUnixTimestamp64Milli({HYPERDX_PARAM_1456399765:Int64}) AND TimestampTime <= fromUnixTimestamp64Milli({HYPERDX_PARAM_1719057412:Int64})) AND (`ResourceAttributes`.`service`.`name` = 'serviceName') ORDER BY TimestampTime DESC LIMIT {HYPERDX_PARAM_49586:Int32} OFFSET {HYPERDX_PARAM_48:Int32}", + params: { + HYPERDX_PARAM_1544803905: 'default', + HYPERDX_PARAM_129845054: 'otel_logs', + HYPERDX_PARAM_1456399765: 1743038742000, + HYPERDX_PARAM_1719057412: 1743040542000, + HYPERDX_PARAM_49586: 200, + HYPERDX_PARAM_48: 0, + }, + }; + const res = chSqlToAliasMap(chSqlInput); + const aliasMap = { + ts: 'Timestamp', + service: 'ResourceAttributes.service.name', + start_time: 'toStartOfDay(LogAttributes.start.`time`)', + }; + expect(res).toEqual(aliasMap); + }); }); describe('computeRatio', () => { diff --git a/packages/common-utils/src/__tests__/utils.test.ts b/packages/common-utils/src/__tests__/utils.test.ts index a56066de8..90482d9b7 100644 --- a/packages/common-utils/src/__tests__/utils.test.ts +++ b/packages/common-utils/src/__tests__/utils.test.ts @@ -10,11 +10,13 @@ import { import { convertToDashboardTemplate, + findJsonExpressions, formatDate, getFirstOrderingItem, isFirstOrderByAscending, + isJsonExpression, isTimestampExpressionInFirstOrderBy, - removeTrailingDirection, + replaceJsonExpressions, splitAndTrimCSV, splitAndTrimWithBracket, } from '../utils'; @@ -674,4 +676,443 @@ describe('utils', () => { }); }); }); + + describe('isJsonExpression', () => { + it('should return false for expressions without dots', () => { + expect(isJsonExpression('col')).toBe(false); + expect(isJsonExpression('columnName')).toBe(false); + expect(isJsonExpression('column_name')).toBe(false); + }); + + it('should return true for simple JSON expressions', () => { + expect(isJsonExpression('col.key')).toBe(true); + expect(isJsonExpression('column.property')).toBe(true); + }); + + it('should return true for nested JSON expressions', () => { + expect(isJsonExpression('col.key.nestedKey')).toBe(true); + expect(isJsonExpression('a.b.c')).toBe(true); + expect(isJsonExpression('json_col.col3.c')).toBe(true); + }); + + it('should return true for JSON expressions with double quotes', () => { + expect(isJsonExpression('"json_col"."key"')).toBe(true); + expect(isJsonExpression('"a"."b"."cde"')).toBe(true); + expect(isJsonExpression('"a_b.2c".b."c."')).toBe(true); + }); + + it('should return true for JSON expressions with backticks', () => { + expect(isJsonExpression('`a`.`b`.`cde`')).toBe(true); + expect(isJsonExpression('`col`.`key`')).toBe(true); + }); + + it('should return true for mixed quoting styles', () => { + expect(isJsonExpression('"a".b.`c`')).toBe(true); + expect(isJsonExpression('a."b".c')).toBe(true); + }); + + it('should return false for expressions with only one non-numeric part', () => { + expect(isJsonExpression('col.')).toBe(false); + expect(isJsonExpression('.col')).toBe(false); + }); + + it('should return false for decimal numbers', () => { + expect(isJsonExpression('10.50')).toBe(false); + expect(isJsonExpression('2.3')).toBe(false); + expect(isJsonExpression('1.5')).toBe(false); + }); + + it('should return false for table.column references with numeric column', () => { + expect(isJsonExpression('table.1')).toBe(false); + }); + + it('should return false for expressions with empty parts', () => { + expect(isJsonExpression('.')).toBe(false); + expect(isJsonExpression('..')).toBe(false); + expect(isJsonExpression('a..')).toBe(false); + }); + + it('should handle dots inside double quotes correctly', () => { + expect(isJsonExpression('"a.b.c"')).toBe(false); + expect(isJsonExpression('"a.b"."c.d"')).toBe(true); + }); + + it('should handle dots inside backticks correctly', () => { + expect(isJsonExpression('`a.b.c`')).toBe(false); + expect(isJsonExpression('`a.b`.`c.d`')).toBe(true); + }); + + it('should return true for mixed quoted and unquoted parts', () => { + expect(isJsonExpression('"col.with.dots".key')).toBe(true); + expect(isJsonExpression('col."key.with.dots"')).toBe(true); + }); + + it('should handle complex quoted identifiers', () => { + expect(isJsonExpression('"table.name"."column.name"."nested"')).toBe( + true, + ); + }); + + it('should handle expressions with underscores and numbers', () => { + expect(isJsonExpression('col_1.key_2.nested_3')).toBe(true); + expect(isJsonExpression('table123.column456')).toBe(true); + }); + + it('should return false for single quoted identifier', () => { + expect(isJsonExpression('"singleColumn"')).toBe(false); + expect(isJsonExpression('`singleColumn`')).toBe(false); + }); + + it('should handle type specifiers', () => { + expect(isJsonExpression('a.b.:UInt64')).toBe(true); + expect(isJsonExpression('col.key.:String')).toBe(true); + }); + + it('should handle whitespace in parts', () => { + expect(isJsonExpression('a . b')).toBe(true); + expect(isJsonExpression('a.b. c')).toBe(true); + }); + + it('should handle leading whitespace', () => { + expect(isJsonExpression(' a.b.c')).toBe(true); + expect(isJsonExpression(' col.key')).toBe(true); + expect(isJsonExpression('\ta.b')).toBe(true); + }); + + it('should handle trailing whitespace', () => { + expect(isJsonExpression('a.b.c ')).toBe(true); + expect(isJsonExpression('col.key ')).toBe(true); + expect(isJsonExpression('a.b\t')).toBe(true); + }); + + it('should handle leading and trailing whitespace', () => { + expect(isJsonExpression(' a.b.c ')).toBe(true); + expect(isJsonExpression(' col.key ')).toBe(true); + expect(isJsonExpression('\ta.b\t')).toBe(true); + }); + + it('should correctly handle single quoted strings', () => { + expect(isJsonExpression("'a'.b.c")).toBe(false); + expect(isJsonExpression("'a'.'b'")).toBe(false); + expect(isJsonExpression("'a' . 'b'")).toBe(false); + expect(isJsonExpression("'")).toBe(false); + expect(isJsonExpression("''")).toBe(false); + expect(isJsonExpression("`'a'`.b")).toBe(true); + expect(isJsonExpression("`'a`.b")).toBe(true); + }); + }); + + describe('findJsonExpressions', () => { + it('should handle empty expression', () => { + const sql = ''; + const actual = findJsonExpressions(sql); + const expected = []; + expect(actual).toEqual(expected); + }); + + it('should find a single JSON expression', () => { + const sql = 'SELECT a.b.c as alias1, col2 as alias2 FROM table'; + const actual = findJsonExpressions(sql); + const expected = [{ index: 7, expr: 'a.b.c' }]; + expect(actual).toEqual(expected); + }); + + it('should find multiple JSON expression', () => { + const sql = 'SELECT a.b.c, d.e, col2 FROM table'; + const actual = findJsonExpressions(sql); + const expected = [ + { index: 7, expr: 'a.b.c' }, + { index: 14, expr: 'd.e' }, + ]; + expect(actual).toEqual(expected); + }); + + it('should find JSON expression with type specifier', () => { + const sql = 'SELECT a.b.:UInt64, col2 FROM table'; + const actual = findJsonExpressions(sql); + const expected = [{ index: 7, expr: 'a.b.:UInt64' }]; + expect(actual).toEqual(expected); + }); + + it('should find JSON expression with complex type specifier', () => { + const sql = 'SELECT a.b.:Array(String) , col2 FROM table'; + const actual = findJsonExpressions(sql); + const expected = [{ index: 7, expr: 'a.b.:Array(String)' }]; + expect(actual).toEqual(expected); + }); + + it('should find JSON expressions in WHERE clause ', () => { + const sql = + 'SELECT col2 FROM table WHERE a.b.:UInt64 = 1 AND toStartOfDay(a.date) = today()'; + const actual = findJsonExpressions(sql); + const expected = [ + { index: 29, expr: 'a.b.:UInt64' }, + { index: 62, expr: 'a.date' }, + ]; + expect(actual).toEqual(expected); + }); + + it('should find JSON expressions in function calls', () => { + const sql = "SELECT JSONExtractString(a.b.c, 'key') FROM table"; + const actual = findJsonExpressions(sql); + const expected = [{ index: 25, expr: 'a.b.c' }]; + expect(actual).toEqual(expected); + }); + + it('should not find JSON expressions in quoted strings', () => { + const sql = + "SELECT a.b.c, ResourceAttributes['key.key2'], 'a.b.c' FROM table"; + const actual = findJsonExpressions(sql); + const expected = [ + { + index: 7, + expr: 'a.b.c', + }, + ]; + expect(actual).toEqual(expected); + }); + + it('should find JSON expressions in math expression', () => { + const sql = + 'SELECT toStartOfDay(a.date + INTERVAL 1 DAY), toStartOfDay(a.date+INTERVAL 1 DAY)'; + const actual = findJsonExpressions(sql); + const expected = [ + { index: 20, expr: 'a.date' }, + { index: 59, expr: 'a.date' }, + ]; + expect(actual).toEqual(expected); + }); + + it('should not infinite loop due to unterminated strings', () => { + const sql = 'SELECT "'; + const actual = findJsonExpressions(sql); + const expected = []; + expect(actual).toEqual(expected); + }); + + it('should not infinite loop due to trailing whitespace', () => { + const sql = 'SELECT '; + const actual = findJsonExpressions(sql); + const expected = []; + expect(actual).toEqual(expected); + }); + + it('should not infinite loop due to mismatched parenthesis', () => { + const sql = 'SELECT ('; + const actual = findJsonExpressions(sql); + const expected = []; + expect(actual).toEqual(expected); + }); + + it('should not infinite loop due to trailing json type specifier', () => { + const sql = 'SELECT a.b.:UInt64'; + const actual = findJsonExpressions(sql); + const expected = [{ index: 7, expr: 'a.b.:UInt64' }]; + expect(actual).toEqual(expected); + }); + + it('should not find JSON expressions in string that has escaped single quote', () => { + const sql = "SELECT 'a.b''''a.b.:UInt64', col2, c.d FROM table"; + const actual = findJsonExpressions(sql); + const expected = [{ index: 35, expr: 'c.d' }]; + expect(actual).toEqual(expected); + }); + + it('should not find JSON expressions in string that has escaped single quote 2', () => { + const sql = "SELECT '\\'a.b', col2, c.d FROM table"; + const actual = findJsonExpressions(sql); + const expected = [{ index: 22, expr: 'c.d' }]; + expect(actual).toEqual(expected); + }); + + it('should find JSON expressions with underscores and numbers', () => { + const sql = 'SELECT json_col.col3.c FROM table'; + const actual = findJsonExpressions(sql); + const expected = [{ index: 7, expr: 'json_col.col3.c' }]; + expect(actual).toEqual(expected); + }); + + it('should find JSON expressions with backticks', () => { + const sql = 'SELECT `a`.`b`.`cde` FROM table'; + const actual = findJsonExpressions(sql); + const expected = [{ index: 7, expr: '`a`.`b`.`cde`' }]; + expect(actual).toEqual(expected); + }); + + it('should find JSON expressions with double quotes', () => { + const sql = 'SELECT "a"."b"."cde" FROM table'; + const actual = findJsonExpressions(sql); + const expected = [{ index: 7, expr: '"a"."b"."cde"' }]; + expect(actual).toEqual(expected); + }); + + it('should find JSON expressions in tuple', () => { + const sql = 'SELECT (a.b, c.d.e) FROM table'; + const actual = findJsonExpressions(sql); + const expected = [ + { index: 8, expr: 'a.b' }, + { index: 13, expr: 'c.d.e' }, + ]; + expect(actual).toEqual(expected); + }); + + it('should not find JSON expressions inside identifiers', () => { + const sql = 'SELECT "a.b.c" FROM table'; + const actual = findJsonExpressions(sql); + const expected = []; + expect(actual).toEqual(expected); + }); + + it('should find JSON expressions with weird identifier quoting', () => { + const sql = 'SELECT "a_b.2c".b."c." FROM table'; + const actual = findJsonExpressions(sql); + const expected = [{ index: 7, expr: '"a_b.2c".b."c."' }]; + expect(actual).toEqual(expected); + }); + + it('should find JSON expressions after *', () => { + const sql = 'SELECT *, a.b.c FROM table'; + const actual = findJsonExpressions(sql); + const expected = [{ index: 10, expr: 'a.b.c' }]; + expect(actual).toEqual(expected); + }); + + it('should not find a decimal number expression', () => { + const sql = 'SELECT 10.50, 2.3, 2, 1.5 - a.b FROM table'; + const actual = findJsonExpressions(sql); + const expected = [{ index: 28, expr: 'a.b' }]; + expect(actual).toEqual(expected); + }); + + it('should not find a . as a JSON expression', () => { + const sql = 'SELECT . FROM table'; + const actual = findJsonExpressions(sql); + const expected = []; + expect(actual).toEqual(expected); + }); + + it('should find a JSON expression with an identifier containing a single-quote', () => { + const sql = `SELECT Timestamp,ServiceName,SeverityText,Body,ResourceAttributes.hyperdx.distro."version'" FROM default.otel_logs WHERE (Timestamp >= fromUnixTimestamp64Milli(1759756098000) AND Timestamp <= fromUnixTimestamp64Milli(1759756998000)) ORDER BY Timestamp DESC`; + const actual = findJsonExpressions(sql); + const expected = [ + { index: 47, expr: `ResourceAttributes.hyperdx.distro."version'"` }, + { index: 97, expr: `default.otel_logs` }, + ]; + expect(actual).toEqual(expected); + }); + + it('should find a JSON expression with an identifier containing a double-quote', () => { + const sql = + 'SELECT Timestamp,ServiceName,SeverityText,Body,ResourceAttributes.hyperdx.distro.`"version"`'; + const actual = findJsonExpressions(sql); + const expected = [ + { index: 47, expr: 'ResourceAttributes.hyperdx.distro.`"version"`' }, + ]; + expect(actual).toEqual(expected); + }); + + it('should find a JSON expression with an identifier containing a backtick', () => { + const sql = + 'SELECT Timestamp,ServiceName,SeverityText,Body,ResourceAttributes.hyperdx.distro."`version`"'; + const actual = findJsonExpressions(sql); + const expected = [ + { index: 47, expr: 'ResourceAttributes.hyperdx.distro."`version`"' }, + ]; + expect(actual).toEqual(expected); + }); + }); + + describe('replaceJsonAccesses', () => { + it('should handle empty expression', () => { + const sql = ''; + const actual = replaceJsonExpressions(sql); + const expected = { replacements: new Map(), sqlWithReplacements: '' }; + expect(actual).toEqual(expected); + }); + + it('should replace a single JSON access', () => { + const sql = 'SELECT a.b.c as alias1, col2 as alias2 FROM table'; + const actual = replaceJsonExpressions(sql); + const expected = { + replacements: new Map([['__hdx_json_replacement_0', 'a.b.c']]), + sqlWithReplacements: + 'SELECT __hdx_json_replacement_0 as alias1, col2 as alias2 FROM table', + }; + expect(actual).toEqual(expected); + }); + + it('should replace multiple JSON access', () => { + const sql = 'SELECT a.b.c, d.e, col2 FROM table'; + const actual = replaceJsonExpressions(sql); + const expected = { + replacements: new Map([ + ['__hdx_json_replacement_0', 'a.b.c'], + ['__hdx_json_replacement_1', 'd.e'], + ]), + sqlWithReplacements: + 'SELECT __hdx_json_replacement_0, __hdx_json_replacement_1, col2 FROM table', + }; + expect(actual).toEqual(expected); + }); + + it('should replace JSON access with type specifier', () => { + const sql = 'SELECT a.b.:UInt64, col2 FROM table'; + const actual = replaceJsonExpressions(sql); + const expected = { + replacements: new Map([['__hdx_json_replacement_0', 'a.b.:UInt64']]), + sqlWithReplacements: 'SELECT __hdx_json_replacement_0, col2 FROM table', + }; + expect(actual).toEqual(expected); + }); + + it('should replace JSON access with complex type specifier', () => { + const sql = 'SELECT a.b.:Array(String), col2 FROM table'; + const actual = replaceJsonExpressions(sql); + const expected = { + replacements: new Map([ + ['__hdx_json_replacement_0', 'a.b.:Array(String)'], + ]), + sqlWithReplacements: 'SELECT __hdx_json_replacement_0, col2 FROM table', + }; + expect(actual).toEqual(expected); + }); + + it('should replace JSON expressions in WHERE clause ', () => { + const sql = + 'SELECT col2 FROM table WHERE a.b.:UInt64 = 1 AND toStartOfDay(a.date) = today()'; + const actual = replaceJsonExpressions(sql); + const expected = { + replacements: new Map([ + ['__hdx_json_replacement_0', 'a.b.:UInt64'], + ['__hdx_json_replacement_1', 'a.date'], + ]), + sqlWithReplacements: + 'SELECT col2 FROM table WHERE __hdx_json_replacement_0 = 1 AND toStartOfDay(__hdx_json_replacement_1) = today()', + }; + expect(actual).toEqual(expected); + }); + + it('should replace JSON expressions in function calls', () => { + const sql = "SELECT JSONExtractString(a.b.c, 'key') FROM table"; + const actual = replaceJsonExpressions(sql); + const expected = { + replacements: new Map([['__hdx_json_replacement_0', 'a.b.c']]), + sqlWithReplacements: + "SELECT JSONExtractString(__hdx_json_replacement_0, 'key') FROM table", + }; + expect(actual).toEqual(expected); + }); + + it('should not replace JSON expressions in quoted strings', () => { + const sql = + "SELECT a.b.c, ResourceAttributes['key.key2'], 'a.b.c' FROM table"; + const actual = replaceJsonExpressions(sql); + const expected = { + replacements: new Map([['__hdx_json_replacement_0', 'a.b.c']]), + sqlWithReplacements: + "SELECT __hdx_json_replacement_0, ResourceAttributes['key.key2'], 'a.b.c' FROM table", + }; + expect(actual).toEqual(expected); + }); + }); }); diff --git a/packages/common-utils/src/clickhouse/index.ts b/packages/common-utils/src/clickhouse/index.ts index aca6850e2..c829246c8 100644 --- a/packages/common-utils/src/clickhouse/index.ts +++ b/packages/common-utils/src/clickhouse/index.ts @@ -7,7 +7,6 @@ import type { ResponseJSON, Row, } from '@clickhouse/client-common'; -import { isSuccessfulResponse } from '@clickhouse/client-common'; import type { ClickHouseClient as WebClickHouseClient } from '@clickhouse/client-web'; import * as SQLParser from 'node-sql-parser'; import objectHash from 'object-hash'; @@ -18,8 +17,12 @@ import { setChartSelectsAlias, splitChartConfigs, } from '@/renderChartConfig'; -import { ChartConfigWithOptDateRange, SQLInterval } from '@/types'; -import { hashCode, splitAndTrimWithBracket } from '@/utils'; +import { ChartConfigWithOptDateRange } from '@/types'; +import { + hashCode, + replaceJsonExpressions, + splitAndTrimWithBracket, +} from '@/utils'; // export @clickhouse/client-common types export type { @@ -686,8 +689,13 @@ export function chSqlToAliasMap( try { const sql = parameterizedQueryToSql(chSql); + + // Replace JSON expressions with replacement tokens so that node-sql-parser can parse the SQL + const { sqlWithReplacements, replacements: jsonReplacementsToExpressions } = + replaceJsonExpressions(sql); + const parser = new SQLParser.Parser(); - const ast = parser.astify(sql, { + const ast = parser.astify(sqlWithReplacements, { database: 'Postgresql', parseOptions: { includeLocations: true }, }) as SQLParser.Select; @@ -703,7 +711,7 @@ export function chSqlToAliasMap( : // normal alias column.expr.column.expr.value; } else if (column.expr.loc != null) { - aliasMap[column.as] = sql.slice( + aliasMap[column.as] = sqlWithReplacements.slice( column.expr.loc.start.offset, column.expr.loc.end.offset, ); @@ -713,8 +721,23 @@ export function chSqlToAliasMap( } }); } + + // Replace the JSON replacement tokens with the original JSON expressions + for (const [alias, aliasExpression] of Object.entries(aliasMap)) { + for (const [replacement, original] of jsonReplacementsToExpressions) { + if (aliasExpression.includes(replacement)) { + aliasMap[alias] = aliasExpression.replaceAll(replacement, original); + } + } + } + return aliasMap; } catch (e) { - console.error('Error parsing alias map', e, 'for query', chSql); + console.error( + 'Error parsing alias map with JSON removed', + e, + 'for query', + chSql, + ); } return aliasMap; diff --git a/packages/common-utils/src/utils.ts b/packages/common-utils/src/utils.ts index 8ba32959b..1535c5949 100644 --- a/packages/common-utils/src/utils.ts +++ b/packages/common-utils/src/utils.ts @@ -87,6 +87,134 @@ export function getFirstTimestampValueExpression(valueExpression: string) { return splitAndTrimWithBracket(valueExpression)[0]; } +/** Returns true if the given expression is a JSON expression, eg. `col.key.nestedKey` or "json_col"."key" */ +export const isJsonExpression = (expr: string) => { + if (!expr.includes('.')) return false; + + let isInDoubleQuote = false; + let isInBacktick = false; + let isInSingleQuote = false; + + const parts: string[] = []; + let current = ''; + for (const c of expr) { + if (c === "'" && !isInDoubleQuote && !isInBacktick) { + isInSingleQuote = !isInSingleQuote; + } else if (isInSingleQuote) { + continue; + } else if (c === '"' && !isInBacktick) { + isInDoubleQuote = !isInDoubleQuote; + current += c; + } else if (c === '`' && !isInDoubleQuote) { + isInBacktick = !isInBacktick; + current += c; + } else if (c === '.' && !isInDoubleQuote && !isInBacktick) { + parts.push(current); + current = ''; + } else { + current += c; + } + } + + if (!isInDoubleQuote && !isInBacktick) { + parts.push(current); + } + + if (parts.some(p => p.trim().length === 0)) return false; + + return ( + parts.filter( + p => + p.trim().length > 0 && + isNaN(Number(p)) && + !(p.startsWith("'") && p.endsWith("'")), + ).length > 1 + ); +}; + +/** + * Finds and returns expressions within the given SQL string that represent JSON references (eg. `col.key.nestedKey`) + * + * Note - This function does not distinguish between json references and `table.column` references - both are returned. + */ +export function findJsonExpressions(sql: string) { + const expressions: { index: number; expr: string }[] = []; + + let isInDoubleQuote = false; + let isInBacktick = false; + + let currentExpr = ''; + const finishExpression = (expr: string, endIndex: number) => { + if (isJsonExpression(expr)) { + expressions.push({ index: endIndex - expr.length, expr }); + } + currentExpr = ''; + }; + + let i = 0; + let isInJsonTypeSpecifier = false; + while (i < sql.length) { + const c = sql.charAt(i); + if (c === "'" && !isInDoubleQuote && !isInBacktick) { + // Skip string literals + while (i < sql.length && sql.charAt(i) !== c) { + i++; + } + currentExpr = ''; + } else if (c === '"' && !isInBacktick) { + isInDoubleQuote = !isInDoubleQuote; + currentExpr += c; + } else if (c === '`' && !isInDoubleQuote) { + isInBacktick = !isInBacktick; + currentExpr += c; + } else if (/[\s{},+*/[\]]/.test(c)) { + isInJsonTypeSpecifier = false; + finishExpression(currentExpr, i); + } else if ('()'.includes(c) && !isInJsonTypeSpecifier) { + finishExpression(currentExpr, i); + } else if (c === ':') { + isInJsonTypeSpecifier = true; + currentExpr += c; + } else { + currentExpr += c; + } + + i++; + } + + finishExpression(currentExpr, i); + return expressions; +} + +/** + * Replaces expressions within the given SQL string that represent JSON expressions (eg. `col.key.nestedKey`). + * Such expression are replaced with placeholders like `__hdx_json_replacement_0`. The resulting string and a + * map of replacements --> original expressions is returned. + * + * Note - This function does not distinguish between json references and `table.column` references - both are replaced. + */ +export function replaceJsonExpressions(sql: string) { + const jsonExpressions = findJsonExpressions(sql); + + const replacements = new Map(); + let sqlWithReplacements = sql; + let indexOffsetFromInserts = 0; + let replacementCounter = 0; + for (const { expr, index } of jsonExpressions) { + const replacement = `__hdx_json_replacement_${replacementCounter++}`; + replacements.set(replacement, expr); + + const effectiveIndex = index + indexOffsetFromInserts; + sqlWithReplacements = + sqlWithReplacements.slice(0, effectiveIndex) + + replacement + + sqlWithReplacements.slice(effectiveIndex + expr.length); + indexOffsetFromInserts += replacement.length - expr.length; + } + + return { sqlWithReplacements, replacements }; +} + export enum Granularity { FifteenSecond = '15 second', ThirtySecond = '30 second',