From 51d49e48e93420f3022883e7962b53dafc6af6a8 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 10 May 2022 09:08:56 -0400 Subject: [PATCH 1/4] fix: Update parser for indexed field --- datafusion/core/src/sql/planner.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/datafusion/core/src/sql/planner.rs b/datafusion/core/src/sql/planner.rs index af8329018f67..cc8da7d0de3d 100644 --- a/datafusion/core/src/sql/planner.rs +++ b/datafusion/core/src/sql/planner.rs @@ -97,9 +97,10 @@ fn plan_key(key: SQLExpr) -> Result { ScalarValue::Int64(Some(s.parse().unwrap())) } SQLExpr::Value(Value::SingleQuotedString(s)) => ScalarValue::Utf8(Some(s)), + SQLExpr::Identifier(Ident { value, .. }) => ScalarValue::Utf8(Some(value)), _ => { return Err(DataFusionError::SQL(ParserError(format!( - "Unsuported index key expression: {}", + "Unsuported index key expression: {:?}", key )))) } @@ -1645,6 +1646,11 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { } } + SQLExpr::ArrayIndex { obj, indexs } => { + let expr = self.sql_expr_to_logical_expr(*obj, schema, ctes)?; + plan_indexed(expr, indexs) + } + SQLExpr::CompoundIdentifier(ids) => { let mut var_names: Vec<_> = ids.into_iter().map(|s| normalize_ident(&s)).collect(); From 23edcd2d017a4da990fb22b23c9883ec8a2c5811 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 10 May 2022 09:38:12 -0400 Subject: [PATCH 2/4] Update to sqlparser 0.17 --- datafusion-cli/Cargo.lock | 108 +++++++++++++++++++---------------- datafusion/common/Cargo.toml | 2 +- datafusion/core/Cargo.toml | 2 +- datafusion/expr/Cargo.toml | 2 +- 4 files changed, 61 insertions(+), 53 deletions(-) diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock index 06f978c3ebb4..9a84e35c2309 100644 --- a/datafusion-cli/Cargo.lock +++ b/datafusion-cli/Cargo.lock @@ -155,9 +155,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "axum" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4af7447fc1214c1f3a1ace861d0216a6c8bb13965b64bbad9650f375b67689a" +checksum = "00f1e8a972137fad81e2a1a60b86ff17ce0338f8017264e45a9723d0083c39a1" dependencies = [ "async-trait", "axum-core", @@ -205,7 +205,7 @@ dependencies = [ "futures", "log", "parking_lot", - "sqlparser", + "sqlparser 0.16.0", "tempfile", "tokio", ] @@ -232,7 +232,7 @@ dependencies = [ "prost-types", "rustc_version", "serde", - "sqlparser", + "sqlparser 0.16.0", "tokio", "tonic", "tonic-build", @@ -358,9 +358,9 @@ dependencies = [ [[package]] name = "clap" -version = "3.1.15" +version = "3.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85a35a599b11c089a7f49105658d089b8f2cf0882993c17daf6de15285c2c35d" +checksum = "47582c09be7c8b32c0ab3a6181825ababb713fde6fff20fc573a3870dd45c6a0" dependencies = [ "atty", "bitflags", @@ -494,6 +494,7 @@ dependencies = [ "datafusion-data-access", "datafusion-expr", "datafusion-physical-expr", + "datafusion-row", "futures", "hashbrown 0.12.1", "lazy_static", @@ -506,7 +507,7 @@ dependencies = [ "pin-project-lite", "rand", "smallvec", - "sqlparser", + "sqlparser 0.17.0", "tempfile", "tokio", "tokio-stream", @@ -535,7 +536,7 @@ dependencies = [ "arrow", "ordered-float 3.0.0", "parquet", - "sqlparser", + "sqlparser 0.17.0", ] [[package]] @@ -545,6 +546,7 @@ dependencies = [ "async-trait", "chrono", "futures", + "glob", "parking_lot", "tempfile", "tokio", @@ -557,7 +559,7 @@ dependencies = [ "ahash", "arrow", "datafusion-common", - "sqlparser", + "sqlparser 0.17.0", ] [[package]] @@ -571,6 +573,7 @@ dependencies = [ "chrono", "datafusion-common", "datafusion-expr", + "datafusion-row", "hashbrown 0.12.1", "lazy_static", "md-5", @@ -591,6 +594,16 @@ dependencies = [ "tonic-build", ] +[[package]] +name = "datafusion-row" +version = "7.0.0" +dependencies = [ + "arrow", + "datafusion-common", + "paste", + "rand", +] + [[package]] name = "digest" version = "0.10.3" @@ -864,6 +877,12 @@ dependencies = [ "wasi 0.10.2+wasi-snapshot-preview1", ] +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + [[package]] name = "h2" version = "0.3.13" @@ -1273,25 +1292,14 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52da4364ffb0e4fe33a9841a98a3f3014fb964045ce4f7a45a398243c8d6b0c9" +checksum = "713d550d9b44d89174e066b7a6217ae06234c10cb47819a88290d2b353c31799" dependencies = [ "libc", "log", - "miow", - "ntapi", "wasi 0.11.0+wasi-snapshot-preview1", - "winapi", -] - -[[package]] -name = "miow" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" -dependencies = [ - "winapi", + "windows-sys 0.36.1", ] [[package]] @@ -1342,15 +1350,6 @@ dependencies = [ "memoffset", ] -[[package]] -name = "ntapi" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28774a7fd2fbb4f0babd8237ce554b73af68021b5f695a3cebd6c59bac0980f" -dependencies = [ - "winapi", -] - [[package]] name = "num" version = "0.4.0" @@ -1623,18 +1622,18 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec757218438d5fda206afc041538b2f6d889286160d649a86a24d37e1235afd1" +checksum = "9027b48e9d4c9175fa2218adf3557f91c1137021739951d4932f5f8268ac48aa" dependencies = [ "unicode-xid", ] [[package]] name = "prost" -version = "0.10.1" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a07b0857a71a8cb765763950499cae2413c3f9cede1133478c43600d9e146890" +checksum = "bc03e116981ff7d8da8e5c220e374587b98d294af7ba7dd7fda761158f00086f" dependencies = [ "bytes", "prost-derive", @@ -1642,9 +1641,9 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.10.1" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "120fbe7988713f39d780a58cf1a7ef0d7ef66c6d87e5aa3438940c05357929f4" +checksum = "65a1118354442de7feb8a2a76f3d80ef01426bd45542c8c1fdffca41a758f846" dependencies = [ "bytes", "cfg-if", @@ -1888,9 +1887,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.80" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f972498cf015f7c0746cac89ebe1d6ef10c293b94175a243a2d9442c163d9944" +checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c" dependencies = [ "indexmap", "itoa 1.0.1", @@ -1946,6 +1945,15 @@ dependencies = [ "log", ] +[[package]] +name = "sqlparser" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddc2739f3a9bfc68e2f7b7695589f6cb0181c88af73ceaee0c84215cd2a2ae28" +dependencies = [ + "log", +] + [[package]] name = "static_assertions" version = "1.1.0" @@ -1991,9 +1999,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.92" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ff7c592601f11445996a06f8ad0c27f094a58857c2f89e97974ab9235b92c52" +checksum = "04066589568b72ec65f42d65a1a52436e954b168773148893c020269563decf2" dependencies = [ "proc-macro2", "quote", @@ -2079,9 +2087,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.18.1" +version = "1.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dce653fb475565de9f6fb0614b28bca8df2c430c0cf84bcd9c843f15de5414cc" +checksum = "4903bf0427cf68dddd5aa6a93220756f8be0c34fcfa9f5e6191e103e15a31395" dependencies = [ "bytes", "libc", @@ -2144,9 +2152,9 @@ dependencies = [ [[package]] name = "tonic" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30fb54bf1e446f44d870d260d99957e7d11fb9d0a0f5bd1a662ad1411cc103f9" +checksum = "5be9d60db39854b30b835107500cf0aca0b0d14d6e1c3de124217c23a29c2ddb" dependencies = [ "async-stream", "async-trait", @@ -2176,9 +2184,9 @@ dependencies = [ [[package]] name = "tonic-build" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c03447cdc9eaf8feffb6412dcb27baf2db11669a6c4789f29da799aabfb99547" +checksum = "d9263bf4c9bfaae7317c1c2faf7f18491d2fe476f70c414b73bf5d445b00ffa1" dependencies = [ "prettyplease", "proc-macro2", @@ -2209,9 +2217,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e980386f06883cf4d0578d6c9178c81f68b45d77d00f2c2c1bc034b3439c2c56" +checksum = "7d342c6d58709c0a6d48d48dabbb62d4ef955cf5f0f3bbfd845838e7ae88dbae" dependencies = [ "bitflags", "bytes", diff --git a/datafusion/common/Cargo.toml b/datafusion/common/Cargo.toml index 7318cafe3e9d..4520847a4cdf 100644 --- a/datafusion/common/Cargo.toml +++ b/datafusion/common/Cargo.toml @@ -44,4 +44,4 @@ cranelift-module = { version = "0.83.0", optional = true } ordered-float = "3.0" parquet = { version = "13", features = ["arrow"], optional = true } pyo3 = { version = "0.16", optional = true } -sqlparser = "0.16" +sqlparser = "0.17" diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml index e11e02e95bdf..65d450b9ad32 100644 --- a/datafusion/core/Cargo.toml +++ b/datafusion/core/Cargo.toml @@ -80,7 +80,7 @@ pyo3 = { version = "0.16", optional = true } rand = "0.8" rayon = { version = "1.5", optional = true } smallvec = { version = "1.6", features = ["union"] } -sqlparser = "0.16" +sqlparser = "0.17" tempfile = "3" tokio = { version = "1.0", features = ["macros", "rt", "rt-multi-thread", "sync", "fs", "parking_lot"] } tokio-stream = "0.1" diff --git a/datafusion/expr/Cargo.toml b/datafusion/expr/Cargo.toml index 8d24f9617241..8f84be35ea62 100644 --- a/datafusion/expr/Cargo.toml +++ b/datafusion/expr/Cargo.toml @@ -38,4 +38,4 @@ path = "src/lib.rs" ahash = { version = "0.7", default-features = false } arrow = { version = "13", features = ["prettyprint"] } datafusion-common = { path = "../common", version = "7.0.0" } -sqlparser = "0.16" +sqlparser = "0.17" From 108a727b6c57c9fbc8b6116513b86edc15afed26 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 10 May 2022 10:05:43 -0400 Subject: [PATCH 3/4] Update more cargo --- ballista/rust/client/Cargo.toml | 2 +- ballista/rust/core/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ballista/rust/client/Cargo.toml b/ballista/rust/client/Cargo.toml index 9052380f14af..e9805a7b0f4e 100644 --- a/ballista/rust/client/Cargo.toml +++ b/ballista/rust/client/Cargo.toml @@ -36,7 +36,7 @@ datafusion = { path = "../../../datafusion/core", version = "7.0.0" } futures = "0.3" log = "0.4" parking_lot = "0.12" -sqlparser = "0.16" +sqlparser = "0.17" tempfile = "3" tokio = "1.0" diff --git a/ballista/rust/core/Cargo.toml b/ballista/rust/core/Cargo.toml index df0c500cf815..05a6f1922c73 100644 --- a/ballista/rust/core/Cargo.toml +++ b/ballista/rust/core/Cargo.toml @@ -53,7 +53,7 @@ parse_arg = "0.1.3" prost = "0.10" prost-types = "0.10" serde = { version = "1", features = ["derive"] } -sqlparser = "0.16" +sqlparser = "0.17" tokio = "1.0" tonic = "0.7" uuid = { version = "1.0", features = ["v4"] } From 09d972221cc9dafef8a1d41c7afea84f1c068d2d Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 10 May 2022 10:07:34 -0400 Subject: [PATCH 4/4] Fix test to use string key rather than identifier --- datafusion/core/src/sql/planner.rs | 1 - datafusion/core/tests/sql/select.rs | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/datafusion/core/src/sql/planner.rs b/datafusion/core/src/sql/planner.rs index cc8da7d0de3d..6861f3bed629 100644 --- a/datafusion/core/src/sql/planner.rs +++ b/datafusion/core/src/sql/planner.rs @@ -97,7 +97,6 @@ fn plan_key(key: SQLExpr) -> Result { ScalarValue::Int64(Some(s.parse().unwrap())) } SQLExpr::Value(Value::SingleQuotedString(s)) => ScalarValue::Utf8(Some(s)), - SQLExpr::Identifier(Ident { value, .. }) => ScalarValue::Utf8(Some(value)), _ => { return Err(DataFusionError::SQL(ParserError(format!( "Unsuported index key expression: {:?}", diff --git a/datafusion/core/tests/sql/select.rs b/datafusion/core/tests/sql/select.rs index 1f1595693f28..3eb297ed719a 100644 --- a/datafusion/core/tests/sql/select.rs +++ b/datafusion/core/tests/sql/select.rs @@ -639,7 +639,7 @@ async fn query_nested_get_indexed_field_on_struct() -> Result<()> { ctx.register_table("structs", table_a)?; // Original column is micros, convert to millis and check timestamp - let sql = "SELECT some_struct[\"bar\"] as l0 FROM structs LIMIT 3"; + let sql = "SELECT some_struct['bar'] as l0 FROM structs LIMIT 3"; let actual = execute_to_batches(&ctx, sql).await; let expected = vec![ "+----------------+", @@ -666,7 +666,7 @@ async fn query_nested_get_indexed_field_on_struct() -> Result<()> { ]; assert_batches_eq!(expected, &actual); - let sql = "SELECT some_struct[\"bar\"][0] as i0 FROM structs LIMIT 3"; + let sql = "SELECT some_struct['bar'][0] as i0 FROM structs LIMIT 3"; let actual = execute_to_batches(&ctx, sql).await; #[rustfmt::skip] let expected = vec![