diff --git a/docs/changelog/127797.yaml b/docs/changelog/127797.yaml new file mode 100644 index 0000000000000..8fca3da004130 --- /dev/null +++ b/docs/changelog/127797.yaml @@ -0,0 +1,6 @@ +pr: 127797 +summary: "Date nanos implicit casting in union types option #2" +area: ES|QL +type: enhancement +issues: + - 110009 diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/DateUtils.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/DateUtils.java index 20f7b400e9364..ae316460dd11c 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/DateUtils.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/DateUtils.java @@ -65,7 +65,7 @@ public class DateUtils { .appendValue(MINUTE_OF_HOUR, 2) .appendLiteral(':') .appendValue(SECOND_OF_MINUTE, 2) - .appendFraction(NANO_OF_SECOND, 3, 9, true) + .appendFraction(NANO_OF_SECOND, 0, 9, true) .appendOffsetId() .toFormatter(Locale.ROOT); diff --git a/x-pack/plugin/esql/qa/server/single-node/build.gradle b/x-pack/plugin/esql/qa/server/single-node/build.gradle index ff14956317a58..977955ed69e52 100644 --- a/x-pack/plugin/esql/qa/server/single-node/build.gradle +++ b/x-pack/plugin/esql/qa/server/single-node/build.gradle @@ -13,6 +13,7 @@ dependencies { javaRestTestImplementation project(xpackModule('esql:qa:testFixtures')) javaRestTestImplementation project(xpackModule('esql:qa:server')) javaRestTestImplementation project(xpackModule('esql:tools')) + javaRestTestImplementation project(xpackModule('esql')) yamlRestTestImplementation project(xpackModule('esql:qa:server')) javaRestTestImplementation('org.apache.arrow:arrow-vector:16.1.0') diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java index cb6780f8359dc..1ef49652c3afc 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java @@ -55,6 +55,8 @@ import static org.elasticsearch.test.ListMatcher.matchesList; import static org.elasticsearch.test.MapMatcher.assertMap; import static org.elasticsearch.test.MapMatcher.matchesMap; +import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.IMPLICIT_CASTING_DATE_AND_DATE_NANOS; +import static org.elasticsearch.xpack.esql.core.type.DataType.isMillisOrNanos; import static org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase.Mode.SYNC; import static org.elasticsearch.xpack.esql.tools.ProfileParser.parseProfile; import static org.elasticsearch.xpack.esql.tools.ProfileParser.readProfileFromResponse; @@ -736,19 +738,26 @@ public void testSuggestedCast() throws IOException { Map results = entityAsMap(resp); List columns = (List) results.get("columns"); DataType suggestedCast = DataType.suggestedCast(Set.of(listOfTypes.get(i), listOfTypes.get(j))); - assertThat( - columns, - equalTo( - List.of( - Map.ofEntries( - Map.entry("name", "my_field"), - Map.entry("type", "unsupported"), - Map.entry("original_types", List.of(listOfTypes.get(i).typeName(), listOfTypes.get(j).typeName())), - Map.entry("suggested_cast", suggestedCast.typeName()) + if (IMPLICIT_CASTING_DATE_AND_DATE_NANOS.isEnabled() + && isMillisOrNanos(listOfTypes.get(i)) + && isMillisOrNanos(listOfTypes.get(j))) { + // datetime and date_nanos are casted to date_nanos implicitly + assertThat(columns, equalTo(List.of(Map.ofEntries(Map.entry("name", "my_field"), Map.entry("type", "date_nanos"))))); + } else { + assertThat( + columns, + equalTo( + List.of( + Map.ofEntries( + Map.entry("name", "my_field"), + Map.entry("type", "unsupported"), + Map.entry("original_types", List.of(listOfTypes.get(i).typeName(), listOfTypes.get(j).typeName())), + Map.entry("suggested_cast", suggestedCast.typeName()) + ) ) ) - ) - ); + ); + } String castedQuery = String.format( Locale.ROOT, diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index 92fe597362bb0..152eb8dbeeda4 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -128,6 +128,7 @@ public class CsvTestsDataLoader { ); private static final TestDataset AIRPORTS_WEB = new TestDataset("airports_web"); private static final TestDataset DATE_NANOS = new TestDataset("date_nanos"); + private static final TestDataset DATE_NANOS_UNION_TYPES = new TestDataset("date_nanos_union_types"); private static final TestDataset COUNTRIES_BBOX = new TestDataset("countries_bbox"); private static final TestDataset COUNTRIES_BBOX_WEB = new TestDataset("countries_bbox_web"); private static final TestDataset AIRPORT_CITY_BOUNDARIES = new TestDataset("airport_city_boundaries"); @@ -197,6 +198,7 @@ public class CsvTestsDataLoader { Map.entry(MULTIVALUE_GEOMETRIES.indexName, MULTIVALUE_GEOMETRIES), Map.entry(MULTIVALUE_POINTS.indexName, MULTIVALUE_POINTS), Map.entry(DATE_NANOS.indexName, DATE_NANOS), + Map.entry(DATE_NANOS_UNION_TYPES.indexName, DATE_NANOS_UNION_TYPES), Map.entry(K8S.indexName, K8S), Map.entry(DISTANCES.indexName, DISTANCES), Map.entry(ADDRESSES.indexName, ADDRESSES), diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/date_nanos_union_types.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/date_nanos_union_types.csv new file mode 100644 index 0000000000000..a212dc72ccf80 --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/date_nanos_union_types.csv @@ -0,0 +1,13 @@ +millis:date_nanos,nanos:date,num:long +2023-10-23T13:55:01.543123456Z,2023-10-23T13:55:01.543Z,1698069301543123456 +2023-10-23T13:55:01.543123456Z,2023-10-23T13:55:01.543Z,1698069301543123456 +2023-10-23T13:53:55.832987654Z,2023-10-23T13:53:55.832Z,1698069235832987654 +2023-10-23T13:52:55.015787878Z,2023-10-23T13:52:55.015Z,1698069175015787878 +2023-10-23T13:51:54.732102837Z,2023-10-23T13:51:54.732Z,1698069114732102837 +2023-10-23T13:33:34.937193000Z,2023-10-23T13:33:34.937Z,1698068014937193000 +2023-10-23T12:27:28.948000000Z,2023-10-23T12:27:28.948Z,1698064048948000000 +2023-10-23T12:15:03.360103847Z,2023-10-23T12:15:03.360Z,1698063303360103847 +2023-10-23T12:15:03.360103847Z,2023-10-23T12:15:03.360Z,1698063303360103847 +1999-10-23T12:15:03.360103847Z,[2023-03-23T12:15:03.360Z, 2023-02-23T13:33:34.937Z, 2023-01-23T13:55:01.543Z], 0 +1999-10-22T12:15:03.360103847Z,[2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z], 0 +2023-10-23T12:15:03.360103847Z,1923-10-23T12:15:03.360Z,1698063303360103847 diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/employees_incompatible.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/employees_incompatible.csv index ddbdb89476c4c..718bb3add9b1c 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/employees_incompatible.csv +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/employees_incompatible.csv @@ -1,4 +1,4 @@ -birth_date:date_nanos ,emp_no:long,first_name:text,gender:text,hire_date:date_nanos,languages:byte,languages.long:long,languages.short:short,languages.byte:byte,last_name:text,salary:long,height:float,height.double:double,height.scaled_float:scaled_float,height.half_float:half_float,still_hired:keyword,avg_worked_seconds:unsigned_long,job_positions:text,is_rehired:keyword,salary_change:float,salary_change.int:integer,salary_change.long:long,salary_change.keyword:keyword +birth_date:date ,emp_no:long,first_name:text,gender:text,hire_date:date_nanos,languages:byte,languages.long:long,languages.short:short,languages.byte:byte,last_name:text,salary:long,height:float,height.double:double,height.scaled_float:scaled_float,height.half_float:half_float,still_hired:keyword,avg_worked_seconds:unsigned_long,job_positions:text,is_rehired:keyword,salary_change:float,salary_change.int:integer,salary_change.long:long,salary_change.keyword:keyword 1953-09-02T00:00:00Z,10001,Georgi ,M,1986-06-26T00:00:00Z,2,2,2,2,Facello ,57305,2.03,2.03,2.03,2.03,true ,268728049,[Senior Python Developer,Accountant],[false,true],[1.19],[1],[1],[1.19] 1964-06-02T00:00:00Z,10002,Bezalel ,F,1985-11-21T00:00:00Z,5,5,5,5,Simmel ,56371,2.08,2.08,2.08,2.08,true ,328922887,[Senior Team Lead],[false,false],[-7.23,11.17],[-7,11],[-7,11],[-7.23,11.17] 1959-12-03T00:00:00Z,10003,Parto ,M,1986-08-28T00:00:00Z,4,4,4,4,Bamford ,61805,1.83,1.83,1.83,1.83,false,200296405,[],[],[14.68,12.82],[14,12],[14,12],[14.68,12.82] diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-date_nanos_union_types.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-date_nanos_union_types.json new file mode 100644 index 0000000000000..1017639bb5cfc --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-date_nanos_union_types.json @@ -0,0 +1,13 @@ +{ + "properties": { + "millis": { + "type": "date_nanos" + }, + "nanos": { + "type": "date" + }, + "num": { + "type": "long" + } + } +} diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec index aa08799943ade..ce933bd6eba7f 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec @@ -1706,3 +1706,808 @@ id:integer | name:keyword | count:long 13 | lllll | 2 14 | mmmmm | 2 ; + +ImplicitCastingMultiTypedFieldsKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL x = emp_no::long, avg_worked_seconds = avg_worked_seconds::unsigned_long, y = hire_date::datetime, z = hire_date::date_nanos, w = hire_date +| KEEP x, languages, hire_date, avg_worked_seconds, y, z, w +| SORT x, z +| LIMIT 2 +; + +x:long |languages:unsupported |hire_date:date_nanos |avg_worked_seconds:unsigned_long |y:datetime |z:date_nanos |w:date_nanos +10001 |null |1986-06-26T00:00:00.000Z |268728049 |1986-06-26T00:00:00.000Z |1986-06-26T00:00:00.000Z |1986-06-26T00:00:00.000Z +10001 |null |1986-06-26T00:00:00.000Z |268728049 |1986-06-26T00:00:00.000Z |1986-06-26T00:00:00.000Z |1986-06-26T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsDropKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL emp_no = emp_no::long, avg_worked_seconds = avg_worked_seconds::unsigned_long, x = hire_date::datetime, y = hire_date +| DROP salary +| KEEP emp_no, languages, hire_date, avg_worked_seconds, x, y +| SORT emp_no, y +| limit 2 +; + +emp_no:long |languages:unsupported |hire_date:date_nanos |avg_worked_seconds:unsigned_long |x:datetime |y:date_nanos +10001 |null |1986-06-26T00:00:00.000Z |268728049 |1986-06-26T00:00:00.000Z |1986-06-26T00:00:00.000Z +10001 |null |1986-06-26T00:00:00.000Z |268728049 |1986-06-26T00:00:00.000Z |1986-06-26T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsRenameKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +from employees, employees_incompatible +| EVAL emp_no = emp_no::long, avg_worked_seconds = avg_worked_seconds::unsigned_long, languages = languages::integer, x = hire_date::datetime +| RENAME emp_no as new_emp_no, languages as new_languages, hire_date as new_hire_date, avg_worked_seconds as new_avg_worked_seconds, x as y +| KEEP new_emp_no, new_languages, new_hire_date, new_avg_worked_seconds, y +| SORT new_emp_no, new_hire_date +| limit 2 +; + +new_emp_no:long |new_languages:integer |new_hire_date:date_nanos |new_avg_worked_seconds:unsigned_long |y:datetime +10001 |2 |1986-06-26T00:00:00.000Z |268728049 |1986-06-26T00:00:00.000Z +10001 |2 |1986-06-26T00:00:00.000Z |268728049 |1986-06-26T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsDropEvalKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| DROP birth_date +| EVAL x = emp_no::long + 1, y = hire_date + 1 year, z = hire_date::datetime +| KEEP x, hire_date, y, z +| SORT x, hire_date +| limit 2 +; + +x:long |hire_date:date_nanos |y:date_nanos |z:date +10002 |1986-06-26T00:00:00.000Z |1987-06-26T00:00:00.000Z |1986-06-26T00:00:00.000Z +10002 |1986-06-26T00:00:00.000Z |1987-06-26T00:00:00.000Z |1986-06-26T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsEvalFilterEqualKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL x = emp_no::long + 1, y = hire_date + 1 day +| WHERE x > 10011 AND hire_date == "1993-08-03" AND languages::integer < 3 AND height::double > 1.2 +| KEEP x, hire_date +| SORT x, hire_date +| limit 4 +; + +x:long |hire_date:date_nanos +10018 | 1993-08-03T00:00:00.000Z +10018 | 1993-08-03T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsEvalFilterNotEqualKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL x = emp_no::long + 1, y = hire_date + 1 day +| WHERE x > 10011 AND hire_date != "1993-08-03" AND languages::integer < 3 AND height::double > 1.2 +| KEEP x, hire_date +| SORT x, hire_date +| limit 4 +; + +x:long |hire_date:date_nanos +10014 | 1985-10-20T00:00:00.000Z +10014 | 1985-10-20T00:00:00.000Z +10017 | 1995-01-27T00:00:00.000Z +10017 | 1995-01-27T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsEvalFilterGreaterThanKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL x = emp_no::long + 1, y = hire_date + 1 day +| WHERE x > 10011 AND hire_date > "1993-08-03" AND languages::integer < 3 AND height::double > 1.2 +| KEEP x, hire_date +| SORT x, hire_date +| limit 4 +; + +x:long | hire_date:date_nanos +10017 | 1995-01-27T00:00:00.000Z +10017 | 1995-01-27T00:00:00.000Z +10020 | 1999-04-30T00:00:00.000Z +10020 | 1999-04-30T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsEvalFilterGreaterThanOrEqualKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL x = emp_no::long + 1, y = hire_date + 1 day +| WHERE x > 10011 AND hire_date >= "1993-08-03" AND languages::integer < 3 AND height::double > 1.2 +| KEEP x, hire_date +| SORT x, hire_date +| limit 4 +; + +x:long |hire_date:date_nanos +10017 | 1995-01-27T00:00:00.000Z +10017 | 1995-01-27T00:00:00.000Z +10018 | 1993-08-03T00:00:00.000Z +10018 | 1993-08-03T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsEvalFilterLessThanKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL x = emp_no::long + 1, y = hire_date + 1 day +| WHERE x > 10011 AND hire_date < "1993-08-03" AND languages::integer < 3 AND height::double > 1.2 +| KEEP x, hire_date +| SORT x, hire_date +| limit 4 +; + +x:long |hire_date:date_nanos +10014 | 1985-10-20T00:00:00.000Z +10014 | 1985-10-20T00:00:00.000Z +10019 | 1987-04-03T00:00:00.000Z +10019 | 1987-04-03T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsEvalFilterLessThanOrEqualKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL x = emp_no::long + 1, y = hire_date + 1 day +| WHERE x > 10011 AND hire_date <= "1993-08-03" AND languages::integer < 3 AND height::double > 1.2 +| KEEP x, hire_date +| SORT x, hire_date +| limit 4 +; + +x:long | hire_date:date_nanos +10014 | 1985-10-20T00:00:00.000Z +10014 | 1985-10-20T00:00:00.000Z +10018 | 1993-08-03T00:00:00.000Z +10018 | 1993-08-03T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsEvalFilterLessThanOrEqualNotEqualKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL x = emp_no::long + 1, y = hire_date + 1 day +| WHERE x > 10011 AND hire_date <= "1993-08-03" AND hire_date != "1985-10-20" AND languages::integer < 3 AND height::double > 1.2 +| KEEP x, hire_date +| SORT x, hire_date +| limit 4 +; + +x:long | hire_date:date_nanos +10018 | 1993-08-03T00:00:00.000Z +10018 | 1993-08-03T00:00:00.000Z +10019 | 1987-04-03T00:00:00.000Z +10019 | 1987-04-03T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsEvalFilterRangeExcludeLowerExcludeUpperKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL x = emp_no::long + 1, y = hire_date + 1 day +| WHERE x > 10011 AND hire_date < "1995-01-27" AND hire_date > "1993-08-03" AND languages::integer < 3 AND height::double > 1.2 +| KEEP x, hire_date +| SORT x, hire_date +| limit 4 +; + +x:long |hire_date:date_nanos +10045 | 1994-05-21T00:00:00.000Z +10045 | 1994-05-21T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsEvalFilterRangeIncludeLowerIncludeUpperKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL x = emp_no::long + 1, y = hire_date + 1 day +| WHERE x > 10011 AND hire_date <= "1995-01-27" AND hire_date >= "1993-08-03" AND languages::integer < 3 AND height::double > 1.2 +| KEEP x, hire_date +| SORT x, hire_date +| limit 4 +; + +x:long |hire_date:date_nanos +10017 | 1995-01-27T00:00:00.000Z +10017 | 1995-01-27T00:00:00.000Z +10018 | 1993-08-03T00:00:00.000Z +10018 | 1993-08-03T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsEvalFilterRangeExcludeLowerIncludeUpperKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL x = emp_no::long + 1, y = hire_date + 1 day +| WHERE x > 10011 AND hire_date <= "1995-01-27" AND hire_date > "1993-08-03" AND languages::integer < 3 AND height::double > 1.2 +| KEEP x, hire_date +| SORT x, hire_date +| limit 4 +; + +x:long |hire_date:date_nanos +10017 | 1995-01-27T00:00:00.000Z +10017 | 1995-01-27T00:00:00.000Z +10045 | 1994-05-21T00:00:00.000Z +10045 | 1994-05-21T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsEvalFilterRangeIncludeLowerExcludeUpperKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL x = emp_no::long + 1, y = hire_date + 1 day +| WHERE x > 10011 AND hire_date < "1995-01-27" AND hire_date >= "1993-08-03" AND languages::integer < 3 AND height::double > 1.2 +| KEEP x, hire_date +| SORT x, hire_date +| limit 4 +; + +x:long |hire_date:date_nanos +10018 | 1993-08-03T00:00:00.000Z +10018 | 1993-08-03T00:00:00.000Z +10045 | 1994-05-21T00:00:00.000Z +10045 | 1994-05-21T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsStatsByNumeric +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| STATS x=max(hire_date), y = min(hire_date) BY languages = languages::integer +| SORT languages +; + +x:date_nanos | y:date_nanos | languages:integer +1999-04-30T00:00:00.000Z | 1985-02-18T00:00:00.000Z | 1 +1995-01-27T00:00:00.000Z | 1986-03-14T00:00:00.000Z | 2 +1996-11-05T00:00:00.000Z | 1985-02-24T00:00:00.000Z | 3 +1995-03-13T00:00:00.000Z | 1985-05-13T00:00:00.000Z | 4 +1994-04-09T00:00:00.000Z | 1985-11-19T00:00:00.000Z | 5 +1997-05-19T00:00:00.000Z | 1985-11-20T00:00:00.000Z | null +; + +ImplicitCastingMultiTypedFieldsStatsByNumericWithFilter +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| STATS x=max(hire_date) where hire_date < "1993-08-03", y = min(hire_date) where hire_date >= "1993-08-03" BY languages = languages::integer +| SORT languages +; + +x:date_nanos | y:date_nanos | languages:integer +1990-10-20T00:00:00.000Z | 1994-05-21T00:00:00.000Z | 1 +1991-06-26T00:00:00.000Z | 1993-08-03T00:00:00.000Z | 2 +1993-03-21T00:00:00.000Z | 1994-02-17T00:00:00.000Z | 3 +1993-02-14T00:00:00.000Z | 1995-03-13T00:00:00.000Z | 4 +1992-12-18T00:00:00.000Z | 1994-04-09T00:00:00.000Z | 5 +1991-10-22T00:00:00.000Z | 1995-03-20T00:00:00.000Z | null +; + +ImplicitCastingMultiTypedFieldsStatsByDateNanos +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| STATS x=count(emp_no::long), y=avg(salary_change::double), z=max(height::double) BY hire_date +| Eval y = round(y, 1), z = round(z, 1) +| KEEP x, y, z, hire_date +| SORT hire_date +| LIMIT 5 +; + +x:long | y:double | z:double | hire_date:date_nanos +2 | null | 1.9 | 1985-02-18T00:00:00.000Z +2 | null | 2.0 | 1985-02-24T00:00:00.000Z +2 | 3.3 | 2.0 | 1985-05-13T00:00:00.000Z +2 | 0.2 | 1.8 | 1985-07-09T00:00:00.000Z +2 | 3.6 | 1.5 | 1985-09-17T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsStatsByDateNanosWithFilter +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| STATS x=count(emp_no::long) where hire_date > "1985-05-01", y=avg(salary_change::double) where hire_date > "1985-05-01", z=max(height::double) where hire_date > "1985-05-01" BY hire_date +| Eval y = round(y, 1), z = round(z, 1) +| KEEP x, y, z, hire_date +| SORT hire_date +| LIMIT 5 +; + +x:long | y:double | z:double | hire_date:date_nanos +0 | null | null | 1985-02-18T00:00:00.000Z +0 | null | null | 1985-02-24T00:00:00.000Z +2 | 3.3 | 2.0 | 1985-05-13T00:00:00.000Z +2 | 0.2 | 1.8 | 1985-07-09T00:00:00.000Z +2 | 3.6 | 1.5 | 1985-09-17T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsMvExpandKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM date_nanos, date_nanos_union_types +| MV_EXPAND nanos +| SORT nanos +| LIMIT 4 +; + +warningRegex:evaluation of \[FROM .*date_nanos.*date_nanos_union_types.*\] failed, treating result as null. Only first 20 failures recorded. +warningRegex:java.lang.IllegalArgumentException: milliSeconds \[-1457696696640\] are before the epoch in 1970 and cannot be converted to nanoseconds + +millis:date_nanos | nanos:date_nanos | num:long +1999-10-23T12:15:03.360103847Z | 2023-01-23T13:55:01.543Z | 0 +1999-10-23T12:15:03.360Z | 2023-01-23T13:55:01.543123456Z | 0 +1999-10-23T12:15:03.360103847Z | 2023-02-23T13:33:34.937Z | 0 +1999-10-23T12:15:03.360Z | 2023-02-23T13:33:34.937193Z | 0 +; + +ImplicitCastingMultiTypedFieldsBackAndForth +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM date_nanos, date_nanos_union_types +| MV_EXPAND nanos +| EVAL a = nanos::datetime, b = nanos::date_nanos, c = nanos, d = nanos::datetime::date_nanos, e = nanos::date_nanos::datetime +| KEEP millis, nanos, a, b, c, d, e +| SORT nanos, millis +| LIMIT 14 +; + +warningRegex:evaluation of \[FROM .*date_nanos.*date_nanos_union_types.*\] failed, treating result as null. Only first 20 failures recorded. +warningRegex:java.lang.IllegalArgumentException: milliSeconds \[-1457696696640\] are before the epoch in 1970 and cannot be converted to nanoseconds + +millis:date_nanos | nanos:date_nanos | a:datetime | b:date_nanos | c:date_nanos | d:date_nanos | e:datetime +1999-10-23T12:15:03.360103847Z | 2023-01-23T13:55:01.543Z | 2023-01-23T13:55:01.543Z | 2023-01-23T13:55:01.543Z | 2023-01-23T13:55:01.543Z | 2023-01-23T13:55:01.543Z | 2023-01-23T13:55:01.543Z +1999-10-23T12:15:03.360Z | 2023-01-23T13:55:01.543123456Z | 2023-01-23T13:55:01.543Z | 2023-01-23T13:55:01.543123456Z | 2023-01-23T13:55:01.543123456Z | 2023-01-23T13:55:01.543Z | 2023-01-23T13:55:01.543Z +1999-10-23T12:15:03.360103847Z | 2023-02-23T13:33:34.937Z | 2023-02-23T13:33:34.937Z | 2023-02-23T13:33:34.937Z | 2023-02-23T13:33:34.937Z | 2023-02-23T13:33:34.937Z | 2023-02-23T13:33:34.937Z +1999-10-23T12:15:03.360Z | 2023-02-23T13:33:34.937193Z | 2023-02-23T13:33:34.937Z | 2023-02-23T13:33:34.937193Z | 2023-02-23T13:33:34.937193Z | 2023-02-23T13:33:34.937Z | 2023-02-23T13:33:34.937Z +1999-10-22T12:15:03.360103847Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z +1999-10-22T12:15:03.360103847Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z +1999-10-22T12:15:03.360103847Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z +1999-10-23T12:15:03.360103847Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z +1999-10-22T12:15:03.360Z | 2023-03-23T12:15:03.360103847Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360103847Z | 2023-03-23T12:15:03.360103847Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z +1999-10-22T12:15:03.360Z | 2023-03-23T12:15:03.360103847Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360103847Z | 2023-03-23T12:15:03.360103847Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z +1999-10-22T12:15:03.360Z | 2023-03-23T12:15:03.360103847Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360103847Z | 2023-03-23T12:15:03.360103847Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z +1999-10-23T12:15:03.360Z | 2023-03-23T12:15:03.360103847Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360103847Z | 2023-03-23T12:15:03.360103847Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z +2023-10-23T12:15:03.360103847Z | 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360Z +2023-10-23T12:15:03.360103847Z | 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360Z +; + +ImplicitCastingMultiTypedFieldsInvalidNanosValidMillis +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM date_nanos, date_nanos_union_types +| EVAL a = nanos::datetime +| MV_EXPAND a +| SORT a, millis, nanos +| LIMIT 14 +; + +warningRegex:evaluation of \[FROM .*date_nanos.*date_nanos_union_types.*\] failed, treating result as null. Only first 20 failures recorded. +warningRegex:java.lang.IllegalArgumentException: milliSeconds \[-1457696696640\] are before the epoch in 1970 and cannot be converted to nanoseconds + +millis:date_nanos | nanos:date_nanos | num:long | a:datetime +2023-10-23T12:15:03.360103847Z | null | 1698063303360103847 | 1923-10-23T12:15:03.360Z +1999-10-23T12:15:03.360Z | [2023-01-23T13:55:01.543123456Z, 2023-02-23T13:33:34.937193Z, 2023-03-23T12:15:03.360103847Z] | 0 | 2023-01-23T13:55:01.543Z +1999-10-23T12:15:03.360103847Z | [2023-01-23T13:55:01.543Z, 2023-02-23T13:33:34.937Z, 2023-03-23T12:15:03.360Z] | 0 | 2023-01-23T13:55:01.543Z +1999-10-23T12:15:03.360Z | [2023-01-23T13:55:01.543123456Z, 2023-02-23T13:33:34.937193Z, 2023-03-23T12:15:03.360103847Z] | 0 | 2023-02-23T13:33:34.937Z +1999-10-23T12:15:03.360103847Z | [2023-01-23T13:55:01.543Z, 2023-02-23T13:33:34.937Z, 2023-03-23T12:15:03.360Z] | 0 | 2023-02-23T13:33:34.937Z +1999-10-22T12:15:03.360Z | [2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z] | 0 | 2023-03-23T12:15:03.360Z +1999-10-22T12:15:03.360Z | [2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z] | 0 | 2023-03-23T12:15:03.360Z +1999-10-22T12:15:03.360Z | [2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z] | 0 | 2023-03-23T12:15:03.360Z +1999-10-22T12:15:03.360103847Z | [2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z] | 0 | 2023-03-23T12:15:03.360Z +1999-10-22T12:15:03.360103847Z | [2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z] | 0 | 2023-03-23T12:15:03.360Z +1999-10-22T12:15:03.360103847Z | [2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z] | 0 | 2023-03-23T12:15:03.360Z +1999-10-23T12:15:03.360Z | [2023-01-23T13:55:01.543123456Z, 2023-02-23T13:33:34.937193Z, 2023-03-23T12:15:03.360103847Z] | 0 | 2023-03-23T12:15:03.360Z +1999-10-23T12:15:03.360103847Z | [2023-01-23T13:55:01.543Z, 2023-02-23T13:33:34.937Z, 2023-03-23T12:15:03.360Z] | 0 | 2023-03-23T12:15:03.360Z +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 | 2023-10-23T12:15:03.360Z +; + +ImplicitCastingMultiTypedFieldsInvalidNanosValidMillisExplicitCastToNanos +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM date_nanos, date_nanos_union_types +| EVAL a = nanos::date_nanos::datetime +| WHERE millis < "2023-10-23T13:00:00" +| MV_EXPAND a +| SORT a, millis, nanos +; + +warningRegex:evaluation of \[FROM .*date_nanos.*date_nanos_union_types.*\] failed, treating result as null. Only first 20 failures recorded. +warningRegex:java.lang.IllegalArgumentException: milliSeconds \[-1457696696640\] are before the epoch in 1970 and cannot be converted to nanoseconds + +millis:date_nanos | nanos:date_nanos | num:long | a:datetime +1999-10-23T12:15:03.360Z | [2023-01-23T13:55:01.543123456Z, 2023-02-23T13:33:34.937193Z, 2023-03-23T12:15:03.360103847Z] | 0 | 2023-01-23T13:55:01.543Z +1999-10-23T12:15:03.360103847Z | [2023-01-23T13:55:01.543Z, 2023-02-23T13:33:34.937Z, 2023-03-23T12:15:03.360Z] | 0 | 2023-01-23T13:55:01.543Z +1999-10-23T12:15:03.360Z | [2023-01-23T13:55:01.543123456Z, 2023-02-23T13:33:34.937193Z, 2023-03-23T12:15:03.360103847Z] | 0 | 2023-02-23T13:33:34.937Z +1999-10-23T12:15:03.360103847Z | [2023-01-23T13:55:01.543Z, 2023-02-23T13:33:34.937Z, 2023-03-23T12:15:03.360Z] | 0 | 2023-02-23T13:33:34.937Z +1999-10-22T12:15:03.360Z | [2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z] | 0 | 2023-03-23T12:15:03.360Z +1999-10-22T12:15:03.360Z | [2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z] | 0 | 2023-03-23T12:15:03.360Z +1999-10-22T12:15:03.360Z | [2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z] | 0 | 2023-03-23T12:15:03.360Z +1999-10-22T12:15:03.360103847Z | [2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z] | 0 | 2023-03-23T12:15:03.360Z +1999-10-22T12:15:03.360103847Z | [2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z] | 0 | 2023-03-23T12:15:03.360Z +1999-10-22T12:15:03.360103847Z | [2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z] | 0 | 2023-03-23T12:15:03.360Z +1999-10-23T12:15:03.360Z | [2023-01-23T13:55:01.543123456Z, 2023-02-23T13:33:34.937193Z, 2023-03-23T12:15:03.360103847Z] | 0 | 2023-03-23T12:15:03.360Z +1999-10-23T12:15:03.360103847Z | [2023-01-23T13:55:01.543Z, 2023-02-23T13:33:34.937Z, 2023-03-23T12:15:03.360Z] | 0 | 2023-03-23T12:15:03.360Z +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 | 2023-10-23T12:15:03.360Z +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 | 2023-10-23T12:15:03.360Z +2023-10-23T12:15:03.360103847Z | 2023-10-23T12:15:03.360Z | 1698063303360103847 | 2023-10-23T12:15:03.360Z +2023-10-23T12:15:03.360103847Z | 2023-10-23T12:15:03.360Z | 1698063303360103847 | 2023-10-23T12:15:03.360Z +2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948Z | 1698064048948000000 | 2023-10-23T12:27:28.948Z +2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948Z | 1698064048948000000 | 2023-10-23T12:27:28.948Z +2023-10-23T12:15:03.360103847Z | null | 1698063303360103847 | null +; + +ImplicitCastingMultiTypedMVFieldsEval +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM date_nanos, date_nanos_union_types +| EVAL nanos = MV_MAX(nanos) +| SORT nanos, millis +| LIMIT 4 +; + +warningRegex:evaluation of \[FROM .*date_nanos.*date_nanos_union_types.*\] failed, treating result as null. Only first 20 failures recorded. +warningRegex:java.lang.IllegalArgumentException: milliSeconds \[-1457696696640\] are before the epoch in 1970 and cannot be converted to nanoseconds +millis:date_nanos | num:long | nanos:date_nanos +1999-10-22T12:15:03.360103847Z | 0 | 2023-03-23T12:15:03.360Z +1999-10-23T12:15:03.360103847Z | 0 | 2023-03-23T12:15:03.360Z +1999-10-22T12:15:03.360Z | 0 | 2023-03-23T12:15:03.360103847Z +1999-10-23T12:15:03.360Z | 0 | 2023-03-23T12:15:03.360103847Z +; + +ImplicitCastingMultiTypedMVFieldsWhere +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM date_nanos, date_nanos_union_types +| WHERE millis <= "2023-10-23T13:00:00" AND MV_MAX(nanos) > "2023-03-23T13:00:00" +| KEEP millis, nanos, num +| SORT millis +; + +warningRegex:evaluation of \[FROM .*date_nanos.*date_nanos_union_types.*\] failed, treating result as null. Only first 20 failures recorded. +warningRegex:java.lang.IllegalArgumentException: milliSeconds \[-1457696696640\] are before the epoch in 1970 and cannot be converted to nanoseconds +millis:date_nanos | nanos:date_nanos | num:long +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 +2023-10-23T12:15:03.360103847Z | 2023-10-23T12:15:03.360Z | 1698063303360103847 +2023-10-23T12:15:03.360103847Z | 2023-10-23T12:15:03.360Z | 1698063303360103847 +2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948Z | 1698064048948000000 +2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948Z | 1698064048948000000 +; + +ImplicitCastingMultiTypedMVFieldsStatsMaxMin +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM date_nanos, date_nanos_union_types +| STATS max = MAX(millis), min = MIN(nanos) +; + +warningRegex:evaluation of \[FROM .*date_nanos.*date_nanos_union_types.*\] failed, treating result as null. Only first 20 failures recorded. +warningRegex:java.lang.IllegalArgumentException: milliSeconds \[-1457696696640\] are before the epoch in 1970 and cannot be converted to nanoseconds + +max:date_nanos | min:date_nanos +2023-10-23T13:55:01.543123456Z | 2023-01-23T13:55:01.543Z +; + +ImplicitCastingMultiTypedMVFieldsStatsValues +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM date_nanos, date_nanos_union_types +| STATS c = MV_COUNT(VALUES(nanos)) +; + +warningRegex:evaluation of \[FROM .*date_nanos.*date_nanos_union_types.*\] failed, treating result as null. Only first 20 failures recorded. +warningRegex:java.lang.IllegalArgumentException: milliSeconds \[-1457696696640\] are before the epoch in 1970 and cannot be converted to nanoseconds + +c:integer +19 +; + +ImplicitCastingMultiTypedDateDiff +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL emp_no = emp_no::long +| WHERE emp_no <= 10003 +| EVAL diff_sec_1 = DATE_DIFF("seconds", TO_DATE_NANOS("1986-01-23T12:15:03.360103847Z"), hire_date) +| EVAL diff_sec_2 = DATE_DIFF("seconds", TO_DATETIME("1986-01-23T12:15:03.360103847Z"), hire_date) +| KEEP emp_no, hire_date, diff_sec_1, diff_sec_2 +| SORT emp_no, hire_date +; + +emp_no:long | hire_date:date_nanos | diff_sec_1:integer | diff_sec_2:integer +10001 | 1986-06-26T00:00:00.000Z | 13261496 | 13261496 +10001 | 1986-06-26T00:00:00.000Z | 13261496 | 13261496 +10002 | 1985-11-21T00:00:00.000Z | -5487303 | -5487303 +10002 | 1985-11-21T00:00:00.000Z | -5487303 | -5487303 +10003 | 1986-08-28T00:00:00.000Z | 18704696 | 18704696 +10003 | 1986-08-28T00:00:00.000Z | 18704696 | 18704696 +; + +ImplicitCastingMultiTypedDateFormat +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL emp_no = emp_no::long +| WHERE emp_no <= 10003 +| EVAL a = DATE_FORMAT(hire_date), b = DATE_FORMAT("yyyy-MM-dd", hire_date), c = DATE_FORMAT("strict_date_optional_time_nanos", hire_date) +| KEEP emp_no, hire_date, a, b, c +| SORT emp_no, hire_date +; + +emp_no:long | hire_date:date_nanos | a:keyword | b:keyword | c:keyword +10001 | 1986-06-26T00:00:00.000Z | 1986-06-26T00:00:00.000Z | 1986-06-26 | 1986-06-26T00:00:00.000Z +10001 | 1986-06-26T00:00:00.000Z | 1986-06-26T00:00:00.000Z | 1986-06-26 | 1986-06-26T00:00:00.000Z +10002 | 1985-11-21T00:00:00.000Z | 1985-11-21T00:00:00.000Z | 1985-11-21 | 1985-11-21T00:00:00.000Z +10002 | 1985-11-21T00:00:00.000Z | 1985-11-21T00:00:00.000Z | 1985-11-21 | 1985-11-21T00:00:00.000Z +10003 | 1986-08-28T00:00:00.000Z | 1986-08-28T00:00:00.000Z | 1986-08-28 | 1986-08-28T00:00:00.000Z +10003 | 1986-08-28T00:00:00.000Z | 1986-08-28T00:00:00.000Z | 1986-08-28 | 1986-08-28T00:00:00.000Z +; + +ImplicitCastingMultiTypedDateTrunc +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL emp_no = emp_no::long +| WHERE emp_no <= 10003 +| EVAL yr = DATE_TRUNC(1 year, hire_date), mo = DATE_TRUNC(1 month, hire_date) +| SORT hire_date DESC +| KEEP emp_no, hire_date, yr, mo +; + +emp_no:long | hire_date:date_nanos | yr:date_nanos | mo:date_nanos +10003 | 1986-08-28T00:00:00.000Z | 1986-01-01T00:00:00.000Z | 1986-08-01T00:00:00.000Z +10003 | 1986-08-28T00:00:00.000Z | 1986-01-01T00:00:00.000Z | 1986-08-01T00:00:00.000Z +10001 | 1986-06-26T00:00:00.000Z | 1986-01-01T00:00:00.000Z | 1986-06-01T00:00:00.000Z +10001 | 1986-06-26T00:00:00.000Z | 1986-01-01T00:00:00.000Z | 1986-06-01T00:00:00.000Z +10002 | 1985-11-21T00:00:00.000Z | 1985-01-01T00:00:00.000Z | 1985-11-01T00:00:00.000Z +10002 | 1985-11-21T00:00:00.000Z | 1985-01-01T00:00:00.000Z | 1985-11-01T00:00:00.000Z +; + +ImplicitCastingMultiTypedDateTruncStatsBy +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| STATS c = count(emp_no::long) BY yr = DATE_TRUNC(1 year, hire_date) +| SORT yr DESC +| LIMIT 5 +; + +c:long | yr:date_nanos +2 | 1999-01-01T00:00:00.000Z +2 | 1997-01-01T00:00:00.000Z +2 | 1996-01-01T00:00:00.000Z +10 | 1995-01-01T00:00:00.000Z +8 | 1994-01-01T00:00:00.000Z +; + +ImplicitCastingMultiTypedDateTruncStatsByWithFilter +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| STATS c = count(emp_no::long) where hire_date > "1996-01-01" BY yr = DATE_TRUNC(1 year, hire_date) +| SORT yr DESC +| LIMIT 5 +; + +c:long | yr:date_nanos +2 | 1999-01-01T00:00:00.000Z +2 | 1997-01-01T00:00:00.000Z +2 | 1996-01-01T00:00:00.000Z +0 | 1995-01-01T00:00:00.000Z +0 | 1994-01-01T00:00:00.000Z +; + +ImplicitCastingMultiTypedDateTruncStatsByWithEval +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL yr = DATE_TRUNC(1 year, hire_date) +| STATS c = count(emp_no::long) BY yr +| SORT yr DESC +| LIMIT 5 +; + +c:long | yr:date_nanos +2 | 1999-01-01T00:00:00.000Z +2 | 1997-01-01T00:00:00.000Z +2 | 1996-01-01T00:00:00.000Z +10 | 1995-01-01T00:00:00.000Z +8 | 1994-01-01T00:00:00.000Z +; + +ImplicitCastingMultiTypedDateTruncStatsByWithEvalWithFilter +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL yr = DATE_TRUNC(1 year, hire_date) +| STATS c = count(emp_no::long) where hire_date > "1991-01-01" BY yr +| SORT yr DESC +| LIMIT 5 +; + +c:long | yr:date_nanos +2 | 1999-01-01T00:00:00.000Z +2 | 1997-01-01T00:00:00.000Z +2 | 1996-01-01T00:00:00.000Z +10 | 1995-01-01T00:00:00.000Z +8 | 1994-01-01T00:00:00.000Z +; + +ImplicitCastingMultiTypedBucketDateNanosByYear +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| STATS c = count(*) BY yr = BUCKET(hire_date, 1 year) +| SORT yr DESC, c +; + +c:long | yr:date_nanos +2 | 1999-01-01T00:00:00.000Z +2 | 1997-01-01T00:00:00.000Z +2 | 1996-01-01T00:00:00.000Z +10 | 1995-01-01T00:00:00.000Z +8 | 1994-01-01T00:00:00.000Z +6 | 1993-01-01T00:00:00.000Z +16 | 1992-01-01T00:00:00.000Z +12 | 1991-01-01T00:00:00.000Z +24 | 1990-01-01T00:00:00.000Z +26 | 1989-01-01T00:00:00.000Z +18 | 1988-01-01T00:00:00.000Z +30 | 1987-01-01T00:00:00.000Z +22 | 1986-01-01T00:00:00.000Z +22 | 1985-01-01T00:00:00.000Z +; + +ImplicitCastingMultiTypedBucketDateNanosByYearWithFilter +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| STATS c = count(*) where hire_date > "1991-01-01" BY yr = BUCKET(hire_date, 1 year) +| SORT yr DESC, c +; + +c:long | yr:date_nanos +2 | 1999-01-01T00:00:00.000Z +2 | 1997-01-01T00:00:00.000Z +2 | 1996-01-01T00:00:00.000Z +10 | 1995-01-01T00:00:00.000Z +8 | 1994-01-01T00:00:00.000Z +6 | 1993-01-01T00:00:00.000Z +16 | 1992-01-01T00:00:00.000Z +12 | 1991-01-01T00:00:00.000Z +0 | 1990-01-01T00:00:00.000Z +0 | 1989-01-01T00:00:00.000Z +0 | 1988-01-01T00:00:00.000Z +0 | 1987-01-01T00:00:00.000Z +0 | 1986-01-01T00:00:00.000Z +0 | 1985-01-01T00:00:00.000Z +; + +ImplicitCastingMultiTypedBucketDateNanosByMonth +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| STATS c = count(*) BY mo = BUCKET(hire_date, 20, "1986-01-01", "1999-12-31") +| SORT mo DESC, c +; + +c:long | mo:date_nanos +2 | 1999-01-01T00:00:00.000Z +2 | 1997-01-01T00:00:00.000Z +2 | 1996-01-01T00:00:00.000Z +10 | 1995-01-01T00:00:00.000Z +8 | 1994-01-01T00:00:00.000Z +6 | 1993-01-01T00:00:00.000Z +16 | 1992-01-01T00:00:00.000Z +12 | 1991-01-01T00:00:00.000Z +24 | 1990-01-01T00:00:00.000Z +26 | 1989-01-01T00:00:00.000Z +18 | 1988-01-01T00:00:00.000Z +30 | 1987-01-01T00:00:00.000Z +22 | 1986-01-01T00:00:00.000Z +22 | 1985-01-01T00:00:00.000Z +; + +ImplicitCastingMultiTypedBucketDateNanosInBothStatsAndBy +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| STATS c = count(*), b = BUCKET(hire_date, 1 year) + 1 year BY yr = BUCKET(hire_date, 1 year) +| SORT yr DESC, c +; + +c:long | b:date_nanos | yr:date_nanos +2 | 2000-01-01T00:00:00.000Z | 1999-01-01T00:00:00.000Z +2 | 1998-01-01T00:00:00.000Z | 1997-01-01T00:00:00.000Z +2 | 1997-01-01T00:00:00.000Z | 1996-01-01T00:00:00.000Z +10 | 1996-01-01T00:00:00.000Z | 1995-01-01T00:00:00.000Z +8 | 1995-01-01T00:00:00.000Z | 1994-01-01T00:00:00.000Z +6 | 1994-01-01T00:00:00.000Z | 1993-01-01T00:00:00.000Z +16 | 1993-01-01T00:00:00.000Z | 1992-01-01T00:00:00.000Z +12 | 1992-01-01T00:00:00.000Z | 1991-01-01T00:00:00.000Z +24 | 1991-01-01T00:00:00.000Z | 1990-01-01T00:00:00.000Z +26 | 1990-01-01T00:00:00.000Z | 1989-01-01T00:00:00.000Z +18 | 1989-01-01T00:00:00.000Z | 1988-01-01T00:00:00.000Z +30 | 1988-01-01T00:00:00.000Z | 1987-01-01T00:00:00.000Z +22 | 1987-01-01T00:00:00.000Z | 1986-01-01T00:00:00.000Z +22 | 1986-01-01T00:00:00.000Z | 1985-01-01T00:00:00.000Z +; + +ImplicitCastingMultiTypedBucketDateNanosInBothStatsAndByWithAlias +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| STATS c = count(*), b = yr + 1 year BY yr = BUCKET(hire_date, 1 year) +| SORT yr DESC, c +; + +c:long | b:date_nanos | yr:date_nanos +2 | 2000-01-01T00:00:00.000Z | 1999-01-01T00:00:00.000Z +2 | 1998-01-01T00:00:00.000Z | 1997-01-01T00:00:00.000Z +2 | 1997-01-01T00:00:00.000Z | 1996-01-01T00:00:00.000Z +10 | 1996-01-01T00:00:00.000Z | 1995-01-01T00:00:00.000Z +8 | 1995-01-01T00:00:00.000Z | 1994-01-01T00:00:00.000Z +6 | 1994-01-01T00:00:00.000Z | 1993-01-01T00:00:00.000Z +16 | 1993-01-01T00:00:00.000Z | 1992-01-01T00:00:00.000Z +12 | 1992-01-01T00:00:00.000Z | 1991-01-01T00:00:00.000Z +24 | 1991-01-01T00:00:00.000Z | 1990-01-01T00:00:00.000Z +26 | 1990-01-01T00:00:00.000Z | 1989-01-01T00:00:00.000Z +18 | 1989-01-01T00:00:00.000Z | 1988-01-01T00:00:00.000Z +30 | 1988-01-01T00:00:00.000Z | 1987-01-01T00:00:00.000Z +22 | 1987-01-01T00:00:00.000Z | 1986-01-01T00:00:00.000Z +22 | 1986-01-01T00:00:00.000Z | 1985-01-01T00:00:00.000Z +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 1687189d92c01..f909308f05e10 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -318,6 +318,11 @@ public enum Cap { */ STRING_LITERAL_AUTO_CASTING_TO_DATETIME_ADD_SUB, + /** + * Support implicit casting for union typed fields that are mixed with date and date_nanos type. + */ + IMPLICIT_CASTING_DATE_AND_DATE_NANOS(Build.current().isSnapshot()), + /** * Support for named or positional parameters in EsqlQueryRequest. */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index a265e86adc943..9790d1f1ddedc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -60,6 +60,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ConvertFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.FoldablesConvertFunction; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDateNanos; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDouble; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToInteger; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToLong; @@ -143,8 +144,10 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.IP; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; +import static org.elasticsearch.xpack.esql.core.type.DataType.NULL; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.TIME_DURATION; +import static org.elasticsearch.xpack.esql.core.type.DataType.UNSUPPORTED; import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION; import static org.elasticsearch.xpack.esql.core.type.DataType.isTemporalAmount; import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.LIMIT; @@ -158,7 +161,7 @@ public class Analyzer extends ParameterizedRuleExecutor NO_FIELDS = List.of( - new ReferenceAttribute(Source.EMPTY, NO_FIELDS_NAME, DataType.NULL, Nullability.TRUE, null, true) + new ReferenceAttribute(Source.EMPTY, NO_FIELDS_NAME, NULL, Nullability.TRUE, null, true) ); private static final List> RULES = List.of( @@ -169,7 +172,8 @@ public class Analyzer extends ParameterizedRuleExecutor( "Resolution", @@ -678,7 +682,7 @@ private LogicalPlan resolveLookup(Lookup l, List childrenOutput) { */ boolean dataTypesOk = joinedAttribute.dataType().equals(attr.dataType()); if (false == dataTypesOk) { - dataTypesOk = joinedAttribute.dataType() == DataType.NULL || attr.dataType() == DataType.NULL; + dataTypesOk = joinedAttribute.dataType() == NULL || attr.dataType() == NULL; } if (false == dataTypesOk) { dataTypesOk = joinedAttribute.dataType().equals(KEYWORD) && attr.dataType().equals(TEXT); @@ -1363,7 +1367,7 @@ private BitSet gatherPreAnalysisMetrics(LogicalPlan plan, BitSet b) { *
  • date_trunc("1 minute", dateField)
    • * * If the inputs to Coalesce are mixed numeric types, cast the rest of the numeric field or value to the first numeric data type if - * applicable. For example, implicit casting converts: + * applicable, the same applies to Case, Greatest, Least. For example, implicit casting converts: *
      *
    • Coalesce(Long, Int) to Coalesce(Long, Long)
      • *
    • Coalesce(null, Long, Int) to Coalesce(null, Long, Long)
      • @@ -1375,6 +1379,7 @@ private BitSet gatherPreAnalysisMetrics(LogicalPlan plan, BitSet b) { private static class ImplicitCasting extends ParameterizedRule { @Override public LogicalPlan apply(LogicalPlan plan, AnalyzerContext context) { + // do implicit casting for function arguments return plan.transformExpressionsUp( org.elasticsearch.xpack.esql.core.expression.function.Function.class, e -> ImplicitCasting.cast(e, context.functionRegistry().snapshotRegistry()) @@ -1405,7 +1410,7 @@ private static Expression processScalarOrGroupingFunction( } List newChildren = new ArrayList<>(args.size()); boolean childrenChanged = false; - DataType targetDataType = DataType.NULL; + DataType targetDataType = NULL; Expression arg; DataType targetNumericType = null; boolean castNumericArgs = true; @@ -1418,7 +1423,7 @@ private static Expression processScalarOrGroupingFunction( if (i < targetDataTypes.size()) { targetDataType = targetDataTypes.get(i); } - if (targetDataType != DataType.NULL && targetDataType != DataType.UNSUPPORTED) { + if (targetDataType != NULL && targetDataType != UNSUPPORTED) { Expression e = castStringLiteral(arg, targetDataType); if (e != arg) { childrenChanged = true; @@ -1451,7 +1456,7 @@ private static Expression processBinaryOperator(BinaryOperator o) { } List newChildren = new ArrayList<>(2); boolean childrenChanged = false; - DataType targetDataType = DataType.NULL; + DataType targetDataType = NULL; Expression from = Literal.NULL; if (left.dataType() == KEYWORD && left.foldable() && (left instanceof EsqlScalarFunction == false)) { @@ -1613,7 +1618,7 @@ public LogicalPlan apply(LogicalPlan plan) { // Collect field attributes from previous runs plan.forEachUp(EsRelation.class, rel -> { for (Attribute attr : rel.output()) { - if (attr instanceof FieldAttribute fa && fa.field() instanceof MultiTypeEsField) { + if (attr instanceof FieldAttribute fa && fa.field() instanceof MultiTypeEsField && fa.synthetic()) { unionFieldAttributes.add(fa); } } @@ -1679,9 +1684,7 @@ private Expression resolveConvertFunction(ConvertFunction convert, List { if (supportedTypes.contains(type.widenSmallNumeric())) { - TypeResolutionKey key = new TypeResolutionKey(fa.name(), type); - var concreteConvert = typeSpecificConvert(convert, fa.source(), type, imf); - typeResolutions.put(key, concreteConvert); + typeResolutions(fa, convert, type, imf, typeResolutions); } }); // If all mapped types were resolved, create a new FieldAttribute with the resolved MultiTypeEsField @@ -1689,11 +1692,45 @@ private Expression resolveConvertFunction(ConvertFunction convert, List supportedTypes = convert.supportedTypes(); + if (supportedTypes.contains(fa.dataType()) && canConvertOriginalTypes(mtf, supportedTypes)) { + // Build the mapping between index name and conversion expressions + Map indexToConversionExpressions = new HashMap<>(); + for (Map.Entry entry : mtf.getIndexToConversionExpressions().entrySet()) { + String indexName = entry.getKey(); + AbstractConvertFunction originalConversionFunction = (AbstractConvertFunction) entry.getValue(); + Expression originalField = originalConversionFunction.field(); + Expression newConvertFunction = convertExpression.replaceChildren(Collections.singletonList(originalField)); + indexToConversionExpressions.put(indexName, newConvertFunction); + } + MultiTypeEsField multiTypeEsField = new MultiTypeEsField( + fa.fieldName(), + convertExpression.dataType(), + false, + indexToConversionExpressions + ); + return createIfDoesNotAlreadyExist(fa, multiTypeEsField, unionFieldAttributes); + } + } else if (convert.field() instanceof AbstractConvertFunction subConvert) { + return convertExpression.replaceChildren( + Collections.singletonList(resolveConvertFunction(subConvert, unionFieldAttributes)) + ); + } return convertExpression; } @@ -1717,7 +1754,10 @@ private Expression createIfDoesNotAlreadyExist( } } - private MultiTypeEsField resolvedMultiTypeEsField(FieldAttribute fa, HashMap typeResolutions) { + private static MultiTypeEsField resolvedMultiTypeEsField( + FieldAttribute fa, + HashMap typeResolutions + ) { Map typesToConversionExpressions = new HashMap<>(); InvalidMappedField imf = (InvalidMappedField) fa.field(); imf.getTypesToIndices().forEach((typeName, indexNames) -> { @@ -1730,8 +1770,22 @@ private MultiTypeEsField resolvedMultiTypeEsField(FieldAttribute fa, HashMap supportedTypes) { + return multiTypeEsField.getIndexToConversionExpressions() + .values() + .stream() + .allMatch( + e -> e instanceof AbstractConvertFunction convertFunction + && supportedTypes.contains(convertFunction.field().dataType().widenSmallNumeric()) + ); + } + + private static Expression typeSpecificConvert(ConvertFunction convert, Source source, DataType type, InvalidMappedField mtf) { EsField field = new EsField(mtf.getName(), type, mtf.getProperties(), mtf.isAggregatable()); + return typeSpecificConvert(convert, source, field); + } + + private static Expression typeSpecificConvert(ConvertFunction convert, Source source, EsField field) { FieldAttribute originalFieldAttr = (FieldAttribute) convert.field(); FieldAttribute resolvedAttr = new FieldAttribute( source, @@ -1806,4 +1860,40 @@ private static LogicalPlan planWithoutSyntheticAttributes(LogicalPlan plan) { return newOutput.size() == output.size() ? plan : new Project(Source.EMPTY, plan, newOutput); } } + + /** + * Cast the union typed fields in EsRelation to date_nanos if they are mixed date and date_nanos types. + */ + private static class DateMillisToNanosInEsRelation extends Rule { + @Override + public LogicalPlan apply(LogicalPlan plan) { + return plan.transformUp(EsRelation.class, relation -> { + if (relation.indexMode() == IndexMode.LOOKUP) { + return relation; + } + return relation.transformExpressionsUp(FieldAttribute.class, f -> { + if (f.field() instanceof InvalidMappedField imf && imf.types().stream().allMatch(DataType::isDate)) { + HashMap typeResolutions = new HashMap<>(); + var convert = new ToDateNanos(f.source(), f); + imf.types().forEach(type -> typeResolutions(f, convert, type, imf, typeResolutions)); + var resolvedField = ResolveUnionTypes.resolvedMultiTypeEsField(f, typeResolutions); + return new FieldAttribute(f.source(), f.parentName(), f.name(), resolvedField, f.nullable(), f.id(), f.synthetic()); + } + return f; + }); + }); + } + } + + private static void typeResolutions( + FieldAttribute fieldAttribute, + ConvertFunction convert, + DataType type, + InvalidMappedField imf, + HashMap typeResolutions + ) { + ResolveUnionTypes.TypeResolutionKey key = new ResolveUnionTypes.TypeResolutionKey(fieldAttribute.name(), type); + var concreteConvert = ResolveUnionTypes.typeSpecificConvert(convert, fieldAttribute.source(), type, imf); + typeResolutions.put(key, concreteConvert); + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java index 704a0395e97b4..87b5adbfde8b4 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java @@ -11,6 +11,8 @@ import org.elasticsearch.inference.TaskType; import org.elasticsearch.xpack.core.enrich.EnrichPolicy; import org.elasticsearch.xpack.esql.EsqlTestUtils; +import org.elasticsearch.xpack.esql.core.type.EsField; +import org.elasticsearch.xpack.esql.core.type.InvalidMappedField; import org.elasticsearch.xpack.esql.enrich.ResolvedEnrichPolicy; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import org.elasticsearch.xpack.esql.index.EsIndex; @@ -24,8 +26,10 @@ import org.elasticsearch.xpack.esql.session.Configuration; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Predicate; import java.util.function.Supplier; @@ -220,4 +224,25 @@ public static E randomValueOtherThanTest(Predicate exclude, Supplier s } } } + + public static IndexResolution indexWithDateDateNanosUnionType() { + // this method is shared by AnalyzerTest, QueryTranslatorTests and LocalPhysicalPlanOptimizerTests + String dateDateNanos = "date_and_date_nanos"; // mixed date and date_nanos + String dateDateNanosLong = "date_and_date_nanos_and_long"; // mixed date, date_nanos and long + LinkedHashMap> typesToIndices1 = new LinkedHashMap<>(); + typesToIndices1.put("date", Set.of("index1", "index2")); + typesToIndices1.put("date_nanos", Set.of("index3")); + LinkedHashMap> typesToIndices2 = new LinkedHashMap<>(); + typesToIndices2.put("date", Set.of("index1")); + typesToIndices2.put("date_nanos", Set.of("index2")); + typesToIndices2.put("long", Set.of("index3")); + EsField dateDateNanosField = new InvalidMappedField(dateDateNanos, typesToIndices1); + EsField dateDateNanosLongField = new InvalidMappedField(dateDateNanosLong, typesToIndices2); + EsIndex index = new EsIndex( + "test*", + Map.of(dateDateNanos, dateDateNanosField, dateDateNanosLong, dateDateNanosLongField), + Map.of("index1", IndexMode.STANDARD, "index2", IndexMode.STANDARD, "index3", IndexMode.STANDARD) + ); + return IndexResolution.valid(index); + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 54dde6d716217..3b3164ad3dfee 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -35,6 +35,7 @@ import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField; import org.elasticsearch.xpack.esql.core.type.PotentiallyUnmappedKeywordEsField; import org.elasticsearch.xpack.esql.enrich.ResolvedEnrichPolicy; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; @@ -48,7 +49,10 @@ import org.elasticsearch.xpack.esql.expression.function.fulltext.MultiMatch; import org.elasticsearch.xpack.esql.expression.function.fulltext.QueryString; import org.elasticsearch.xpack.esql.expression.function.grouping.Bucket; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDateNanos; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDatetime; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToInteger; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToLong; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Concat; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Substring; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add; @@ -73,6 +77,7 @@ import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.esql.plan.logical.Lookup; import org.elasticsearch.xpack.esql.plan.logical.OrderBy; +import org.elasticsearch.xpack.esql.plan.logical.Project; import org.elasticsearch.xpack.esql.plan.logical.Row; import org.elasticsearch.xpack.esql.plan.logical.RrfScoreEval; import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation; @@ -110,12 +115,16 @@ import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.analyzer; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.analyzerDefaultMapping; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.defaultEnrichResolution; +import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.indexWithDateDateNanosUnionType; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.loadMapping; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.randomValueOtherThanTest; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.tsdbIndexResolution; import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; +import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS; import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_PERIOD; +import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; +import static org.elasticsearch.xpack.esql.core.type.DataType.UNSUPPORTED; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToString; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; @@ -3962,6 +3971,146 @@ public void testBucketWithIntervalInStringInGroupingReferencedInAggregation() { assertEquals(oneYear, literal); } + public void testImplicitCastingForDateAndDateNanosFields() { + assumeTrue("requires snapshot", EsqlCapabilities.Cap.IMPLICIT_CASTING_DATE_AND_DATE_NANOS.isEnabled()); + IndexResolution indexWithUnionTypedFields = indexWithDateDateNanosUnionType(); + Analyzer analyzer = AnalyzerTestUtils.analyzer(indexWithUnionTypedFields); + + // Validate if a union typed field is cast to a type explicitly, implicit casting won't be applied again, and include some cases of + // nested casting as well. + LogicalPlan plan = analyze(""" + FROM tests + | Eval a = date_and_date_nanos, b = date_and_date_nanos::datetime, c = date_and_date_nanos::date_nanos, + d = date_and_date_nanos::datetime::datetime, e = date_and_date_nanos::datetime::date_nanos, + f = date_and_date_nanos::date_nanos::datetime, g = date_and_date_nanos::date_nanos::date_nanos, + h = date_and_date_nanos::datetime::long, i = date_and_date_nanos::date_nanos::long, + j = date_and_date_nanos::long::datetime, k = date_and_date_nanos::long::date_nanos + """, analyzer); + + Project project = as(plan, Project.class); + List projections = project.projections(); + assertEquals(13, projections.size()); + // implicit casting + FieldAttribute fa = as(projections.get(0), FieldAttribute.class); + verifyNameAndTypeAndMultiTypeEsField(fa.name(), fa.dataType(), "date_and_date_nanos", DATE_NANOS, fa); + // long is not cast to date_nanos + UnsupportedAttribute ua = as(projections.get(1), UnsupportedAttribute.class); + verifyNameAndType(ua.name(), ua.dataType(), "date_and_date_nanos_and_long", UNSUPPORTED); + // implicit casting + ReferenceAttribute ra = as(projections.get(2), ReferenceAttribute.class); + verifyNameAndType(ra.name(), ra.dataType(), "a", DATE_NANOS); + // explicit casting + ra = as(projections.get(3), ReferenceAttribute.class); + verifyNameAndType(ra.name(), ra.dataType(), "b", DATETIME); + ra = as(projections.get(4), ReferenceAttribute.class); + verifyNameAndType(ra.name(), ra.dataType(), "c", DATE_NANOS); + ra = as(projections.get(5), ReferenceAttribute.class); + verifyNameAndType(ra.name(), ra.dataType(), "d", DATETIME); + ra = as(projections.get(6), ReferenceAttribute.class); + verifyNameAndType(ra.name(), ra.dataType(), "e", DATE_NANOS); + ra = as(projections.get(7), ReferenceAttribute.class); + verifyNameAndType(ra.name(), ra.dataType(), "f", DATETIME); + ra = as(projections.get(8), ReferenceAttribute.class); + verifyNameAndType(ra.name(), ra.dataType(), "g", DATE_NANOS); + ra = as(projections.get(9), ReferenceAttribute.class); + verifyNameAndType(ra.name(), ra.dataType(), "h", LONG); + ra = as(projections.get(10), ReferenceAttribute.class); + verifyNameAndType(ra.name(), ra.dataType(), "i", LONG); + ra = as(projections.get(11), ReferenceAttribute.class); + verifyNameAndType(ra.name(), ra.dataType(), "j", DATETIME); + ra = as(projections.get(12), ReferenceAttribute.class); + verifyNameAndType(ra.name(), ra.dataType(), "k", DATE_NANOS); + + Limit limit = as(project.child(), Limit.class); + // original Eval coded in the query + Eval eval = as(limit.child(), Eval.class); + List aliases = eval.fields(); + assertEquals(11, aliases.size()); + // implicit casting + Alias a = aliases.get(0); // a = date_and_date_nanos + verifyNameAndTypeAndMultiTypeEsField(a.name(), a.dataType(), "a", DATE_NANOS, a.child()); + // explicit casting + a = aliases.get(1); // b = date_and_date_nanos::datetime + verifyNameAndTypeAndMultiTypeEsField(a.name(), a.dataType(), "b", DATETIME, a.child()); + a = aliases.get(2); // c = date_and_date_nanos::date_nanos + verifyNameAndTypeAndMultiTypeEsField(a.name(), a.dataType(), "c", DATE_NANOS, a.child()); + a = aliases.get(3); // d = date_and_date_nanos::datetime::datetime + verifyNameAndType(a.name(), a.dataType(), "d", DATETIME); + ToDatetime toDatetime = as(a.child(), ToDatetime.class); + fa = as(toDatetime.field(), FieldAttribute.class); + verifyNameAndTypeAndMultiTypeEsField( + fa.name(), + fa.dataType(), + "$$date_and_date_nanos$converted_to$datetime", + DATETIME, + toDatetime.field() + ); + a = aliases.get(4); // e = date_and_date_nanos::datetime::date_nanos + verifyNameAndType(a.name(), a.dataType(), "e", DATE_NANOS); + ToDateNanos toDateNanos = as(a.child(), ToDateNanos.class); + fa = as(toDateNanos.field(), FieldAttribute.class); + verifyNameAndTypeAndMultiTypeEsField( + fa.name(), + fa.dataType(), + "$$date_and_date_nanos$converted_to$datetime", + DATETIME, + toDateNanos.field() + ); + a = aliases.get(5); // f = date_and_date_nanos::date_nanos::datetime + verifyNameAndType(a.name(), a.dataType(), "f", DATETIME); + toDatetime = as(a.child(), ToDatetime.class); + fa = as(toDatetime.field(), FieldAttribute.class); + verifyNameAndTypeAndMultiTypeEsField(fa.name(), fa.dataType(), "$$date_and_date_nanos$converted_to$date_nanos", DATE_NANOS, fa); + a = aliases.get(6); // g = date_and_date_nanos::date_nanos::date_nanos + verifyNameAndType(a.name(), a.dataType(), "g", DATE_NANOS); + toDateNanos = as(a.child(), ToDateNanos.class); + fa = as(toDateNanos.field(), FieldAttribute.class); + verifyNameAndTypeAndMultiTypeEsField(fa.name(), fa.dataType(), "$$date_and_date_nanos$converted_to$date_nanos", DATE_NANOS, fa); + a = aliases.get(7); // h = date_and_date_nanos::datetime::long + verifyNameAndType(a.name(), a.dataType(), "h", LONG); + ToLong toLong = as(a.child(), ToLong.class); + fa = as(toLong.field(), FieldAttribute.class); + verifyNameAndTypeAndMultiTypeEsField(fa.name(), fa.dataType(), "$$date_and_date_nanos$converted_to$datetime", DATETIME, fa); + a = aliases.get(8); // i = date_and_date_nanos::date_nanos::long + verifyNameAndType(a.name(), a.dataType(), "i", LONG); + toLong = as(a.child(), ToLong.class); + fa = as(toLong.field(), FieldAttribute.class); + verifyNameAndTypeAndMultiTypeEsField(fa.name(), fa.dataType(), "$$date_and_date_nanos$converted_to$date_nanos", DATE_NANOS, fa); + a = aliases.get(9); // j = date_and_date_nanos::long::datetime + verifyNameAndType(a.name(), a.dataType(), "j", DATETIME); + toDatetime = as(a.child(), ToDatetime.class); + fa = as(toDatetime.field(), FieldAttribute.class); + verifyNameAndTypeAndMultiTypeEsField(fa.name(), fa.dataType(), "$$date_and_date_nanos$converted_to$long", LONG, fa); + a = aliases.get(10); // k = date_and_date_nanos::long::date_nanos + verifyNameAndType(a.name(), a.dataType(), "k", DATE_NANOS); + toDateNanos = as(a.child(), ToDateNanos.class); + fa = as(toDateNanos.field(), FieldAttribute.class); + verifyNameAndTypeAndMultiTypeEsField(fa.name(), fa.dataType(), "$$date_and_date_nanos$converted_to$long", LONG, fa); + EsRelation esRelation = as(eval.child(), EsRelation.class); + assertEquals("test*", esRelation.indexPattern()); + } + + private void verifyNameAndType(String actualName, DataType actualType, String expectedName, DataType expectedType) { + assertEquals(expectedName, actualName); + assertEquals(expectedType, actualType); + } + + private void verifyNameAndTypeAndMultiTypeEsField( + String actualName, + DataType actualType, + String expectedName, + DataType expectedType, + Expression e + ) { + assertEquals(expectedName, actualName); + assertEquals(expectedType, actualType); + assertTrue(isMultiTypeEsField(e)); + } + + private boolean isMultiTypeEsField(Expression e) { + return e instanceof FieldAttribute fa && fa.field() instanceof MultiTypeEsField; + } + @Override protected IndexAnalyzers createDefaultIndexAnalyzers() { return super.createDefaultIndexAnalyzers(); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java index cc60ed92acbb4..08c9a2f8b4fee 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java @@ -38,6 +38,7 @@ import org.elasticsearch.xpack.esql.analysis.EnrichResolution; import org.elasticsearch.xpack.esql.analysis.Verifier; import org.elasticsearch.xpack.esql.core.expression.Alias; +import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.Literal; @@ -46,9 +47,11 @@ import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; +import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField; import org.elasticsearch.xpack.esql.core.util.Holder; import org.elasticsearch.xpack.esql.enrich.ResolvedEnrichPolicy; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; +import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute; import org.elasticsearch.xpack.esql.expression.function.aggregate.Count; import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction; import org.elasticsearch.xpack.esql.expression.function.fulltext.Kql; @@ -57,6 +60,7 @@ import org.elasticsearch.xpack.esql.expression.function.fulltext.QueryString; import org.elasticsearch.xpack.esql.expression.predicate.logical.Or; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThanOrEqual; import org.elasticsearch.xpack.esql.index.EsIndex; import org.elasticsearch.xpack.esql.index.IndexResolution; import org.elasticsearch.xpack.esql.optimizer.rules.logical.ExtractAggregateCommonFilter; @@ -122,7 +126,9 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.defaultLookupResolution; +import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.indexWithDateDateNanosUnionType; import static org.elasticsearch.xpack.esql.core.querydsl.query.Query.unscore; +import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS; import static org.elasticsearch.xpack.esql.plan.physical.EsStatsQueryExec.StatsType; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; @@ -152,6 +158,7 @@ public class LocalPhysicalPlanOptimizerTests extends MapperServiceTestCase { public static final String MATCH_FUNCTION_QUERY = "from test | where match(%s, %s)"; private TestPlannerOptimizer plannerOptimizer; + private TestPlannerOptimizer plannerOptimizerDateDateNanosUnionTypes; private Analyzer timeSeriesAnalyzer; private final Configuration config; private final SearchStats IS_SV_STATS = new TestSearchStats() { @@ -243,6 +250,13 @@ private Analyzer makeAnalyzer(String mappingFileName) { return makeAnalyzer(mappingFileName, new EnrichResolution()); } + private Analyzer makeAnalyzer(IndexResolution indexResolution) { + return new Analyzer( + new AnalyzerContext(config, new EsqlFunctionRegistry(), indexResolution, new EnrichResolution(), emptyInferenceResolution()), + new Verifier(new Metrics(new EsqlFunctionRegistry()), new XPackLicenseState(() -> 0L)) + ); + } + /** * Expects * LimitExec[1000[INTEGER]] @@ -1911,6 +1925,113 @@ public void testConstantKeywordDissectFilter() { assertNull(query.query()); } + public void testMatchFunctionWithStatsWherePushable() { + String query = """ + from test + | stats c = count(*) where match(last_name, "Smith") + """; + var plan = plannerOptimizer.plan(query); + + var limit = as(plan, LimitExec.class); + var agg = as(limit.child(), AggregateExec.class); + var exchange = as(agg.child(), ExchangeExec.class); + var stats = as(exchange.child(), EsStatsQueryExec.class); + QueryBuilder expected = new MatchQueryBuilder("last_name", "Smith").lenient(true); + assertThat(stats.query().toString(), equalTo(expected.toString())); + } + + public void testMatchFunctionWithStatsPushableAndNonPushableCondition() { + String query = """ + from test + | where length(first_name) > 10 + | stats c = count(*) where match(last_name, "Smith") + """; + var plan = plannerOptimizer.plan(query); + + var limit = as(plan, LimitExec.class); + var agg = as(limit.child(), AggregateExec.class); + var exchange = as(agg.child(), ExchangeExec.class); + var aggExec = as(exchange.child(), AggregateExec.class); + var filter = as(aggExec.child(), FilterExec.class); + assertTrue(filter.condition() instanceof GreaterThan); + var fieldExtract = as(filter.child(), FieldExtractExec.class); + var esQuery = as(fieldExtract.child(), EsQueryExec.class); + QueryBuilder expected = new MatchQueryBuilder("last_name", "Smith").lenient(true); + assertThat(esQuery.query().toString(), equalTo(expected.toString())); + } + + public void testMatchFunctionStatisWithNonPushableCondition() { + String query = """ + from test + | stats c = count(*) where match(last_name, "Smith"), d = count(*) where match(first_name, "Anna") + """; + var plan = plannerOptimizer.plan(query); + + var limit = as(plan, LimitExec.class); + var agg = as(limit.child(), AggregateExec.class); + var aggregates = agg.aggregates(); + assertThat(aggregates.size(), is(2)); + for (NamedExpression aggregate : aggregates) { + var alias = as(aggregate, Alias.class); + var count = as(alias.child(), Count.class); + var match = as(count.filter(), Match.class); + } + var exchange = as(agg.child(), ExchangeExec.class); + var aggExec = as(exchange.child(), AggregateExec.class); + var fieldExtract = as(aggExec.child(), FieldExtractExec.class); + var esQuery = as(fieldExtract.child(), EsQueryExec.class); + assertNull(esQuery.query()); + } + + public void testToDateNanosPushDown() { + assumeTrue("requires snapshot", EsqlCapabilities.Cap.IMPLICIT_CASTING_DATE_AND_DATE_NANOS.isEnabled()); + IndexResolution indexWithUnionTypedFields = indexWithDateDateNanosUnionType(); + plannerOptimizerDateDateNanosUnionTypes = new TestPlannerOptimizer(EsqlTestUtils.TEST_CFG, makeAnalyzer(indexWithUnionTypedFields)); + var stats = EsqlTestUtils.statsForExistingField("date_and_date_nanos", "date_and_date_nanos_and_long"); + String query = """ + from test* + | where date_and_date_nanos < "2025-01-01" and date_and_date_nanos_and_long::date_nanos >= "2024-01-01\""""; + var plan = plannerOptimizerDateDateNanosUnionTypes.plan(query, stats); + + // date_and_date_nanos should be pushed down to EsQueryExec, date_and_date_nanos_and_long should not be pushed down + var project = as(plan, ProjectExec.class); + List projections = project.projections(); + assertEquals(2, projections.size()); + FieldAttribute fa = as(projections.get(0), FieldAttribute.class); + assertEquals(DATE_NANOS, fa.dataType()); + assertEquals("date_and_date_nanos", fa.fieldName()); + assertTrue(isMultiTypeEsField(fa)); // mixed date and date_nanos are auto-casted + UnsupportedAttribute ua = as(projections.get(1), UnsupportedAttribute.class); // mixed date, date_nanos and long are not auto-casted + assertEquals("date_and_date_nanos_and_long", ua.fieldName()); + var limit = as(project.child(), LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + project = as(exchange.child(), ProjectExec.class); + var fieldExtract = as(project.child(), FieldExtractExec.class); + limit = as(fieldExtract.child(), LimitExec.class); + // date_and_date_nanos_and_long::date_nanos >= "2024-01-01" is not pushed down + var filter = as(limit.child(), FilterExec.class); + GreaterThanOrEqual gt = as(filter.condition(), GreaterThanOrEqual.class); + fa = as(gt.left(), FieldAttribute.class); + assertTrue(isMultiTypeEsField(fa)); + assertEquals("date_and_date_nanos_and_long", fa.fieldName()); + fieldExtract = as(filter.child(), FieldExtractExec.class); // extract date_and_date_nanos_and_long + var esQuery = as(fieldExtract.child(), EsQueryExec.class); + var source = ((SingleValueQuery.Builder) esQuery.query()).source(); + var expected = wrapWithSingleQuery( + query, + unscore( + rangeQuery("date_and_date_nanos").lt("2025-01-01T00:00:00.000Z").timeZone("Z").format("strict_date_optional_time_nanos") + ), + "date_and_date_nanos", + source + ); // date_and_date_nanos is pushed down + assertThat(expected.toString(), is(esQuery.query().toString())); + } + + private boolean isMultiTypeEsField(Expression e) { + return e instanceof FieldAttribute fa && fa.field() instanceof MultiTypeEsField; + } + private QueryBuilder wrapWithSingleQuery(String query, QueryBuilder inner, String fieldName, Source source) { return FilterTests.singleValueQuery(query, inner, fieldName, source); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java index 02b108dcf6adb..5683f57c37965 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.EsqlTestUtils; +import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.analysis.Analyzer; import org.elasticsearch.xpack.esql.analysis.AnalyzerContext; import org.elasticsearch.xpack.esql.analysis.Verifier; @@ -20,6 +21,7 @@ import org.elasticsearch.xpack.esql.optimizer.TestPlannerOptimizer; import org.elasticsearch.xpack.esql.plan.physical.EsQueryExec; import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; +import org.elasticsearch.xpack.esql.stats.SearchStats; import org.elasticsearch.xpack.esql.telemetry.Metrics; import org.hamcrest.Matcher; import org.junit.BeforeClass; @@ -31,6 +33,7 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.emptyPolicyResolution; import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping; import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; +import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.indexWithDateDateNanosUnionType; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.matchesRegex; @@ -41,6 +44,8 @@ public class QueryTranslatorTests extends ESTestCase { private static TestPlannerOptimizer plannerOptimizerIPs; + private static TestPlannerOptimizer plannerOptimizerDateDateNanosUnionTypes; + private static Analyzer makeAnalyzer(String mappingFileName) { var mapping = loadMapping(mappingFileName); EsIndex test = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD)); @@ -58,6 +63,19 @@ private static Analyzer makeAnalyzer(String mappingFileName) { ); } + public static Analyzer makeAnalyzer(IndexResolution indexResolution) { + return new Analyzer( + new AnalyzerContext( + EsqlTestUtils.TEST_CFG, + new EsqlFunctionRegistry(), + indexResolution, + emptyPolicyResolution(), + emptyInferenceResolution() + ), + new Verifier(new Metrics(new EsqlFunctionRegistry()), new XPackLicenseState(() -> 0L)) + ); + } + @BeforeClass public static void init() { plannerOptimizer = new TestPlannerOptimizer(EsqlTestUtils.TEST_CFG, makeAnalyzer("mapping-all-types.json")); @@ -83,6 +101,13 @@ public void assertQueryTranslationIPs(String query, Matcher translationM assertThat(translatedQuery, translationMatcher); } + private void assertQueryTranslationDateDateNanosUnionTypes(String query, SearchStats stats, Matcher translationMatcher) { + PhysicalPlan optimized = plannerOptimizerDateDateNanosUnionTypes.plan(query, stats); + EsQueryExec eqe = (EsQueryExec) optimized.collectLeaves().get(0); + final String translatedQuery = eqe.query().toString().replaceAll("\\s+", ""); + assertThat(translatedQuery, translationMatcher); + } + public void testBinaryComparisons() { assertQueryTranslation(""" FROM test | WHERE 10 < integer""", containsString(""" @@ -299,4 +324,128 @@ OR CIDR_MATCH(ip0, "fe80::cae2:65ff:fece:feb9") OR host == "beta\"""", matchesRe esql_single_value":\\{"field":"ip1".*"terms":\\{"ip1":\\["127.0.0.3/32","127.0.0.2".*""" + """ esql_single_value":\\{"field":"ip0".*"terms":\\{"ip0":\\["127.0.0.1","128.0.0.1","fe80::cae2:65ff:fece:feb9".*""")); } + + public void testToDateNanos() { + assumeTrue("requires snapshot", EsqlCapabilities.Cap.IMPLICIT_CASTING_DATE_AND_DATE_NANOS.isEnabled()); + IndexResolution indexWithUnionTypedFields = indexWithDateDateNanosUnionType(); + plannerOptimizerDateDateNanosUnionTypes = new TestPlannerOptimizer(EsqlTestUtils.TEST_CFG, makeAnalyzer(indexWithUnionTypedFields)); + var stats = EsqlTestUtils.statsForExistingField("date_and_date_nanos", "date_and_date_nanos_and_long"); + + // == term + assertQueryTranslationDateDateNanosUnionTypes(""" + FROM test* | WHERE date_and_date_nanos == "2025-01-01\"""", stats, containsString(""" + "esql_single_value":{"field":"date_and_date_nanos",\ + "next":{"term":{"date_and_date_nanos":{"value":"2025-01-01T00:00:00.000Z","boost":0.0}}}""")); + + // != term + assertQueryTranslationDateDateNanosUnionTypes(""" + FROM test* | WHERE date_and_date_nanos != "2025-01-01\"""", stats, containsString(""" + "esql_single_value":{"field":"date_and_date_nanos",\ + "next":{"bool":{"must_not":[{"term":{"date_and_date_nanos":{"value":"2025-01-01T00:00:00.000Z","boost":0.0}}}],\ + "boost":0.0}}""")); + + // > range + assertQueryTranslationDateDateNanosUnionTypes(""" + FROM test* | WHERE date_and_date_nanos > "2025-01-01\"""", stats, containsString(""" + "esql_single_value":{"field":"date_and_date_nanos",\ + "next":{"range":{"date_and_date_nanos":{"gt":"2025-01-01T00:00:00.000Z","time_zone":"Z",\ + "format":"strict_date_optional_time_nanos","boost":0.0}}}""")); + + // >= range + assertQueryTranslationDateDateNanosUnionTypes(""" + FROM test* | WHERE date_and_date_nanos >= "2025-01-01\"""", stats, containsString(""" + "esql_single_value":{"field":"date_and_date_nanos",\ + "next":{"range":{"date_and_date_nanos":{"gte":"2025-01-01T00:00:00.000Z","time_zone":"Z",\ + "format":"strict_date_optional_time_nanos","boost":0.0}}}""")); + + // < range + assertQueryTranslationDateDateNanosUnionTypes( + """ + FROM test* | WHERE date_and_date_nanos < "2025-01-01" and date_and_date_nanos_and_long::date_nanos > "2025-01-01\"""", + stats, + containsString(""" + "esql_single_value":{"field":"date_and_date_nanos",\ + "next":{"range":{"date_and_date_nanos":{"lt":"2025-01-01T00:00:00.000Z","time_zone":"Z",\ + "format":"strict_date_optional_time_nanos","boost":0.0}}}""") + ); + + // <= range + assertQueryTranslationDateDateNanosUnionTypes(""" + FROM test* | WHERE date_and_date_nanos <= "2025-01-01\"""", stats, containsString(""" + "esql_single_value":{"field":"date_and_date_nanos",\ + "next":{"range":{"date_and_date_nanos":{"lte":"2025-01-01T00:00:00.000Z","time_zone":"Z",\ + "format":"strict_date_optional_time_nanos","boost":0.0}}}""")); + + // <= and >= + assertQueryTranslationDateDateNanosUnionTypes(""" + FROM test* | WHERE date_and_date_nanos <= "2025-01-01" and date_and_date_nanos > "2020-01-01\"""", stats, containsString(""" + "esql_single_value":{"field":"date_and_date_nanos",\ + "next":{"range":{"date_and_date_nanos":{"gt":"2020-01-01T00:00:00.000Z","lte":"2025-01-01T00:00:00.000Z","time_zone":"Z",\ + "format":"strict_date_optional_time_nanos","boost":0.0}}}""")); + + // >= or < + assertQueryTranslationDateDateNanosUnionTypes(""" + FROM test* | WHERE date_and_date_nanos >= "2025-01-01" or date_and_date_nanos < "2020-01-01\"""", stats, matchesRegex(""" + .*bool.*should.*""" + """ + esql_single_value":\\{"field":"date_and_date_nanos".*"range":\\{"date_and_date_nanos":\\{"gte":"2025-01-01T00:00:00.000Z",\ + "time_zone":"Z","format":"strict_date_optional_time_nanos","boost":0.0.*""" + """ + esql_single_value":\\{"field":"date_and_date_nanos".*"range":\\{"date_and_date_nanos":\\{"lt":"2020-01-01T00:00:00.000Z",\ + "time_zone":"Z","format":"strict_date_optional_time_nanos","boost":0.0.*""")); + + // > or = + assertQueryTranslationDateDateNanosUnionTypes(""" + FROM test* | WHERE date_and_date_nanos > "2025-01-01" or date_and_date_nanos == "2020-01-01\"""", stats, matchesRegex(""" + .*bool.*should.*""" + """ + esql_single_value":\\{"field":"date_and_date_nanos".*"range":\\{"date_and_date_nanos":\\{"gt":"2025-01-01T00:00:00.000Z",\ + "time_zone":"Z","format":"strict_date_optional_time_nanos","boost":0.0.*""" + """ + esql_single_value":\\{"field":"date_and_date_nanos".*"term":\\{"date_and_date_nanos":\\{"value":"2020-01-01T00:00:00.000Z",\ + "boost":0.0.*""")); + + // < or != + assertQueryTranslationDateDateNanosUnionTypes(""" + FROM test* | WHERE date_and_date_nanos < "2020-01-01" or date_and_date_nanos != "2025-01-01\"""", stats, matchesRegex(""" + .*bool.*should.*""" + """ + esql_single_value":\\{"field":"date_and_date_nanos".*"range":\\{"date_and_date_nanos":\\{"lt":"2020-01-01T00:00:00.000Z",\ + "time_zone":"Z","format":"strict_date_optional_time_nanos","boost":0.0.*""" + """ + esql_single_value":\\{"field":"date_and_date_nanos".*"must_not".*"term":\\{"date_and_date_nanos":\\{"value":\ + "2025-01-01T00:00:00.000Z","boost":0.0.*""")); + + // == or == + assertQueryTranslationDateDateNanosUnionTypes(""" + FROM test* | WHERE date_and_date_nanos == "2020-01-01" or date_and_date_nanos == "2025-01-01\"""", stats, matchesRegex(""" + .*bool.*should.*""" + """ + esql_single_value":\\{"field":"date_and_date_nanos".*"term":\\{"date_and_date_nanos":\\{"value":"2020-01-01T00:00:00.000Z",\ + "boost":0.0.*""" + """ + esql_single_value":\\{"field":"date_and_date_nanos".*"term":\\{"date_and_date_nanos":\\{"value":"2025-01-01T00:00:00.000Z",\ + "boost":0.0.*""")); + + // != or != + assertQueryTranslationDateDateNanosUnionTypes(""" + FROM test* | WHERE date_and_date_nanos != "2020-01-01" or date_and_date_nanos != "2025-01-01\"""", stats, matchesRegex(""" + .*bool.*should.*""" + """ + esql_single_value":\\{"field":"date_and_date_nanos".*"must_not".*"term":\\{"date_and_date_nanos":\\{"value":\ + "2020-01-01T00:00:00.000Z","boost":0.0.*""" + """ + esql_single_value":\\{"field":"date_and_date_nanos".*"must_not".*"term":\\{"date_and_date_nanos":\\{"value":\ + "2025-01-01T00:00:00.000Z","boost":0.0.*""")); + + // = or != + assertQueryTranslationDateDateNanosUnionTypes(""" + FROM test* | WHERE date_and_date_nanos == "2020-01-01" or date_and_date_nanos != "2025-01-01\"""", stats, matchesRegex(""" + .*bool.*should.*""" + """ + esql_single_value":\\{"field":"date_and_date_nanos".*"term":\\{"date_and_date_nanos":\\{"value":\ + "2020-01-01T00:00:00.000Z","boost":0.0.*""" + """ + esql_single_value":\\{"field":"date_and_date_nanos".*"must_not".*"term":\\{"date_and_date_nanos":\\{"value":\ + "2025-01-01T00:00:00.000Z","boost":0.0.*""")); + + // explicit casting + assertQueryTranslationDateDateNanosUnionTypes( + """ + FROM test* | WHERE date_and_date_nanos::datetime < "2025-12-31" and date_and_date_nanos > "2025-01-01\"""", + stats, + containsString(""" + "esql_single_value":{"field":"date_and_date_nanos",\ + "next":{"range":{"date_and_date_nanos":{"gt":"2025-01-01T00:00:00.000Z","time_zone":"Z",\ + "format":"strict_date_optional_time_nanos","boost":0.0}}}""") + ); + } }