From 42ce293ca5b02e0539c1ea619f7fbceba2723995 Mon Sep 17 00:00:00 2001 From: Fang Xing Date: Tue, 25 Feb 2025 20:17:25 -0500 Subject: [PATCH 01/24] implicit casting for date nanos --- .../src/main/resources/date_nanos.csv-spec | 11 + .../xpack/esql/analysis/Analyzer.java | 254 +++++++++++++++++- .../xpack/esql/analysis/Verifier.java | 12 + .../esql/type/EsqlDataTypeConverter.java | 3 + 4 files changed, 271 insertions(+), 9 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec index a41c9b8cd0b16..4268179469203 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec @@ -1296,3 +1296,14 @@ diff_sec:integer | diff_sec_m:integer | n:date_nanos -18489600 | -18489599 | 2023-03-23T12:15:03.360103847Z -18489600 | -18489599 | 2023-03-23T12:15:03.360103847Z ; + +DateAndDateNanos +from employees,employees_incompatible +| keep emp_no, birth_date +| sort emp_no +| limit 1 +; + +emp_no:integer |birth_date:date +10001 |1953-09-02T00:00:00Z +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 74242cf9be3f9..0b243d2e32ac4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -45,6 +45,7 @@ import org.elasticsearch.xpack.esql.core.util.Holder; import org.elasticsearch.xpack.esql.core.util.StringUtils; import org.elasticsearch.xpack.esql.expression.NamedExpressions; +import org.elasticsearch.xpack.esql.expression.Order; import org.elasticsearch.xpack.esql.expression.UnresolvedNamePattern; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import org.elasticsearch.xpack.esql.expression.function.FunctionDefinition; @@ -54,6 +55,8 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.FoldablesConvertFunction; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDateNanos; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDatetime; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDouble; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToInteger; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToLong; @@ -77,6 +80,7 @@ import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.esql.plan.logical.Lookup; import org.elasticsearch.xpack.esql.plan.logical.MvExpand; +import org.elasticsearch.xpack.esql.plan.logical.OrderBy; import org.elasticsearch.xpack.esql.plan.logical.Project; import org.elasticsearch.xpack.esql.plan.logical.Rename; import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation; @@ -119,6 +123,7 @@ import static java.util.Collections.singletonList; import static org.elasticsearch.common.logging.LoggerMessageFormat.format; import static org.elasticsearch.xpack.core.enrich.EnrichPolicy.GEO_MATCH_TYPE; +import static org.elasticsearch.xpack.esql.core.expression.Attribute.rawTemporaryName; import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN; import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS; @@ -131,9 +136,13 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.IP; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; +import static org.elasticsearch.xpack.esql.core.type.DataType.NULL; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.TIME_DURATION; +import static org.elasticsearch.xpack.esql.core.type.DataType.UNSUPPORTED; import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION; +import static org.elasticsearch.xpack.esql.core.type.DataType.isMillisOrNanos; +import static org.elasticsearch.xpack.esql.core.type.DataType.isRepresentable; import static org.elasticsearch.xpack.esql.core.type.DataType.isTemporalAmount; import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.LIMIT; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.maybeParseTemporalAmount; @@ -145,7 +154,7 @@ public class Analyzer extends ParameterizedRuleExecutor NO_FIELDS = List.of( - new ReferenceAttribute(Source.EMPTY, "", DataType.NULL, Nullability.TRUE, null, true) + new ReferenceAttribute(Source.EMPTY, "", NULL, Nullability.TRUE, null, true) ); private static final Iterable> rules; @@ -167,9 +176,14 @@ public class Analyzer extends ParameterizedRuleExecutor("Finish Analysis", Limiter.ONCE, new AddImplicitLimit(), new UnionTypesCleanup()); + new ResolveUnionTypes(), // Must be after ResolveRefs, so union types can be found + new AddImplicitProject() + ); + var finish = new Batch<>( + "Finish Analysis", + Limiter.ONCE, + new AddImplicitLimit(), + new UnionTypesCleanup()); rules = List.of(init, resolution, finish); } @@ -594,7 +608,7 @@ private LogicalPlan resolveLookup(Lookup l, List childrenOutput) { */ boolean dataTypesOk = joinedAttribute.dataType().equals(attr.dataType()); if (false == dataTypesOk) { - dataTypesOk = joinedAttribute.dataType() == DataType.NULL || attr.dataType() == DataType.NULL; + dataTypesOk = joinedAttribute.dataType() == NULL || attr.dataType() == NULL; } if (false == dataTypesOk) { dataTypesOk = joinedAttribute.dataType().equals(KEYWORD) && attr.dataType().equals(TEXT); @@ -1125,13 +1139,129 @@ private BitSet gatherPreAnalysisMetrics(LogicalPlan plan, BitSet b) { private static class ImplicitCasting extends ParameterizedRule { @Override public LogicalPlan apply(LogicalPlan plan, AnalyzerContext context) { - return plan.transformExpressionsUp( + // do implicit casting for union typed fields in sort keys + LogicalPlan newPlan = plan.transformUp(p -> { + if (p instanceof OrderBy ob) { + return castInvalidMappedFieldInOrderBy(ob); + } + if (p instanceof EsqlProject proj) { + return castInvalidMappedFieldInProjection(proj); + } + return p; + } + ); + // do implicit casting for function arguments + return newPlan.transformExpressionsUp( org.elasticsearch.xpack.esql.core.expression.function.Function.class, e -> ImplicitCasting.cast(e, context.functionRegistry()) ); } + private static EsqlProject castInvalidMappedFieldInProjection(EsqlProject esqlProject) { + List newProjections = new ArrayList<>(esqlProject.projections().size()); + List aliases = new ArrayList<>(esqlProject.projections().size()); + int counter = 0; + projectionLoop: for (NamedExpression projection : esqlProject.projections()) { + Expression e = projection instanceof Alias a ? a.child() : projection; + if (e.resolved() && e.dataType() == UNSUPPORTED + && e instanceof FieldAttribute fa && fa.field() instanceof InvalidMappedField imf) { + // this is an invalid mapped field, find a common data type and cast to it + DataType targetType = null; + for (DataType type : imf.types()) { + if (targetType == null) { // initialize the target type to the first type + targetType = type; + } else { + targetType = EsqlDataTypeConverter.commonType(targetType, type); + if (targetType == null) { // if there is no common type, continue to the next argument + newProjections.add(projection); + continue projectionLoop; + } + } + } + if (targetType != null && isRepresentable(targetType) && (isMillisOrNanos(targetType) || targetType.isNumeric())) { + // create an eval to cast union type to a common type + var name = rawTemporaryName("keep", String.valueOf(counter++), targetType.name()); + Alias alias; + switch (targetType) { + case INTEGER -> alias = new Alias(fa.source(), name, new ToInteger(fa.source(), fa)); + case LONG -> alias = new Alias(fa.source(), name, new ToLong(fa.source(), fa)); + case DOUBLE -> alias = new Alias(fa.source(), name, new ToDouble(fa.source(), fa)); + case UNSIGNED_LONG -> alias = new Alias(fa.source(), name, new ToUnsignedLong(fa.source(), fa)); + case DATETIME -> alias = new Alias(fa.source(), name, new ToDatetime(fa.source(), fa)); + case DATE_NANOS -> alias = new Alias(fa.source(), name, new ToDateNanos(fa.source(), fa)); + default -> throw new EsqlIllegalArgumentException("unexpected data type: " + targetType); + } + aliases.add(alias); + newProjections.add(alias); + } + } else { + newProjections.add(projection); + } + } + if (aliases.isEmpty() == false) { + Eval eval = new Eval(esqlProject.source(), esqlProject.child(), aliases); + return new EsqlProject(esqlProject.source(), eval, newProjections); + } else { + return esqlProject; + } + } + + private static OrderBy castInvalidMappedFieldInOrderBy(OrderBy ob) { + List newOrder = new ArrayList<>(ob.order().size()); + boolean changed = false; + orderLoop: for (Order o : ob.order()) { + DataType targetType = null; + if (o.child().resolved() && o.child().dataType() == UNSUPPORTED + && o.child() instanceof FieldAttribute fa + && fa.field() instanceof InvalidMappedField imf) { + // this is an invalid mapped field, find a common data type and cast to it + for (DataType type : imf.types()) { + if (targetType == null) { // initialize the target type to the first type + targetType = type; + } else { + targetType = EsqlDataTypeConverter.commonType(targetType, type); + if (targetType == null) { // if there is no common type, continue to the next argument + newOrder.add(o); + continue orderLoop; + } + } + } + if (targetType != null && isRepresentable(targetType) && (isMillisOrNanos(targetType) || targetType.isNumeric())) { + changed = true; + switch (targetType) { + case INTEGER -> newOrder.add( + new Order(o.source(), new ToInteger(fa.source(), fa), o.direction(), o.nullsPosition()) + ); + case LONG -> newOrder.add( + new Order(o.source(), new ToLong(fa.source(), fa), o.direction(), o.nullsPosition()) + ); + case DOUBLE -> newOrder.add( + new Order(o.source(), new ToDouble(fa.source(), fa), o.direction(), o.nullsPosition()) + ); + case UNSIGNED_LONG -> newOrder.add( + new Order(o.source(), new ToUnsignedLong(fa.source(), fa), o.direction(), o.nullsPosition()) + ); + case DATETIME -> newOrder.add( + new Order(o.source(), new ToDatetime(fa.source(), fa), o.direction(), o.nullsPosition()) + ); + case DATE_NANOS -> newOrder.add( + new Order(o.source(), new ToDateNanos(fa.source(), fa), o.direction(), o.nullsPosition()) + ); + default -> throw new EsqlIllegalArgumentException("unexpected data type: " + targetType); + } + } + } else { + newOrder.add(o); + } + } + + return changed ? new OrderBy(ob.source(), ob.child(), newOrder) : ob; + } + private static Expression cast(org.elasticsearch.xpack.esql.core.expression.function.Function f, EsqlFunctionRegistry registry) { + // Add cast functions to InvalidMappedField if there isn't one yet + f = castInvalidMappedField(f); + if (f instanceof In in) { return processIn(in); } @@ -1144,6 +1274,54 @@ private static Expression cast(org.elasticsearch.xpack.esql.core.expression.func return f; } + private static org.elasticsearch.xpack.esql.core.expression.function.Function castInvalidMappedField( + org.elasticsearch.xpack.esql.core.expression.function.Function f + ) { + if (f instanceof AbstractConvertFunction) { + return f; + } + List args = f.arguments(); + List newChildren = new ArrayList<>(f.children().size()); + boolean childrenChanged = false; + Expression arg; + argLoop: for (Expression expression : args) { + DataType targetType = null; + arg = expression; + if (arg.resolved() + && arg.dataType() == UNSUPPORTED + && arg instanceof FieldAttribute fa + && fa.field() instanceof InvalidMappedField imf) { + // this is an invalid mapped field, find a common data type and cast to it + for (DataType type : imf.types()) { + if (targetType == null) { // initialize the target type to the first type + targetType = type; + } else { + targetType = EsqlDataTypeConverter.commonType(targetType, type); + if (targetType == null) { // if there is no common type, continue to the next argument + newChildren.add(arg); + continue argLoop; + } + } + } + if (targetType != null && isRepresentable(targetType) && (isMillisOrNanos(targetType) || targetType.isNumeric())) { + childrenChanged = true; + switch (targetType) { + case INTEGER -> newChildren.add(new ToInteger(arg.source(), arg)); + case LONG -> newChildren.add(new ToLong(arg.source(), arg)); + case DOUBLE -> newChildren.add(new ToDouble(arg.source(), arg)); + case UNSIGNED_LONG -> newChildren.add(new ToUnsignedLong(arg.source(), arg)); + case DATETIME -> newChildren.add(new ToDatetime(arg.source(), arg)); + case DATE_NANOS -> newChildren.add(new ToDateNanos(arg.source(), arg)); + default -> throw new EsqlIllegalArgumentException("unexpected data type: " + targetType); + } + } + } else { + newChildren.add(arg); + } + } + return childrenChanged ? (org.elasticsearch.xpack.esql.core.expression.function.Function) f.replaceChildren(newChildren) : f; + } + private static Expression processScalarOrGroupingFunction( org.elasticsearch.xpack.esql.core.expression.function.Function f, EsqlFunctionRegistry registry @@ -1155,7 +1333,7 @@ private static Expression processScalarOrGroupingFunction( } List newChildren = new ArrayList<>(args.size()); boolean childrenChanged = false; - DataType targetDataType = DataType.NULL; + DataType targetDataType = NULL; Expression arg; DataType targetNumericType = null; boolean castNumericArgs = true; @@ -1168,7 +1346,7 @@ private static Expression processScalarOrGroupingFunction( if (i < targetDataTypes.size()) { targetDataType = targetDataTypes.get(i); } - if (targetDataType != DataType.NULL && targetDataType != DataType.UNSUPPORTED) { + if (targetDataType != NULL && targetDataType != DataType.UNSUPPORTED) { Expression e = castStringLiteral(arg, targetDataType); if (e != arg) { childrenChanged = true; @@ -1201,7 +1379,7 @@ private static Expression processBinaryOperator(BinaryOperator o) { } List newChildren = new ArrayList<>(2); boolean childrenChanged = false; - DataType targetDataType = DataType.NULL; + DataType targetDataType = NULL; Expression from = Literal.NULL; if (left.dataType() == KEYWORD && left.foldable() && (left instanceof EsqlScalarFunction == false)) { @@ -1545,4 +1723,62 @@ private static LogicalPlan planWithoutSyntheticAttributes(LogicalPlan plan) { return newOutput.size() == output.size() ? plan : new Project(Source.EMPTY, plan, newOutput); } } + + private static class AddImplicitProject extends ParameterizedRule { + @Override + public LogicalPlan apply(LogicalPlan logicalPlan, AnalyzerContext context) { + if (logicalPlan.resolved() == false) { + return logicalPlan; + } + List projections = logicalPlan.collectFirstChildren(EsqlProject.class::isInstance); + if (projections.isEmpty()) { + List newProjections = new ArrayList<>(logicalPlan.output().size()); + List aliases = new ArrayList<>(logicalPlan.output().size()); + int counter = 0; + projectionLoop: for (Attribute e : logicalPlan.output()) { + if (e.resolved() && e.dataType() == UNSUPPORTED + && e instanceof FieldAttribute fa && fa.field() instanceof InvalidMappedField imf) { + // this is an invalid mapped field, find a common data type and cast to it + DataType targetType = null; + for (DataType type : imf.types()) { + if (targetType == null) { // initialize the target type to the first type + targetType = type; + } else { + targetType = EsqlDataTypeConverter.commonType(targetType, type); + if (targetType == null) { // if there is no common type, continue to the next argument + newProjections.add(e); + continue projectionLoop; + } + } + } + if (targetType != null && isRepresentable(targetType) && (isMillisOrNanos(targetType) || targetType.isNumeric())) { + // create an eval to cast union type to a common type + var name = rawTemporaryName("keep", String.valueOf(counter++), targetType.name()); + Alias alias; + switch (targetType) { + case INTEGER -> alias = new Alias(fa.source(), name, new ToInteger(fa.source(), fa)); + case LONG -> alias = new Alias(fa.source(), name, new ToLong(fa.source(), fa)); + case DOUBLE -> alias = new Alias(fa.source(), name, new ToDouble(fa.source(), fa)); + case UNSIGNED_LONG -> alias = new Alias(fa.source(), name, new ToUnsignedLong(fa.source(), fa)); + case DATETIME -> alias = new Alias(fa.source(), name, new ToDatetime(fa.source(), fa)); + case DATE_NANOS -> alias = new Alias(fa.source(), name, new ToDateNanos(fa.source(), fa)); + default -> throw new EsqlIllegalArgumentException("unexpected data type: " + targetType); + } + aliases.add(alias); + newProjections.add(alias); + } + } else { + newProjections.add(e); + } + } + if (aliases.isEmpty() == false) { + Eval eval = new Eval(logicalPlan.source(), logicalPlan, aliases); + return new EsqlProject(logicalPlan.source(), eval, newProjections); + } else { + return logicalPlan; + } + } + return logicalPlan; + } + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index 87e555e8d2f7f..db0b115f10607 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -17,6 +17,7 @@ import org.elasticsearch.xpack.esql.core.capabilities.Unresolvable; import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; import org.elasticsearch.xpack.esql.core.expression.function.Function; import org.elasticsearch.xpack.esql.core.expression.predicate.BinaryOperator; @@ -34,6 +35,7 @@ import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.esql.plan.logical.Lookup; import org.elasticsearch.xpack.esql.plan.logical.Project; +import org.elasticsearch.xpack.esql.plan.logical.local.EsqlProject; import org.elasticsearch.xpack.esql.telemetry.FeatureMetric; import org.elasticsearch.xpack.esql.telemetry.Metrics; @@ -93,6 +95,16 @@ Collection verify(LogicalPlan plan, BitSet partialMetrics) { return; } + if (p instanceof EsqlProject proj) { + for (NamedExpression projection : proj.projections()) { + // don't call dataType() - it will fail on UnresolvedAttribute + String className = String.valueOf(projection.getClass()); + if (projection.resolved() == false && projection instanceof UnsupportedAttribute == false) { + + } + } + } + planCheckers.forEach(c -> c.accept(p, failures)); checkOperationsOnUnsignedLong(p, failures); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java index 4259de7347abd..7f2a729879f7e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java @@ -396,6 +396,9 @@ public static DataType commonType(DataType left, DataType right) { if (isNullOrDatePeriod(left) && isNullOrDatePeriod(right)) { return DATE_PERIOD; } + if ((isDateTime(left) && right == DATE_NANOS) || (left == DATE_NANOS && isDateTime(right))) { + return DATE_NANOS; + } } if (isString(left) && isString(right)) { // Both TEXT and SEMANTIC_TEXT are processed as KEYWORD From 5436afbe98970abec8ec9441d714cdb94a86742c Mon Sep 17 00:00:00 2001 From: Fang Xing Date: Thu, 27 Feb 2025 23:29:06 -0500 Subject: [PATCH 02/24] implicit casting for union typed field in logical plan --- .../esql/qa/rest/FieldExtractorTestCase.java | 6 + .../resources/data/employees_incompatible.csv | 202 +++++++------- .../src/main/resources/date_nanos.csv-spec | 11 - .../src/main/resources/union_types.csv-spec | 64 +++++ .../xpack/esql/action/EsqlCapabilities.java | 7 +- .../xpack/esql/analysis/Analyzer.java | 261 +++++++----------- .../xpack/esql/analysis/Verifier.java | 12 - .../esql/type/EsqlDataTypeConverterTests.java | 2 +- 8 files changed, 283 insertions(+), 282 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/FieldExtractorTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/FieldExtractorTestCase.java index 11aa9a43fa5af..d5b092a6f292f 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/FieldExtractorTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/FieldExtractorTestCase.java @@ -937,6 +937,9 @@ public void testLongIntegerConflict() throws IOException { "order of fields in error message inconsistent before 8.14", getCachedNodesVersions().stream().allMatch(v -> Version.fromString(v).onOrAfter(Version.V_8_14_0)) ); + if (EsqlCapabilities.Cap.IMPLICIT_CASTING_UNION_TYPED_NUMERIC_AND_DATE.isEnabled()) { + return; + } longTest().sourceMode(SourceMode.DEFAULT).createIndex("test1", "emp_no"); index("test1", """ {"emp_no": 1}"""); @@ -979,6 +982,9 @@ public void testIntegerShortConflict() throws IOException { "order of fields in error message inconsistent before 8.14", getCachedNodesVersions().stream().allMatch(v -> Version.fromString(v).onOrAfter(Version.V_8_14_0)) ); + if (EsqlCapabilities.Cap.IMPLICIT_CASTING_UNION_TYPED_NUMERIC_AND_DATE.isEnabled()) { + return; + } intTest().sourceMode(SourceMode.DEFAULT).createIndex("test1", "emp_no"); index("test1", """ {"emp_no": 1}"""); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/employees_incompatible.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/employees_incompatible.csv index ddbdb89476c4c..cf810f27b245d 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/employees_incompatible.csv +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/employees_incompatible.csv @@ -1,101 +1,101 @@ -birth_date:date_nanos ,emp_no:long,first_name:text,gender:text,hire_date:date_nanos,languages:byte,languages.long:long,languages.short:short,languages.byte:byte,last_name:text,salary:long,height:float,height.double:double,height.scaled_float:scaled_float,height.half_float:half_float,still_hired:keyword,avg_worked_seconds:unsigned_long,job_positions:text,is_rehired:keyword,salary_change:float,salary_change.int:integer,salary_change.long:long,salary_change.keyword:keyword -1953-09-02T00:00:00Z,10001,Georgi ,M,1986-06-26T00:00:00Z,2,2,2,2,Facello ,57305,2.03,2.03,2.03,2.03,true ,268728049,[Senior Python Developer,Accountant],[false,true],[1.19],[1],[1],[1.19] -1964-06-02T00:00:00Z,10002,Bezalel ,F,1985-11-21T00:00:00Z,5,5,5,5,Simmel ,56371,2.08,2.08,2.08,2.08,true ,328922887,[Senior Team Lead],[false,false],[-7.23,11.17],[-7,11],[-7,11],[-7.23,11.17] -1959-12-03T00:00:00Z,10003,Parto ,M,1986-08-28T00:00:00Z,4,4,4,4,Bamford ,61805,1.83,1.83,1.83,1.83,false,200296405,[],[],[14.68,12.82],[14,12],[14,12],[14.68,12.82] -1954-05-01T00:00:00Z,10004,Chirstian ,M,1986-12-01T00:00:00Z,5,5,5,5,Koblick ,36174,1.78,1.78,1.78,1.78,true ,311267831,[Reporting Analyst,Tech Lead,Head Human Resources,Support Engineer],[true],[3.65,-0.35,1.13,13.48],[3,0,1,13],[3,0,1,13],[3.65,-0.35,1.13,13.48] -1955-01-21T00:00:00Z,10005,Kyoichi ,M,1989-09-12T00:00:00Z,1,1,1,1,Maliniak ,63528,2.05,2.05,2.05,2.05,true ,244294991,[],[false,false,false,true],[-2.14,13.07],[-2,13],[-2,13],[-2.14,13.07] -1953-04-20T00:00:00Z,10006,Anneke ,F,1989-06-02T00:00:00Z,3,3,3,3,Preusig ,60335,1.56,1.56,1.56,1.56,false,372957040,[Tech Lead,Principal Support Engineer,Senior Team Lead],[],[-3.90],[-3],[-3],[-3.90] -1957-05-23T00:00:00Z,10007,Tzvetan ,F,1989-02-10T00:00:00Z,4,4,4,4,Zielinski ,74572,1.70,1.70,1.70,1.70,true ,393084805,[],[true,false,true,false],[-7.06,1.99,0.57],[-7,1,0],[-7,1,0],[-7.06,1.99,0.57] -1958-02-19T00:00:00Z,10008,Saniya ,M,1994-09-15T00:00:00Z,2,2,2,2,Kalloufi ,43906,2.10,2.10,2.10,2.10,true ,283074758,[Senior Python Developer,Junior Developer,Purchase Manager,Internship],[true,false],[12.68,3.54,0.75,-2.92],[12,3,0,-2],[12,3,0,-2],[12.68,3.54,0.75,-2.92] -1952-04-19T00:00:00Z,10009,Sumant ,F,1985-02-18T00:00:00Z,1,1,1,1,Peac ,66174,1.85,1.85,1.85,1.85,false,236805489,[Senior Python Developer,Internship],[],[],[],[],[] -1963-06-01T00:00:00Z,10010,Duangkaew , ,1989-08-24T00:00:00Z,4,4,4,4,Piveteau ,45797,1.70,1.70,1.70,1.70,false,315236372,[Architect,Reporting Analyst,Tech Lead,Purchase Manager],[true,true,false,false],[5.05,-6.77,4.69,12.15],[5,-6,4,12],[5,-6,4,12],[5.05,-6.77,4.69,12.15] -1953-11-07T00:00:00Z,10011,Mary , ,1990-01-22T00:00:00Z,5,5,5,5,Sluis ,31120,1.50,1.50,1.50,1.50,true ,239615525,[Architect,Reporting Analyst,Tech Lead,Senior Team Lead],[true,true],[10.35,-7.82,8.73,3.48],[10,-7,8,3],[10,-7,8,3],[10.35,-7.82,8.73,3.48] -1960-10-04T00:00:00Z,10012,Patricio , ,1992-12-18T00:00:00Z,5,5,5,5,Bridgland ,48942,1.97,1.97,1.97,1.97,false,365510850,[Head Human Resources,Accountant],[false,true,true,false],[0.04],[0],[0],[0.04] -1963-06-07T00:00:00Z,10013,Eberhardt , ,1985-10-20T00:00:00Z,1,1,1,1,Terkki ,48735,1.94,1.94,1.94,1.94,true ,253864340,[Reporting Analyst],[true,true],[],[],[],[] -1956-02-12T00:00:00Z,10014,Berni , ,1987-03-11T00:00:00Z,5,5,5,5,Genin ,37137,1.99,1.99,1.99,1.99,false,225049139,[Reporting Analyst,Data Scientist,Head Human Resources],[],[-1.89,9.07],[-1,9],[-1,9],[-1.89,9.07] -1959-08-19T00:00:00Z,10015,Guoxiang , ,1987-07-02T00:00:00Z,5,5,5,5,Nooteboom ,25324,1.66,1.66,1.66,1.66,true ,390266432,[Principal Support Engineer,Junior Developer,Head Human Resources,Support Engineer],[true,false,false,false],[14.25,12.40],[14,12],[14,12],[14.25,12.40] -1961-05-02T00:00:00Z,10016,Kazuhito , ,1995-01-27T00:00:00Z,2,2,2,2,Cappelletti ,61358,1.54,1.54,1.54,1.54,false,253029411,[Reporting Analyst,Python Developer,Accountant,Purchase Manager],[false,false],[-5.18,7.69],[-5,7],[-5,7],[-5.18,7.69] -1958-07-06T00:00:00Z,10017,Cristinel , ,1993-08-03T00:00:00Z,2,2,2,2,Bouloucos ,58715,1.74,1.74,1.74,1.74,false,236703986,[Data Scientist,Head Human Resources,Purchase Manager],[true,false,true,true],[-6.33],[-6],[-6],[-6.33] -1954-06-19T00:00:00Z,10018,Kazuhide , ,1987-04-03T00:00:00Z,2,2,2,2,Peha ,56760,1.97,1.97,1.97,1.97,false,309604079,[Junior Developer],[false,false,true,true],[-1.64,11.51,-5.32],[-1,11,-5],[-1,11,-5],[-1.64,11.51,-5.32] -1953-01-23T00:00:00Z,10019,Lillian , ,1999-04-30T00:00:00Z,1,1,1,1,Haddadi ,73717,2.06,2.06,2.06,2.06,false,342855721,[Purchase Manager],[false,false],[-6.84,8.42,-7.26],[-6,8,-7],[-6,8,-7],[-6.84,8.42,-7.26] -1952-12-24T00:00:00Z,10020,Mayuko ,M,1991-01-26T00:00:00Z, , , , ,Warwick ,40031,1.41,1.41,1.41,1.41,false,373309605,[Tech Lead],[true,true,false],[-5.81],[-5],[-5],[-5.81] -1960-02-20T00:00:00Z,10021,Ramzi ,M,1988-02-10T00:00:00Z, , , , ,Erde ,60408,1.47,1.47,1.47,1.47,false,287654610,[Support Engineer],[true],[],[],[],[] -1952-07-08T00:00:00Z,10022,Shahaf ,M,1995-08-22T00:00:00Z, , , , ,Famili ,48233,1.82,1.82,1.82,1.82,false,233521306,[Reporting Analyst,Data Scientist,Python Developer,Internship],[true,false],[12.09,2.85],[12,2],[12,2],[12.09,2.85] -1953-09-29T00:00:00Z,10023,Bojan ,F,1989-12-17T00:00:00Z, , , , ,Montemayor ,47896,1.75,1.75,1.75,1.75,true ,330870342,[Accountant,Support Engineer,Purchase Manager],[true,true,false],[14.63,0.80],[14,0],[14,0],[14.63,0.80] -1958-09-05T00:00:00Z,10024,Suzette ,F,1997-05-19T00:00:00Z, , , , ,Pettey ,64675,2.08,2.08,2.08,2.08,true ,367717671,[Junior Developer],[true,true,true,true],[],[],[],[] -1958-10-31T00:00:00Z,10025,Prasadram ,M,1987-08-17T00:00:00Z, , , , ,Heyers ,47411,1.87,1.87,1.87,1.87,false,371270797,[Accountant],[true,false],[-4.33,-2.90,12.06,-3.46],[-4,-2,12,-3],[-4,-2,12,-3],[-4.33,-2.90,12.06,-3.46] -1953-04-03T00:00:00Z,10026,Yongqiao ,M,1995-03-20T00:00:00Z, , , , ,Berztiss ,28336,2.10,2.10,2.10,2.10,true ,359208133,[Reporting Analyst],[false,true],[-7.37,10.62,11.20],[-7,10,11],[-7,10,11],[-7.37,10.62,11.20] -1962-07-10T00:00:00Z,10027,Divier ,F,1989-07-07T00:00:00Z, , , , ,Reistad ,73851,1.53,1.53,1.53,1.53,false,374037782,[Senior Python Developer],[false],[],[],[],[] -1963-11-26T00:00:00Z,10028,Domenick ,M,1991-10-22T00:00:00Z, , , , ,Tempesti ,39356,2.07,2.07,2.07,2.07,true ,226435054,[Tech Lead,Python Developer,Accountant,Internship],[true,false,false,true],[],[],[],[] -1956-12-13T00:00:00Z,10029,Otmar ,M,1985-11-20T00:00:00Z, , , , ,Herbst ,74999,1.99,1.99,1.99,1.99,false,257694181,[Senior Python Developer,Data Scientist,Principal Support Engineer],[true],[-0.32,-1.90,-8.19],[0,-1,-8],[0,-1,-8],[-0.32,-1.90,-8.19] -1958-07-14T00:00:00Z,10030, ,M,1994-02-17T00:00:00Z,3,3,3,3,Demeyer ,67492,1.92,1.92,1.92,1.92,false,394597613,[Tech Lead,Data Scientist,Senior Team Lead],[true,false,false],[-0.40],[0],[0],[-0.40] -1959-01-27T00:00:00Z,10031, ,M,1991-09-01T00:00:00Z,4,4,4,4,Joslin ,37716,1.68,1.68,1.68,1.68,false,348545109,[Architect,Senior Python Developer,Purchase Manager,Senior Team Lead],[false],[],[],[],[] -1960-08-09T00:00:00Z,10032, ,F,1990-06-20T00:00:00Z,3,3,3,3,Reistad ,62233,2.10,2.10,2.10,2.10,false,277622619,[Architect,Senior Python Developer,Junior Developer,Purchase Manager],[false,false],[9.32,-4.92],[9,-4],[9,-4],[9.32,-4.92] -1956-11-14T00:00:00Z,10033, ,M,1987-03-18T00:00:00Z,1,1,1,1,Merlo ,70011,1.63,1.63,1.63,1.63,false,208374744,[],[true],[],[],[],[] -1962-12-29T00:00:00Z,10034, ,M,1988-09-21T00:00:00Z,1,1,1,1,Swan ,39878,1.46,1.46,1.46,1.46,false,214393176,[Business Analyst,Data Scientist,Python Developer,Accountant],[false],[-8.46],[-8],[-8],[-8.46] -1953-02-08T00:00:00Z,10035, ,M,1988-09-05T00:00:00Z,5,5,5,5,Chappelet ,25945,1.81,1.81,1.81,1.81,false,203838153,[Senior Python Developer,Data Scientist],[false],[-2.54,-6.58],[-2,-6],[-2,-6],[-2.54,-6.58] -1959-08-10T00:00:00Z,10036, ,M,1992-01-03T00:00:00Z,4,4,4,4,Portugali ,60781,1.61,1.61,1.61,1.61,false,305493131,[Senior Python Developer],[true,false,false],[],[],[],[] -1963-07-22T00:00:00Z,10037, ,M,1990-12-05T00:00:00Z,2,2,2,2,Makrucki ,37691,2.00,2.00,2.00,2.00,true ,359217000,[Senior Python Developer,Tech Lead,Accountant],[false],[-7.08],[-7],[-7],[-7.08] -1960-07-20T00:00:00Z,10038, ,M,1989-09-20T00:00:00Z,4,4,4,4,Lortz ,35222,1.53,1.53,1.53,1.53,true ,314036411,[Senior Python Developer,Python Developer,Support Engineer],[],[],[],[],[] -1959-10-01T00:00:00Z,10039, ,M,1988-01-19T00:00:00Z,2,2,2,2,Brender ,36051,1.55,1.55,1.55,1.55,false,243221262,[Business Analyst,Python Developer,Principal Support Engineer],[true,true],[-6.90],[-6],[-6],[-6.90] - ,10040,Weiyi ,F,1993-02-14T00:00:00Z,4,4,4,4,Meriste ,37112,1.90,1.90,1.90,1.90,false,244478622,[Principal Support Engineer],[true,false,true,true],[6.97,14.74,-8.94,1.92],[6,14,-8,1],[6,14,-8,1],[6.97,14.74,-8.94,1.92] - ,10041,Uri ,F,1989-11-12T00:00:00Z,1,1,1,1,Lenart ,56415,1.75,1.75,1.75,1.75,false,287789442,[Data Scientist,Head Human Resources,Internship,Senior Team Lead],[],[9.21,0.05,7.29,-2.94],[9,0,7,-2],[9,0,7,-2],[9.21,0.05,7.29,-2.94] - ,10042,Magy ,F,1993-03-21T00:00:00Z,3,3,3,3,Stamatiou ,30404,1.44,1.44,1.44,1.44,true ,246355863,[Architect,Business Analyst,Junior Developer,Internship],[],[-9.28,9.42],[-9,9],[-9,9],[-9.28,9.42] - ,10043,Yishay ,M,1990-10-20T00:00:00Z,1,1,1,1,Tzvieli ,34341,1.52,1.52,1.52,1.52,true ,287222180,[Data Scientist,Python Developer,Support Engineer],[false,true,true],[-5.17,4.62,7.42],[-5,4,7],[-5,4,7],[-5.17,4.62,7.42] - ,10044,Mingsen ,F,1994-05-21T00:00:00Z,1,1,1,1,Casley ,39728,2.06,2.06,2.06,2.06,false,387408356,[Tech Lead,Principal Support Engineer,Accountant,Support Engineer],[true,true],[8.09],[8],[8],[8.09] - ,10045,Moss ,M,1989-09-02T00:00:00Z,3,3,3,3,Shanbhogue ,74970,1.70,1.70,1.70,1.70,false,371418933,[Principal Support Engineer,Junior Developer,Accountant,Purchase Manager],[true,false],[],[],[],[] - ,10046,Lucien ,M,1992-06-20T00:00:00Z,4,4,4,4,Rosenbaum ,50064,1.52,1.52,1.52,1.52,true ,302353405,[Principal Support Engineer,Junior Developer,Head Human Resources,Internship],[true,true,false,true],[2.39],[2],[2],[2.39] - ,10047,Zvonko ,M,1989-03-31T00:00:00Z,4,4,4,4,Nyanchama ,42716,1.52,1.52,1.52,1.52,true ,306369346,[Architect,Data Scientist,Principal Support Engineer,Senior Team Lead],[true],[-6.36,12.12],[-6,12],[-6,12],[-6.36,12.12] - ,10048,Florian ,M,1985-02-24T00:00:00Z,3,3,3,3,Syrotiuk ,26436,2.00,2.00,2.00,2.00,false,248451647,[Internship],[true,true],[],[],[],[] - ,10049,Basil ,F,1992-05-04T00:00:00Z,5,5,5,5,Tramer ,37853,1.52,1.52,1.52,1.52,true ,320725709,[Senior Python Developer,Business Analyst],[],[-1.05],[-1],[-1],[-1.05] -1958-05-21T00:00:00Z,10050,Yinghua ,M,1990-12-25T00:00:00Z,2,2,2,2,Dredge ,43026,1.96,1.96,1.96,1.96,true ,242731798,[Reporting Analyst,Junior Developer,Accountant,Support Engineer],[true],[8.70,10.94],[8,10],[8,10],[8.70,10.94] -1953-07-28T00:00:00Z,10051,Hidefumi ,M,1992-10-15T00:00:00Z,3,3,3,3,Caine ,58121,1.89,1.89,1.89,1.89,true ,374753122,[Business Analyst,Accountant,Purchase Manager],[],[],[],[],[] -1961-02-26T00:00:00Z,10052,Heping ,M,1988-05-21T00:00:00Z,1,1,1,1,Nitsch ,55360,1.79,1.79,1.79,1.79,true ,299654717,[],[true,true,false],[-0.55,-1.89,-4.22,-6.03],[0,-1,-4,-6],[0,-1,-4,-6],[-0.55,-1.89,-4.22,-6.03] -1954-09-13T00:00:00Z,10053,Sanjiv ,F,1986-02-04T00:00:00Z,3,3,3,3,Zschoche ,54462,1.58,1.58,1.58,1.58,false,368103911,[Support Engineer],[true,false,true,false],[-7.67,-3.25],[-7,-3],[-7,-3],[-7.67,-3.25] -1957-04-04T00:00:00Z,10054,Mayumi ,M,1995-03-13T00:00:00Z,4,4,4,4,Schueller ,65367,1.82,1.82,1.82,1.82,false,297441693,[Principal Support Engineer],[false,false],[],[],[],[] -1956-06-06T00:00:00Z,10055,Georgy ,M,1992-04-27T00:00:00Z,5,5,5,5,Dredge ,49281,2.04,2.04,2.04,2.04,false,283157844,[Senior Python Developer,Head Human Resources,Internship,Support Engineer],[false,false,true],[7.34,12.99,3.17],[7,12,3],[7,12,3],[7.34,12.99,3.17] -1961-09-01T00:00:00Z,10056,Brendon ,F,1990-02-01T00:00:00Z,2,2,2,2,Bernini ,33370,1.57,1.57,1.57,1.57,true ,349086555,[Senior Team Lead],[true,false,false],[10.99,-5.17],[10,-5],[10,-5],[10.99,-5.17] -1954-05-30T00:00:00Z,10057,Ebbe ,F,1992-01-15T00:00:00Z,4,4,4,4,Callaway ,27215,1.59,1.59,1.59,1.59,true ,324356269,[Python Developer,Head Human Resources],[],[-6.73,-2.43,-5.27,1.03],[-6,-2,-5,1],[-6,-2,-5,1],[-6.73,-2.43,-5.27,1.03] -1954-10-01T00:00:00Z,10058,Berhard ,M,1987-04-13T00:00:00Z,3,3,3,3,McFarlin ,38376,1.83,1.83,1.83,1.83,false,268378108,[Principal Support Engineer],[],[-4.89],[-4],[-4],[-4.89] -1953-09-19T00:00:00Z,10059,Alejandro ,F,1991-06-26T00:00:00Z,2,2,2,2,McAlpine ,44307,1.48,1.48,1.48,1.48,false,237368465,[Architect,Principal Support Engineer,Purchase Manager,Senior Team Lead],[false],[5.53,13.38,-4.69,6.27],[5,13,-4,6],[5,13,-4,6],[5.53,13.38,-4.69,6.27] -1961-10-15T00:00:00Z,10060,Breannda ,M,1987-11-02T00:00:00Z,2,2,2,2,Billingsley ,29175,1.42,1.42,1.42,1.42,true ,341158890,[Business Analyst,Data Scientist,Senior Team Lead],[false,false,true,false],[-1.76,-0.85],[-1,0],[-1,0],[-1.76,-0.85] -1962-10-19T00:00:00Z,10061,Tse ,M,1985-09-17T00:00:00Z,1,1,1,1,Herber ,49095,1.45,1.45,1.45,1.45,false,327550310,[Purchase Manager,Senior Team Lead],[false,true],[14.39,-2.58,-0.95],[14,-2,0],[14,-2,0],[14.39,-2.58,-0.95] -1961-11-02T00:00:00Z,10062,Anoosh ,M,1991-08-30T00:00:00Z,3,3,3,3,Peyn ,65030,1.70,1.70,1.70,1.70,false,203989706,[Python Developer,Senior Team Lead],[false,true,true],[-1.17],[-1],[-1],[-1.171] -1952-08-06T00:00:00Z,10063,Gino ,F,1989-04-08T00:00:00Z,3,3,3,3,Leonhardt ,52121,1.78,1.78,1.78,1.78,true ,214068302,[],[true],[],[],[],[] -1959-04-07T00:00:00Z,10064,Udi ,M,1985-11-20T00:00:00Z,5,5,5,5,Jansch ,33956,1.93,1.93,1.93,1.93,false,307364077,[Purchase Manager],[false,false,true,false],[-8.66,-2.52],[-8,-2],[-8,-2],[-8.66,-2.52] -1963-04-14T00:00:00Z,10065,Satosi ,M,1988-05-18T00:00:00Z,2,2,2,2,Awdeh ,50249,1.59,1.59,1.59,1.59,false,372660279,[Business Analyst,Data Scientist,Principal Support Engineer],[false,true],[-1.47,14.44,-9.81],[-1,14,-9],[-1,14,-9],[-1.47,14.44,-9.81] -1952-11-13T00:00:00Z,10066,Kwee ,M,1986-02-26T00:00:00Z,5,5,5,5,Schusler ,31897,2.10,2.10,2.10,2.10,true ,360906451,[Senior Python Developer,Data Scientist,Accountant,Internship],[true,true,true],[5.94],[5],[5],[5.94] -1953-01-07T00:00:00Z,10067,Claudi ,M,1987-03-04T00:00:00Z,2,2,2,2,Stavenow ,52044,1.77,1.77,1.77,1.77,true ,347664141,[Tech Lead,Principal Support Engineer],[false,false],[8.72,4.44],[8,4],[8,4],[8.72,4.44] -1962-11-26T00:00:00Z,10068,Charlene ,M,1987-08-07T00:00:00Z,3,3,3,3,Brattka ,28941,1.58,1.58,1.58,1.58,true ,233999584,[Architect],[true],[3.43,-5.61,-5.29],[3,-5,-5],[3,-5,-5],[3.43,-5.61,-5.29] -1960-09-06T00:00:00Z,10069,Margareta ,F,1989-11-05T00:00:00Z,5,5,5,5,Bierman ,41933,1.77,1.77,1.77,1.77,true ,366512352,[Business Analyst,Junior Developer,Purchase Manager,Support Engineer],[false],[-3.34,-6.33,6.23,-0.31],[-3,-6,6,0],[-3,-6,6,0],[-3.34,-6.33,6.23,-0.31] -1955-08-20T00:00:00Z,10070,Reuven ,M,1985-10-14T00:00:00Z,3,3,3,3,Garigliano ,54329,1.77,1.77,1.77,1.77,true ,347188604,[],[true,true,true],[-5.90],[-5],[-5],[-5.90] -1958-01-21T00:00:00Z,10071,Hisao ,M,1987-10-01T00:00:00Z,2,2,2,2,Lipner ,40612,2.07,2.07,2.07,2.07,false,306671693,[Business Analyst,Reporting Analyst,Senior Team Lead],[false,false,false],[-2.69],[-2],[-2],[-2.69] -1952-05-15T00:00:00Z,10072,Hironoby ,F,1988-07-21T00:00:00Z,5,5,5,5,Sidou ,54518,1.82,1.82,1.82,1.82,true ,209506065,[Architect,Tech Lead,Python Developer,Senior Team Lead],[false,false,true,false],[11.21,-2.30,2.22,-5.44],[11,-2,2,-5],[11,-2,2,-5],[11.21,-2.30,2.22,-5.44] -1954-02-23T00:00:00Z,10073,Shir ,M,1991-12-01T00:00:00Z,4,4,4,4,McClurg ,32568,1.66,1.66,1.66,1.66,false,314930367,[Principal Support Engineer,Python Developer,Junior Developer,Purchase Manager],[true,false],[-5.67],[-5],[-5],[-5.67] -1955-08-28T00:00:00Z,10074,Mokhtar ,F,1990-08-13T00:00:00Z,5,5,5,5,Bernatsky ,38992,1.64,1.64,1.64,1.64,true ,382397583,[Senior Python Developer,Python Developer],[true,false,false,true],[6.70,1.98,-5.64,2.96],[6,1,-5,2],[6,1,-5,2],[6.70,1.98,-5.64,2.96] -1960-03-09T00:00:00Z,10075,Gao ,F,1987-03-19T00:00:00Z,5,5,5,5,Dolinsky ,51956,1.94,1.94,1.94,1.94,false,370238919,[Purchase Manager],[true],[9.63,-3.29,8.42],[9,-3,8],[9,-3,8],[9.63,-3.29,8.42] -1952-06-13T00:00:00Z,10076,Erez ,F,1985-07-09T00:00:00Z,3,3,3,3,Ritzmann ,62405,1.83,1.83,1.83,1.83,false,376240317,[Architect,Senior Python Developer],[false],[-6.90,-1.30,8.75],[-6,-1,8],[-6,-1,8],[-6.90,-1.30,8.75] -1964-04-18T00:00:00Z,10077,Mona ,M,1990-03-02T00:00:00Z,5,5,5,5,Azuma ,46595,1.68,1.68,1.68,1.68,false,351960222,[Internship],[],[-0.01],[0],[0],[-0.01] -1959-12-25T00:00:00Z,10078,Danel ,F,1987-05-26T00:00:00Z,2,2,2,2,Mondadori ,69904,1.81,1.81,1.81,1.81,true ,377116038,[Architect,Principal Support Engineer,Internship],[true],[-7.88,9.98,12.52],[-7,9,12],[-7,9,12],[-7.88,9.98,12.52] -1961-10-05T00:00:00Z,10079,Kshitij ,F,1986-03-27T00:00:00Z,2,2,2,2,Gils ,32263,1.59,1.59,1.59,1.59,false,320953330,[],[false],[7.58],[7],[7],[7.58] -1957-12-03T00:00:00Z,10080,Premal ,M,1985-11-19T00:00:00Z,5,5,5,5,Baek ,52833,1.80,1.80,1.80,1.80,false,239266137,[Senior Python Developer],[],[-4.35,7.36,5.56],[-4,7,5],[-4,7,5],[-4.35,7.36,5.56] -1960-12-17T00:00:00Z,10081,Zhongwei ,M,1986-10-30T00:00:00Z,2,2,2,2,Rosen ,50128,1.44,1.44,1.44,1.44,true ,321375511,[Accountant,Internship],[false,false,false],[],[],[],[] -1963-09-09T00:00:00Z,10082,Parviz ,M,1990-01-03T00:00:00Z,4,4,4,4,Lortz ,49818,1.61,1.61,1.61,1.61,false,232522994,[Principal Support Engineer],[false],[1.19,-3.39],[1,-3],[1,-3],[1.19,-3.39] -1959-07-23T00:00:00Z,10083,Vishv ,M,1987-03-31T00:00:00Z,1,1,1,1,Zockler ,39110,1.42,1.42,1.42,1.42,false,331236443,[Head Human Resources],[],[],[],[],[] -1960-05-25T00:00:00Z,10084,Tuval ,M,1995-12-15T00:00:00Z,1,1,1,1,Kalloufi ,28035,1.51,1.51,1.51,1.51,true ,359067056,[Principal Support Engineer],[false],[],[],[],[] -1962-11-07T00:00:00Z,10085,Kenroku ,M,1994-04-09T00:00:00Z,5,5,5,5,Malabarba ,35742,2.01,2.01,2.01,2.01,true ,353404008,[Senior Python Developer,Business Analyst,Tech Lead,Accountant],[],[11.67,6.75,8.40],[11,6,8],[11,6,8],[11.67,6.75,8.40] -1962-11-19T00:00:00Z,10086,Somnath ,M,1990-02-16T00:00:00Z,1,1,1,1,Foote ,68547,1.74,1.74,1.74,1.74,true ,328580163,[Senior Python Developer],[false,true],[13.61],[13],[13],[13.61] -1959-07-23T00:00:00Z,10087,Xinglin ,F,1986-09-08T00:00:00Z,5,5,5,5,Eugenio ,32272,1.74,1.74,1.74,1.74,true ,305782871,[Junior Developer,Internship],[false,false],[-2.05],[-2],[-2],[-2.05] -1954-02-25T00:00:00Z,10088,Jungsoon ,F,1988-09-02T00:00:00Z,5,5,5,5,Syrzycki ,39638,1.91,1.91,1.91,1.91,false,330714423,[Reporting Analyst,Business Analyst,Tech Lead],[true],[],[],[],[] -1963-03-21T00:00:00Z,10089,Sudharsan ,F,1986-08-12T00:00:00Z,4,4,4,4,Flasterstein,43602,1.57,1.57,1.57,1.57,true ,232951673,[Junior Developer,Accountant],[true,false,false,false],[],[],[],[] -1961-05-30T00:00:00Z,10090,Kendra ,M,1986-03-14T00:00:00Z,2,2,2,2,Hofting ,44956,2.03,2.03,2.03,2.03,true ,212460105,[],[false,false,false,true],[7.15,-1.85,3.60],[7,-1,3],[7,-1,3],[7.15,-1.85,3.60] -1955-10-04T00:00:00Z,10091,Amabile ,M,1992-11-18T00:00:00Z,3,3,3,3,Gomatam ,38645,2.09,2.09,2.09,2.09,true ,242582807,[Reporting Analyst,Python Developer],[true,true,false,false],[-9.23,7.50,5.85,5.19],[-9,7,5,5],[-9,7,5,5],[-9.23,7.50,5.85,5.19] -1964-10-18T00:00:00Z,10092,Valdiodio ,F,1989-09-22T00:00:00Z,1,1,1,1,Niizuma ,25976,1.75,1.75,1.75,1.75,false,313407352,[Junior Developer,Accountant],[false,false,true,true],[8.78,0.39,-6.77,8.30],[8,0,-6,8],[8,0,-6,8],[8.78,0.39,-6.77,8.30] -1964-06-11T00:00:00Z,10093,Sailaja ,M,1996-11-05T00:00:00Z,3,3,3,3,Desikan ,45656,1.69,1.69,1.69,1.69,false,315904921,[Reporting Analyst,Tech Lead,Principal Support Engineer,Purchase Manager],[],[-0.88],[0],[0],[-0.88] -1957-05-25T00:00:00Z,10094,Arumugam ,F,1987-04-18T00:00:00Z,5,5,5,5,Ossenbruggen,66817,2.10,2.10,2.10,2.10,false,332920135,[Senior Python Developer,Principal Support Engineer,Accountant],[true,false,true],[2.22,7.92],[2,7],[2,7],[2.22,7.92] -1965-01-03T00:00:00Z,10095,Hilari ,M,1986-07-15T00:00:00Z,4,4,4,4,Morton ,37702,1.55,1.55,1.55,1.55,false,321850475,[],[true,true,false,false],[-3.93,-6.66],[-3,-6],[-3,-6],[-3.93,-6.66] -1954-09-16T00:00:00Z,10096,Jayson ,M,1990-01-14T00:00:00Z,4,4,4,4,Mandell ,43889,1.94,1.94,1.94,1.94,false,204381503,[Architect,Reporting Analyst],[false,false,false],[],[],[],[] -1952-02-27T00:00:00Z,10097,Remzi ,M,1990-09-15T00:00:00Z,3,3,3,3,Waschkowski ,71165,1.53,1.53,1.53,1.53,false,206258084,[Reporting Analyst,Tech Lead],[true,false],[-1.12],[-1],[-1],[-1.12] -1961-09-23T00:00:00Z,10098,Sreekrishna,F,1985-05-13T00:00:00Z,4,4,4,4,Servieres ,44817,2.00,2.00,2.00,2.00,false,272392146,[Architect,Internship,Senior Team Lead],[false],[-2.83,8.31,4.38],[-2,8,4],[-2,8,4],[-2.83,8.31,4.38] -1956-05-25T00:00:00Z,10099,Valter ,F,1988-10-18T00:00:00Z,2,2,2,2,Sullins ,73578,1.81,1.81,1.81,1.81,true ,377713748,[],[true,true],[10.71,14.26,-8.78,-3.98],[10,14,-8,-3],[10,14,-8,-3],[10.71,14.26,-8.78,-3.98] -1953-04-21T00:00:00Z,10100,Hironobu ,F,1987-09-21T00:00:00Z,4,4,4,4,Haraldson ,68431,1.77,1.77,1.77,1.77,true ,223910853,[Purchase Manager],[false,true,true,false],[13.97,-7.49],[13,-7],[13,-7],[13.97,-7.49] +birth_date:date ,emp_no:long,first_name:text,gender:text,hire_date:date_nanos,languages:byte,languages.long:long,languages.short:short,languages.byte:byte,last_name:text,salary:long,height:float,height.double:double,height.scaled_float:scaled_float,height.half_float:half_float,still_hired:keyword,avg_worked_seconds:unsigned_long,job_positions:text,is_rehired:keyword,salary_change:float,salary_change.int:integer,salary_change.long:long,salary_change.keyword:keyword +1953-09-02T00:00:00Z,10001,Georgi ,M,1986-06-26T00:00:00.000Z,2,2,2,2,Facello ,57305,2.03,2.03,2.03,2.03,true ,268728049,[Senior Python Developer,Accountant],[false,true],[1.19],[1],[1],[1.19] +1964-06-02T00:00:00Z,10002,Bezalel ,F,1985-11-21T00:00:00.000Z,5,5,5,5,Simmel ,56371,2.08,2.08,2.08,2.08,true ,328922887,[Senior Team Lead],[false,false],[-7.23,11.17],[-7,11],[-7,11],[-7.23,11.17] +1959-12-03T00:00:00Z,10003,Parto ,M,1986-08-28T00:00:00.000Z,4,4,4,4,Bamford ,61805,1.83,1.83,1.83,1.83,false,200296405,[],[],[14.68,12.82],[14,12],[14,12],[14.68,12.82] +1954-05-01T00:00:00Z,10004,Chirstian ,M,1986-12-01T00:00:00.000Z,5,5,5,5,Koblick ,36174,1.78,1.78,1.78,1.78,true ,311267831,[Reporting Analyst,Tech Lead,Head Human Resources,Support Engineer],[true],[3.65,-0.35,1.13,13.48],[3,0,1,13],[3,0,1,13],[3.65,-0.35,1.13,13.48] +1955-01-21T00:00:00Z,10005,Kyoichi ,M,1989-09-12T00:00:00.000Z,1,1,1,1,Maliniak ,63528,2.05,2.05,2.05,2.05,true ,244294991,[],[false,false,false,true],[-2.14,13.07],[-2,13],[-2,13],[-2.14,13.07] +1953-04-20T00:00:00Z,10006,Anneke ,F,1989-06-02T00:00:00.000Z,3,3,3,3,Preusig ,60335,1.56,1.56,1.56,1.56,false,372957040,[Tech Lead,Principal Support Engineer,Senior Team Lead],[],[-3.90],[-3],[-3],[-3.90] +1957-05-23T00:00:00Z,10007,Tzvetan ,F,1989-02-10T00:00:00.000Z,4,4,4,4,Zielinski ,74572,1.70,1.70,1.70,1.70,true ,393084805,[],[true,false,true,false],[-7.06,1.99,0.57],[-7,1,0],[-7,1,0],[-7.06,1.99,0.57] +1958-02-19T00:00:00Z,10008,Saniya ,M,1994-09-15T00:00:00.000Z,2,2,2,2,Kalloufi ,43906,2.10,2.10,2.10,2.10,true ,283074758,[Senior Python Developer,Junior Developer,Purchase Manager,Internship],[true,false],[12.68,3.54,0.75,-2.92],[12,3,0,-2],[12,3,0,-2],[12.68,3.54,0.75,-2.92] +1952-04-19T00:00:00Z,10009,Sumant ,F,1985-02-18T00:00:00.000Z,1,1,1,1,Peac ,66174,1.85,1.85,1.85,1.85,false,236805489,[Senior Python Developer,Internship],[],[],[],[],[] +1963-06-01T00:00:00Z,10010,Duangkaew , ,1989-08-24T00:00:00.000Z,4,4,4,4,Piveteau ,45797,1.70,1.70,1.70,1.70,false,315236372,[Architect,Reporting Analyst,Tech Lead,Purchase Manager],[true,true,false,false],[5.05,-6.77,4.69,12.15],[5,-6,4,12],[5,-6,4,12],[5.05,-6.77,4.69,12.15] +1953-11-07T00:00:00Z,10011,Mary , ,1990-01-22T00:00:00.000Z,5,5,5,5,Sluis ,31120,1.50,1.50,1.50,1.50,true ,239615525,[Architect,Reporting Analyst,Tech Lead,Senior Team Lead],[true,true],[10.35,-7.82,8.73,3.48],[10,-7,8,3],[10,-7,8,3],[10.35,-7.82,8.73,3.48] +1960-10-04T00:00:00Z,10012,Patricio , ,1992-12-18T00:00:00.000Z,5,5,5,5,Bridgland ,48942,1.97,1.97,1.97,1.97,false,365510850,[Head Human Resources,Accountant],[false,true,true,false],[0.04],[0],[0],[0.04] +1963-06-07T00:00:00Z,10013,Eberhardt , ,1985-10-20T00:00:00.000Z,1,1,1,1,Terkki ,48735,1.94,1.94,1.94,1.94,true ,253864340,[Reporting Analyst],[true,true],[],[],[],[] +1956-02-12T00:00:00Z,10014,Berni , ,1987-03-11T00:00:00.000Z,5,5,5,5,Genin ,37137,1.99,1.99,1.99,1.99,false,225049139,[Reporting Analyst,Data Scientist,Head Human Resources],[],[-1.89,9.07],[-1,9],[-1,9],[-1.89,9.07] +1959-08-19T00:00:00Z,10015,Guoxiang , ,1987-07-02T00:00:00.000Z,5,5,5,5,Nooteboom ,25324,1.66,1.66,1.66,1.66,true ,390266432,[Principal Support Engineer,Junior Developer,Head Human Resources,Support Engineer],[true,false,false,false],[14.25,12.40],[14,12],[14,12],[14.25,12.40] +1961-05-02T00:00:00Z,10016,Kazuhito , ,1995-01-27T00:00:00.000Z,2,2,2,2,Cappelletti ,61358,1.54,1.54,1.54,1.54,false,253029411,[Reporting Analyst,Python Developer,Accountant,Purchase Manager],[false,false],[-5.18,7.69],[-5,7],[-5,7],[-5.18,7.69] +1958-07-06T00:00:00Z,10017,Cristinel , ,1993-08-03T00:00:00.000Z,2,2,2,2,Bouloucos ,58715,1.74,1.74,1.74,1.74,false,236703986,[Data Scientist,Head Human Resources,Purchase Manager],[true,false,true,true],[-6.33],[-6],[-6],[-6.33] +1954-06-19T00:00:00Z,10018,Kazuhide , ,1987-04-03T00:00:00.000Z,2,2,2,2,Peha ,56760,1.97,1.97,1.97,1.97,false,309604079,[Junior Developer],[false,false,true,true],[-1.64,11.51,-5.32],[-1,11,-5],[-1,11,-5],[-1.64,11.51,-5.32] +1953-01-23T00:00:00Z,10019,Lillian , ,1999-04-30T00:00:00.000Z,1,1,1,1,Haddadi ,73717,2.06,2.06,2.06,2.06,false,342855721,[Purchase Manager],[false,false],[-6.84,8.42,-7.26],[-6,8,-7],[-6,8,-7],[-6.84,8.42,-7.26] +1952-12-24T00:00:00Z,10020,Mayuko ,M,1991-01-26T00:00:00.000Z, , , , ,Warwick ,40031,1.41,1.41,1.41,1.41,false,373309605,[Tech Lead],[true,true,false],[-5.81],[-5],[-5],[-5.81] +1960-02-20T00:00:00Z,10021,Ramzi ,M,1988-02-10T00:00:00.000Z, , , , ,Erde ,60408,1.47,1.47,1.47,1.47,false,287654610,[Support Engineer],[true],[],[],[],[] +1952-07-08T00:00:00Z,10022,Shahaf ,M,1995-08-22T00:00:00.000Z, , , , ,Famili ,48233,1.82,1.82,1.82,1.82,false,233521306,[Reporting Analyst,Data Scientist,Python Developer,Internship],[true,false],[12.09,2.85],[12,2],[12,2],[12.09,2.85] +1953-09-29T00:00:00Z,10023,Bojan ,F,1989-12-17T00:00:00.000Z, , , , ,Montemayor ,47896,1.75,1.75,1.75,1.75,true ,330870342,[Accountant,Support Engineer,Purchase Manager],[true,true,false],[14.63,0.80],[14,0],[14,0],[14.63,0.80] +1958-09-05T00:00:00Z,10024,Suzette ,F,1997-05-19T00:00:00.000Z, , , , ,Pettey ,64675,2.08,2.08,2.08,2.08,true ,367717671,[Junior Developer],[true,true,true,true],[],[],[],[] +1958-10-31T00:00:00Z,10025,Prasadram ,M,1987-08-17T00:00:00.000Z, , , , ,Heyers ,47411,1.87,1.87,1.87,1.87,false,371270797,[Accountant],[true,false],[-4.33,-2.90,12.06,-3.46],[-4,-2,12,-3],[-4,-2,12,-3],[-4.33,-2.90,12.06,-3.46] +1953-04-03T00:00:00Z,10026,Yongqiao ,M,1995-03-20T00:00:00.000Z, , , , ,Berztiss ,28336,2.10,2.10,2.10,2.10,true ,359208133,[Reporting Analyst],[false,true],[-7.37,10.62,11.20],[-7,10,11],[-7,10,11],[-7.37,10.62,11.20] +1962-07-10T00:00:00Z,10027,Divier ,F,1989-07-07T00:00:00.000Z, , , , ,Reistad ,73851,1.53,1.53,1.53,1.53,false,374037782,[Senior Python Developer],[false],[],[],[],[] +1963-11-26T00:00:00Z,10028,Domenick ,M,1991-10-22T00:00:00.000Z, , , , ,Tempesti ,39356,2.07,2.07,2.07,2.07,true ,226435054,[Tech Lead,Python Developer,Accountant,Internship],[true,false,false,true],[],[],[],[] +1956-12-13T00:00:00Z,10029,Otmar ,M,1985-11-20T00:00:00.000Z, , , , ,Herbst ,74999,1.99,1.99,1.99,1.99,false,257694181,[Senior Python Developer,Data Scientist,Principal Support Engineer],[true],[-0.32,-1.90,-8.19],[0,-1,-8],[0,-1,-8],[-0.32,-1.90,-8.19] +1958-07-14T00:00:00Z,10030, ,M,1994-02-17T00:00:00.000Z,3,3,3,3,Demeyer ,67492,1.92,1.92,1.92,1.92,false,394597613,[Tech Lead,Data Scientist,Senior Team Lead],[true,false,false],[-0.40],[0],[0],[-0.40] +1959-01-27T00:00:00Z,10031, ,M,1991-09-01T00:00:00.000Z,4,4,4,4,Joslin ,37716,1.68,1.68,1.68,1.68,false,348545109,[Architect,Senior Python Developer,Purchase Manager,Senior Team Lead],[false],[],[],[],[] +1960-08-09T00:00:00Z,10032, ,F,1990-06-20T00:00:00.000Z,3,3,3,3,Reistad ,62233,2.10,2.10,2.10,2.10,false,277622619,[Architect,Senior Python Developer,Junior Developer,Purchase Manager],[false,false],[9.32,-4.92],[9,-4],[9,-4],[9.32,-4.92] +1956-11-14T00:00:00Z,10033, ,M,1987-03-18T00:00:00.000Z,1,1,1,1,Merlo ,70011,1.63,1.63,1.63,1.63,false,208374744,[],[true],[],[],[],[] +1962-12-29T00:00:00Z,10034, ,M,1988-09-21T00:00:00.000Z,1,1,1,1,Swan ,39878,1.46,1.46,1.46,1.46,false,214393176,[Business Analyst,Data Scientist,Python Developer,Accountant],[false],[-8.46],[-8],[-8],[-8.46] +1953-02-08T00:00:00Z,10035, ,M,1988-09-05T00:00:00.000Z,5,5,5,5,Chappelet ,25945,1.81,1.81,1.81,1.81,false,203838153,[Senior Python Developer,Data Scientist],[false],[-2.54,-6.58],[-2,-6],[-2,-6],[-2.54,-6.58] +1959-08-10T00:00:00Z,10036, ,M,1992-01-03T00:00:00.000Z,4,4,4,4,Portugali ,60781,1.61,1.61,1.61,1.61,false,305493131,[Senior Python Developer],[true,false,false],[],[],[],[] +1963-07-22T00:00:00Z,10037, ,M,1990-12-05T00:00:00.000Z,2,2,2,2,Makrucki ,37691,2.00,2.00,2.00,2.00,true ,359217000,[Senior Python Developer,Tech Lead,Accountant],[false],[-7.08],[-7],[-7],[-7.08] +1960-07-20T00:00:00Z,10038, ,M,1989-09-20T00:00:00.000Z,4,4,4,4,Lortz ,35222,1.53,1.53,1.53,1.53,true ,314036411,[Senior Python Developer,Python Developer,Support Engineer],[],[],[],[],[] +1959-10-01T00:00:00Z,10039, ,M,1988-01-19T00:00:00.000Z,2,2,2,2,Brender ,36051,1.55,1.55,1.55,1.55,false,243221262,[Business Analyst,Python Developer,Principal Support Engineer],[true,true],[-6.90],[-6],[-6],[-6.90] + ,10040,Weiyi ,F,1993-02-14T00:00:00.000Z,4,4,4,4,Meriste ,37112,1.90,1.90,1.90,1.90,false,244478622,[Principal Support Engineer],[true,false,true,true],[6.97,14.74,-8.94,1.92],[6,14,-8,1],[6,14,-8,1],[6.97,14.74,-8.94,1.92] + ,10041,Uri ,F,1989-11-12T00:00:00.000Z,1,1,1,1,Lenart ,56415,1.75,1.75,1.75,1.75,false,287789442,[Data Scientist,Head Human Resources,Internship,Senior Team Lead],[],[9.21,0.05,7.29,-2.94],[9,0,7,-2],[9,0,7,-2],[9.21,0.05,7.29,-2.94] + ,10042,Magy ,F,1993-03-21T00:00:00.000Z,3,3,3,3,Stamatiou ,30404,1.44,1.44,1.44,1.44,true ,246355863,[Architect,Business Analyst,Junior Developer,Internship],[],[-9.28,9.42],[-9,9],[-9,9],[-9.28,9.42] + ,10043,Yishay ,M,1990-10-20T00:00:00.000Z,1,1,1,1,Tzvieli ,34341,1.52,1.52,1.52,1.52,true ,287222180,[Data Scientist,Python Developer,Support Engineer],[false,true,true],[-5.17,4.62,7.42],[-5,4,7],[-5,4,7],[-5.17,4.62,7.42] + ,10044,Mingsen ,F,1994-05-21T00:00:00.000Z,1,1,1,1,Casley ,39728,2.06,2.06,2.06,2.06,false,387408356,[Tech Lead,Principal Support Engineer,Accountant,Support Engineer],[true,true],[8.09],[8],[8],[8.09] + ,10045,Moss ,M,1989-09-02T00:00:00.000Z,3,3,3,3,Shanbhogue ,74970,1.70,1.70,1.70,1.70,false,371418933,[Principal Support Engineer,Junior Developer,Accountant,Purchase Manager],[true,false],[],[],[],[] + ,10046,Lucien ,M,1992-06-20T00:00:00.000Z,4,4,4,4,Rosenbaum ,50064,1.52,1.52,1.52,1.52,true ,302353405,[Principal Support Engineer,Junior Developer,Head Human Resources,Internship],[true,true,false,true],[2.39],[2],[2],[2.39] + ,10047,Zvonko ,M,1989-03-31T00:00:00.000Z,4,4,4,4,Nyanchama ,42716,1.52,1.52,1.52,1.52,true ,306369346,[Architect,Data Scientist,Principal Support Engineer,Senior Team Lead],[true],[-6.36,12.12],[-6,12],[-6,12],[-6.36,12.12] + ,10048,Florian ,M,1985-02-24T00:00:00.000Z,3,3,3,3,Syrotiuk ,26436,2.00,2.00,2.00,2.00,false,248451647,[Internship],[true,true],[],[],[],[] + ,10049,Basil ,F,1992-05-04T00:00:00.000Z,5,5,5,5,Tramer ,37853,1.52,1.52,1.52,1.52,true ,320725709,[Senior Python Developer,Business Analyst],[],[-1.05],[-1],[-1],[-1.05] +1958-05-21T00:00:00Z,10050,Yinghua ,M,1990-12-25T00:00:00.000Z,2,2,2,2,Dredge ,43026,1.96,1.96,1.96,1.96,true ,242731798,[Reporting Analyst,Junior Developer,Accountant,Support Engineer],[true],[8.70,10.94],[8,10],[8,10],[8.70,10.94] +1953-07-28T00:00:00Z,10051,Hidefumi ,M,1992-10-15T00:00:00.000Z,3,3,3,3,Caine ,58121,1.89,1.89,1.89,1.89,true ,374753122,[Business Analyst,Accountant,Purchase Manager],[],[],[],[],[] +1961-02-26T00:00:00Z,10052,Heping ,M,1988-05-21T00:00:00.000Z,1,1,1,1,Nitsch ,55360,1.79,1.79,1.79,1.79,true ,299654717,[],[true,true,false],[-0.55,-1.89,-4.22,-6.03],[0,-1,-4,-6],[0,-1,-4,-6],[-0.55,-1.89,-4.22,-6.03] +1954-09-13T00:00:00Z,10053,Sanjiv ,F,1986-02-04T00:00:00.000Z,3,3,3,3,Zschoche ,54462,1.58,1.58,1.58,1.58,false,368103911,[Support Engineer],[true,false,true,false],[-7.67,-3.25],[-7,-3],[-7,-3],[-7.67,-3.25] +1957-04-04T00:00:00Z,10054,Mayumi ,M,1995-03-13T00:00:00.000Z,4,4,4,4,Schueller ,65367,1.82,1.82,1.82,1.82,false,297441693,[Principal Support Engineer],[false,false],[],[],[],[] +1956-06-06T00:00:00Z,10055,Georgy ,M,1992-04-27T00:00:00.000Z,5,5,5,5,Dredge ,49281,2.04,2.04,2.04,2.04,false,283157844,[Senior Python Developer,Head Human Resources,Internship,Support Engineer],[false,false,true],[7.34,12.99,3.17],[7,12,3],[7,12,3],[7.34,12.99,3.17] +1961-09-01T00:00:00Z,10056,Brendon ,F,1990-02-01T00:00:00.000Z,2,2,2,2,Bernini ,33370,1.57,1.57,1.57,1.57,true ,349086555,[Senior Team Lead],[true,false,false],[10.99,-5.17],[10,-5],[10,-5],[10.99,-5.17] +1954-05-30T00:00:00Z,10057,Ebbe ,F,1992-01-15T00:00:00.000Z,4,4,4,4,Callaway ,27215,1.59,1.59,1.59,1.59,true ,324356269,[Python Developer,Head Human Resources],[],[-6.73,-2.43,-5.27,1.03],[-6,-2,-5,1],[-6,-2,-5,1],[-6.73,-2.43,-5.27,1.03] +1954-10-01T00:00:00Z,10058,Berhard ,M,1987-04-13T00:00:00.000Z,3,3,3,3,McFarlin ,38376,1.83,1.83,1.83,1.83,false,268378108,[Principal Support Engineer],[],[-4.89],[-4],[-4],[-4.89] +1953-09-19T00:00:00Z,10059,Alejandro ,F,1991-06-26T00:00:00.000Z,2,2,2,2,McAlpine ,44307,1.48,1.48,1.48,1.48,false,237368465,[Architect,Principal Support Engineer,Purchase Manager,Senior Team Lead],[false],[5.53,13.38,-4.69,6.27],[5,13,-4,6],[5,13,-4,6],[5.53,13.38,-4.69,6.27] +1961-10-15T00:00:00Z,10060,Breannda ,M,1987-11-02T00:00:00.000Z,2,2,2,2,Billingsley ,29175,1.42,1.42,1.42,1.42,true ,341158890,[Business Analyst,Data Scientist,Senior Team Lead],[false,false,true,false],[-1.76,-0.85],[-1,0],[-1,0],[-1.76,-0.85] +1962-10-19T00:00:00Z,10061,Tse ,M,1985-09-17T00:00:00.000Z,1,1,1,1,Herber ,49095,1.45,1.45,1.45,1.45,false,327550310,[Purchase Manager,Senior Team Lead],[false,true],[14.39,-2.58,-0.95],[14,-2,0],[14,-2,0],[14.39,-2.58,-0.95] +1961-11-02T00:00:00Z,10062,Anoosh ,M,1991-08-30T00:00:00.000Z,3,3,3,3,Peyn ,65030,1.70,1.70,1.70,1.70,false,203989706,[Python Developer,Senior Team Lead],[false,true,true],[-1.17],[-1],[-1],[-1.171] +1952-08-06T00:00:00Z,10063,Gino ,F,1989-04-08T00:00:00.000Z,3,3,3,3,Leonhardt ,52121,1.78,1.78,1.78,1.78,true ,214068302,[],[true],[],[],[],[] +1959-04-07T00:00:00Z,10064,Udi ,M,1985-11-20T00:00:00.000Z,5,5,5,5,Jansch ,33956,1.93,1.93,1.93,1.93,false,307364077,[Purchase Manager],[false,false,true,false],[-8.66,-2.52],[-8,-2],[-8,-2],[-8.66,-2.52] +1963-04-14T00:00:00Z,10065,Satosi ,M,1988-05-18T00:00:00.000Z,2,2,2,2,Awdeh ,50249,1.59,1.59,1.59,1.59,false,372660279,[Business Analyst,Data Scientist,Principal Support Engineer],[false,true],[-1.47,14.44,-9.81],[-1,14,-9],[-1,14,-9],[-1.47,14.44,-9.81] +1952-11-13T00:00:00Z,10066,Kwee ,M,1986-02-26T00:00:00.000Z,5,5,5,5,Schusler ,31897,2.10,2.10,2.10,2.10,true ,360906451,[Senior Python Developer,Data Scientist,Accountant,Internship],[true,true,true],[5.94],[5],[5],[5.94] +1953-01-07T00:00:00Z,10067,Claudi ,M,1987-03-04T00:00:00.000Z,2,2,2,2,Stavenow ,52044,1.77,1.77,1.77,1.77,true ,347664141,[Tech Lead,Principal Support Engineer],[false,false],[8.72,4.44],[8,4],[8,4],[8.72,4.44] +1962-11-26T00:00:00Z,10068,Charlene ,M,1987-08-07T00:00:00.000Z,3,3,3,3,Brattka ,28941,1.58,1.58,1.58,1.58,true ,233999584,[Architect],[true],[3.43,-5.61,-5.29],[3,-5,-5],[3,-5,-5],[3.43,-5.61,-5.29] +1960-09-06T00:00:00Z,10069,Margareta ,F,1989-11-05T00:00:00.000Z,5,5,5,5,Bierman ,41933,1.77,1.77,1.77,1.77,true ,366512352,[Business Analyst,Junior Developer,Purchase Manager,Support Engineer],[false],[-3.34,-6.33,6.23,-0.31],[-3,-6,6,0],[-3,-6,6,0],[-3.34,-6.33,6.23,-0.31] +1955-08-20T00:00:00Z,10070,Reuven ,M,1985-10-14T00:00:00.000Z,3,3,3,3,Garigliano ,54329,1.77,1.77,1.77,1.77,true ,347188604,[],[true,true,true],[-5.90],[-5],[-5],[-5.90] +1958-01-21T00:00:00Z,10071,Hisao ,M,1987-10-01T00:00:00.000Z,2,2,2,2,Lipner ,40612,2.07,2.07,2.07,2.07,false,306671693,[Business Analyst,Reporting Analyst,Senior Team Lead],[false,false,false],[-2.69],[-2],[-2],[-2.69] +1952-05-15T00:00:00Z,10072,Hironoby ,F,1988-07-21T00:00:00.000Z,5,5,5,5,Sidou ,54518,1.82,1.82,1.82,1.82,true ,209506065,[Architect,Tech Lead,Python Developer,Senior Team Lead],[false,false,true,false],[11.21,-2.30,2.22,-5.44],[11,-2,2,-5],[11,-2,2,-5],[11.21,-2.30,2.22,-5.44] +1954-02-23T00:00:00Z,10073,Shir ,M,1991-12-01T00:00:00.000Z,4,4,4,4,McClurg ,32568,1.66,1.66,1.66,1.66,false,314930367,[Principal Support Engineer,Python Developer,Junior Developer,Purchase Manager],[true,false],[-5.67],[-5],[-5],[-5.67] +1955-08-28T00:00:00Z,10074,Mokhtar ,F,1990-08-13T00:00:00.000Z,5,5,5,5,Bernatsky ,38992,1.64,1.64,1.64,1.64,true ,382397583,[Senior Python Developer,Python Developer],[true,false,false,true],[6.70,1.98,-5.64,2.96],[6,1,-5,2],[6,1,-5,2],[6.70,1.98,-5.64,2.96] +1960-03-09T00:00:00Z,10075,Gao ,F,1987-03-19T00:00:00.000Z,5,5,5,5,Dolinsky ,51956,1.94,1.94,1.94,1.94,false,370238919,[Purchase Manager],[true],[9.63,-3.29,8.42],[9,-3,8],[9,-3,8],[9.63,-3.29,8.42] +1952-06-13T00:00:00Z,10076,Erez ,F,1985-07-09T00:00:00.000Z,3,3,3,3,Ritzmann ,62405,1.83,1.83,1.83,1.83,false,376240317,[Architect,Senior Python Developer],[false],[-6.90,-1.30,8.75],[-6,-1,8],[-6,-1,8],[-6.90,-1.30,8.75] +1964-04-18T00:00:00Z,10077,Mona ,M,1990-03-02T00:00:00.000Z,5,5,5,5,Azuma ,46595,1.68,1.68,1.68,1.68,false,351960222,[Internship],[],[-0.01],[0],[0],[-0.01] +1959-12-25T00:00:00Z,10078,Danel ,F,1987-05-26T00:00:00.000Z,2,2,2,2,Mondadori ,69904,1.81,1.81,1.81,1.81,true ,377116038,[Architect,Principal Support Engineer,Internship],[true],[-7.88,9.98,12.52],[-7,9,12],[-7,9,12],[-7.88,9.98,12.52] +1961-10-05T00:00:00Z,10079,Kshitij ,F,1986-03-27T00:00:00.000Z,2,2,2,2,Gils ,32263,1.59,1.59,1.59,1.59,false,320953330,[],[false],[7.58],[7],[7],[7.58] +1957-12-03T00:00:00Z,10080,Premal ,M,1985-11-19T00:00:00.000Z,5,5,5,5,Baek ,52833,1.80,1.80,1.80,1.80,false,239266137,[Senior Python Developer],[],[-4.35,7.36,5.56],[-4,7,5],[-4,7,5],[-4.35,7.36,5.56] +1960-12-17T00:00:00Z,10081,Zhongwei ,M,1986-10-30T00:00:00.000Z,2,2,2,2,Rosen ,50128,1.44,1.44,1.44,1.44,true ,321375511,[Accountant,Internship],[false,false,false],[],[],[],[] +1963-09-09T00:00:00Z,10082,Parviz ,M,1990-01-03T00:00:00.000Z,4,4,4,4,Lortz ,49818,1.61,1.61,1.61,1.61,false,232522994,[Principal Support Engineer],[false],[1.19,-3.39],[1,-3],[1,-3],[1.19,-3.39] +1959-07-23T00:00:00Z,10083,Vishv ,M,1987-03-31T00:00:00.000Z,1,1,1,1,Zockler ,39110,1.42,1.42,1.42,1.42,false,331236443,[Head Human Resources],[],[],[],[],[] +1960-05-25T00:00:00Z,10084,Tuval ,M,1995-12-15T00:00:00.000Z,1,1,1,1,Kalloufi ,28035,1.51,1.51,1.51,1.51,true ,359067056,[Principal Support Engineer],[false],[],[],[],[] +1962-11-07T00:00:00Z,10085,Kenroku ,M,1994-04-09T00:00:00.000Z,5,5,5,5,Malabarba ,35742,2.01,2.01,2.01,2.01,true ,353404008,[Senior Python Developer,Business Analyst,Tech Lead,Accountant],[],[11.67,6.75,8.40],[11,6,8],[11,6,8],[11.67,6.75,8.40] +1962-11-19T00:00:00Z,10086,Somnath ,M,1990-02-16T00:00:00.000Z,1,1,1,1,Foote ,68547,1.74,1.74,1.74,1.74,true ,328580163,[Senior Python Developer],[false,true],[13.61],[13],[13],[13.61] +1959-07-23T00:00:00Z,10087,Xinglin ,F,1986-09-08T00:00:00.000Z,5,5,5,5,Eugenio ,32272,1.74,1.74,1.74,1.74,true ,305782871,[Junior Developer,Internship],[false,false],[-2.05],[-2],[-2],[-2.05] +1954-02-25T00:00:00Z,10088,Jungsoon ,F,1988-09-02T00:00:00.000Z,5,5,5,5,Syrzycki ,39638,1.91,1.91,1.91,1.91,false,330714423,[Reporting Analyst,Business Analyst,Tech Lead],[true],[],[],[],[] +1963-03-21T00:00:00Z,10089,Sudharsan ,F,1986-08-12T00:00:00.000Z,4,4,4,4,Flasterstein,43602,1.57,1.57,1.57,1.57,true ,232951673,[Junior Developer,Accountant],[true,false,false,false],[],[],[],[] +1961-05-30T00:00:00Z,10090,Kendra ,M,1986-03-14T00:00:00.000Z,2,2,2,2,Hofting ,44956,2.03,2.03,2.03,2.03,true ,212460105,[],[false,false,false,true],[7.15,-1.85,3.60],[7,-1,3],[7,-1,3],[7.15,-1.85,3.60] +1955-10-04T00:00:00Z,10091,Amabile ,M,1992-11-18T00:00:00.000Z,3,3,3,3,Gomatam ,38645,2.09,2.09,2.09,2.09,true ,242582807,[Reporting Analyst,Python Developer],[true,true,false,false],[-9.23,7.50,5.85,5.19],[-9,7,5,5],[-9,7,5,5],[-9.23,7.50,5.85,5.19] +1964-10-18T00:00:00Z,10092,Valdiodio ,F,1989-09-22T00:00:00.000Z,1,1,1,1,Niizuma ,25976,1.75,1.75,1.75,1.75,false,313407352,[Junior Developer,Accountant],[false,false,true,true],[8.78,0.39,-6.77,8.30],[8,0,-6,8],[8,0,-6,8],[8.78,0.39,-6.77,8.30] +1964-06-11T00:00:00Z,10093,Sailaja ,M,1996-11-05T00:00:00.000Z,3,3,3,3,Desikan ,45656,1.69,1.69,1.69,1.69,false,315904921,[Reporting Analyst,Tech Lead,Principal Support Engineer,Purchase Manager],[],[-0.88],[0],[0],[-0.88] +1957-05-25T00:00:00Z,10094,Arumugam ,F,1987-04-18T00:00:00.000Z,5,5,5,5,Ossenbruggen,66817,2.10,2.10,2.10,2.10,false,332920135,[Senior Python Developer,Principal Support Engineer,Accountant],[true,false,true],[2.22,7.92],[2,7],[2,7],[2.22,7.92] +1965-01-03T00:00:00Z,10095,Hilari ,M,1986-07-15T00:00:00.000Z,4,4,4,4,Morton ,37702,1.55,1.55,1.55,1.55,false,321850475,[],[true,true,false,false],[-3.93,-6.66],[-3,-6],[-3,-6],[-3.93,-6.66] +1954-09-16T00:00:00Z,10096,Jayson ,M,1990-01-14T00:00:00.000Z,4,4,4,4,Mandell ,43889,1.94,1.94,1.94,1.94,false,204381503,[Architect,Reporting Analyst],[false,false,false],[],[],[],[] +1952-02-27T00:00:00Z,10097,Remzi ,M,1990-09-15T00:00:00.000Z,3,3,3,3,Waschkowski ,71165,1.53,1.53,1.53,1.53,false,206258084,[Reporting Analyst,Tech Lead],[true,false],[-1.12],[-1],[-1],[-1.12] +1961-09-23T00:00:00Z,10098,Sreekrishna,F,1985-05-13T00:00:00.000Z,4,4,4,4,Servieres ,44817,2.00,2.00,2.00,2.00,false,272392146,[Architect,Internship,Senior Team Lead],[false],[-2.83,8.31,4.38],[-2,8,4],[-2,8,4],[-2.83,8.31,4.38] +1956-05-25T00:00:00Z,10099,Valter ,F,1988-10-18T00:00:00.000Z,2,2,2,2,Sullins ,73578,1.81,1.81,1.81,1.81,true ,377713748,[],[true,true],[10.71,14.26,-8.78,-3.98],[10,14,-8,-3],[10,14,-8,-3],[10.71,14.26,-8.78,-3.98] +1953-04-21T00:00:00Z,10100,Hironobu ,F,1987-09-21T00:00:00.000Z,4,4,4,4,Haraldson ,68431,1.77,1.77,1.77,1.77,true ,223910853,[Purchase Manager],[false,true,true,false],[13.97,-7.49],[13,-7],[13,-7],[13.97,-7.49] diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec index 4268179469203..a41c9b8cd0b16 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec @@ -1296,14 +1296,3 @@ diff_sec:integer | diff_sec_m:integer | n:date_nanos -18489600 | -18489599 | 2023-03-23T12:15:03.360103847Z -18489600 | -18489599 | 2023-03-23T12:15:03.360103847Z ; - -DateAndDateNanos -from employees,employees_incompatible -| keep emp_no, birth_date -| sort emp_no -| limit 1 -; - -emp_no:integer |birth_date:date -10001 |1953-09-02T00:00:00Z -; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec index 8b19bc589fcff..443a800d40a36 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec @@ -1639,3 +1639,67 @@ id:integer | name:keyword | count:long 13 | lllll | 2 14 | mmmmm | 2 ; + +MultiTypedFieldsKeepDropSort +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| DROP salary +| KEEP emp_no, languages, hire_date, avg_worked_seconds +| SORT emp_no, hire_date +| limit 2 +; + +emp_no:long |languages:integer |hire_date:date_nanos |avg_worked_seconds:unsigned_long +10001 |2 |1986-06-26T00:00:00.000Z |268728049 +10001 |2 |1986-06-26T00:00:00.000Z |268728049 +; + +MultiTypedFieldsRenameKeep +required_capability: implicit_casting_union_typed_numeric_and_date + +from employees, employees_incompatible +| RENAME emp_no as new_emp_no, languages as new_languages, hire_date as new_hire_date, avg_worked_seconds as new_avg_worked_seconds +| KEEP new_emp_no, new_languages, new_hire_date, new_avg_worked_seconds +| sort new_emp_no, new_hire_date +| limit 2 +; + +new_emp_no:long |new_languages:integer |new_hire_date:date_nanos |new_avg_worked_seconds:unsigned_long +10001 |2 |1986-06-26T00:00:00.000Z |268728049 +10001 |2 |1986-06-26T00:00:00.000Z |268728049 +; + +MultiTypedFieldsEvalKeepSort +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| DROP birth_date +| EVAL x = emp_no + 1, y = hire_date + 1 year, z = hire_date::datetime +| KEEP emp_no, x, hire_date, y, z +| SORT emp_no, hire_date +| limit 2 +; + +emp_no:long |x:long |hire_date:date_nanos |y:date_nanos |z:date +10001 |10002 |1986-06-26T00:00:00.000Z |1987-06-26T00:00:00.000Z |1986-06-26T00:00:00.000Z +10001 |10002 |1986-06-26T00:00:00.000Z |1987-06-26T00:00:00.000Z |1986-06-26T00:00:00.000Z +; + +MultiTypedFieldsEvalFilterKeepSort +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| EVAL x = emp_no + 1, y = hire_date + 1 day +| WHERE emp_no > 10010 AND hire_date > "1992-01-01" AND languages < 3 AND height > 1.2 +| KEEP emp_no, hire_date +| SORT emp_no, hire_date +| limit 4 +; + +emp_no:long |hire_date:date_nanos +10016 | 1995-01-27T00:00:00.000Z +10016 | 1995-01-27T00:00:00.000Z +10017 | 1993-08-03T00:00:00.000Z +10017 | 1993-08-03T00:00:00.000Z +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 5f88f9f348276..46a3383324655 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -850,7 +850,12 @@ public enum Cap { /** * Allow mixed numeric types in conditional functions - case, greatest and least */ - MIXED_NUMERIC_TYPES_IN_CASE_GREATEST_LEAST; + MIXED_NUMERIC_TYPES_IN_CASE_GREATEST_LEAST, + + /** + * Support implicit casting for union typed numeric and date/date_nanos fields + */ + IMPLICIT_CASTING_UNION_TYPED_NUMERIC_AND_DATE; private final boolean enabled; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 9dff4072c8e0e..402dfd9b27969 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -51,6 +51,7 @@ import org.elasticsearch.xpack.esql.expression.function.FunctionDefinition; import org.elasticsearch.xpack.esql.expression.function.UnresolvedFunction; import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute; +import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction; import org.elasticsearch.xpack.esql.expression.function.grouping.GroupingFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Case; @@ -127,7 +128,6 @@ import static java.util.Collections.singletonList; import static org.elasticsearch.common.logging.LoggerMessageFormat.format; import static org.elasticsearch.xpack.core.enrich.EnrichPolicy.GEO_MATCH_TYPE; -import static org.elasticsearch.xpack.esql.core.expression.Attribute.rawTemporaryName; import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN; import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS; @@ -183,6 +183,10 @@ public class Analyzer extends ParameterizedRuleExecutordate_trunc("1 minute", dateField) * * If the inputs to Coalesce are mixed numeric types, cast the rest of the numeric field or value to the first numeric data type if - * applicable. For example, implicit casting converts: + * applicable, the same applies to Case, Greatest, Least. For example, implicit casting converts: *
    *
  • Coalesce(Long, Int) to Coalesce(Long, Long)
    • *
  • Coalesce(null, Long, Int) to Coalesce(null, Long, Long)
    • @@ -1196,13 +1200,10 @@ private BitSet gatherPreAnalysisMetrics(LogicalPlan plan, BitSet b) { private static class ImplicitCasting extends ParameterizedRule { @Override public LogicalPlan apply(LogicalPlan plan, AnalyzerContext context) { - // do implicit casting for union typed fields in sort keys + // do implicit casting for union typed fields in sort, keep, rename LogicalPlan newPlan = plan.transformUp(p -> { - if (p instanceof OrderBy ob) { - return castInvalidMappedFieldInOrderBy(ob); - } - if (p instanceof EsqlProject proj) { - return castInvalidMappedFieldInProjection(proj); + if (p instanceof OrderBy | p instanceof EsqlProject) { + return castInvalidMappedFieldInLogicalPlan(p); } return p; }); @@ -1213,108 +1214,6 @@ public LogicalPlan apply(LogicalPlan plan, AnalyzerContext context) { ); } - private static EsqlProject castInvalidMappedFieldInProjection(EsqlProject esqlProject) { - List newProjections = new ArrayList<>(esqlProject.projections().size()); - List aliases = new ArrayList<>(esqlProject.projections().size()); - int counter = 0; - projectionLoop: for (NamedExpression projection : esqlProject.projections()) { - Expression e = projection instanceof Alias a ? a.child() : projection; - if (e.resolved() - && e.dataType() == UNSUPPORTED - && e instanceof FieldAttribute fa - && fa.field() instanceof InvalidMappedField imf) { - // this is an invalid mapped field, find a common data type and cast to it - DataType targetType = null; - for (DataType type : imf.types()) { - if (targetType == null) { // initialize the target type to the first type - targetType = type; - } else { - targetType = EsqlDataTypeConverter.commonType(targetType, type); - if (targetType == null) { // if there is no common type, continue to the next argument - newProjections.add(projection); - continue projectionLoop; - } - } - } - if (targetType != null && isRepresentable(targetType) && (isMillisOrNanos(targetType) || targetType.isNumeric())) { - // create an eval to cast union type to a common type - var name = rawTemporaryName("keep", String.valueOf(counter++), targetType.name()); - Alias alias; - switch (targetType) { - case INTEGER -> alias = new Alias(fa.source(), name, new ToInteger(fa.source(), fa)); - case LONG -> alias = new Alias(fa.source(), name, new ToLong(fa.source(), fa)); - case DOUBLE -> alias = new Alias(fa.source(), name, new ToDouble(fa.source(), fa)); - case UNSIGNED_LONG -> alias = new Alias(fa.source(), name, new ToUnsignedLong(fa.source(), fa)); - case DATETIME -> alias = new Alias(fa.source(), name, new ToDatetime(fa.source(), fa)); - case DATE_NANOS -> alias = new Alias(fa.source(), name, new ToDateNanos(fa.source(), fa)); - default -> throw new EsqlIllegalArgumentException("unexpected data type: " + targetType); - } - aliases.add(alias); - newProjections.add(alias); - } - } else { - newProjections.add(projection); - } - } - if (aliases.isEmpty() == false) { - Eval eval = new Eval(esqlProject.source(), esqlProject.child(), aliases); - return new EsqlProject(esqlProject.source(), eval, newProjections); - } else { - return esqlProject; - } - } - - private static OrderBy castInvalidMappedFieldInOrderBy(OrderBy ob) { - List newOrder = new ArrayList<>(ob.order().size()); - boolean changed = false; - orderLoop: for (Order o : ob.order()) { - DataType targetType = null; - if (o.child().resolved() - && o.child().dataType() == UNSUPPORTED - && o.child() instanceof FieldAttribute fa - && fa.field() instanceof InvalidMappedField imf) { - // this is an invalid mapped field, find a common data type and cast to it - for (DataType type : imf.types()) { - if (targetType == null) { // initialize the target type to the first type - targetType = type; - } else { - targetType = EsqlDataTypeConverter.commonType(targetType, type); - if (targetType == null) { // if there is no common type, continue to the next argument - newOrder.add(o); - continue orderLoop; - } - } - } - if (targetType != null && isRepresentable(targetType) && (isMillisOrNanos(targetType) || targetType.isNumeric())) { - changed = true; - switch (targetType) { - case INTEGER -> newOrder.add( - new Order(o.source(), new ToInteger(fa.source(), fa), o.direction(), o.nullsPosition()) - ); - case LONG -> newOrder.add(new Order(o.source(), new ToLong(fa.source(), fa), o.direction(), o.nullsPosition())); - case DOUBLE -> newOrder.add( - new Order(o.source(), new ToDouble(fa.source(), fa), o.direction(), o.nullsPosition()) - ); - case UNSIGNED_LONG -> newOrder.add( - new Order(o.source(), new ToUnsignedLong(fa.source(), fa), o.direction(), o.nullsPosition()) - ); - case DATETIME -> newOrder.add( - new Order(o.source(), new ToDatetime(fa.source(), fa), o.direction(), o.nullsPosition()) - ); - case DATE_NANOS -> newOrder.add( - new Order(o.source(), new ToDateNanos(fa.source(), fa), o.direction(), o.nullsPosition()) - ); - default -> throw new EsqlIllegalArgumentException("unexpected data type: " + targetType); - } - } - } else { - newOrder.add(o); - } - } - - return changed ? new OrderBy(ob.source(), ob.child(), newOrder) : ob; - } - private static Expression cast(org.elasticsearch.xpack.esql.core.expression.function.Function f, EsqlFunctionRegistry registry) { // Add cast functions to InvalidMappedField if there isn't one yet f = castInvalidMappedField(f); @@ -1334,7 +1233,7 @@ private static Expression cast(org.elasticsearch.xpack.esql.core.expression.func private static org.elasticsearch.xpack.esql.core.expression.function.Function castInvalidMappedField( org.elasticsearch.xpack.esql.core.expression.function.Function f ) { - if (f instanceof AbstractConvertFunction) { + if (f instanceof AbstractConvertFunction || f instanceof FullTextFunction) { return f; } List args = f.arguments(); @@ -1805,56 +1704,106 @@ public LogicalPlan apply(LogicalPlan logicalPlan, AnalyzerContext context) { return logicalPlan; } List projections = logicalPlan.collectFirstChildren(EsqlProject.class::isInstance); - if (projections.isEmpty()) { - List newProjections = new ArrayList<>(logicalPlan.output().size()); - List aliases = new ArrayList<>(logicalPlan.output().size()); - int counter = 0; - projectionLoop: for (Attribute e : logicalPlan.output()) { - if (e.resolved() - && e.dataType() == UNSUPPORTED - && e instanceof FieldAttribute fa - && fa.field() instanceof InvalidMappedField imf) { - // this is an invalid mapped field, find a common data type and cast to it - DataType targetType = null; - for (DataType type : imf.types()) { - if (targetType == null) { // initialize the target type to the first type - targetType = type; - } else { - targetType = EsqlDataTypeConverter.commonType(targetType, type); - if (targetType == null) { // if there is no common type, continue to the next argument - newProjections.add(e); - continue projectionLoop; - } - } - } - if (targetType != null && isRepresentable(targetType) && (isMillisOrNanos(targetType) || targetType.isNumeric())) { - // create an eval to cast union type to a common type - var name = rawTemporaryName("keep", String.valueOf(counter++), targetType.name()); - Alias alias; - switch (targetType) { - case INTEGER -> alias = new Alias(fa.source(), name, new ToInteger(fa.source(), fa)); - case LONG -> alias = new Alias(fa.source(), name, new ToLong(fa.source(), fa)); - case DOUBLE -> alias = new Alias(fa.source(), name, new ToDouble(fa.source(), fa)); - case UNSIGNED_LONG -> alias = new Alias(fa.source(), name, new ToUnsignedLong(fa.source(), fa)); - case DATETIME -> alias = new Alias(fa.source(), name, new ToDatetime(fa.source(), fa)); - case DATE_NANOS -> alias = new Alias(fa.source(), name, new ToDateNanos(fa.source(), fa)); - default -> throw new EsqlIllegalArgumentException("unexpected data type: " + targetType); - } - aliases.add(alias); - newProjections.add(alias); - } + return projections.isEmpty() ? castInvalidMappedFieldInLogicalPlan(logicalPlan) : logicalPlan; + } + } + + private static LogicalPlan castInvalidMappedFieldInLogicalPlan(LogicalPlan plan) { + List projections; + if (plan instanceof EsqlProject project) { + projections = project.projections(); + } else if (plan instanceof OrderBy ob) { + projections = ob.order(); + } else { + projections = plan.output(); + } + List newProjections = new ArrayList<>(projections.size()); + List aliases = new ArrayList<>(projections.size()); + projectionLoop: for (Expression projection : projections) { + Expression e = projection; + String alias = null; + if (projection instanceof Alias a) { + e = a.child(); + alias = a.name(); + } else if (projection instanceof Order o) { + e = o.child(); + } + if (e.resolved() + && e.dataType() == UNSUPPORTED + && e instanceof FieldAttribute fa + && fa.field() instanceof InvalidMappedField imf) { + // this is an invalid mapped field, find a common data type and cast to it + DataType targetType = null; + for (DataType type : imf.types()) { + if (targetType == null) { // initialize the target type to the first type + targetType = type; } else { - newProjections.add(e); + targetType = EsqlDataTypeConverter.commonType(targetType, type); + if (targetType == null) { // if there is no common type, continue to the next argument + newProjections.add(projection); + continue projectionLoop; + } } } - if (aliases.isEmpty() == false) { - Eval eval = new Eval(logicalPlan.source(), logicalPlan, aliases); - return new EsqlProject(logicalPlan.source(), eval, newProjections); - } else { - return logicalPlan; + if (targetType != null && isRepresentable(targetType) && (isMillisOrNanos(targetType) || targetType.isNumeric())) { + // create an eval to cast union type to a common type + String name = alias != null ? alias : fa.name(); + Expression newChild; + Source source = fa.source(); + switch (targetType) { + case INTEGER -> newChild = new ToInteger(source, fa); + case LONG -> newChild = new ToLong(source, fa); + case DOUBLE -> newChild = new ToDouble(source, fa); + case UNSIGNED_LONG -> newChild = new ToUnsignedLong(source, fa); + case DATETIME -> newChild = new ToDatetime(source, fa); + case DATE_NANOS -> newChild = new ToDateNanos(source, fa); + default -> throw new EsqlIllegalArgumentException("unexpected data type: " + targetType); + } + Alias newAlias = new Alias(source, name, newChild); + aliases.add(newAlias); + if (projection instanceof Alias a) { + newProjections.add(a.replaceChild(newAlias.toAttribute())); + } else if (projection instanceof Order o) { + newProjections.add(new Order(o.source(), newChild, o.direction(), o.nullsPosition())); + } else { + newProjections.add(newAlias.toAttribute()); + } } + } else { + newProjections.add(projection); } - return logicalPlan; + } + if (aliases.isEmpty() == false) { + if (plan instanceof EsqlProject p) { + Eval eval = new Eval(plan.source(), p.child(), aliases); + return new EsqlProject( + p.source(), + eval, + newProjections.stream() + .filter(e -> e instanceof NamedExpression) + .map(e -> (NamedExpression) e) + .collect(Collectors.toList()) + ); + } else if (plan instanceof OrderBy o) { + // Eval eval = new Eval(plan.source(), o.child(), aliases); + return new OrderBy( + o.source(), + o.child(), + newProjections.stream().filter(e -> e instanceof Order).map(e -> (Order) e).collect(Collectors.toList()) + ); + } else { + Eval eval = new Eval(plan.source(), plan, aliases); + return new EsqlProject( + plan.source(), + eval, + newProjections.stream() + .filter(e -> e instanceof NamedExpression) + .map(e -> (NamedExpression) e) + .collect(Collectors.toList()) + ); + } + } else { + return plan; } } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index a2bffb51413ed..9143cf5ddf1f4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -17,7 +17,6 @@ import org.elasticsearch.xpack.esql.core.capabilities.Unresolvable; import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Expression; -import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; import org.elasticsearch.xpack.esql.core.expression.function.Function; import org.elasticsearch.xpack.esql.core.expression.predicate.BinaryOperator; @@ -36,7 +35,6 @@ import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.esql.plan.logical.Lookup; import org.elasticsearch.xpack.esql.plan.logical.Project; -import org.elasticsearch.xpack.esql.plan.logical.local.EsqlProject; import org.elasticsearch.xpack.esql.telemetry.FeatureMetric; import org.elasticsearch.xpack.esql.telemetry.Metrics; @@ -96,16 +94,6 @@ Collection verify(LogicalPlan plan, BitSet partialMetrics) { return; } - if (p instanceof EsqlProject proj) { - for (NamedExpression projection : proj.projections()) { - // don't call dataType() - it will fail on UnresolvedAttribute - String className = String.valueOf(projection.getClass()); - if (projection.resolved() == false && projection instanceof UnsupportedAttribute == false) { - - } - } - } - planCheckers.forEach(c -> c.accept(p, failures)); checkOperationsOnUnsignedLong(p, failures); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverterTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverterTests.java index 9a30c2281d742..487a2ee111c03 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverterTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverterTests.java @@ -97,7 +97,7 @@ public void testCommonTypeDateTimeIntervals() { assertEqualsCommonType(dataType1, NULL, dataType1); } else if (isDateTimeOrNanosOrTemporal(dataType2)) { if ((dataType1 == DATE_NANOS && dataType2 == DATETIME) || (dataType1 == DATETIME && dataType2 == DATE_NANOS)) { - assertNullCommonType(dataType1, dataType2); + assertEqualsCommonType(dataType1, dataType2, DATE_NANOS); } else if (isDateTime(dataType1) || isDateTime(dataType2)) { assertEqualsCommonType(dataType1, dataType2, DATETIME); } else if (dataType1 == DATE_NANOS || dataType2 == DATE_NANOS) { From 08d395c5ec7b39235a99b8df7f357ec01a12c11f Mon Sep 17 00:00:00 2001 From: Fang Xing <155562079+fang-xing-esql@users.noreply.github.com> Date: Fri, 28 Feb 2025 00:07:40 -0500 Subject: [PATCH 03/24] Update docs/changelog/123678.yaml --- docs/changelog/123678.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docs/changelog/123678.yaml diff --git a/docs/changelog/123678.yaml b/docs/changelog/123678.yaml new file mode 100644 index 0000000000000..f32b0f966fb7c --- /dev/null +++ b/docs/changelog/123678.yaml @@ -0,0 +1,6 @@ +pr: 123678 +summary: Date nanos implicit casting +area: ES|QL +type: enhancement +issues: + - 110009 From 3552ae3f879146dd2c369c342b42378536d0220a Mon Sep 17 00:00:00 2001 From: Fang Xing Date: Thu, 6 Mar 2025 00:01:42 -0500 Subject: [PATCH 04/24] more implicit casting for union typed field in logical plan --- .../src/main/resources/union_types.csv-spec | 122 +++++++++++- .../xpack/esql/analysis/Analyzer.java | 188 ++++++++++++------ .../xpack/esql/plan/logical/Eval.java | 8 +- .../xpack/esql/plan/logical/MvExpand.java | 3 +- .../esql/plan/logical/join/JoinConfig.java | 4 +- 5 files changed, 257 insertions(+), 68 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec index 443a800d40a36..12a84ed6a9355 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec @@ -1640,7 +1640,22 @@ id:integer | name:keyword | count:long 14 | mmmmm | 2 ; -MultiTypedFieldsKeepDropSort +MultiTypedFieldsEvalKeepSort +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| Eval x = emp_no +| KEEP x, languages, hire_date, avg_worked_seconds +| SORT x, hire_date +| limit 2 +; + +x:long |languages:integer |hire_date:date_nanos |avg_worked_seconds:unsigned_long +10001 |2 |1986-06-26T00:00:00.000Z |268728049 +10001 |2 |1986-06-26T00:00:00.000Z |268728049 +; + +MultiTypedFieldsDropKeepSort required_capability: implicit_casting_union_typed_numeric_and_date FROM employees, employees_incompatible @@ -1655,13 +1670,13 @@ emp_no:long |languages:integer |hire_date:date_nanos |avg_worked_seconds:uns 10001 |2 |1986-06-26T00:00:00.000Z |268728049 ; -MultiTypedFieldsRenameKeep +MultiTypedFieldsRenameKeepSort required_capability: implicit_casting_union_typed_numeric_and_date from employees, employees_incompatible | RENAME emp_no as new_emp_no, languages as new_languages, hire_date as new_hire_date, avg_worked_seconds as new_avg_worked_seconds | KEEP new_emp_no, new_languages, new_hire_date, new_avg_worked_seconds -| sort new_emp_no, new_hire_date +| SORT new_emp_no, new_hire_date | limit 2 ; @@ -1670,7 +1685,7 @@ new_emp_no:long |new_languages:integer |new_hire_date:date_nanos |new_avg_worked 10001 |2 |1986-06-26T00:00:00.000Z |268728049 ; -MultiTypedFieldsEvalKeepSort +MultiTypedFieldsDropEvalKeepSort required_capability: implicit_casting_union_typed_numeric_and_date FROM employees, employees_incompatible @@ -1703,3 +1718,102 @@ emp_no:long |hire_date:date_nanos 10017 | 1993-08-03T00:00:00.000Z 10017 | 1993-08-03T00:00:00.000Z ; + +MultiTypedFieldsStatsByNumeric +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| STATS x=count(emp_no), y=max(hire_date), z=max(height) BY languages +| SORT languages +; + +x:long | y:date_nanos | z:double | languages:integer +30 | 1999-04-30T00:00:00.000Z | 2.06 | 1 +38 | 1995-01-27T00:00:00.000Z | 2.1 | 2 +34 | 1996-11-05T00:00:00.000Z | 2.1 | 3 +36 | 1995-03-13T00:00:00.000Z | 2.0 | 4 +42 | 1994-04-09T00:00:00.000Z | 2.1 | 5 +20 | 1997-05-19T00:00:00.000Z | 2.1 | null +; + +MultiTypedFieldsStatsByDateNanos +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| STATS x=count(emp_no), y=avg(salary_change), z=max(height) BY hire_date +| Eval y = round(y, 1), z = round(z, 1) +| KEEP x, y, z, hire_date +| SORT hire_date +| LIMIT 5 +; + +x:long | y:double | z:double | hire_date:date_nanos +2 | null | 1.9 | 1985-02-18T00:00:00.000Z +2 | null | 2.0 | 1985-02-24T00:00:00.000Z +2 | 3.3 | 2.0 | 1985-05-13T00:00:00.000Z +2 | 0.2 | 1.8 | 1985-07-09T00:00:00.000Z +2 | 3.6 | 1.5 | 1985-09-17T00:00:00.000Z +; + +MultiTypedFieldsWhereMvExpandKeepSortNumeric +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| WHERE emp_no == 10003 +| MV_EXPAND salary_change +| KEEP emp_no, salary_change +| EVAL salary_change = round(salary_change, 2) +| SORT salary_change +; + +emp_no:long | salary_change:double +10003 | 12.82 +10003 | 12.82 +10003 | 14.68 +10003 | 14.68 +; + +MultiTypedFieldsMvExpandKeepSortNumeric +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| MV_EXPAND salary_change +| KEEP emp_no, salary_change +| EVAL salary_change = round(salary_change, 2) +| SORT emp_no, salary_change +| LIMIT 10 +; + +emp_no:long | salary_change:double +10001 | 1.19 +10001 | 1.19 +10002 | -7.23 +10002 | -7.23 +10002 | 11.17 +10002 | 11.17 +10003 | 12.82 +10003 | 12.82 +10003 | 14.68 +10003 | 14.68 +; + +MultiTypedFieldsEvalLookupJoinNumeric +required_capability: join_lookup_v12 +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| EVAL language_code = languages +| LOOKUP JOIN languages_lookup ON language_code +| WHERE emp_no >= 10091 AND emp_no < 10094 +| KEEP emp_no, language_code, language_name +| SORT emp_no +; + +emp_no:long | language_code:integer | language_name:keyword +10091 | 3 | Spanish +10091 | 3 | Spanish +10092 | 1 | English +10092 | 1 | English +10093 | 3 | Spanish +10093 | 3 | Spanish +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 406a7fde5b37b..f45b5d35aa22a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -74,11 +74,13 @@ import org.elasticsearch.xpack.esql.parser.ParsingException; import org.elasticsearch.xpack.esql.plan.IndexPattern; import org.elasticsearch.xpack.esql.plan.logical.Aggregate; +import org.elasticsearch.xpack.esql.plan.logical.Dissect; import org.elasticsearch.xpack.esql.plan.logical.Drop; import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.plan.logical.EsRelation; import org.elasticsearch.xpack.esql.plan.logical.Eval; import org.elasticsearch.xpack.esql.plan.logical.Fork; +import org.elasticsearch.xpack.esql.plan.logical.Grok; import org.elasticsearch.xpack.esql.plan.logical.Insist; import org.elasticsearch.xpack.esql.plan.logical.Keep; import org.elasticsearch.xpack.esql.plan.logical.Limit; @@ -87,6 +89,7 @@ import org.elasticsearch.xpack.esql.plan.logical.MvExpand; import org.elasticsearch.xpack.esql.plan.logical.OrderBy; import org.elasticsearch.xpack.esql.plan.logical.Project; +import org.elasticsearch.xpack.esql.plan.logical.RegexExtract; import org.elasticsearch.xpack.esql.plan.logical.Rename; import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation; import org.elasticsearch.xpack.esql.plan.logical.join.Join; @@ -1202,8 +1205,14 @@ private static class ImplicitCasting extends ParameterizedRule { - if (p instanceof OrderBy | p instanceof EsqlProject) { - return castInvalidMappedFieldInLogicalPlan(p); + if (p instanceof OrderBy + || p instanceof EsqlProject + || p instanceof Aggregate + || p instanceof RegexExtract + || p instanceof MvExpand + || p instanceof Eval + || p instanceof LookupJoin) { + return castInvalidMappedFieldInLogicalPlan(p, false); } return p; }); @@ -1216,7 +1225,7 @@ public LogicalPlan apply(LogicalPlan plan, AnalyzerContext context) { private static Expression cast(org.elasticsearch.xpack.esql.core.expression.function.Function f, EsqlFunctionRegistry registry) { // Add cast functions to InvalidMappedField if there isn't one yet - f = castInvalidMappedField(f); + f = castInvalidMappedFieldInFunction(f); if (f instanceof In in) { return processIn(in); @@ -1230,7 +1239,7 @@ private static Expression cast(org.elasticsearch.xpack.esql.core.expression.func return f; } - private static org.elasticsearch.xpack.esql.core.expression.function.Function castInvalidMappedField( + private static org.elasticsearch.xpack.esql.core.expression.function.Function castInvalidMappedFieldInFunction( org.elasticsearch.xpack.esql.core.expression.function.Function f ) { if (f instanceof AbstractConvertFunction || f instanceof FullTextFunction) { @@ -1261,15 +1270,8 @@ private static org.elasticsearch.xpack.esql.core.expression.function.Function ca } if (targetType != null && isRepresentable(targetType) && (isMillisOrNanos(targetType) || targetType.isNumeric())) { childrenChanged = true; - switch (targetType) { - case INTEGER -> newChildren.add(new ToInteger(arg.source(), arg)); - case LONG -> newChildren.add(new ToLong(arg.source(), arg)); - case DOUBLE -> newChildren.add(new ToDouble(arg.source(), arg)); - case UNSIGNED_LONG -> newChildren.add(new ToUnsignedLong(arg.source(), arg)); - case DATETIME -> newChildren.add(new ToDatetime(arg.source(), arg)); - case DATE_NANOS -> newChildren.add(new ToDateNanos(arg.source(), arg)); - default -> throw new EsqlIllegalArgumentException("unexpected data type: " + targetType); - } + Expression newChild = castInvalidMappedField(targetType, fa); + newChildren.add(newChild); } } else { newChildren.add(arg); @@ -1704,28 +1706,35 @@ public LogicalPlan apply(LogicalPlan logicalPlan, AnalyzerContext context) { return logicalPlan; } List projections = logicalPlan.collectFirstChildren(EsqlProject.class::isInstance); - return projections.isEmpty() ? castInvalidMappedFieldInLogicalPlan(logicalPlan) : logicalPlan; + return projections.isEmpty() ? castInvalidMappedFieldInLogicalPlan(logicalPlan, true) : logicalPlan; } } - private static LogicalPlan castInvalidMappedFieldInLogicalPlan(LogicalPlan plan) { - List projections; - if (plan instanceof EsqlProject project) { - projections = project.projections(); - } else if (plan instanceof OrderBy ob) { - projections = ob.order(); + private static LogicalPlan castInvalidMappedFieldInLogicalPlan(LogicalPlan plan, boolean addProject) { + List fields; + if (addProject) { + fields = plan.output(); } else { - projections = plan.output(); - } - List newProjections = new ArrayList<>(projections.size()); - List aliases = new ArrayList<>(projections.size()); - projectionLoop: for (Expression projection : projections) { - Expression e = projection; + fields = switch (plan) { + case EsqlProject project -> project.projections(); + case OrderBy ob -> ob.order(); + case Aggregate agg -> agg.groupings(); + case RegexExtract re -> List.of(re.input()); + case MvExpand me -> List.of(me.target()); + case Eval e -> e.fields(); + case LookupJoin lj -> lj.config().leftFields(); + default -> plan.output(); + }; + } + List newProjections = new ArrayList<>(fields.size()); + List aliases = new ArrayList<>(fields.size()); + projectionLoop: for (Expression field : fields) { + Expression e = field; String alias = null; - if (projection instanceof Alias a) { + if (field instanceof Alias a) { e = a.child(); alias = a.name(); - } else if (projection instanceof Order o) { + } else if (field instanceof Order o) { e = o.child(); } if (e.resolved() @@ -1740,7 +1749,7 @@ private static LogicalPlan castInvalidMappedFieldInLogicalPlan(LogicalPlan plan) } else { targetType = EsqlDataTypeConverter.commonType(targetType, type); if (targetType == null) { // if there is no common type, continue to the next argument - newProjections.add(projection); + newProjections.add(field); continue projectionLoop; } } @@ -1748,50 +1757,24 @@ private static LogicalPlan castInvalidMappedFieldInLogicalPlan(LogicalPlan plan) if (targetType != null && isRepresentable(targetType) && (isMillisOrNanos(targetType) || targetType.isNumeric())) { // create an eval to cast union type to a common type String name = alias != null ? alias : fa.name(); - Expression newChild; Source source = fa.source(); - switch (targetType) { - case INTEGER -> newChild = new ToInteger(source, fa); - case LONG -> newChild = new ToLong(source, fa); - case DOUBLE -> newChild = new ToDouble(source, fa); - case UNSIGNED_LONG -> newChild = new ToUnsignedLong(source, fa); - case DATETIME -> newChild = new ToDatetime(source, fa); - case DATE_NANOS -> newChild = new ToDateNanos(source, fa); - default -> throw new EsqlIllegalArgumentException("unexpected data type: " + targetType); - } + Expression newChild = castInvalidMappedField(targetType, fa); Alias newAlias = new Alias(source, name, newChild); aliases.add(newAlias); - if (projection instanceof Alias a) { + if (field instanceof Alias a) { newProjections.add(a.replaceChild(newAlias.toAttribute())); - } else if (projection instanceof Order o) { + } else if (field instanceof Order o) { newProjections.add(new Order(o.source(), newChild, o.direction(), o.nullsPosition())); } else { newProjections.add(newAlias.toAttribute()); } } } else { - newProjections.add(projection); + newProjections.add(field); } } if (aliases.isEmpty() == false) { - if (plan instanceof EsqlProject p) { - Eval eval = new Eval(plan.source(), p.child(), aliases); - return new EsqlProject( - p.source(), - eval, - newProjections.stream() - .filter(e -> e instanceof NamedExpression) - .map(e -> (NamedExpression) e) - .collect(Collectors.toList()) - ); - } else if (plan instanceof OrderBy o) { - // Eval eval = new Eval(plan.source(), o.child(), aliases); - return new OrderBy( - o.source(), - o.child(), - newProjections.stream().filter(e -> e instanceof Order).map(e -> (Order) e).collect(Collectors.toList()) - ); - } else { + if (addProject) { Eval eval = new Eval(plan.source(), plan, aliases); return new EsqlProject( plan.source(), @@ -1802,8 +1785,93 @@ private static LogicalPlan castInvalidMappedFieldInLogicalPlan(LogicalPlan plan) .collect(Collectors.toList()) ); } + switch (plan) { + case EsqlProject p -> { + Eval eval = new Eval(plan.source(), p.child(), aliases); + return new EsqlProject( + p.source(), + eval, + newProjections.stream() + .filter(e -> e instanceof NamedExpression) + .map(e -> (NamedExpression) e) + .collect(Collectors.toList()) + ); + } + case OrderBy o -> { + // Eval eval = new Eval(plan.source(), o.child(), aliases); + return new OrderBy( + o.source(), + o.child(), + newProjections.stream().filter(e -> e instanceof Order).map(e -> (Order) e).collect(Collectors.toList()) + ); + } + case Aggregate agg -> { + // both groupings and aggregates need to be replaced + Eval eval = new Eval(plan.source(), agg.child(), aliases); + // create new aggregates according to new groupings + List origAggs = agg.aggregates(); + List newAggs = new ArrayList<>(origAggs.size()); + for (int i = 0; i < origAggs.size() - newProjections.size(); i++) { // add aggregate functions + newAggs.add(origAggs.get(i)); + } + for (Expression e : newProjections) { // add new groupings + newAggs.add(Expressions.attribute(e)); + } + return new Aggregate(agg.source(), eval, agg.aggregateType(), newProjections, newAggs); + } + case Dissect d -> { + Eval eval = new Eval(plan.source(), d.child(), aliases); + return new Dissect(plan.source(), eval, newProjections.get(0), d.parser(), d.extractedFields()); + } + case Grok g -> { + Eval eval = new Eval(plan.source(), g.child(), aliases); + return new Grok(plan.source(), eval, newProjections.get(0), g.parser(), g.extractedFields()); + } + case MvExpand me -> { + Eval eval = new Eval(plan.source(), me.child(), aliases); + NamedExpression newTarget = Expressions.attribute(newProjections.get(0)); + return new MvExpand(plan.source(), eval, newTarget, newTarget.toAttribute()); + } + case Eval ev -> { + return new Eval(plan.source(), ev.child(), aliases); + } + case LookupJoin lj -> { + Eval eval = new Eval(plan.source(), lj.left(), aliases); + JoinConfig oldJoinConfig = lj.config(); + List leftKeys = newProjections.stream() + .filter(e -> e instanceof Attribute) + .map(e -> (Attribute) e) + .collect(Collectors.toList()); + JoinConfig newJoinConfig = new JoinConfig(oldJoinConfig.type(), leftKeys, leftKeys, oldJoinConfig.rightFields()); + return new LookupJoin(lj.source(), eval, lj.right(), newJoinConfig); + } + default -> { + Eval eval = new Eval(plan.source(), plan, aliases); + return new EsqlProject( + plan.source(), + eval, + newProjections.stream() + .filter(e -> e instanceof NamedExpression) + .map(e -> (NamedExpression) e) + .collect(Collectors.toList()) + ); + } + } } else { return plan; } } + + private static Expression castInvalidMappedField(DataType targetType, FieldAttribute fa) { + Source source = fa.source(); + return switch (targetType) { + case INTEGER -> new ToInteger(source, fa); + case LONG -> new ToLong(source, fa); + case DOUBLE -> new ToDouble(source, fa); + case UNSIGNED_LONG -> new ToUnsignedLong(source, fa); + case DATETIME -> new ToDatetime(source, fa); + case DATE_NANOS -> new ToDateNanos(source, fa); + default -> throw new EsqlIllegalArgumentException("unexpected data type: " + targetType); + }; + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Eval.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Eval.java index af81e26d57c60..11221ff32145c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Eval.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Eval.java @@ -13,7 +13,6 @@ import org.elasticsearch.xpack.esql.capabilities.PostAnalysisVerificationAware; import org.elasticsearch.xpack.esql.capabilities.TelemetryAware; import org.elasticsearch.xpack.esql.common.Failures; -import org.elasticsearch.xpack.esql.core.capabilities.Resolvables; import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.AttributeMap; @@ -134,7 +133,12 @@ private List renameAliases(List originalAttributes, List n @Override public boolean expressionsResolved() { - return Resolvables.resolved(fields); + for (Alias a : fields) { + if (a.resolved() == false || a.dataType() == DataType.UNSUPPORTED) { + return false; + } + } + return true; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/MvExpand.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/MvExpand.java index f65811fc26526..52941018ffae8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/MvExpand.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/MvExpand.java @@ -16,6 +16,7 @@ import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import java.io.IOException; @@ -90,7 +91,7 @@ public String telemetryLabel() { @Override public boolean expressionsResolved() { - return target.resolved(); + return target.resolved() && target.dataType() != DataType.UNSUPPORTED; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/JoinConfig.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/JoinConfig.java index 383606d6ccbed..d9c5a9fc2d063 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/JoinConfig.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/JoinConfig.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.xpack.esql.core.capabilities.Resolvables; import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.type.DataType; import java.io.IOException; import java.util.List; @@ -50,6 +51,7 @@ public boolean expressionsResolved() { return type.resolved() && Resolvables.resolved(matchFields) && Resolvables.resolved(leftFields) - && Resolvables.resolved(rightFields); + && Resolvables.resolved(rightFields) + && leftFields.stream().noneMatch(e -> e.dataType() == DataType.UNSUPPORTED); } } From 29f7afe1ec01720af0119f419e556ac4ea265fe0 Mon Sep 17 00:00:00 2001 From: Fang Xing Date: Tue, 18 Mar 2025 00:14:17 -0400 Subject: [PATCH 05/24] refactor --- .../xpack/esql/analysis/Analyzer.java | 405 +++++++++--------- 1 file changed, 203 insertions(+), 202 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 132db5391acb5..e23753c544082 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.logging.HeaderWarning; import org.elasticsearch.compute.data.Block; import org.elasticsearch.core.Strings; +import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexMode; import org.elasticsearch.logging.Logger; import org.elasticsearch.xpack.core.enrich.EnrichPolicy; @@ -187,12 +188,7 @@ public class Analyzer extends ParameterizedRuleExecutor( @@ -1261,7 +1257,7 @@ private BitSet gatherPreAnalysisMetrics(LogicalPlan plan, BitSet b) { private static class ImplicitCasting extends ParameterizedRule { @Override public LogicalPlan apply(LogicalPlan plan, AnalyzerContext context) { - // do implicit casting for union typed fields in sort, keep, rename + // Do implicit casting for union typed fields in the following commands LogicalPlan newPlan = plan.transformUp(p -> { if (p instanceof OrderBy || p instanceof EsqlProject @@ -1274,6 +1270,11 @@ public LogicalPlan apply(LogicalPlan plan, AnalyzerContext context) { } return p; }); + + // Add an implicit EsqlProject on top of the whole query if there isn't one yet, + // without explicit or implicit casting, a union typed field is returned as a null. + newPlan = addImplicitProjectForInvalidMappedFields(newPlan); + // do implicit casting for function arguments return newPlan.transformExpressionsUp( org.elasticsearch.xpack.esql.core.expression.function.Function.class, @@ -1297,45 +1298,221 @@ private static Expression cast(org.elasticsearch.xpack.esql.core.expression.func return f; } + /** + * Both the logical plan's children and the logical plan itself can be changed, so we cannot just to a replaceChild here. + */ + private static LogicalPlan castInvalidMappedFieldInLogicalPlan(LogicalPlan plan, boolean addProject) { + List originalExpressions; + if (addProject) { + originalExpressions = plan.output(); + } else { + originalExpressions = switch (plan) { + case EsqlProject project -> project.projections(); + case OrderBy ob -> ob.order(); + case Aggregate agg -> agg.groupings(); + case RegexExtract re -> List.of(re.input()); + case MvExpand me -> List.of(me.target()); + case Eval e -> e.fields(); + case LookupJoin lj -> lj.config().leftFields(); + // The other types of plans are unexpected + default -> throw new EsqlIllegalArgumentException("unexpected logical plan: " + plan); + }; + } + Tuple, List> newChildren = castInvalidMappedFields(originalExpressions, true); + List aliases = newChildren.v1(); + List newProjections = newChildren.v2(); + if (aliases.isEmpty() == false) { + if (addProject) { + return esqlProjectForInvalidMappedField(plan.source(), plan, aliases, newProjections); + } + switch (plan) { + case EsqlProject p -> { + return esqlProjectForInvalidMappedField(p.source(), p.child(), aliases, newProjections); + } + case OrderBy o -> { + return new OrderBy( + o.source(), + o.child(), + newProjections.stream().filter(e -> e instanceof Order).map(e -> (Order) e).collect(Collectors.toList()) + ); + } + case Aggregate agg -> { + // both groupings and aggregates need to be replaced + // create new aggregates according to new groupings + List origAggs = agg.aggregates(); + List newAggs = new ArrayList<>(origAggs.size()); + for (int i = 0; i < origAggs.size() - newProjections.size(); i++) { // add aggregate functions + newAggs.add(origAggs.get(i)); + } + for (Expression e : newProjections) { // add new groupings + newAggs.add(Expressions.attribute(e)); + } + return new Aggregate( + agg.source(), + evalForInvalidMappedField(agg.source(), agg.child(), aliases), + agg.aggregateType(), + newProjections, + newAggs + ); + } + case Dissect d -> { + return new Dissect( + plan.source(), + evalForInvalidMappedField(d.source(), d.child(), aliases), + newProjections.get(0), + d.parser(), + d.extractedFields() + ); + } + case Grok g -> { + return new Grok( + plan.source(), + evalForInvalidMappedField(g.source(), g.child(), aliases), + newProjections.get(0), + g.parser(), + g.extractedFields() + ); + } + case MvExpand mve -> { + NamedExpression newTarget = Expressions.attribute(newProjections.get(0)); + return new MvExpand( + plan.source(), + evalForInvalidMappedField(mve.source(), mve.child(), aliases), + newTarget, + newTarget.toAttribute() + ); + } + case Eval ev -> { + return evalForInvalidMappedField(ev.source(), ev.child(), aliases); + } + case LookupJoin lj -> { + JoinConfig oldJoinConfig = lj.config(); + List leftKeys = newProjections.stream() + .filter(e -> e instanceof Attribute) + .map(e -> (Attribute) e) + .collect(Collectors.toList()); + JoinConfig newJoinConfig = new JoinConfig(oldJoinConfig.type(), leftKeys, leftKeys, oldJoinConfig.rightFields()); + return new LookupJoin( + lj.source(), + evalForInvalidMappedField(lj.source(), lj.left(), aliases), + lj.right(), + newJoinConfig + ); + } + // The other types of plans are unexpected + default -> throw new EsqlIllegalArgumentException("unexpected logical plan: " + plan); + } + } else { + return plan; + } + } + + private static LogicalPlan addImplicitProjectForInvalidMappedFields(LogicalPlan logicalPlan) { + if (logicalPlan.resolved() == false) { + return logicalPlan; + } + List projections = logicalPlan.collectFirstChildren(EsqlProject.class::isInstance); + return projections.isEmpty() ? castInvalidMappedFieldInLogicalPlan(logicalPlan, true) : logicalPlan; + } + private static org.elasticsearch.xpack.esql.core.expression.function.Function castInvalidMappedFieldInFunction( org.elasticsearch.xpack.esql.core.expression.function.Function f ) { + // No need to add implicit casting for union typed fields that already have explicit casting on them. + // Full text functions are pushdown only functions, implicit or explicit casting may fail the query. if (f instanceof AbstractConvertFunction || f instanceof FullTextFunction) { return f; } - List args = f.arguments(); - List newChildren = new ArrayList<>(f.children().size()); - boolean childrenChanged = false; - Expression arg; - argLoop: for (Expression expression : args) { - DataType targetType = null; - arg = expression; - if (arg.resolved() - && arg.dataType() == UNSUPPORTED - && arg instanceof FieldAttribute fa + Tuple, List> newChildren = castInvalidMappedFields(f.arguments(), false); + return newChildren.v1().isEmpty() + ? f + : (org.elasticsearch.xpack.esql.core.expression.function.Function) f.replaceChildren(newChildren.v2()); + } + + private static Tuple, List> castInvalidMappedFields( + List originalExpressions, + boolean createNewChildPlan + ) { + List newAliases = new ArrayList<>(originalExpressions.size()); + List newExpressions = new ArrayList<>(originalExpressions.size()); + expressionLoop: for (Expression exp : originalExpressions) { + Expression e = Alias.unwrap(exp); + e = e instanceof Order o ? o.child() : e; + String alias = exp instanceof Alias a ? a.name() : null; + if (e.resolved() + && e.dataType() == UNSUPPORTED + && e instanceof FieldAttribute fa && fa.field() instanceof InvalidMappedField imf) { - // this is an invalid mapped field, find a common data type and cast to it + // This is an invalid mapped field, find a common data type and cast to it. + DataType targetType = null; for (DataType type : imf.types()) { - if (targetType == null) { // initialize the target type to the first type + if (targetType == null) { // Initialize the target type to the first type. targetType = type; } else { targetType = EsqlDataTypeConverter.commonType(targetType, type); - if (targetType == null) { // if there is no common type, continue to the next argument - newChildren.add(arg); - continue argLoop; + if (targetType == null) { // If there is no common type, continue to the next expression. + newExpressions.add(exp); + continue expressionLoop; } } } if (targetType != null && isRepresentable(targetType) && (isMillisOrNanos(targetType) || targetType.isNumeric())) { - childrenChanged = true; + alias = alias != null ? alias : fa.name(); + Source source = fa.source(); Expression newChild = castInvalidMappedField(targetType, fa); - newChildren.add(newChild); + Alias newAlias = new Alias(source, alias, newChild); + newAliases.add(newAlias); + if (createNewChildPlan) { + // Cast union typed fields in a logical plan, a new child plan(Eval) is needed for the implicit casting and new + // aliases. The newExpressions with all the fields and references are need to create a new plan. + switch (exp) { + case Alias a -> newExpressions.add(a.replaceChild(newAlias.toAttribute())); + case Order o -> newExpressions.add(new Order(o.source(), newChild, o.direction(), o.nullsPosition())); + default -> newExpressions.add(newAlias.toAttribute()); + } + } else { // Cast union typed fields in a function, there is no need to create a new child plan + newExpressions.add(newChild); + } } } else { - newChildren.add(arg); + newExpressions.add(exp); } } - return childrenChanged ? (org.elasticsearch.xpack.esql.core.expression.function.Function) f.replaceChildren(newChildren) : f; + return Tuple.tuple(newAliases, newExpressions); + } + + private static EsqlProject esqlProjectForInvalidMappedField( + Source source, + LogicalPlan childPlan, + List aliases, + List newProjections + ) { + Eval eval = evalForInvalidMappedField(source, childPlan, aliases); + return new EsqlProject( + source, + eval, + newProjections.stream().filter(e -> e instanceof NamedExpression).map(e -> (NamedExpression) e).collect(Collectors.toList()) + ); + } + + private static Eval evalForInvalidMappedField(Source source, LogicalPlan childPlan, List aliases) { + return new Eval(source, childPlan, aliases); + } + + /** + * Do implicit casting for data, date_nanos and numeric types only + */ + private static Expression castInvalidMappedField(DataType targetType, FieldAttribute fa) { + Source source = fa.source(); + return switch (targetType) { + case INTEGER -> new ToInteger(source, fa); + case LONG -> new ToLong(source, fa); + case DOUBLE -> new ToDouble(source, fa); + case UNSIGNED_LONG -> new ToUnsignedLong(source, fa); + case DATETIME -> new ToDatetime(source, fa); + case DATE_NANOS -> new ToDateNanos(source, fa); + default -> throw new EsqlIllegalArgumentException("unexpected data type: " + targetType); + }; } private static Expression processScalarOrGroupingFunction( @@ -1756,180 +1933,4 @@ private static LogicalPlan planWithoutSyntheticAttributes(LogicalPlan plan) { return newOutput.size() == output.size() ? plan : new Project(Source.EMPTY, plan, newOutput); } } - - private static class AddImplicitProject extends ParameterizedRule { - @Override - public LogicalPlan apply(LogicalPlan logicalPlan, AnalyzerContext context) { - if (logicalPlan.resolved() == false) { - return logicalPlan; - } - List projections = logicalPlan.collectFirstChildren(EsqlProject.class::isInstance); - return projections.isEmpty() ? castInvalidMappedFieldInLogicalPlan(logicalPlan, true) : logicalPlan; - } - } - - private static LogicalPlan castInvalidMappedFieldInLogicalPlan(LogicalPlan plan, boolean addProject) { - List fields; - if (addProject) { - fields = plan.output(); - } else { - fields = switch (plan) { - case EsqlProject project -> project.projections(); - case OrderBy ob -> ob.order(); - case Aggregate agg -> agg.groupings(); - case RegexExtract re -> List.of(re.input()); - case MvExpand me -> List.of(me.target()); - case Eval e -> e.fields(); - case LookupJoin lj -> lj.config().leftFields(); - default -> plan.output(); - }; - } - List newProjections = new ArrayList<>(fields.size()); - List aliases = new ArrayList<>(fields.size()); - projectionLoop: for (Expression field : fields) { - Expression e = field; - String alias = null; - if (field instanceof Alias a) { - e = a.child(); - alias = a.name(); - } else if (field instanceof Order o) { - e = o.child(); - } - if (e.resolved() - && e.dataType() == UNSUPPORTED - && e instanceof FieldAttribute fa - && fa.field() instanceof InvalidMappedField imf) { - // this is an invalid mapped field, find a common data type and cast to it - DataType targetType = null; - for (DataType type : imf.types()) { - if (targetType == null) { // initialize the target type to the first type - targetType = type; - } else { - targetType = EsqlDataTypeConverter.commonType(targetType, type); - if (targetType == null) { // if there is no common type, continue to the next argument - newProjections.add(field); - continue projectionLoop; - } - } - } - if (targetType != null && isRepresentable(targetType) && (isMillisOrNanos(targetType) || targetType.isNumeric())) { - // create an eval to cast union type to a common type - String name = alias != null ? alias : fa.name(); - Source source = fa.source(); - Expression newChild = castInvalidMappedField(targetType, fa); - Alias newAlias = new Alias(source, name, newChild); - aliases.add(newAlias); - if (field instanceof Alias a) { - newProjections.add(a.replaceChild(newAlias.toAttribute())); - } else if (field instanceof Order o) { - newProjections.add(new Order(o.source(), newChild, o.direction(), o.nullsPosition())); - } else { - newProjections.add(newAlias.toAttribute()); - } - } - } else { - newProjections.add(field); - } - } - if (aliases.isEmpty() == false) { - if (addProject) { - Eval eval = new Eval(plan.source(), plan, aliases); - return new EsqlProject( - plan.source(), - eval, - newProjections.stream() - .filter(e -> e instanceof NamedExpression) - .map(e -> (NamedExpression) e) - .collect(Collectors.toList()) - ); - } - switch (plan) { - case EsqlProject p -> { - Eval eval = new Eval(plan.source(), p.child(), aliases); - return new EsqlProject( - p.source(), - eval, - newProjections.stream() - .filter(e -> e instanceof NamedExpression) - .map(e -> (NamedExpression) e) - .collect(Collectors.toList()) - ); - } - case OrderBy o -> { - // Eval eval = new Eval(plan.source(), o.child(), aliases); - return new OrderBy( - o.source(), - o.child(), - newProjections.stream().filter(e -> e instanceof Order).map(e -> (Order) e).collect(Collectors.toList()) - ); - } - case Aggregate agg -> { - // both groupings and aggregates need to be replaced - Eval eval = new Eval(plan.source(), agg.child(), aliases); - // create new aggregates according to new groupings - List origAggs = agg.aggregates(); - List newAggs = new ArrayList<>(origAggs.size()); - for (int i = 0; i < origAggs.size() - newProjections.size(); i++) { // add aggregate functions - newAggs.add(origAggs.get(i)); - } - for (Expression e : newProjections) { // add new groupings - newAggs.add(Expressions.attribute(e)); - } - return new Aggregate(agg.source(), eval, agg.aggregateType(), newProjections, newAggs); - } - case Dissect d -> { - Eval eval = new Eval(plan.source(), d.child(), aliases); - return new Dissect(plan.source(), eval, newProjections.get(0), d.parser(), d.extractedFields()); - } - case Grok g -> { - Eval eval = new Eval(plan.source(), g.child(), aliases); - return new Grok(plan.source(), eval, newProjections.get(0), g.parser(), g.extractedFields()); - } - case MvExpand me -> { - Eval eval = new Eval(plan.source(), me.child(), aliases); - NamedExpression newTarget = Expressions.attribute(newProjections.get(0)); - return new MvExpand(plan.source(), eval, newTarget, newTarget.toAttribute()); - } - case Eval ev -> { - return new Eval(plan.source(), ev.child(), aliases); - } - case LookupJoin lj -> { - Eval eval = new Eval(plan.source(), lj.left(), aliases); - JoinConfig oldJoinConfig = lj.config(); - List leftKeys = newProjections.stream() - .filter(e -> e instanceof Attribute) - .map(e -> (Attribute) e) - .collect(Collectors.toList()); - JoinConfig newJoinConfig = new JoinConfig(oldJoinConfig.type(), leftKeys, leftKeys, oldJoinConfig.rightFields()); - return new LookupJoin(lj.source(), eval, lj.right(), newJoinConfig); - } - default -> { - Eval eval = new Eval(plan.source(), plan, aliases); - return new EsqlProject( - plan.source(), - eval, - newProjections.stream() - .filter(e -> e instanceof NamedExpression) - .map(e -> (NamedExpression) e) - .collect(Collectors.toList()) - ); - } - } - } else { - return plan; - } - } - - private static Expression castInvalidMappedField(DataType targetType, FieldAttribute fa) { - Source source = fa.source(); - return switch (targetType) { - case INTEGER -> new ToInteger(source, fa); - case LONG -> new ToLong(source, fa); - case DOUBLE -> new ToDouble(source, fa); - case UNSIGNED_LONG -> new ToUnsignedLong(source, fa); - case DATETIME -> new ToDatetime(source, fa); - case DATE_NANOS -> new ToDateNanos(source, fa); - default -> throw new EsqlIllegalArgumentException("unexpected data type: " + targetType); - }; - } } From c13e83ce7952bb3b158d6880c822d160434d32ac Mon Sep 17 00:00:00 2001 From: Fang Xing Date: Wed, 19 Mar 2025 07:48:38 -0400 Subject: [PATCH 06/24] support aggregations with bucket and more tests --- .../src/main/resources/union_types.csv-spec | 165 ++++++++++++++++++ .../xpack/esql/analysis/Analyzer.java | 37 ++-- 2 files changed, 191 insertions(+), 11 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec index 12a84ed6a9355..e691e7e9604d2 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec @@ -1817,3 +1817,168 @@ emp_no:long | language_code:integer | language_name:keyword 10093 | 3 | Spanish 10093 | 3 | Spanish ; + +MultiTypedMVFields +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| WHERE emp_no <= 10002 +| EVAL x = ROUND(MV_MAX(salary_change),2) +| KEEP emp_no, x +| SORT emp_no +| LIMIT 4 +; + +emp_no:long | x:double +10001 | 1.19 +10001 | 1.19 +10002 | 11.17 +10002 | 11.17 +; + +MultiTypedMVFieldsWhere +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| WHERE emp_no <= 10002 AND ROUND(MV_MAX(salary_change),2) > 10 +| KEEP emp_no +; + +emp_no:long +10002 +10002 +; + +MultiTypedMVFieldsStatsMaxMin +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| WHERE emp_no <= 10002 +| STATS max = MAX(salary_change), min = MIN(salary_change) +| EVAL x = round(max, 2), y = round(min, 2) +| KEEP x, y +; + +x:double |y:double +11.17 |-7.23 +; + +MultiTypedMVFieldsStatsValues +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| WHERE emp_no <= 10002 +| STATS c = MV_COUNT(VALUES(salary_change)) +; + +c:integerq +6 +; + +MultiTypedDateDiff +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| WHERE emp_no <= 10003 +| EVAL diff_sec_1 = DATE_DIFF("seconds", TO_DATE_NANOS("1986-01-23T12:15:03.360103847Z"), hire_date) +| EVAL diff_sec_2 = DATE_DIFF("seconds", TO_DATETIME("1986-01-23T12:15:03.360103847Z"), hire_date) +| KEEP emp_no, hire_date, diff_sec_1, diff_sec_2 +| SORT emp_no, hire_date +; + +emp_no:long | hire_date:date_nanos | diff_sec_1:integer | diff_sec_2:integer +10001 | 1986-06-26T00:00:00.000Z | 13261496 | 13261496 +10001 | 1986-06-26T00:00:00.000Z | 13261496 | 13261496 +10002 | 1985-11-21T00:00:00.000Z | -5487303 | -5487303 +10002 | 1985-11-21T00:00:00.000Z | -5487303 | -5487303 +10003 | 1986-08-28T00:00:00.000Z | 18704696 | 18704696 +10003 | 1986-08-28T00:00:00.000Z | 18704696 | 18704696 +; + +MultiTypedDateFormat +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| WHERE emp_no <= 10003 +| EVAL a = DATE_FORMAT(hire_date), b = DATE_FORMAT("yyyy-MM-dd", hire_date), c = DATE_FORMAT("strict_date_optional_time_nanos", hire_date) +| KEEP emp_no, hire_date, a, b, c +| SORT emp_no, hire_date +; + +emp_no:long | hire_date:date_nanos | a:keyword | b:keyword | c:keyword +10001 | 1986-06-26T00:00:00.000Z | 1986-06-26T00:00:00.000Z | 1986-06-26 | 1986-06-26T00:00:00.000Z +10001 | 1986-06-26T00:00:00.000Z | 1986-06-26T00:00:00.000Z | 1986-06-26 | 1986-06-26T00:00:00.000Z +10002 | 1985-11-21T00:00:00.000Z | 1985-11-21T00:00:00.000Z | 1985-11-21 | 1985-11-21T00:00:00.000Z +10002 | 1985-11-21T00:00:00.000Z | 1985-11-21T00:00:00.000Z | 1985-11-21 | 1985-11-21T00:00:00.000Z +10003 | 1986-08-28T00:00:00.000Z | 1986-08-28T00:00:00.000Z | 1986-08-28 | 1986-08-28T00:00:00.000Z +10003 | 1986-08-28T00:00:00.000Z | 1986-08-28T00:00:00.000Z | 1986-08-28 | 1986-08-28T00:00:00.000Z +; + +MultiTypedDateTrunc +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| WHERE emp_no <= 10003 +| EVAL yr = DATE_TRUNC(1 year, hire_date), mo = DATE_TRUNC(1 month, hire_date) +| SORT hire_date DESC +| KEEP emp_no, hire_date, yr, mo +; + +emp_no:long | hire_date:date_nanos | yr:date_nanos | mo:date_nanos +10003 | 1986-08-28T00:00:00.000Z | 1986-01-01T00:00:00.000Z | 1986-08-01T00:00:00.000Z +10003 | 1986-08-28T00:00:00.000Z | 1986-01-01T00:00:00.000Z | 1986-08-01T00:00:00.000Z +10001 | 1986-06-26T00:00:00.000Z | 1986-01-01T00:00:00.000Z | 1986-06-01T00:00:00.000Z +10001 | 1986-06-26T00:00:00.000Z | 1986-01-01T00:00:00.000Z | 1986-06-01T00:00:00.000Z +10002 | 1985-11-21T00:00:00.000Z | 1985-01-01T00:00:00.000Z | 1985-11-01T00:00:00.000Z +10002 | 1985-11-21T00:00:00.000Z | 1985-01-01T00:00:00.000Z | 1985-11-01T00:00:00.000Z +; + +MultiTypedBucketDateNanosByYear +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| STATS c = count(*) BY yr = BUCKET(hire_date, 1 year) +| SORT yr DESC, c +; + +c:long | yr:date_nanos +2 | 1999-01-01T00:00:00.000Z +2 | 1997-01-01T00:00:00.000Z +2 | 1996-01-01T00:00:00.000Z +10 | 1995-01-01T00:00:00.000Z +8 | 1994-01-01T00:00:00.000Z +6 | 1993-01-01T00:00:00.000Z +16 | 1992-01-01T00:00:00.000Z +12 | 1991-01-01T00:00:00.000Z +24 | 1990-01-01T00:00:00.000Z +26 | 1989-01-01T00:00:00.000Z +18 | 1988-01-01T00:00:00.000Z +30 | 1987-01-01T00:00:00.000Z +22 | 1986-01-01T00:00:00.000Z +22 | 1985-01-01T00:00:00.000Z +; + +MultiTypedBucketDateNanosByMonth +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| STATS c = count(*) BY mo = BUCKET(hire_date, 20, "1986-01-01", "1999-12-31") +| SORT mo DESC, c +; + +c:long | mo:date_nanos +2 | 1999-01-01T00:00:00.000Z +2 | 1997-01-01T00:00:00.000Z +2 | 1996-01-01T00:00:00.000Z +10 | 1995-01-01T00:00:00.000Z +8 | 1994-01-01T00:00:00.000Z +6 | 1993-01-01T00:00:00.000Z +16 | 1992-01-01T00:00:00.000Z +12 | 1991-01-01T00:00:00.000Z +24 | 1990-01-01T00:00:00.000Z +26 | 1989-01-01T00:00:00.000Z +18 | 1988-01-01T00:00:00.000Z +30 | 1987-01-01T00:00:00.000Z +22 | 1986-01-01T00:00:00.000Z +22 | 1985-01-01T00:00:00.000Z +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index e23753c544082..2809bcd92e90d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -1337,23 +1337,16 @@ private static LogicalPlan castInvalidMappedFieldInLogicalPlan(LogicalPlan plan, ); } case Aggregate agg -> { - // both groupings and aggregates need to be replaced - // create new aggregates according to new groupings + // Both groupings and aggregates need to be updated, create new aggregates according to new groupings List origAggs = agg.aggregates(); List newAggs = new ArrayList<>(origAggs.size()); - for (int i = 0; i < origAggs.size() - newProjections.size(); i++) { // add aggregate functions + for (int i = 0; i < origAggs.size() - newProjections.size(); i++) { // Add aggregate functions newAggs.add(origAggs.get(i)); } - for (Expression e : newProjections) { // add new groupings + for (Expression e : newProjections) { // Add groupings newAggs.add(Expressions.attribute(e)); } - return new Aggregate( - agg.source(), - evalForInvalidMappedField(agg.source(), agg.child(), aliases), - agg.aggregateType(), - newProjections, - newAggs - ); + return agg.with(evalForInvalidMappedField(agg.source(), agg.child(), aliases), newProjections, newAggs); } case Dissect d -> { return new Dissect( @@ -1403,6 +1396,28 @@ private static LogicalPlan castInvalidMappedFieldInLogicalPlan(LogicalPlan plan, default -> throw new EsqlIllegalArgumentException("unexpected logical plan: " + plan); } } else { + // Double check Aggs, when Bucket is used together with Aggs, the Aggs may need two rounds of resolutions + if (plan instanceof Aggregate agg) { + // If there is unresolved reference in the aggregates try to get it resolved again by removing the custom message + List origAggs = agg.aggregates(); + List groupings = agg.groupings(); + int aggsCount = origAggs.size() - groupings.size(); + List newAggs = new ArrayList<>(origAggs.size()); + for (int i = 0; i < origAggs.size(); i++) { + var e = origAggs.get(i); + if (i < aggsCount) { // Add aggregate functions + newAggs.add(e); + } else { // Add groupings + if (e instanceof UnresolvedAttribute ua && ua.customMessage()) { + // Try to make ResolveRefs resolve the aggregates again by removing the custom message + newAggs.add(Expressions.attribute(groupings.get(i - aggsCount))); + } else { + newAggs.add(e); + } + } + } + return agg.with(groupings, newAggs); + } return plan; } } From e26d25dc72e7f83f405d912291ead9501086376c Mon Sep 17 00:00:00 2001 From: Fang Xing Date: Wed, 19 Mar 2025 08:38:21 -0400 Subject: [PATCH 07/24] correct typo --- .../qa/testFixtures/src/main/resources/union_types.csv-spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec index e691e7e9604d2..2b0a20e9a95f7 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec @@ -1871,7 +1871,7 @@ FROM employees, employees_incompatible | STATS c = MV_COUNT(VALUES(salary_change)) ; -c:integerq +c:integer 6 ; From 29b1155be4b681a2ef322e4f250113db411ae928 Mon Sep 17 00:00:00 2001 From: Fang Xing Date: Wed, 2 Apr 2025 16:51:55 -0400 Subject: [PATCH 08/24] remove grok and dissect, add enrich --- .../src/main/resources/union_types.csv-spec | 42 ++++++++++ .../xpack/esql/analysis/Analyzer.java | 80 ++++++++++--------- .../xpack/esql/plan/logical/Enrich.java | 2 + .../xpack/esql/analysis/VerifierTests.java | 4 +- 4 files changed, 88 insertions(+), 40 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec index 2b0a20e9a95f7..f7a56fc8b37e6 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec @@ -1982,3 +1982,45 @@ c:long | mo:date_nanos 22 | 1986-01-01T00:00:00.000Z 22 | 1985-01-01T00:00:00.000Z ; + + +MultiTypedEnrichOnNumericField +required_capability: enrich_load +required_capability: implicit_casting_union_typed_numeric_and_date + +from employees, employees_incompatible +| enrich languages_policy on languages +| keep emp_no, language_name +| sort emp_no +| limit 6 +; + +emp_no:long |language_name:keyword +10001 | French +10001 | French +10002 | null +10002 | null +10003 | German +10003 | German +; + +MultiTypedEnrichOnNumericFieldWithEval +required_capability: enrich_load +required_capability: implicit_casting_union_typed_numeric_and_date + +from employees, employees_incompatible +| eval languages = languages + 1 +| enrich languages_policy on languages +| keep emp_no, language_name +| sort emp_no +| limit 6 +; + +emp_no:long |language_name:keyword +10001 | Spanish +10001 | Spanish +10002 | null +10002 | null +10003 | null +10003 | null +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 487f34dc744a3..f0c21ef092ab9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -76,13 +76,11 @@ import org.elasticsearch.xpack.esql.plan.IndexPattern; import org.elasticsearch.xpack.esql.plan.logical.Aggregate; import org.elasticsearch.xpack.esql.plan.logical.Dedup; -import org.elasticsearch.xpack.esql.plan.logical.Dissect; import org.elasticsearch.xpack.esql.plan.logical.Drop; import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.plan.logical.EsRelation; import org.elasticsearch.xpack.esql.plan.logical.Eval; import org.elasticsearch.xpack.esql.plan.logical.Fork; -import org.elasticsearch.xpack.esql.plan.logical.Grok; import org.elasticsearch.xpack.esql.plan.logical.Insist; import org.elasticsearch.xpack.esql.plan.logical.Keep; import org.elasticsearch.xpack.esql.plan.logical.Limit; @@ -91,7 +89,6 @@ import org.elasticsearch.xpack.esql.plan.logical.MvExpand; import org.elasticsearch.xpack.esql.plan.logical.OrderBy; import org.elasticsearch.xpack.esql.plan.logical.Project; -import org.elasticsearch.xpack.esql.plan.logical.RegexExtract; import org.elasticsearch.xpack.esql.plan.logical.Rename; import org.elasticsearch.xpack.esql.plan.logical.RrfScoreEval; import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation; @@ -1065,7 +1062,10 @@ private LogicalPlan resolveEnrich(Enrich enrich, List childrenOutput) final DataType dataType = resolved.dataType(); String matchType = enrich.policy().getType(); DataType[] allowed = allowedEnrichTypes(matchType); - if (Arrays.asList(allowed).contains(dataType) == false) { + if (Arrays.asList(allowed).contains(dataType) == false && multiTypedField(resolved) == null) { // leave multi-typed + // fields to + // ImplicitCasting to + // deal with String suffix = "only [" + Arrays.stream(allowed).map(DataType::typeName).collect(Collectors.joining(", ")) + "] allowed for type [" @@ -1269,10 +1269,10 @@ public LogicalPlan apply(LogicalPlan plan, AnalyzerContext context) { if (p instanceof OrderBy || p instanceof EsqlProject || p instanceof Aggregate - || p instanceof RegexExtract || p instanceof MvExpand || p instanceof Eval - || p instanceof LookupJoin) { + || p instanceof LookupJoin + || p instanceof Enrich) { return castInvalidMappedFieldInLogicalPlan(p, false); } return p; @@ -1317,10 +1317,10 @@ private static LogicalPlan castInvalidMappedFieldInLogicalPlan(LogicalPlan plan, case EsqlProject project -> project.projections(); case OrderBy ob -> ob.order(); case Aggregate agg -> agg.groupings(); - case RegexExtract re -> List.of(re.input()); case MvExpand me -> List.of(me.target()); - case Eval e -> e.fields(); + case Eval eval -> eval.fields(); case LookupJoin lj -> lj.config().leftFields(); + case Enrich enrich -> List.of(enrich.matchField()); // The other types of plans are unexpected default -> throw new EsqlIllegalArgumentException("unexpected logical plan: " + plan); }; @@ -1355,32 +1355,13 @@ private static LogicalPlan castInvalidMappedFieldInLogicalPlan(LogicalPlan plan, } return agg.with(evalForInvalidMappedField(agg.source(), agg.child(), aliases), newProjections, newAggs); } - case Dissect d -> { - return new Dissect( - plan.source(), - evalForInvalidMappedField(d.source(), d.child(), aliases), - newProjections.get(0), - d.parser(), - d.extractedFields() - ); - } - case Grok g -> { - return new Grok( - plan.source(), - evalForInvalidMappedField(g.source(), g.child(), aliases), - newProjections.get(0), - g.parser(), - g.extractedFields() - ); - } case MvExpand mve -> { NamedExpression newTarget = Expressions.attribute(newProjections.get(0)); return new MvExpand( - plan.source(), + mve.source(), evalForInvalidMappedField(mve.source(), mve.child(), aliases), newTarget, - new ReferenceAttribute(newTarget.source(), newTarget.name(), newTarget.dataType(), newTarget.nullable(), - mve.expanded().id(), false) + newTarget.toAttribute() ); } case Eval ev -> { @@ -1400,6 +1381,20 @@ private static LogicalPlan castInvalidMappedFieldInLogicalPlan(LogicalPlan plan, newJoinConfig ); } + case Enrich enrich -> { + NamedExpression newMatchField = Expressions.attribute(newProjections.get(0)); + return new Enrich( + enrich.source(), + evalForInvalidMappedField(enrich.source(), enrich.child(), aliases), + enrich.mode(), + enrich.policyName(), + // Let resolveEnrich check whether the data type is supported, e.g. Enrich does not support data_nanos + new UnresolvedAttribute(newMatchField.source(), newMatchField.name()), + enrich.policy(), + enrich.concreteIndices(), + enrich.enrichFields() + ); + } // The other types of plans are unexpected default -> throw new EsqlIllegalArgumentException("unexpected logical plan: " + plan); } @@ -1462,13 +1457,11 @@ private static Tuple, List> castInvalidMappedFields( Expression e = Alias.unwrap(exp); e = e instanceof Order o ? o.child() : e; String alias = exp instanceof Alias a ? a.name() : null; - if (e.resolved() - && e.dataType() == UNSUPPORTED - && e instanceof FieldAttribute fa - && fa.field() instanceof InvalidMappedField imf) { + InvalidMappedField multiTypedField = multiTypedField(e); + if (multiTypedField != null) { // This is an invalid mapped field, find a common data type and cast to it. DataType targetType = null; - for (DataType type : imf.types()) { + for (DataType type : multiTypedField.types()) { if (targetType == null) { // Initialize the target type to the first type. targetType = type; } else { @@ -1480,9 +1473,9 @@ private static Tuple, List> castInvalidMappedFields( } } if (targetType != null && isRepresentable(targetType) && (isMillisOrNanos(targetType) || targetType.isNumeric())) { - alias = alias != null ? alias : fa.name(); - Source source = fa.source(); - Expression newChild = castInvalidMappedField(targetType, fa); + alias = alias != null ? alias : multiTypedField.getName(); + Source source = e.source(); + Expression newChild = castInvalidMappedField(targetType, e); Alias newAlias = new Alias(source, alias, newChild); newAliases.add(newAlias); if (createNewChildPlan) { @@ -1525,7 +1518,7 @@ private static Eval evalForInvalidMappedField(Source source, LogicalPlan childPl /** * Do implicit casting for data, date_nanos and numeric types only */ - private static Expression castInvalidMappedField(DataType targetType, FieldAttribute fa) { + private static Expression castInvalidMappedField(DataType targetType, Expression fa) { Source source = fa.source(); return switch (targetType) { case INTEGER -> new ToInteger(source, fa); @@ -1956,4 +1949,15 @@ private static LogicalPlan planWithoutSyntheticAttributes(LogicalPlan plan) { return newOutput.size() == output.size() ? plan : new Project(Source.EMPTY, plan, newOutput); } } + + private static InvalidMappedField multiTypedField(Expression e) { + Expression exp = Alias.unwrap(e); + if (exp.resolved() + && exp.dataType() == UNSUPPORTED + && exp instanceof FieldAttribute fa + && fa.field() instanceof InvalidMappedField imf) { + return imf; + } + return null; + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java index 11e9a57064e5b..e9defff9e6525 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java @@ -32,6 +32,7 @@ import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.index.EsIndex; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import org.elasticsearch.xpack.esql.plan.GeneratingPlan; @@ -208,6 +209,7 @@ public boolean expressionsResolved() { return policyName.resolved() && matchField instanceof EmptyAttribute == false // matchField not defined in the query, needs to be resolved from the policy && matchField.resolved() + && matchField.dataType() != DataType.UNSUPPORTED && Resolvables.resolved(enrichFields()); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 1b3e1d1e5051b..ab99c1de80621 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -133,8 +133,8 @@ public void testUnsupportedAndMultiTypedFields() { error("from test* | enrich client_cidr on unsupported", analyzer) ); assertEquals( - "1:36: Unsupported type [unsupported] for enrich matching field [multi_typed];" - + " only [keyword, text, ip, long, integer, float, double, datetime] allowed for type [range]", + "1:36: Cannot use field [multi_typed] due to ambiguities being mapped as [2] incompatible types: " + + "[ip] in [test1, test2, test3] and [2] other indices, [keyword] in [test6]", error("from test* | enrich client_cidr on multi_typed", analyzer) ); From 654e6cea445b18e056e9b3c2598fa1e56dd10d69 Mon Sep 17 00:00:00 2001 From: Fang Xing Date: Fri, 4 Apr 2025 13:25:27 -0400 Subject: [PATCH 09/24] more tests around date functions --- .../src/main/resources/union_types.csv-spec | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec index f7a56fc8b37e6..38ea1ee22c79b 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec @@ -1933,6 +1933,41 @@ emp_no:long | hire_date:date_nanos | yr:date_nanos | mo:date_nano 10002 | 1985-11-21T00:00:00.000Z | 1985-01-01T00:00:00.000Z | 1985-11-01T00:00:00.000Z ; +MultiTypedDateTruncStatsBy +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| STATS c = count(emp_no) BY yr = DATE_TRUNC(1 year, hire_date) +| SORT yr DESC +| LIMIT 5 +; + +c:long | yr:date_nanos +2 | 1999-01-01T00:00:00.000Z +2 | 1997-01-01T00:00:00.000Z +2 | 1996-01-01T00:00:00.000Z +10 | 1995-01-01T00:00:00.000Z +8 | 1994-01-01T00:00:00.000Z +; + +MultiTypedDateTruncStatsByWithEval +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| EVAL yr = DATE_TRUNC(1 year, hire_date) +| STATS c = count(emp_no) BY yr +| SORT yr DESC +| LIMIT 5 +; + +c:long | yr:date_nanos +2 | 1999-01-01T00:00:00.000Z +2 | 1997-01-01T00:00:00.000Z +2 | 1996-01-01T00:00:00.000Z +10 | 1995-01-01T00:00:00.000Z +8 | 1994-01-01T00:00:00.000Z +; + MultiTypedBucketDateNanosByYear required_capability: implicit_casting_union_typed_numeric_and_date @@ -1983,6 +2018,55 @@ c:long | mo:date_nanos 22 | 1985-01-01T00:00:00.000Z ; +MultiTypedBucketDateNanosInBothStatsAndBy +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| STATS c = count(*), b = BUCKET(hire_date, 1 year) + 1 year BY yr = BUCKET(hire_date, 1 year) +| SORT yr DESC, c +; + +c:long | b:date_nanos | yr:date_nanos +2 | 2000-01-01T00:00:00.000Z | 1999-01-01T00:00:00.000Z +2 | 1998-01-01T00:00:00.000Z | 1997-01-01T00:00:00.000Z +2 | 1997-01-01T00:00:00.000Z | 1996-01-01T00:00:00.000Z +10 | 1996-01-01T00:00:00.000Z | 1995-01-01T00:00:00.000Z +8 | 1995-01-01T00:00:00.000Z | 1994-01-01T00:00:00.000Z +6 | 1994-01-01T00:00:00.000Z | 1993-01-01T00:00:00.000Z +16 | 1993-01-01T00:00:00.000Z | 1992-01-01T00:00:00.000Z +12 | 1992-01-01T00:00:00.000Z | 1991-01-01T00:00:00.000Z +24 | 1991-01-01T00:00:00.000Z | 1990-01-01T00:00:00.000Z +26 | 1990-01-01T00:00:00.000Z | 1989-01-01T00:00:00.000Z +18 | 1989-01-01T00:00:00.000Z | 1988-01-01T00:00:00.000Z +30 | 1988-01-01T00:00:00.000Z | 1987-01-01T00:00:00.000Z +22 | 1987-01-01T00:00:00.000Z | 1986-01-01T00:00:00.000Z +22 | 1986-01-01T00:00:00.000Z | 1985-01-01T00:00:00.000Z +; + +MultiTypedBucketDateNanosInBothStatsAndByWithAlias +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| STATS c = count(*), b = yr + 1 year BY yr = BUCKET(hire_date, 1 year) +| SORT yr DESC, c +; + +c:long | b:date_nanos | yr:date_nanos +2 | 2000-01-01T00:00:00.000Z | 1999-01-01T00:00:00.000Z +2 | 1998-01-01T00:00:00.000Z | 1997-01-01T00:00:00.000Z +2 | 1997-01-01T00:00:00.000Z | 1996-01-01T00:00:00.000Z +10 | 1996-01-01T00:00:00.000Z | 1995-01-01T00:00:00.000Z +8 | 1995-01-01T00:00:00.000Z | 1994-01-01T00:00:00.000Z +6 | 1994-01-01T00:00:00.000Z | 1993-01-01T00:00:00.000Z +16 | 1993-01-01T00:00:00.000Z | 1992-01-01T00:00:00.000Z +12 | 1992-01-01T00:00:00.000Z | 1991-01-01T00:00:00.000Z +24 | 1991-01-01T00:00:00.000Z | 1990-01-01T00:00:00.000Z +26 | 1990-01-01T00:00:00.000Z | 1989-01-01T00:00:00.000Z +18 | 1989-01-01T00:00:00.000Z | 1988-01-01T00:00:00.000Z +30 | 1988-01-01T00:00:00.000Z | 1987-01-01T00:00:00.000Z +22 | 1987-01-01T00:00:00.000Z | 1986-01-01T00:00:00.000Z +22 | 1986-01-01T00:00:00.000Z | 1985-01-01T00:00:00.000Z +; MultiTypedEnrichOnNumericField required_capability: enrich_load From 2c5da6ebe78dd40cb7dcdd67a76e63e976466370 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Fri, 11 Apr 2025 16:04:56 +0200 Subject: [PATCH 10/24] DateNanos should now require a minimum of three decimal places A date in the valid date-nanos range should be acceptable to parse. This issue was showing up in a difference in behaviour between Elasticsearch ingest and the CsvTests. This fix makes CsvTests behave the same as Elasticearch itself. --- .../xpack/esql/core/util/DateUtils.java | 2 +- .../resources/data/employees_incompatible.csv | 200 +++++++++--------- 2 files changed, 101 insertions(+), 101 deletions(-) diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/DateUtils.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/DateUtils.java index 20f7b400e9364..ae316460dd11c 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/DateUtils.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/DateUtils.java @@ -65,7 +65,7 @@ public class DateUtils { .appendValue(MINUTE_OF_HOUR, 2) .appendLiteral(':') .appendValue(SECOND_OF_MINUTE, 2) - .appendFraction(NANO_OF_SECOND, 3, 9, true) + .appendFraction(NANO_OF_SECOND, 0, 9, true) .appendOffsetId() .toFormatter(Locale.ROOT); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/employees_incompatible.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/employees_incompatible.csv index cf810f27b245d..718bb3add9b1c 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/employees_incompatible.csv +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/employees_incompatible.csv @@ -1,101 +1,101 @@ birth_date:date ,emp_no:long,first_name:text,gender:text,hire_date:date_nanos,languages:byte,languages.long:long,languages.short:short,languages.byte:byte,last_name:text,salary:long,height:float,height.double:double,height.scaled_float:scaled_float,height.half_float:half_float,still_hired:keyword,avg_worked_seconds:unsigned_long,job_positions:text,is_rehired:keyword,salary_change:float,salary_change.int:integer,salary_change.long:long,salary_change.keyword:keyword -1953-09-02T00:00:00Z,10001,Georgi ,M,1986-06-26T00:00:00.000Z,2,2,2,2,Facello ,57305,2.03,2.03,2.03,2.03,true ,268728049,[Senior Python Developer,Accountant],[false,true],[1.19],[1],[1],[1.19] -1964-06-02T00:00:00Z,10002,Bezalel ,F,1985-11-21T00:00:00.000Z,5,5,5,5,Simmel ,56371,2.08,2.08,2.08,2.08,true ,328922887,[Senior Team Lead],[false,false],[-7.23,11.17],[-7,11],[-7,11],[-7.23,11.17] -1959-12-03T00:00:00Z,10003,Parto ,M,1986-08-28T00:00:00.000Z,4,4,4,4,Bamford ,61805,1.83,1.83,1.83,1.83,false,200296405,[],[],[14.68,12.82],[14,12],[14,12],[14.68,12.82] -1954-05-01T00:00:00Z,10004,Chirstian ,M,1986-12-01T00:00:00.000Z,5,5,5,5,Koblick ,36174,1.78,1.78,1.78,1.78,true ,311267831,[Reporting Analyst,Tech Lead,Head Human Resources,Support Engineer],[true],[3.65,-0.35,1.13,13.48],[3,0,1,13],[3,0,1,13],[3.65,-0.35,1.13,13.48] -1955-01-21T00:00:00Z,10005,Kyoichi ,M,1989-09-12T00:00:00.000Z,1,1,1,1,Maliniak ,63528,2.05,2.05,2.05,2.05,true ,244294991,[],[false,false,false,true],[-2.14,13.07],[-2,13],[-2,13],[-2.14,13.07] -1953-04-20T00:00:00Z,10006,Anneke ,F,1989-06-02T00:00:00.000Z,3,3,3,3,Preusig ,60335,1.56,1.56,1.56,1.56,false,372957040,[Tech Lead,Principal Support Engineer,Senior Team Lead],[],[-3.90],[-3],[-3],[-3.90] -1957-05-23T00:00:00Z,10007,Tzvetan ,F,1989-02-10T00:00:00.000Z,4,4,4,4,Zielinski ,74572,1.70,1.70,1.70,1.70,true ,393084805,[],[true,false,true,false],[-7.06,1.99,0.57],[-7,1,0],[-7,1,0],[-7.06,1.99,0.57] -1958-02-19T00:00:00Z,10008,Saniya ,M,1994-09-15T00:00:00.000Z,2,2,2,2,Kalloufi ,43906,2.10,2.10,2.10,2.10,true ,283074758,[Senior Python Developer,Junior Developer,Purchase Manager,Internship],[true,false],[12.68,3.54,0.75,-2.92],[12,3,0,-2],[12,3,0,-2],[12.68,3.54,0.75,-2.92] -1952-04-19T00:00:00Z,10009,Sumant ,F,1985-02-18T00:00:00.000Z,1,1,1,1,Peac ,66174,1.85,1.85,1.85,1.85,false,236805489,[Senior Python Developer,Internship],[],[],[],[],[] -1963-06-01T00:00:00Z,10010,Duangkaew , ,1989-08-24T00:00:00.000Z,4,4,4,4,Piveteau ,45797,1.70,1.70,1.70,1.70,false,315236372,[Architect,Reporting Analyst,Tech Lead,Purchase Manager],[true,true,false,false],[5.05,-6.77,4.69,12.15],[5,-6,4,12],[5,-6,4,12],[5.05,-6.77,4.69,12.15] -1953-11-07T00:00:00Z,10011,Mary , ,1990-01-22T00:00:00.000Z,5,5,5,5,Sluis ,31120,1.50,1.50,1.50,1.50,true ,239615525,[Architect,Reporting Analyst,Tech Lead,Senior Team Lead],[true,true],[10.35,-7.82,8.73,3.48],[10,-7,8,3],[10,-7,8,3],[10.35,-7.82,8.73,3.48] -1960-10-04T00:00:00Z,10012,Patricio , ,1992-12-18T00:00:00.000Z,5,5,5,5,Bridgland ,48942,1.97,1.97,1.97,1.97,false,365510850,[Head Human Resources,Accountant],[false,true,true,false],[0.04],[0],[0],[0.04] -1963-06-07T00:00:00Z,10013,Eberhardt , ,1985-10-20T00:00:00.000Z,1,1,1,1,Terkki ,48735,1.94,1.94,1.94,1.94,true ,253864340,[Reporting Analyst],[true,true],[],[],[],[] -1956-02-12T00:00:00Z,10014,Berni , ,1987-03-11T00:00:00.000Z,5,5,5,5,Genin ,37137,1.99,1.99,1.99,1.99,false,225049139,[Reporting Analyst,Data Scientist,Head Human Resources],[],[-1.89,9.07],[-1,9],[-1,9],[-1.89,9.07] -1959-08-19T00:00:00Z,10015,Guoxiang , ,1987-07-02T00:00:00.000Z,5,5,5,5,Nooteboom ,25324,1.66,1.66,1.66,1.66,true ,390266432,[Principal Support Engineer,Junior Developer,Head Human Resources,Support Engineer],[true,false,false,false],[14.25,12.40],[14,12],[14,12],[14.25,12.40] -1961-05-02T00:00:00Z,10016,Kazuhito , ,1995-01-27T00:00:00.000Z,2,2,2,2,Cappelletti ,61358,1.54,1.54,1.54,1.54,false,253029411,[Reporting Analyst,Python Developer,Accountant,Purchase Manager],[false,false],[-5.18,7.69],[-5,7],[-5,7],[-5.18,7.69] -1958-07-06T00:00:00Z,10017,Cristinel , ,1993-08-03T00:00:00.000Z,2,2,2,2,Bouloucos ,58715,1.74,1.74,1.74,1.74,false,236703986,[Data Scientist,Head Human Resources,Purchase Manager],[true,false,true,true],[-6.33],[-6],[-6],[-6.33] -1954-06-19T00:00:00Z,10018,Kazuhide , ,1987-04-03T00:00:00.000Z,2,2,2,2,Peha ,56760,1.97,1.97,1.97,1.97,false,309604079,[Junior Developer],[false,false,true,true],[-1.64,11.51,-5.32],[-1,11,-5],[-1,11,-5],[-1.64,11.51,-5.32] -1953-01-23T00:00:00Z,10019,Lillian , ,1999-04-30T00:00:00.000Z,1,1,1,1,Haddadi ,73717,2.06,2.06,2.06,2.06,false,342855721,[Purchase Manager],[false,false],[-6.84,8.42,-7.26],[-6,8,-7],[-6,8,-7],[-6.84,8.42,-7.26] -1952-12-24T00:00:00Z,10020,Mayuko ,M,1991-01-26T00:00:00.000Z, , , , ,Warwick ,40031,1.41,1.41,1.41,1.41,false,373309605,[Tech Lead],[true,true,false],[-5.81],[-5],[-5],[-5.81] -1960-02-20T00:00:00Z,10021,Ramzi ,M,1988-02-10T00:00:00.000Z, , , , ,Erde ,60408,1.47,1.47,1.47,1.47,false,287654610,[Support Engineer],[true],[],[],[],[] -1952-07-08T00:00:00Z,10022,Shahaf ,M,1995-08-22T00:00:00.000Z, , , , ,Famili ,48233,1.82,1.82,1.82,1.82,false,233521306,[Reporting Analyst,Data Scientist,Python Developer,Internship],[true,false],[12.09,2.85],[12,2],[12,2],[12.09,2.85] -1953-09-29T00:00:00Z,10023,Bojan ,F,1989-12-17T00:00:00.000Z, , , , ,Montemayor ,47896,1.75,1.75,1.75,1.75,true ,330870342,[Accountant,Support Engineer,Purchase Manager],[true,true,false],[14.63,0.80],[14,0],[14,0],[14.63,0.80] -1958-09-05T00:00:00Z,10024,Suzette ,F,1997-05-19T00:00:00.000Z, , , , ,Pettey ,64675,2.08,2.08,2.08,2.08,true ,367717671,[Junior Developer],[true,true,true,true],[],[],[],[] -1958-10-31T00:00:00Z,10025,Prasadram ,M,1987-08-17T00:00:00.000Z, , , , ,Heyers ,47411,1.87,1.87,1.87,1.87,false,371270797,[Accountant],[true,false],[-4.33,-2.90,12.06,-3.46],[-4,-2,12,-3],[-4,-2,12,-3],[-4.33,-2.90,12.06,-3.46] -1953-04-03T00:00:00Z,10026,Yongqiao ,M,1995-03-20T00:00:00.000Z, , , , ,Berztiss ,28336,2.10,2.10,2.10,2.10,true ,359208133,[Reporting Analyst],[false,true],[-7.37,10.62,11.20],[-7,10,11],[-7,10,11],[-7.37,10.62,11.20] -1962-07-10T00:00:00Z,10027,Divier ,F,1989-07-07T00:00:00.000Z, , , , ,Reistad ,73851,1.53,1.53,1.53,1.53,false,374037782,[Senior Python Developer],[false],[],[],[],[] -1963-11-26T00:00:00Z,10028,Domenick ,M,1991-10-22T00:00:00.000Z, , , , ,Tempesti ,39356,2.07,2.07,2.07,2.07,true ,226435054,[Tech Lead,Python Developer,Accountant,Internship],[true,false,false,true],[],[],[],[] -1956-12-13T00:00:00Z,10029,Otmar ,M,1985-11-20T00:00:00.000Z, , , , ,Herbst ,74999,1.99,1.99,1.99,1.99,false,257694181,[Senior Python Developer,Data Scientist,Principal Support Engineer],[true],[-0.32,-1.90,-8.19],[0,-1,-8],[0,-1,-8],[-0.32,-1.90,-8.19] -1958-07-14T00:00:00Z,10030, ,M,1994-02-17T00:00:00.000Z,3,3,3,3,Demeyer ,67492,1.92,1.92,1.92,1.92,false,394597613,[Tech Lead,Data Scientist,Senior Team Lead],[true,false,false],[-0.40],[0],[0],[-0.40] -1959-01-27T00:00:00Z,10031, ,M,1991-09-01T00:00:00.000Z,4,4,4,4,Joslin ,37716,1.68,1.68,1.68,1.68,false,348545109,[Architect,Senior Python Developer,Purchase Manager,Senior Team Lead],[false],[],[],[],[] -1960-08-09T00:00:00Z,10032, ,F,1990-06-20T00:00:00.000Z,3,3,3,3,Reistad ,62233,2.10,2.10,2.10,2.10,false,277622619,[Architect,Senior Python Developer,Junior Developer,Purchase Manager],[false,false],[9.32,-4.92],[9,-4],[9,-4],[9.32,-4.92] -1956-11-14T00:00:00Z,10033, ,M,1987-03-18T00:00:00.000Z,1,1,1,1,Merlo ,70011,1.63,1.63,1.63,1.63,false,208374744,[],[true],[],[],[],[] -1962-12-29T00:00:00Z,10034, ,M,1988-09-21T00:00:00.000Z,1,1,1,1,Swan ,39878,1.46,1.46,1.46,1.46,false,214393176,[Business Analyst,Data Scientist,Python Developer,Accountant],[false],[-8.46],[-8],[-8],[-8.46] -1953-02-08T00:00:00Z,10035, ,M,1988-09-05T00:00:00.000Z,5,5,5,5,Chappelet ,25945,1.81,1.81,1.81,1.81,false,203838153,[Senior Python Developer,Data Scientist],[false],[-2.54,-6.58],[-2,-6],[-2,-6],[-2.54,-6.58] -1959-08-10T00:00:00Z,10036, ,M,1992-01-03T00:00:00.000Z,4,4,4,4,Portugali ,60781,1.61,1.61,1.61,1.61,false,305493131,[Senior Python Developer],[true,false,false],[],[],[],[] -1963-07-22T00:00:00Z,10037, ,M,1990-12-05T00:00:00.000Z,2,2,2,2,Makrucki ,37691,2.00,2.00,2.00,2.00,true ,359217000,[Senior Python Developer,Tech Lead,Accountant],[false],[-7.08],[-7],[-7],[-7.08] -1960-07-20T00:00:00Z,10038, ,M,1989-09-20T00:00:00.000Z,4,4,4,4,Lortz ,35222,1.53,1.53,1.53,1.53,true ,314036411,[Senior Python Developer,Python Developer,Support Engineer],[],[],[],[],[] -1959-10-01T00:00:00Z,10039, ,M,1988-01-19T00:00:00.000Z,2,2,2,2,Brender ,36051,1.55,1.55,1.55,1.55,false,243221262,[Business Analyst,Python Developer,Principal Support Engineer],[true,true],[-6.90],[-6],[-6],[-6.90] - ,10040,Weiyi ,F,1993-02-14T00:00:00.000Z,4,4,4,4,Meriste ,37112,1.90,1.90,1.90,1.90,false,244478622,[Principal Support Engineer],[true,false,true,true],[6.97,14.74,-8.94,1.92],[6,14,-8,1],[6,14,-8,1],[6.97,14.74,-8.94,1.92] - ,10041,Uri ,F,1989-11-12T00:00:00.000Z,1,1,1,1,Lenart ,56415,1.75,1.75,1.75,1.75,false,287789442,[Data Scientist,Head Human Resources,Internship,Senior Team Lead],[],[9.21,0.05,7.29,-2.94],[9,0,7,-2],[9,0,7,-2],[9.21,0.05,7.29,-2.94] - ,10042,Magy ,F,1993-03-21T00:00:00.000Z,3,3,3,3,Stamatiou ,30404,1.44,1.44,1.44,1.44,true ,246355863,[Architect,Business Analyst,Junior Developer,Internship],[],[-9.28,9.42],[-9,9],[-9,9],[-9.28,9.42] - ,10043,Yishay ,M,1990-10-20T00:00:00.000Z,1,1,1,1,Tzvieli ,34341,1.52,1.52,1.52,1.52,true ,287222180,[Data Scientist,Python Developer,Support Engineer],[false,true,true],[-5.17,4.62,7.42],[-5,4,7],[-5,4,7],[-5.17,4.62,7.42] - ,10044,Mingsen ,F,1994-05-21T00:00:00.000Z,1,1,1,1,Casley ,39728,2.06,2.06,2.06,2.06,false,387408356,[Tech Lead,Principal Support Engineer,Accountant,Support Engineer],[true,true],[8.09],[8],[8],[8.09] - ,10045,Moss ,M,1989-09-02T00:00:00.000Z,3,3,3,3,Shanbhogue ,74970,1.70,1.70,1.70,1.70,false,371418933,[Principal Support Engineer,Junior Developer,Accountant,Purchase Manager],[true,false],[],[],[],[] - ,10046,Lucien ,M,1992-06-20T00:00:00.000Z,4,4,4,4,Rosenbaum ,50064,1.52,1.52,1.52,1.52,true ,302353405,[Principal Support Engineer,Junior Developer,Head Human Resources,Internship],[true,true,false,true],[2.39],[2],[2],[2.39] - ,10047,Zvonko ,M,1989-03-31T00:00:00.000Z,4,4,4,4,Nyanchama ,42716,1.52,1.52,1.52,1.52,true ,306369346,[Architect,Data Scientist,Principal Support Engineer,Senior Team Lead],[true],[-6.36,12.12],[-6,12],[-6,12],[-6.36,12.12] - ,10048,Florian ,M,1985-02-24T00:00:00.000Z,3,3,3,3,Syrotiuk ,26436,2.00,2.00,2.00,2.00,false,248451647,[Internship],[true,true],[],[],[],[] - ,10049,Basil ,F,1992-05-04T00:00:00.000Z,5,5,5,5,Tramer ,37853,1.52,1.52,1.52,1.52,true ,320725709,[Senior Python Developer,Business Analyst],[],[-1.05],[-1],[-1],[-1.05] -1958-05-21T00:00:00Z,10050,Yinghua ,M,1990-12-25T00:00:00.000Z,2,2,2,2,Dredge ,43026,1.96,1.96,1.96,1.96,true ,242731798,[Reporting Analyst,Junior Developer,Accountant,Support Engineer],[true],[8.70,10.94],[8,10],[8,10],[8.70,10.94] -1953-07-28T00:00:00Z,10051,Hidefumi ,M,1992-10-15T00:00:00.000Z,3,3,3,3,Caine ,58121,1.89,1.89,1.89,1.89,true ,374753122,[Business Analyst,Accountant,Purchase Manager],[],[],[],[],[] -1961-02-26T00:00:00Z,10052,Heping ,M,1988-05-21T00:00:00.000Z,1,1,1,1,Nitsch ,55360,1.79,1.79,1.79,1.79,true ,299654717,[],[true,true,false],[-0.55,-1.89,-4.22,-6.03],[0,-1,-4,-6],[0,-1,-4,-6],[-0.55,-1.89,-4.22,-6.03] -1954-09-13T00:00:00Z,10053,Sanjiv ,F,1986-02-04T00:00:00.000Z,3,3,3,3,Zschoche ,54462,1.58,1.58,1.58,1.58,false,368103911,[Support Engineer],[true,false,true,false],[-7.67,-3.25],[-7,-3],[-7,-3],[-7.67,-3.25] -1957-04-04T00:00:00Z,10054,Mayumi ,M,1995-03-13T00:00:00.000Z,4,4,4,4,Schueller ,65367,1.82,1.82,1.82,1.82,false,297441693,[Principal Support Engineer],[false,false],[],[],[],[] -1956-06-06T00:00:00Z,10055,Georgy ,M,1992-04-27T00:00:00.000Z,5,5,5,5,Dredge ,49281,2.04,2.04,2.04,2.04,false,283157844,[Senior Python Developer,Head Human Resources,Internship,Support Engineer],[false,false,true],[7.34,12.99,3.17],[7,12,3],[7,12,3],[7.34,12.99,3.17] -1961-09-01T00:00:00Z,10056,Brendon ,F,1990-02-01T00:00:00.000Z,2,2,2,2,Bernini ,33370,1.57,1.57,1.57,1.57,true ,349086555,[Senior Team Lead],[true,false,false],[10.99,-5.17],[10,-5],[10,-5],[10.99,-5.17] -1954-05-30T00:00:00Z,10057,Ebbe ,F,1992-01-15T00:00:00.000Z,4,4,4,4,Callaway ,27215,1.59,1.59,1.59,1.59,true ,324356269,[Python Developer,Head Human Resources],[],[-6.73,-2.43,-5.27,1.03],[-6,-2,-5,1],[-6,-2,-5,1],[-6.73,-2.43,-5.27,1.03] -1954-10-01T00:00:00Z,10058,Berhard ,M,1987-04-13T00:00:00.000Z,3,3,3,3,McFarlin ,38376,1.83,1.83,1.83,1.83,false,268378108,[Principal Support Engineer],[],[-4.89],[-4],[-4],[-4.89] -1953-09-19T00:00:00Z,10059,Alejandro ,F,1991-06-26T00:00:00.000Z,2,2,2,2,McAlpine ,44307,1.48,1.48,1.48,1.48,false,237368465,[Architect,Principal Support Engineer,Purchase Manager,Senior Team Lead],[false],[5.53,13.38,-4.69,6.27],[5,13,-4,6],[5,13,-4,6],[5.53,13.38,-4.69,6.27] -1961-10-15T00:00:00Z,10060,Breannda ,M,1987-11-02T00:00:00.000Z,2,2,2,2,Billingsley ,29175,1.42,1.42,1.42,1.42,true ,341158890,[Business Analyst,Data Scientist,Senior Team Lead],[false,false,true,false],[-1.76,-0.85],[-1,0],[-1,0],[-1.76,-0.85] -1962-10-19T00:00:00Z,10061,Tse ,M,1985-09-17T00:00:00.000Z,1,1,1,1,Herber ,49095,1.45,1.45,1.45,1.45,false,327550310,[Purchase Manager,Senior Team Lead],[false,true],[14.39,-2.58,-0.95],[14,-2,0],[14,-2,0],[14.39,-2.58,-0.95] -1961-11-02T00:00:00Z,10062,Anoosh ,M,1991-08-30T00:00:00.000Z,3,3,3,3,Peyn ,65030,1.70,1.70,1.70,1.70,false,203989706,[Python Developer,Senior Team Lead],[false,true,true],[-1.17],[-1],[-1],[-1.171] -1952-08-06T00:00:00Z,10063,Gino ,F,1989-04-08T00:00:00.000Z,3,3,3,3,Leonhardt ,52121,1.78,1.78,1.78,1.78,true ,214068302,[],[true],[],[],[],[] -1959-04-07T00:00:00Z,10064,Udi ,M,1985-11-20T00:00:00.000Z,5,5,5,5,Jansch ,33956,1.93,1.93,1.93,1.93,false,307364077,[Purchase Manager],[false,false,true,false],[-8.66,-2.52],[-8,-2],[-8,-2],[-8.66,-2.52] -1963-04-14T00:00:00Z,10065,Satosi ,M,1988-05-18T00:00:00.000Z,2,2,2,2,Awdeh ,50249,1.59,1.59,1.59,1.59,false,372660279,[Business Analyst,Data Scientist,Principal Support Engineer],[false,true],[-1.47,14.44,-9.81],[-1,14,-9],[-1,14,-9],[-1.47,14.44,-9.81] -1952-11-13T00:00:00Z,10066,Kwee ,M,1986-02-26T00:00:00.000Z,5,5,5,5,Schusler ,31897,2.10,2.10,2.10,2.10,true ,360906451,[Senior Python Developer,Data Scientist,Accountant,Internship],[true,true,true],[5.94],[5],[5],[5.94] -1953-01-07T00:00:00Z,10067,Claudi ,M,1987-03-04T00:00:00.000Z,2,2,2,2,Stavenow ,52044,1.77,1.77,1.77,1.77,true ,347664141,[Tech Lead,Principal Support Engineer],[false,false],[8.72,4.44],[8,4],[8,4],[8.72,4.44] -1962-11-26T00:00:00Z,10068,Charlene ,M,1987-08-07T00:00:00.000Z,3,3,3,3,Brattka ,28941,1.58,1.58,1.58,1.58,true ,233999584,[Architect],[true],[3.43,-5.61,-5.29],[3,-5,-5],[3,-5,-5],[3.43,-5.61,-5.29] -1960-09-06T00:00:00Z,10069,Margareta ,F,1989-11-05T00:00:00.000Z,5,5,5,5,Bierman ,41933,1.77,1.77,1.77,1.77,true ,366512352,[Business Analyst,Junior Developer,Purchase Manager,Support Engineer],[false],[-3.34,-6.33,6.23,-0.31],[-3,-6,6,0],[-3,-6,6,0],[-3.34,-6.33,6.23,-0.31] -1955-08-20T00:00:00Z,10070,Reuven ,M,1985-10-14T00:00:00.000Z,3,3,3,3,Garigliano ,54329,1.77,1.77,1.77,1.77,true ,347188604,[],[true,true,true],[-5.90],[-5],[-5],[-5.90] -1958-01-21T00:00:00Z,10071,Hisao ,M,1987-10-01T00:00:00.000Z,2,2,2,2,Lipner ,40612,2.07,2.07,2.07,2.07,false,306671693,[Business Analyst,Reporting Analyst,Senior Team Lead],[false,false,false],[-2.69],[-2],[-2],[-2.69] -1952-05-15T00:00:00Z,10072,Hironoby ,F,1988-07-21T00:00:00.000Z,5,5,5,5,Sidou ,54518,1.82,1.82,1.82,1.82,true ,209506065,[Architect,Tech Lead,Python Developer,Senior Team Lead],[false,false,true,false],[11.21,-2.30,2.22,-5.44],[11,-2,2,-5],[11,-2,2,-5],[11.21,-2.30,2.22,-5.44] -1954-02-23T00:00:00Z,10073,Shir ,M,1991-12-01T00:00:00.000Z,4,4,4,4,McClurg ,32568,1.66,1.66,1.66,1.66,false,314930367,[Principal Support Engineer,Python Developer,Junior Developer,Purchase Manager],[true,false],[-5.67],[-5],[-5],[-5.67] -1955-08-28T00:00:00Z,10074,Mokhtar ,F,1990-08-13T00:00:00.000Z,5,5,5,5,Bernatsky ,38992,1.64,1.64,1.64,1.64,true ,382397583,[Senior Python Developer,Python Developer],[true,false,false,true],[6.70,1.98,-5.64,2.96],[6,1,-5,2],[6,1,-5,2],[6.70,1.98,-5.64,2.96] -1960-03-09T00:00:00Z,10075,Gao ,F,1987-03-19T00:00:00.000Z,5,5,5,5,Dolinsky ,51956,1.94,1.94,1.94,1.94,false,370238919,[Purchase Manager],[true],[9.63,-3.29,8.42],[9,-3,8],[9,-3,8],[9.63,-3.29,8.42] -1952-06-13T00:00:00Z,10076,Erez ,F,1985-07-09T00:00:00.000Z,3,3,3,3,Ritzmann ,62405,1.83,1.83,1.83,1.83,false,376240317,[Architect,Senior Python Developer],[false],[-6.90,-1.30,8.75],[-6,-1,8],[-6,-1,8],[-6.90,-1.30,8.75] -1964-04-18T00:00:00Z,10077,Mona ,M,1990-03-02T00:00:00.000Z,5,5,5,5,Azuma ,46595,1.68,1.68,1.68,1.68,false,351960222,[Internship],[],[-0.01],[0],[0],[-0.01] -1959-12-25T00:00:00Z,10078,Danel ,F,1987-05-26T00:00:00.000Z,2,2,2,2,Mondadori ,69904,1.81,1.81,1.81,1.81,true ,377116038,[Architect,Principal Support Engineer,Internship],[true],[-7.88,9.98,12.52],[-7,9,12],[-7,9,12],[-7.88,9.98,12.52] -1961-10-05T00:00:00Z,10079,Kshitij ,F,1986-03-27T00:00:00.000Z,2,2,2,2,Gils ,32263,1.59,1.59,1.59,1.59,false,320953330,[],[false],[7.58],[7],[7],[7.58] -1957-12-03T00:00:00Z,10080,Premal ,M,1985-11-19T00:00:00.000Z,5,5,5,5,Baek ,52833,1.80,1.80,1.80,1.80,false,239266137,[Senior Python Developer],[],[-4.35,7.36,5.56],[-4,7,5],[-4,7,5],[-4.35,7.36,5.56] -1960-12-17T00:00:00Z,10081,Zhongwei ,M,1986-10-30T00:00:00.000Z,2,2,2,2,Rosen ,50128,1.44,1.44,1.44,1.44,true ,321375511,[Accountant,Internship],[false,false,false],[],[],[],[] -1963-09-09T00:00:00Z,10082,Parviz ,M,1990-01-03T00:00:00.000Z,4,4,4,4,Lortz ,49818,1.61,1.61,1.61,1.61,false,232522994,[Principal Support Engineer],[false],[1.19,-3.39],[1,-3],[1,-3],[1.19,-3.39] -1959-07-23T00:00:00Z,10083,Vishv ,M,1987-03-31T00:00:00.000Z,1,1,1,1,Zockler ,39110,1.42,1.42,1.42,1.42,false,331236443,[Head Human Resources],[],[],[],[],[] -1960-05-25T00:00:00Z,10084,Tuval ,M,1995-12-15T00:00:00.000Z,1,1,1,1,Kalloufi ,28035,1.51,1.51,1.51,1.51,true ,359067056,[Principal Support Engineer],[false],[],[],[],[] -1962-11-07T00:00:00Z,10085,Kenroku ,M,1994-04-09T00:00:00.000Z,5,5,5,5,Malabarba ,35742,2.01,2.01,2.01,2.01,true ,353404008,[Senior Python Developer,Business Analyst,Tech Lead,Accountant],[],[11.67,6.75,8.40],[11,6,8],[11,6,8],[11.67,6.75,8.40] -1962-11-19T00:00:00Z,10086,Somnath ,M,1990-02-16T00:00:00.000Z,1,1,1,1,Foote ,68547,1.74,1.74,1.74,1.74,true ,328580163,[Senior Python Developer],[false,true],[13.61],[13],[13],[13.61] -1959-07-23T00:00:00Z,10087,Xinglin ,F,1986-09-08T00:00:00.000Z,5,5,5,5,Eugenio ,32272,1.74,1.74,1.74,1.74,true ,305782871,[Junior Developer,Internship],[false,false],[-2.05],[-2],[-2],[-2.05] -1954-02-25T00:00:00Z,10088,Jungsoon ,F,1988-09-02T00:00:00.000Z,5,5,5,5,Syrzycki ,39638,1.91,1.91,1.91,1.91,false,330714423,[Reporting Analyst,Business Analyst,Tech Lead],[true],[],[],[],[] -1963-03-21T00:00:00Z,10089,Sudharsan ,F,1986-08-12T00:00:00.000Z,4,4,4,4,Flasterstein,43602,1.57,1.57,1.57,1.57,true ,232951673,[Junior Developer,Accountant],[true,false,false,false],[],[],[],[] -1961-05-30T00:00:00Z,10090,Kendra ,M,1986-03-14T00:00:00.000Z,2,2,2,2,Hofting ,44956,2.03,2.03,2.03,2.03,true ,212460105,[],[false,false,false,true],[7.15,-1.85,3.60],[7,-1,3],[7,-1,3],[7.15,-1.85,3.60] -1955-10-04T00:00:00Z,10091,Amabile ,M,1992-11-18T00:00:00.000Z,3,3,3,3,Gomatam ,38645,2.09,2.09,2.09,2.09,true ,242582807,[Reporting Analyst,Python Developer],[true,true,false,false],[-9.23,7.50,5.85,5.19],[-9,7,5,5],[-9,7,5,5],[-9.23,7.50,5.85,5.19] -1964-10-18T00:00:00Z,10092,Valdiodio ,F,1989-09-22T00:00:00.000Z,1,1,1,1,Niizuma ,25976,1.75,1.75,1.75,1.75,false,313407352,[Junior Developer,Accountant],[false,false,true,true],[8.78,0.39,-6.77,8.30],[8,0,-6,8],[8,0,-6,8],[8.78,0.39,-6.77,8.30] -1964-06-11T00:00:00Z,10093,Sailaja ,M,1996-11-05T00:00:00.000Z,3,3,3,3,Desikan ,45656,1.69,1.69,1.69,1.69,false,315904921,[Reporting Analyst,Tech Lead,Principal Support Engineer,Purchase Manager],[],[-0.88],[0],[0],[-0.88] -1957-05-25T00:00:00Z,10094,Arumugam ,F,1987-04-18T00:00:00.000Z,5,5,5,5,Ossenbruggen,66817,2.10,2.10,2.10,2.10,false,332920135,[Senior Python Developer,Principal Support Engineer,Accountant],[true,false,true],[2.22,7.92],[2,7],[2,7],[2.22,7.92] -1965-01-03T00:00:00Z,10095,Hilari ,M,1986-07-15T00:00:00.000Z,4,4,4,4,Morton ,37702,1.55,1.55,1.55,1.55,false,321850475,[],[true,true,false,false],[-3.93,-6.66],[-3,-6],[-3,-6],[-3.93,-6.66] -1954-09-16T00:00:00Z,10096,Jayson ,M,1990-01-14T00:00:00.000Z,4,4,4,4,Mandell ,43889,1.94,1.94,1.94,1.94,false,204381503,[Architect,Reporting Analyst],[false,false,false],[],[],[],[] -1952-02-27T00:00:00Z,10097,Remzi ,M,1990-09-15T00:00:00.000Z,3,3,3,3,Waschkowski ,71165,1.53,1.53,1.53,1.53,false,206258084,[Reporting Analyst,Tech Lead],[true,false],[-1.12],[-1],[-1],[-1.12] -1961-09-23T00:00:00Z,10098,Sreekrishna,F,1985-05-13T00:00:00.000Z,4,4,4,4,Servieres ,44817,2.00,2.00,2.00,2.00,false,272392146,[Architect,Internship,Senior Team Lead],[false],[-2.83,8.31,4.38],[-2,8,4],[-2,8,4],[-2.83,8.31,4.38] -1956-05-25T00:00:00Z,10099,Valter ,F,1988-10-18T00:00:00.000Z,2,2,2,2,Sullins ,73578,1.81,1.81,1.81,1.81,true ,377713748,[],[true,true],[10.71,14.26,-8.78,-3.98],[10,14,-8,-3],[10,14,-8,-3],[10.71,14.26,-8.78,-3.98] -1953-04-21T00:00:00Z,10100,Hironobu ,F,1987-09-21T00:00:00.000Z,4,4,4,4,Haraldson ,68431,1.77,1.77,1.77,1.77,true ,223910853,[Purchase Manager],[false,true,true,false],[13.97,-7.49],[13,-7],[13,-7],[13.97,-7.49] +1953-09-02T00:00:00Z,10001,Georgi ,M,1986-06-26T00:00:00Z,2,2,2,2,Facello ,57305,2.03,2.03,2.03,2.03,true ,268728049,[Senior Python Developer,Accountant],[false,true],[1.19],[1],[1],[1.19] +1964-06-02T00:00:00Z,10002,Bezalel ,F,1985-11-21T00:00:00Z,5,5,5,5,Simmel ,56371,2.08,2.08,2.08,2.08,true ,328922887,[Senior Team Lead],[false,false],[-7.23,11.17],[-7,11],[-7,11],[-7.23,11.17] +1959-12-03T00:00:00Z,10003,Parto ,M,1986-08-28T00:00:00Z,4,4,4,4,Bamford ,61805,1.83,1.83,1.83,1.83,false,200296405,[],[],[14.68,12.82],[14,12],[14,12],[14.68,12.82] +1954-05-01T00:00:00Z,10004,Chirstian ,M,1986-12-01T00:00:00Z,5,5,5,5,Koblick ,36174,1.78,1.78,1.78,1.78,true ,311267831,[Reporting Analyst,Tech Lead,Head Human Resources,Support Engineer],[true],[3.65,-0.35,1.13,13.48],[3,0,1,13],[3,0,1,13],[3.65,-0.35,1.13,13.48] +1955-01-21T00:00:00Z,10005,Kyoichi ,M,1989-09-12T00:00:00Z,1,1,1,1,Maliniak ,63528,2.05,2.05,2.05,2.05,true ,244294991,[],[false,false,false,true],[-2.14,13.07],[-2,13],[-2,13],[-2.14,13.07] +1953-04-20T00:00:00Z,10006,Anneke ,F,1989-06-02T00:00:00Z,3,3,3,3,Preusig ,60335,1.56,1.56,1.56,1.56,false,372957040,[Tech Lead,Principal Support Engineer,Senior Team Lead],[],[-3.90],[-3],[-3],[-3.90] +1957-05-23T00:00:00Z,10007,Tzvetan ,F,1989-02-10T00:00:00Z,4,4,4,4,Zielinski ,74572,1.70,1.70,1.70,1.70,true ,393084805,[],[true,false,true,false],[-7.06,1.99,0.57],[-7,1,0],[-7,1,0],[-7.06,1.99,0.57] +1958-02-19T00:00:00Z,10008,Saniya ,M,1994-09-15T00:00:00Z,2,2,2,2,Kalloufi ,43906,2.10,2.10,2.10,2.10,true ,283074758,[Senior Python Developer,Junior Developer,Purchase Manager,Internship],[true,false],[12.68,3.54,0.75,-2.92],[12,3,0,-2],[12,3,0,-2],[12.68,3.54,0.75,-2.92] +1952-04-19T00:00:00Z,10009,Sumant ,F,1985-02-18T00:00:00Z,1,1,1,1,Peac ,66174,1.85,1.85,1.85,1.85,false,236805489,[Senior Python Developer,Internship],[],[],[],[],[] +1963-06-01T00:00:00Z,10010,Duangkaew , ,1989-08-24T00:00:00Z,4,4,4,4,Piveteau ,45797,1.70,1.70,1.70,1.70,false,315236372,[Architect,Reporting Analyst,Tech Lead,Purchase Manager],[true,true,false,false],[5.05,-6.77,4.69,12.15],[5,-6,4,12],[5,-6,4,12],[5.05,-6.77,4.69,12.15] +1953-11-07T00:00:00Z,10011,Mary , ,1990-01-22T00:00:00Z,5,5,5,5,Sluis ,31120,1.50,1.50,1.50,1.50,true ,239615525,[Architect,Reporting Analyst,Tech Lead,Senior Team Lead],[true,true],[10.35,-7.82,8.73,3.48],[10,-7,8,3],[10,-7,8,3],[10.35,-7.82,8.73,3.48] +1960-10-04T00:00:00Z,10012,Patricio , ,1992-12-18T00:00:00Z,5,5,5,5,Bridgland ,48942,1.97,1.97,1.97,1.97,false,365510850,[Head Human Resources,Accountant],[false,true,true,false],[0.04],[0],[0],[0.04] +1963-06-07T00:00:00Z,10013,Eberhardt , ,1985-10-20T00:00:00Z,1,1,1,1,Terkki ,48735,1.94,1.94,1.94,1.94,true ,253864340,[Reporting Analyst],[true,true],[],[],[],[] +1956-02-12T00:00:00Z,10014,Berni , ,1987-03-11T00:00:00Z,5,5,5,5,Genin ,37137,1.99,1.99,1.99,1.99,false,225049139,[Reporting Analyst,Data Scientist,Head Human Resources],[],[-1.89,9.07],[-1,9],[-1,9],[-1.89,9.07] +1959-08-19T00:00:00Z,10015,Guoxiang , ,1987-07-02T00:00:00Z,5,5,5,5,Nooteboom ,25324,1.66,1.66,1.66,1.66,true ,390266432,[Principal Support Engineer,Junior Developer,Head Human Resources,Support Engineer],[true,false,false,false],[14.25,12.40],[14,12],[14,12],[14.25,12.40] +1961-05-02T00:00:00Z,10016,Kazuhito , ,1995-01-27T00:00:00Z,2,2,2,2,Cappelletti ,61358,1.54,1.54,1.54,1.54,false,253029411,[Reporting Analyst,Python Developer,Accountant,Purchase Manager],[false,false],[-5.18,7.69],[-5,7],[-5,7],[-5.18,7.69] +1958-07-06T00:00:00Z,10017,Cristinel , ,1993-08-03T00:00:00Z,2,2,2,2,Bouloucos ,58715,1.74,1.74,1.74,1.74,false,236703986,[Data Scientist,Head Human Resources,Purchase Manager],[true,false,true,true],[-6.33],[-6],[-6],[-6.33] +1954-06-19T00:00:00Z,10018,Kazuhide , ,1987-04-03T00:00:00Z,2,2,2,2,Peha ,56760,1.97,1.97,1.97,1.97,false,309604079,[Junior Developer],[false,false,true,true],[-1.64,11.51,-5.32],[-1,11,-5],[-1,11,-5],[-1.64,11.51,-5.32] +1953-01-23T00:00:00Z,10019,Lillian , ,1999-04-30T00:00:00Z,1,1,1,1,Haddadi ,73717,2.06,2.06,2.06,2.06,false,342855721,[Purchase Manager],[false,false],[-6.84,8.42,-7.26],[-6,8,-7],[-6,8,-7],[-6.84,8.42,-7.26] +1952-12-24T00:00:00Z,10020,Mayuko ,M,1991-01-26T00:00:00Z, , , , ,Warwick ,40031,1.41,1.41,1.41,1.41,false,373309605,[Tech Lead],[true,true,false],[-5.81],[-5],[-5],[-5.81] +1960-02-20T00:00:00Z,10021,Ramzi ,M,1988-02-10T00:00:00Z, , , , ,Erde ,60408,1.47,1.47,1.47,1.47,false,287654610,[Support Engineer],[true],[],[],[],[] +1952-07-08T00:00:00Z,10022,Shahaf ,M,1995-08-22T00:00:00Z, , , , ,Famili ,48233,1.82,1.82,1.82,1.82,false,233521306,[Reporting Analyst,Data Scientist,Python Developer,Internship],[true,false],[12.09,2.85],[12,2],[12,2],[12.09,2.85] +1953-09-29T00:00:00Z,10023,Bojan ,F,1989-12-17T00:00:00Z, , , , ,Montemayor ,47896,1.75,1.75,1.75,1.75,true ,330870342,[Accountant,Support Engineer,Purchase Manager],[true,true,false],[14.63,0.80],[14,0],[14,0],[14.63,0.80] +1958-09-05T00:00:00Z,10024,Suzette ,F,1997-05-19T00:00:00Z, , , , ,Pettey ,64675,2.08,2.08,2.08,2.08,true ,367717671,[Junior Developer],[true,true,true,true],[],[],[],[] +1958-10-31T00:00:00Z,10025,Prasadram ,M,1987-08-17T00:00:00Z, , , , ,Heyers ,47411,1.87,1.87,1.87,1.87,false,371270797,[Accountant],[true,false],[-4.33,-2.90,12.06,-3.46],[-4,-2,12,-3],[-4,-2,12,-3],[-4.33,-2.90,12.06,-3.46] +1953-04-03T00:00:00Z,10026,Yongqiao ,M,1995-03-20T00:00:00Z, , , , ,Berztiss ,28336,2.10,2.10,2.10,2.10,true ,359208133,[Reporting Analyst],[false,true],[-7.37,10.62,11.20],[-7,10,11],[-7,10,11],[-7.37,10.62,11.20] +1962-07-10T00:00:00Z,10027,Divier ,F,1989-07-07T00:00:00Z, , , , ,Reistad ,73851,1.53,1.53,1.53,1.53,false,374037782,[Senior Python Developer],[false],[],[],[],[] +1963-11-26T00:00:00Z,10028,Domenick ,M,1991-10-22T00:00:00Z, , , , ,Tempesti ,39356,2.07,2.07,2.07,2.07,true ,226435054,[Tech Lead,Python Developer,Accountant,Internship],[true,false,false,true],[],[],[],[] +1956-12-13T00:00:00Z,10029,Otmar ,M,1985-11-20T00:00:00Z, , , , ,Herbst ,74999,1.99,1.99,1.99,1.99,false,257694181,[Senior Python Developer,Data Scientist,Principal Support Engineer],[true],[-0.32,-1.90,-8.19],[0,-1,-8],[0,-1,-8],[-0.32,-1.90,-8.19] +1958-07-14T00:00:00Z,10030, ,M,1994-02-17T00:00:00Z,3,3,3,3,Demeyer ,67492,1.92,1.92,1.92,1.92,false,394597613,[Tech Lead,Data Scientist,Senior Team Lead],[true,false,false],[-0.40],[0],[0],[-0.40] +1959-01-27T00:00:00Z,10031, ,M,1991-09-01T00:00:00Z,4,4,4,4,Joslin ,37716,1.68,1.68,1.68,1.68,false,348545109,[Architect,Senior Python Developer,Purchase Manager,Senior Team Lead],[false],[],[],[],[] +1960-08-09T00:00:00Z,10032, ,F,1990-06-20T00:00:00Z,3,3,3,3,Reistad ,62233,2.10,2.10,2.10,2.10,false,277622619,[Architect,Senior Python Developer,Junior Developer,Purchase Manager],[false,false],[9.32,-4.92],[9,-4],[9,-4],[9.32,-4.92] +1956-11-14T00:00:00Z,10033, ,M,1987-03-18T00:00:00Z,1,1,1,1,Merlo ,70011,1.63,1.63,1.63,1.63,false,208374744,[],[true],[],[],[],[] +1962-12-29T00:00:00Z,10034, ,M,1988-09-21T00:00:00Z,1,1,1,1,Swan ,39878,1.46,1.46,1.46,1.46,false,214393176,[Business Analyst,Data Scientist,Python Developer,Accountant],[false],[-8.46],[-8],[-8],[-8.46] +1953-02-08T00:00:00Z,10035, ,M,1988-09-05T00:00:00Z,5,5,5,5,Chappelet ,25945,1.81,1.81,1.81,1.81,false,203838153,[Senior Python Developer,Data Scientist],[false],[-2.54,-6.58],[-2,-6],[-2,-6],[-2.54,-6.58] +1959-08-10T00:00:00Z,10036, ,M,1992-01-03T00:00:00Z,4,4,4,4,Portugali ,60781,1.61,1.61,1.61,1.61,false,305493131,[Senior Python Developer],[true,false,false],[],[],[],[] +1963-07-22T00:00:00Z,10037, ,M,1990-12-05T00:00:00Z,2,2,2,2,Makrucki ,37691,2.00,2.00,2.00,2.00,true ,359217000,[Senior Python Developer,Tech Lead,Accountant],[false],[-7.08],[-7],[-7],[-7.08] +1960-07-20T00:00:00Z,10038, ,M,1989-09-20T00:00:00Z,4,4,4,4,Lortz ,35222,1.53,1.53,1.53,1.53,true ,314036411,[Senior Python Developer,Python Developer,Support Engineer],[],[],[],[],[] +1959-10-01T00:00:00Z,10039, ,M,1988-01-19T00:00:00Z,2,2,2,2,Brender ,36051,1.55,1.55,1.55,1.55,false,243221262,[Business Analyst,Python Developer,Principal Support Engineer],[true,true],[-6.90],[-6],[-6],[-6.90] + ,10040,Weiyi ,F,1993-02-14T00:00:00Z,4,4,4,4,Meriste ,37112,1.90,1.90,1.90,1.90,false,244478622,[Principal Support Engineer],[true,false,true,true],[6.97,14.74,-8.94,1.92],[6,14,-8,1],[6,14,-8,1],[6.97,14.74,-8.94,1.92] + ,10041,Uri ,F,1989-11-12T00:00:00Z,1,1,1,1,Lenart ,56415,1.75,1.75,1.75,1.75,false,287789442,[Data Scientist,Head Human Resources,Internship,Senior Team Lead],[],[9.21,0.05,7.29,-2.94],[9,0,7,-2],[9,0,7,-2],[9.21,0.05,7.29,-2.94] + ,10042,Magy ,F,1993-03-21T00:00:00Z,3,3,3,3,Stamatiou ,30404,1.44,1.44,1.44,1.44,true ,246355863,[Architect,Business Analyst,Junior Developer,Internship],[],[-9.28,9.42],[-9,9],[-9,9],[-9.28,9.42] + ,10043,Yishay ,M,1990-10-20T00:00:00Z,1,1,1,1,Tzvieli ,34341,1.52,1.52,1.52,1.52,true ,287222180,[Data Scientist,Python Developer,Support Engineer],[false,true,true],[-5.17,4.62,7.42],[-5,4,7],[-5,4,7],[-5.17,4.62,7.42] + ,10044,Mingsen ,F,1994-05-21T00:00:00Z,1,1,1,1,Casley ,39728,2.06,2.06,2.06,2.06,false,387408356,[Tech Lead,Principal Support Engineer,Accountant,Support Engineer],[true,true],[8.09],[8],[8],[8.09] + ,10045,Moss ,M,1989-09-02T00:00:00Z,3,3,3,3,Shanbhogue ,74970,1.70,1.70,1.70,1.70,false,371418933,[Principal Support Engineer,Junior Developer,Accountant,Purchase Manager],[true,false],[],[],[],[] + ,10046,Lucien ,M,1992-06-20T00:00:00Z,4,4,4,4,Rosenbaum ,50064,1.52,1.52,1.52,1.52,true ,302353405,[Principal Support Engineer,Junior Developer,Head Human Resources,Internship],[true,true,false,true],[2.39],[2],[2],[2.39] + ,10047,Zvonko ,M,1989-03-31T00:00:00Z,4,4,4,4,Nyanchama ,42716,1.52,1.52,1.52,1.52,true ,306369346,[Architect,Data Scientist,Principal Support Engineer,Senior Team Lead],[true],[-6.36,12.12],[-6,12],[-6,12],[-6.36,12.12] + ,10048,Florian ,M,1985-02-24T00:00:00Z,3,3,3,3,Syrotiuk ,26436,2.00,2.00,2.00,2.00,false,248451647,[Internship],[true,true],[],[],[],[] + ,10049,Basil ,F,1992-05-04T00:00:00Z,5,5,5,5,Tramer ,37853,1.52,1.52,1.52,1.52,true ,320725709,[Senior Python Developer,Business Analyst],[],[-1.05],[-1],[-1],[-1.05] +1958-05-21T00:00:00Z,10050,Yinghua ,M,1990-12-25T00:00:00Z,2,2,2,2,Dredge ,43026,1.96,1.96,1.96,1.96,true ,242731798,[Reporting Analyst,Junior Developer,Accountant,Support Engineer],[true],[8.70,10.94],[8,10],[8,10],[8.70,10.94] +1953-07-28T00:00:00Z,10051,Hidefumi ,M,1992-10-15T00:00:00Z,3,3,3,3,Caine ,58121,1.89,1.89,1.89,1.89,true ,374753122,[Business Analyst,Accountant,Purchase Manager],[],[],[],[],[] +1961-02-26T00:00:00Z,10052,Heping ,M,1988-05-21T00:00:00Z,1,1,1,1,Nitsch ,55360,1.79,1.79,1.79,1.79,true ,299654717,[],[true,true,false],[-0.55,-1.89,-4.22,-6.03],[0,-1,-4,-6],[0,-1,-4,-6],[-0.55,-1.89,-4.22,-6.03] +1954-09-13T00:00:00Z,10053,Sanjiv ,F,1986-02-04T00:00:00Z,3,3,3,3,Zschoche ,54462,1.58,1.58,1.58,1.58,false,368103911,[Support Engineer],[true,false,true,false],[-7.67,-3.25],[-7,-3],[-7,-3],[-7.67,-3.25] +1957-04-04T00:00:00Z,10054,Mayumi ,M,1995-03-13T00:00:00Z,4,4,4,4,Schueller ,65367,1.82,1.82,1.82,1.82,false,297441693,[Principal Support Engineer],[false,false],[],[],[],[] +1956-06-06T00:00:00Z,10055,Georgy ,M,1992-04-27T00:00:00Z,5,5,5,5,Dredge ,49281,2.04,2.04,2.04,2.04,false,283157844,[Senior Python Developer,Head Human Resources,Internship,Support Engineer],[false,false,true],[7.34,12.99,3.17],[7,12,3],[7,12,3],[7.34,12.99,3.17] +1961-09-01T00:00:00Z,10056,Brendon ,F,1990-02-01T00:00:00Z,2,2,2,2,Bernini ,33370,1.57,1.57,1.57,1.57,true ,349086555,[Senior Team Lead],[true,false,false],[10.99,-5.17],[10,-5],[10,-5],[10.99,-5.17] +1954-05-30T00:00:00Z,10057,Ebbe ,F,1992-01-15T00:00:00Z,4,4,4,4,Callaway ,27215,1.59,1.59,1.59,1.59,true ,324356269,[Python Developer,Head Human Resources],[],[-6.73,-2.43,-5.27,1.03],[-6,-2,-5,1],[-6,-2,-5,1],[-6.73,-2.43,-5.27,1.03] +1954-10-01T00:00:00Z,10058,Berhard ,M,1987-04-13T00:00:00Z,3,3,3,3,McFarlin ,38376,1.83,1.83,1.83,1.83,false,268378108,[Principal Support Engineer],[],[-4.89],[-4],[-4],[-4.89] +1953-09-19T00:00:00Z,10059,Alejandro ,F,1991-06-26T00:00:00Z,2,2,2,2,McAlpine ,44307,1.48,1.48,1.48,1.48,false,237368465,[Architect,Principal Support Engineer,Purchase Manager,Senior Team Lead],[false],[5.53,13.38,-4.69,6.27],[5,13,-4,6],[5,13,-4,6],[5.53,13.38,-4.69,6.27] +1961-10-15T00:00:00Z,10060,Breannda ,M,1987-11-02T00:00:00Z,2,2,2,2,Billingsley ,29175,1.42,1.42,1.42,1.42,true ,341158890,[Business Analyst,Data Scientist,Senior Team Lead],[false,false,true,false],[-1.76,-0.85],[-1,0],[-1,0],[-1.76,-0.85] +1962-10-19T00:00:00Z,10061,Tse ,M,1985-09-17T00:00:00Z,1,1,1,1,Herber ,49095,1.45,1.45,1.45,1.45,false,327550310,[Purchase Manager,Senior Team Lead],[false,true],[14.39,-2.58,-0.95],[14,-2,0],[14,-2,0],[14.39,-2.58,-0.95] +1961-11-02T00:00:00Z,10062,Anoosh ,M,1991-08-30T00:00:00Z,3,3,3,3,Peyn ,65030,1.70,1.70,1.70,1.70,false,203989706,[Python Developer,Senior Team Lead],[false,true,true],[-1.17],[-1],[-1],[-1.171] +1952-08-06T00:00:00Z,10063,Gino ,F,1989-04-08T00:00:00Z,3,3,3,3,Leonhardt ,52121,1.78,1.78,1.78,1.78,true ,214068302,[],[true],[],[],[],[] +1959-04-07T00:00:00Z,10064,Udi ,M,1985-11-20T00:00:00Z,5,5,5,5,Jansch ,33956,1.93,1.93,1.93,1.93,false,307364077,[Purchase Manager],[false,false,true,false],[-8.66,-2.52],[-8,-2],[-8,-2],[-8.66,-2.52] +1963-04-14T00:00:00Z,10065,Satosi ,M,1988-05-18T00:00:00Z,2,2,2,2,Awdeh ,50249,1.59,1.59,1.59,1.59,false,372660279,[Business Analyst,Data Scientist,Principal Support Engineer],[false,true],[-1.47,14.44,-9.81],[-1,14,-9],[-1,14,-9],[-1.47,14.44,-9.81] +1952-11-13T00:00:00Z,10066,Kwee ,M,1986-02-26T00:00:00Z,5,5,5,5,Schusler ,31897,2.10,2.10,2.10,2.10,true ,360906451,[Senior Python Developer,Data Scientist,Accountant,Internship],[true,true,true],[5.94],[5],[5],[5.94] +1953-01-07T00:00:00Z,10067,Claudi ,M,1987-03-04T00:00:00Z,2,2,2,2,Stavenow ,52044,1.77,1.77,1.77,1.77,true ,347664141,[Tech Lead,Principal Support Engineer],[false,false],[8.72,4.44],[8,4],[8,4],[8.72,4.44] +1962-11-26T00:00:00Z,10068,Charlene ,M,1987-08-07T00:00:00Z,3,3,3,3,Brattka ,28941,1.58,1.58,1.58,1.58,true ,233999584,[Architect],[true],[3.43,-5.61,-5.29],[3,-5,-5],[3,-5,-5],[3.43,-5.61,-5.29] +1960-09-06T00:00:00Z,10069,Margareta ,F,1989-11-05T00:00:00Z,5,5,5,5,Bierman ,41933,1.77,1.77,1.77,1.77,true ,366512352,[Business Analyst,Junior Developer,Purchase Manager,Support Engineer],[false],[-3.34,-6.33,6.23,-0.31],[-3,-6,6,0],[-3,-6,6,0],[-3.34,-6.33,6.23,-0.31] +1955-08-20T00:00:00Z,10070,Reuven ,M,1985-10-14T00:00:00Z,3,3,3,3,Garigliano ,54329,1.77,1.77,1.77,1.77,true ,347188604,[],[true,true,true],[-5.90],[-5],[-5],[-5.90] +1958-01-21T00:00:00Z,10071,Hisao ,M,1987-10-01T00:00:00Z,2,2,2,2,Lipner ,40612,2.07,2.07,2.07,2.07,false,306671693,[Business Analyst,Reporting Analyst,Senior Team Lead],[false,false,false],[-2.69],[-2],[-2],[-2.69] +1952-05-15T00:00:00Z,10072,Hironoby ,F,1988-07-21T00:00:00Z,5,5,5,5,Sidou ,54518,1.82,1.82,1.82,1.82,true ,209506065,[Architect,Tech Lead,Python Developer,Senior Team Lead],[false,false,true,false],[11.21,-2.30,2.22,-5.44],[11,-2,2,-5],[11,-2,2,-5],[11.21,-2.30,2.22,-5.44] +1954-02-23T00:00:00Z,10073,Shir ,M,1991-12-01T00:00:00Z,4,4,4,4,McClurg ,32568,1.66,1.66,1.66,1.66,false,314930367,[Principal Support Engineer,Python Developer,Junior Developer,Purchase Manager],[true,false],[-5.67],[-5],[-5],[-5.67] +1955-08-28T00:00:00Z,10074,Mokhtar ,F,1990-08-13T00:00:00Z,5,5,5,5,Bernatsky ,38992,1.64,1.64,1.64,1.64,true ,382397583,[Senior Python Developer,Python Developer],[true,false,false,true],[6.70,1.98,-5.64,2.96],[6,1,-5,2],[6,1,-5,2],[6.70,1.98,-5.64,2.96] +1960-03-09T00:00:00Z,10075,Gao ,F,1987-03-19T00:00:00Z,5,5,5,5,Dolinsky ,51956,1.94,1.94,1.94,1.94,false,370238919,[Purchase Manager],[true],[9.63,-3.29,8.42],[9,-3,8],[9,-3,8],[9.63,-3.29,8.42] +1952-06-13T00:00:00Z,10076,Erez ,F,1985-07-09T00:00:00Z,3,3,3,3,Ritzmann ,62405,1.83,1.83,1.83,1.83,false,376240317,[Architect,Senior Python Developer],[false],[-6.90,-1.30,8.75],[-6,-1,8],[-6,-1,8],[-6.90,-1.30,8.75] +1964-04-18T00:00:00Z,10077,Mona ,M,1990-03-02T00:00:00Z,5,5,5,5,Azuma ,46595,1.68,1.68,1.68,1.68,false,351960222,[Internship],[],[-0.01],[0],[0],[-0.01] +1959-12-25T00:00:00Z,10078,Danel ,F,1987-05-26T00:00:00Z,2,2,2,2,Mondadori ,69904,1.81,1.81,1.81,1.81,true ,377116038,[Architect,Principal Support Engineer,Internship],[true],[-7.88,9.98,12.52],[-7,9,12],[-7,9,12],[-7.88,9.98,12.52] +1961-10-05T00:00:00Z,10079,Kshitij ,F,1986-03-27T00:00:00Z,2,2,2,2,Gils ,32263,1.59,1.59,1.59,1.59,false,320953330,[],[false],[7.58],[7],[7],[7.58] +1957-12-03T00:00:00Z,10080,Premal ,M,1985-11-19T00:00:00Z,5,5,5,5,Baek ,52833,1.80,1.80,1.80,1.80,false,239266137,[Senior Python Developer],[],[-4.35,7.36,5.56],[-4,7,5],[-4,7,5],[-4.35,7.36,5.56] +1960-12-17T00:00:00Z,10081,Zhongwei ,M,1986-10-30T00:00:00Z,2,2,2,2,Rosen ,50128,1.44,1.44,1.44,1.44,true ,321375511,[Accountant,Internship],[false,false,false],[],[],[],[] +1963-09-09T00:00:00Z,10082,Parviz ,M,1990-01-03T00:00:00Z,4,4,4,4,Lortz ,49818,1.61,1.61,1.61,1.61,false,232522994,[Principal Support Engineer],[false],[1.19,-3.39],[1,-3],[1,-3],[1.19,-3.39] +1959-07-23T00:00:00Z,10083,Vishv ,M,1987-03-31T00:00:00Z,1,1,1,1,Zockler ,39110,1.42,1.42,1.42,1.42,false,331236443,[Head Human Resources],[],[],[],[],[] +1960-05-25T00:00:00Z,10084,Tuval ,M,1995-12-15T00:00:00Z,1,1,1,1,Kalloufi ,28035,1.51,1.51,1.51,1.51,true ,359067056,[Principal Support Engineer],[false],[],[],[],[] +1962-11-07T00:00:00Z,10085,Kenroku ,M,1994-04-09T00:00:00Z,5,5,5,5,Malabarba ,35742,2.01,2.01,2.01,2.01,true ,353404008,[Senior Python Developer,Business Analyst,Tech Lead,Accountant],[],[11.67,6.75,8.40],[11,6,8],[11,6,8],[11.67,6.75,8.40] +1962-11-19T00:00:00Z,10086,Somnath ,M,1990-02-16T00:00:00Z,1,1,1,1,Foote ,68547,1.74,1.74,1.74,1.74,true ,328580163,[Senior Python Developer],[false,true],[13.61],[13],[13],[13.61] +1959-07-23T00:00:00Z,10087,Xinglin ,F,1986-09-08T00:00:00Z,5,5,5,5,Eugenio ,32272,1.74,1.74,1.74,1.74,true ,305782871,[Junior Developer,Internship],[false,false],[-2.05],[-2],[-2],[-2.05] +1954-02-25T00:00:00Z,10088,Jungsoon ,F,1988-09-02T00:00:00Z,5,5,5,5,Syrzycki ,39638,1.91,1.91,1.91,1.91,false,330714423,[Reporting Analyst,Business Analyst,Tech Lead],[true],[],[],[],[] +1963-03-21T00:00:00Z,10089,Sudharsan ,F,1986-08-12T00:00:00Z,4,4,4,4,Flasterstein,43602,1.57,1.57,1.57,1.57,true ,232951673,[Junior Developer,Accountant],[true,false,false,false],[],[],[],[] +1961-05-30T00:00:00Z,10090,Kendra ,M,1986-03-14T00:00:00Z,2,2,2,2,Hofting ,44956,2.03,2.03,2.03,2.03,true ,212460105,[],[false,false,false,true],[7.15,-1.85,3.60],[7,-1,3],[7,-1,3],[7.15,-1.85,3.60] +1955-10-04T00:00:00Z,10091,Amabile ,M,1992-11-18T00:00:00Z,3,3,3,3,Gomatam ,38645,2.09,2.09,2.09,2.09,true ,242582807,[Reporting Analyst,Python Developer],[true,true,false,false],[-9.23,7.50,5.85,5.19],[-9,7,5,5],[-9,7,5,5],[-9.23,7.50,5.85,5.19] +1964-10-18T00:00:00Z,10092,Valdiodio ,F,1989-09-22T00:00:00Z,1,1,1,1,Niizuma ,25976,1.75,1.75,1.75,1.75,false,313407352,[Junior Developer,Accountant],[false,false,true,true],[8.78,0.39,-6.77,8.30],[8,0,-6,8],[8,0,-6,8],[8.78,0.39,-6.77,8.30] +1964-06-11T00:00:00Z,10093,Sailaja ,M,1996-11-05T00:00:00Z,3,3,3,3,Desikan ,45656,1.69,1.69,1.69,1.69,false,315904921,[Reporting Analyst,Tech Lead,Principal Support Engineer,Purchase Manager],[],[-0.88],[0],[0],[-0.88] +1957-05-25T00:00:00Z,10094,Arumugam ,F,1987-04-18T00:00:00Z,5,5,5,5,Ossenbruggen,66817,2.10,2.10,2.10,2.10,false,332920135,[Senior Python Developer,Principal Support Engineer,Accountant],[true,false,true],[2.22,7.92],[2,7],[2,7],[2.22,7.92] +1965-01-03T00:00:00Z,10095,Hilari ,M,1986-07-15T00:00:00Z,4,4,4,4,Morton ,37702,1.55,1.55,1.55,1.55,false,321850475,[],[true,true,false,false],[-3.93,-6.66],[-3,-6],[-3,-6],[-3.93,-6.66] +1954-09-16T00:00:00Z,10096,Jayson ,M,1990-01-14T00:00:00Z,4,4,4,4,Mandell ,43889,1.94,1.94,1.94,1.94,false,204381503,[Architect,Reporting Analyst],[false,false,false],[],[],[],[] +1952-02-27T00:00:00Z,10097,Remzi ,M,1990-09-15T00:00:00Z,3,3,3,3,Waschkowski ,71165,1.53,1.53,1.53,1.53,false,206258084,[Reporting Analyst,Tech Lead],[true,false],[-1.12],[-1],[-1],[-1.12] +1961-09-23T00:00:00Z,10098,Sreekrishna,F,1985-05-13T00:00:00Z,4,4,4,4,Servieres ,44817,2.00,2.00,2.00,2.00,false,272392146,[Architect,Internship,Senior Team Lead],[false],[-2.83,8.31,4.38],[-2,8,4],[-2,8,4],[-2.83,8.31,4.38] +1956-05-25T00:00:00Z,10099,Valter ,F,1988-10-18T00:00:00Z,2,2,2,2,Sullins ,73578,1.81,1.81,1.81,1.81,true ,377713748,[],[true,true],[10.71,14.26,-8.78,-3.98],[10,14,-8,-3],[10,14,-8,-3],[10.71,14.26,-8.78,-3.98] +1953-04-21T00:00:00Z,10100,Hironobu ,F,1987-09-21T00:00:00Z,4,4,4,4,Haraldson ,68431,1.77,1.77,1.77,1.77,true ,223910853,[Purchase Manager],[false,true,true,false],[13.97,-7.49],[13,-7],[13,-7],[13.97,-7.49] From 279d368632bb3858c9760591914e6ab9bff065fc Mon Sep 17 00:00:00 2001 From: Fang Xing Date: Mon, 21 Apr 2025 12:00:08 -0400 Subject: [PATCH 11/24] refactor and remove implicit casting for numeric types --- .../esql/qa/rest/FieldExtractorTestCase.java | 6 - .../xpack/esql/CsvTestsDataLoader.java | 2 + .../main/resources/data/date_nanos_lookup.csv | 13 + .../resources/mapping-date_nanos_lookup.json | 13 + .../src/main/resources/union_types.csv-spec | 327 ++++++------- .../xpack/esql/action/EsqlCapabilities.java | 4 +- .../xpack/esql/analysis/Analyzer.java | 436 ++++++++---------- .../esql/type/EsqlDataTypeConverter.java | 3 - .../xpack/esql/analysis/AnalyzerTests.java | 70 +++ .../xpack/esql/analysis/VerifierTests.java | 4 +- .../esql/type/EsqlDataTypeConverterTests.java | 2 +- 11 files changed, 422 insertions(+), 458 deletions(-) create mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/date_nanos_lookup.csv create mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-date_nanos_lookup.json diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/FieldExtractorTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/FieldExtractorTestCase.java index 8d063dda416d6..4cf8997f77be1 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/FieldExtractorTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/FieldExtractorTestCase.java @@ -959,9 +959,6 @@ public void testIntegerDocValuesConflict() throws IOException { * In an ideal world we'd promote the {@code integer} to an {@code long} and just go. */ public void testLongIntegerConflict() throws IOException { - if (EsqlCapabilities.Cap.IMPLICIT_CASTING_UNION_TYPED_NUMERIC_AND_DATE.isEnabled()) { - return; - } assumeOriginalTypesReported(); longTest().sourceMode(SourceMode.DEFAULT).createIndex("test1", "emp_no"); index("test1", """ @@ -1005,9 +1002,6 @@ public void testLongIntegerConflict() throws IOException { * In an ideal world we'd promote the {@code short} to an {@code integer} and just go. */ public void testIntegerShortConflict() throws IOException { - if (EsqlCapabilities.Cap.IMPLICIT_CASTING_UNION_TYPED_NUMERIC_AND_DATE.isEnabled()) { - return; - } assumeOriginalTypesReported(); intTest().sourceMode(SourceMode.DEFAULT).createIndex("test1", "emp_no"); index("test1", """ diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index 68b0818284f05..f9da5c60c754d 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -125,6 +125,7 @@ public class CsvTestsDataLoader { ); private static final TestDataset AIRPORTS_WEB = new TestDataset("airports_web"); private static final TestDataset DATE_NANOS = new TestDataset("date_nanos"); + private static final TestDataset DATE_NANOS_LOOKUP = new TestDataset("date_nanos_lookup").withSetting("lookup-settings.json"); private static final TestDataset COUNTRIES_BBOX = new TestDataset("countries_bbox"); private static final TestDataset COUNTRIES_BBOX_WEB = new TestDataset("countries_bbox_web"); private static final TestDataset AIRPORT_CITY_BOUNDARIES = new TestDataset("airport_city_boundaries"); @@ -191,6 +192,7 @@ public class CsvTestsDataLoader { Map.entry(MULTIVALUE_GEOMETRIES.indexName, MULTIVALUE_GEOMETRIES), Map.entry(MULTIVALUE_POINTS.indexName, MULTIVALUE_POINTS), Map.entry(DATE_NANOS.indexName, DATE_NANOS), + Map.entry(DATE_NANOS_LOOKUP.indexName, DATE_NANOS_LOOKUP), Map.entry(K8S.indexName, K8S), Map.entry(DISTANCES.indexName, DISTANCES), Map.entry(ADDRESSES.indexName, ADDRESSES), diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/date_nanos_lookup.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/date_nanos_lookup.csv new file mode 100644 index 0000000000000..a212dc72ccf80 --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/date_nanos_lookup.csv @@ -0,0 +1,13 @@ +millis:date_nanos,nanos:date,num:long +2023-10-23T13:55:01.543123456Z,2023-10-23T13:55:01.543Z,1698069301543123456 +2023-10-23T13:55:01.543123456Z,2023-10-23T13:55:01.543Z,1698069301543123456 +2023-10-23T13:53:55.832987654Z,2023-10-23T13:53:55.832Z,1698069235832987654 +2023-10-23T13:52:55.015787878Z,2023-10-23T13:52:55.015Z,1698069175015787878 +2023-10-23T13:51:54.732102837Z,2023-10-23T13:51:54.732Z,1698069114732102837 +2023-10-23T13:33:34.937193000Z,2023-10-23T13:33:34.937Z,1698068014937193000 +2023-10-23T12:27:28.948000000Z,2023-10-23T12:27:28.948Z,1698064048948000000 +2023-10-23T12:15:03.360103847Z,2023-10-23T12:15:03.360Z,1698063303360103847 +2023-10-23T12:15:03.360103847Z,2023-10-23T12:15:03.360Z,1698063303360103847 +1999-10-23T12:15:03.360103847Z,[2023-03-23T12:15:03.360Z, 2023-02-23T13:33:34.937Z, 2023-01-23T13:55:01.543Z], 0 +1999-10-22T12:15:03.360103847Z,[2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z], 0 +2023-10-23T12:15:03.360103847Z,1923-10-23T12:15:03.360Z,1698063303360103847 diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-date_nanos_lookup.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-date_nanos_lookup.json new file mode 100644 index 0000000000000..1017639bb5cfc --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-date_nanos_lookup.json @@ -0,0 +1,13 @@ +{ + "properties": { + "millis": { + "type": "date_nanos" + }, + "nanos": { + "type": "date" + }, + "num": { + "type": "long" + } + } +} diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec index 38ea1ee22c79b..9a593c5ce49f9 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec @@ -1640,107 +1640,116 @@ id:integer | name:keyword | count:long 14 | mmmmm | 2 ; -MultiTypedFieldsEvalKeepSort -required_capability: implicit_casting_union_typed_numeric_and_date +MultiTypedFieldsKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos FROM employees, employees_incompatible -| Eval x = emp_no -| KEEP x, languages, hire_date, avg_worked_seconds +| EVAL x = emp_no::long, avg_worked_seconds = avg_worked_seconds::unsigned_long, z = hire_date::datetime +| KEEP x, languages, hire_date, avg_worked_seconds, z | SORT x, hire_date -| limit 2 +| LIMIT 2 ; -x:long |languages:integer |hire_date:date_nanos |avg_worked_seconds:unsigned_long -10001 |2 |1986-06-26T00:00:00.000Z |268728049 -10001 |2 |1986-06-26T00:00:00.000Z |268728049 +x:long |languages:unsupported |hire_date:date_nanos |avg_worked_seconds:unsigned_long |z:datetime +10001 |null |1986-06-26T00:00:00.000Z |268728049 |1986-06-26T00:00:00.000Z +10001 |null |1986-06-26T00:00:00.000Z |268728049 |1986-06-26T00:00:00.000Z ; MultiTypedFieldsDropKeepSort -required_capability: implicit_casting_union_typed_numeric_and_date +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos FROM employees, employees_incompatible +| EVAL emp_no = emp_no::long, avg_worked_seconds = avg_worked_seconds::unsigned_long, x = hire_date::datetime | DROP salary -| KEEP emp_no, languages, hire_date, avg_worked_seconds +| KEEP emp_no, languages, hire_date, avg_worked_seconds, x | SORT emp_no, hire_date | limit 2 ; -emp_no:long |languages:integer |hire_date:date_nanos |avg_worked_seconds:unsigned_long -10001 |2 |1986-06-26T00:00:00.000Z |268728049 -10001 |2 |1986-06-26T00:00:00.000Z |268728049 +emp_no:long |languages:unsupported |hire_date:date_nanos |avg_worked_seconds:unsigned_long |x:datetime +10001 |null |1986-06-26T00:00:00.000Z |268728049 |1986-06-26T00:00:00.000Z +10001 |null |1986-06-26T00:00:00.000Z |268728049 |1986-06-26T00:00:00.000Z ; MultiTypedFieldsRenameKeepSort -required_capability: implicit_casting_union_typed_numeric_and_date +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos from employees, employees_incompatible -| RENAME emp_no as new_emp_no, languages as new_languages, hire_date as new_hire_date, avg_worked_seconds as new_avg_worked_seconds -| KEEP new_emp_no, new_languages, new_hire_date, new_avg_worked_seconds +| EVAL emp_no = emp_no::long, avg_worked_seconds = avg_worked_seconds::unsigned_long, languages = languages::integer, x = hire_date::datetime +| RENAME emp_no as new_emp_no, languages as new_languages, hire_date as new_hire_date, avg_worked_seconds as new_avg_worked_seconds, x as y +| KEEP new_emp_no, new_languages, new_hire_date, new_avg_worked_seconds, y | SORT new_emp_no, new_hire_date | limit 2 ; -new_emp_no:long |new_languages:integer |new_hire_date:date_nanos |new_avg_worked_seconds:unsigned_long -10001 |2 |1986-06-26T00:00:00.000Z |268728049 -10001 |2 |1986-06-26T00:00:00.000Z |268728049 +new_emp_no:long |new_languages:integer |new_hire_date:date_nanos |new_avg_worked_seconds:unsigned_long |y:datetime +10001 |2 |1986-06-26T00:00:00.000Z |268728049 |1986-06-26T00:00:00.000Z +10001 |2 |1986-06-26T00:00:00.000Z |268728049 |1986-06-26T00:00:00.000Z ; MultiTypedFieldsDropEvalKeepSort -required_capability: implicit_casting_union_typed_numeric_and_date +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos FROM employees, employees_incompatible | DROP birth_date -| EVAL x = emp_no + 1, y = hire_date + 1 year, z = hire_date::datetime -| KEEP emp_no, x, hire_date, y, z -| SORT emp_no, hire_date +| EVAL x = emp_no::long + 1, y = hire_date + 1 year, z = hire_date::datetime +| KEEP x, hire_date, y, z +| SORT x, hire_date | limit 2 ; -emp_no:long |x:long |hire_date:date_nanos |y:date_nanos |z:date -10001 |10002 |1986-06-26T00:00:00.000Z |1987-06-26T00:00:00.000Z |1986-06-26T00:00:00.000Z -10001 |10002 |1986-06-26T00:00:00.000Z |1987-06-26T00:00:00.000Z |1986-06-26T00:00:00.000Z +x:long |hire_date:date_nanos |y:date_nanos |z:date +10002 |1986-06-26T00:00:00.000Z |1987-06-26T00:00:00.000Z |1986-06-26T00:00:00.000Z +10002 |1986-06-26T00:00:00.000Z |1987-06-26T00:00:00.000Z |1986-06-26T00:00:00.000Z ; MultiTypedFieldsEvalFilterKeepSort -required_capability: implicit_casting_union_typed_numeric_and_date +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos FROM employees, employees_incompatible -| EVAL x = emp_no + 1, y = hire_date + 1 day -| WHERE emp_no > 10010 AND hire_date > "1992-01-01" AND languages < 3 AND height > 1.2 -| KEEP emp_no, hire_date -| SORT emp_no, hire_date +| EVAL x = emp_no::long + 1, y = hire_date + 1 day +| WHERE x > 10011 AND hire_date > "1992-01-01" AND languages::integer < 3 AND height::double > 1.2 +| KEEP x, hire_date +| SORT x, hire_date | limit 4 ; -emp_no:long |hire_date:date_nanos -10016 | 1995-01-27T00:00:00.000Z -10016 | 1995-01-27T00:00:00.000Z -10017 | 1993-08-03T00:00:00.000Z -10017 | 1993-08-03T00:00:00.000Z +x:long |hire_date:date_nanos +10017 | 1995-01-27T00:00:00.000Z +10017 | 1995-01-27T00:00:00.000Z +10018 | 1993-08-03T00:00:00.000Z +10018 | 1993-08-03T00:00:00.000Z ; MultiTypedFieldsStatsByNumeric -required_capability: implicit_casting_union_typed_numeric_and_date +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos FROM employees, employees_incompatible -| STATS x=count(emp_no), y=max(hire_date), z=max(height) BY languages +| STATS x=max(hire_date) BY languages = languages::integer | SORT languages ; -x:long | y:date_nanos | z:double | languages:integer -30 | 1999-04-30T00:00:00.000Z | 2.06 | 1 -38 | 1995-01-27T00:00:00.000Z | 2.1 | 2 -34 | 1996-11-05T00:00:00.000Z | 2.1 | 3 -36 | 1995-03-13T00:00:00.000Z | 2.0 | 4 -42 | 1994-04-09T00:00:00.000Z | 2.1 | 5 -20 | 1997-05-19T00:00:00.000Z | 2.1 | null +x:date_nanos | languages:integer +1999-04-30T00:00:00.000Z | 1 +1995-01-27T00:00:00.000Z | 2 +1996-11-05T00:00:00.000Z | 3 +1995-03-13T00:00:00.000Z | 4 +1994-04-09T00:00:00.000Z | 5 +1997-05-19T00:00:00.000Z | null ; MultiTypedFieldsStatsByDateNanos -required_capability: implicit_casting_union_typed_numeric_and_date +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos FROM employees, employees_incompatible -| STATS x=count(emp_no), y=avg(salary_change), z=max(height) BY hire_date +| STATS x=count(emp_no::long), y=avg(salary_change::double), z=max(height::double) BY hire_date | Eval y = round(y, 1), z = round(z, 1) | KEEP x, y, z, hire_date | SORT hire_date @@ -1755,130 +1764,103 @@ x:long | y:double | z:double | hire_date:date_nanos 2 | 3.6 | 1.5 | 1985-09-17T00:00:00.000Z ; -MultiTypedFieldsWhereMvExpandKeepSortNumeric -required_capability: implicit_casting_union_typed_numeric_and_date +MultiTypedFieldsMvExpandKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos -FROM employees, employees_incompatible -| WHERE emp_no == 10003 -| MV_EXPAND salary_change -| KEEP emp_no, salary_change -| EVAL salary_change = round(salary_change, 2) -| SORT salary_change -; - -emp_no:long | salary_change:double -10003 | 12.82 -10003 | 12.82 -10003 | 14.68 -10003 | 14.68 +FROM date_nanos, date_nanos_lookup +| MV_EXPAND nanos +| SORT nanos +| LIMIT 4 ; -MultiTypedFieldsMvExpandKeepSortNumeric -required_capability: implicit_casting_union_typed_numeric_and_date +warning:Line 2:13: evaluation of [nanos] failed, treating result as null. Only first 20 failures recorded. +warning:Line 2:13: java.lang.IllegalArgumentException: milliSeconds [-1457696696640] are before the epoch in 1970 and cannot be converted to nanoseconds -FROM employees, employees_incompatible -| MV_EXPAND salary_change -| KEEP emp_no, salary_change -| EVAL salary_change = round(salary_change, 2) -| SORT emp_no, salary_change -| LIMIT 10 -; - -emp_no:long | salary_change:double -10001 | 1.19 -10001 | 1.19 -10002 | -7.23 -10002 | -7.23 -10002 | 11.17 -10002 | 11.17 -10003 | 12.82 -10003 | 12.82 -10003 | 14.68 -10003 | 14.68 -; - -MultiTypedFieldsEvalLookupJoinNumeric -required_capability: join_lookup_v12 -required_capability: implicit_casting_union_typed_numeric_and_date - -FROM employees, employees_incompatible -| EVAL language_code = languages -| LOOKUP JOIN languages_lookup ON language_code -| WHERE emp_no >= 10091 AND emp_no < 10094 -| KEEP emp_no, language_code, language_name -| SORT emp_no -; - -emp_no:long | language_code:integer | language_name:keyword -10091 | 3 | Spanish -10091 | 3 | Spanish -10092 | 1 | English -10092 | 1 | English -10093 | 3 | Spanish -10093 | 3 | Spanish +num:long | nanos:date_nanos | millis:date_nanos +0 | 2023-01-23T13:55:01.543Z | 1999-10-23T12:15:03.360103847Z +0 | 2023-01-23T13:55:01.543123456Z | 1999-10-23T12:15:03.360Z +0 | 2023-02-23T13:33:34.937Z | 1999-10-23T12:15:03.360103847Z +0 | 2023-02-23T13:33:34.937193Z | 1999-10-23T12:15:03.360Z ; -MultiTypedMVFields -required_capability: implicit_casting_union_typed_numeric_and_date +MultiTypedMVEval +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos -FROM employees, employees_incompatible -| WHERE emp_no <= 10002 -| EVAL x = ROUND(MV_MAX(salary_change),2) -| KEEP emp_no, x -| SORT emp_no +FROM date_nanos, date_nanos_lookup +| EVAL nanos = MV_MAX(nanos) +| SORT nanos | LIMIT 4 ; -emp_no:long | x:double -10001 | 1.19 -10001 | 1.19 -10002 | 11.17 -10002 | 11.17 +warning:Line 2:23: evaluation of [nanos] failed, treating result as null. Only first 20 failures recorded. +warning:Line 2:23: java.lang.IllegalArgumentException: milliSeconds [-1457696696640] are before the epoch in 1970 and cannot be converted to nanoseconds + +num:long | nanos:date_nanos | millis:date_nanos +0 | 2023-03-23T12:15:03.360Z | 1999-10-23T12:15:03.360103847Z +0 | 2023-03-23T12:15:03.360Z | 1999-10-22T12:15:03.360103847Z +0 | 2023-03-23T12:15:03.360103847Z | 1999-10-22T12:15:03.360Z +0 | 2023-03-23T12:15:03.360103847Z | 1999-10-23T12:15:03.360Z ; MultiTypedMVFieldsWhere -required_capability: implicit_casting_union_typed_numeric_and_date +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos -FROM employees, employees_incompatible -| WHERE emp_no <= 10002 AND ROUND(MV_MAX(salary_change),2) > 10 -| KEEP emp_no +FROM date_nanos, date_nanos_lookup +| WHERE millis <= "2023-10-23T13:00:00" AND MV_MAX(nanos) > "2023-03-23T13:00:00" +| KEEP millis, nanos, num +| SORT millis ; -emp_no:long -10002 -10002 +warning:Line 2:52: evaluation of [nanos] failed, treating result as null. Only first 20 failures recorded. +warning:Line 2:52: java.lang.IllegalArgumentException: milliSeconds [-1457696696640] are before the epoch in 1970 and cannot be converted to nanoseconds +millis:date_nanos | nanos:date_nanos | num:long +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 +2023-10-23T12:15:03.360103847Z | 2023-10-23T12:15:03.360Z | 1698063303360103847 +2023-10-23T12:15:03.360103847Z | 2023-10-23T12:15:03.360Z | 1698063303360103847 +2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948Z | 1698064048948000000 +2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948Z | 1698064048948000000 ; MultiTypedMVFieldsStatsMaxMin -required_capability: implicit_casting_union_typed_numeric_and_date +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos -FROM employees, employees_incompatible -| WHERE emp_no <= 10002 -| STATS max = MAX(salary_change), min = MIN(salary_change) -| EVAL x = round(max, 2), y = round(min, 2) -| KEEP x, y +FROM date_nanos, date_nanos_lookup +| STATS max = MAX(millis), min = MIN(nanos) ; -x:double |y:double -11.17 |-7.23 +warning:Line 2:38: evaluation of [nanos] failed, treating result as null. Only first 20 failures recorded. +warning:Line 2:38: java.lang.IllegalArgumentException: milliSeconds [-1457696696640] are before the epoch in 1970 and cannot be converted to nanoseconds + +max:date_nanos | min:date_nanos +2023-10-23T13:55:01.543123456Z | 2023-01-23T13:55:01.543Z ; MultiTypedMVFieldsStatsValues -required_capability: implicit_casting_union_typed_numeric_and_date +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos -FROM employees, employees_incompatible -| WHERE emp_no <= 10002 -| STATS c = MV_COUNT(VALUES(salary_change)) +FROM date_nanos, date_nanos_lookup +| STATS c = MV_COUNT(VALUES(nanos)) ; +warning:Line 2:29: evaluation of [nanos] failed, treating result as null. Only first 20 failures recorded. +warning:Line 2:29: java.lang.IllegalArgumentException: milliSeconds [-1457696696640] are before the epoch in 1970 and cannot be converted to nanoseconds + c:integer -6 +19 ; MultiTypedDateDiff -required_capability: implicit_casting_union_typed_numeric_and_date +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos FROM employees, employees_incompatible +| EVAL emp_no = emp_no::long | WHERE emp_no <= 10003 | EVAL diff_sec_1 = DATE_DIFF("seconds", TO_DATE_NANOS("1986-01-23T12:15:03.360103847Z"), hire_date) | EVAL diff_sec_2 = DATE_DIFF("seconds", TO_DATETIME("1986-01-23T12:15:03.360103847Z"), hire_date) @@ -1896,9 +1878,11 @@ emp_no:long | hire_date:date_nanos | diff_sec_1:integer | diff_sec_2:integer ; MultiTypedDateFormat -required_capability: implicit_casting_union_typed_numeric_and_date +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos FROM employees, employees_incompatible +| EVAL emp_no = emp_no::long | WHERE emp_no <= 10003 | EVAL a = DATE_FORMAT(hire_date), b = DATE_FORMAT("yyyy-MM-dd", hire_date), c = DATE_FORMAT("strict_date_optional_time_nanos", hire_date) | KEEP emp_no, hire_date, a, b, c @@ -1915,9 +1899,11 @@ emp_no:long | hire_date:date_nanos | a:keyword | b:keyword | ; MultiTypedDateTrunc -required_capability: implicit_casting_union_typed_numeric_and_date +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos FROM employees, employees_incompatible +| EVAL emp_no = emp_no::long | WHERE emp_no <= 10003 | EVAL yr = DATE_TRUNC(1 year, hire_date), mo = DATE_TRUNC(1 month, hire_date) | SORT hire_date DESC @@ -1934,10 +1920,11 @@ emp_no:long | hire_date:date_nanos | yr:date_nanos | mo:date_nano ; MultiTypedDateTruncStatsBy -required_capability: implicit_casting_union_typed_numeric_and_date +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos FROM employees, employees_incompatible -| STATS c = count(emp_no) BY yr = DATE_TRUNC(1 year, hire_date) +| STATS c = count(emp_no::long) BY yr = DATE_TRUNC(1 year, hire_date) | SORT yr DESC | LIMIT 5 ; @@ -1951,11 +1938,12 @@ c:long | yr:date_nanos ; MultiTypedDateTruncStatsByWithEval -required_capability: implicit_casting_union_typed_numeric_and_date +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos FROM employees, employees_incompatible | EVAL yr = DATE_TRUNC(1 year, hire_date) -| STATS c = count(emp_no) BY yr +| STATS c = count(emp_no::long) BY yr | SORT yr DESC | LIMIT 5 ; @@ -1969,7 +1957,8 @@ c:long | yr:date_nanos ; MultiTypedBucketDateNanosByYear -required_capability: implicit_casting_union_typed_numeric_and_date +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos FROM employees, employees_incompatible | STATS c = count(*) BY yr = BUCKET(hire_date, 1 year) @@ -1994,7 +1983,8 @@ c:long | yr:date_nanos ; MultiTypedBucketDateNanosByMonth -required_capability: implicit_casting_union_typed_numeric_and_date +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos FROM employees, employees_incompatible | STATS c = count(*) BY mo = BUCKET(hire_date, 20, "1986-01-01", "1999-12-31") @@ -2019,7 +2009,8 @@ c:long | mo:date_nanos ; MultiTypedBucketDateNanosInBothStatsAndBy -required_capability: implicit_casting_union_typed_numeric_and_date +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos FROM employees, employees_incompatible | STATS c = count(*), b = BUCKET(hire_date, 1 year) + 1 year BY yr = BUCKET(hire_date, 1 year) @@ -2044,7 +2035,8 @@ c:long | b:date_nanos | yr:date_nanos ; MultiTypedBucketDateNanosInBothStatsAndByWithAlias -required_capability: implicit_casting_union_typed_numeric_and_date +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos FROM employees, employees_incompatible | STATS c = count(*), b = yr + 1 year BY yr = BUCKET(hire_date, 1 year) @@ -2067,44 +2059,3 @@ c:long | b:date_nanos | yr:date_nanos 22 | 1987-01-01T00:00:00.000Z | 1986-01-01T00:00:00.000Z 22 | 1986-01-01T00:00:00.000Z | 1985-01-01T00:00:00.000Z ; - -MultiTypedEnrichOnNumericField -required_capability: enrich_load -required_capability: implicit_casting_union_typed_numeric_and_date - -from employees, employees_incompatible -| enrich languages_policy on languages -| keep emp_no, language_name -| sort emp_no -| limit 6 -; - -emp_no:long |language_name:keyword -10001 | French -10001 | French -10002 | null -10002 | null -10003 | German -10003 | German -; - -MultiTypedEnrichOnNumericFieldWithEval -required_capability: enrich_load -required_capability: implicit_casting_union_typed_numeric_and_date - -from employees, employees_incompatible -| eval languages = languages + 1 -| enrich languages_policy on languages -| keep emp_no, language_name -| sort emp_no -| limit 6 -; - -emp_no:long |language_name:keyword -10001 | Spanish -10001 | Spanish -10002 | null -10002 | null -10003 | null -10003 | null -; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 94eeb09daa847..f87ab9b8641c1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -308,9 +308,9 @@ public enum Cap { STRING_LITERAL_AUTO_CASTING_TO_DATETIME_ADD_SUB, /** - * Support implicit casting for union typed numeric and date/date_nanos fields + * Support implicit casting for union typed fields that are mixed with date and date_nanos type. */ - IMPLICIT_CASTING_UNION_TYPED_NUMERIC_AND_DATE, + IMPLICIT_CASTING_DATE_AND_DATE_NANOS(Build.current().isSnapshot()), /** * Support for named or positional parameters in EsqlQueryRequest. diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index fe933479dcb3c..396e6a3bebbbe 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -9,9 +9,9 @@ import org.elasticsearch.common.logging.HeaderWarning; import org.elasticsearch.common.logging.LoggerMessageFormat; +import org.elasticsearch.common.util.Maps; import org.elasticsearch.compute.data.Block; import org.elasticsearch.core.Strings; -import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexMode; import org.elasticsearch.logging.Logger; import org.elasticsearch.xpack.core.enrich.EnrichPolicy; @@ -48,13 +48,11 @@ import org.elasticsearch.xpack.esql.core.util.Holder; import org.elasticsearch.xpack.esql.core.util.StringUtils; import org.elasticsearch.xpack.esql.expression.NamedExpressions; -import org.elasticsearch.xpack.esql.expression.Order; import org.elasticsearch.xpack.esql.expression.UnresolvedNamePattern; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import org.elasticsearch.xpack.esql.expression.function.FunctionDefinition; import org.elasticsearch.xpack.esql.expression.function.UnresolvedFunction; import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute; -import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction; import org.elasticsearch.xpack.esql.expression.function.grouping.GroupingFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Case; @@ -92,10 +90,10 @@ import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.esql.plan.logical.Lookup; import org.elasticsearch.xpack.esql.plan.logical.MvExpand; -import org.elasticsearch.xpack.esql.plan.logical.OrderBy; import org.elasticsearch.xpack.esql.plan.logical.Project; import org.elasticsearch.xpack.esql.plan.logical.Rename; import org.elasticsearch.xpack.esql.plan.logical.RrfScoreEval; +import org.elasticsearch.xpack.esql.plan.logical.UnaryPlan; import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation; import org.elasticsearch.xpack.esql.plan.logical.inference.Completion; import org.elasticsearch.xpack.esql.plan.logical.inference.InferencePlan; @@ -155,7 +153,6 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.UNSUPPORTED; import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION; import static org.elasticsearch.xpack.esql.core.type.DataType.isMillisOrNanos; -import static org.elasticsearch.xpack.esql.core.type.DataType.isRepresentable; import static org.elasticsearch.xpack.esql.core.type.DataType.isTemporalAmount; import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.LIMIT; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.maybeParseTemporalAmount; @@ -190,7 +187,9 @@ public class Analyzer extends ParameterizedRuleExecutor("Finish Analysis", Limiter.ONCE, new AddImplicitLimit(), new AddImplicitForkLimit(), new UnionTypesCleanup()) ); @@ -1177,8 +1176,7 @@ private LogicalPlan resolveEnrich(Enrich enrich, List childrenOutput) final DataType dataType = resolved.dataType(); String matchType = enrich.policy().getType(); DataType[] allowed = allowedEnrichTypes(matchType); - // leave multi-typed fields to ImplicitCasting to cast to a common type - if (Arrays.asList(allowed).contains(dataType) == false && multiTypedField(resolved) == null) { + if (Arrays.asList(allowed).contains(dataType) == false) { String suffix = "only [" + Arrays.stream(allowed).map(DataType::typeName).collect(Collectors.joining(", ")) + "] allowed for type [" @@ -1360,35 +1358,14 @@ private BitSet gatherPreAnalysisMetrics(LogicalPlan plan, BitSet b) { private static class ImplicitCasting extends ParameterizedRule { @Override public LogicalPlan apply(LogicalPlan plan, AnalyzerContext context) { - // Do implicit casting for union typed fields in the following commands - LogicalPlan newPlan = plan.transformUp(p -> { - if (p instanceof OrderBy - || p instanceof EsqlProject - || p instanceof Aggregate - || p instanceof MvExpand - || p instanceof Eval - || p instanceof LookupJoin - || p instanceof Enrich) { - return castInvalidMappedFieldInLogicalPlan(p, false); - } - return p; - }); - - // Add an implicit EsqlProject on top of the whole query if there isn't one yet, - // without explicit or implicit casting, a union typed field is returned as a null. - newPlan = addImplicitProjectForInvalidMappedFields(newPlan); - // do implicit casting for function arguments - return newPlan.transformExpressionsUp( + return plan.transformExpressionsUp( org.elasticsearch.xpack.esql.core.expression.function.Function.class, e -> ImplicitCasting.cast(e, context.functionRegistry().snapshotRegistry()) ); } private static Expression cast(org.elasticsearch.xpack.esql.core.expression.function.Function f, EsqlFunctionRegistry registry) { - // Add cast functions to InvalidMappedField if there isn't one yet - f = castInvalidMappedFieldInFunction(f); - if (f instanceof In in) { return processIn(in); } @@ -1401,232 +1378,6 @@ private static Expression cast(org.elasticsearch.xpack.esql.core.expression.func return f; } - /** - * Both the logical plan's children and the logical plan itself can be changed, so we cannot just to a replaceChild here. - */ - private static LogicalPlan castInvalidMappedFieldInLogicalPlan(LogicalPlan plan, boolean addProject) { - List originalExpressions; - if (addProject) { - originalExpressions = plan.output(); - } else { - originalExpressions = switch (plan) { - case EsqlProject project -> project.projections(); - case OrderBy ob -> ob.order(); - case Aggregate agg -> agg.groupings(); - case MvExpand me -> List.of(me.target()); - case Eval eval -> eval.fields(); - case LookupJoin lj -> lj.config().leftFields(); - case Enrich enrich -> List.of(enrich.matchField()); - // The other types of plans are unexpected - default -> throw new EsqlIllegalArgumentException("unexpected logical plan: " + plan); - }; - } - Tuple, List> newChildren = castInvalidMappedFields(originalExpressions, true); - List aliases = newChildren.v1(); - List newProjections = newChildren.v2(); - if (aliases.isEmpty() == false) { - if (addProject) { - return esqlProjectForInvalidMappedField(plan.source(), plan, aliases, newProjections); - } - switch (plan) { - case EsqlProject p -> { - return esqlProjectForInvalidMappedField(p.source(), p.child(), aliases, newProjections); - } - case OrderBy o -> { - return new OrderBy( - o.source(), - o.child(), - newProjections.stream().filter(e -> e instanceof Order).map(e -> (Order) e).collect(Collectors.toList()) - ); - } - case Aggregate agg -> { - // Both groupings and aggregates need to be updated, create new aggregates according to new groupings - List origAggs = agg.aggregates(); - List newAggs = new ArrayList<>(origAggs.size()); - for (int i = 0; i < origAggs.size() - newProjections.size(); i++) { // Add aggregate functions - newAggs.add(origAggs.get(i)); - } - for (Expression e : newProjections) { // Add groupings - newAggs.add(Expressions.attribute(e)); - } - return agg.with(evalForInvalidMappedField(agg.source(), agg.child(), aliases), newProjections, newAggs); - } - case MvExpand mve -> { - NamedExpression newTarget = Expressions.attribute(newProjections.get(0)); - return new MvExpand( - mve.source(), - evalForInvalidMappedField(mve.source(), mve.child(), aliases), - newTarget, - newTarget.toAttribute() - ); - } - case Eval ev -> { - return evalForInvalidMappedField(ev.source(), ev.child(), aliases); - } - case LookupJoin lj -> { - JoinConfig oldJoinConfig = lj.config(); - List leftKeys = newProjections.stream() - .filter(e -> e instanceof Attribute) - .map(e -> (Attribute) e) - .collect(Collectors.toList()); - JoinConfig newJoinConfig = new JoinConfig(oldJoinConfig.type(), leftKeys, leftKeys, oldJoinConfig.rightFields()); - return new LookupJoin( - lj.source(), - evalForInvalidMappedField(lj.source(), lj.left(), aliases), - lj.right(), - newJoinConfig - ); - } - case Enrich enrich -> { - NamedExpression newMatchField = Expressions.attribute(newProjections.get(0)); - return new Enrich( - enrich.source(), - evalForInvalidMappedField(enrich.source(), enrich.child(), aliases), - enrich.mode(), - enrich.policyName(), - // Let resolveEnrich check whether the data type is supported, e.g. Enrich does not support data_nanos - new UnresolvedAttribute(newMatchField.source(), newMatchField.name()), - enrich.policy(), - enrich.concreteIndices(), - enrich.enrichFields() - ); - } - // The other types of plans are unexpected - default -> throw new EsqlIllegalArgumentException("unexpected logical plan: " + plan); - } - } else { - // Double check Aggs, when Bucket is used together with Aggs, the Aggs may need two rounds of resolutions - if (plan instanceof Aggregate agg) { - // If there is unresolved reference in the aggregates try to get it resolved again by removing the custom message - List origAggs = agg.aggregates(); - List groupings = agg.groupings(); - int aggsCount = origAggs.size() - groupings.size(); - List newAggs = new ArrayList<>(origAggs.size()); - for (int i = 0; i < origAggs.size(); i++) { - var e = origAggs.get(i); - if (i < aggsCount) { // Add aggregate functions - newAggs.add(e); - } else { // Add groupings - if (e instanceof UnresolvedAttribute ua && ua.customMessage()) { - // Try to make ResolveRefs resolve the aggregates again by removing the custom message - newAggs.add(Expressions.attribute(groupings.get(i - aggsCount))); - } else { - newAggs.add(e); - } - } - } - return agg.with(groupings, newAggs); - } - return plan; - } - } - - private static LogicalPlan addImplicitProjectForInvalidMappedFields(LogicalPlan logicalPlan) { - if (logicalPlan.resolved() == false) { - return logicalPlan; - } - List projections = logicalPlan.collectFirstChildren(EsqlProject.class::isInstance); - return projections.isEmpty() ? castInvalidMappedFieldInLogicalPlan(logicalPlan, true) : logicalPlan; - } - - private static org.elasticsearch.xpack.esql.core.expression.function.Function castInvalidMappedFieldInFunction( - org.elasticsearch.xpack.esql.core.expression.function.Function f - ) { - // No need to add implicit casting for union typed fields that already have explicit casting on them. - // Full text functions are pushdown only functions, implicit or explicit casting may fail the query. - if (f instanceof AbstractConvertFunction || f instanceof FullTextFunction) { - return f; - } - Tuple, List> newChildren = castInvalidMappedFields(f.arguments(), false); - return newChildren.v1().isEmpty() - ? f - : (org.elasticsearch.xpack.esql.core.expression.function.Function) f.replaceChildren(newChildren.v2()); - } - - private static Tuple, List> castInvalidMappedFields( - List originalExpressions, - boolean createNewChildPlan - ) { - List newAliases = new ArrayList<>(originalExpressions.size()); - List newExpressions = new ArrayList<>(originalExpressions.size()); - expressionLoop: for (Expression exp : originalExpressions) { - Expression e = Alias.unwrap(exp); - e = e instanceof Order o ? o.child() : e; - String alias = exp instanceof Alias a ? a.name() : null; - InvalidMappedField multiTypedField = multiTypedField(e); - if (multiTypedField != null) { - // This is an invalid mapped field, find a common data type and cast to it. - DataType targetType = null; - for (DataType type : multiTypedField.types()) { - if (targetType == null) { // Initialize the target type to the first type. - targetType = type; - } else { - targetType = EsqlDataTypeConverter.commonType(targetType, type); - if (targetType == null) { // If there is no common type, continue to the next expression. - newExpressions.add(exp); - continue expressionLoop; - } - } - } - if (targetType != null && isRepresentable(targetType) && (isMillisOrNanos(targetType) || targetType.isNumeric())) { - alias = alias != null ? alias : multiTypedField.getName(); - Source source = e.source(); - Expression newChild = castInvalidMappedField(targetType, e); - Alias newAlias = new Alias(source, alias, newChild); - newAliases.add(newAlias); - if (createNewChildPlan) { - // Cast union typed fields in a logical plan, a new child plan(Eval) is needed for the implicit casting and new - // aliases. The newExpressions with all the fields and references are need to create a new plan. - switch (exp) { - case Alias a -> newExpressions.add(a.replaceChild(newAlias.toAttribute())); - case Order o -> newExpressions.add(new Order(o.source(), newChild, o.direction(), o.nullsPosition())); - default -> newExpressions.add(newAlias.toAttribute()); - } - } else { // Cast union typed fields in a function, there is no need to create a new child plan - newExpressions.add(newChild); - } - } - } else { - newExpressions.add(exp); - } - } - return Tuple.tuple(newAliases, newExpressions); - } - - private static EsqlProject esqlProjectForInvalidMappedField( - Source source, - LogicalPlan childPlan, - List aliases, - List newProjections - ) { - Eval eval = evalForInvalidMappedField(source, childPlan, aliases); - return new EsqlProject( - source, - eval, - newProjections.stream().filter(e -> e instanceof NamedExpression).map(e -> (NamedExpression) e).collect(Collectors.toList()) - ); - } - - private static Eval evalForInvalidMappedField(Source source, LogicalPlan childPlan, List aliases) { - return new Eval(source, childPlan, aliases); - } - - /** - * Do implicit casting for data, date_nanos and numeric types only - */ - private static Expression castInvalidMappedField(DataType targetType, Expression fa) { - Source source = fa.source(); - return switch (targetType) { - case INTEGER -> new ToInteger(source, fa); - case LONG -> new ToLong(source, fa); - case DOUBLE -> new ToDouble(source, fa); - case UNSIGNED_LONG -> new ToUnsignedLong(source, fa); - case DATETIME -> new ToDatetime(source, fa); - case DATE_NANOS -> new ToDateNanos(source, fa); - default -> throw new EsqlIllegalArgumentException("unexpected data type: " + targetType); - }; - } - private static Expression processScalarOrGroupingFunction( org.elasticsearch.xpack.esql.core.expression.function.Function f, EsqlFunctionRegistry registry @@ -2050,4 +1801,177 @@ private static InvalidMappedField multiTypedField(Expression e) { } return null; } + + /** + * Cast union typed fields that are mixed of date and date_nanos types into date_nanos. + */ + private static class ImplicitCastingForUnionTypedFields extends ParameterizedRule { + @Override + public LogicalPlan apply(LogicalPlan plan, AnalyzerContext context) { + // This rule should be applied after ResolveUnionTypes, so that the InvalidMappedFields with explicit casting are converted into + // MultiTypeEsField, and don't be double cast here. + Map invalidMappedFieldCasted = new HashMap<>(); + LogicalPlan transformedPlan = plan.transformUp(LogicalPlan.class, p -> { + if (p instanceof UnaryPlan == false && p instanceof LookupJoin == false) { + return p; + } + Set invalidMappedFields = invalidMappedFieldsInLogicalPlan(p); + if (invalidMappedFields.isEmpty() == false) { + // If we are at a plan node that has invalid mapped fields, we need to either add an EVAL, or if that has been done + // we should instead replace with the already cast field + Map newAliases = Maps.newHashMapWithExpectedSize(invalidMappedFields.size()); + Map existingAliases = Maps.newHashMapWithExpectedSize(invalidMappedFields.size()); + for (FieldAttribute fa : invalidMappedFields) { + if (invalidMappedFieldCasted.containsKey(fa)) { + // There is already an eval plan created for the implicit cast field, just reference to it + Alias alias = invalidMappedFieldCasted.get(fa); + existingAliases.put(fa, alias); + } else { + // Create a new alias and later on add a new EVAL with this new aliases for implicit casting + DataType targetType = commonDataType(fa); + if (targetType != null) { + Expression conversionFunction = castInvalidMappedField(targetType, fa); + Alias alias = new Alias(fa.source(), fa.name(), conversionFunction); + newAliases.put(fa, alias); + invalidMappedFieldCasted.put(fa, alias); + } + } + } + // If there are new aliases created, create a new eval child with new aliases for the current plan。 + // How many children does a LogicalPlan have? Only deal with UnaryPlan and LookupJoin for now. + if (newAliases.isEmpty() == false) { // create a new eval child plan + if (p instanceof UnaryPlan u) { // unary plan + Eval eval = new Eval(u.source(), u.child(), newAliases.values().stream().toList()); + p = u.replaceChild(eval); + } + // TODO Lookup join does not work on date_nanos field today, joining on a date_nanos field does not find a match. + // And lookup up join is a special case as a lookup join has two children, after date_nanos is supported as a join + // key, the transformation needs to take it into account. + } + // If there are new or existing aliases identified, combine them into one map + Map allAliases = Maps.newHashMapWithExpectedSize(invalidMappedFields.size()); + allAliases.putAll(newAliases); + allAliases.putAll(existingAliases); + if (allAliases.isEmpty() == false) { // there is already eval plan for that union typed field, reference to the aliases + p = p.transformExpressionsOnly(FieldAttribute.class, fa -> { + Alias alias = allAliases.get(fa); + return alias != null ? alias.toAttribute() : fa; + }); + // MvExpand and Stats have ReferenceAttribute referencing the FieldAttribute in the same plan. + // The ReferenceAttribute need to be updated to point to the casting expression. + if (p instanceof MvExpand mvExpand) { + p = transformMvExpand(mvExpand); + } else if (p instanceof Aggregate aggregate) { + p = transformAggregate(aggregate); + } + } + } + return p; + }); + transformedPlan = castInvalidMappedFieldInFinalOutput(transformedPlan); + return transformedPlan; + } + + /** + * Find a common data type that the union typed field can cast to, only date and date_nanos types are supported. + * This method can be extended to support implicit casting for the other data types. + */ + private static DataType commonDataType(FieldAttribute unionTypedField) { + DataType targetType = null; + if (unionTypedField.field() instanceof InvalidMappedField imf) { + for (DataType type : imf.types()) { + if (isMillisOrNanos(type) == false) { // if there is field that is no date or date_nanos, don't do implicit casting + return null; + } + if (targetType == null) { // initialize the target type to the first type + targetType = type; + } else if (targetType == DATE_NANOS || type == DATE_NANOS) { + targetType = DATE_NANOS; + } + } + } + return targetType; + } + + /** + * Do implicit casting for date and date_nanos only. + */ + private static Expression castInvalidMappedField(DataType targetType, FieldAttribute fa) { + Source source = fa.source(); + return switch (targetType) { + case DATETIME -> new ToDatetime(source, fa); // in case we decided to use DATE as a common type instead of DATE_NANOS + case DATE_NANOS -> new ToDateNanos(source, fa); + default -> throw new EsqlIllegalArgumentException("unexpected data type: " + targetType); + }; + } + + /** + * Return all the FieldAttribute that contain InvalidMappedField in the current plan. + */ + private static Set invalidMappedFieldsInLogicalPlan(LogicalPlan plan) { + Set fas = new HashSet<>(); + if (plan instanceof EsRelation == false) { // not all the union typed fields from indices need to be cast implicitly + plan.forEachExpression(FieldAttribute.class, fa -> { + if (fa.field() instanceof InvalidMappedField) { + fas.add(fa); + } + }); + } + return fas; + } + + /** + * Cast the InvalidMappedFields in the final output of the query, this is needed when these fields are not referenced in the query + * explicitly, so there is no chance to cast them to a common type earlier, an example of such query is from index*. + */ + private static LogicalPlan castInvalidMappedFieldInFinalOutput(LogicalPlan logicalPlan) { + // Check the output of the query, if the top level plan is resolved, check if there is InvalidMappedField in its output, + // if so add a project with eval, so that a not null value can be returned for a union typed field + if (logicalPlan.resolved()) { + List output = logicalPlan.output(); + Map newAliases = Maps.newHashMapWithExpectedSize(output.size()); + output.forEach(a -> { + if (a instanceof FieldAttribute fa && fa.field() instanceof InvalidMappedField) { + DataType targetType = commonDataType(fa); + if (targetType != null) { + Expression conversionFunction = castInvalidMappedField(targetType, fa); + Alias alias = new Alias(fa.source(), fa.name(), conversionFunction); + newAliases.put(fa, alias); + } + } + }); + if (newAliases.isEmpty() == false) { // add an Eval for the union typed fields left that are not cast implicitly yet + if (logicalPlan instanceof EsRelation esr) { + // EsRelation does not have a child, we should not see row here, add a eval on top of it + logicalPlan = new Eval(esr.source(), esr, newAliases.values().stream().toList()); + } else if (logicalPlan instanceof UnaryPlan unary) { + // Add an Eval as the child of this plan + Eval eval = new Eval(unary.source(), unary.child(), newAliases.values().stream().toList()); + logicalPlan = unary.replaceChild(eval); + } + // TODO LookupJoin is a binary plan, it does not create a new field, ideally adding an Eval on top of it should be fine, + // however because the output of a LookupJoin does not include InvalidMappedFields, it is a bug need to be addressed + } + } + return logicalPlan; + } + + private static MvExpand transformMvExpand(MvExpand mvExpand) { + NamedExpression target = mvExpand.target(); + return new MvExpand(mvExpand.source(), mvExpand.child(), target, target.toAttribute()); + } + + private static Aggregate transformAggregate(Aggregate aggregate) { + List aggregates = aggregate.aggregates(); + List groupings = aggregate.groupings(); + List aggregatesWithNewRefs = new ArrayList<>(aggregates.size()); + for (int i = 0; i < aggregates.size() - groupings.size(); i++) { + aggregatesWithNewRefs.add(aggregates.get(i)); + } + for (Expression e : groupings) { // Add groupings + aggregatesWithNewRefs.add(Expressions.attribute(e)); + } + return aggregate.with(groupings, aggregatesWithNewRefs); + } + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java index e1d501dab8ac9..6cde1f28535dd 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java @@ -404,9 +404,6 @@ public static DataType commonType(DataType left, DataType right) { if (isNullOrDatePeriod(left) && isNullOrDatePeriod(right)) { return DATE_PERIOD; } - if ((isDateTime(left) && right == DATE_NANOS) || (left == DATE_NANOS && isDateTime(right))) { - return DATE_NANOS; - } } if (isString(left) && isString(right)) { // Both TEXT and SEMANTIC_TEXT are processed as KEYWORD diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index dab59a17805b1..8781c49ccc435 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -35,6 +35,9 @@ import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.type.EsField; +import org.elasticsearch.xpack.esql.core.type.InvalidMappedField; +import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField; import org.elasticsearch.xpack.esql.core.type.PotentiallyUnmappedKeywordEsField; import org.elasticsearch.xpack.esql.enrich.ResolvedEnrichPolicy; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; @@ -70,6 +73,7 @@ import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.esql.plan.logical.Lookup; import org.elasticsearch.xpack.esql.plan.logical.OrderBy; +import org.elasticsearch.xpack.esql.plan.logical.Project; import org.elasticsearch.xpack.esql.plan.logical.Row; import org.elasticsearch.xpack.esql.plan.logical.RrfScoreEval; import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation; @@ -81,6 +85,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -3760,6 +3765,71 @@ public void testResolveCompletionOutputField() { assertThat(getAttributeByName(esRelation.output(), "description"), not(equalTo(completion.targetField()))); } + public void testImplicitCastingForDateAndDateNanosFields() { + String dateDateNanos = "date_and_date_nanos"; // mixed date and date_nanos + String dateDateNanosLong = "date_and_date_nanos_and_long"; // mixed date, date_nanos and long + LinkedHashMap> typesToIndices1 = new LinkedHashMap<>(); + typesToIndices1.put("date", Set.of("index1", "index2")); + typesToIndices1.put("date_nanos", Set.of("index3")); + LinkedHashMap> typesToIndices2 = new LinkedHashMap<>(); + typesToIndices2.put("date", Set.of("index1")); + typesToIndices2.put("date_nanos", Set.of("index2")); + typesToIndices2.put("long", Set.of("index3")); + EsField dateDateNanosField = new InvalidMappedField(dateDateNanos, typesToIndices1); + EsField dateDateNanosLongField = new InvalidMappedField(dateDateNanosLong, typesToIndices2); + + IndexResolution indexWithUnionTypedFields = IndexResolution.valid( + new EsIndex("test*", Map.of(dateDateNanos, dateDateNanosField, dateDateNanosLong, dateDateNanosLongField)) + ); + Analyzer analyzer = AnalyzerTestUtils.analyzer(indexWithUnionTypedFields); + + // validate if a union typed field is cast to a type explicitly, implicit casting won't be applied again + LogicalPlan plan = analyze(""" + FROM tests + | Eval x = date_and_date_nanos::datetime, y = date_and_date_nanos + """, analyzer); + + Project project = as(plan, Project.class); + List projections = project.projections(); + assertEquals(4, projections.size()); + UnsupportedAttribute ua = as(projections.get(0), UnsupportedAttribute.class); // long is not casted to date_nanos + assertEquals("date_and_date_nanos_and_long", ua.name()); + assertEquals(DataType.UNSUPPORTED, ua.dataType()); + ReferenceAttribute ra = as(projections.get(1), ReferenceAttribute.class); + assertEquals("date_and_date_nanos", ra.name()); + assertEquals(DataType.DATE_NANOS, ra.dataType()); // implicit casting + ra = as(projections.get(2), ReferenceAttribute.class); + assertEquals("x", ra.name()); + assertEquals(DataType.DATETIME, ra.dataType()); // explicit casting + ra = as(projections.get(3), ReferenceAttribute.class); + assertEquals("y", ra.name()); + assertEquals(DataType.DATE_NANOS, ra.dataType()); // implicit casting + Limit limit = as(project.child(), Limit.class); + Eval eval = as(limit.child(), Eval.class); // eval coded in the query + List aliases = eval.fields(); + assertEquals(2, aliases.size()); + Alias a = aliases.get(0); + assertEquals("x", a.name()); + assertEquals(DataType.DATETIME, a.dataType()); // explicit casting + assertTrue(isMultiTypeEsField(a.child())); // no double casting + a = aliases.get(1); + assertEquals("y", a.name()); + assertEquals(DataType.DATE_NANOS, a.dataType()); // implicit casting + assertTrue(a.child() instanceof ReferenceAttribute); + eval = as(eval.child(), Eval.class); // a new eval added for implicit casting + aliases = eval.fields(); + assertEquals(1, aliases.size()); + a = aliases.get(0); + assertEquals("date_and_date_nanos", a.name()); + assertEquals(DataType.DATE_NANOS, a.dataType()); + assertTrue(isMultiTypeEsField(a.child())); + EsRelation esRelation = as(eval.child(), EsRelation.class); + } + + private boolean isMultiTypeEsField(Expression e) { + return e instanceof FieldAttribute fa && fa.field() instanceof MultiTypeEsField; + } + @Override protected IndexAnalyzers createDefaultIndexAnalyzers() { return super.createDefaultIndexAnalyzers(); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 56846a5adb017..e75cd5ea05a6c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -135,8 +135,8 @@ public void testUnsupportedAndMultiTypedFields() { error("from test* | enrich client_cidr on unsupported", analyzer) ); assertEquals( - "1:36: Cannot use field [multi_typed] due to ambiguities being mapped as [2] incompatible types: " - + "[ip] in [test1, test2, test3] and [2] other indices, [keyword] in [test6]", + "1:36: Unsupported type [unsupported] for enrich matching field [multi_typed];" + + " only [keyword, text, ip, long, integer, float, double, datetime] allowed for type [range]", error("from test* | enrich client_cidr on multi_typed", analyzer) ); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverterTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverterTests.java index 487a2ee111c03..9a30c2281d742 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverterTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverterTests.java @@ -97,7 +97,7 @@ public void testCommonTypeDateTimeIntervals() { assertEqualsCommonType(dataType1, NULL, dataType1); } else if (isDateTimeOrNanosOrTemporal(dataType2)) { if ((dataType1 == DATE_NANOS && dataType2 == DATETIME) || (dataType1 == DATETIME && dataType2 == DATE_NANOS)) { - assertEqualsCommonType(dataType1, dataType2, DATE_NANOS); + assertNullCommonType(dataType1, dataType2); } else if (isDateTime(dataType1) || isDateTime(dataType2)) { assertEqualsCommonType(dataType1, dataType2, DATETIME); } else if (dataType1 == DATE_NANOS || dataType2 == DATE_NANOS) { From 4fb446bd0bd7f6964faf8f8006ff5c6082aae5ea Mon Sep 17 00:00:00 2001 From: Fang Xing Date: Mon, 21 Apr 2025 14:11:17 -0400 Subject: [PATCH 12/24] update tests --- .../src/main/resources/union_types.csv-spec | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec index 9a593c5ce49f9..f56eb079a8646 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec @@ -1640,7 +1640,7 @@ id:integer | name:keyword | count:long 14 | mmmmm | 2 ; -MultiTypedFieldsKeepSort +ImplicitCastingMultiTypedFieldsKeepSort required_capability: date_nanos_type required_capability: implicit_casting_date_and_date_nanos @@ -1656,7 +1656,7 @@ x:long |languages:unsupported |hire_date:date_nanos |avg_worked_seconds:unsi 10001 |null |1986-06-26T00:00:00.000Z |268728049 |1986-06-26T00:00:00.000Z ; -MultiTypedFieldsDropKeepSort +ImplicitCastingMultiTypedFieldsDropKeepSort required_capability: date_nanos_type required_capability: implicit_casting_date_and_date_nanos @@ -1673,7 +1673,7 @@ emp_no:long |languages:unsupported |hire_date:date_nanos |avg_worked_seconds 10001 |null |1986-06-26T00:00:00.000Z |268728049 |1986-06-26T00:00:00.000Z ; -MultiTypedFieldsRenameKeepSort +ImplicitCastingMultiTypedFieldsRenameKeepSort required_capability: date_nanos_type required_capability: implicit_casting_date_and_date_nanos @@ -1690,7 +1690,7 @@ new_emp_no:long |new_languages:integer |new_hire_date:date_nanos |new_avg_worked 10001 |2 |1986-06-26T00:00:00.000Z |268728049 |1986-06-26T00:00:00.000Z ; -MultiTypedFieldsDropEvalKeepSort +ImplicitCastingMultiTypedFieldsDropEvalKeepSort required_capability: date_nanos_type required_capability: implicit_casting_date_and_date_nanos @@ -1707,7 +1707,7 @@ x:long |hire_date:date_nanos |y:date_nanos |z:date 10002 |1986-06-26T00:00:00.000Z |1987-06-26T00:00:00.000Z |1986-06-26T00:00:00.000Z ; -MultiTypedFieldsEvalFilterKeepSort +ImplicitCastingMultiTypedFieldsEvalFilterKeepSort required_capability: date_nanos_type required_capability: implicit_casting_date_and_date_nanos @@ -1726,7 +1726,7 @@ x:long |hire_date:date_nanos 10018 | 1993-08-03T00:00:00.000Z ; -MultiTypedFieldsStatsByNumeric +ImplicitCastingMultiTypedFieldsStatsByNumeric required_capability: date_nanos_type required_capability: implicit_casting_date_and_date_nanos @@ -1744,7 +1744,7 @@ x:date_nanos | languages:integer 1997-05-19T00:00:00.000Z | null ; -MultiTypedFieldsStatsByDateNanos +ImplicitCastingMultiTypedFieldsStatsByDateNanos required_capability: date_nanos_type required_capability: implicit_casting_date_and_date_nanos @@ -1764,7 +1764,7 @@ x:long | y:double | z:double | hire_date:date_nanos 2 | 3.6 | 1.5 | 1985-09-17T00:00:00.000Z ; -MultiTypedFieldsMvExpandKeepSort +ImplicitCastingMultiTypedFieldsMvExpandKeepSort required_capability: date_nanos_type required_capability: implicit_casting_date_and_date_nanos @@ -1784,13 +1784,13 @@ num:long | nanos:date_nanos | millis:date_nanos 0 | 2023-02-23T13:33:34.937193Z | 1999-10-23T12:15:03.360Z ; -MultiTypedMVEval +ImplicitCastingMultiTypedMVFieldsEval required_capability: date_nanos_type required_capability: implicit_casting_date_and_date_nanos FROM date_nanos, date_nanos_lookup | EVAL nanos = MV_MAX(nanos) -| SORT nanos +| SORT nanos, millis | LIMIT 4 ; @@ -1798,13 +1798,13 @@ warning:Line 2:23: evaluation of [nanos] failed, treating result as null. Only f warning:Line 2:23: java.lang.IllegalArgumentException: milliSeconds [-1457696696640] are before the epoch in 1970 and cannot be converted to nanoseconds num:long | nanos:date_nanos | millis:date_nanos -0 | 2023-03-23T12:15:03.360Z | 1999-10-23T12:15:03.360103847Z 0 | 2023-03-23T12:15:03.360Z | 1999-10-22T12:15:03.360103847Z +0 | 2023-03-23T12:15:03.360Z | 1999-10-23T12:15:03.360103847Z 0 | 2023-03-23T12:15:03.360103847Z | 1999-10-22T12:15:03.360Z 0 | 2023-03-23T12:15:03.360103847Z | 1999-10-23T12:15:03.360Z ; -MultiTypedMVFieldsWhere +ImplicitCastingMultiTypedMVFieldsWhere required_capability: date_nanos_type required_capability: implicit_casting_date_and_date_nanos @@ -1825,7 +1825,7 @@ millis:date_nanos | nanos:date_nanos | num:long 2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948Z | 1698064048948000000 ; -MultiTypedMVFieldsStatsMaxMin +ImplicitCastingMultiTypedMVFieldsStatsMaxMin required_capability: date_nanos_type required_capability: implicit_casting_date_and_date_nanos @@ -1840,7 +1840,7 @@ max:date_nanos | min:date_nanos 2023-10-23T13:55:01.543123456Z | 2023-01-23T13:55:01.543Z ; -MultiTypedMVFieldsStatsValues +ImplicitCastingMultiTypedMVFieldsStatsValues required_capability: date_nanos_type required_capability: implicit_casting_date_and_date_nanos @@ -1855,7 +1855,7 @@ c:integer 19 ; -MultiTypedDateDiff +ImplicitCastingMultiTypedDateDiff required_capability: date_nanos_type required_capability: implicit_casting_date_and_date_nanos @@ -1877,7 +1877,7 @@ emp_no:long | hire_date:date_nanos | diff_sec_1:integer | diff_sec_2:integer 10003 | 1986-08-28T00:00:00.000Z | 18704696 | 18704696 ; -MultiTypedDateFormat +ImplicitCastingMultiTypedDateFormat required_capability: date_nanos_type required_capability: implicit_casting_date_and_date_nanos @@ -1898,7 +1898,7 @@ emp_no:long | hire_date:date_nanos | a:keyword | b:keyword | 10003 | 1986-08-28T00:00:00.000Z | 1986-08-28T00:00:00.000Z | 1986-08-28 | 1986-08-28T00:00:00.000Z ; -MultiTypedDateTrunc +ImplicitCastingMultiTypedDateTrunc required_capability: date_nanos_type required_capability: implicit_casting_date_and_date_nanos @@ -1919,7 +1919,7 @@ emp_no:long | hire_date:date_nanos | yr:date_nanos | mo:date_nano 10002 | 1985-11-21T00:00:00.000Z | 1985-01-01T00:00:00.000Z | 1985-11-01T00:00:00.000Z ; -MultiTypedDateTruncStatsBy +ImplicitCastingMultiTypedDateTruncStatsBy required_capability: date_nanos_type required_capability: implicit_casting_date_and_date_nanos @@ -1937,7 +1937,7 @@ c:long | yr:date_nanos 8 | 1994-01-01T00:00:00.000Z ; -MultiTypedDateTruncStatsByWithEval +ImplicitCastingMultiTypedDateTruncStatsByWithEval required_capability: date_nanos_type required_capability: implicit_casting_date_and_date_nanos @@ -1956,7 +1956,7 @@ c:long | yr:date_nanos 8 | 1994-01-01T00:00:00.000Z ; -MultiTypedBucketDateNanosByYear +ImplicitCastingMultiTypedBucketDateNanosByYear required_capability: date_nanos_type required_capability: implicit_casting_date_and_date_nanos @@ -1982,7 +1982,7 @@ c:long | yr:date_nanos 22 | 1985-01-01T00:00:00.000Z ; -MultiTypedBucketDateNanosByMonth +ImplicitCastingMultiTypedBucketDateNanosByMonth required_capability: date_nanos_type required_capability: implicit_casting_date_and_date_nanos @@ -2008,7 +2008,7 @@ c:long | mo:date_nanos 22 | 1985-01-01T00:00:00.000Z ; -MultiTypedBucketDateNanosInBothStatsAndBy +ImplicitCastingMultiTypedBucketDateNanosInBothStatsAndBy required_capability: date_nanos_type required_capability: implicit_casting_date_and_date_nanos @@ -2034,7 +2034,7 @@ c:long | b:date_nanos | yr:date_nanos 22 | 1986-01-01T00:00:00.000Z | 1985-01-01T00:00:00.000Z ; -MultiTypedBucketDateNanosInBothStatsAndByWithAlias +ImplicitCastingMultiTypedBucketDateNanosInBothStatsAndByWithAlias required_capability: date_nanos_type required_capability: implicit_casting_date_and_date_nanos From 00b62295ff0216cf665cad1a529980a849586ebb Mon Sep 17 00:00:00 2001 From: Fang Xing Date: Mon, 21 Apr 2025 16:13:21 -0400 Subject: [PATCH 13/24] fix tests --- .../elasticsearch/xpack/esql/CsvTestsDataLoader.java | 4 ++-- ...te_nanos_lookup.csv => date_nanos_union_types.csv} | 0 ...ookup.json => mapping-date_nanos_union_types.json} | 0 .../src/main/resources/union_types.csv-spec | 10 +++++----- .../elasticsearch/xpack/esql/analysis/Analyzer.java | 11 ----------- 5 files changed, 7 insertions(+), 18 deletions(-) rename x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/{date_nanos_lookup.csv => date_nanos_union_types.csv} (100%) rename x-pack/plugin/esql/qa/testFixtures/src/main/resources/{mapping-date_nanos_lookup.json => mapping-date_nanos_union_types.json} (100%) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index f9da5c60c754d..210ff765e8497 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -125,7 +125,7 @@ public class CsvTestsDataLoader { ); private static final TestDataset AIRPORTS_WEB = new TestDataset("airports_web"); private static final TestDataset DATE_NANOS = new TestDataset("date_nanos"); - private static final TestDataset DATE_NANOS_LOOKUP = new TestDataset("date_nanos_lookup").withSetting("lookup-settings.json"); + private static final TestDataset DATE_NANOS_UNION_TYPES = new TestDataset("date_nanos_union_types"); private static final TestDataset COUNTRIES_BBOX = new TestDataset("countries_bbox"); private static final TestDataset COUNTRIES_BBOX_WEB = new TestDataset("countries_bbox_web"); private static final TestDataset AIRPORT_CITY_BOUNDARIES = new TestDataset("airport_city_boundaries"); @@ -192,7 +192,7 @@ public class CsvTestsDataLoader { Map.entry(MULTIVALUE_GEOMETRIES.indexName, MULTIVALUE_GEOMETRIES), Map.entry(MULTIVALUE_POINTS.indexName, MULTIVALUE_POINTS), Map.entry(DATE_NANOS.indexName, DATE_NANOS), - Map.entry(DATE_NANOS_LOOKUP.indexName, DATE_NANOS_LOOKUP), + Map.entry(DATE_NANOS_UNION_TYPES.indexName, DATE_NANOS_UNION_TYPES), Map.entry(K8S.indexName, K8S), Map.entry(DISTANCES.indexName, DISTANCES), Map.entry(ADDRESSES.indexName, ADDRESSES), diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/date_nanos_lookup.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/date_nanos_union_types.csv similarity index 100% rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/date_nanos_lookup.csv rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/date_nanos_union_types.csv diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-date_nanos_lookup.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-date_nanos_union_types.json similarity index 100% rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-date_nanos_lookup.json rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-date_nanos_union_types.json diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec index f56eb079a8646..cf5f996d26cf8 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec @@ -1768,7 +1768,7 @@ ImplicitCastingMultiTypedFieldsMvExpandKeepSort required_capability: date_nanos_type required_capability: implicit_casting_date_and_date_nanos -FROM date_nanos, date_nanos_lookup +FROM date_nanos, date_nanos_union_types | MV_EXPAND nanos | SORT nanos | LIMIT 4 @@ -1788,7 +1788,7 @@ ImplicitCastingMultiTypedMVFieldsEval required_capability: date_nanos_type required_capability: implicit_casting_date_and_date_nanos -FROM date_nanos, date_nanos_lookup +FROM date_nanos, date_nanos_union_types | EVAL nanos = MV_MAX(nanos) | SORT nanos, millis | LIMIT 4 @@ -1808,7 +1808,7 @@ ImplicitCastingMultiTypedMVFieldsWhere required_capability: date_nanos_type required_capability: implicit_casting_date_and_date_nanos -FROM date_nanos, date_nanos_lookup +FROM date_nanos, date_nanos_union_types | WHERE millis <= "2023-10-23T13:00:00" AND MV_MAX(nanos) > "2023-03-23T13:00:00" | KEEP millis, nanos, num | SORT millis @@ -1829,7 +1829,7 @@ ImplicitCastingMultiTypedMVFieldsStatsMaxMin required_capability: date_nanos_type required_capability: implicit_casting_date_and_date_nanos -FROM date_nanos, date_nanos_lookup +FROM date_nanos, date_nanos_union_types | STATS max = MAX(millis), min = MIN(nanos) ; @@ -1844,7 +1844,7 @@ ImplicitCastingMultiTypedMVFieldsStatsValues required_capability: date_nanos_type required_capability: implicit_casting_date_and_date_nanos -FROM date_nanos, date_nanos_lookup +FROM date_nanos, date_nanos_union_types | STATS c = MV_COUNT(VALUES(nanos)) ; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 396e6a3bebbbe..ccbb3d8dbec14 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -1791,17 +1791,6 @@ private static LogicalPlan planWithoutSyntheticAttributes(LogicalPlan plan) { } } - private static InvalidMappedField multiTypedField(Expression e) { - Expression exp = Alias.unwrap(e); - if (exp.resolved() - && exp.dataType() == UNSUPPORTED - && exp instanceof FieldAttribute fa - && fa.field() instanceof InvalidMappedField imf) { - return imf; - } - return null; - } - /** * Cast union typed fields that are mixed of date and date_nanos types into date_nanos. */ From 2d68f851c6329e3ba16e4a0afb7de5222be4b710 Mon Sep 17 00:00:00 2001 From: Fang Xing Date: Thu, 24 Apr 2025 21:15:47 -0400 Subject: [PATCH 14/24] push down binary comparisons on date and date nanos union type fields --- .../src/main/resources/union_types.csv-spec | 332 ++++++++++++++++-- .../xpack/esql/analysis/Analyzer.java | 26 +- .../esql/expression/predicate/Range.java | 9 +- .../comparison/EsqlBinaryComparison.java | 8 +- .../local/LucenePushdownPredicates.java | 54 ++- .../xpack/esql/plan/logical/Enrich.java | 2 - .../xpack/esql/plan/logical/Eval.java | 8 +- .../esql/plan/logical/join/JoinConfig.java | 4 +- .../xpack/esql/planner/TranslatorHandler.java | 6 +- .../esql/analysis/AnalyzerTestUtils.java | 26 ++ .../xpack/esql/analysis/AnalyzerTests.java | 64 ++-- .../LocalPhysicalPlanOptimizerTests.java | 65 ++++ .../esql/planner/QueryTranslatorTests.java | 138 ++++++++ 13 files changed, 665 insertions(+), 77 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec index cf5f996d26cf8..2cceae723adec 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec @@ -1645,15 +1645,15 @@ required_capability: date_nanos_type required_capability: implicit_casting_date_and_date_nanos FROM employees, employees_incompatible -| EVAL x = emp_no::long, avg_worked_seconds = avg_worked_seconds::unsigned_long, z = hire_date::datetime -| KEEP x, languages, hire_date, avg_worked_seconds, z -| SORT x, hire_date +| EVAL x = emp_no::long, avg_worked_seconds = avg_worked_seconds::unsigned_long, y = hire_date::datetime, z = hire_date::date_nanos +| KEEP x, languages, hire_date, avg_worked_seconds, y, z +| SORT x, z | LIMIT 2 ; -x:long |languages:unsupported |hire_date:date_nanos |avg_worked_seconds:unsigned_long |z:datetime -10001 |null |1986-06-26T00:00:00.000Z |268728049 |1986-06-26T00:00:00.000Z -10001 |null |1986-06-26T00:00:00.000Z |268728049 |1986-06-26T00:00:00.000Z +x:long |languages:unsupported |hire_date:date_nanos |avg_worked_seconds:unsigned_long |y:datetime |z:date_nanos +10001 |null |1986-06-26T00:00:00.000Z |268728049 |1986-06-26T00:00:00.000Z |1986-06-26T00:00:00.000Z +10001 |null |1986-06-26T00:00:00.000Z |268728049 |1986-06-26T00:00:00.000Z |1986-06-26T00:00:00.000Z ; ImplicitCastingMultiTypedFieldsDropKeepSort @@ -1661,16 +1661,16 @@ required_capability: date_nanos_type required_capability: implicit_casting_date_and_date_nanos FROM employees, employees_incompatible -| EVAL emp_no = emp_no::long, avg_worked_seconds = avg_worked_seconds::unsigned_long, x = hire_date::datetime +| EVAL emp_no = emp_no::long, avg_worked_seconds = avg_worked_seconds::unsigned_long, x = hire_date::datetime, y = hire_date | DROP salary -| KEEP emp_no, languages, hire_date, avg_worked_seconds, x -| SORT emp_no, hire_date +| KEEP emp_no, languages, hire_date, avg_worked_seconds, x, y +| SORT emp_no, y | limit 2 ; -emp_no:long |languages:unsupported |hire_date:date_nanos |avg_worked_seconds:unsigned_long |x:datetime -10001 |null |1986-06-26T00:00:00.000Z |268728049 |1986-06-26T00:00:00.000Z -10001 |null |1986-06-26T00:00:00.000Z |268728049 |1986-06-26T00:00:00.000Z +emp_no:long |languages:unsupported |hire_date:date_nanos |avg_worked_seconds:unsigned_long |x:datetime |y:date_nanos +10001 |null |1986-06-26T00:00:00.000Z |268728049 |1986-06-26T00:00:00.000Z |1986-06-26T00:00:00.000Z +10001 |null |1986-06-26T00:00:00.000Z |268728049 |1986-06-26T00:00:00.000Z |1986-06-26T00:00:00.000Z ; ImplicitCastingMultiTypedFieldsRenameKeepSort @@ -1707,13 +1707,180 @@ x:long |hire_date:date_nanos |y:date_nanos |z:date 10002 |1986-06-26T00:00:00.000Z |1987-06-26T00:00:00.000Z |1986-06-26T00:00:00.000Z ; -ImplicitCastingMultiTypedFieldsEvalFilterKeepSort +ImplicitCastingMultiTypedFieldsEvalFilterEqualKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL x = emp_no::long + 1, y = hire_date + 1 day +| WHERE x > 10011 AND hire_date == "1993-08-03" AND languages::integer < 3 AND height::double > 1.2 +| KEEP x, hire_date +| SORT x, hire_date +| limit 4 +; + +x:long |hire_date:date_nanos +10018 | 1993-08-03T00:00:00.000Z +10018 | 1993-08-03T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsEvalFilterNotEqualKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL x = emp_no::long + 1, y = hire_date + 1 day +| WHERE x > 10011 AND hire_date != "1993-08-03" AND languages::integer < 3 AND height::double > 1.2 +| KEEP x, hire_date +| SORT x, hire_date +| limit 4 +; + +x:long |hire_date:date_nanos +10014 | 1985-10-20T00:00:00.000Z +10014 | 1985-10-20T00:00:00.000Z +10017 | 1995-01-27T00:00:00.000Z +10017 | 1995-01-27T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsEvalFilterGreaterThanKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL x = emp_no::long + 1, y = hire_date + 1 day +| WHERE x > 10011 AND hire_date > "1993-08-03" AND languages::integer < 3 AND height::double > 1.2 +| KEEP x, hire_date +| SORT x, hire_date +| limit 4 +; + +x:long | hire_date:date_nanos +10017 | 1995-01-27T00:00:00.000Z +10017 | 1995-01-27T00:00:00.000Z +10020 | 1999-04-30T00:00:00.000Z +10020 | 1999-04-30T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsEvalFilterGreaterThanOrEqualKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL x = emp_no::long + 1, y = hire_date + 1 day +| WHERE x > 10011 AND hire_date >= "1993-08-03" AND languages::integer < 3 AND height::double > 1.2 +| KEEP x, hire_date +| SORT x, hire_date +| limit 4 +; + +x:long |hire_date:date_nanos +10017 | 1995-01-27T00:00:00.000Z +10017 | 1995-01-27T00:00:00.000Z +10018 | 1993-08-03T00:00:00.000Z +10018 | 1993-08-03T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsEvalFilterLessThanKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL x = emp_no::long + 1, y = hire_date + 1 day +| WHERE x > 10011 AND hire_date < "1993-08-03" AND languages::integer < 3 AND height::double > 1.2 +| KEEP x, hire_date +| SORT x, hire_date +| limit 4 +; + +x:long |hire_date:date_nanos +10014 | 1985-10-20T00:00:00.000Z +10014 | 1985-10-20T00:00:00.000Z +10019 | 1987-04-03T00:00:00.000Z +10019 | 1987-04-03T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsEvalFilterLessThanOrEqualKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL x = emp_no::long + 1, y = hire_date + 1 day +| WHERE x > 10011 AND hire_date <= "1993-08-03" AND languages::integer < 3 AND height::double > 1.2 +| KEEP x, hire_date +| SORT x, hire_date +| limit 4 +; + +x:long | hire_date:date_nanos +10014 | 1985-10-20T00:00:00.000Z +10014 | 1985-10-20T00:00:00.000Z +10018 | 1993-08-03T00:00:00.000Z +10018 | 1993-08-03T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsEvalFilterLessThanOrEqualNotEqualKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL x = emp_no::long + 1, y = hire_date + 1 day +| WHERE x > 10011 AND hire_date <= "1993-08-03" AND hire_date != "1985-10-20" AND languages::integer < 3 AND height::double > 1.2 +| KEEP x, hire_date +| SORT x, hire_date +| limit 4 +; + +x:long | hire_date:date_nanos +10018 | 1993-08-03T00:00:00.000Z +10018 | 1993-08-03T00:00:00.000Z +10019 | 1987-04-03T00:00:00.000Z +10019 | 1987-04-03T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsEvalFilterRangeExcludeLowerExcludeUpperKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL x = emp_no::long + 1, y = hire_date + 1 day +| WHERE x > 10011 AND hire_date < "1995-01-27" AND hire_date > "1993-08-03" AND languages::integer < 3 AND height::double > 1.2 +| KEEP x, hire_date +| SORT x, hire_date +| limit 4 +; + +x:long |hire_date:date_nanos +10045 | 1994-05-21T00:00:00.000Z +10045 | 1994-05-21T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsEvalFilterRangeIncludeLowerIncludeUpperKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL x = emp_no::long + 1, y = hire_date + 1 day +| WHERE x > 10011 AND hire_date <= "1995-01-27" AND hire_date >= "1993-08-03" AND languages::integer < 3 AND height::double > 1.2 +| KEEP x, hire_date +| SORT x, hire_date +| limit 4 +; + +x:long |hire_date:date_nanos +10017 | 1995-01-27T00:00:00.000Z +10017 | 1995-01-27T00:00:00.000Z +10018 | 1993-08-03T00:00:00.000Z +10018 | 1993-08-03T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsEvalFilterRangeExcludeLowerIncludeUpperKeepSort required_capability: date_nanos_type required_capability: implicit_casting_date_and_date_nanos FROM employees, employees_incompatible | EVAL x = emp_no::long + 1, y = hire_date + 1 day -| WHERE x > 10011 AND hire_date > "1992-01-01" AND languages::integer < 3 AND height::double > 1.2 +| WHERE x > 10011 AND hire_date <= "1995-01-27" AND hire_date > "1993-08-03" AND languages::integer < 3 AND height::double > 1.2 | KEEP x, hire_date | SORT x, hire_date | limit 4 @@ -1722,8 +1889,27 @@ FROM employees, employees_incompatible x:long |hire_date:date_nanos 10017 | 1995-01-27T00:00:00.000Z 10017 | 1995-01-27T00:00:00.000Z +10045 | 1994-05-21T00:00:00.000Z +10045 | 1994-05-21T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsEvalFilterRangeIncludeLowerExcludeUpperKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL x = emp_no::long + 1, y = hire_date + 1 day +| WHERE x > 10011 AND hire_date < "1995-01-27" AND hire_date >= "1993-08-03" AND languages::integer < 3 AND height::double > 1.2 +| KEEP x, hire_date +| SORT x, hire_date +| limit 4 +; + +x:long |hire_date:date_nanos 10018 | 1993-08-03T00:00:00.000Z 10018 | 1993-08-03T00:00:00.000Z +10045 | 1994-05-21T00:00:00.000Z +10045 | 1994-05-21T00:00:00.000Z ; ImplicitCastingMultiTypedFieldsStatsByNumeric @@ -1731,17 +1917,35 @@ required_capability: date_nanos_type required_capability: implicit_casting_date_and_date_nanos FROM employees, employees_incompatible -| STATS x=max(hire_date) BY languages = languages::integer +| STATS x=max(hire_date), y = min(hire_date) BY languages = languages::integer +| SORT languages +; + +x:date_nanos | y:date_nanos | languages:integer +1999-04-30T00:00:00.000Z | 1985-02-18T00:00:00.000Z | 1 +1995-01-27T00:00:00.000Z | 1986-03-14T00:00:00.000Z | 2 +1996-11-05T00:00:00.000Z | 1985-02-24T00:00:00.000Z | 3 +1995-03-13T00:00:00.000Z | 1985-05-13T00:00:00.000Z | 4 +1994-04-09T00:00:00.000Z | 1985-11-19T00:00:00.000Z | 5 +1997-05-19T00:00:00.000Z | 1985-11-20T00:00:00.000Z | null +; + +ImplicitCastingMultiTypedFieldsStatsByNumericWithFilter +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| STATS x=max(hire_date) where hire_date < "1993-08-03", y = min(hire_date) where hire_date >= "1993-08-03" BY languages = languages::integer | SORT languages ; -x:date_nanos | languages:integer -1999-04-30T00:00:00.000Z | 1 -1995-01-27T00:00:00.000Z | 2 -1996-11-05T00:00:00.000Z | 3 -1995-03-13T00:00:00.000Z | 4 -1994-04-09T00:00:00.000Z | 5 -1997-05-19T00:00:00.000Z | null +x:date_nanos | y:date_nanos | languages:integer +1990-10-20T00:00:00.000Z | 1994-05-21T00:00:00.000Z | 1 +1991-06-26T00:00:00.000Z | 1993-08-03T00:00:00.000Z | 2 +1993-03-21T00:00:00.000Z | 1994-02-17T00:00:00.000Z | 3 +1993-02-14T00:00:00.000Z | 1995-03-13T00:00:00.000Z | 4 +1992-12-18T00:00:00.000Z | 1994-04-09T00:00:00.000Z | 5 +1991-10-22T00:00:00.000Z | 1995-03-20T00:00:00.000Z | null ; ImplicitCastingMultiTypedFieldsStatsByDateNanos @@ -1764,6 +1968,26 @@ x:long | y:double | z:double | hire_date:date_nanos 2 | 3.6 | 1.5 | 1985-09-17T00:00:00.000Z ; +ImplicitCastingMultiTypedFieldsStatsByDateNanosWithFilter +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| STATS x=count(emp_no::long) where hire_date > "1985-05-01", y=avg(salary_change::double) where hire_date > "1985-05-01", z=max(height::double) where hire_date > "1985-05-01" BY hire_date +| Eval y = round(y, 1), z = round(z, 1) +| KEEP x, y, z, hire_date +| SORT hire_date +| LIMIT 5 +; + +x:long | y:double | z:double | hire_date:date_nanos +0 | null | null | 1985-02-18T00:00:00.000Z +0 | null | null | 1985-02-24T00:00:00.000Z +2 | 3.3 | 2.0 | 1985-05-13T00:00:00.000Z +2 | 0.2 | 1.8 | 1985-07-09T00:00:00.000Z +2 | 3.6 | 1.5 | 1985-09-17T00:00:00.000Z +; + ImplicitCastingMultiTypedFieldsMvExpandKeepSort required_capability: date_nanos_type required_capability: implicit_casting_date_and_date_nanos @@ -1796,7 +2020,6 @@ FROM date_nanos, date_nanos_union_types warning:Line 2:23: evaluation of [nanos] failed, treating result as null. Only first 20 failures recorded. warning:Line 2:23: java.lang.IllegalArgumentException: milliSeconds [-1457696696640] are before the epoch in 1970 and cannot be converted to nanoseconds - num:long | nanos:date_nanos | millis:date_nanos 0 | 2023-03-23T12:15:03.360Z | 1999-10-22T12:15:03.360103847Z 0 | 2023-03-23T12:15:03.360Z | 1999-10-23T12:15:03.360103847Z @@ -1937,6 +2160,24 @@ c:long | yr:date_nanos 8 | 1994-01-01T00:00:00.000Z ; +ImplicitCastingMultiTypedDateTruncStatsByWithFilter +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| STATS c = count(emp_no::long) where hire_date > "1996-01-01" BY yr = DATE_TRUNC(1 year, hire_date) +| SORT yr DESC +| LIMIT 5 +; + +c:long | yr:date_nanos +2 | 1999-01-01T00:00:00.000Z +2 | 1997-01-01T00:00:00.000Z +2 | 1996-01-01T00:00:00.000Z +0 | 1995-01-01T00:00:00.000Z +0 | 1994-01-01T00:00:00.000Z +; + ImplicitCastingMultiTypedDateTruncStatsByWithEval required_capability: date_nanos_type required_capability: implicit_casting_date_and_date_nanos @@ -1956,6 +2197,25 @@ c:long | yr:date_nanos 8 | 1994-01-01T00:00:00.000Z ; +ImplicitCastingMultiTypedDateTruncStatsByWithEvalWithFilter +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL yr = DATE_TRUNC(1 year, hire_date) +| STATS c = count(emp_no::long) where hire_date > "1991-01-01" BY yr +| SORT yr DESC +| LIMIT 5 +; + +c:long | yr:date_nanos +2 | 1999-01-01T00:00:00.000Z +2 | 1997-01-01T00:00:00.000Z +2 | 1996-01-01T00:00:00.000Z +10 | 1995-01-01T00:00:00.000Z +8 | 1994-01-01T00:00:00.000Z +; + ImplicitCastingMultiTypedBucketDateNanosByYear required_capability: date_nanos_type required_capability: implicit_casting_date_and_date_nanos @@ -1982,6 +2242,32 @@ c:long | yr:date_nanos 22 | 1985-01-01T00:00:00.000Z ; +ImplicitCastingMultiTypedBucketDateNanosByYearWithFilter +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| STATS c = count(*) where hire_date > "1991-01-01" BY yr = BUCKET(hire_date, 1 year) +| SORT yr DESC, c +; + +c:long | yr:date_nanos +2 | 1999-01-01T00:00:00.000Z +2 | 1997-01-01T00:00:00.000Z +2 | 1996-01-01T00:00:00.000Z +10 | 1995-01-01T00:00:00.000Z +8 | 1994-01-01T00:00:00.000Z +6 | 1993-01-01T00:00:00.000Z +16 | 1992-01-01T00:00:00.000Z +12 | 1991-01-01T00:00:00.000Z +0 | 1990-01-01T00:00:00.000Z +0 | 1989-01-01T00:00:00.000Z +0 | 1988-01-01T00:00:00.000Z +0 | 1987-01-01T00:00:00.000Z +0 | 1986-01-01T00:00:00.000Z +0 | 1985-01-01T00:00:00.000Z +; + ImplicitCastingMultiTypedBucketDateNanosByMonth required_capability: date_nanos_type required_capability: implicit_casting_date_and_date_nanos diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index ccbb3d8dbec14..c2e0991145af7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -18,6 +18,7 @@ import org.elasticsearch.xpack.esql.Column; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.VerificationException; +import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.analysis.AnalyzerRules.ParameterizedAnalyzerRule; import org.elasticsearch.xpack.esql.common.Failure; import org.elasticsearch.xpack.esql.core.capabilities.Resolvables; @@ -1797,11 +1798,15 @@ private static LogicalPlan planWithoutSyntheticAttributes(LogicalPlan plan) { private static class ImplicitCastingForUnionTypedFields extends ParameterizedRule { @Override public LogicalPlan apply(LogicalPlan plan, AnalyzerContext context) { + if (EsqlCapabilities.Cap.IMPLICIT_CASTING_DATE_AND_DATE_NANOS.isEnabled() == false) { + return plan; + } // This rule should be applied after ResolveUnionTypes, so that the InvalidMappedFields with explicit casting are converted into - // MultiTypeEsField, and don't be double cast here. + // MultiTypeEsField, and don't get double cast here. Map invalidMappedFieldCasted = new HashMap<>(); LogicalPlan transformedPlan = plan.transformUp(LogicalPlan.class, p -> { - if (p instanceof UnaryPlan == false && p instanceof LookupJoin == false) { + // exclude LookupJoin for now, as it doesn't support date_nanos as join key yet + if (p instanceof UnaryPlan == false) { return p; } Set invalidMappedFields = invalidMappedFieldsInLogicalPlan(p); @@ -1829,11 +1834,10 @@ public LogicalPlan apply(LogicalPlan plan, AnalyzerContext context) { // If there are new aliases created, create a new eval child with new aliases for the current plan。 // How many children does a LogicalPlan have? Only deal with UnaryPlan and LookupJoin for now. if (newAliases.isEmpty() == false) { // create a new eval child plan - if (p instanceof UnaryPlan u) { // unary plan - Eval eval = new Eval(u.source(), u.child(), newAliases.values().stream().toList()); - p = u.replaceChild(eval); - } - // TODO Lookup join does not work on date_nanos field today, joining on a date_nanos field does not find a match. + UnaryPlan u = (UnaryPlan) p; // this must be a unary plan, as it is checked at the beginning of plan loop + Eval eval = new Eval(u.source(), u.child(), newAliases.values().stream().toList()); + p = u.replaceChild(eval); + // TODO Lookup join does not work on date_nanos field yet, joining on a date_nanos field does not find a match. // And lookup up join is a special case as a lookup join has two children, after date_nanos is supported as a join // key, the transformation needs to take it into account. } @@ -1899,7 +1903,10 @@ private static Expression castInvalidMappedField(DataType targetType, FieldAttri */ private static Set invalidMappedFieldsInLogicalPlan(LogicalPlan plan) { Set fas = new HashSet<>(); - if (plan instanceof EsRelation == false) { // not all the union typed fields from indices need to be cast implicitly + // Invalid mapped fields are legal at EsRelation level, as long as they are not used elsewhere. In the final output, if they + // have not been dropped, implicit cast will be added for them, so that we can return not null values, the implicit casting is + // deferred to when the fields are used or returned. + if (plan instanceof EsRelation == false) { plan.forEachExpression(FieldAttribute.class, fa -> { if (fa.field() instanceof InvalidMappedField) { fas.add(fa); @@ -1939,7 +1946,8 @@ private static LogicalPlan castInvalidMappedFieldInFinalOutput(LogicalPlan logic logicalPlan = unary.replaceChild(eval); } // TODO LookupJoin is a binary plan, it does not create a new field, ideally adding an Eval on top of it should be fine, - // however because the output of a LookupJoin does not include InvalidMappedFields, it is a bug need to be addressed + // however because the output of a LookupJoin does not include InvalidMappedFields even the LHS output has + // InvalidMappedFields, it is a bug need to be addressed } } return logicalPlan; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/Range.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/Range.java index e523bf4ddfb42..d56a383e7052a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/Range.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/Range.java @@ -14,6 +14,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.expression.TypedAttribute; import org.elasticsearch.xpack.esql.core.expression.function.scalar.ScalarFunction; import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.esql.core.querydsl.query.Query; @@ -279,7 +280,13 @@ private RangeQuery translate(TranslatorHandler handler) { } } logger.trace("Building range query with format string [{}]", format); - return new RangeQuery(source(), handler.nameOf(value), l, includeLower(), u, includeUpper(), format, zoneId); + // This is a similar check as in EsqlBinaryComparison + // Extract the real field name from MultiTypeEsField, and use it in the push down query if it is found + TypedAttribute attribute = LucenePushdownPredicates.checkIsPushableAttribute(value); + String name = handler.nameOf(attribute); + String fieldNameFromMultiTypeEsField = LucenePushdownPredicates.extractFieldNameFromMultiTypeEsField(attribute); + name = fieldNameFromMultiTypeEsField != null ? fieldNameFromMultiTypeEsField : name; + return new RangeQuery(source(), name, l, includeLower(), u, includeUpper(), format, zoneId); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java index d9d5aa985ded1..b135d3eaea818 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java @@ -378,6 +378,9 @@ public Expression singleValueField() { private Query translate(TranslatorHandler handler) { TypedAttribute attribute = LucenePushdownPredicates.checkIsPushableAttribute(left()); String name = handler.nameOf(attribute); + // Extract the real field name from MultiTypeEsField, and use it in the push down query if it is found + String fieldNameFromMultiTypeEsField = LucenePushdownPredicates.extractFieldNameFromMultiTypeEsField(attribute); + name = fieldNameFromMultiTypeEsField != null ? fieldNameFromMultiTypeEsField : name; Object value = valueOf(FoldContext.small() /* TODO remove me */, right()); String format = null; boolean isDateLiteralComparison = false; @@ -452,7 +455,10 @@ private Query translate(TranslatorHandler handler) { return new RangeQuery(source(), name, null, false, value, true, format, zoneId); } if (this instanceof Equals || this instanceof NotEquals) { - name = LucenePushdownPredicates.pushableAttributeName(attribute); + // Extract the real field name from MultiTypeEsField, and use it in the push down query if it is found + name = fieldNameFromMultiTypeEsField != null + ? fieldNameFromMultiTypeEsField + : LucenePushdownPredicates.pushableAttributeName(attribute); Query query; if (isDateLiteralComparison) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java index f7fad1cdb984c..7da4f8038895f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java @@ -7,14 +7,22 @@ package org.elasticsearch.xpack.esql.optimizer.rules.physical.local; +import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.expression.TypedAttribute; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField; import org.elasticsearch.xpack.esql.core.util.Check; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; import org.elasticsearch.xpack.esql.stats.SearchStats; +import java.util.Map; +import java.util.function.Predicate; + +import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS; + /** * When deciding if a filter or topN can be pushed down to Lucene, we need to check a few things on the field. * Exactly what is checked depends on the type of field and the query. For example, we have the following possible combinations: @@ -94,6 +102,46 @@ static String pushableAttributeName(TypedAttribute attribute) { : attribute.name(); } + /** + * Extract the real field name from a MultiTypeEsField, limit to MultiTypeEsField that has date_nanos type only. + * + * For example, the name of a MultiTypeEsField can be $$myfield$converted_to$date_nanos, and the real field name extract from the + * MultiTypeEsField is myfield, this method return myfield given a MultiTypeEsField. + * + * If the real field name is found, and the original field data types contain only date and date_nanos types, return the real field + * name, so that the real field name will be used to check for eligibility of being pushed down, and the real field name will be used + * in the push down query, instead of the name of the MultiTypeEsField, which should not match any field in an index. + * + * This method can be extended to support the other data types in the future if there is a need. + */ + static String extractFieldNameFromMultiTypeEsField(TypedAttribute attribute) { + if (EsqlCapabilities.Cap.IMPLICIT_CASTING_DATE_AND_DATE_NANOS.isEnabled() + && attribute instanceof FieldAttribute fa + && fa.field() instanceof MultiTypeEsField multiTypeEsField + && fa.dataType() == DATE_NANOS + && // limit to casting to date_nanos only + mixedDateAndDateNanosOnly(multiTypeEsField, DataType::isMillisOrNanos) // limit to mixed date and date_nanos only + ) { + return fa.fieldName(); + } + return null; + } + + /** + * Check if the original field types in a MultiTypeEsField satisfy the required data types defined in the predicate. + */ + private static boolean mixedDateAndDateNanosOnly(MultiTypeEsField multiTypeEsField, Predicate predicate) { + Map indexToConversionExpressions = multiTypeEsField.getIndexToConversionExpressions(); + for (Map.Entry entry : indexToConversionExpressions.entrySet()) { + Expression conversionFunction = entry.getValue(); + if (conversionFunction instanceof AbstractConvertFunction abstractConvertFunction + && predicate.test(abstractConvertFunction.field().dataType()) == false) { + return false; + } + } + return true; + } + /** * The default implementation of this has no access to SearchStats, so it can only make decisions based on the FieldAttribute itself. * In particular, it assumes TEXT fields have no exact subfields (underlying keyword field), @@ -131,10 +179,14 @@ public boolean hasExactSubfield(FieldAttribute attr) { @Override public boolean isIndexedAndHasDocValues(FieldAttribute attr) { + // If this is a MultiTypeEsField cast to date_nanos, make it eligible for being pushed down by checking the real + // field name against SearchStats + String fieldNameFromMultiTypeEsField = LucenePushdownPredicates.extractFieldNameFromMultiTypeEsField(attr); + String name = fieldNameFromMultiTypeEsField != null ? fieldNameFromMultiTypeEsField : attr.name(); // We still consider the value of isAggregatable here, because some fields like ScriptFieldTypes are always aggregatable // But this could hide issues with fields that are not indexed but are aggregatable // This is the original behaviour for ES|QL, but is it correct? - return attr.field().isAggregatable() || stats.isIndexed(attr.name()) && stats.hasDocValues(attr.name()); + return attr.field().isAggregatable() || stats.isIndexed(name) && stats.hasDocValues(name); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java index e9defff9e6525..11e9a57064e5b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java @@ -32,7 +32,6 @@ import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; -import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.index.EsIndex; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import org.elasticsearch.xpack.esql.plan.GeneratingPlan; @@ -209,7 +208,6 @@ public boolean expressionsResolved() { return policyName.resolved() && matchField instanceof EmptyAttribute == false // matchField not defined in the query, needs to be resolved from the policy && matchField.resolved() - && matchField.dataType() != DataType.UNSUPPORTED && Resolvables.resolved(enrichFields()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Eval.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Eval.java index 90e35dfc73af9..5c0aa35f13880 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Eval.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Eval.java @@ -13,6 +13,7 @@ import org.elasticsearch.xpack.esql.capabilities.PostAnalysisVerificationAware; import org.elasticsearch.xpack.esql.capabilities.TelemetryAware; import org.elasticsearch.xpack.esql.common.Failures; +import org.elasticsearch.xpack.esql.core.capabilities.Resolvables; import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.AttributeMap; @@ -131,12 +132,7 @@ private List renameAliases(List originalAttributes, List n @Override public boolean expressionsResolved() { - for (Alias a : fields) { - if (a.resolved() == false || a.dataType() == DataType.UNSUPPORTED) { - return false; - } - } - return true; + return Resolvables.resolved(fields); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/JoinConfig.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/JoinConfig.java index d9c5a9fc2d063..383606d6ccbed 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/JoinConfig.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/JoinConfig.java @@ -12,7 +12,6 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.xpack.esql.core.capabilities.Resolvables; import org.elasticsearch.xpack.esql.core.expression.Attribute; -import org.elasticsearch.xpack.esql.core.type.DataType; import java.io.IOException; import java.util.List; @@ -51,7 +50,6 @@ public boolean expressionsResolved() { return type.resolved() && Resolvables.resolved(matchFields) && Resolvables.resolved(leftFields) - && Resolvables.resolved(rightFields) - && leftFields.stream().noneMatch(e -> e.dataType() == DataType.UNSUPPORTED); + && Resolvables.resolved(rightFields); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/TranslatorHandler.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/TranslatorHandler.java index f7f09d36a4296..26eded735e9d6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/TranslatorHandler.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/TranslatorHandler.java @@ -15,6 +15,7 @@ import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.querydsl.query.Query; +import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates; import org.elasticsearch.xpack.esql.querydsl.query.SingleValueQuery; /** @@ -41,7 +42,10 @@ public Query asQuery(Expression e) { private static Query wrapFunctionQuery(Expression field, Query query) { if (field instanceof FieldAttribute fa) { fa = fa.getExactInfo().hasExact() ? fa.exactAttribute() : fa; - return new SingleValueQuery(query, fa.name()); + // Extract the real field name from MultiTypeEsField, and use it in the push down query if it is found + String fieldNameFromMultiTypeEsField = LucenePushdownPredicates.extractFieldNameFromMultiTypeEsField(fa); + String fieldName = fieldNameFromMultiTypeEsField != null ? fieldNameFromMultiTypeEsField : fa.name(); + return new SingleValueQuery(query, fieldName); } if (field instanceof MetadataAttribute) { return query; // MetadataAttributes are always single valued diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java index db9047be3f065..4195a6380eff8 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java @@ -11,6 +11,8 @@ import org.elasticsearch.inference.TaskType; import org.elasticsearch.xpack.core.enrich.EnrichPolicy; import org.elasticsearch.xpack.esql.EsqlTestUtils; +import org.elasticsearch.xpack.esql.core.type.EsField; +import org.elasticsearch.xpack.esql.core.type.InvalidMappedField; import org.elasticsearch.xpack.esql.enrich.ResolvedEnrichPolicy; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import org.elasticsearch.xpack.esql.index.EsIndex; @@ -24,8 +26,10 @@ import org.elasticsearch.xpack.esql.session.Configuration; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import static org.elasticsearch.xpack.core.enrich.EnrichPolicy.GEO_MATCH_TYPE; import static org.elasticsearch.xpack.core.enrich.EnrichPolicy.MATCH_TYPE; @@ -209,4 +213,26 @@ public static void loadEnrichPolicyResolution(EnrichResolution enrich, String po public static IndexResolution tsdbIndexResolution() { return loadMapping("tsdb-mapping.json", "test"); } + + public static IndexResolution indexWithDateDateNanosUnionType() { + // this method is shared by AnalyzerTest, QueryTranslatorTests and LocalPhysicalPlanOptimizerTests + String dateDateNanos = "date_and_date_nanos"; // mixed date and date_nanos + String dateDateNanosLong = "date_and_date_nanos_and_long"; // mixed date, date_nanos and long + LinkedHashMap> typesToIndices1 = new LinkedHashMap<>(); + typesToIndices1.put("date", Set.of("index1", "index2")); + typesToIndices1.put("date_nanos", Set.of("index3")); + LinkedHashMap> typesToIndices2 = new LinkedHashMap<>(); + typesToIndices2.put("date", Set.of("index1")); + typesToIndices2.put("date_nanos", Set.of("index2")); + typesToIndices2.put("long", Set.of("index3")); + EsField dateDateNanosField = new InvalidMappedField(dateDateNanos, typesToIndices1); + EsField dateDateNanosLongField = new InvalidMappedField(dateDateNanosLong, typesToIndices2); + EsIndex index = new EsIndex( + "test*", + Map.of(dateDateNanos, dateDateNanosField, dateDateNanosLong, dateDateNanosLongField), + Map.of("index1", IndexMode.STANDARD, "index2", IndexMode.STANDARD, "index3", IndexMode.STANDARD) + ); + return IndexResolution.valid(index); + + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 8781c49ccc435..fd5536afbfbbc 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -35,8 +35,6 @@ import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; -import org.elasticsearch.xpack.esql.core.type.EsField; -import org.elasticsearch.xpack.esql.core.type.InvalidMappedField; import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField; import org.elasticsearch.xpack.esql.core.type.PotentiallyUnmappedKeywordEsField; import org.elasticsearch.xpack.esql.enrich.ResolvedEnrichPolicy; @@ -85,7 +83,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -111,6 +108,7 @@ import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.analyzer; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.analyzerDefaultMapping; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.defaultEnrichResolution; +import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.indexWithDateDateNanosUnionType; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.loadMapping; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.tsdbIndexResolution; import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; @@ -3766,64 +3764,70 @@ public void testResolveCompletionOutputField() { } public void testImplicitCastingForDateAndDateNanosFields() { - String dateDateNanos = "date_and_date_nanos"; // mixed date and date_nanos - String dateDateNanosLong = "date_and_date_nanos_and_long"; // mixed date, date_nanos and long - LinkedHashMap> typesToIndices1 = new LinkedHashMap<>(); - typesToIndices1.put("date", Set.of("index1", "index2")); - typesToIndices1.put("date_nanos", Set.of("index3")); - LinkedHashMap> typesToIndices2 = new LinkedHashMap<>(); - typesToIndices2.put("date", Set.of("index1")); - typesToIndices2.put("date_nanos", Set.of("index2")); - typesToIndices2.put("long", Set.of("index3")); - EsField dateDateNanosField = new InvalidMappedField(dateDateNanos, typesToIndices1); - EsField dateDateNanosLongField = new InvalidMappedField(dateDateNanosLong, typesToIndices2); - - IndexResolution indexWithUnionTypedFields = IndexResolution.valid( - new EsIndex("test*", Map.of(dateDateNanos, dateDateNanosField, dateDateNanosLong, dateDateNanosLongField)) - ); + assumeTrue("requires snapshot", EsqlCapabilities.Cap.IMPLICIT_CASTING_DATE_AND_DATE_NANOS.isEnabled()); + IndexResolution indexWithUnionTypedFields = indexWithDateDateNanosUnionType(); Analyzer analyzer = AnalyzerTestUtils.analyzer(indexWithUnionTypedFields); // validate if a union typed field is cast to a type explicitly, implicit casting won't be applied again LogicalPlan plan = analyze(""" FROM tests - | Eval x = date_and_date_nanos::datetime, y = date_and_date_nanos + | Eval x = date_and_date_nanos::datetime, y = date_and_date_nanos, z = date_and_date_nanos::date_nanos """, analyzer); Project project = as(plan, Project.class); List projections = project.projections(); - assertEquals(4, projections.size()); - UnsupportedAttribute ua = as(projections.get(0), UnsupportedAttribute.class); // long is not casted to date_nanos + assertEquals(5, projections.size()); + // long is not casted to date_nanos + UnsupportedAttribute ua = as(projections.get(0), UnsupportedAttribute.class); assertEquals("date_and_date_nanos_and_long", ua.name()); assertEquals(DataType.UNSUPPORTED, ua.dataType()); + // implicit casting ReferenceAttribute ra = as(projections.get(1), ReferenceAttribute.class); assertEquals("date_and_date_nanos", ra.name()); - assertEquals(DataType.DATE_NANOS, ra.dataType()); // implicit casting + assertEquals(DataType.DATE_NANOS, ra.dataType()); + // explicit casting ra = as(projections.get(2), ReferenceAttribute.class); assertEquals("x", ra.name()); - assertEquals(DataType.DATETIME, ra.dataType()); // explicit casting + assertEquals(DataType.DATETIME, ra.dataType()); + // implicit casting ra = as(projections.get(3), ReferenceAttribute.class); assertEquals("y", ra.name()); - assertEquals(DataType.DATE_NANOS, ra.dataType()); // implicit casting + assertEquals(DataType.DATE_NANOS, ra.dataType()); + // explicit casting + ra = as(projections.get(4), ReferenceAttribute.class); + assertEquals("z", ra.name()); + assertEquals(DataType.DATE_NANOS, ra.dataType()); Limit limit = as(project.child(), Limit.class); - Eval eval = as(limit.child(), Eval.class); // eval coded in the query + // original Eval coded in the query + Eval eval = as(limit.child(), Eval.class); List aliases = eval.fields(); - assertEquals(2, aliases.size()); + assertEquals(3, aliases.size()); + // explicit casting Alias a = aliases.get(0); assertEquals("x", a.name()); - assertEquals(DataType.DATETIME, a.dataType()); // explicit casting - assertTrue(isMultiTypeEsField(a.child())); // no double casting + assertEquals(DataType.DATETIME, a.dataType()); + assertTrue(isMultiTypeEsField(a.child())); + // implicit casting a = aliases.get(1); assertEquals("y", a.name()); - assertEquals(DataType.DATE_NANOS, a.dataType()); // implicit casting + assertEquals(DataType.DATE_NANOS, a.dataType()); assertTrue(a.child() instanceof ReferenceAttribute); - eval = as(eval.child(), Eval.class); // a new eval added for implicit casting + // explicit casting + a = aliases.get(2); + assertEquals("z", a.name()); + assertEquals(DataType.DATE_NANOS, a.dataType()); + assertTrue(isMultiTypeEsField(a.child())); + // a new eval added for implicit casting + eval = as(eval.child(), Eval.class); aliases = eval.fields(); assertEquals(1, aliases.size()); a = aliases.get(0); assertEquals("date_and_date_nanos", a.name()); assertEquals(DataType.DATE_NANOS, a.dataType()); assertTrue(isMultiTypeEsField(a.child())); + EsRelation esRelation = as(eval.child(), EsRelation.class); + assertEquals("test*", esRelation.indexPattern()); } private boolean isMultiTypeEsField(Expression e) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java index ddbecf52a3f12..88c2ffa2d3ab4 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java @@ -38,18 +38,23 @@ import org.elasticsearch.xpack.esql.analysis.EnrichResolution; import org.elasticsearch.xpack.esql.analysis.Verifier; import org.elasticsearch.xpack.esql.core.expression.Alias; +import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; +import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField; import org.elasticsearch.xpack.esql.core.util.Holder; import org.elasticsearch.xpack.esql.enrich.ResolvedEnrichPolicy; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; +import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute; import org.elasticsearch.xpack.esql.expression.function.fulltext.Match; import org.elasticsearch.xpack.esql.expression.predicate.logical.Or; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThanOrEqual; import org.elasticsearch.xpack.esql.index.EsIndex; import org.elasticsearch.xpack.esql.index.IndexResolution; import org.elasticsearch.xpack.esql.optimizer.rules.logical.ExtractAggregateCommonFilter; @@ -106,7 +111,9 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping; import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; +import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.indexWithDateDateNanosUnionType; import static org.elasticsearch.xpack.esql.core.querydsl.query.Query.unscore; +import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS; import static org.elasticsearch.xpack.esql.plan.physical.EsStatsQueryExec.StatsType; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; @@ -136,6 +143,7 @@ public class LocalPhysicalPlanOptimizerTests extends MapperServiceTestCase { public static final String MATCH_FUNCTION_QUERY = "from test | where match(%s, %s)"; private TestPlannerOptimizer plannerOptimizer; + private TestPlannerOptimizer plannerOptimizerDateDateNanosUnionTypes; private final Configuration config; private final SearchStats IS_SV_STATS = new TestSearchStats() { @Override @@ -195,6 +203,13 @@ private Analyzer makeAnalyzer(String mappingFileName) { return makeAnalyzer(mappingFileName, new EnrichResolution()); } + private Analyzer makeAnalyzer(IndexResolution indexResolution) { + return new Analyzer( + new AnalyzerContext(config, new EsqlFunctionRegistry(), indexResolution, new EnrichResolution(), emptyInferenceResolution()), + new Verifier(new Metrics(new EsqlFunctionRegistry()), new XPackLicenseState(() -> 0L)) + ); + } + /** * Expects * LimitExec[1000[INTEGER]] @@ -1818,6 +1833,56 @@ public void testMatchFunctionWithPushableDisjunction() { assertThat(esQuery.query().toString(), equalTo(expected.toString())); } + public void testToDateNanosPushDown() { + assumeTrue("requires snapshot", EsqlCapabilities.Cap.IMPLICIT_CASTING_DATE_AND_DATE_NANOS.isEnabled()); + IndexResolution indexWithUnionTypedFields = indexWithDateDateNanosUnionType(); + plannerOptimizerDateDateNanosUnionTypes = new TestPlannerOptimizer(EsqlTestUtils.TEST_CFG, makeAnalyzer(indexWithUnionTypedFields)); + var stats = EsqlTestUtils.statsForExistingField("date_and_date_nanos", "date_and_date_nanos_and_long"); + String query = """ + from test* + | where date_and_date_nanos < "2025-01-01" and date_and_date_nanos_and_long::date_nanos >= "2024-01-01\""""; + var plan = plannerOptimizerDateDateNanosUnionTypes.plan(query, stats); + + // date_and_date_nanos should be pushed down to EsQueryExec, date_and_date_nanos_and_long should not be pushed down + var project = as(plan, ProjectExec.class); + List projections = project.projections(); + assertEquals(2, projections.size()); + UnsupportedAttribute ua = as(projections.get(0), UnsupportedAttribute.class); // mixed date, date_nanos and long are not auto-casted + assertEquals("date_and_date_nanos_and_long", ua.fieldName()); + Alias alias = as(projections.get(1), Alias.class); + assertEquals(DATE_NANOS, alias.dataType()); + FieldAttribute ra = as(alias.child(), FieldAttribute.class); + assertEquals("date_and_date_nanos", ra.fieldName()); + assertTrue(isMultiTypeEsField(ra)); // mixed date and date_nanos are auto-casted + var limit = as(project.child(), LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + project = as(exchange.child(), ProjectExec.class); + var fieldExtract = as(project.child(), FieldExtractExec.class); + limit = as(fieldExtract.child(), LimitExec.class); + // date_and_date_nanos_and_long::date_nanos >= "2024-01-01" is not pushed down + var filter = as(limit.child(), FilterExec.class); + GreaterThanOrEqual gt = as(filter.condition(), GreaterThanOrEqual.class); + FieldAttribute fa = as(gt.left(), FieldAttribute.class); + assertTrue(isMultiTypeEsField(fa)); + assertEquals("date_and_date_nanos_and_long", fa.fieldName()); + fieldExtract = as(filter.child(), FieldExtractExec.class); // extract date_and_date_nanos_and_long + var esQuery = as(fieldExtract.child(), EsQueryExec.class); + var source = ((SingleValueQuery.Builder) esQuery.query()).source(); + var expected = wrapWithSingleQuery( + query, + unscore( + rangeQuery("date_and_date_nanos").lt("2025-01-01T00:00:00.000Z").timeZone("Z").format("strict_date_optional_time_nanos") + ), + "date_and_date_nanos", + source + ); // date_and_date_nanos is pushed down + assertThat(expected.toString(), is(esQuery.query().toString())); + } + + private boolean isMultiTypeEsField(Expression e) { + return e instanceof FieldAttribute fa && fa.field() instanceof MultiTypeEsField; + } + private QueryBuilder wrapWithSingleQuery(String query, QueryBuilder inner, String fieldName, Source source) { return FilterTests.singleValueQuery(query, inner, fieldName, source); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java index 02b108dcf6adb..ff47227cd25f8 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.EsqlTestUtils; +import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.analysis.Analyzer; import org.elasticsearch.xpack.esql.analysis.AnalyzerContext; import org.elasticsearch.xpack.esql.analysis.Verifier; @@ -20,6 +21,7 @@ import org.elasticsearch.xpack.esql.optimizer.TestPlannerOptimizer; import org.elasticsearch.xpack.esql.plan.physical.EsQueryExec; import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; +import org.elasticsearch.xpack.esql.stats.SearchStats; import org.elasticsearch.xpack.esql.telemetry.Metrics; import org.hamcrest.Matcher; import org.junit.BeforeClass; @@ -31,6 +33,7 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.emptyPolicyResolution; import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping; import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; +import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.indexWithDateDateNanosUnionType; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.matchesRegex; @@ -41,6 +44,8 @@ public class QueryTranslatorTests extends ESTestCase { private static TestPlannerOptimizer plannerOptimizerIPs; + private static TestPlannerOptimizer plannerOptimizerDateDateNanosUnionTypes; + private static Analyzer makeAnalyzer(String mappingFileName) { var mapping = loadMapping(mappingFileName); EsIndex test = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD)); @@ -58,6 +63,19 @@ private static Analyzer makeAnalyzer(String mappingFileName) { ); } + public static Analyzer makeAnalyzer(IndexResolution indexResolution) { + return new Analyzer( + new AnalyzerContext( + EsqlTestUtils.TEST_CFG, + new EsqlFunctionRegistry(), + indexResolution, + emptyPolicyResolution(), + emptyInferenceResolution() + ), + new Verifier(new Metrics(new EsqlFunctionRegistry()), new XPackLicenseState(() -> 0L)) + ); + } + @BeforeClass public static void init() { plannerOptimizer = new TestPlannerOptimizer(EsqlTestUtils.TEST_CFG, makeAnalyzer("mapping-all-types.json")); @@ -83,6 +101,13 @@ public void assertQueryTranslationIPs(String query, Matcher translationM assertThat(translatedQuery, translationMatcher); } + private void assertQueryTranslationDateDateNanosUnionTypes(String query, SearchStats stats, Matcher translationMatcher) { + PhysicalPlan optimized = plannerOptimizerDateDateNanosUnionTypes.plan(query, stats); + EsQueryExec eqe = (EsQueryExec) optimized.collectLeaves().get(0); + final String translatedQuery = eqe.query().toString().replaceAll("\\s+", ""); + assertThat(translatedQuery, translationMatcher); + } + public void testBinaryComparisons() { assertQueryTranslation(""" FROM test | WHERE 10 < integer""", containsString(""" @@ -299,4 +324,117 @@ OR CIDR_MATCH(ip0, "fe80::cae2:65ff:fece:feb9") OR host == "beta\"""", matchesRe esql_single_value":\\{"field":"ip1".*"terms":\\{"ip1":\\["127.0.0.3/32","127.0.0.2".*""" + """ esql_single_value":\\{"field":"ip0".*"terms":\\{"ip0":\\["127.0.0.1","128.0.0.1","fe80::cae2:65ff:fece:feb9".*""")); } + + public void testToDateNanos() { + assumeTrue("requires snapshot", EsqlCapabilities.Cap.IMPLICIT_CASTING_DATE_AND_DATE_NANOS.isEnabled()); + IndexResolution indexWithUnionTypedFields = indexWithDateDateNanosUnionType(); + plannerOptimizerDateDateNanosUnionTypes = new TestPlannerOptimizer(EsqlTestUtils.TEST_CFG, makeAnalyzer(indexWithUnionTypedFields)); + var stats = EsqlTestUtils.statsForExistingField("date_and_date_nanos", "date_and_date_nanos_and_long"); + + // == term + assertQueryTranslationDateDateNanosUnionTypes(""" + FROM test* | WHERE date_and_date_nanos == "2025-01-01\"""", stats, containsString(""" + "esql_single_value":{"field":"date_and_date_nanos",\ + "next":{"term":{"date_and_date_nanos":{"value":"2025-01-01T00:00:00.000Z","boost":0.0}}}""")); + + // != term + assertQueryTranslationDateDateNanosUnionTypes(""" + FROM test* | WHERE date_and_date_nanos != "2025-01-01\"""", stats, containsString(""" + "esql_single_value":{"field":"date_and_date_nanos",\ + "next":{"bool":{"must_not":[{"term":{"date_and_date_nanos":{"value":"2025-01-01T00:00:00.000Z","boost":0.0}}}],\ + "boost":0.0}}""")); + + // > range + assertQueryTranslationDateDateNanosUnionTypes(""" + FROM test* | WHERE date_and_date_nanos > "2025-01-01\"""", stats, containsString(""" + "esql_single_value":{"field":"date_and_date_nanos",\ + "next":{"range":{"date_and_date_nanos":{"gt":"2025-01-01T00:00:00.000Z","time_zone":"Z",\ + "format":"strict_date_optional_time_nanos","boost":0.0}}}""")); + + // >= range + assertQueryTranslationDateDateNanosUnionTypes(""" + FROM test* | WHERE date_and_date_nanos >= "2025-01-01\"""", stats, containsString(""" + "esql_single_value":{"field":"date_and_date_nanos",\ + "next":{"range":{"date_and_date_nanos":{"gte":"2025-01-01T00:00:00.000Z","time_zone":"Z",\ + "format":"strict_date_optional_time_nanos","boost":0.0}}}""")); + + // < range + assertQueryTranslationDateDateNanosUnionTypes( + """ + FROM test* | WHERE date_and_date_nanos < "2025-01-01" and date_and_date_nanos_and_long::date_nanos > "2025-01-01\"""", + stats, + containsString(""" + "esql_single_value":{"field":"date_and_date_nanos",\ + "next":{"range":{"date_and_date_nanos":{"lt":"2025-01-01T00:00:00.000Z","time_zone":"Z",\ + "format":"strict_date_optional_time_nanos","boost":0.0}}}""") + ); + + // <= range + assertQueryTranslationDateDateNanosUnionTypes(""" + FROM test* | WHERE date_and_date_nanos <= "2025-01-01\"""", stats, containsString(""" + "esql_single_value":{"field":"date_and_date_nanos",\ + "next":{"range":{"date_and_date_nanos":{"lte":"2025-01-01T00:00:00.000Z","time_zone":"Z",\ + "format":"strict_date_optional_time_nanos","boost":0.0}}}""")); + + // <= and >= + assertQueryTranslationDateDateNanosUnionTypes(""" + FROM test* | WHERE date_and_date_nanos <= "2025-01-01" and date_and_date_nanos > "2020-01-01\"""", stats, containsString(""" + "esql_single_value":{"field":"date_and_date_nanos",\ + "next":{"range":{"date_and_date_nanos":{"gt":"2020-01-01T00:00:00.000Z","lte":"2025-01-01T00:00:00.000Z","time_zone":"Z",\ + "format":"strict_date_optional_time_nanos","boost":0.0}}}""")); + + // >= or < + assertQueryTranslationDateDateNanosUnionTypes(""" + FROM test* | WHERE date_and_date_nanos >= "2025-01-01" or date_and_date_nanos < "2020-01-01\"""", stats, matchesRegex(""" + .*bool.*should.*""" + """ + esql_single_value":\\{"field":"date_and_date_nanos".*"range":\\{"date_and_date_nanos":\\{"gte":"2025-01-01T00:00:00.000Z",\ + "time_zone":"Z","format":"strict_date_optional_time_nanos","boost":0.0.*""" + """ + esql_single_value":\\{"field":"date_and_date_nanos".*"range":\\{"date_and_date_nanos":\\{"lt":"2020-01-01T00:00:00.000Z",\ + "time_zone":"Z","format":"strict_date_optional_time_nanos","boost":0.0.*""")); + + // > or = + assertQueryTranslationDateDateNanosUnionTypes(""" + FROM test* | WHERE date_and_date_nanos > "2025-01-01" or date_and_date_nanos == "2020-01-01\"""", stats, matchesRegex(""" + .*bool.*should.*""" + """ + esql_single_value":\\{"field":"date_and_date_nanos".*"range":\\{"date_and_date_nanos":\\{"gt":"2025-01-01T00:00:00.000Z",\ + "time_zone":"Z","format":"strict_date_optional_time_nanos","boost":0.0.*""" + """ + esql_single_value":\\{"field":"date_and_date_nanos".*"term":\\{"date_and_date_nanos":\\{"value":"2020-01-01T00:00:00.000Z",\ + "boost":0.0.*""")); + + // < or != + assertQueryTranslationDateDateNanosUnionTypes(""" + FROM test* | WHERE date_and_date_nanos < "2020-01-01" or date_and_date_nanos != "2025-01-01\"""", stats, matchesRegex(""" + .*bool.*should.*""" + """ + esql_single_value":\\{"field":"date_and_date_nanos".*"range":\\{"date_and_date_nanos":\\{"lt":"2020-01-01T00:00:00.000Z",\ + "time_zone":"Z","format":"strict_date_optional_time_nanos","boost":0.0.*""" + """ + esql_single_value":\\{"field":"date_and_date_nanos".*"must_not".*"term":\\{"date_and_date_nanos":\\{"value":\ + "2025-01-01T00:00:00.000Z","boost":0.0.*""")); + + // == or == + assertQueryTranslationDateDateNanosUnionTypes(""" + FROM test* | WHERE date_and_date_nanos == "2020-01-01" or date_and_date_nanos == "2025-01-01\"""", stats, matchesRegex(""" + .*bool.*should.*""" + """ + esql_single_value":\\{"field":"date_and_date_nanos".*"term":\\{"date_and_date_nanos":\\{"value":"2020-01-01T00:00:00.000Z",\ + "boost":0.0.*""" + """ + esql_single_value":\\{"field":"date_and_date_nanos".*"term":\\{"date_and_date_nanos":\\{"value":"2025-01-01T00:00:00.000Z",\ + "boost":0.0.*""")); + + // != or != + assertQueryTranslationDateDateNanosUnionTypes(""" + FROM test* | WHERE date_and_date_nanos != "2020-01-01" or date_and_date_nanos != "2025-01-01\"""", stats, matchesRegex(""" + .*bool.*should.*""" + """ + esql_single_value":\\{"field":"date_and_date_nanos".*"must_not".*"term":\\{"date_and_date_nanos":\\{"value":\ + "2020-01-01T00:00:00.000Z","boost":0.0.*""" + """ + esql_single_value":\\{"field":"date_and_date_nanos".*"must_not".*"term":\\{"date_and_date_nanos":\\{"value":\ + "2025-01-01T00:00:00.000Z","boost":0.0.*""")); + + // = or != + assertQueryTranslationDateDateNanosUnionTypes(""" + FROM test* | WHERE date_and_date_nanos == "2020-01-01" or date_and_date_nanos != "2025-01-01\"""", stats, matchesRegex(""" + .*bool.*should.*""" + """ + esql_single_value":\\{"field":"date_and_date_nanos".*"term":\\{"date_and_date_nanos":\\{"value":\ + "2020-01-01T00:00:00.000Z","boost":0.0.*""" + """ + esql_single_value":\\{"field":"date_and_date_nanos".*"must_not".*"term":\\{"date_and_date_nanos":\\{"value":\ + "2025-01-01T00:00:00.000Z","boost":0.0.*""")); + } } From 3b997625d70d66485bd78157b0a33939fdd38a0a Mon Sep 17 00:00:00 2001 From: Fang Xing Date: Tue, 6 May 2025 23:13:14 -0400 Subject: [PATCH 15/24] implicit casting for date and date_nanos in EsRelation --- .../src/main/resources/union_types.csv-spec | 42 ++--- .../xpack/esql/analysis/Analyzer.java | 153 ++++++++++++++---- .../xpack/esql/analysis/AnalyzerTests.java | 25 +-- .../LocalPhysicalPlanOptimizerTests.java | 13 +- 4 files changed, 155 insertions(+), 78 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec index 2cceae723adec..f58e1d8dd090c 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec @@ -1998,14 +1998,14 @@ FROM date_nanos, date_nanos_union_types | LIMIT 4 ; -warning:Line 2:13: evaluation of [nanos] failed, treating result as null. Only first 20 failures recorded. -warning:Line 2:13: java.lang.IllegalArgumentException: milliSeconds [-1457696696640] are before the epoch in 1970 and cannot be converted to nanoseconds +warning:Line 1:1: evaluation of [FROM date_nanos, date_nanos_union_types] failed, treating result as null. Only first 20 failures recorded. +warning:Line 1:1: java.lang.IllegalArgumentException: milliSeconds [-1457696696640] are before the epoch in 1970 and cannot be converted to nanoseconds -num:long | nanos:date_nanos | millis:date_nanos -0 | 2023-01-23T13:55:01.543Z | 1999-10-23T12:15:03.360103847Z -0 | 2023-01-23T13:55:01.543123456Z | 1999-10-23T12:15:03.360Z -0 | 2023-02-23T13:33:34.937Z | 1999-10-23T12:15:03.360103847Z -0 | 2023-02-23T13:33:34.937193Z | 1999-10-23T12:15:03.360Z +millis:date_nanos | nanos:date_nanos | num:long +1999-10-23T12:15:03.360103847Z | 2023-01-23T13:55:01.543Z | 0 +1999-10-23T12:15:03.360Z | 2023-01-23T13:55:01.543123456Z | 0 +1999-10-23T12:15:03.360103847Z | 2023-02-23T13:33:34.937Z | 0 +1999-10-23T12:15:03.360Z | 2023-02-23T13:33:34.937193Z | 0 ; ImplicitCastingMultiTypedMVFieldsEval @@ -2018,13 +2018,13 @@ FROM date_nanos, date_nanos_union_types | LIMIT 4 ; -warning:Line 2:23: evaluation of [nanos] failed, treating result as null. Only first 20 failures recorded. -warning:Line 2:23: java.lang.IllegalArgumentException: milliSeconds [-1457696696640] are before the epoch in 1970 and cannot be converted to nanoseconds -num:long | nanos:date_nanos | millis:date_nanos -0 | 2023-03-23T12:15:03.360Z | 1999-10-22T12:15:03.360103847Z -0 | 2023-03-23T12:15:03.360Z | 1999-10-23T12:15:03.360103847Z -0 | 2023-03-23T12:15:03.360103847Z | 1999-10-22T12:15:03.360Z -0 | 2023-03-23T12:15:03.360103847Z | 1999-10-23T12:15:03.360Z +warning:Line 1:1: evaluation of [FROM date_nanos, date_nanos_union_types] failed, treating result as null. Only first 20 failures recorded. +warning:Line 1:1: java.lang.IllegalArgumentException: milliSeconds [-1457696696640] are before the epoch in 1970 and cannot be converted to nanoseconds +millis:date_nanos | num:long | nanos:date_nanos +1999-10-22T12:15:03.360103847Z | 0 | 2023-03-23T12:15:03.360Z +1999-10-23T12:15:03.360103847Z | 0 | 2023-03-23T12:15:03.360Z +1999-10-22T12:15:03.360Z | 0 | 2023-03-23T12:15:03.360103847Z +1999-10-23T12:15:03.360Z | 0 | 2023-03-23T12:15:03.360103847Z ; ImplicitCastingMultiTypedMVFieldsWhere @@ -2037,8 +2037,8 @@ FROM date_nanos, date_nanos_union_types | SORT millis ; -warning:Line 2:52: evaluation of [nanos] failed, treating result as null. Only first 20 failures recorded. -warning:Line 2:52: java.lang.IllegalArgumentException: milliSeconds [-1457696696640] are before the epoch in 1970 and cannot be converted to nanoseconds +warning:Line 1:1: evaluation of [FROM date_nanos, date_nanos_union_types] failed, treating result as null. Only first 20 failures recorded. +warning:Line 1:1: java.lang.IllegalArgumentException: milliSeconds [-1457696696640] are before the epoch in 1970 and cannot be converted to nanoseconds millis:date_nanos | nanos:date_nanos | num:long 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 @@ -2056,8 +2056,8 @@ FROM date_nanos, date_nanos_union_types | STATS max = MAX(millis), min = MIN(nanos) ; -warning:Line 2:38: evaluation of [nanos] failed, treating result as null. Only first 20 failures recorded. -warning:Line 2:38: java.lang.IllegalArgumentException: milliSeconds [-1457696696640] are before the epoch in 1970 and cannot be converted to nanoseconds +warning:Line 1:1: evaluation of [FROM date_nanos, date_nanos_union_types] failed, treating result as null. Only first 20 failures recorded. +warning:Line 1:1: java.lang.IllegalArgumentException: milliSeconds [-1457696696640] are before the epoch in 1970 and cannot be converted to nanoseconds max:date_nanos | min:date_nanos 2023-10-23T13:55:01.543123456Z | 2023-01-23T13:55:01.543Z @@ -2067,12 +2067,12 @@ ImplicitCastingMultiTypedMVFieldsStatsValues required_capability: date_nanos_type required_capability: implicit_casting_date_and_date_nanos -FROM date_nanos, date_nanos_union_types +FROM date_nanos, date_nanos_union_types | STATS c = MV_COUNT(VALUES(nanos)) ; -warning:Line 2:29: evaluation of [nanos] failed, treating result as null. Only first 20 failures recorded. -warning:Line 2:29: java.lang.IllegalArgumentException: milliSeconds [-1457696696640] are before the epoch in 1970 and cannot be converted to nanoseconds +warning:Line 1:1: evaluation of [FROM date_nanos, date_nanos_union_types] failed, treating result as null. Only first 20 failures recorded. +warning:Line 1:1: java.lang.IllegalArgumentException: milliSeconds [-1457696696640] are before the epoch in 1970 and cannot be converted to nanoseconds c:integer 19 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index c2e0991145af7..0fb1fde5be578 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -177,20 +177,13 @@ public class Analyzer extends ParameterizedRuleExecutor( - "Resolution", - /* - * ImplicitCasting must be before ResolveRefs. Because a reference is created for a Bucket in Aggregate's aggregates, - * resolving this reference before implicit casting may cause this reference to have customMessage=true, it prevents further - * attempts to resolve this reference. - */ - new ImplicitCasting(), - new ResolveRefs(), - new ResolveUnionTypes(), // Must be after ResolveRefs, so union types can be found - // Must be after ResolveUnionTypes, if there is explicit casting on the union typed fields, implicit casting won't be added - new ImplicitCastingForUnionTypedFields() + new Batch<>("Resolution", new ResolveRefs(), new ImplicitCasting(), new ResolveUnionTypes() // Must be after ResolveRefs, so union + // types can be found + // Must be after ResolveUnionTypes, if there is explicit casting on the union typed fields, implicit casting won't be added + // new ImplicitCastingForUnionTypedFields() ), new Batch<>("Finish Analysis", Limiter.ONCE, new AddImplicitLimit(), new AddImplicitForkLimit(), new UnionTypesCleanup()) ); @@ -584,7 +577,7 @@ private Aggregate resolveAggregate(Aggregate aggregate, List children } } - if (Resolvables.resolved(groupings) == false || (Resolvables.resolved(aggregates) == false)) { + if (Resolvables.resolved(groupings) == false || Resolvables.resolved(aggregates) == false) { ArrayList resolved = new ArrayList<>(); for (Expression e : groupings) { Attribute attr = Expressions.attribute(e); @@ -595,17 +588,29 @@ private Aggregate resolveAggregate(Aggregate aggregate, List children List resolvedList = NamedExpressions.mergeOutputAttributes(resolved, childrenOutput); List newAggregates = new ArrayList<>(); - for (NamedExpression ag : aggregate.aggregates()) { - var agg = (NamedExpression) ag.transformUp(UnresolvedAttribute.class, ua -> { - Expression ne = ua; - Attribute maybeResolved = maybeResolveAttribute(ua, resolvedList); - if (maybeResolved != null) { - changed.set(true); - ne = maybeResolved; - } - return ne; - }); - newAggregates.add(agg); + // If the groupings are not resolved, skip the resolution of the references to groupings in the aggregates, resolve the + // aggregations that do not reference to groupings, so that the fields/attributes referenced by the aggregations can be + // resolved, and verifier doesn't report field/reference/column not found errors for them. + boolean groupingResolved = Resolvables.resolved(groupings); + int size = groupingResolved ? aggregates.size() : aggregates.size() - groupings.size(); + for (int i = 0; i < aggregates.size(); i++) { + NamedExpression maybeResolvedAgg = aggregates.get(i); + if (i < size) { // Skip resolving references to groupings in the aggregations if the groupings are not resolved yet. + maybeResolvedAgg = (NamedExpression) maybeResolvedAgg.transformUp(UnresolvedAttribute.class, ua -> { + Expression ne = ua; + Attribute maybeResolved = maybeResolveAttribute(ua, resolvedList); + // An item in aggregations can reference to groupings explicitly, if groupings are not resolved yet and + // maybeResolved is not resolved, return the original UnresolvedAttribute, so that it has another chance + // to get resolved in the next iteration. + // For example STATS c = count(emp_no), x = d::int + 1 BY d = (date == "2025-01-01") + if (groupingResolved || maybeResolved.resolved()) { + changed.set(true); + ne = maybeResolved; + } + return ne; + }); + } + newAggregates.add(maybeResolvedAgg); } // TODO: remove this when Stats interface is removed @@ -1598,7 +1603,7 @@ public LogicalPlan apply(LogicalPlan plan) { // Collect field attributes from previous runs plan.forEachUp(EsRelation.class, rel -> { for (Attribute attr : rel.output()) { - if (attr instanceof FieldAttribute fa && fa.field() instanceof MultiTypeEsField) { + if (attr instanceof FieldAttribute fa && fa.field() instanceof MultiTypeEsField && fa.synthetic()) { unionFieldAttributes.add(fa); } } @@ -1674,11 +1679,58 @@ private Expression resolveConvertFunction(ConvertFunction convert, List typeResolutions = new HashMap<>(); + Set supportedTypes = convert.supportedTypes(); + Holder conversionSupported = new Holder<>(true); + // Get the type of each field in the multi typed field and check if the explicit conversion is supported + mtf.getIndexToConversionExpressions().forEach((indexName, conversionExpression) -> { + AbstractConvertFunction c = (AbstractConvertFunction) conversionExpression; + FieldAttribute fieldAttribute = (FieldAttribute) c.field(); + DataType type = fieldAttribute.dataType(); + if (supportedTypes.contains(type.widenSmallNumeric())) { + TypeResolutionKey key = new TypeResolutionKey(fa.name(), type); + var concreteConvert = typeSpecificConvert(convert, fa.source(), fieldAttribute.field()); + typeResolutions.putIfAbsent(key, concreteConvert); + } else { + conversionSupported.set(false); + } + }); + // If the conversions are supported, create a new FieldAttribute with a new MultiTypeEsField, and add it to + // unionFieldAttributes. + if (conversionSupported.get()) { + // build the map between index name and conversion expressions + Map indexToConversionExpressions = new HashMap<>(); + for (Map.Entry entry : mtf.getIndexToConversionExpressions().entrySet()) { + String indexName = entry.getKey(); + AbstractConvertFunction originalConversionFunction = (AbstractConvertFunction) entry.getValue(); + TypeResolutionKey key = new TypeResolutionKey(fa.name(), originalConversionFunction.field().dataType()); + Expression newConversionFunction = typeResolutions.get(key); + indexToConversionExpressions.put(indexName, newConversionFunction); + } + MultiTypeEsField multiTypeEsField = new MultiTypeEsField( + fa.name(), + ((Expression) convert).dataType(), + false, + indexToConversionExpressions + ); + return createIfDoesNotAlreadyExist(fa, multiTypeEsField, unionFieldAttributes); + } + } + } else if (convert.field() instanceof AbstractConvertFunction subConvert) { + return convertExpression.replaceChildren( + Collections.singletonList(resolveConvertFunction(subConvert, unionFieldAttributes)) + ); + } return convertExpression; } @@ -1702,7 +1754,10 @@ private Expression createIfDoesNotAlreadyExist( } } - private MultiTypeEsField resolvedMultiTypeEsField(FieldAttribute fa, HashMap typeResolutions) { + private static MultiTypeEsField resolvedMultiTypeEsField( + FieldAttribute fa, + HashMap typeResolutions + ) { Map typesToConversionExpressions = new HashMap<>(); InvalidMappedField imf = (InvalidMappedField) fa.field(); imf.getTypesToIndices().forEach((typeName, indexNames) -> { @@ -1715,8 +1770,12 @@ private MultiTypeEsField resolvedMultiTypeEsField(FieldAttribute fa, HashMap { + @Override + public LogicalPlan apply(LogicalPlan plan) { + return plan.transformUp(EsRelation.class, relation -> { + if (relation.indexMode() == IndexMode.LOOKUP) { + return relation; + } + return relation.transformExpressionsUp(FieldAttribute.class, f -> { + if (f.field() instanceof InvalidMappedField imf && imf.types().stream().allMatch(DataType::isDate)) { + HashMap typeResolutions = new HashMap<>(); + var convert = new ToDateNanos(f.source(), f); + imf.types().forEach(type -> { + ResolveUnionTypes.TypeResolutionKey key = new ResolveUnionTypes.TypeResolutionKey(f.name(), type); + var concreteConvert = ResolveUnionTypes.typeSpecificConvert(convert, f.source(), type, imf); + typeResolutions.put(key, concreteConvert); + }); + var resolvedField = ResolveUnionTypes.resolvedMultiTypeEsField(f, typeResolutions); + return new FieldAttribute(f.source(), f.parentName(), f.name(), resolvedField, f.nullable(), f.id(), f.synthetic()); + } + return f; + }); + }); + } + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index b7a0cd465e96c..7dcfc6fa6a5b0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -3793,16 +3793,16 @@ public void testImplicitCastingForDateAndDateNanosFields() { Project project = as(plan, Project.class); List projections = project.projections(); assertEquals(5, projections.size()); - // long is not casted to date_nanos - UnsupportedAttribute ua = as(projections.get(0), UnsupportedAttribute.class); + // implicit casting + FieldAttribute fa = as(projections.get(0), FieldAttribute.class); + assertEquals("date_and_date_nanos", fa.name()); + assertEquals(DataType.DATE_NANOS, fa.dataType()); + // long is not cast to date_nanos + UnsupportedAttribute ua = as(projections.get(1), UnsupportedAttribute.class); assertEquals("date_and_date_nanos_and_long", ua.name()); assertEquals(DataType.UNSUPPORTED, ua.dataType()); - // implicit casting - ReferenceAttribute ra = as(projections.get(1), ReferenceAttribute.class); - assertEquals("date_and_date_nanos", ra.name()); - assertEquals(DataType.DATE_NANOS, ra.dataType()); // explicit casting - ra = as(projections.get(2), ReferenceAttribute.class); + ReferenceAttribute ra = as(projections.get(2), ReferenceAttribute.class); assertEquals("x", ra.name()); assertEquals(DataType.DATETIME, ra.dataType()); // implicit casting @@ -3827,21 +3827,12 @@ public void testImplicitCastingForDateAndDateNanosFields() { a = aliases.get(1); assertEquals("y", a.name()); assertEquals(DataType.DATE_NANOS, a.dataType()); - assertTrue(a.child() instanceof ReferenceAttribute); + assertTrue(a.child() instanceof FieldAttribute); // explicit casting a = aliases.get(2); assertEquals("z", a.name()); assertEquals(DataType.DATE_NANOS, a.dataType()); assertTrue(isMultiTypeEsField(a.child())); - // a new eval added for implicit casting - eval = as(eval.child(), Eval.class); - aliases = eval.fields(); - assertEquals(1, aliases.size()); - a = aliases.get(0); - assertEquals("date_and_date_nanos", a.name()); - assertEquals(DataType.DATE_NANOS, a.dataType()); - assertTrue(isMultiTypeEsField(a.child())); - EsRelation esRelation = as(eval.child(), EsRelation.class); assertEquals("test*", esRelation.indexPattern()); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java index 0b28af3d3d73e..648a20834a6da 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java @@ -1849,13 +1849,12 @@ public void testToDateNanosPushDown() { var project = as(plan, ProjectExec.class); List projections = project.projections(); assertEquals(2, projections.size()); - UnsupportedAttribute ua = as(projections.get(0), UnsupportedAttribute.class); // mixed date, date_nanos and long are not auto-casted + FieldAttribute fa = as(projections.get(0), FieldAttribute.class); + assertEquals(DATE_NANOS, fa.dataType()); + assertEquals("date_and_date_nanos", fa.fieldName()); + assertTrue(isMultiTypeEsField(fa)); // mixed date and date_nanos are auto-casted + UnsupportedAttribute ua = as(projections.get(1), UnsupportedAttribute.class); // mixed date, date_nanos and long are not auto-casted assertEquals("date_and_date_nanos_and_long", ua.fieldName()); - Alias alias = as(projections.get(1), Alias.class); - assertEquals(DATE_NANOS, alias.dataType()); - FieldAttribute ra = as(alias.child(), FieldAttribute.class); - assertEquals("date_and_date_nanos", ra.fieldName()); - assertTrue(isMultiTypeEsField(ra)); // mixed date and date_nanos are auto-casted var limit = as(project.child(), LimitExec.class); var exchange = as(limit.child(), ExchangeExec.class); project = as(exchange.child(), ProjectExec.class); @@ -1864,7 +1863,7 @@ public void testToDateNanosPushDown() { // date_and_date_nanos_and_long::date_nanos >= "2024-01-01" is not pushed down var filter = as(limit.child(), FilterExec.class); GreaterThanOrEqual gt = as(filter.condition(), GreaterThanOrEqual.class); - FieldAttribute fa = as(gt.left(), FieldAttribute.class); + fa = as(gt.left(), FieldAttribute.class); assertTrue(isMultiTypeEsField(fa)); assertEquals("date_and_date_nanos_and_long", fa.fieldName()); fieldExtract = as(filter.child(), FieldExtractExec.class); // extract date_and_date_nanos_and_long From 437bfdc39ae7b8bc48eb09f8f947060a3b61a874 Mon Sep 17 00:00:00 2001 From: Fang Xing Date: Wed, 7 May 2025 00:14:22 -0400 Subject: [PATCH 16/24] update testSuggestedCast as date and date_nanos are cast to date_nanos in EsRelation --- .../elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java index d902f32c16c45..545b01f4f3066 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java @@ -55,6 +55,7 @@ import static org.elasticsearch.test.ListMatcher.matchesList; import static org.elasticsearch.test.MapMatcher.assertMap; import static org.elasticsearch.test.MapMatcher.matchesMap; +import static org.elasticsearch.xpack.esql.core.type.DataType.isMillisOrNanos; import static org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase.Mode.SYNC; import static org.elasticsearch.xpack.esql.tools.ProfileParser.parseProfile; import static org.elasticsearch.xpack.esql.tools.ProfileParser.readProfileFromResponse; @@ -724,6 +725,10 @@ public void testSuggestedCast() throws IOException { for (int i = 0; i < listOfTypes.size(); i++) { for (int j = i + 1; j < listOfTypes.size(); j++) { + if (isMillisOrNanos(listOfTypes.get(i)) && isMillisOrNanos(listOfTypes.get(j))) { + // datetime and date_nanos are casted to date_nanos implicitly + continue; + } String query = String.format(Locale.ROOT, """ { "query": "FROM index-%s,index-%s | LIMIT 100 | KEEP my_field" From fc8862662c03ef1294efcf2957815523da6f38bc Mon Sep 17 00:00:00 2001 From: Fang Xing <155562079+fang-xing-esql@users.noreply.github.com> Date: Wed, 7 May 2025 06:20:13 +0200 Subject: [PATCH 17/24] Update docs/changelog/127797.yaml --- docs/changelog/127797.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docs/changelog/127797.yaml diff --git a/docs/changelog/127797.yaml b/docs/changelog/127797.yaml new file mode 100644 index 0000000000000..8fca3da004130 --- /dev/null +++ b/docs/changelog/127797.yaml @@ -0,0 +1,6 @@ +pr: 127797 +summary: "Date nanos implicit casting in union types option #2" +area: ES|QL +type: enhancement +issues: + - 110009 From 83650e9ebd7353a37d9abec8c2a3a6407e7d1ef3 Mon Sep 17 00:00:00 2001 From: Fang Xing Date: Wed, 7 May 2025 10:24:28 -0400 Subject: [PATCH 18/24] clean up --- .../xpack/esql/analysis/Analyzer.java | 210 ++---------------- 1 file changed, 14 insertions(+), 196 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 0fb1fde5be578..f64f7ed75f36e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -9,7 +9,6 @@ import org.elasticsearch.common.logging.HeaderWarning; import org.elasticsearch.common.logging.LoggerMessageFormat; -import org.elasticsearch.common.util.Maps; import org.elasticsearch.compute.data.Block; import org.elasticsearch.core.Strings; import org.elasticsearch.index.IndexMode; @@ -18,7 +17,6 @@ import org.elasticsearch.xpack.esql.Column; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.VerificationException; -import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.analysis.AnalyzerRules.ParameterizedAnalyzerRule; import org.elasticsearch.xpack.esql.common.Failure; import org.elasticsearch.xpack.esql.core.capabilities.Resolvables; @@ -63,7 +61,6 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ConvertFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.FoldablesConvertFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDateNanos; -import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDatetime; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDouble; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToInteger; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToLong; @@ -94,7 +91,6 @@ import org.elasticsearch.xpack.esql.plan.logical.Project; import org.elasticsearch.xpack.esql.plan.logical.Rename; import org.elasticsearch.xpack.esql.plan.logical.RrfScoreEval; -import org.elasticsearch.xpack.esql.plan.logical.UnaryPlan; import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation; import org.elasticsearch.xpack.esql.plan.logical.inference.Completion; import org.elasticsearch.xpack.esql.plan.logical.inference.InferencePlan; @@ -153,7 +149,6 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.TIME_DURATION; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSUPPORTED; import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION; -import static org.elasticsearch.xpack.esql.core.type.DataType.isMillisOrNanos; import static org.elasticsearch.xpack.esql.core.type.DataType.isTemporalAmount; import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.LIMIT; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.maybeParseTemporalAmount; @@ -180,10 +175,11 @@ public class Analyzer extends ParameterizedRuleExecutor("Resolution", new ResolveRefs(), new ImplicitCasting(), new ResolveUnionTypes() // Must be after ResolveRefs, so union - // types can be found - // Must be after ResolveUnionTypes, if there is explicit casting on the union typed fields, implicit casting won't be added - // new ImplicitCastingForUnionTypedFields() + new Batch<>( + "Resolution", + new ResolveRefs(), + new ImplicitCasting(), + new ResolveUnionTypes() // Must be after ResolveRefs, so union types can be found ), new Batch<>("Finish Analysis", Limiter.ONCE, new AddImplicitLimit(), new AddImplicitForkLimit(), new UnionTypesCleanup()) ); @@ -1680,15 +1676,16 @@ private Expression resolveConvertFunction(ConvertFunction convert, List typeResolutions = new HashMap<>(); Set supportedTypes = convert.supportedTypes(); Holder conversionSupported = new Holder<>(true); @@ -1705,10 +1702,11 @@ private Expression resolveConvertFunction(ConvertFunction convert, List indexToConversionExpressions = new HashMap<>(); for (Map.Entry entry : mtf.getIndexToConversionExpressions().entrySet()) { String indexName = entry.getKey(); @@ -1851,186 +1849,6 @@ private static LogicalPlan planWithoutSyntheticAttributes(LogicalPlan plan) { } } - /** - * Cast union typed fields that are mixed of date and date_nanos types into date_nanos. - */ - private static class ImplicitCastingForUnionTypedFields extends ParameterizedRule { - @Override - public LogicalPlan apply(LogicalPlan plan, AnalyzerContext context) { - if (EsqlCapabilities.Cap.IMPLICIT_CASTING_DATE_AND_DATE_NANOS.isEnabled() == false) { - return plan; - } - // This rule should be applied after ResolveUnionTypes, so that the InvalidMappedFields with explicit casting are converted into - // MultiTypeEsField, and don't get double cast here. - Map invalidMappedFieldCasted = new HashMap<>(); - LogicalPlan transformedPlan = plan.transformUp(LogicalPlan.class, p -> { - // exclude LookupJoin for now, as it doesn't support date_nanos as join key yet - if (p instanceof UnaryPlan == false) { - return p; - } - Set invalidMappedFields = invalidMappedFieldsInLogicalPlan(p); - if (invalidMappedFields.isEmpty() == false) { - // If we are at a plan node that has invalid mapped fields, we need to either add an EVAL, or if that has been done - // we should instead replace with the already cast field - Map newAliases = Maps.newHashMapWithExpectedSize(invalidMappedFields.size()); - Map existingAliases = Maps.newHashMapWithExpectedSize(invalidMappedFields.size()); - for (FieldAttribute fa : invalidMappedFields) { - if (invalidMappedFieldCasted.containsKey(fa)) { - // There is already an eval plan created for the implicit cast field, just reference to it - Alias alias = invalidMappedFieldCasted.get(fa); - existingAliases.put(fa, alias); - } else { - // Create a new alias and later on add a new EVAL with this new aliases for implicit casting - DataType targetType = commonDataType(fa); - if (targetType != null) { - Expression conversionFunction = castInvalidMappedField(targetType, fa); - Alias alias = new Alias(fa.source(), fa.name(), conversionFunction); - newAliases.put(fa, alias); - invalidMappedFieldCasted.put(fa, alias); - } - } - } - // If there are new aliases created, create a new eval child with new aliases for the current plan。 - // How many children does a LogicalPlan have? Only deal with UnaryPlan and LookupJoin for now. - if (newAliases.isEmpty() == false) { // create a new eval child plan - UnaryPlan u = (UnaryPlan) p; // this must be a unary plan, as it is checked at the beginning of plan loop - Eval eval = new Eval(u.source(), u.child(), newAliases.values().stream().toList()); - p = u.replaceChild(eval); - // TODO Lookup join does not work on date_nanos field yet, joining on a date_nanos field does not find a match. - // And lookup up join is a special case as a lookup join has two children, after date_nanos is supported as a join - // key, the transformation needs to take it into account. - } - // If there are new or existing aliases identified, combine them into one map - Map allAliases = Maps.newHashMapWithExpectedSize(invalidMappedFields.size()); - allAliases.putAll(newAliases); - allAliases.putAll(existingAliases); - if (allAliases.isEmpty() == false) { // there is already eval plan for that union typed field, reference to the aliases - p = p.transformExpressionsOnly(FieldAttribute.class, fa -> { - Alias alias = allAliases.get(fa); - return alias != null ? alias.toAttribute() : fa; - }); - // MvExpand and Stats have ReferenceAttribute referencing the FieldAttribute in the same plan. - // The ReferenceAttribute need to be updated to point to the casting expression. - if (p instanceof MvExpand mvExpand) { - p = transformMvExpand(mvExpand); - } else if (p instanceof Aggregate aggregate) { - p = transformAggregate(aggregate); - } - } - } - return p; - }); - transformedPlan = castInvalidMappedFieldInFinalOutput(transformedPlan); - return transformedPlan; - } - - /** - * Find a common data type that the union typed field can cast to, only date and date_nanos types are supported. - * This method can be extended to support implicit casting for the other data types. - */ - private static DataType commonDataType(FieldAttribute unionTypedField) { - DataType targetType = null; - if (unionTypedField.field() instanceof InvalidMappedField imf) { - for (DataType type : imf.types()) { - if (isMillisOrNanos(type) == false) { // if there is field that is no date or date_nanos, don't do implicit casting - return null; - } - if (targetType == null) { // initialize the target type to the first type - targetType = type; - } else if (targetType == DATE_NANOS || type == DATE_NANOS) { - targetType = DATE_NANOS; - } - } - } - return targetType; - } - - /** - * Do implicit casting for date and date_nanos only. - */ - private static Expression castInvalidMappedField(DataType targetType, FieldAttribute fa) { - Source source = fa.source(); - return switch (targetType) { - case DATETIME -> new ToDatetime(source, fa); // in case we decided to use DATE as a common type instead of DATE_NANOS - case DATE_NANOS -> new ToDateNanos(source, fa); - default -> throw new EsqlIllegalArgumentException("unexpected data type: " + targetType); - }; - } - - /** - * Return all the FieldAttribute that contain InvalidMappedField in the current plan. - */ - private static Set invalidMappedFieldsInLogicalPlan(LogicalPlan plan) { - Set fas = new HashSet<>(); - // Invalid mapped fields are legal at EsRelation level, as long as they are not used elsewhere. In the final output, if they - // have not been dropped, implicit cast will be added for them, so that we can return not null values, the implicit casting is - // deferred to when the fields are used or returned. - if (plan instanceof EsRelation == false) { - plan.forEachExpression(FieldAttribute.class, fa -> { - if (fa.field() instanceof InvalidMappedField) { - fas.add(fa); - } - }); - } - return fas; - } - - /** - * Cast the InvalidMappedFields in the final output of the query, this is needed when these fields are not referenced in the query - * explicitly, so there is no chance to cast them to a common type earlier, an example of such query is from index*. - */ - private static LogicalPlan castInvalidMappedFieldInFinalOutput(LogicalPlan logicalPlan) { - // Check the output of the query, if the top level plan is resolved, check if there is InvalidMappedField in its output, - // if so add a project with eval, so that a not null value can be returned for a union typed field - if (logicalPlan.resolved()) { - List output = logicalPlan.output(); - Map newAliases = Maps.newHashMapWithExpectedSize(output.size()); - output.forEach(a -> { - if (a instanceof FieldAttribute fa && fa.field() instanceof InvalidMappedField) { - DataType targetType = commonDataType(fa); - if (targetType != null) { - Expression conversionFunction = castInvalidMappedField(targetType, fa); - Alias alias = new Alias(fa.source(), fa.name(), conversionFunction); - newAliases.put(fa, alias); - } - } - }); - if (newAliases.isEmpty() == false) { // add an Eval for the union typed fields left that are not cast implicitly yet - if (logicalPlan instanceof EsRelation esr) { - // EsRelation does not have a child, we should not see row here, add a eval on top of it - logicalPlan = new Eval(esr.source(), esr, newAliases.values().stream().toList()); - } else if (logicalPlan instanceof UnaryPlan unary) { - // Add an Eval as the child of this plan - Eval eval = new Eval(unary.source(), unary.child(), newAliases.values().stream().toList()); - logicalPlan = unary.replaceChild(eval); - } - // TODO LookupJoin is a binary plan, it does not create a new field, ideally adding an Eval on top of it should be fine, - // however because the output of a LookupJoin does not include InvalidMappedFields even the LHS output has - // InvalidMappedFields, it is a bug need to be addressed - } - } - return logicalPlan; - } - - private static MvExpand transformMvExpand(MvExpand mvExpand) { - NamedExpression target = mvExpand.target(); - return new MvExpand(mvExpand.source(), mvExpand.child(), target, target.toAttribute()); - } - - private static Aggregate transformAggregate(Aggregate aggregate) { - List aggregates = aggregate.aggregates(); - List groupings = aggregate.groupings(); - List aggregatesWithNewRefs = new ArrayList<>(aggregates.size()); - for (int i = 0; i < aggregates.size() - groupings.size(); i++) { - aggregatesWithNewRefs.add(aggregates.get(i)); - } - for (Expression e : groupings) { // Add groupings - aggregatesWithNewRefs.add(Expressions.attribute(e)); - } - return aggregate.with(groupings, aggregatesWithNewRefs); - } - } - /** * Cast the union typed fields in EsRelation to date_nanos if they are mixed date and date_nanos types. */ From f9cf6fb5a2e3662f3de71a15b1e5a26cef539eb8 Mon Sep 17 00:00:00 2001 From: Fang Xing Date: Wed, 7 May 2025 12:17:34 -0400 Subject: [PATCH 19/24] fix multi cluster tests --- .../src/main/resources/union_types.csv-spec | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec index a2e5f0d588648..f9449c2ca9d81 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec @@ -2065,8 +2065,8 @@ FROM date_nanos, date_nanos_union_types | LIMIT 4 ; -warning:Line 1:1: evaluation of [FROM date_nanos, date_nanos_union_types] failed, treating result as null. Only first 20 failures recorded. -warning:Line 1:1: java.lang.IllegalArgumentException: milliSeconds [-1457696696640] are before the epoch in 1970 and cannot be converted to nanoseconds +warningRegex:evaluation of \[FROM .*date_nanos.*date_nanos_union_types.*\] failed, treating result as null. Only first 20 failures recorded. +warningRegex:java.lang.IllegalArgumentException: milliSeconds \[-1457696696640\] are before the epoch in 1970 and cannot be converted to nanoseconds millis:date_nanos | nanos:date_nanos | num:long 1999-10-23T12:15:03.360103847Z | 2023-01-23T13:55:01.543Z | 0 @@ -2085,8 +2085,8 @@ FROM date_nanos, date_nanos_union_types | LIMIT 4 ; -warning:Line 1:1: evaluation of [FROM date_nanos, date_nanos_union_types] failed, treating result as null. Only first 20 failures recorded. -warning:Line 1:1: java.lang.IllegalArgumentException: milliSeconds [-1457696696640] are before the epoch in 1970 and cannot be converted to nanoseconds +warningRegex:evaluation of \[FROM .*date_nanos.*date_nanos_union_types.*\] failed, treating result as null. Only first 20 failures recorded. +warningRegex:java.lang.IllegalArgumentException: milliSeconds \[-1457696696640\] are before the epoch in 1970 and cannot be converted to nanoseconds millis:date_nanos | num:long | nanos:date_nanos 1999-10-22T12:15:03.360103847Z | 0 | 2023-03-23T12:15:03.360Z 1999-10-23T12:15:03.360103847Z | 0 | 2023-03-23T12:15:03.360Z @@ -2104,8 +2104,8 @@ FROM date_nanos, date_nanos_union_types | SORT millis ; -warning:Line 1:1: evaluation of [FROM date_nanos, date_nanos_union_types] failed, treating result as null. Only first 20 failures recorded. -warning:Line 1:1: java.lang.IllegalArgumentException: milliSeconds [-1457696696640] are before the epoch in 1970 and cannot be converted to nanoseconds +warningRegex:evaluation of \[FROM .*date_nanos.*date_nanos_union_types.*\] failed, treating result as null. Only first 20 failures recorded. +warningRegex:java.lang.IllegalArgumentException: milliSeconds \[-1457696696640\] are before the epoch in 1970 and cannot be converted to nanoseconds millis:date_nanos | nanos:date_nanos | num:long 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 @@ -2123,8 +2123,8 @@ FROM date_nanos, date_nanos_union_types | STATS max = MAX(millis), min = MIN(nanos) ; -warning:Line 1:1: evaluation of [FROM date_nanos, date_nanos_union_types] failed, treating result as null. Only first 20 failures recorded. -warning:Line 1:1: java.lang.IllegalArgumentException: milliSeconds [-1457696696640] are before the epoch in 1970 and cannot be converted to nanoseconds +warningRegex:evaluation of \[FROM .*date_nanos.*date_nanos_union_types.*\] failed, treating result as null. Only first 20 failures recorded. +warningRegex:java.lang.IllegalArgumentException: milliSeconds \[-1457696696640\] are before the epoch in 1970 and cannot be converted to nanoseconds max:date_nanos | min:date_nanos 2023-10-23T13:55:01.543123456Z | 2023-01-23T13:55:01.543Z @@ -2138,8 +2138,8 @@ FROM date_nanos, date_nanos_union_types | STATS c = MV_COUNT(VALUES(nanos)) ; -warning:Line 1:1: evaluation of [FROM date_nanos, date_nanos_union_types] failed, treating result as null. Only first 20 failures recorded. -warning:Line 1:1: java.lang.IllegalArgumentException: milliSeconds [-1457696696640] are before the epoch in 1970 and cannot be converted to nanoseconds +warningRegex:evaluation of \[FROM .*date_nanos.*date_nanos_union_types.*\] failed, treating result as null. Only first 20 failures recorded. +warningRegex:java.lang.IllegalArgumentException: milliSeconds \[-1457696696640\] are before the epoch in 1970 and cannot be converted to nanoseconds c:integer 19 From 5dd906e85e3e6fab864d98b07bfee50d5b2cf47b Mon Sep 17 00:00:00 2001 From: Fang Xing Date: Thu, 22 May 2025 23:39:37 -0400 Subject: [PATCH 20/24] refactor and remove pushdown optimization --- docs/changelog/123678.yaml | 6 - .../esql/qa/server/single-node/build.gradle | 1 + .../xpack/esql/qa/single_node/RestEsqlIT.java | 34 +++-- .../src/main/resources/union_types.csv-spec | 43 +++++- .../xpack/esql/analysis/Analyzer.java | 93 +++++------ .../esql/expression/predicate/Range.java | 9 +- .../comparison/EsqlBinaryComparison.java | 8 +- .../local/LucenePushdownPredicates.java | 54 +------ .../xpack/esql/plan/logical/MvExpand.java | 3 +- .../xpack/esql/planner/TranslatorHandler.java | 5 +- .../xpack/esql/analysis/AnalyzerTests.java | 144 ++++++++++++++---- 11 files changed, 218 insertions(+), 182 deletions(-) delete mode 100644 docs/changelog/123678.yaml diff --git a/docs/changelog/123678.yaml b/docs/changelog/123678.yaml deleted file mode 100644 index f32b0f966fb7c..0000000000000 --- a/docs/changelog/123678.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 123678 -summary: Date nanos implicit casting -area: ES|QL -type: enhancement -issues: - - 110009 diff --git a/x-pack/plugin/esql/qa/server/single-node/build.gradle b/x-pack/plugin/esql/qa/server/single-node/build.gradle index 03f2383ecfcd9..81f8ace464e1a 100644 --- a/x-pack/plugin/esql/qa/server/single-node/build.gradle +++ b/x-pack/plugin/esql/qa/server/single-node/build.gradle @@ -13,6 +13,7 @@ dependencies { javaRestTestImplementation project(xpackModule('esql:qa:testFixtures')) javaRestTestImplementation project(xpackModule('esql:qa:server')) javaRestTestImplementation project(xpackModule('esql:tools')) + javaRestTestImplementation project(xpackModule('esql')) yamlRestTestImplementation project(xpackModule('esql:qa:server')) javaRestTestImplementation('org.apache.arrow:arrow-vector:16.1.0') diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java index 545b01f4f3066..4bfd07bbf35ae 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java @@ -55,6 +55,7 @@ import static org.elasticsearch.test.ListMatcher.matchesList; import static org.elasticsearch.test.MapMatcher.assertMap; import static org.elasticsearch.test.MapMatcher.matchesMap; +import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.IMPLICIT_CASTING_DATE_AND_DATE_NANOS; import static org.elasticsearch.xpack.esql.core.type.DataType.isMillisOrNanos; import static org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase.Mode.SYNC; import static org.elasticsearch.xpack.esql.tools.ProfileParser.parseProfile; @@ -725,10 +726,6 @@ public void testSuggestedCast() throws IOException { for (int i = 0; i < listOfTypes.size(); i++) { for (int j = i + 1; j < listOfTypes.size(); j++) { - if (isMillisOrNanos(listOfTypes.get(i)) && isMillisOrNanos(listOfTypes.get(j))) { - // datetime and date_nanos are casted to date_nanos implicitly - continue; - } String query = String.format(Locale.ROOT, """ { "query": "FROM index-%s,index-%s | LIMIT 100 | KEEP my_field" @@ -740,19 +737,26 @@ public void testSuggestedCast() throws IOException { Map results = entityAsMap(resp); List columns = (List) results.get("columns"); DataType suggestedCast = DataType.suggestedCast(Set.of(listOfTypes.get(i), listOfTypes.get(j))); - assertThat( - columns, - equalTo( - List.of( - Map.ofEntries( - Map.entry("name", "my_field"), - Map.entry("type", "unsupported"), - Map.entry("original_types", List.of(listOfTypes.get(i).typeName(), listOfTypes.get(j).typeName())), - Map.entry("suggested_cast", suggestedCast.typeName()) + if (IMPLICIT_CASTING_DATE_AND_DATE_NANOS.isEnabled() + && isMillisOrNanos(listOfTypes.get(i)) + && isMillisOrNanos(listOfTypes.get(j))) { + // datetime and date_nanos are casted to date_nanos implicitly + assertThat(columns, equalTo(List.of(Map.ofEntries(Map.entry("name", "my_field"), Map.entry("type", "date_nanos"))))); + } else { + assertThat( + columns, + equalTo( + List.of( + Map.ofEntries( + Map.entry("name", "my_field"), + Map.entry("type", "unsupported"), + Map.entry("original_types", List.of(listOfTypes.get(i).typeName(), listOfTypes.get(j).typeName())), + Map.entry("suggested_cast", suggestedCast.typeName()) + ) ) ) - ) - ); + ); + } String castedQuery = String.format( Locale.ROOT, diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec index f9449c2ca9d81..7919955db1ec7 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec @@ -1712,15 +1712,15 @@ required_capability: date_nanos_type required_capability: implicit_casting_date_and_date_nanos FROM employees, employees_incompatible -| EVAL x = emp_no::long, avg_worked_seconds = avg_worked_seconds::unsigned_long, y = hire_date::datetime, z = hire_date::date_nanos -| KEEP x, languages, hire_date, avg_worked_seconds, y, z +| EVAL x = emp_no::long, avg_worked_seconds = avg_worked_seconds::unsigned_long, y = hire_date::datetime, z = hire_date::date_nanos, w = hire_date +| KEEP x, languages, hire_date, avg_worked_seconds, y, z, w | SORT x, z | LIMIT 2 ; -x:long |languages:unsupported |hire_date:date_nanos |avg_worked_seconds:unsigned_long |y:datetime |z:date_nanos -10001 |null |1986-06-26T00:00:00.000Z |268728049 |1986-06-26T00:00:00.000Z |1986-06-26T00:00:00.000Z -10001 |null |1986-06-26T00:00:00.000Z |268728049 |1986-06-26T00:00:00.000Z |1986-06-26T00:00:00.000Z +x:long |languages:unsupported |hire_date:date_nanos |avg_worked_seconds:unsigned_long |y:datetime |z:date_nanos |w:date_nanos +10001 |null |1986-06-26T00:00:00.000Z |268728049 |1986-06-26T00:00:00.000Z |1986-06-26T00:00:00.000Z |1986-06-26T00:00:00.000Z +10001 |null |1986-06-26T00:00:00.000Z |268728049 |1986-06-26T00:00:00.000Z |1986-06-26T00:00:00.000Z |1986-06-26T00:00:00.000Z ; ImplicitCastingMultiTypedFieldsDropKeepSort @@ -2075,6 +2075,39 @@ millis:date_nanos | nanos:date_nanos | num:long 1999-10-23T12:15:03.360Z | 2023-02-23T13:33:34.937193Z | 0 ; +ImplicitCastingMultiTypedFieldsBackAndForth +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM date_nanos, date_nanos_union_types +| MV_EXPAND nanos +| EVAL a = nanos::datetime, b = nanos::date_nanos, c = nanos, d = nanos::datetime::date_nanos, e = nanos::date_nanos::datetime +| KEEP millis, nanos, a, b, c, d, e +| SORT nanos, millis +| LIMIT 14 +; + +warningRegex:evaluation of \[FROM .*date_nanos.*date_nanos_union_types.*\] failed, treating result as null. Only first 20 failures recorded. +warningRegex:java.lang.IllegalArgumentException: milliSeconds \[-1457696696640\] are before the epoch in 1970 and cannot be converted to nanoseconds + +millis:date_nanos | nanos:date_nanos | a:datetime | b:date_nanos | c:date_nanos | d:date_nanos | e:datetime +1999-10-23T12:15:03.360103847Z | 2023-01-23T13:55:01.543Z | 2023-01-23T13:55:01.543Z | 2023-01-23T13:55:01.543Z | 2023-01-23T13:55:01.543Z | 2023-01-23T13:55:01.543Z | 2023-01-23T13:55:01.543Z +1999-10-23T12:15:03.360Z | 2023-01-23T13:55:01.543123456Z | 2023-01-23T13:55:01.543Z | 2023-01-23T13:55:01.543123456Z | 2023-01-23T13:55:01.543123456Z | 2023-01-23T13:55:01.543Z | 2023-01-23T13:55:01.543Z +1999-10-23T12:15:03.360103847Z | 2023-02-23T13:33:34.937Z | 2023-02-23T13:33:34.937Z | 2023-02-23T13:33:34.937Z | 2023-02-23T13:33:34.937Z | 2023-02-23T13:33:34.937Z | 2023-02-23T13:33:34.937Z +1999-10-23T12:15:03.360Z | 2023-02-23T13:33:34.937193Z | 2023-02-23T13:33:34.937Z | 2023-02-23T13:33:34.937193Z | 2023-02-23T13:33:34.937193Z | 2023-02-23T13:33:34.937Z | 2023-02-23T13:33:34.937Z +1999-10-22T12:15:03.360103847Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z +1999-10-22T12:15:03.360103847Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z +1999-10-22T12:15:03.360103847Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z +1999-10-23T12:15:03.360103847Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z +1999-10-22T12:15:03.360Z | 2023-03-23T12:15:03.360103847Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360103847Z | 2023-03-23T12:15:03.360103847Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z +1999-10-22T12:15:03.360Z | 2023-03-23T12:15:03.360103847Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360103847Z | 2023-03-23T12:15:03.360103847Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z +1999-10-22T12:15:03.360Z | 2023-03-23T12:15:03.360103847Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360103847Z | 2023-03-23T12:15:03.360103847Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z +1999-10-23T12:15:03.360Z | 2023-03-23T12:15:03.360103847Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360103847Z | 2023-03-23T12:15:03.360103847Z | 2023-03-23T12:15:03.360Z | 2023-03-23T12:15:03.360Z +2023-10-23T12:15:03.360103847Z | 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360Z +2023-10-23T12:15:03.360103847Z | 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360Z +; + + ImplicitCastingMultiTypedMVFieldsEval required_capability: date_nanos_type required_capability: implicit_casting_date_and_date_nanos diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index f64f7ed75f36e..06af72bbb7a56 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -173,7 +173,7 @@ public class Analyzer extends ParameterizedRuleExecutor( "Resolution", @@ -1665,9 +1665,7 @@ private Expression resolveConvertFunction(ConvertFunction convert, List { if (supportedTypes.contains(type.widenSmallNumeric())) { - TypeResolutionKey key = new TypeResolutionKey(fa.name(), type); - var concreteConvert = typeSpecificConvert(convert, fa.source(), type, imf); - typeResolutions.put(key, concreteConvert); + typeResolutions(fa, convert, type, imf, typeResolutions); } }); // If all mapped types were resolved, create a new FieldAttribute with the resolved MultiTypeEsField @@ -1676,53 +1674,36 @@ private Expression resolveConvertFunction(ConvertFunction convert, List typeResolutions = new HashMap<>(); - Set supportedTypes = convert.supportedTypes(); - Holder conversionSupported = new Holder<>(true); - // Get the type of each field in the multi typed field and check if the explicit conversion is supported - mtf.getIndexToConversionExpressions().forEach((indexName, conversionExpression) -> { - AbstractConvertFunction c = (AbstractConvertFunction) conversionExpression; - FieldAttribute fieldAttribute = (FieldAttribute) c.field(); - DataType type = fieldAttribute.dataType(); - if (supportedTypes.contains(type.widenSmallNumeric())) { - TypeResolutionKey key = new TypeResolutionKey(fa.name(), type); - var concreteConvert = typeSpecificConvert(convert, fa.source(), fieldAttribute.field()); - typeResolutions.putIfAbsent(key, concreteConvert); - } else { - conversionSupported.set(false); - } - }); - // If the conversions are supported, all the data types in a MultiTypeEsField can be cast to the explicit casting - // data type, create a new FieldAttribute with a new MultiTypeEsField, and add it to unionFieldAttributes. - if (conversionSupported.get()) { - // Build the mapping between index name and conversion expressions, as a MultiTypeEsField does not store the - // mapping between data types and index names, - Map indexToConversionExpressions = new HashMap<>(); - for (Map.Entry entry : mtf.getIndexToConversionExpressions().entrySet()) { - String indexName = entry.getKey(); - AbstractConvertFunction originalConversionFunction = (AbstractConvertFunction) entry.getValue(); - TypeResolutionKey key = new TypeResolutionKey(fa.name(), originalConversionFunction.field().dataType()); - Expression newConversionFunction = typeResolutions.get(key); - indexToConversionExpressions.put(indexName, newConversionFunction); - } - MultiTypeEsField multiTypeEsField = new MultiTypeEsField( - fa.name(), - ((Expression) convert).dataType(), - false, - indexToConversionExpressions - ); - return createIfDoesNotAlreadyExist(fa, multiTypeEsField, unionFieldAttributes); + // The same data type between implicit(date_nanos) and explicit casting, explicit conversion is not needed, mark is + // as synthetic = true as it is an explicit conversion + return createIfDoesNotAlreadyExist(fa, mtf, unionFieldAttributes); + } + + // Data type is different between implicit(date_nanos) and explicit casting, if the conversion is supported, create a + // new MultiTypeEsField with explicit casting type, and add it to unionFieldAttributes. + Set supportedTypes = convert.supportedTypes(); + if (supportedTypes.contains(fa.dataType())) { + // Build the mapping between index name and conversion expressions + Map indexToConversionExpressions = new HashMap<>(); + for (Map.Entry entry : mtf.getIndexToConversionExpressions().entrySet()) { + String indexName = entry.getKey(); + AbstractConvertFunction originalConversionFunction = (AbstractConvertFunction) entry.getValue(); + Expression originalField = originalConversionFunction.field(); + Expression newConvertFunction = convertExpression.replaceChildren(Collections.singletonList(originalField)); + indexToConversionExpressions.put(indexName, newConvertFunction); } + MultiTypeEsField multiTypeEsField = new MultiTypeEsField( + fa.name(), + convertExpression.dataType(), + false, + indexToConversionExpressions + ); + return createIfDoesNotAlreadyExist(fa, multiTypeEsField, unionFieldAttributes); } } else if (convert.field() instanceof AbstractConvertFunction subConvert) { return convertExpression.replaceChildren( @@ -1852,7 +1833,7 @@ private static LogicalPlan planWithoutSyntheticAttributes(LogicalPlan plan) { /** * Cast the union typed fields in EsRelation to date_nanos if they are mixed date and date_nanos types. */ - private static class ResolveUnionTypesInEsRelation extends Rule { + private static class DateMillisToNanosInEsRelation extends Rule { @Override public LogicalPlan apply(LogicalPlan plan) { return plan.transformUp(EsRelation.class, relation -> { @@ -1863,11 +1844,7 @@ public LogicalPlan apply(LogicalPlan plan) { if (f.field() instanceof InvalidMappedField imf && imf.types().stream().allMatch(DataType::isDate)) { HashMap typeResolutions = new HashMap<>(); var convert = new ToDateNanos(f.source(), f); - imf.types().forEach(type -> { - ResolveUnionTypes.TypeResolutionKey key = new ResolveUnionTypes.TypeResolutionKey(f.name(), type); - var concreteConvert = ResolveUnionTypes.typeSpecificConvert(convert, f.source(), type, imf); - typeResolutions.put(key, concreteConvert); - }); + imf.types().forEach(type -> typeResolutions(f, convert, type, imf, typeResolutions)); var resolvedField = ResolveUnionTypes.resolvedMultiTypeEsField(f, typeResolutions); return new FieldAttribute(f.source(), f.parentName(), f.name(), resolvedField, f.nullable(), f.id(), f.synthetic()); } @@ -1876,4 +1853,16 @@ public LogicalPlan apply(LogicalPlan plan) { }); } } + + private static void typeResolutions( + FieldAttribute fieldAttribute, + ConvertFunction convert, + DataType type, + InvalidMappedField imf, + HashMap typeResolutions + ) { + ResolveUnionTypes.TypeResolutionKey key = new ResolveUnionTypes.TypeResolutionKey(fieldAttribute.name(), type); + var concreteConvert = ResolveUnionTypes.typeSpecificConvert(convert, fieldAttribute.source(), type, imf); + typeResolutions.put(key, concreteConvert); + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/Range.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/Range.java index 89d37eb37b29b..6427449add799 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/Range.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/Range.java @@ -14,7 +14,6 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; -import org.elasticsearch.xpack.esql.core.expression.TypedAttribute; import org.elasticsearch.xpack.esql.core.expression.function.scalar.ScalarFunction; import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.esql.core.querydsl.query.Query; @@ -280,13 +279,7 @@ private RangeQuery translate(TranslatorHandler handler) { } } logger.trace("Building range query with format string [{}]", format); - // This is a similar check as in EsqlBinaryComparison - // Extract the real field name from MultiTypeEsField, and use it in the push down query if it is found - TypedAttribute attribute = LucenePushdownPredicates.checkIsPushableAttribute(value); - String name = handler.nameOf(attribute); - String fieldNameFromMultiTypeEsField = LucenePushdownPredicates.extractFieldNameFromMultiTypeEsField(attribute); - name = fieldNameFromMultiTypeEsField != null ? fieldNameFromMultiTypeEsField : name; - return new RangeQuery(source(), name, l, includeLower(), u, includeUpper(), format, zoneId); + return new RangeQuery(source(), handler.nameOf(value), l, includeLower(), u, includeUpper(), format, zoneId); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java index 362ea95bce8f2..69ef99ba04d15 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java @@ -378,9 +378,6 @@ public Expression singleValueField() { private Query translate(TranslatorHandler handler) { TypedAttribute attribute = LucenePushdownPredicates.checkIsPushableAttribute(left()); String name = handler.nameOf(attribute); - // Extract the real field name from MultiTypeEsField, and use it in the push down query if it is found - String fieldNameFromMultiTypeEsField = LucenePushdownPredicates.extractFieldNameFromMultiTypeEsField(attribute); - name = fieldNameFromMultiTypeEsField != null ? fieldNameFromMultiTypeEsField : name; Object value = valueOf(FoldContext.small() /* TODO remove me */, right()); String format = null; boolean isDateLiteralComparison = false; @@ -455,10 +452,7 @@ private Query translate(TranslatorHandler handler) { return new RangeQuery(source(), name, null, false, value, true, format, zoneId); } if (this instanceof Equals || this instanceof NotEquals) { - // Extract the real field name from MultiTypeEsField, and use it in the push down query if it is found - name = fieldNameFromMultiTypeEsField != null - ? fieldNameFromMultiTypeEsField - : LucenePushdownPredicates.pushableAttributeName(attribute); + name = LucenePushdownPredicates.pushableAttributeName(attribute); Query query; if (isDateLiteralComparison) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java index a51a7f1a51154..7843f8a6cfe04 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java @@ -7,22 +7,14 @@ package org.elasticsearch.xpack.esql.optimizer.rules.physical.local; -import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.expression.TypedAttribute; import org.elasticsearch.xpack.esql.core.type.DataType; -import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField; import org.elasticsearch.xpack.esql.core.util.Check; -import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; import org.elasticsearch.xpack.esql.stats.SearchStats; -import java.util.Map; -import java.util.function.Predicate; - -import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS; - /** * When deciding if a filter or topN can be pushed down to Lucene, we need to check a few things on the field. * Exactly what is checked depends on the type of field and the query. For example, we have the following possible combinations: @@ -104,46 +96,6 @@ static String pushableAttributeName(TypedAttribute attribute) { : attribute.name(); } - /** - * Extract the real field name from a MultiTypeEsField, limit to MultiTypeEsField that has date_nanos type only. - * - * For example, the name of a MultiTypeEsField can be $$myfield$converted_to$date_nanos, and the real field name extract from the - * MultiTypeEsField is myfield, this method return myfield given a MultiTypeEsField. - * - * If the real field name is found, and the original field data types contain only date and date_nanos types, return the real field - * name, so that the real field name will be used to check for eligibility of being pushed down, and the real field name will be used - * in the push down query, instead of the name of the MultiTypeEsField, which should not match any field in an index. - * - * This method can be extended to support the other data types in the future if there is a need. - */ - static String extractFieldNameFromMultiTypeEsField(TypedAttribute attribute) { - if (EsqlCapabilities.Cap.IMPLICIT_CASTING_DATE_AND_DATE_NANOS.isEnabled() - && attribute instanceof FieldAttribute fa - && fa.field() instanceof MultiTypeEsField multiTypeEsField - && fa.dataType() == DATE_NANOS - && // limit to casting to date_nanos only - mixedDateAndDateNanosOnly(multiTypeEsField, DataType::isMillisOrNanos) // limit to mixed date and date_nanos only - ) { - return fa.fieldName(); - } - return null; - } - - /** - * Check if the original field types in a MultiTypeEsField satisfy the required data types defined in the predicate. - */ - private static boolean mixedDateAndDateNanosOnly(MultiTypeEsField multiTypeEsField, Predicate predicate) { - Map indexToConversionExpressions = multiTypeEsField.getIndexToConversionExpressions(); - for (Map.Entry entry : indexToConversionExpressions.entrySet()) { - Expression conversionFunction = entry.getValue(); - if (conversionFunction instanceof AbstractConvertFunction abstractConvertFunction - && predicate.test(abstractConvertFunction.field().dataType()) == false) { - return false; - } - } - return true; - } - /** * The default implementation of this has no access to SearchStats, so it can only make decisions based on the FieldAttribute itself. * In particular, it assumes TEXT fields have no exact subfields (underlying keyword field), @@ -186,14 +138,10 @@ public boolean hasExactSubfield(FieldAttribute attr) { @Override public boolean isIndexedAndHasDocValues(FieldAttribute attr) { - // If this is a MultiTypeEsField cast to date_nanos, make it eligible for being pushed down by checking the real - // field name against SearchStats - String fieldNameFromMultiTypeEsField = LucenePushdownPredicates.extractFieldNameFromMultiTypeEsField(attr); - String name = fieldNameFromMultiTypeEsField != null ? fieldNameFromMultiTypeEsField : attr.name(); // We still consider the value of isAggregatable here, because some fields like ScriptFieldTypes are always aggregatable // But this could hide issues with fields that are not indexed but are aggregatable // This is the original behaviour for ES|QL, but is it correct? - return attr.field().isAggregatable() || stats.isIndexed(name) && stats.hasDocValues(name); + return attr.field().isAggregatable() || stats.isIndexed(attr.name()) && stats.hasDocValues(attr.name()); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/MvExpand.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/MvExpand.java index 52941018ffae8..f65811fc26526 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/MvExpand.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/MvExpand.java @@ -16,7 +16,6 @@ import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; -import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import java.io.IOException; @@ -91,7 +90,7 @@ public String telemetryLabel() { @Override public boolean expressionsResolved() { - return target.resolved() && target.dataType() != DataType.UNSUPPORTED; + return target.resolved(); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/TranslatorHandler.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/TranslatorHandler.java index e06d2531cc4a9..4b7af5bf49de8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/TranslatorHandler.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/TranslatorHandler.java @@ -46,10 +46,7 @@ private static Query wrapFunctionQuery(Expression field, Query query) { } if (field instanceof FieldAttribute fa) { fa = fa.getExactInfo().hasExact() ? fa.exactAttribute() : fa; - // Extract the real field name from MultiTypeEsField, and use it in the push down query if it is found - String fieldNameFromMultiTypeEsField = LucenePushdownPredicates.extractFieldNameFromMultiTypeEsField(fa); - String fieldName = fieldNameFromMultiTypeEsField != null ? fieldNameFromMultiTypeEsField : fa.name(); - return new SingleValueQuery(query, fieldName, false); + return new SingleValueQuery(query, fa.name(), false); } if (field instanceof MetadataAttribute) { return query; // MetadataAttributes are always single valued diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index aa4a0a7f3657e..4ed6462b6e233 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -49,7 +49,10 @@ import org.elasticsearch.xpack.esql.expression.function.fulltext.MultiMatch; import org.elasticsearch.xpack.esql.expression.function.fulltext.QueryString; import org.elasticsearch.xpack.esql.expression.function.grouping.Bucket; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDateNanos; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDatetime; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToInteger; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToLong; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Concat; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Substring; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add; @@ -118,7 +121,10 @@ import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.tsdbIndexResolution; import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; +import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS; import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_PERIOD; +import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; +import static org.elasticsearch.xpack.esql.core.type.DataType.UNSUPPORTED; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToString; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; @@ -3945,59 +3951,137 @@ public void testImplicitCastingForDateAndDateNanosFields() { IndexResolution indexWithUnionTypedFields = indexWithDateDateNanosUnionType(); Analyzer analyzer = AnalyzerTestUtils.analyzer(indexWithUnionTypedFields); - // validate if a union typed field is cast to a type explicitly, implicit casting won't be applied again + // Validate if a union typed field is cast to a type explicitly, implicit casting won't be applied again, and include some cases of + // nested casting as well. LogicalPlan plan = analyze(""" FROM tests - | Eval x = date_and_date_nanos::datetime, y = date_and_date_nanos, z = date_and_date_nanos::date_nanos + | Eval a = date_and_date_nanos, b = date_and_date_nanos::datetime, c = date_and_date_nanos::date_nanos, + d = date_and_date_nanos::datetime::datetime, e = date_and_date_nanos::datetime::date_nanos, + f = date_and_date_nanos::date_nanos::datetime, g = date_and_date_nanos::date_nanos::date_nanos, + h = date_and_date_nanos::datetime::long, i = date_and_date_nanos::date_nanos::long, + j = date_and_date_nanos::long::datetime, k = date_and_date_nanos::long::date_nanos """, analyzer); Project project = as(plan, Project.class); List projections = project.projections(); - assertEquals(5, projections.size()); + assertEquals(13, projections.size()); // implicit casting FieldAttribute fa = as(projections.get(0), FieldAttribute.class); - assertEquals("date_and_date_nanos", fa.name()); - assertEquals(DataType.DATE_NANOS, fa.dataType()); + verifyNameAndTypeAndMultiTypeEsField(fa.name(), fa.dataType(), "date_and_date_nanos", DATE_NANOS, fa); // long is not cast to date_nanos UnsupportedAttribute ua = as(projections.get(1), UnsupportedAttribute.class); - assertEquals("date_and_date_nanos_and_long", ua.name()); - assertEquals(DataType.UNSUPPORTED, ua.dataType()); - // explicit casting - ReferenceAttribute ra = as(projections.get(2), ReferenceAttribute.class); - assertEquals("x", ra.name()); - assertEquals(DataType.DATETIME, ra.dataType()); + verifyNameAndType(ua.name(), ua.dataType(), "date_and_date_nanos_and_long", UNSUPPORTED); // implicit casting - ra = as(projections.get(3), ReferenceAttribute.class); - assertEquals("y", ra.name()); - assertEquals(DataType.DATE_NANOS, ra.dataType()); + ReferenceAttribute ra = as(projections.get(2), ReferenceAttribute.class); + verifyNameAndType(ra.name(), ra.dataType(), "a", DATE_NANOS); // explicit casting + ra = as(projections.get(3), ReferenceAttribute.class); + verifyNameAndType(ra.name(), ra.dataType(), "b", DATETIME); ra = as(projections.get(4), ReferenceAttribute.class); - assertEquals("z", ra.name()); - assertEquals(DataType.DATE_NANOS, ra.dataType()); + verifyNameAndType(ra.name(), ra.dataType(), "c", DATE_NANOS); + ra = as(projections.get(5), ReferenceAttribute.class); + verifyNameAndType(ra.name(), ra.dataType(), "d", DATETIME); + ra = as(projections.get(6), ReferenceAttribute.class); + verifyNameAndType(ra.name(), ra.dataType(), "e", DATE_NANOS); + ra = as(projections.get(7), ReferenceAttribute.class); + verifyNameAndType(ra.name(), ra.dataType(), "f", DATETIME); + ra = as(projections.get(8), ReferenceAttribute.class); + verifyNameAndType(ra.name(), ra.dataType(), "g", DATE_NANOS); + ra = as(projections.get(9), ReferenceAttribute.class); + verifyNameAndType(ra.name(), ra.dataType(), "h", LONG); + ra = as(projections.get(10), ReferenceAttribute.class); + verifyNameAndType(ra.name(), ra.dataType(), "i", LONG); + ra = as(projections.get(11), ReferenceAttribute.class); + verifyNameAndType(ra.name(), ra.dataType(), "j", DATETIME); + ra = as(projections.get(12), ReferenceAttribute.class); + verifyNameAndType(ra.name(), ra.dataType(), "k", DATE_NANOS); + Limit limit = as(project.child(), Limit.class); // original Eval coded in the query Eval eval = as(limit.child(), Eval.class); List aliases = eval.fields(); - assertEquals(3, aliases.size()); - // explicit casting - Alias a = aliases.get(0); - assertEquals("x", a.name()); - assertEquals(DataType.DATETIME, a.dataType()); - assertTrue(isMultiTypeEsField(a.child())); + assertEquals(11, aliases.size()); // implicit casting - a = aliases.get(1); - assertEquals("y", a.name()); - assertEquals(DataType.DATE_NANOS, a.dataType()); - assertTrue(a.child() instanceof FieldAttribute); + Alias a = aliases.get(0); // a = date_and_date_nanos + verifyNameAndTypeAndMultiTypeEsField(a.name(), a.dataType(), "a", DATE_NANOS, a.child()); // explicit casting - a = aliases.get(2); - assertEquals("z", a.name()); - assertEquals(DataType.DATE_NANOS, a.dataType()); - assertTrue(isMultiTypeEsField(a.child())); + a = aliases.get(1); // b = date_and_date_nanos::datetime + verifyNameAndTypeAndMultiTypeEsField(a.name(), a.dataType(), "b", DATETIME, a.child()); + a = aliases.get(2); // c = date_and_date_nanos::date_nanos + verifyNameAndTypeAndMultiTypeEsField(a.name(), a.dataType(), "c", DATE_NANOS, a.child()); + a = aliases.get(3); // d = date_and_date_nanos::datetime::datetime + verifyNameAndType(a.name(), a.dataType(), "d", DATETIME); + ToDatetime toDatetime = as(a.child(), ToDatetime.class); + fa = as(toDatetime.field(), FieldAttribute.class); + verifyNameAndTypeAndMultiTypeEsField( + fa.name(), + fa.dataType(), + "$$date_and_date_nanos$converted_to$datetime", + DATETIME, + toDatetime.field() + ); + a = aliases.get(4); // e = date_and_date_nanos::datetime::date_nanos + verifyNameAndType(a.name(), a.dataType(), "e", DATE_NANOS); + ToDateNanos toDateNanos = as(a.child(), ToDateNanos.class); + fa = as(toDateNanos.field(), FieldAttribute.class); + verifyNameAndTypeAndMultiTypeEsField( + fa.name(), + fa.dataType(), + "$$date_and_date_nanos$converted_to$datetime", + DATETIME, + toDateNanos.field() + ); + a = aliases.get(5); // f = date_and_date_nanos::date_nanos::datetime + verifyNameAndType(a.name(), a.dataType(), "f", DATETIME); + toDatetime = as(a.child(), ToDatetime.class); + fa = as(toDatetime.field(), FieldAttribute.class); + verifyNameAndTypeAndMultiTypeEsField(fa.name(), fa.dataType(), "$$date_and_date_nanos$converted_to$date_nanos", DATE_NANOS, fa); + a = aliases.get(6); // g = date_and_date_nanos::date_nanos::date_nanos + verifyNameAndType(a.name(), a.dataType(), "g", DATE_NANOS); + toDateNanos = as(a.child(), ToDateNanos.class); + fa = as(toDateNanos.field(), FieldAttribute.class); + verifyNameAndTypeAndMultiTypeEsField(fa.name(), fa.dataType(), "$$date_and_date_nanos$converted_to$date_nanos", DATE_NANOS, fa); + a = aliases.get(7); // h = date_and_date_nanos::datetime::long + verifyNameAndType(a.name(), a.dataType(), "h", LONG); + ToLong toLong = as(a.child(), ToLong.class); + fa = as(toLong.field(), FieldAttribute.class); + verifyNameAndTypeAndMultiTypeEsField(fa.name(), fa.dataType(), "$$date_and_date_nanos$converted_to$datetime", DATETIME, fa); + a = aliases.get(8); // i = date_and_date_nanos::date_nanos::long + verifyNameAndType(a.name(), a.dataType(), "i", LONG); + toLong = as(a.child(), ToLong.class); + fa = as(toLong.field(), FieldAttribute.class); + verifyNameAndTypeAndMultiTypeEsField(fa.name(), fa.dataType(), "$$date_and_date_nanos$converted_to$date_nanos", DATE_NANOS, fa); + a = aliases.get(9); // j = date_and_date_nanos::long::datetime + verifyNameAndType(a.name(), a.dataType(), "j", DATETIME); + toDatetime = as(a.child(), ToDatetime.class); + fa = as(toDatetime.field(), FieldAttribute.class); + verifyNameAndTypeAndMultiTypeEsField(fa.name(), fa.dataType(), "$$date_and_date_nanos$converted_to$long", LONG, fa); + a = aliases.get(10); // k = date_and_date_nanos::long::date_nanos + verifyNameAndType(a.name(), a.dataType(), "k", DATE_NANOS); + toDateNanos = as(a.child(), ToDateNanos.class); + fa = as(toDateNanos.field(), FieldAttribute.class); + verifyNameAndTypeAndMultiTypeEsField(fa.name(), fa.dataType(), "$$date_and_date_nanos$converted_to$long", LONG, fa); EsRelation esRelation = as(eval.child(), EsRelation.class); assertEquals("test*", esRelation.indexPattern()); } + private void verifyNameAndType(String actualName, DataType actualType, String expectedName, DataType expectedType) { + assertEquals(expectedName, actualName); + assertEquals(expectedType, actualType); + } + + private void verifyNameAndTypeAndMultiTypeEsField( + String actualName, + DataType actualType, + String expectedName, + DataType expectedType, + Expression e + ) { + assertEquals(expectedName, actualName); + assertEquals(expectedType, actualType); + assertTrue(isMultiTypeEsField(e)); + } + private boolean isMultiTypeEsField(Expression e) { return e instanceof FieldAttribute fa && fa.field() instanceof MultiTypeEsField; } From 56db2a597239bf8d487d402c0688c399e2e6b05f Mon Sep 17 00:00:00 2001 From: Fang Xing Date: Fri, 23 May 2025 11:57:15 -0400 Subject: [PATCH 21/24] more tests on invalid nanos valid millis --- .../src/main/resources/union_types.csv-spec | 31 +++++++++++++++++++ .../xpack/esql/analysis/Analyzer.java | 6 ++-- .../xpack/esql/analysis/AnalyzerTests.java | 15 ++------- 3 files changed, 36 insertions(+), 16 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec index 7919955db1ec7..ed913e747965d 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec @@ -2107,6 +2107,37 @@ millis:date_nanos | nanos:date_nanos | a:datetime 2023-10-23T12:15:03.360103847Z | 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360Z ; +ImplicitCastingMultiTypedFieldsInvalidNanosValidMillis +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM date_nanos, date_nanos_union_types +| EVAL a = nanos::datetime +| MV_EXPAND a +| KEEP a, millis, nanos +| SORT a, millis, nanos +| LIMIT 14 +; + +warningRegex:evaluation of \[FROM .*date_nanos.*date_nanos_union_types.*\] failed, treating result as null. Only first 20 failures recorded. +warningRegex:java.lang.IllegalArgumentException: milliSeconds \[-1457696696640\] are before the epoch in 1970 and cannot be converted to nanoseconds + +a:datetime | millis:date_nanos | nanos:date_nanos +1923-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | null +2023-01-23T13:55:01.543Z | 1999-10-23T12:15:03.360Z | [2023-01-23T13:55:01.543123456Z, 2023-02-23T13:33:34.937193Z, 2023-03-23T12:15:03.360103847Z] +2023-01-23T13:55:01.543Z | 1999-10-23T12:15:03.360103847Z | [2023-01-23T13:55:01.543Z, 2023-02-23T13:33:34.937Z, 2023-03-23T12:15:03.360Z] +2023-02-23T13:33:34.937Z | 1999-10-23T12:15:03.360Z | [2023-01-23T13:55:01.543123456Z, 2023-02-23T13:33:34.937193Z, 2023-03-23T12:15:03.360103847Z] +2023-02-23T13:33:34.937Z | 1999-10-23T12:15:03.360103847Z | [2023-01-23T13:55:01.543Z, 2023-02-23T13:33:34.937Z, 2023-03-23T12:15:03.360Z] +2023-03-23T12:15:03.360Z | 1999-10-22T12:15:03.360Z | [2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z] +2023-03-23T12:15:03.360Z | 1999-10-22T12:15:03.360Z | [2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z] +2023-03-23T12:15:03.360Z | 1999-10-22T12:15:03.360Z | [2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z] +2023-03-23T12:15:03.360Z | 1999-10-22T12:15:03.360103847Z | [2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z] +2023-03-23T12:15:03.360Z | 1999-10-22T12:15:03.360103847Z | [2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z] +2023-03-23T12:15:03.360Z | 1999-10-22T12:15:03.360103847Z | [2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z] +2023-03-23T12:15:03.360Z | 1999-10-23T12:15:03.360Z | [2023-01-23T13:55:01.543123456Z, 2023-02-23T13:33:34.937193Z, 2023-03-23T12:15:03.360103847Z] +2023-03-23T12:15:03.360Z | 1999-10-23T12:15:03.360103847Z | [2023-01-23T13:55:01.543Z, 2023-02-23T13:33:34.937Z, 2023-03-23T12:15:03.360Z] +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z +; ImplicitCastingMultiTypedMVFieldsEval required_capability: date_nanos_type diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index d13bc25210f95..6e8c2543c339c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -1690,11 +1690,9 @@ private Expression resolveConvertFunction(ConvertFunction convert, List Date: Fri, 23 May 2025 17:53:40 -0400 Subject: [PATCH 22/24] more tests on implicit and explicit casting --- .../src/main/resources/union_types.csv-spec | 64 ++++++++++++++----- .../xpack/esql/analysis/Analyzer.java | 5 +- .../xpack/esql/analysis/AnalyzerTests.java | 15 ++++- 3 files changed, 62 insertions(+), 22 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec index ed913e747965d..ada984e39e81a 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec @@ -2090,7 +2090,7 @@ FROM date_nanos, date_nanos_union_types warningRegex:evaluation of \[FROM .*date_nanos.*date_nanos_union_types.*\] failed, treating result as null. Only first 20 failures recorded. warningRegex:java.lang.IllegalArgumentException: milliSeconds \[-1457696696640\] are before the epoch in 1970 and cannot be converted to nanoseconds -millis:date_nanos | nanos:date_nanos | a:datetime | b:date_nanos | c:date_nanos | d:date_nanos | e:datetime +millis:date_nanos | nanos:date_nanos | a:datetime | b:date_nanos | c:date_nanos | d:date_nanos | e:datetime 1999-10-23T12:15:03.360103847Z | 2023-01-23T13:55:01.543Z | 2023-01-23T13:55:01.543Z | 2023-01-23T13:55:01.543Z | 2023-01-23T13:55:01.543Z | 2023-01-23T13:55:01.543Z | 2023-01-23T13:55:01.543Z 1999-10-23T12:15:03.360Z | 2023-01-23T13:55:01.543123456Z | 2023-01-23T13:55:01.543Z | 2023-01-23T13:55:01.543123456Z | 2023-01-23T13:55:01.543123456Z | 2023-01-23T13:55:01.543Z | 2023-01-23T13:55:01.543Z 1999-10-23T12:15:03.360103847Z | 2023-02-23T13:33:34.937Z | 2023-02-23T13:33:34.937Z | 2023-02-23T13:33:34.937Z | 2023-02-23T13:33:34.937Z | 2023-02-23T13:33:34.937Z | 2023-02-23T13:33:34.937Z @@ -2114,7 +2114,6 @@ required_capability: implicit_casting_date_and_date_nanos FROM date_nanos, date_nanos_union_types | EVAL a = nanos::datetime | MV_EXPAND a -| KEEP a, millis, nanos | SORT a, millis, nanos | LIMIT 14 ; @@ -2122,21 +2121,52 @@ FROM date_nanos, date_nanos_union_types warningRegex:evaluation of \[FROM .*date_nanos.*date_nanos_union_types.*\] failed, treating result as null. Only first 20 failures recorded. warningRegex:java.lang.IllegalArgumentException: milliSeconds \[-1457696696640\] are before the epoch in 1970 and cannot be converted to nanoseconds -a:datetime | millis:date_nanos | nanos:date_nanos -1923-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | null -2023-01-23T13:55:01.543Z | 1999-10-23T12:15:03.360Z | [2023-01-23T13:55:01.543123456Z, 2023-02-23T13:33:34.937193Z, 2023-03-23T12:15:03.360103847Z] -2023-01-23T13:55:01.543Z | 1999-10-23T12:15:03.360103847Z | [2023-01-23T13:55:01.543Z, 2023-02-23T13:33:34.937Z, 2023-03-23T12:15:03.360Z] -2023-02-23T13:33:34.937Z | 1999-10-23T12:15:03.360Z | [2023-01-23T13:55:01.543123456Z, 2023-02-23T13:33:34.937193Z, 2023-03-23T12:15:03.360103847Z] -2023-02-23T13:33:34.937Z | 1999-10-23T12:15:03.360103847Z | [2023-01-23T13:55:01.543Z, 2023-02-23T13:33:34.937Z, 2023-03-23T12:15:03.360Z] -2023-03-23T12:15:03.360Z | 1999-10-22T12:15:03.360Z | [2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z] -2023-03-23T12:15:03.360Z | 1999-10-22T12:15:03.360Z | [2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z] -2023-03-23T12:15:03.360Z | 1999-10-22T12:15:03.360Z | [2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z] -2023-03-23T12:15:03.360Z | 1999-10-22T12:15:03.360103847Z | [2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z] -2023-03-23T12:15:03.360Z | 1999-10-22T12:15:03.360103847Z | [2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z] -2023-03-23T12:15:03.360Z | 1999-10-22T12:15:03.360103847Z | [2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z] -2023-03-23T12:15:03.360Z | 1999-10-23T12:15:03.360Z | [2023-01-23T13:55:01.543123456Z, 2023-02-23T13:33:34.937193Z, 2023-03-23T12:15:03.360103847Z] -2023-03-23T12:15:03.360Z | 1999-10-23T12:15:03.360103847Z | [2023-01-23T13:55:01.543Z, 2023-02-23T13:33:34.937Z, 2023-03-23T12:15:03.360Z] -2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z +millis:date_nanos | nanos:date_nanos | num:long | a:datetime +2023-10-23T12:15:03.360103847Z | null | 1698063303360103847 | 1923-10-23T12:15:03.360Z +1999-10-23T12:15:03.360Z | [2023-01-23T13:55:01.543123456Z, 2023-02-23T13:33:34.937193Z, 2023-03-23T12:15:03.360103847Z] | 0 | 2023-01-23T13:55:01.543Z +1999-10-23T12:15:03.360103847Z | [2023-01-23T13:55:01.543Z, 2023-02-23T13:33:34.937Z, 2023-03-23T12:15:03.360Z] | 0 | 2023-01-23T13:55:01.543Z +1999-10-23T12:15:03.360Z | [2023-01-23T13:55:01.543123456Z, 2023-02-23T13:33:34.937193Z, 2023-03-23T12:15:03.360103847Z] | 0 | 2023-02-23T13:33:34.937Z +1999-10-23T12:15:03.360103847Z | [2023-01-23T13:55:01.543Z, 2023-02-23T13:33:34.937Z, 2023-03-23T12:15:03.360Z] | 0 | 2023-02-23T13:33:34.937Z +1999-10-22T12:15:03.360Z | [2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z] | 0 | 2023-03-23T12:15:03.360Z +1999-10-22T12:15:03.360Z | [2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z] | 0 | 2023-03-23T12:15:03.360Z +1999-10-22T12:15:03.360Z | [2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z] | 0 | 2023-03-23T12:15:03.360Z +1999-10-22T12:15:03.360103847Z | [2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z] | 0 | 2023-03-23T12:15:03.360Z +1999-10-22T12:15:03.360103847Z | [2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z] | 0 | 2023-03-23T12:15:03.360Z +1999-10-22T12:15:03.360103847Z | [2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z] | 0 | 2023-03-23T12:15:03.360Z +1999-10-23T12:15:03.360Z | [2023-01-23T13:55:01.543123456Z, 2023-02-23T13:33:34.937193Z, 2023-03-23T12:15:03.360103847Z] | 0 | 2023-03-23T12:15:03.360Z +1999-10-23T12:15:03.360103847Z | [2023-01-23T13:55:01.543Z, 2023-02-23T13:33:34.937Z, 2023-03-23T12:15:03.360Z] | 0 | 2023-03-23T12:15:03.360Z +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 | 2023-10-23T12:15:03.360Z +; + +ImplicitCastingMultiTypedFieldsInvalidNanosValidMillisExplicitCastToNanos +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM date_nanos, date_nanos_union_types +| EVAL a = nanos::date_nanos::datetime +| MV_EXPAND a +| SORT a, millis, nanos +| LIMIT 14 +; + +warningRegex:evaluation of \[FROM .*date_nanos.*date_nanos_union_types.*\] failed, treating result as null. Only first 20 failures recorded. +warningRegex:java.lang.IllegalArgumentException: milliSeconds \[-1457696696640\] are before the epoch in 1970 and cannot be converted to nanoseconds + +millis:date_nanos | nanos:date_nanos | num:long | a:datetime +1999-10-23T12:15:03.360Z | [2023-01-23T13:55:01.543123456Z, 2023-02-23T13:33:34.937193Z, 2023-03-23T12:15:03.360103847Z] | 0 | 2023-01-23T13:55:01.543Z +1999-10-23T12:15:03.360103847Z | [2023-01-23T13:55:01.543Z, 2023-02-23T13:33:34.937Z, 2023-03-23T12:15:03.360Z] | 0 | 2023-01-23T13:55:01.543Z +1999-10-23T12:15:03.360Z | [2023-01-23T13:55:01.543123456Z, 2023-02-23T13:33:34.937193Z, 2023-03-23T12:15:03.360103847Z] | 0 | 2023-02-23T13:33:34.937Z +1999-10-23T12:15:03.360103847Z | [2023-01-23T13:55:01.543Z, 2023-02-23T13:33:34.937Z, 2023-03-23T12:15:03.360Z] | 0 | 2023-02-23T13:33:34.937Z +1999-10-22T12:15:03.360Z | [2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z] | 0 | 2023-03-23T12:15:03.360Z +1999-10-22T12:15:03.360Z | [2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z] | 0 | 2023-03-23T12:15:03.360Z +1999-10-22T12:15:03.360Z | [2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z] | 0 | 2023-03-23T12:15:03.360Z +1999-10-22T12:15:03.360103847Z | [2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z] | 0 | 2023-03-23T12:15:03.360Z +1999-10-22T12:15:03.360103847Z | [2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z] | 0 | 2023-03-23T12:15:03.360Z +1999-10-22T12:15:03.360103847Z | [2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z] | 0 | 2023-03-23T12:15:03.360Z +1999-10-23T12:15:03.360Z | [2023-01-23T13:55:01.543123456Z, 2023-02-23T13:33:34.937193Z, 2023-03-23T12:15:03.360103847Z] | 0 | 2023-03-23T12:15:03.360Z +1999-10-23T12:15:03.360103847Z | [2023-01-23T13:55:01.543Z, 2023-02-23T13:33:34.937Z, 2023-03-23T12:15:03.360Z] | 0 | 2023-03-23T12:15:03.360Z +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 | 2023-10-23T12:15:03.360Z +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 | 2023-10-23T12:15:03.360Z ; ImplicitCastingMultiTypedMVFieldsEval diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 6e8c2543c339c..f2d47390e82af 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -1690,9 +1690,10 @@ private Expression resolveConvertFunction(ConvertFunction convert, List Date: Fri, 23 May 2025 22:01:20 +0000 Subject: [PATCH 23/24] [CI] Auto commit changes from spotless --- .../java/org/elasticsearch/xpack/esql/analysis/Analyzer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index f2d47390e82af..7b76d764ef049 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -1691,7 +1691,7 @@ private Expression resolveConvertFunction(ConvertFunction convert, List Date: Tue, 3 Jun 2025 22:58:08 -0400 Subject: [PATCH 24/24] more tests and validation of data type conversions --- .../src/main/resources/union_types.csv-spec | 7 ++++++- .../xpack/esql/analysis/Analyzer.java | 19 ++++++++++++++++--- .../esql/planner/QueryTranslatorTests.java | 11 +++++++++++ 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec index ada984e39e81a..ce933bd6eba7f 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec @@ -2144,9 +2144,9 @@ required_capability: implicit_casting_date_and_date_nanos FROM date_nanos, date_nanos_union_types | EVAL a = nanos::date_nanos::datetime +| WHERE millis < "2023-10-23T13:00:00" | MV_EXPAND a | SORT a, millis, nanos -| LIMIT 14 ; warningRegex:evaluation of \[FROM .*date_nanos.*date_nanos_union_types.*\] failed, treating result as null. Only first 20 failures recorded. @@ -2167,6 +2167,11 @@ millis:date_nanos | nanos:date_nanos 1999-10-23T12:15:03.360103847Z | [2023-01-23T13:55:01.543Z, 2023-02-23T13:33:34.937Z, 2023-03-23T12:15:03.360Z] | 0 | 2023-03-23T12:15:03.360Z 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 | 2023-10-23T12:15:03.360Z 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 | 2023-10-23T12:15:03.360Z +2023-10-23T12:15:03.360103847Z | 2023-10-23T12:15:03.360Z | 1698063303360103847 | 2023-10-23T12:15:03.360Z +2023-10-23T12:15:03.360103847Z | 2023-10-23T12:15:03.360Z | 1698063303360103847 | 2023-10-23T12:15:03.360Z +2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948Z | 1698064048948000000 | 2023-10-23T12:27:28.948Z +2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948Z | 1698064048948000000 | 2023-10-23T12:27:28.948Z +2023-10-23T12:15:03.360103847Z | null | 1698063303360103847 | null ; ImplicitCastingMultiTypedMVFieldsEval diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index d6f80a1b33639..9790d1f1ddedc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -1697,7 +1697,10 @@ private Expression resolveConvertFunction(ConvertFunction convert, List supportedTypes = convert.supportedTypes(); - if (supportedTypes.contains(fa.dataType())) { + if (supportedTypes.contains(fa.dataType()) && canConvertOriginalTypes(mtf, supportedTypes)) { // Build the mapping between index name and conversion expressions Map indexToConversionExpressions = new HashMap<>(); for (Map.Entry entry : mtf.getIndexToConversionExpressions().entrySet()) { @@ -1716,7 +1719,7 @@ private Expression resolveConvertFunction(ConvertFunction convert, List supportedTypes) { + return multiTypeEsField.getIndexToConversionExpressions() + .values() + .stream() + .allMatch( + e -> e instanceof AbstractConvertFunction convertFunction + && supportedTypes.contains(convertFunction.field().dataType().widenSmallNumeric()) + ); + } + private static Expression typeSpecificConvert(ConvertFunction convert, Source source, DataType type, InvalidMappedField mtf) { EsField field = new EsField(mtf.getName(), type, mtf.getProperties(), mtf.isAggregatable()); return typeSpecificConvert(convert, source, field); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java index ff47227cd25f8..5683f57c37965 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java @@ -436,5 +436,16 @@ public void testToDateNanos() { "2020-01-01T00:00:00.000Z","boost":0.0.*""" + """ esql_single_value":\\{"field":"date_and_date_nanos".*"must_not".*"term":\\{"date_and_date_nanos":\\{"value":\ "2025-01-01T00:00:00.000Z","boost":0.0.*""")); + + // explicit casting + assertQueryTranslationDateDateNanosUnionTypes( + """ + FROM test* | WHERE date_and_date_nanos::datetime < "2025-12-31" and date_and_date_nanos > "2025-01-01\"""", + stats, + containsString(""" + "esql_single_value":{"field":"date_and_date_nanos",\ + "next":{"range":{"date_and_date_nanos":{"gt":"2025-01-01T00:00:00.000Z","time_zone":"Z",\ + "format":"strict_date_optional_time_nanos","boost":0.0}}}""") + ); } }