Skip to content

ESQL: Add telemetry support for Lookup Join On Expression #134942

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Sep 19, 2025
Merged

ESQL: Add telemetry support for Lookup Join On Expression #134942

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
eacaf40
Add telemetry for expression join
julian-elastic Sep 16, 2025
ff43ea3
Update docs/changelog/134942.yaml
julian-elastic Sep 17, 2025
cf67bf2
Merge branch 'main' into exprJoinTelemetry
julian-elastic Sep 18, 2025
a972adb
Keep the old name for field join to preserve BWC
julian-elastic Sep 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs/changelog/134942.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
pr: 134942
summary: Add telemetry support for Lookup Join On Expression
area: ES|QL
type: enhancement
issues: []
23 changes: 23 additions & 0 deletions ...in/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TelemetryIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,23 @@ public static Iterable<Object[]> parameters() {
Map.ofEntries(Map.entry("TO_STRING", 1)),
true
) },
new Object[] {
new Test(
"""
FROM idx
| EVAL y = to_str(host)
| RENAME host as host_left
| LOOKUP JOIN lookup_idx ON host_left == host
""",
Map.ofEntries(
Map.entry("RENAME", 1),
Map.entry("FROM", 1),
Map.entry("EVAL", 1),
Map.entry("LOOKUP JOIN ON EXPRESSION", 1)
),
Map.ofEntries(Map.entry("TO_STRING", 1)),
true
) },
new Object[] {
new Test(
"TS time_series_idx | LIMIT 10",
Expand Down Expand Up @@ -184,6 +201,12 @@ public void init() {
}

public void testMetrics() throws Exception {
if (testCase.query().contains("LOOKUP JOIN lookup_idx ON host_left == host")) {
assumeTrue(
"requires LOOKUP JOIN ON boolean expression capability",
EsqlCapabilities.Cap.LOOKUP_JOIN_ON_BOOLEAN_EXPRESSION.isEnabled()
);
}
DiscoveryNode dataNode = randomDataNode();
testQuery(dataNode, testCase);
}
Expand Down
3 changes: 2 additions & 1 deletion x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/telemetry/FeatureMetric.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,8 @@ public enum FeatureMetric {
DROP(Drop.class::isInstance),
KEEP(Keep.class::isInstance),
RENAME(Rename.class::isInstance),
LOOKUP_JOIN(LookupJoin.class::isInstance),
LOOKUP_JOIN(plan -> plan instanceof LookupJoin lookupJoin && lookupJoin.config().joinOnConditions() == null),
LOOKUP_JOIN_ON_EXPRESSION(plan -> plan instanceof LookupJoin lookupJoin && lookupJoin.config().joinOnConditions() != null),
LOOKUP(Lookup.class::isInstance),
CHANGE_POINT(ChangePoint.class::isInstance),
INLINE_STATS(InlineStats.class::isInstance),
Expand Down
99 changes: 78 additions & 21 deletions ...lugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/VerifierMetricsTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.KEEP;
import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.LIMIT;
import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.LOOKUP_JOIN;
import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.LOOKUP_JOIN_ON_EXPRESSION;
import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.MV_EXPAND;
import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.RENAME;
import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.ROW;
Expand Down Expand Up @@ -65,7 +66,8 @@ public void testDissectQuery() {
assertEquals(0, drop(c));
assertEquals(0, keep(c));
assertEquals(0, rename(c));
assertEquals(0, lookupjoin(c));
assertEquals(0, lookupJoinOnFields(c));
assertEquals(0, lookupJoinOnExpression(c));
assertEquals(0, inlineStats(c));
assertEquals(1, function("concat", c));
}
Expand All @@ -87,7 +89,8 @@ public void testEvalQuery() {
assertEquals(0, drop(c));
assertEquals(0, keep(c));
assertEquals(0, rename(c));
assertEquals(0, lookupjoin(c));
assertEquals(0, lookupJoinOnFields(c));
assertEquals(0, lookupJoinOnExpression(c));
assertEquals(0, inlineStats(c));
assertEquals(1, function("length", c));
}
Expand All @@ -109,7 +112,8 @@ public void testGrokQuery() {
assertEquals(0, drop(c));
assertEquals(0, keep(c));
assertEquals(0, rename(c));
assertEquals(0, lookupjoin(c));
assertEquals(0, lookupJoinOnFields(c));
assertEquals(0, lookupJoinOnExpression(c));
assertEquals(0, inlineStats(c));
assertEquals(1, function("concat", c));
}
Expand All @@ -131,7 +135,8 @@ public void testLimitQuery() {
assertEquals(0, drop(c));
assertEquals(0, keep(c));
assertEquals(0, rename(c));
assertEquals(0, lookupjoin(c));
assertEquals(0, lookupJoinOnFields(c));
assertEquals(0, lookupJoinOnExpression(c));
assertEquals(0, inlineStats(c));
}

Expand All @@ -152,7 +157,8 @@ public void testSortQuery() {
assertEquals(0, drop(c));
assertEquals(0, keep(c));
assertEquals(0, rename(c));
assertEquals(0, lookupjoin(c));
assertEquals(0, lookupJoinOnFields(c));
assertEquals(0, lookupJoinOnExpression(c));
assertEquals(0, inlineStats(c));
}

Expand All @@ -173,7 +179,8 @@ public void testStatsQuery() {
assertEquals(0, drop(c));
assertEquals(0, keep(c));
assertEquals(0, rename(c));
assertEquals(0, lookupjoin(c));
assertEquals(0, lookupJoinOnFields(c));
assertEquals(0, lookupJoinOnExpression(c));
assertEquals(0, inlineStats(c));
assertEquals(1, function("max", c));
}
Expand All @@ -195,7 +202,8 @@ public void testWhereQuery() {
assertEquals(0, drop(c));
assertEquals(0, keep(c));
assertEquals(0, rename(c));
assertEquals(0, lookupjoin(c));
assertEquals(0, lookupJoinOnFields(c));
assertEquals(0, lookupJoinOnExpression(c));
assertEquals(0, inlineStats(c));
}

Expand All @@ -216,7 +224,8 @@ public void testTwoWhereQuery() {
assertEquals(0, drop(c));
assertEquals(0, keep(c));
assertEquals(0, rename(c));
assertEquals(0, lookupjoin(c));
assertEquals(0, lookupJoinOnFields(c));
assertEquals(0, lookupJoinOnExpression(c));
assertEquals(0, inlineStats(c));
}

Expand Down Expand Up @@ -257,7 +266,8 @@ public void testTwoQueriesExecuted() {
assertEquals(0, drop(c));
assertEquals(0, keep(c));
assertEquals(0, rename(c));
assertEquals(0, lookupjoin(c));
assertEquals(0, lookupJoinOnFields(c));
assertEquals(0, lookupJoinOnExpression(c));
assertEquals(0, inlineStats(c));

assertEquals(1, function("length", c));
Expand Down Expand Up @@ -343,7 +353,8 @@ public void testEnrich() {
assertEquals(1L, keep(c));
assertEquals(0, rename(c));
assertEquals(0, inlineStats(c));
assertEquals(0, lookupjoin(c));
assertEquals(0, lookupJoinOnFields(c));
assertEquals(0, lookupJoinOnExpression(c));
assertEquals(1, function("to_string", c));
}

Expand Down Expand Up @@ -374,7 +385,8 @@ public void testMvExpand() {
assertEquals(1L, keep(c));
assertEquals(0, rename(c));
assertEquals(0, inlineStats(c));
assertEquals(0, lookupjoin(c));
assertEquals(0, lookupJoinOnFields(c));
assertEquals(0, lookupJoinOnExpression(c));
}

public void testShowInfo() {
Expand All @@ -395,7 +407,8 @@ public void testShowInfo() {
assertEquals(0, keep(c));
assertEquals(0, rename(c));
assertEquals(0, inlineStats(c));
assertEquals(0, lookupjoin(c));
assertEquals(0, lookupJoinOnFields(c));
assertEquals(0, lookupJoinOnExpression(c));
assertEquals(1, function("count", c));
}

Expand All @@ -417,7 +430,8 @@ public void testRow() {
assertEquals(0, keep(c));
assertEquals(0, rename(c));
assertEquals(0, inlineStats(c));
assertEquals(0, lookupjoin(c));
assertEquals(0, lookupJoinOnFields(c));
assertEquals(0, lookupJoinOnExpression(c));
}

public void testDropAndRename() {
Expand All @@ -438,7 +452,8 @@ public void testDropAndRename() {
assertEquals(0, keep(c));
assertEquals(1L, rename(c));
assertEquals(0, inlineStats(c));
assertEquals(0, lookupjoin(c));
assertEquals(0, lookupJoinOnFields(c));
assertEquals(0, lookupJoinOnExpression(c));
assertEquals(1, function("count", c));
}

Expand All @@ -465,7 +480,8 @@ public void testKeep() {
assertEquals(1L, keep(c));
assertEquals(0, rename(c));
assertEquals(0, inlineStats(c));
assertEquals(0, lookupjoin(c));
assertEquals(0, lookupJoinOnFields(c));
assertEquals(0, lookupJoinOnExpression(c));
}

public void testCategorize() {
Expand All @@ -490,7 +506,8 @@ public void testCategorize() {
assertEquals(1L, keep(c));
assertEquals(0, rename(c));
assertEquals(0, inlineStats(c));
assertEquals(0, lookupjoin(c));
assertEquals(0, lookupJoinOnFields(c));
assertEquals(0, lookupJoinOnExpression(c));
assertEquals(1, function("count", c));
assertEquals(1, function("categorize", c));
}
Expand All @@ -517,7 +534,8 @@ public void testInlineStatsStandalone() {
assertEquals(0, keep(c));
assertEquals(0, rename(c));
assertEquals(1L, inlineStats(c));
assertEquals(0, lookupjoin(c));
assertEquals(0, lookupJoinOnFields(c));
assertEquals(0, lookupJoinOnExpression(c));
assertEquals(1, function("max", c));
}

Expand All @@ -544,7 +562,8 @@ public void testInlineStatsWithOtherStats() {
assertEquals(0, keep(c));
assertEquals(0, rename(c));
assertEquals(1L, inlineStats(c));
assertEquals(0, lookupjoin(c));
assertEquals(0, lookupJoinOnFields(c));
assertEquals(0, lookupJoinOnExpression(c));
assertEquals(1, function("max", c));
}

Expand All @@ -570,7 +589,40 @@ public void testBinaryPlanAfterStats() {
assertEquals(0, keep(c));
assertEquals(0, rename(c));
assertEquals(0, inlineStats(c));
assertEquals(1L, lookupjoin(c));
assertEquals(1L, lookupJoinOnFields(c));
assertEquals(0, lookupJoinOnExpression(c));
assertEquals(1, function("max", c));
}

public void testBinaryPlanAfterStatsExpressionJoin() {
assumeTrue(
"requires LOOKUP JOIN ON boolean expression capability",
EsqlCapabilities.Cap.LOOKUP_JOIN_ON_BOOLEAN_EXPRESSION.isEnabled()
);
Counters c = esql("""
from employees
| eval language_code = languages
| stats m = max(salary) by language_code
| rename language_code as language_code_left
| lookup join languages_lookup on language_code_left >= language_code""");
assertEquals(0, dissect(c));
assertEquals(1L, eval(c));
assertEquals(0, grok(c));
assertEquals(0, limit(c));
assertEquals(0, sort(c));
assertEquals(1L, stats(c));
assertEquals(0, where(c));
assertEquals(0, enrich(c));
assertEquals(0, mvExpand(c));
assertEquals(0, show(c));
assertEquals(0, row(c));
assertEquals(1L, from(c));
assertEquals(0, drop(c));
assertEquals(0, keep(c));
assertEquals(1L, rename(c));
assertEquals(0, inlineStats(c));
assertEquals(0, lookupJoinOnFields(c));
assertEquals(1L, lookupJoinOnExpression(c));
assertEquals(1, function("max", c));
}

Expand All @@ -597,7 +649,8 @@ public void testBinaryPlanAfterInlineStats() {
assertEquals(0, keep(c));
assertEquals(0, rename(c));
assertEquals(1L, inlineStats(c));
assertEquals(1L, lookupjoin(c));
assertEquals(1L, lookupJoinOnFields(c));
assertEquals(0, lookupJoinOnExpression(c));
assertEquals(1, function("max", c));
}

Expand Down Expand Up @@ -665,10 +718,14 @@ private long inlineStats(Counters c) {
return c.get(FEATURES_PREFIX + INLINE_STATS);
}

private long lookupjoin(Counters c) {
private long lookupJoinOnFields(Counters c) {
return c.get(FEATURES_PREFIX + LOOKUP_JOIN);
}

private long lookupJoinOnExpression(Counters c) {
return c.get(FEATURES_PREFIX + LOOKUP_JOIN_ON_EXPRESSION);
}

private long function(String function, Counters c) {
return c.get(FUNC_PREFIX + function);
}
Expand Down
8 changes: 6 additions & 2 deletions x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ setup:
- do: { xpack.usage: { } }
- match: { esql.available: true }
- match: { esql.enabled: true }
- length: { esql.features: 26 }
- length: { esql.features: 27 }
- set: { esql.features.dissect: dissect_counter }
- set: { esql.features.drop: drop_counter }
- set: { esql.features.eval: eval_counter }
Expand All @@ -67,6 +67,7 @@ setup:
- set: { esql.features.stats: stats_counter }
- set: { esql.features.where: where_counter }
- set: { esql.features.lookup_join: lookup_join_counter }
- set: { esql.features.lookup_join_on_expression: lookup_join_on_expression_counter }
- set: { esql.features.lookup: lookup_counter }
- set: { esql.features.change_point: change_point_counter }
- set: { esql.features.inline_stats: inline_stats_counter }
Expand Down Expand Up @@ -110,6 +111,7 @@ setup:
- gt: { esql.features.stats: $stats_counter }
- gt: { esql.features.where: $where_counter }
- match: { esql.features.lookup_join: $lookup_join_counter }
- match: { esql.features.lookup_join_on_expression: $lookup_join_on_expression_counter }
- match: { esql.features.lookup: $lookup_counter }
- match: { esql.features.change_point: $change_point_counter }
- match: { esql.features.inline_stats: $inline_stats_counter }
Expand Down Expand Up @@ -147,7 +149,7 @@ setup:
- do: { xpack.usage: { } }
- match: { esql.available: true }
- match: { esql.enabled: true }
- length: { esql.features: 26 }
- length: { esql.features: 27 }
- set: { esql.features.dissect: dissect_counter }
- set: { esql.features.drop: drop_counter }
- set: { esql.features.eval: eval_counter }
Expand All @@ -164,6 +166,7 @@ setup:
- set: { esql.features.stats: stats_counter }
- set: { esql.features.where: where_counter }
- set: { esql.features.lookup_join: lookup_join_counter }
- set: { esql.features.lookup_join_on_expression: lookup_join_on_expression_counter }
- set: { esql.features.lookup: lookup_counter }
- set: { esql.features.change_point: change_point_counter }
- set: { esql.features.inline_stats: inline_stats_counter }
Expand Down Expand Up @@ -207,6 +210,7 @@ setup:
- gt: { esql.features.stats: $stats_counter }
- gt: { esql.features.where: $where_counter }
- match: { esql.features.lookup_join: $lookup_join_counter }
- match: { esql.features.lookup_join_on_expression: $lookup_join_on_expression_counter }
- match: { esql.features.lookup: $lookup_counter }
- match: { esql.features.change_point: $change_point_counter }
- match: { esql.features.inline_stats: $inline_stats_counter }
Expand Down