Skip to content

Commit 0b876b8

Browse files
tvernumtvernum
authored
Change reindex to use ::es-redacted:: filtering (#135414)
In audit logs we redact certain fields from the body of rest requests. This commit changes the way we redact fields in the reindex request. Previously the only form of redaction we supported was total removal of fields, however that can be problematic when an admin wants to know whether a field was supplied or not. Here we change the way we redact requests for reindexing to replace fields with `::es-redacted::` instead of removing them.
1 parent c803bdb commit 0b876b8

File tree

5 files changed

+267
-47
lines changed

5 files changed

+267
-47
lines changed

‎docs/changelog/135414.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 135414
2+
summary: "Change reindex to use ::es-redacted:: filtering"
3+
area: Audit
4+
type: enhancement
5+
issues: []

‎modules/reindex/src/main/java/org/elasticsearch/reindex/RestReindexAction.java‎

Lines changed: 39 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import org.elasticsearch.features.NodeFeature;
1515
import org.elasticsearch.index.reindex.ReindexAction;
1616
import org.elasticsearch.index.reindex.ReindexRequest;
17+
import org.elasticsearch.rest.FilteredRestRequest;
1718
import org.elasticsearch.rest.RestRequest;
1819
import org.elasticsearch.rest.RestRequestFilter;
1920
import org.elasticsearch.rest.Scope;
@@ -22,8 +23,10 @@
2223

2324
import java.io.IOException;
2425
import java.util.List;
26+
import java.util.Map;
2527
import java.util.Set;
2628
import java.util.function.Predicate;
29+
import java.util.stream.Collectors;
2730

2831
import static org.elasticsearch.core.TimeValue.parseTimeValue;
2932
import static org.elasticsearch.rest.RestRequest.Method.POST;
@@ -79,10 +82,43 @@ protected ReindexRequest buildRequest(RestRequest request) throws IOException {
7982
return internal;
8083
}
8184

82-
private static final Set<String> FILTERED_FIELDS = Set.of("source.remote.host.password");
83-
85+
/**
86+
* This method isn't used because we implement {@link #getFilteredRequest(RestRequest)} instead
87+
*/
8488
@Override
8589
public Set<String> getFilteredFields() {
86-
return FILTERED_FIELDS;
90+
assert false : "This method should never be called";
91+
throw new UnsupportedOperationException();
92+
}
93+
94+
@Override
95+
public RestRequest getFilteredRequest(RestRequest restRequest) {
96+
if (restRequest.hasContent()) {
97+
return new FilteredRestRequest(restRequest, Set.of()) {
98+
@Override
99+
@SuppressWarnings({ "rawtypes", "unchecked" })
100+
protected Map<String, Object> transformBody(Map<String, Object> map) {
101+
final var source = map.get("source");
102+
if (source instanceof Map sourceMap) {
103+
final var remote = sourceMap.get("remote");
104+
if (remote instanceof Map remoteMap) {
105+
remoteMap.computeIfPresent("password", (key, value) -> "::es-redacted::");
106+
remoteMap.computeIfPresent("headers", (key, value) -> {
107+
if (value instanceof Map<?, ?> headers) {
108+
return headers.entrySet()
109+
.stream()
110+
.collect(Collectors.toMap(Map.Entry::getKey, ignore -> "::es-redacted::"));
111+
} else {
112+
return null;
113+
}
114+
});
115+
}
116+
}
117+
return map;
118+
}
119+
};
120+
} else {
121+
return restRequest;
122+
}
87123
}
88124
}

‎modules/reindex/src/test/java/org/elasticsearch/reindex/RestReindexActionTests.java‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,10 @@
1111

1212
import org.elasticsearch.common.bytes.BytesArray;
1313
import org.elasticsearch.common.bytes.BytesReference;
14+
import org.elasticsearch.common.xcontent.XContentHelper;
1415
import org.elasticsearch.index.reindex.AbstractBulkByScrollRequest;
1516
import org.elasticsearch.index.reindex.ReindexRequest;
17+
import org.elasticsearch.rest.RestRequest;
1618
import org.elasticsearch.test.rest.FakeRestRequest;
1719
import org.elasticsearch.test.rest.RestActionTestCase;
1820
import org.elasticsearch.xcontent.XContentBuilder;
@@ -21,8 +23,16 @@
2123
import org.junit.Before;
2224

2325
import java.io.IOException;
26+
import java.util.List;
27+
import java.util.Map;
2428

2529
import static java.util.Collections.singletonMap;
30+
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
31+
import static org.hamcrest.Matchers.aMapWithSize;
32+
import static org.hamcrest.Matchers.equalTo;
33+
import static org.hamcrest.Matchers.hasEntry;
34+
import static org.hamcrest.Matchers.hasKey;
35+
import static org.hamcrest.Matchers.notNullValue;
2636

2737
public class RestReindexActionTests extends RestActionTestCase {
2838

@@ -74,4 +84,150 @@ public void testSetScrollTimeout() throws IOException {
7484
assertEquals("10m", request.getScrollTime().toString());
7585
}
7686
}
87+
88+
public void testFilterSource() throws IOException {
89+
final FakeRestRequest.Builder requestBuilder = new FakeRestRequest.Builder(xContentRegistry());
90+
final var body = """
91+
{
92+
"source" : {
93+
"index": "photos",
94+
"remote" : {
95+
"host": "https://bugle.example.net:2400/",
96+
"username": "peter.parker",
97+
"password": "mj4ever!",
98+
"headers": {
99+
"X-Hero-Name": "spiderman"
100+
}
101+
}
102+
},
103+
"dest": {
104+
"index": "webshots"
105+
}
106+
}
107+
""";
108+
requestBuilder.withContent(new BytesArray(body), XContentType.JSON);
109+
110+
final FakeRestRequest restRequest = requestBuilder.build();
111+
ReindexRequest request = action.buildRequest(restRequest);
112+
113+
// Check that the request parsed correctly
114+
assertThat(request.getRemoteInfo().getScheme(), equalTo("https"));
115+
assertThat(request.getRemoteInfo().getHost(), equalTo("bugle.example.net"));
116+
assertThat(request.getRemoteInfo().getPort(), equalTo(2400));
117+
assertThat(request.getRemoteInfo().getUsername(), equalTo("peter.parker"));
118+
assertThat(request.getRemoteInfo().getPassword(), equalTo("mj4ever!"));
119+
assertThat(request.getRemoteInfo().getHeaders(), hasEntry("X-Hero-Name", "spiderman"));
120+
assertThat(request.getRemoteInfo().getHeaders(), aMapWithSize(1));
121+
122+
final RestRequest filtered = action.getFilteredRequest(restRequest);
123+
assertToXContentEquivalent(new BytesArray("""
124+
{
125+
"source" : {
126+
"index": "photos",
127+
"remote" : {
128+
"host": "https://bugle.example.net:2400/",
129+
"username": "peter.parker",
130+
"password": "::es-redacted::",
131+
"headers": {
132+
"X-Hero-Name": "::es-redacted::"
133+
}
134+
}
135+
},
136+
"dest": {
137+
"index": "webshots"
138+
}
139+
}
140+
"""), filtered.content(), XContentType.JSON);
141+
}
142+
143+
public void testUnfilteredSource() throws IOException {
144+
final FakeRestRequest.Builder requestBuilder = new FakeRestRequest.Builder(xContentRegistry());
145+
final var empty1 = "";
146+
final var empty2 = "{}";
147+
final var nonRemote = """
148+
{
149+
"source" : { "index": "your-index" },
150+
"dest" : { "index": "my-index" }
151+
}
152+
""";
153+
final var noCredentials = """
154+
{
155+
"source" : {
156+
"index": "remote-index",
157+
"remote" : {
158+
"host": "https://es.example.net:12345/",
159+
"headers": {}
160+
}
161+
},
162+
"dest": {
163+
"index": "my-index"
164+
}
165+
}
166+
""";
167+
for (String body : List.of(empty1, empty2, nonRemote, noCredentials)) {
168+
final BytesArray bodyAsBytes = new BytesArray(body);
169+
requestBuilder.withContent(bodyAsBytes, XContentType.JSON);
170+
final FakeRestRequest restRequest = requestBuilder.build();
171+
final RestRequest filtered = action.getFilteredRequest(restRequest);
172+
assertToXContentEquivalent(bodyAsBytes, filtered.content(), XContentType.JSON);
173+
}
174+
}
175+
176+
public void testFilteringBadlyStructureSourceIsSafe() throws IOException {
177+
final FakeRestRequest.Builder requestBuilder = new FakeRestRequest.Builder(xContentRegistry());
178+
final var remoteAsString = """
179+
{
180+
"source" : {
181+
"index": "remote-index",
182+
"remote" : "https://es.example.net:12345/"
183+
},
184+
"dest": {
185+
"index": "my-index"
186+
}
187+
}
188+
""";
189+
final var passwordAsNumber = """
190+
{
191+
"source" : {
192+
"index": "remote-index",
193+
"remote" : {
194+
"host": "https://es.example.net:12345/",
195+
"username": "skroob",
196+
"password": 12345
197+
}
198+
},
199+
"dest": {
200+
"index": "my-index"
201+
}
202+
}
203+
""";
204+
final var headersAsList = """
205+
{
206+
"source" : {
207+
"index": "remote-index",
208+
"remote" : {
209+
"host": "https://es.example.net:12345/",
210+
"headers": [ "bogus" ]
211+
}
212+
},
213+
"dest": {
214+
"index": "my-index"
215+
}
216+
}
217+
""";
218+
for (String body : List.of(remoteAsString, passwordAsNumber, headersAsList)) {
219+
final BytesArray bodyAsBytes = new BytesArray(body);
220+
requestBuilder.withContent(bodyAsBytes, XContentType.JSON);
221+
final FakeRestRequest restRequest = requestBuilder.build();
222+
223+
final RestRequest filtered = action.getFilteredRequest(restRequest);
224+
assertThat(filtered, notNullValue());
225+
226+
// We will redacted some parts of these bodies, so just check that they end up as valid JSON with the right top level fields
227+
final Map<String, Object> filteredMap = XContentHelper.convertToMap(filtered.content(), false, XContentType.JSON).v2();
228+
assertThat(filteredMap, notNullValue());
229+
assertThat(filteredMap, hasKey("source"));
230+
assertThat(filteredMap, hasKey("dest"));
231+
}
232+
}
77233
}

‎server/src/main/java/org/elasticsearch/rest/FilteredRestRequest.java‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.rest;
11+
12+
import org.elasticsearch.ElasticsearchException;
13+
import org.elasticsearch.common.bytes.BytesReference;
14+
import org.elasticsearch.common.bytes.ReleasableBytesReference;
15+
import org.elasticsearch.common.xcontent.XContentHelper;
16+
import org.elasticsearch.common.xcontent.support.XContentMapValues;
17+
import org.elasticsearch.core.Tuple;
18+
import org.elasticsearch.xcontent.XContentBuilder;
19+
import org.elasticsearch.xcontent.XContentType;
20+
21+
import java.io.IOException;
22+
import java.util.Map;
23+
import java.util.Set;
24+
25+
public class FilteredRestRequest extends RestRequest {
26+
27+
private final RestRequest restRequest;
28+
private final String[] excludeFields;
29+
private BytesReference filteredBytes;
30+
31+
public FilteredRestRequest(RestRequest restRequest, Set<String> excludeFields) {
32+
super(restRequest);
33+
this.restRequest = restRequest;
34+
this.excludeFields = excludeFields.toArray(String[]::new);
35+
this.filteredBytes = null;
36+
}
37+
38+
@Override
39+
public boolean hasContent() {
40+
return true;
41+
}
42+
43+
@Override
44+
public ReleasableBytesReference content() {
45+
if (filteredBytes == null) {
46+
Tuple<XContentType, Map<String, Object>> result = XContentHelper.convertToMap(
47+
restRequest.requiredContent(),
48+
true,
49+
restRequest.getXContentType()
50+
);
51+
final Map<String, Object> transformedSource = transformBody(result.v2());
52+
try {
53+
XContentBuilder xContentBuilder = XContentBuilder.builder(result.v1().xContent()).map(transformedSource);
54+
filteredBytes = BytesReference.bytes(xContentBuilder);
55+
} catch (IOException e) {
56+
throw new ElasticsearchException("failed to parse request", e);
57+
}
58+
}
59+
return ReleasableBytesReference.wrap(filteredBytes);
60+
}
61+
62+
protected Map<String, Object> transformBody(Map<String, Object> map) {
63+
return XContentMapValues.filter(map, null, excludeFields);
64+
}
65+
}

‎server/src/main/java/org/elasticsearch/rest/RestRequestFilter.java‎

Lines changed: 2 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -9,18 +9,6 @@
99

1010
package org.elasticsearch.rest;
1111

12-
import org.elasticsearch.ElasticsearchException;
13-
import org.elasticsearch.common.Strings;
14-
import org.elasticsearch.common.bytes.BytesReference;
15-
import org.elasticsearch.common.bytes.ReleasableBytesReference;
16-
import org.elasticsearch.common.xcontent.XContentHelper;
17-
import org.elasticsearch.common.xcontent.support.XContentMapValues;
18-
import org.elasticsearch.core.Tuple;
19-
import org.elasticsearch.xcontent.XContentBuilder;
20-
import org.elasticsearch.xcontent.XContentType;
21-
22-
import java.io.IOException;
23-
import java.util.Map;
2412
import java.util.Set;
2513

2614
/**
@@ -35,38 +23,7 @@ public interface RestRequestFilter {
3523
default RestRequest getFilteredRequest(RestRequest restRequest) {
3624
Set<String> fields = getFilteredFields();
3725
if (restRequest.hasContent() && fields.isEmpty() == false) {
38-
return new RestRequest(restRequest) {
39-
40-
private BytesReference filteredBytes = null;
41-
42-
@Override
43-
public boolean hasContent() {
44-
return true;
45-
}
46-
47-
@Override
48-
public ReleasableBytesReference content() {
49-
if (filteredBytes == null) {
50-
Tuple<XContentType, Map<String, Object>> result = XContentHelper.convertToMap(
51-
restRequest.requiredContent(),
52-
true,
53-
restRequest.getXContentType()
54-
);
55-
Map<String, Object> transformedSource = XContentMapValues.filter(
56-
result.v2(),
57-
null,
58-
fields.toArray(Strings.EMPTY_ARRAY)
59-
);
60-
try {
61-
XContentBuilder xContentBuilder = XContentBuilder.builder(result.v1().xContent()).map(transformedSource);
62-
filteredBytes = BytesReference.bytes(xContentBuilder);
63-
} catch (IOException e) {
64-
throw new ElasticsearchException("failed to parse request", e);
65-
}
66-
}
67-
return ReleasableBytesReference.wrap(filteredBytes);
68-
}
69-
};
26+
return new FilteredRestRequest(restRequest, fields);
7027
} else {
7128
return restRequest;
7229
}
@@ -76,4 +33,5 @@ public ReleasableBytesReference content() {
7633
* The list of fields that should be filtered. This can be a dot separated pattern to match sub objects and also supports wildcards
7734
*/
7835
Set<String> getFilteredFields();
36+
7937
}

0 commit comments

Comments
 (0)