From 3f6253970e6c35857b3eb70e4521fa10489d0c37 Mon Sep 17 00:00:00 2001 From: David Turner Date: Tue, 20 May 2025 08:14:39 +0100 Subject: [PATCH] Make S3 custom query parameter optional Today Elasticsearch will record the purpose for each request to S3 using a custom query parameter[^1]. This isn't believed to be necessary outside of the ECH/ECE/ECK/... managed services, and it adds rather a lot to the request logs, so with this commit we make the feature optional and disabled by default. Backport of #128043 to `8.18` [^1]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/LogFormat.html#LogFormatCustom --- docs/changelog/128043.yaml | 19 ++ .../s3/S3BlobStoreRepositoryTests.java | 11 +- .../s3/AbstractRepositoryS3RestTestCase.java | 12 +- .../repositories/s3/S3BlobContainer.java | 7 +- .../repositories/s3/S3BlobStore.java | 14 +- .../repositories/s3/S3ClientSettings.java | 22 +++ .../repositories/s3/S3RepositoryPlugin.java | 1 + .../AddPurposeCustomQueryParameterTests.java | 167 ++++++++++++++++++ .../analyze/S3RepositoryAnalysisRestIT.java | 44 ++++- 9 files changed, 279 insertions(+), 18 deletions(-) create mode 100644 docs/changelog/128043.yaml create mode 100644 modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AddPurposeCustomQueryParameterTests.java diff --git a/docs/changelog/128043.yaml b/docs/changelog/128043.yaml new file mode 100644 index 0000000000000..90c2a538d03ca --- /dev/null +++ b/docs/changelog/128043.yaml @@ -0,0 +1,19 @@ +pr: 128043 +summary: Make S3 custom query parameter optional +area: Snapshot/Restore +type: breaking +issues: [] +breaking: + title: Make S3 custom query parameter optional + area: Cluster and node setting + details: >- + Earlier versions of Elasticsearch would record the purpose of each S3 API + call using the `?x-purpose=` custom query parameter. This isn't believed to + be necessary outside of the ECH/ECE/ECK/... managed services, and it adds + rather a lot to the request logs, so with this change we make the feature + optional and disabled by default. + impact: >- + If you wish to reinstate the old behaviour on a S3 repository, set + `s3.client.${CLIENT_NAME}.add_purpose_custom_query_parameter` to `true` + for the relevant client. + notable: false diff --git a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java index 12220dc51ea8f..41f83702b8a0e 100644 --- a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java +++ b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java @@ -172,6 +172,7 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { .put(S3ClientSettings.ENDPOINT_SETTING.getConcreteSettingForNamespace("test").getKey(), httpServerUrl()) // Disable request throttling because some random values in tests might generate too many failures for the S3 client .put(S3ClientSettings.USE_THROTTLE_RETRIES_SETTING.getConcreteSettingForNamespace("test").getKey(), false) + .put(S3ClientSettings.ADD_PURPOSE_CUSTOM_QUERY_PARAMETER.getConcreteSettingForNamespace("test").getKey(), "true") .put(super.nodeSettings(nodeOrdinal, otherSettings)) .setSecureSettings(secureSettings); @@ -495,19 +496,13 @@ public void testMultipartUploadCleanup() { blobStore.bucket(), blobStore.blobContainer(repository.basePath().add("test-multipart-upload")).path().buildAsString() + danglingBlobName ); - initiateMultipartUploadRequest.putCustomQueryParameter( - S3BlobStore.CUSTOM_QUERY_PARAMETER_PURPOSE, - OperationPurpose.SNAPSHOT_DATA.getKey() - ); + blobStore.addPurposeQueryParameter(OperationPurpose.SNAPSHOT_DATA, initiateMultipartUploadRequest); final var multipartUploadResult = clientRef.client().initiateMultipartUpload(initiateMultipartUploadRequest); final var listMultipartUploadsRequest = new ListMultipartUploadsRequest(blobStore.bucket()).withPrefix( repository.basePath().buildAsString() ); - listMultipartUploadsRequest.putCustomQueryParameter( - S3BlobStore.CUSTOM_QUERY_PARAMETER_PURPOSE, - OperationPurpose.SNAPSHOT_DATA.getKey() - ); + blobStore.addPurposeQueryParameter(OperationPurpose.SNAPSHOT_DATA, listMultipartUploadsRequest); assertEquals( List.of(multipartUploadResult.getUploadId()), clientRef.client() diff --git a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/AbstractRepositoryS3RestTestCase.java b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/AbstractRepositoryS3RestTestCase.java index 67ada622efeea..fb7e0215003fb 100644 --- a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/AbstractRepositoryS3RestTestCase.java +++ b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/AbstractRepositoryS3RestTestCase.java @@ -59,6 +59,12 @@ private Request getRegisterRequest(UnaryOperator settingsUnaryOperator .put("canned_acl", "private") .put("storage_class", "standard") .put("disable_chunked_encoding", randomBoolean()) + .put( + randomFrom( + Settings.EMPTY, + Settings.builder().put("add_purpose_custom_query_parameter", randomBoolean()).build() + ) + ) .build() ) ) @@ -183,8 +189,10 @@ private void testNonexistentClient(Boolean readonly) throws Exception { final var responseObjectPath = ObjectPath.createFromResponse(responseException.getResponse()); assertThat(responseObjectPath.evaluate("error.type"), equalTo("repository_verification_exception")); assertThat(responseObjectPath.evaluate("error.reason"), containsString("is not accessible on master node")); - assertThat(responseObjectPath.evaluate("error.caused_by.type"), equalTo("illegal_argument_exception")); - assertThat(responseObjectPath.evaluate("error.caused_by.reason"), containsString("Unknown s3 client name")); + assertThat(responseObjectPath.evaluate("error.caused_by.type"), equalTo("repository_exception")); + assertThat(responseObjectPath.evaluate("error.caused_by.reason"), containsString("cannot create blob store")); + assertThat(responseObjectPath.evaluate("error.caused_by.caused_by.type"), equalTo("illegal_argument_exception")); + assertThat(responseObjectPath.evaluate("error.caused_by.caused_by.reason"), containsString("Unknown s3 client name")); } public void testNonexistentSnapshot() throws Exception { diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java index 13eb2d4dec13c..18eb4457d24d9 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java @@ -928,7 +928,7 @@ ActionListener getMultipartUploadCleanupListener(int maxUploads, RefCounti try (var clientReference = blobStore.clientReference()) { final var bucket = blobStore.bucket(); final var request = new ListMultipartUploadsRequest(bucket).withPrefix(keyPath).withMaxUploads(maxUploads); - request.putCustomQueryParameter(S3BlobStore.CUSTOM_QUERY_PARAMETER_PURPOSE, OperationPurpose.SNAPSHOT_DATA.getKey()); + blobStore.addPurposeQueryParameter(OperationPurpose.SNAPSHOT_DATA, request); final var multipartUploadListing = SocketAccess.doPrivileged(() -> clientReference.client().listMultipartUploads(request)); final var multipartUploads = multipartUploadListing.getMultipartUploads(); if (multipartUploads.isEmpty()) { @@ -968,10 +968,7 @@ private ActionListener newMultipartUploadCleanupListener( public void onResponse(Void unused) { try (var clientReference = blobStore.clientReference()) { for (final var abortMultipartUploadRequest : abortMultipartUploadRequests) { - abortMultipartUploadRequest.putCustomQueryParameter( - S3BlobStore.CUSTOM_QUERY_PARAMETER_PURPOSE, - OperationPurpose.SNAPSHOT_DATA.getKey() - ); + blobStore.addPurposeQueryParameter(OperationPurpose.SNAPSHOT_DATA, abortMultipartUploadRequest); try { SocketAccess.doPrivilegedVoid(() -> clientReference.client().abortMultipartUpload(abortMultipartUploadRequest)); logger.info( diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java index 5237031fcdd6c..9fa6a9f6792fe 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java @@ -91,6 +91,8 @@ class S3BlobStore implements BlobStore { private final int bulkDeletionBatchSize; + private final boolean addPurposeCustomQueryParameter; + S3BlobStore( S3Service service, String bucket, @@ -115,7 +117,7 @@ class S3BlobStore implements BlobStore { this.snapshotExecutor = threadPool.executor(ThreadPool.Names.SNAPSHOT); this.s3RepositoriesMetrics = s3RepositoriesMetrics; this.bulkDeletionBatchSize = S3Repository.DELETION_BATCH_SIZE_SETTING.get(repositoryMetadata.settings()); - + this.addPurposeCustomQueryParameter = service.settings(repositoryMetadata).addPurposeCustomQueryParameter; } RequestMetricCollector getMetricCollector(Operation operation, OperationPurpose purpose) { @@ -523,6 +525,14 @@ static void configureRequestForMetrics( OperationPurpose purpose ) { request.setRequestMetricCollector(blobStore.getMetricCollector(operation, purpose)); - request.putCustomQueryParameter(CUSTOM_QUERY_PARAMETER_PURPOSE, purpose.getKey()); + blobStore.addPurposeQueryParameter(purpose, request); + } + + public void addPurposeQueryParameter(OperationPurpose purpose, AmazonWebServiceRequest request) { + if (addPurposeCustomQueryParameter || purpose == OperationPurpose.REPOSITORY_ANALYSIS) { + // REPOSITORY_ANALYSIS is a strict check for 100% S3 compatibility, including custom query parameter support, so is always added + request.putCustomQueryParameter(CUSTOM_QUERY_PARAMETER_PURPOSE, purpose.getKey()); + } } + } diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java index 6eb253bf5074c..c6421e275b0c0 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java @@ -174,6 +174,13 @@ final class S3ClientSettings { key -> new Setting<>(key, "", Function.identity(), Property.NodeScope) ); + /** Whether to include the {@code x-purpose} custom query parameter in all requests. */ + static final Setting.AffixSetting ADD_PURPOSE_CUSTOM_QUERY_PARAMETER = Setting.affixKeySetting( + PREFIX, + "add_purpose_custom_query_parameter", + key -> Setting.boolSetting(key, false, Property.NodeScope) + ); + /** Credentials to authenticate with s3. */ final S3BasicCredentials credentials; @@ -218,6 +225,9 @@ final class S3ClientSettings { /** Whether chunked encoding should be disabled or not. */ final boolean disableChunkedEncoding; + /** Whether to add the {@code x-purpose} custom query parameter to all requests. */ + final boolean addPurposeCustomQueryParameter; + /** Region to use for signing requests or empty string to use default. */ final String region; @@ -239,6 +249,7 @@ private S3ClientSettings( boolean throttleRetries, boolean pathStyleAccess, boolean disableChunkedEncoding, + boolean addPurposeCustomQueryParameter, String region, String signerOverride ) { @@ -256,6 +267,7 @@ private S3ClientSettings( this.throttleRetries = throttleRetries; this.pathStyleAccess = pathStyleAccess; this.disableChunkedEncoding = disableChunkedEncoding; + this.addPurposeCustomQueryParameter = addPurposeCustomQueryParameter; this.region = region; this.signerOverride = signerOverride; } @@ -290,6 +302,11 @@ S3ClientSettings refine(Settings repositorySettings) { normalizedSettings, disableChunkedEncoding ); + final boolean newAddPurposeCustomQueryParameter = getRepoSettingOrDefault( + ADD_PURPOSE_CUSTOM_QUERY_PARAMETER, + normalizedSettings, + addPurposeCustomQueryParameter + ); final S3BasicCredentials newCredentials; if (checkDeprecatedCredentials(repositorySettings)) { newCredentials = loadDeprecatedCredentials(repositorySettings); @@ -310,6 +327,7 @@ S3ClientSettings refine(Settings repositorySettings) { && Objects.equals(credentials, newCredentials) && newPathStyleAccess == pathStyleAccess && newDisableChunkedEncoding == disableChunkedEncoding + && newAddPurposeCustomQueryParameter == addPurposeCustomQueryParameter && Objects.equals(region, newRegion) && Objects.equals(signerOverride, newSignerOverride)) { return this; @@ -329,6 +347,7 @@ S3ClientSettings refine(Settings repositorySettings) { newThrottleRetries, newPathStyleAccess, newDisableChunkedEncoding, + newAddPurposeCustomQueryParameter, newRegion, newSignerOverride ); @@ -438,6 +457,7 @@ static S3ClientSettings getClientSettings(final Settings settings, final String getConfigValue(settings, clientName, USE_THROTTLE_RETRIES_SETTING), getConfigValue(settings, clientName, USE_PATH_STYLE_ACCESS), getConfigValue(settings, clientName, DISABLE_CHUNKED_ENCODING), + getConfigValue(settings, clientName, ADD_PURPOSE_CUSTOM_QUERY_PARAMETER), getConfigValue(settings, clientName, REGION), getConfigValue(settings, clientName, SIGNER_OVERRIDE) ); @@ -466,6 +486,7 @@ public boolean equals(final Object o) { && Objects.equals(proxyUsername, that.proxyUsername) && Objects.equals(proxyPassword, that.proxyPassword) && Objects.equals(disableChunkedEncoding, that.disableChunkedEncoding) + && Objects.equals(addPurposeCustomQueryParameter, that.addPurposeCustomQueryParameter) && Objects.equals(region, that.region) && Objects.equals(signerOverride, that.signerOverride); } @@ -486,6 +507,7 @@ public int hashCode() { maxConnections, throttleRetries, disableChunkedEncoding, + addPurposeCustomQueryParameter, region, signerOverride ); diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java index 37b960b33eb79..274f645bb6839 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java @@ -131,6 +131,7 @@ public List> getSettings() { S3ClientSettings.USE_THROTTLE_RETRIES_SETTING, S3ClientSettings.USE_PATH_STYLE_ACCESS, S3ClientSettings.SIGNER_OVERRIDE, + S3ClientSettings.ADD_PURPOSE_CUSTOM_QUERY_PARAMETER, S3ClientSettings.REGION, S3Service.REPOSITORY_S3_CAS_TTL_SETTING, S3Service.REPOSITORY_S3_CAS_ANTI_CONTENTION_DELAY_SETTING, diff --git a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AddPurposeCustomQueryParameterTests.java b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AddPurposeCustomQueryParameterTests.java new file mode 100644 index 0000000000000..c4cd95f5e7855 --- /dev/null +++ b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AddPurposeCustomQueryParameterTests.java @@ -0,0 +1,167 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.repositories.s3; + +import fixture.s3.S3HttpFixture; +import fixture.s3.S3HttpHandler; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; + +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest; +import org.elasticsearch.action.admin.cluster.repositories.put.TransportPutRepositoryAction; +import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest; +import org.elasticsearch.action.admin.cluster.snapshots.create.TransportCreateSnapshotAction; +import org.elasticsearch.common.settings.MockSecureSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.snapshots.SnapshotState; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.hamcrest.Matcher; +import org.junit.runner.Description; +import org.junit.runners.model.Statement; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.not; + +public class AddPurposeCustomQueryParameterTests extends ESSingleNodeTestCase { + + @Override + protected Collection> getPlugins() { + return CollectionUtils.appendToCopyNoNullElements(super.getPlugins(), S3RepositoryPlugin.class); + } + + @Override + protected Settings nodeSettings() { + final var secureSettings = new MockSecureSettings(); + for (final var clientName : List.of("default", "with_purpose", "without_purpose")) { + secureSettings.setString( + S3ClientSettings.ACCESS_KEY_SETTING.getConcreteSettingForNamespace(clientName).getKey(), + randomIdentifier() + ); + secureSettings.setString( + S3ClientSettings.SECRET_KEY_SETTING.getConcreteSettingForNamespace(clientName).getKey(), + randomSecretKey() + ); + } + + return Settings.builder() + .put(super.nodeSettings()) + .put(S3ClientSettings.REGION.getConcreteSettingForNamespace("default").getKey(), randomIdentifier()) + .put(S3ClientSettings.ADD_PURPOSE_CUSTOM_QUERY_PARAMETER.getConcreteSettingForNamespace("with_purpose").getKey(), "true") + .put(S3ClientSettings.ADD_PURPOSE_CUSTOM_QUERY_PARAMETER.getConcreteSettingForNamespace("without_purpose").getKey(), "false") + .setSecureSettings(secureSettings) + .build(); + } + + private static final Matcher> HAS_CUSTOM_QUERY_PARAMETER = hasItem(S3BlobStore.CUSTOM_QUERY_PARAMETER_PURPOSE); + private static final Matcher> NO_CUSTOM_QUERY_PARAMETER = not(HAS_CUSTOM_QUERY_PARAMETER); + + public void testCustomQueryParameterConfiguration() throws Throwable { + final var indexName = randomIdentifier(); + createIndex(indexName); + prepareIndex(indexName).setSource("foo", "bar").get(); + + final var bucket = randomIdentifier(); + final var basePath = randomIdentifier(); + + runCustomQueryParameterTest(bucket, basePath, null, Settings.EMPTY, NO_CUSTOM_QUERY_PARAMETER); + runCustomQueryParameterTest(bucket, basePath, "default", Settings.EMPTY, NO_CUSTOM_QUERY_PARAMETER); + runCustomQueryParameterTest(bucket, basePath, "without_purpose", Settings.EMPTY, NO_CUSTOM_QUERY_PARAMETER); + runCustomQueryParameterTest(bucket, basePath, "with_purpose", Settings.EMPTY, HAS_CUSTOM_QUERY_PARAMETER); + + final var falseRepositorySetting = Settings.builder().put("add_purpose_custom_query_parameter", false).build(); + final var trueRepositorySetting = Settings.builder().put("add_purpose_custom_query_parameter", true).build(); + for (final var clientName : new String[] { null, "default", "with_purpose", "without_purpose" }) { + // client name doesn't matter if repository setting specified + runCustomQueryParameterTest(bucket, basePath, clientName, falseRepositorySetting, NO_CUSTOM_QUERY_PARAMETER); + runCustomQueryParameterTest(bucket, basePath, clientName, trueRepositorySetting, HAS_CUSTOM_QUERY_PARAMETER); + } + } + + private void runCustomQueryParameterTest( + String bucket, + String basePath, + String clientName, + Settings extraRepositorySettings, + Matcher> queryParamMatcher + ) throws Throwable { + final var httpFixture = new S3HttpFixture(true, bucket, basePath, (key, token) -> true) { + + @SuppressForbidden(reason = "this test uses a HttpServer to emulate an S3 endpoint") + class AssertingHandler extends S3HttpHandler { + AssertingHandler() { + super(bucket, basePath); + } + + @SuppressForbidden(reason = "this test uses a HttpServer to emulate an S3 endpoint") + @Override + public void handle(HttpExchange exchange) throws IOException { + try { + assertThat( + parseRequestComponents(exchange.getRequestMethod() + " " + exchange.getRequestURI().toString()) + .customQueryParameters() + .keySet(), + queryParamMatcher + ); + super.handle(exchange); + } catch (Error e) { + // HttpServer catches Throwable, so we must throw errors on another thread + ExceptionsHelper.maybeDieOnAnotherThread(e); + throw e; + } + } + } + + @SuppressForbidden(reason = "this test uses a HttpServer to emulate an S3 endpoint") + @Override + protected HttpHandler createHandler() { + return new AssertingHandler(); + } + }; + httpFixture.apply(new Statement() { + @Override + public void evaluate() { + final var repoName = randomIdentifier(); + assertAcked( + client().execute( + TransportPutRepositoryAction.TYPE, + new PutRepositoryRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, repoName).type(S3Repository.TYPE) + .settings( + Settings.builder() + .put("bucket", bucket) + .put("base_path", basePath) + .put("endpoint", httpFixture.getAddress()) + .put(clientName == null ? Settings.EMPTY : Settings.builder().put("client", clientName).build()) + .put(extraRepositorySettings) + ) + ) + ); + + assertEquals( + SnapshotState.SUCCESS, + client().execute( + TransportCreateSnapshotAction.TYPE, + new CreateSnapshotRequest(TEST_REQUEST_TIMEOUT, repoName, randomIdentifier()).waitForCompletion(true) + ).actionGet(SAFE_AWAIT_TIMEOUT).getSnapshotInfo().state() + ); + } + }, Description.EMPTY).evaluate(); + } + +} diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/s3/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/S3RepositoryAnalysisRestIT.java b/x-pack/plugin/snapshot-repo-test-kit/qa/s3/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/S3RepositoryAnalysisRestIT.java index c0f2b40f5a10f..42bc9be86746c 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/qa/s3/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/S3RepositoryAnalysisRestIT.java +++ b/x-pack/plugin/snapshot-repo-test-kit/qa/s3/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/S3RepositoryAnalysisRestIT.java @@ -7,7 +7,12 @@ package org.elasticsearch.repositories.blobstore.testkit.analyze; import fixture.s3.S3HttpFixture; +import fixture.s3.S3HttpHandler; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; + +import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.local.distribution.DistributionType; @@ -15,14 +20,50 @@ import org.junit.rules.RuleChain; import org.junit.rules.TestRule; +import java.util.concurrent.atomic.AtomicBoolean; + import static org.hamcrest.Matchers.blankOrNullString; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.not; public class S3RepositoryAnalysisRestIT extends AbstractRepositoryAnalysisRestTestCase { static final boolean USE_FIXTURE = Boolean.parseBoolean(System.getProperty("tests.use.fixture", "true")); - public static final S3HttpFixture s3Fixture = new S3HttpFixture(USE_FIXTURE); + public static final S3HttpFixture s3Fixture = new S3HttpFixture(USE_FIXTURE) { + @Override + protected HttpHandler createHandler() { + final var delegateHandler = asInstanceOf(S3HttpHandler.class, super.createHandler()); + final var repoAnalysisStarted = new AtomicBoolean(); + return exchange -> { + ensurePurposeParameterPresent(exchange, repoAnalysisStarted); + delegateHandler.handle(exchange); + }; + } + }; + + private static void ensurePurposeParameterPresent(HttpExchange exchange, AtomicBoolean repoAnalysisStarted) { + final var requestPath = exchange.getRequestURI().getPath(); + if (requestPath.startsWith("/bucket/base_path_integration_tests/temp-analysis-")) { + repoAnalysisStarted.set(true); + } + final var queryString = exchange.getRequestURI().getQuery(); + if (repoAnalysisStarted.get() == false) { + if (Regex.simpleMatch("/bucket/base_path_integration_tests/tests-*/master.dat", requestPath) + || Regex.simpleMatch("/bucket/base_path_integration_tests/tests-*/data-*.dat", requestPath) + || queryString.contains("prefix=base_path_integration_tests/tests-") + || queryString.contains("delete")) { + // verify repository is not part of repo analysis so will have different/missing x-purpose parameter + return; + } + if (queryString.contains("prefix=base_path_integration_tests/index-")) { + // getRepositoryData looking for root index-N blob will have different/missing x-purpose parameter + return; + } + repoAnalysisStarted.set(true); + } + assertThat(queryString, containsString("x-purpose=RepositoryAnalysis")); + } public static ElasticsearchCluster cluster = ElasticsearchCluster.local() .distribution(DistributionType.DEFAULT) @@ -59,6 +100,7 @@ protected Settings repositorySettings() { .put("bucket", bucket) .put("base_path", basePath) .put("delete_objects_max_size", between(1, 1000)) + .put(randomFrom(Settings.EMPTY, Settings.builder().put("add_purpose_custom_query_parameter", randomBoolean()).build())) .build(); } }