diff --git a/src/Elastic.Clients.Elasticsearch/Common/UrlParameters/TaskId/TaskId.cs b/src/Elastic.Clients.Elasticsearch/Common/UrlParameters/TaskId/TaskId.cs index 77d8d8e9258..e0b6df13186 100644 --- a/src/Elastic.Clients.Elasticsearch/Common/UrlParameters/TaskId/TaskId.cs +++ b/src/Elastic.Clients.Elasticsearch/Common/UrlParameters/TaskId/TaskId.cs @@ -5,6 +5,7 @@ using System; using System.Diagnostics; using System.Globalization; +using System.Runtime; using System.Text.Json; using System.Text.Json.Serialization; using Elastic.Transport; @@ -71,6 +72,19 @@ public override int GetHashCode() internal sealed class TaskIdConverter : JsonConverter { + public override void WriteAsPropertyName(Utf8JsonWriter writer, TaskId value, JsonSerializerOptions options) + { + if (options.TryGetClientSettings(out var settings)) + { + writer.WritePropertyName(value.GetString(settings)); + return; + } + + throw new JsonException("Unable to retrive client settings during property name serialization."); + } + + public override TaskId ReadAsPropertyName(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => reader.GetString(); + public override TaskId? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { if (reader.TokenType == JsonTokenType.Null) diff --git a/src/Elastic.Clients.Elasticsearch/_Generated/Api/ClearScrollRequest.g.cs b/src/Elastic.Clients.Elasticsearch/_Generated/Api/ClearScrollRequest.g.cs index 62436535269..4bd9d4e0b5a 100644 --- a/src/Elastic.Clients.Elasticsearch/_Generated/Api/ClearScrollRequest.g.cs +++ b/src/Elastic.Clients.Elasticsearch/_Generated/Api/ClearScrollRequest.g.cs @@ -33,7 +33,10 @@ public sealed partial class ClearScrollRequest : PlainRequestBase ApiUrlsLookups.NoNamespaceClearScroll; protected override HttpMethod HttpMethod => HttpMethod.DELETE; - protected override bool SupportsBody => false; + protected override bool SupportsBody => true; + [JsonInclude] + [JsonPropertyName("scroll_id")] + public Elastic.Clients.Elasticsearch.ScrollIds? ScrollId { get; set; } } public sealed partial class ClearScrollRequestDescriptor : RequestDescriptorBase @@ -45,9 +48,25 @@ public ClearScrollRequestDescriptor() internal override ApiUrls ApiUrls => ApiUrlsLookups.NoNamespaceClearScroll; protected override HttpMethod HttpMethod => HttpMethod.DELETE; - protected override bool SupportsBody => false; + protected override bool SupportsBody => true; + private Elastic.Clients.Elasticsearch.ScrollIds? ScrollIdValue { get; set; } + + public ClearScrollRequestDescriptor ScrollId(Elastic.Clients.Elasticsearch.ScrollIds? scrollId) + { + ScrollIdValue = scrollId; + return Self; + } + protected override void Serialize(Utf8JsonWriter writer, JsonSerializerOptions options, IElasticsearchClientSettings settings) { + writer.WriteStartObject(); + if (ScrollIdValue is not null) + { + writer.WritePropertyName("scroll_id"); + JsonSerializer.Serialize(writer, ScrollIdValue, options); + } + + writer.WriteEndObject(); } } } \ No newline at end of file diff --git a/tests/Tests.Configuration/tests.default.yaml b/tests/Tests.Configuration/tests.default.yaml index f0cb38c1948..1731177cc55 100644 --- a/tests/Tests.Configuration/tests.default.yaml +++ b/tests/Tests.Configuration/tests.default.yaml @@ -5,10 +5,10 @@ # tracked by git). # mode either u (unit test), i (integration test) or m (mixed mode) -mode: u +mode: i # the elasticsearch version that should be started # Can be a snapshot version of sonatype or "latest" to get the latest snapshot of sonatype -elasticsearch_version: 8.5.0-SNAPSHOT +elasticsearch_version: 8.4.2 # cluster filter allows you to only run the integration tests of a particular cluster (cluster suffix not needed) # cluster_filter: # whether we want to forcefully reseed on the node, if you are starting the tests with a node already running diff --git a/tests/Tests.Core/ManagedElasticsearch/Clusters/ReindexCluster.cs b/tests/Tests.Core/ManagedElasticsearch/Clusters/ReindexCluster.cs new file mode 100644 index 00000000000..86bb7194206 --- /dev/null +++ b/tests/Tests.Core/ManagedElasticsearch/Clusters/ReindexCluster.cs @@ -0,0 +1,12 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using Tests.Core.ManagedElasticsearch.NodeSeeders; + +namespace Tests.Core.ManagedElasticsearch.Clusters; + +public class ReindexCluster : ClientTestClusterBase +{ + protected override void SeedNode() => new DefaultSeeder(Client).SeedNodeNoData(); +} diff --git a/tests/Tests/AsyncSearch/AsyncSearchApiTests.cs b/tests/Tests/AsyncSearch/AsyncSearchApiTests.cs index e279b65932a..98b4c5752c5 100644 --- a/tests/Tests/AsyncSearch/AsyncSearchApiTests.cs +++ b/tests/Tests/AsyncSearch/AsyncSearchApiTests.cs @@ -14,159 +14,157 @@ using Tests.Framework.EndpointTests.TestState; using System; -namespace Tests.AsyncSearch +namespace Tests.AsyncSearch; + +public class AsyncSearchApiTests : CoordinatedIntegrationTestBase { - public class AsyncSearchApiTests : CoordinatedIntegrationTestBase - { - private const string SubmitStep = nameof(SubmitStep); - private const string StatusStep = nameof(StatusStep); - private const string GetStep = nameof(GetStep); - private const string DeleteStep = nameof(DeleteStep); + private const string SubmitStep = nameof(SubmitStep); + private const string StatusStep = nameof(StatusStep); + private const string GetStep = nameof(GetStep); + private const string DeleteStep = nameof(DeleteStep); - public AsyncSearchApiTests(ReadOnlyCluster cluster, EndpointUsage usage) : base(new CoordinatedUsage(cluster, usage) + public AsyncSearchApiTests(ReadOnlyCluster cluster, EndpointUsage usage) : base(new CoordinatedUsage(cluster, usage) + { { - { - SubmitStep, u => - u.Calls, AsyncSearchSubmitRequest, AsyncSearchSubmitResponse>( - v => new AsyncSearchSubmitRequest + SubmitStep, u => + u.Calls, AsyncSearchSubmitRequest, AsyncSearchSubmitResponse>( + v => new AsyncSearchSubmitRequest + { + Query = new MatchAllQuery(), + KeepOnCompletion = true, + WaitForCompletionTimeout = Duration.MinusOne, + Aggregations = new TermsAggregation("states") { - Query = new MatchAllQuery(), - KeepOnCompletion = true, - WaitForCompletionTimeout = Duration.MinusOne, - Aggregations = new TermsAggregation("states") + Field = Infer.Field(p => p.State.Suffix("keyword")), + MinDocCount = 2, + Size = 5, + ShardSize = 100, + ExecutionHint = TermsAggregationExecutionHint.Map, + //Missing = "n/a", + // TODO - Review terms agg and fix this + //Include = new TermsInclude(new[] { StateOfBeing.Stable.ToString(), StateOfBeing.VeryActive.ToString() }), + Order =new [] { - Field = Infer.Field(p => p.State.Suffix("keyword")), - MinDocCount = 2, - Size = 5, - ShardSize = 100, - ExecutionHint = TermsAggregationExecutionHint.Map, - //Missing = "n/a", + AggregateOrder.KeyAscending, + AggregateOrder.CountDescending + }, + Meta = new Dictionary { { "foo", "bar" } } + } + }, + (v, d) => d + .MatchAll() + .KeepOnCompletion() + .WaitForCompletionTimeout(-1) + .Aggregations(a => a + .Terms("states", st => st + .Field(p => p.State.Suffix("keyword")) + .MinDocCount(2) + .Size(5) + .ShardSize(100) + .ExecutionHint(TermsAggregationExecutionHint.Map) + //.Missing("n/a") // TODO - Review terms agg and fix this - //Include = new TermsInclude(new[] { StateOfBeing.Stable.ToString(), StateOfBeing.VeryActive.ToString() }), - Order =new [] + //.Include(new[] { StateOfBeing.Stable.ToString(), StateOfBeing.VeryActive.ToString() }) + .Order(new [] { AggregateOrder.KeyAscending, AggregateOrder.CountDescending - }, - Meta = new Dictionary { { "foo", "bar" } } - } - }, - (v, d) => d - .MatchAll() - .KeepOnCompletion() - .WaitForCompletionTimeout(-1) - .Aggregations(a => a - .Terms("states", st => st - .Field(p => p.State.Suffix("keyword")) - .MinDocCount(2) - .Size(5) - .ShardSize(100) - .ExecutionHint(TermsAggregationExecutionHint.Map) - //.Missing("n/a") - // TODO - Review terms agg and fix this - //.Include(new[] { StateOfBeing.Stable.ToString(), StateOfBeing.VeryActive.ToString() }) - .Order(new [] - { - AggregateOrder.KeyAscending, - AggregateOrder.CountDescending - }) - .Meta(m => m - .Add("foo", "bar") - ) + }) + .Meta(m => m + .Add("foo", "bar") ) - ), - (v, c, f) => c.AsyncSearch.Submit(f), - (v, c, f) => c.AsyncSearch.SubmitAsync(f), - (v, c, r) => c.AsyncSearch.Submit(r), - (v, c, r) => c.AsyncSearch.SubmitAsync(r), - onResponse: (r, values) => values.ExtendedValue("id", r.Id) - ) - }, - { - StatusStep, u => - u.Calls( - v => new AsyncSearchStatusRequest(v), - (v, d) => d, - (v, c, f) => c.AsyncSearch.Status(v, f), - (v, c, f) => c.AsyncSearch.StatusAsync(v, f), - (v, c, r) => c.AsyncSearch.Status(r), - (v, c, r) => c.AsyncSearch.StatusAsync(r), - uniqueValueSelector: values => values.ExtendedValue("id") - ) - }, - { - GetStep, u => - u.Calls, GetAsyncSearchRequest, GetAsyncSearchResponse>( - v => new GetAsyncSearchRequest(v), - (v, d) => d, - (v, c, f) => c.AsyncSearch.Get(v, f), - (v, c, f) => c.AsyncSearch.GetAsync(v, f), - (v, c, r) => c.AsyncSearch.Get(r), - (v, c, r) => c.AsyncSearch.GetAsync(r), - uniqueValueSelector: values => values.ExtendedValue("id") - ) - }, - { - DeleteStep, u => - u.Calls( - v => new DeleteAsyncSearchRequest(v), - (v, d) => d, - (v, c, f) => c.AsyncSearch.Delete(v, f), - (v, c, f) => c.AsyncSearch.DeleteAsync(v, f), - (v, c, r) => c.AsyncSearch.Delete(r), - (v, c, r) => c.AsyncSearch.DeleteAsync(r), - uniqueValueSelector: values => values.ExtendedValue("id") - ) - }, - }) - { } - - [I] - public async Task AsyncSearchSubmitResponse() => await Assert>(SubmitStep, r => + ) + ), + (v, c, f) => c.AsyncSearch.Submit(f), + (v, c, f) => c.AsyncSearch.SubmitAsync(f), + (v, c, r) => c.AsyncSearch.Submit(r), + (v, c, r) => c.AsyncSearch.SubmitAsync(r), + onResponse: (r, values) => values.ExtendedValue("id", r.Id) + ) + }, { - r.ShouldBeValid(); - r.Id.Should().NotBeNullOrEmpty(); - // TODO - MORE ASSERTIONS - r.Response.Should().NotBeNull(); - r.Response.Took.Should().BeGreaterOrEqualTo(0); - }); - - [I] - public async Task AsyncSearchStatusResponse() => await Assert(StatusStep, r => + StatusStep, u => + u.Calls( + v => new AsyncSearchStatusRequest(v), + (v, d) => d, + (v, c, f) => c.AsyncSearch.Status(v, f), + (v, c, f) => c.AsyncSearch.StatusAsync(v, f), + (v, c, r) => c.AsyncSearch.Status(r), + (v, c, r) => c.AsyncSearch.StatusAsync(r), + uniqueValueSelector: values => values.ExtendedValue("id") + ) + }, + { + GetStep, u => + u.Calls, GetAsyncSearchRequest, GetAsyncSearchResponse>( + v => new GetAsyncSearchRequest(v), + (v, d) => d, + (v, c, f) => c.AsyncSearch.Get(v, f), + (v, c, f) => c.AsyncSearch.GetAsync(v, f), + (v, c, r) => c.AsyncSearch.Get(r), + (v, c, r) => c.AsyncSearch.GetAsync(r), + uniqueValueSelector: values => values.ExtendedValue("id") + ) + }, { - r.ShouldBeValid(); - r.StartTimeInMillis.Should().BeGreaterThan(DateTimeOffset.UtcNow.AddMinutes(-10).ToUnixTimeMilliseconds()); - r.StartTimeInMillis.Should().BeLessOrEqualTo(DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()); - r.ExpirationTimeInMillis.Should().BeGreaterOrEqualTo(DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()); + DeleteStep, u => + u.Calls( + v => new DeleteAsyncSearchRequest(v), + (v, d) => d, + (v, c, f) => c.AsyncSearch.Delete(v, f), + (v, c, f) => c.AsyncSearch.DeleteAsync(v, f), + (v, c, r) => c.AsyncSearch.Delete(r), + (v, c, r) => c.AsyncSearch.DeleteAsync(r), + uniqueValueSelector: values => values.ExtendedValue("id") + ) + }, + }) + { } + + [I] + public async Task AsyncSearchSubmitResponse() => await Assert>(SubmitStep, r => + { + r.ShouldBeValid(); + r.Id.Should().NotBeNullOrEmpty(); + r.Response.Should().NotBeNull(); + r.Response.Took.Should().BeGreaterOrEqualTo(0); + }); - if (r.IsRunning) - r.CompletionStatus.HasValue.Should().BeFalse(); - else - r.CompletionStatus?.Should().Be(200); + [I] + public async Task AsyncSearchStatusResponse() => await Assert(StatusStep, r => + { + r.ShouldBeValid(); + r.StartTimeInMillis.Should().BeGreaterThan(DateTimeOffset.UtcNow.AddMinutes(-10).ToUnixTimeMilliseconds()); + r.StartTimeInMillis.Should().BeLessOrEqualTo(DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()); + r.ExpirationTimeInMillis.Should().BeGreaterOrEqualTo(DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()); - r.Shards.Total.Should().BeGreaterOrEqualTo(1); - }); + if (r.IsRunning) + r.CompletionStatus.HasValue.Should().BeFalse(); + else + r.CompletionStatus?.Should().Be(200); - [I] - public async Task AsyncSearchGetResponse() => await Assert>(GetStep, (s, r) => - { - r.ShouldBeValid(); - r.Id.Should().NotBeNullOrEmpty(); - r.StartTimeInMillis.Should().BeGreaterThan(DateTimeOffset.UtcNow.AddMinutes(-10).ToUnixTimeMilliseconds()); - r.StartTimeInMillis.Should().BeLessOrEqualTo(DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()); - r.ExpirationTimeInMillis.Should().BeGreaterOrEqualTo(DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()); - r.Response.Should().NotBeNull(); - r.Response.Took.Should().BeGreaterOrEqualTo(0); - r.Response.Hits.Should().HaveCount(10); - var terms = r.Response.Aggregations.GetTerms("states"); - terms.Should().NotBeNull(); - }); + r.Shards.Total.Should().BeGreaterOrEqualTo(1); + }); - [I] - public async Task AsyncSearchDeleteResponse() => await Assert(DeleteStep, r => - { - r.ShouldBeValid(); - r.Acknowledged.Should().BeTrue(); - }); - } + [I] + public async Task AsyncSearchGetResponse() => await Assert>(GetStep, (s, r) => + { + r.ShouldBeValid(); + r.Id.Should().NotBeNullOrEmpty(); + r.StartTimeInMillis.Should().BeGreaterThan(DateTimeOffset.UtcNow.AddMinutes(-10).ToUnixTimeMilliseconds()); + r.StartTimeInMillis.Should().BeLessOrEqualTo(DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()); + r.ExpirationTimeInMillis.Should().BeGreaterOrEqualTo(DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()); + r.Response.Should().NotBeNull(); + r.Response.Took.Should().BeGreaterOrEqualTo(0); + r.Response.Hits.Should().HaveCount(10); + var terms = r.Response.Aggregations.GetTerms("states"); + terms.Should().NotBeNull(); + }); + + [I] + public async Task AsyncSearchDeleteResponse() => await Assert(DeleteStep, r => + { + r.ShouldBeValid(); + r.Acknowledged.Should().BeTrue(); + }); } diff --git a/tests/Tests/Document/Multiple/DeleteByQuery/DeleteByQueryApiTests.cs b/tests/Tests/Document/Multiple/DeleteByQuery/DeleteByQueryApiTests.cs new file mode 100644 index 00000000000..32840de025e --- /dev/null +++ b/tests/Tests/Document/Multiple/DeleteByQuery/DeleteByQueryApiTests.cs @@ -0,0 +1,293 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using Elastic.Clients.Elasticsearch.IndexManagement; +using Elastic.Clients.Elasticsearch.Mapping; +using Elastic.Clients.Elasticsearch.QueryDsl; +using Tests.Core.Extensions; +using Tests.Core.ManagedElasticsearch.Clusters; +using Tests.Domain; +using Tests.Framework.EndpointTests; +using Tests.Framework.EndpointTests.TestState; + +namespace Tests.Document.Multiple.DeleteByQuery +{ + public class DeleteByQueryApiTests + : ApiIntegrationTestBase, + DeleteByQueryRequest> + { + public DeleteByQueryApiTests(IntrusiveOperationCluster cluster, EndpointUsage usage) : base(cluster, usage) { } + + protected override bool ExpectIsValid => true; + + protected override object ExpectJson { get; } = new + { + max_docs = Project.Projects.Count, + query = new + { + ids = new + { + values = new[] { Project.First.Name, "x" } + } + } + }; + + protected override int ExpectStatusCode => 200; + + protected override Action> Fluent => d => d + .IgnoreUnavailable() + .MaxDocs(Project.Projects.Count) + .Query(q => q + .Ids(ids => ids + .Values(new Ids(new[] { Project.First.Name, "x" })) + ) + ); + + protected override HttpMethod HttpMethod => HttpMethod.POST; + + protected override DeleteByQueryRequest Initializer => new(Indices) + { + IgnoreUnavailable = true, + MaxDocs = Project.Projects.Count, + Query = QueryContainer.Ids(new IdsQuery + { + Values = new Ids(new[] { Project.First.Name, "x" }) + }) + }; + + protected override bool SupportsDeserialization => false; + + protected override string ExpectedUrlPathAndQuery => $"/{CallIsolatedValue}%2C{SecondIndex}/_delete_by_query?ignore_unavailable=true"; + + private Indices Indices => Infer.Index(CallIsolatedValue).And(SecondIndex); + + private string SecondIndex => $"{CallIsolatedValue}-clone"; + + protected override void IntegrationSetup(ElasticsearchClient client, CallUniqueValues values) + { + foreach (var index in values.Values) + { + // TODO: Reapply settings and mappings once default seeder is updated to use current APIs + Client.Indices.Create(index, c => c + .Settings(s => s + .NumberOfShards(2) + .NumberOfReplicas(0) + //.Analysis(DefaultSeeder.ProjectAnalysisSettings) + ) + //.Mappings(p => p + // //.AutoMap() + // .Properties(DefaultSeeder.ProjectProperties) + //) + .Mappings(m => m.Properties(new Properties(new Dictionary + { + { "name", new KeywordProperty() }, + }))) + ); + + Client.IndexMany(Project.Projects, index); + var cloneIndex = index + "-clone"; + Client.Indices.Create(cloneIndex); + Client.Indices.Refresh(new RefreshRequest(Infer.Index(index).And(cloneIndex))); + } + } + + protected override LazyResponses ClientUsage() => Calls( + (client, f) => client.DeleteByQuery(Indices, f), + (client, f) => client.DeleteByQueryAsync(Indices, f), + (client, r) => client.DeleteByQuery(r), + (client, r) => client.DeleteByQueryAsync(r) + ); + + protected override void OnAfterCall(ElasticsearchClient client) => client.Indices.Refresh(new RefreshRequest(CallIsolatedValue)); + + protected override DeleteByQueryRequestDescriptor NewDescriptor() => new(Indices); + + protected override void ExpectResponse(DeleteByQueryResponse response) + { + response.Took.Should().BeGreaterThan(0); + response.Total.Should().Be(1); + response.Deleted.Should().Be(1); + response.Retries.Should().NotBeNull(); + response.Retries.Bulk.Should().Be(0); + response.Retries.Search.Should().Be(0); + response.RequestsPerSecond.Should().Be(-1); + response.Failures.Should().BeEmpty(); + } + } + + public class DeleteByQueryWaitForCompletionApiTests : DeleteByQueryApiTests + { + public DeleteByQueryWaitForCompletionApiTests(IntrusiveOperationCluster cluster, EndpointUsage usage) : base(cluster, usage) { } + + protected override object ExpectJson => new { query = new { match_all = new { } } }; + + protected override Action> Fluent => d => d + .Indices(CallIsolatedValue) + .Query(q => q.MatchAll()) + .WaitForCompletion(false) + .Conflicts(Conflicts.Proceed); + + protected override DeleteByQueryRequest Initializer => new(CallIsolatedValue) + { + Query = new MatchAllQuery(), + WaitForCompletion = false, + Conflicts = Conflicts.Proceed + }; + + protected override string ExpectedUrlPathAndQuery => $"/{CallIsolatedValue}/_delete_by_query?wait_for_completion=false&conflicts=proceed"; + + protected override DeleteByQueryRequestDescriptor NewDescriptor() => new (CallIsolatedValue); + + protected override void ExpectResponse(DeleteByQueryResponse response) + { + response.Task.Should().NotBeNull(); + response.Task.TaskNumber.Should().BeGreaterThan(0); + response.Task.NodeId.Should().NotBeNullOrWhiteSpace(); + response.Task.FullyQualifiedId.Should().NotBeNullOrWhiteSpace(); + } + } + + public class DeleteByQueryWithFailuresApiTests : DeleteByQueryApiTests + { + public DeleteByQueryWithFailuresApiTests(IntrusiveOperationCluster cluster, EndpointUsage usage) : base(cluster, usage) { } + + protected override bool ExpectIsValid => false; + + protected override object ExpectJson => + new + { + query = new { match = new { description = new { query = "description" } } } + }; + + protected override int ExpectStatusCode => 409; + + protected override Action> Fluent => d => d + .Indices(CallIsolatedValue) + .Query(q => q + .Match(m => m + .Field(p => p.Description) + .Query("description") + ) + ); + + protected override DeleteByQueryRequest Initializer => new(CallIsolatedValue) + { + Query = QueryContainer.Match(new MatchQuery(Infer.Field(p => p.Description)) + { + Query = "description" + }) + }; + + protected override string ExpectedUrlPathAndQuery => $"/{CallIsolatedValue}/_delete_by_query"; + + protected override void IntegrationSetup(ElasticsearchClient client, CallUniqueValues values) + { + foreach (var index in values.Values) + { + Client.Indices.Create(index, c => c + .Settings(s => s + .RefreshInterval(-1) + ) + ); + Client.Index(new Project { Name = "project1", Description = "description" }, + i => i.Index(index).Id(1).Refresh(Refresh.True).Routing(1)); + Client.Index(new Project { Name = "project2", Description = "description" }, + i => i.Index(index).Id(1).Routing(1)); + } + } + + protected override DeleteByQueryRequestDescriptor NewDescriptor() => new(CallIsolatedValue); + + protected override void ExpectResponse(DeleteByQueryResponse response) + { + response.VersionConflicts.Should().Be(1); + response.Failures.Should().NotBeEmpty(); + var failure = response.Failures.First(); + + failure.Index.Should().NotBeNullOrWhiteSpace(); + failure.Status.Should().Be(409); + failure.Id.Should().NotBeNullOrWhiteSpace(); + + failure.Cause.Should().NotBeNull(); + //failure.Cause.IndexUUID.Should().NotBeNullOrWhiteSpace(); // TODO: Specification may be lacking this property + failure.Cause.Reason.Should().NotBeNullOrWhiteSpace(); + //failure.Cause.Index.Should().NotBeNullOrWhiteSpace(); // TODO: Specification may be lacking this property + //failure.Cause.Shard.Should().NotBeNull(); // TODO: Specification may be lacking this property + failure.Cause.Type.Should().NotBeNullOrWhiteSpace(); + } + } + + public class DeleteByQueryWithSlicesApiTests : DeleteByQueryApiTests + { + public DeleteByQueryWithSlicesApiTests(IntrusiveOperationCluster cluster, EndpointUsage usage) : base(cluster, usage) { } + + protected override bool ExpectIsValid => true; + + protected override object ExpectJson => + new + { + slice = new { id = 0, max = 2 }, + query = new { terms = new { name = FirstTenProjectNames } } + }; + + protected override int ExpectStatusCode => 200; + + protected override Action> Fluent => d => d + .Indices(CallIsolatedValue) + .Slice(s => s + .Id(0) + .Max(2) + ) + .Query(q => q + .Terms(m => m + .Field(p => p.Name) + .Terms(new TermsQueryField(FirstTenProjectNames)) + ) + ); + + protected override DeleteByQueryRequest Initializer => new(CallIsolatedValue) + { + Slice = new SlicedScroll + { + Id = 0, + Max = 2 + }, + Query = QueryContainer.Terms(new TermsQuery() + { + Field = Infer.Field(p => p.Name), + Terms = new TermsQueryField(FirstTenProjectNames) + }) + }; + + protected override string ExpectedUrlPathAndQuery => $"/{CallIsolatedValue}/_delete_by_query"; + private static List FirstTenProjectNames => Project.Projects.Take(10).Select(p => p.Name).ToList(); + + protected override DeleteByQueryRequestDescriptor NewDescriptor() => new(CallIsolatedValue); + + protected override void ExpectResponse(DeleteByQueryResponse response) + { + response.ShouldBeValid(); + response.SliceId.Should().Be(0); + + // Since we only executed one slice of the two, some of the documents that + // match the query will still exist. + Client.Indices.Refresh(new RefreshRequest(CallIsolatedValue)); + + var countResponse = Client.Count(c => c + .Indices(CallIsolatedValue) + .Query(q => q + .Terms(m => m + .Field(p => p.Name) + .Terms(new TermsQueryField(FirstTenProjectNames)) + ) + ) + ); + + countResponse.Count.Should().BeGreaterThan(0); + } + } +} diff --git a/tests/Tests/Document/Multiple/DeleteByQuery/DeleteByQueryUrlTests.cs b/tests/Tests/Document/Multiple/DeleteByQuery/DeleteByQueryUrlTests.cs new file mode 100644 index 00000000000..7fc02b3f47d --- /dev/null +++ b/tests/Tests/Document/Multiple/DeleteByQuery/DeleteByQueryUrlTests.cs @@ -0,0 +1,21 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using System.Threading.Tasks; +using Tests.Domain; +using Tests.Framework.EndpointTests; +using static Tests.Framework.EndpointTests.UrlTester; + +namespace Tests.Document.Multiple.DeleteByQuery +{ + public class DeleteByQueryUrlTests : UrlTestsBase + { + [U] public override async Task Urls() => + await POST("/project/_delete_by_query") + .Fluent(c => c.DeleteByQuery("project", d => { })) + .Request(c => c.DeleteByQuery(new DeleteByQueryRequest("project"))) + .FluentAsync(c => c.DeleteByQueryAsync("project", d => { })) + .RequestAsync(c => c.DeleteByQueryAsync(new DeleteByQueryRequest("project"))); + } +} diff --git a/tests/Tests/Document/Multiple/DeleteByQueryRethrottle/DeleteByQueryRethrottleApiTests.cs b/tests/Tests/Document/Multiple/DeleteByQueryRethrottle/DeleteByQueryRethrottleApiTests.cs new file mode 100644 index 00000000000..5a99118e3c5 --- /dev/null +++ b/tests/Tests/Document/Multiple/DeleteByQueryRethrottle/DeleteByQueryRethrottleApiTests.cs @@ -0,0 +1,109 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using System; +using System.Linq; +using Elastic.Clients.Elasticsearch.IndexManagement; +using Tests.Core.Extensions; +using Tests.Core.ManagedElasticsearch.Clusters; +using Tests.Domain; +using Tests.Framework.EndpointTests; +using Tests.Framework.EndpointTests.TestState; + +namespace Tests.Document.Multiple.DeleteByQueryRethrottle; + +public class DeleteByQueryRethrottleApiTests + : ApiIntegrationTestBase +{ + protected const string TaskIdKey = "taskId"; + + public DeleteByQueryRethrottleApiTests(ReindexCluster cluster, EndpointUsage usage) : base(cluster, usage) { } + + protected override bool ExpectIsValid => true; + + protected override object ExpectJson => null; + protected override int ExpectStatusCode => 200; + + protected override Action Fluent => d => d + .RequestsPerSecond(-1); + + protected override HttpMethod HttpMethod => HttpMethod.POST; + + protected override DeleteByQueryRethrottleRequest Initializer => new(TaskId.ToString()) + { + RequestsPerSecond = -1, + }; + + protected override DeleteByQueryRethrottleRequestDescriptor NewDescriptor() => new(TaskId.ToString()); + + protected override bool SupportsDeserialization => false; + protected TaskId TaskId => RanIntegrationSetup ? ExtendedValue(TaskIdKey) : "foo:1"; + + protected override string ExpectedUrlPathAndQuery => RanIntegrationSetup ? $"/_delete_by_query/{TaskId}/_rethrottle?requests_per_second=-1" : $"/_delete_by_query/foo%3A1/_rethrottle?requests_per_second=-1"; + + protected override void IntegrationSetup(ElasticsearchClient client, CallUniqueValues values) + { + foreach (var callUniqueValue in values) + { + client.IndexMany(Project.Projects, callUniqueValue.Value); + client.Indices.Refresh(new RefreshRequest(callUniqueValue.Value)); + } + } + + protected override LazyResponses ClientUsage() => Calls( + (client, f) => client.DeleteByQueryRethrottle(TaskId.ToString(), f), + (client, f) => client.DeleteByQueryRethrottleAsync(TaskId.ToString(), f), + (client, r) => client.DeleteByQueryRethrottle(r), + (client, r) => client.DeleteByQueryRethrottleAsync(r) + ); + + protected override void OnBeforeCall(ElasticsearchClient client) + { + client.IndexMany(Project.Projects, CallIsolatedValue); + client.Indices.Refresh(new RefreshRequest(CallIsolatedValue)); + + var deleteByQuery = client.DeleteByQuery(CallIsolatedValue, u => u + .Conflicts(Conflicts.Proceed) + .Query(q => q.MatchAll()) + .Refresh() + .RequestsPerSecond(1) + .WaitForCompletion(false) + ); + + deleteByQuery.ShouldBeValid(); + ExtendedValue(TaskIdKey, deleteByQuery.Task); + } + + protected override void ExpectResponse(DeleteByQueryRethrottleResponse response) + { + response.ShouldBeValid(); + + response.Nodes.Should().NotBeEmpty().And.HaveCount(1); + var node = response.Nodes.First().Value; + + node.Name.Should().NotBeNullOrEmpty(); + node.TransportAddress.Should().NotBeNullOrEmpty(); + node.Host.Should().NotBeNullOrEmpty(); + node.Ip.Should().NotBeNullOrEmpty(); + node.Roles.Should().NotBeEmpty(); + node.Attributes.Should().NotBeEmpty(); + + node.Tasks.Should().NotBeEmpty().And.HaveCount(1); + node.Tasks.First().Key.Should().Be(TaskId); + + var task = node.Tasks.First().Value; + + task.Node.Should().NotBeNullOrEmpty().And.Be(TaskId.NodeId); + task.Id.Should().Be(TaskId.TaskNumber); + task.Type.Should().NotBeNullOrEmpty(); + task.Action.Should().NotBeNullOrEmpty(); + + task.Status.RequestsPerSecond.Should().Be(-1); + + task.StartTimeInMillis.Should().BeGreaterThan(0); + task.RunningTimeInNanos.Should().BeGreaterThan(0); + task.Cancellable.Should().BeTrue(); + } +} diff --git a/tests/Tests/Document/Multiple/DeleteByQueryRethrottle/DeleteByQueryRethrottleUrlTests.cs b/tests/Tests/Document/Multiple/DeleteByQueryRethrottle/DeleteByQueryRethrottleUrlTests.cs new file mode 100644 index 00000000000..3a4ce456065 --- /dev/null +++ b/tests/Tests/Document/Multiple/DeleteByQueryRethrottle/DeleteByQueryRethrottleUrlTests.cs @@ -0,0 +1,22 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using System.Threading.Tasks; +using Tests.Framework.EndpointTests; +using static Tests.Framework.EndpointTests.UrlTester; + +namespace Tests.Document.Multiple.DeleteByQueryRethrottle +{ + public class DeleteByQueryRethrottleUrlTests : UrlTestsBase + { + private readonly Id _taskId = "rhtoNesNR4aXVIY2bRR4GQ:13056"; + + [U] public override async Task Urls() => + await POST($"/_delete_by_query/{EscapeUriString(_taskId.ToString())}/_rethrottle") + .Fluent(c => c.DeleteByQueryRethrottle(_taskId)) + .Request(c => c.DeleteByQueryRethrottle(new DeleteByQueryRethrottleRequest(_taskId))) + .FluentAsync(c => c.DeleteByQueryRethrottleAsync(_taskId)) + .RequestAsync(c => c.DeleteByQueryRethrottleAsync(new DeleteByQueryRethrottleRequest(_taskId))); + } +} diff --git a/tests/Tests/Search/Scroll/ClearScroll/ClearScrollApiTests.cs b/tests/Tests/Search/Scroll/ClearScroll/ClearScrollApiTests.cs new file mode 100644 index 00000000000..28af7368cd7 --- /dev/null +++ b/tests/Tests/Search/Scroll/ClearScroll/ClearScrollApiTests.cs @@ -0,0 +1,55 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using System; +using Tests.Core.ManagedElasticsearch.Clusters; +using Tests.Domain; +using Tests.Framework.EndpointTests; +using Tests.Framework.EndpointTests.TestState; + +namespace Tests.Search.Scroll.ClearScroll; + +// ReadOnlyCluster because even though its technically a write action, it does not hinder on going reads. +public class ClearScrollApiTests + : ApiIntegrationTestBase +{ + private ScrollId _scrollId = "default-for-unit-tests"; + + public ClearScrollApiTests(ReadOnlyCluster cluster, EndpointUsage usage) : base(cluster, usage) { } + + protected override bool ExpectIsValid => true; + + protected override object ExpectJson => new + { + scroll_id = new[] + { + _scrollId + } + }; + + protected override int ExpectStatusCode => 200; + protected override HttpMethod HttpMethod => HttpMethod.DELETE; + + protected override Action Fluent => cs => cs.ScrollId(_scrollId); + protected override ClearScrollRequest Initializer => new() { ScrollId = _scrollId }; + protected override bool SupportsDeserialization => false; + protected override string ExpectedUrlPathAndQuery => $"/_search/scroll"; + + protected override LazyResponses ClientUsage() => Calls( + (c, f) => c.ClearScroll(f), + (c, f) => c.ClearScrollAsync(f), + (c, r) => c.ClearScroll(r), + (c, r) => c.ClearScrollAsync(r) + ); + + protected override void OnBeforeCall(ElasticsearchClient client) + { + var searchResponse = Client.Search(s => s.Query(q => q.MatchAll()).Scroll(TimeSpan.FromMinutes(1))); + + if (!searchResponse.IsValid) + throw new Exception("Setup: Initial scroll failed."); + + _scrollId = searchResponse.ScrollId ?? _scrollId; + } +} diff --git a/tests/Tests/Search/Scroll/ClearScroll/ClearScrollUrlTests.cs b/tests/Tests/Search/Scroll/ClearScroll/ClearScrollUrlTests.cs new file mode 100644 index 00000000000..8590775659d --- /dev/null +++ b/tests/Tests/Search/Scroll/ClearScroll/ClearScrollUrlTests.cs @@ -0,0 +1,19 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using System.Threading.Tasks; +using Tests.Framework.EndpointTests; +using static Tests.Framework.EndpointTests.UrlTester; + +namespace Tests.Search.Scroll.ClearScroll +{ + public class ClearScrollUrlTests + { + [U] public async Task Urls() => await DELETE("/_search/scroll") + .Fluent(c => c.ClearScroll(cs => cs.ScrollId("scrollid1, scrollid2"))) + .Request(c => c.ClearScroll(new ClearScrollRequest { ScrollId = new(new[] { new ScrollId("scrollid1"), new ScrollId("scrollid2") }) })) + .FluentAsync(c => c.ClearScrollAsync(cs => cs.ScrollId("scrollid1, scrollid2"))) + .RequestAsync(c => c.ClearScrollAsync(new ClearScrollRequest { ScrollId = new(new[] { new ScrollId("scrollid1"), new ScrollId("scrollid2") }) })); + } +} diff --git a/tests/Tests/Search/Scroll/Scroll/ScrollApiTests.cs b/tests/Tests/Search/Scroll/Scroll/ScrollApiTests.cs new file mode 100644 index 00000000000..389718745bf --- /dev/null +++ b/tests/Tests/Search/Scroll/Scroll/ScrollApiTests.cs @@ -0,0 +1,54 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using System; +using Tests.Core.ManagedElasticsearch.Clusters; +using Tests.Domain; +using Tests.Framework.EndpointTests; +using Tests.Framework.EndpointTests.TestState; + +namespace Tests.Search.Scroll.Scroll; + +// ReadOnlyCluster because even though its technically a write action, it does not hinder on going reads. +public class ScrollApiTests + : ApiIntegrationTestBase, ScrollRequestDescriptor, ScrollRequest> +{ + private ScrollId _scrollId = "default-for-unit-tests"; + + public ScrollApiTests(ReadOnlyCluster cluster, EndpointUsage usage) : base(cluster, usage) { } + + protected override bool ExpectIsValid => true; + protected override int ExpectStatusCode => 200; + protected override HttpMethod HttpMethod => HttpMethod.POST; + protected override bool SupportsDeserialization => false; + protected override string ExpectedUrlPathAndQuery => $"/_search/scroll"; + + protected override object ExpectJson => new + { + scroll = "1m", + scroll_id = _scrollId + }; + + protected override ScrollRequest Initializer => new() { Scroll = "1m", ScrollId = _scrollId }; + + protected override Action Fluent => f => f.Scroll("1m").ScrollId(_scrollId); + + protected override LazyResponses ClientUsage() => Calls( + (c, f) => c.Scroll(f), + (c, f) => c.ScrollAsync(f), + (c, r) => c.Scroll(r), + (c, r) => c.ScrollAsync(r) + ); + + protected override void OnBeforeCall(ElasticsearchClient client) + { + var response = client.Search(s => s.Query(q => q.MatchAll()).Scroll(TimeSpan.FromMinutes(1))); + if (!response.IsValid) + throw new Exception("Scroll setup failed"); + + _scrollId = response.ScrollId ?? _scrollId; + } + + protected override void OnAfterCall(ElasticsearchClient client) => client.ClearScroll(cs => cs.ScrollId(_scrollId)); +} diff --git a/tests/Tests/Search/Scroll/Scroll/ScrollUrlTests.cs b/tests/Tests/Search/Scroll/Scroll/ScrollUrlTests.cs new file mode 100644 index 00000000000..8de341729ac --- /dev/null +++ b/tests/Tests/Search/Scroll/Scroll/ScrollUrlTests.cs @@ -0,0 +1,20 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using System; +using System.Threading.Tasks; +using Tests.Domain; +using Tests.Framework.EndpointTests; +using static Tests.Framework.EndpointTests.UrlTester; + +namespace Tests.Search.Scroll.Scroll; + +public class ScrollUrlTests +{ + [U] public async Task Urls() => await POST("/_search/scroll") + .Fluent(c => c.Scroll(s => s.ScrollId("scroll_id").Scroll("1m"))) + .Request(c => c.Scroll(new ScrollRequest { ScrollId = "scroll_id", Scroll = TimeSpan.FromMinutes(1) })) + .FluentAsync(c => c.ScrollAsync(s => s.ScrollId("scroll_id").Scroll("1m"))) + .RequestAsync(c => c.ScrollAsync(new ScrollRequest { ScrollId = "scroll_id", Scroll = TimeSpan.FromMinutes(1) })); +} diff --git a/tests/Tests/Search/Scroll/Scroll/SlicedScrollSearchApiTests.cs b/tests/Tests/Search/Scroll/Scroll/SlicedScrollSearchApiTests.cs new file mode 100644 index 00000000000..374a8768eaf --- /dev/null +++ b/tests/Tests/Search/Scroll/Scroll/SlicedScrollSearchApiTests.cs @@ -0,0 +1,67 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using System; +using System.Threading; +using Tests.Core.ManagedElasticsearch.Clusters; +using Tests.Domain; +using Tests.Framework.EndpointTests; +using Tests.Framework.EndpointTests.TestState; + +namespace Tests.Search.Scroll.Scroll; + +// ReadOnlyCluster because even though its technically a write action, it does not hinder on going reads. +public class SlicedScrollSearchApiTests + : ApiIntegrationTestBase, ScrollRequestDescriptor, ScrollRequest> +{ + private int _slice; + + private ScrollId _scrollId = "default-for-unit-tests"; + + public SlicedScrollSearchApiTests(ReadOnlyCluster cluster, EndpointUsage usage) : base(cluster, usage) { } + + protected override bool ExpectIsValid => true; + protected override bool SupportsDeserialization => false; + protected override string ExpectedUrlPathAndQuery => $"/_search/scroll"; + protected override int ExpectStatusCode => 200; + protected override HttpMethod HttpMethod => HttpMethod.POST; + + protected override object ExpectJson => new + { + scroll = "1m", + scroll_id = _scrollId + }; + + protected override ScrollRequest Initializer => new() { Scroll = "1m", ScrollId = _scrollId }; + + protected override Action Fluent => f => f.Scroll("1m").ScrollId(_scrollId); + + protected override LazyResponses ClientUsage() => Calls( + (c, f) => c.Scroll(f), + (c, f) => c.ScrollAsync(f), + (c, r) => c.Scroll(r), + (c, r) => c.ScrollAsync(r) + ); + + protected override void OnBeforeCall(ElasticsearchClient client) + { + var maxSlices = 2; // number of shards we use by default for test indices + var currentSlice = Interlocked.Increment(ref _slice) % maxSlices; + var scrollTimeout = TimeSpan.FromMinutes(1); + var response = client.Search(s => s + .Scroll(scrollTimeout) + .Slice(ss => ss.Max(maxSlices).Id(currentSlice)) + .Sort(new[] + { + new SortCombinations(SortOptions.Field("_doc", new FieldSort { Order = SortOrder.Asc })) + }) + ); + if (!response.IsValid) + throw new Exception("Scroll setup failed"); + + _scrollId = response.ScrollId ?? _scrollId; + } + + protected override void OnAfterCall(ElasticsearchClient client) => client.ClearScroll(cs => cs.ScrollId(_scrollId)); +}