From 9222ed53a8a0ded46ca916f4c7cbc751f919f110 Mon Sep 17 00:00:00 2001 From: Mashhur Date: Thu, 20 Mar 2025 00:53:02 -0700 Subject: [PATCH 1/6] elasticsearch-ruby client got updated to v8 in LS core. This plugin uses it in integration tests. This change tolerates the elasticsearch-ruby v8 client. --- spec/es_spec_helper.rb | 8 +++++-- spec/integration/outputs/delete_spec.rb | 8 +++---- spec/integration/outputs/ilm_spec.rb | 17 +++++++++----- .../integration/outputs/index_version_spec.rb | 14 +++++------ .../outputs/painless_update_spec.rb | 19 +++++++-------- .../outputs/unsupported_actions_spec.rb | 23 ++++++++----------- spec/integration/outputs/update_spec.rb | 19 +++++++-------- spec/spec_helper.rb | 14 +++++++++++ .../api/actions/put_ilm_policy.rb | 2 +- spec/unit/outputs/error_whitelist_spec.rb | 1 - 10 files changed, 68 insertions(+), 57 deletions(-) diff --git a/spec/es_spec_helper.rb b/spec/es_spec_helper.rb index 9c043afbe..4255d3cd4 100644 --- a/spec/es_spec_helper.rb +++ b/spec/es_spec_helper.rb @@ -20,8 +20,12 @@ def get_host_port end def get_client - Elasticsearch::Client.new(:hosts => [get_host_port]).tap do |client| - allow(client).to receive(:verify_elasticsearch).and_return(true) # bypass client side version checking + if elastic_ruby_v8_client_available? + Elasticsearch::Client.new(:hosts => [get_host_port]) + else + Elasticsearch::Client.new(:hosts => [get_host_port]).tap do |client| + allow(client).to receive(:verify_elasticsearch).and_return(true) # bypass client side version checking + end end end diff --git a/spec/integration/outputs/delete_spec.rb b/spec/integration/outputs/delete_spec.rb index 9fa8afd18..bc253832b 100644 --- a/spec/integration/outputs/delete_spec.rb +++ b/spec/integration/outputs/delete_spec.rb @@ -39,12 +39,12 @@ it "should ignore non-monotonic external version updates" do id = "ev2" subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "index", "message" => "foo", "my_version" => 99)]) - r = es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true) + r = es.get(generate_common_index_params('logstash-delete', id)) expect(r['_version']).to eq(99) expect(r['_source']['message']).to eq('foo') subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "delete", "message" => "foo", "my_version" => 98)]) - r2 = es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true) + r2 = es.get(generate_common_index_params('logstash-delete', id)) expect(r2['_version']).to eq(99) expect(r2['_source']['message']).to eq('foo') end @@ -52,12 +52,12 @@ it "should commit monotonic external version updates" do id = "ev3" subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "index", "message" => "foo", "my_version" => 99)]) - r = es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true) + r = es.get(generate_common_index_params('logstash-delete', id)) expect(r['_version']).to eq(99) expect(r['_source']['message']).to eq('foo') subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "delete", "message" => "foo", "my_version" => 100)]) - expect { es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true) }.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) + expect { es.get(generate_common_index_params('logstash-delete', id)) }.to raise_error(get_expected_error_class) end end end diff --git a/spec/integration/outputs/ilm_spec.rb b/spec/integration/outputs/ilm_spec.rb index e6c2dce10..e288e5397 100644 --- a/spec/integration/outputs/ilm_spec.rb +++ b/spec/integration/outputs/ilm_spec.rb @@ -102,7 +102,7 @@ it 'should not install the default policy' do subject.register sleep(1) - expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) + expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(get_expected_error_class) end it 'should not write the ILM settings into the template' do @@ -287,7 +287,7 @@ end it 'should install it if it is not present' do - expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) + expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(get_expected_error_class) subject.register sleep(1) expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.not_to raise_error @@ -340,14 +340,14 @@ let (:policy) { small_max_doc_policy } before do - expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) + expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(get_expected_error_class) put_policy(@es,ilm_policy_name, policy) end it 'should not install the default policy if it is not used' do subject.register sleep(1) - expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) + expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(get_expected_error_class) end end @@ -357,14 +357,14 @@ let (:policy) { max_age_policy("1d") } before do - expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) + expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(get_expected_error_class) put_policy(@es,ilm_policy_name, policy) end it 'should not install the default policy if it is not used' do subject.register sleep(1) - expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) + expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(get_expected_error_class) end end @@ -532,3 +532,8 @@ end end + +def get_expected_error_class + return Elastic::Transport::Transport::Errors::NotFound if elastic_ruby_v8_client_available? + Elasticsearch::Transport::Transport::Errors::NotFound +end \ No newline at end of file diff --git a/spec/integration/outputs/index_version_spec.rb b/spec/integration/outputs/index_version_spec.rb index 0b1ecda94..44edce09c 100644 --- a/spec/integration/outputs/index_version_spec.rb +++ b/spec/integration/outputs/index_version_spec.rb @@ -36,11 +36,11 @@ it "should default to ES version" do subject.multi_receive([LogStash::Event.new("my_id" => "123", "message" => "foo")]) - r = es.get(:index => 'logstash-index', :type => doc_type, :id => "123", :refresh => true) + r = es.get(generate_common_index_params('logstash-index', '123')) expect(r["_version"]).to eq(1) expect(r["_source"]["message"]).to eq('foo') subject.multi_receive([LogStash::Event.new("my_id" => "123", "message" => "foobar")]) - r2 = es.get(:index => 'logstash-index', :type => doc_type, :id => "123", :refresh => true) + r2 = es.get(generate_common_index_params('logstash-index', '123')) expect(r2["_version"]).to eq(2) expect(r2["_source"]["message"]).to eq('foobar') end @@ -63,7 +63,7 @@ it "should respect the external version" do id = "ev1" subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")]) - r = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true) + r = es.get(generate_common_index_params('logstash-index', id)) expect(r["_version"]).to eq(99) expect(r["_source"]["message"]).to eq('foo') end @@ -71,12 +71,12 @@ it "should ignore non-monotonic external version updates" do id = "ev2" subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")]) - r = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true) + r = es.get(generate_common_index_params('logstash-index', id)) expect(r["_version"]).to eq(99) expect(r["_source"]["message"]).to eq('foo') subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "98", "message" => "foo")]) - r2 = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true) + r2 = es.get(generate_common_index_params('logstash-index', id)) expect(r2["_version"]).to eq(99) expect(r2["_source"]["message"]).to eq('foo') end @@ -84,12 +84,12 @@ it "should commit monotonic external version updates" do id = "ev3" subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")]) - r = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true) + r = es.get(generate_common_index_params('logstash-index', id)) expect(r["_version"]).to eq(99) expect(r["_source"]["message"]).to eq('foo') subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "100", "message" => "foo")]) - r2 = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true) + r2 = es.get(generate_common_index_params('logstash-index', id)) expect(r2["_version"]).to eq(100) expect(r2["_source"]["message"]).to eq('foo') end diff --git a/spec/integration/outputs/painless_update_spec.rb b/spec/integration/outputs/painless_update_spec.rb index 02116fa51..cd2763177 100644 --- a/spec/integration/outputs/painless_update_spec.rb +++ b/spec/integration/outputs/painless_update_spec.rb @@ -21,12 +21,9 @@ def get_es_output( options={} ) @es.indices.delete_template(:name => "*") # This can fail if there are no indexes, ignore failure. @es.indices.delete(:index => "*") rescue nil - @es.index( - :index => 'logstash-update', - :type => doc_type, - :id => "123", - :body => { :message => 'Test', :counter => 1 } - ) + params = generate_common_index_params('logstash-update', '123') + params[:body] = { :message => 'Test', :counter => 1 } + @es.index(params) @es.indices.refresh end @@ -46,7 +43,7 @@ def get_es_output( options={} ) subject = get_es_output(plugin_parameters) subject.register subject.multi_receive([LogStash::Event.new("count" => 4 )]) - r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true) + r = @es.get(:index => 'logstash-update', :id => "123", :refresh => true) expect(r["_source"]["counter"]).to eq(5) end end @@ -57,7 +54,7 @@ def get_es_output( options={} ) subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' }) subject.register subject.multi_receive([LogStash::Event.new("message" => "sample message here")]) - r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true) + r = @es.get(:index => 'logstash-update', :id => "456", :refresh => true) expect(r["_source"]["message"]).to eq('upsert message') end @@ -65,7 +62,7 @@ def get_es_output( options={} ) subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true }) subject.register subject.multi_receive([LogStash::Event.new("message" => "sample message here")]) - r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true) + r = @es.get(:index => 'logstash-update', :id => "456", :refresh => true) expect(r["_source"]["message"]).to eq('sample message here') end @@ -82,7 +79,7 @@ def get_es_output( options={} ) subject.register subject.multi_receive([LogStash::Event.new("message" => "sample message here")]) - r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true) + r = @es.get(:index => 'logstash-update', :id => "456", :refresh => true) expect(r["_source"]["message"]).to eq('upsert message') end @@ -91,7 +88,7 @@ def get_es_output( options={} ) subject.register subject.multi_receive([LogStash::Event.new("counter" => 1)]) @es.indices.refresh - r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true) + r = @es.get(:index => 'logstash-update', :id => "456", :refresh => true) expect(r["_source"]["counter"]).to eq(1) end end diff --git a/spec/integration/outputs/unsupported_actions_spec.rb b/spec/integration/outputs/unsupported_actions_spec.rb index 1ac4f9e3e..fc2ad0266 100644 --- a/spec/integration/outputs/unsupported_actions_spec.rb +++ b/spec/integration/outputs/unsupported_actions_spec.rb @@ -26,18 +26,13 @@ def get_es_output( options={} ) # This can fail if there are no indexes, ignore failure. @es.indices.delete(:index => "*") rescue nil # index single doc for update purpose - @es.index( - :index => INDEX, - :type => doc_type, - :id => "2", - :body => { :message => 'Test to doc indexing', :counter => 1 } - ) - @es.index( - :index => INDEX, - :type => doc_type, - :id => "3", - :body => { :message => 'Test to doc deletion', :counter => 2 } - ) + params_index = generate_common_index_params(INDEX, '2') + params_index[:body] = { :message => 'Test to doc indexing', :counter => 1 } + @es.index(params_index) + + params_delete = generate_common_index_params(INDEX, '3') + params_delete[:body] = { :message => 'Test to doc deletion', :counter => 2 } + @es.index(params_delete) @es.indices.refresh end @@ -63,12 +58,12 @@ def get_es_output( options={} ) rejected_events = events.select { |event| !index_or_update.call(event) } indexed_events.each do |event| - response = @es.get(:index => INDEX, :type => doc_type, :id => event.get("doc_id"), :refresh => true) + response = @es.get(generate_common_index_params(INDEX, event.get("doc_id"))) expect(response['_source']['message']).to eq(event.get("message")) end rejected_events.each do |event| - expect {@es.get(:index => INDEX, :type => doc_type, :id => event.get("doc_id"), :refresh => true)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) + expect {@es.get(generate_common_index_params(INDEX, event.get("doc_id")))}.to raise_error(get_expected_error_class) end end end diff --git a/spec/integration/outputs/update_spec.rb b/spec/integration/outputs/update_spec.rb index 4a86dc77e..38ad6aa2c 100644 --- a/spec/integration/outputs/update_spec.rb +++ b/spec/integration/outputs/update_spec.rb @@ -21,12 +21,9 @@ def get_es_output( options={} ) @es.indices.delete_template(:name => "*") # This can fail if there are no indexes, ignore failure. @es.indices.delete(:index => "*") rescue nil - @es.index( - :index => 'logstash-update', - :type => doc_type, - :id => "123", - :body => { :message => 'Test', :counter => 1 } - ) + params = generate_common_index_params('logstash-update', '123') + params[:body] = { :message => 'Test', :counter => 1 } + @es.index(params) @es.indices.refresh end @@ -40,14 +37,14 @@ def get_es_output( options={} ) subject = get_es_output({ 'document_id' => "456" } ) subject.register subject.multi_receive([LogStash::Event.new("message" => "sample message here")]) - expect {@es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) + expect {@es.get(generate_common_index_params('logstash-update', '456'))}.to raise_error(get_expected_error_class) end it "should update existing document" do subject = get_es_output({ 'document_id' => "123" }) subject.register subject.multi_receive([LogStash::Event.new("message" => "updated message here")]) - r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true) + r = @es.get(generate_common_index_params('logstash-update', '123')) expect(r["_source"]["message"]).to eq('updated message here') end @@ -57,7 +54,7 @@ def get_es_output( options={} ) subject = get_es_output({ 'document_id' => "123" }) subject.register subject.multi_receive([LogStash::Event.new("data" => "updated message here", "message" => "foo")]) - r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true) + r = @es.get(generate_common_index_params('logstash-update', '123')) expect(r["_source"]["data"]).to eq('updated message here') expect(r["_source"]["message"]).to eq('foo') end @@ -94,7 +91,7 @@ def get_es_output( options={} ) subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' }) subject.register subject.multi_receive([LogStash::Event.new("message" => "sample message here")]) - r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true) + r = @es.get(generate_common_index_params('logstash-update', '456')) expect(r["_source"]["message"]).to eq('upsert message') end @@ -102,7 +99,7 @@ def get_es_output( options={} ) subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true }) subject.register subject.multi_receive([LogStash::Event.new("message" => "sample message here")]) - r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true) + r = @es.get(generate_common_index_params('logstash-update', '456')) expect(r["_source"]["message"]).to eq('sample message here') end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 207d83d5b..5b28d9db5 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -8,3 +8,17 @@ module LogStash::Outputs::ElasticSearch::SpecHelper RSpec.configure do |config| config.include LogStash::Outputs::ElasticSearch::SpecHelper end + + +def elastic_ruby_v8_client_available? + Elasticsearch::Transport + false +rescue NameError # NameError: uninitialized constant Elasticsearch::Transport if Elastic Ruby client is not available + true +end + +def generate_common_index_params(index, doc_id) + params = {:index => index, :id => doc_id, :refresh => true} + params[:type] = doc_type unless elastic_ruby_v8_client_available? + params +end \ No newline at end of file diff --git a/spec/support/elasticsearch/api/actions/put_ilm_policy.rb b/spec/support/elasticsearch/api/actions/put_ilm_policy.rb index e670366b8..5164e22ac 100644 --- a/spec/support/elasticsearch/api/actions/put_ilm_policy.rb +++ b/spec/support/elasticsearch/api/actions/put_ilm_policy.rb @@ -15,7 +15,7 @@ def put_ilm_policy(arguments={}) method = HTTP_PUT path = Utils.__pathify '_ilm/policy/', Utils.__escape(arguments[:name]) - params = Utils.__validate_and_extract_params arguments + params = elastic_ruby_v8_client_available?? Utils.process_params(arguments): Utils.__validate_and_extract_params(arguments) body = arguments[:body] perform_request(method, path, params, body.to_json).body diff --git a/spec/unit/outputs/error_whitelist_spec.rb b/spec/unit/outputs/error_whitelist_spec.rb index 9780376fd..e6cbbcfff 100644 --- a/spec/unit/outputs/error_whitelist_spec.rb +++ b/spec/unit/outputs/error_whitelist_spec.rb @@ -4,7 +4,6 @@ describe "whitelisting error types in expected behavior" do let(:template) { '{"template" : "not important, will be updated by :index"}' } let(:event1) { LogStash::Event.new("somevalue" => 100, "@timestamp" => "2014-11-17T20:37:17.223Z") } - let(:action1) { ["index", {:_id=>1, :routing=>nil, :_index=>"logstash-2014.11.17", :_type=> doc_type }, event1] } let(:settings) { {"manage_template" => true, "index" => "logstash-2014.11.17", "template_overwrite" => true, "hosts" => get_host_port() } } subject { LogStash::Outputs::ElasticSearch.new(settings) } From 8c7305608c9cd235dabac58b3ca5d7b4307996ad Mon Sep 17 00:00:00 2001 From: Mashhur Date: Thu, 20 Mar 2025 01:40:07 -0700 Subject: [PATCH 2/6] Fix the ILM spec issue where method was removed, restored internally. --- spec/spec_helper.rb | 19 ++++++++++++++++ .../elasticsearch/api/actions/put_alias.rb | 2 +- .../api/actions/put_ilm_policy.rb | 22 ++++++++++++++++--- 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 5b28d9db5..4d1790ce0 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -21,4 +21,23 @@ def generate_common_index_params(index, doc_id) params = {:index => index, :id => doc_id, :refresh => true} params[:type] = doc_type unless elastic_ruby_v8_client_available? params +end + + +COMMON_QUERY_PARAMS = [ + :ignore, # Client specific parameters + :format, # Search, Cat, ... + :pretty, # Pretty-print the response + :human, # Return numeric values in human readable format + :filter_path # Filter the JSON response +] + +# This method was removed from elasticsearch-ruby client v8 +# Copied from elasticsearch-ruby v7 client to make it available +# +def __extract_params(arguments, params=[], options={}) + result = arguments.select { |k,v| COMMON_QUERY_PARAMS.include?(k) || params.include?(k) } + result = Hash[result] unless result.is_a?(Hash) # Normalize Ruby 1.8 and Ruby 1.9 Hash#select behaviour + result = Hash[result.map { |k,v| v.is_a?(Array) ? [k, Utils.__listify(v, options)] : [k,v] }] # Listify Arrays + result end \ No newline at end of file diff --git a/spec/support/elasticsearch/api/actions/put_alias.rb b/spec/support/elasticsearch/api/actions/put_alias.rb index d0585934f..92afe9b27 100644 --- a/spec/support/elasticsearch/api/actions/put_alias.rb +++ b/spec/support/elasticsearch/api/actions/put_alias.rb @@ -15,7 +15,7 @@ def put_alias(arguments={}) method = HTTP_PUT path = Utils.__pathify Utils.__escape(arguments[:name]) - params = Utils.__validate_and_extract_params arguments + params = __extract_params(arguments) body = arguments[:body] perform_request(method, path, params, body.to_json).body end diff --git a/spec/support/elasticsearch/api/actions/put_ilm_policy.rb b/spec/support/elasticsearch/api/actions/put_ilm_policy.rb index 5164e22ac..7268c191a 100644 --- a/spec/support/elasticsearch/api/actions/put_ilm_policy.rb +++ b/spec/support/elasticsearch/api/actions/put_ilm_policy.rb @@ -6,6 +6,14 @@ module Elasticsearch module API module Actions + COMMON_QUERY_PARAMS = [ + :ignore, # Client specific parameters + :format, # Search, Cat, ... + :pretty, # Pretty-print the response + :human, # Return numeric values in human readable format + :filter_path # Filter the JSON response + ] + # @option arguments [String] :name The name of the policy (*Required*) # @option arguments [Hash] :body The policy definition (*Required*) @@ -14,12 +22,20 @@ def put_ilm_policy(arguments={}) raise ArgumentError, "Required argument 'body' missing" unless arguments[:body] method = HTTP_PUT path = Utils.__pathify '_ilm/policy/', Utils.__escape(arguments[:name]) - - params = elastic_ruby_v8_client_available?? Utils.process_params(arguments): Utils.__validate_and_extract_params(arguments) - + params = __extract_params(arguments) body = arguments[:body] perform_request(method, path, params, body.to_json).body end + + # This method was removed from elasticsearch-ruby client v8 + # Copied from elasticsearch-ruby v7 client to make it available + # + def __extract_params(arguments, params=[], options={}) + result = arguments.select { |k,v| COMMON_QUERY_PARAMS.include?(k) || params.include?(k) } + result = Hash[result] unless result.is_a?(Hash) # Normalize Ruby 1.8 and Ruby 1.9 Hash#select behaviour + result = Hash[result.map { |k,v| v.is_a?(Array) ? [k, Utils.__listify(v, options)] : [k,v] }] # Listify Arrays + result + end end end end From 136971670a3e00c6b2d9d66837a0685c6f125814 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Duarte?= Date: Thu, 20 Mar 2025 16:37:16 +0000 Subject: [PATCH 3/6] fix tests to not require custom es ruby code --- spec/es_spec_helper.rb | 24 ++--------- spec/integration/outputs/ilm_spec.rb | 15 +++---- .../api/actions/delete_ilm_policy.rb | 19 --------- .../elasticsearch/api/actions/get_alias.rb | 18 -------- .../api/actions/get_ilm_policy.rb | 18 -------- .../elasticsearch/api/actions/put_alias.rb | 24 ----------- .../api/actions/put_ilm_policy.rb | 41 ------------------- 7 files changed, 12 insertions(+), 147 deletions(-) delete mode 100644 spec/support/elasticsearch/api/actions/delete_ilm_policy.rb delete mode 100644 spec/support/elasticsearch/api/actions/get_alias.rb delete mode 100644 spec/support/elasticsearch/api/actions/get_ilm_policy.rb delete mode 100644 spec/support/elasticsearch/api/actions/put_alias.rb delete mode 100644 spec/support/elasticsearch/api/actions/put_ilm_policy.rb diff --git a/spec/es_spec_helper.rb b/spec/es_spec_helper.rb index 4255d3cd4..791576886 100644 --- a/spec/es_spec_helper.rb +++ b/spec/es_spec_helper.rb @@ -1,11 +1,6 @@ require_relative './spec_helper' require 'elasticsearch' -require_relative "support/elasticsearch/api/actions/delete_ilm_policy" -require_relative "support/elasticsearch/api/actions/get_alias" -require_relative "support/elasticsearch/api/actions/put_alias" -require_relative "support/elasticsearch/api/actions/get_ilm_policy" -require_relative "support/elasticsearch/api/actions/put_ilm_policy" require 'json' require 'cabin' @@ -130,31 +125,20 @@ def get_cluster_settings(client) end def get_policy(client, policy_name) - client.get_ilm_policy(name: policy_name) + client.index_lifecycle_management.get_lifecycle(policy: policy_name) end def put_policy(client, policy_name, policy) - client.put_ilm_policy({:name => policy_name, :body=> policy}) - end - - def put_alias(client, the_alias, index) - body = { - "aliases" => { - index => { - "is_write_index"=> true - } - } - } - client.put_alias({name: the_alias, body: body}) + client.index_lifecycle_management.put_lifecycle({:policy => policy_name, :body=> policy}) end def clean_ilm(client) - client.get_ilm_policy.each_key { |key| client.delete_ilm_policy(name: key) if key =~ /logstash-policy/ } + client.index_lifecycle_management.get_lifecycle.each_key { |key| client.index_lifecycle_management.delete_lifecycle(policy: key) if key =~ /logstash-policy/ } end def supports_ilm?(client) begin - client.get_ilm_policy + client.index_lifecycle_management.get_lifecycle true rescue false diff --git a/spec/integration/outputs/ilm_spec.rb b/spec/integration/outputs/ilm_spec.rb index e288e5397..0262ab426 100644 --- a/spec/integration/outputs/ilm_spec.rb +++ b/spec/integration/outputs/ilm_spec.rb @@ -282,7 +282,7 @@ subject.register sleep(1) expect(@es.indices.exists_alias(name: "logstash")).to be_truthy - expect(@es.get_alias(name: "logstash")).to include("logstash-000001") + expect(@es.indices.get_alias(name: "logstash")).to include("logstash-000001") end end @@ -298,7 +298,7 @@ subject.register sleep(1) expect(@es.indices.exists_alias(name: "logstash")).to be_truthy - expect(@es.get_alias(name: "logstash")).to include("logstash-#{todays_date}-000001") + expect(@es.indices.get_alias(name: "logstash")).to include("logstash-#{todays_date}-000001") end it 'should ingest into a single index' do @@ -374,7 +374,7 @@ subject.register sleep(1) expect(@es.indices.exists_alias(name: expected_index)).to be_truthy - expect(@es.get_alias(name: expected_index)).to include("#{expected_index}-#{todays_date}-000001") + expect(@es.indices.get_alias(name: expected_index)).to include("#{expected_index}-#{todays_date}-000001") end it 'should write the ILM settings into the template' do @@ -443,17 +443,18 @@ subject.register sleep(1) expect(@es.indices.exists_alias(name: ilm_rollover_alias)).to be_truthy - expect(@es.get_alias(name: ilm_rollover_alias)).to include("#{ilm_rollover_alias}-#{todays_date}-000001") + expect(@es.indices.get_alias(name: ilm_rollover_alias)).to include("#{ilm_rollover_alias}-#{todays_date}-000001") end context 'when the custom rollover alias already exists' do it 'should ignore the already exists error' do expect(@es.indices.exists_alias(name: ilm_rollover_alias)).to be_falsey - put_alias(@es, "#{ilm_rollover_alias}-#{todays_date}-000001", ilm_rollover_alias) + @es.indices.create(index: "#{ilm_rollover_alias}-#{todays_date}-000001") + @es.indices.put_alias(name: ilm_rollover_alias, index: "#{ilm_rollover_alias}-#{todays_date}-000001") expect(@es.indices.exists_alias(name: ilm_rollover_alias)).to be_truthy subject.register sleep(1) - expect(@es.get_alias(name: ilm_rollover_alias)).to include("#{ilm_rollover_alias}-#{todays_date}-000001") + expect(@es.indices.get_alias(name: ilm_rollover_alias)).to include("#{ilm_rollover_alias}-#{todays_date}-000001") end end @@ -536,4 +537,4 @@ def get_expected_error_class return Elastic::Transport::Transport::Errors::NotFound if elastic_ruby_v8_client_available? Elasticsearch::Transport::Transport::Errors::NotFound -end \ No newline at end of file +end diff --git a/spec/support/elasticsearch/api/actions/delete_ilm_policy.rb b/spec/support/elasticsearch/api/actions/delete_ilm_policy.rb deleted file mode 100644 index 83c66f3c7..000000000 --- a/spec/support/elasticsearch/api/actions/delete_ilm_policy.rb +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License; -# you may not use this file except in compliance with the Elastic License. - -module Elasticsearch - module API - module Actions - - # Update the password of the specified user - def delete_ilm_policy(arguments={}) - method = HTTP_DELETE - path = Utils.__pathify '_ilm/policy/', - Utils.__escape(arguments[:name]) - params = {} - perform_request(method, path, params, nil).body - end - end - end -end diff --git a/spec/support/elasticsearch/api/actions/get_alias.rb b/spec/support/elasticsearch/api/actions/get_alias.rb deleted file mode 100644 index ef4ebbd4f..000000000 --- a/spec/support/elasticsearch/api/actions/get_alias.rb +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License; -# you may not use this file except in compliance with the Elastic License. - -module Elasticsearch - module API - module Actions - - # Retrieve the list of index lifecycle management policies - def get_alias(arguments={}) - method = HTTP_GET - path = Utils.__pathify '_alias', Utils.__escape(arguments[:name]) - params = {} - perform_request(method, path, params, nil).body - end - end - end -end \ No newline at end of file diff --git a/spec/support/elasticsearch/api/actions/get_ilm_policy.rb b/spec/support/elasticsearch/api/actions/get_ilm_policy.rb deleted file mode 100644 index accf98466..000000000 --- a/spec/support/elasticsearch/api/actions/get_ilm_policy.rb +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License; -# you may not use this file except in compliance with the Elastic License. - -module Elasticsearch - module API - module Actions - - # Retrieve the list of index lifecycle management policies - def get_ilm_policy(arguments={}) - method = HTTP_GET - path = Utils.__pathify '_ilm/policy', Utils.__escape(arguments[:name]) - params = {} - perform_request(method, path, params, nil).body - end - end - end -end \ No newline at end of file diff --git a/spec/support/elasticsearch/api/actions/put_alias.rb b/spec/support/elasticsearch/api/actions/put_alias.rb deleted file mode 100644 index 92afe9b27..000000000 --- a/spec/support/elasticsearch/api/actions/put_alias.rb +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License; -# you may not use this file except in compliance with the Elastic License. - -module Elasticsearch - module API - module Actions - - # @option arguments [String] :name The name of the alias (*Required*) - # @option arguments [Hash] :The alias definition(*Required*) - - def put_alias(arguments={}) - raise ArgumentError, "Required argument 'name' missing" unless arguments[:name] - raise ArgumentError, "Required argument 'body' missing" unless arguments[:body] - method = HTTP_PUT - path = Utils.__pathify Utils.__escape(arguments[:name]) - - params = __extract_params(arguments) - body = arguments[:body] - perform_request(method, path, params, body.to_json).body - end - end - end -end diff --git a/spec/support/elasticsearch/api/actions/put_ilm_policy.rb b/spec/support/elasticsearch/api/actions/put_ilm_policy.rb deleted file mode 100644 index 7268c191a..000000000 --- a/spec/support/elasticsearch/api/actions/put_ilm_policy.rb +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License; -# you may not use this file except in compliance with the Elastic License. - -module Elasticsearch - module API - module Actions - - COMMON_QUERY_PARAMS = [ - :ignore, # Client specific parameters - :format, # Search, Cat, ... - :pretty, # Pretty-print the response - :human, # Return numeric values in human readable format - :filter_path # Filter the JSON response - ] - - # @option arguments [String] :name The name of the policy (*Required*) - # @option arguments [Hash] :body The policy definition (*Required*) - - def put_ilm_policy(arguments={}) - raise ArgumentError, "Required argument 'name' missing" unless arguments[:name] - raise ArgumentError, "Required argument 'body' missing" unless arguments[:body] - method = HTTP_PUT - path = Utils.__pathify '_ilm/policy/', Utils.__escape(arguments[:name]) - params = __extract_params(arguments) - body = arguments[:body] - perform_request(method, path, params, body.to_json).body - end - - # This method was removed from elasticsearch-ruby client v8 - # Copied from elasticsearch-ruby v7 client to make it available - # - def __extract_params(arguments, params=[], options={}) - result = arguments.select { |k,v| COMMON_QUERY_PARAMS.include?(k) || params.include?(k) } - result = Hash[result] unless result.is_a?(Hash) # Normalize Ruby 1.8 and Ruby 1.9 Hash#select behaviour - result = Hash[result.map { |k,v| v.is_a?(Array) ? [k, Utils.__listify(v, options)] : [k,v] }] # Listify Arrays - result - end - end - end -end From 755651a2a5415e8cf8bd5f891e06b6795c53b46c Mon Sep 17 00:00:00 2001 From: Mashhur Date: Fri, 21 Mar 2025 07:34:40 -0700 Subject: [PATCH 4/6] Simplifications. --- spec/integration/outputs/delete_spec.rb | 8 +++--- .../integration/outputs/index_version_spec.rb | 14 +++++----- .../outputs/painless_update_spec.rb | 10 ++++--- .../outputs/unsupported_actions_spec.rb | 26 +++++++++++++------ spec/integration/outputs/update_spec.rb | 21 +++++++++------ spec/spec_helper.rb | 25 ------------------ 6 files changed, 49 insertions(+), 55 deletions(-) diff --git a/spec/integration/outputs/delete_spec.rb b/spec/integration/outputs/delete_spec.rb index bc253832b..5d4104279 100644 --- a/spec/integration/outputs/delete_spec.rb +++ b/spec/integration/outputs/delete_spec.rb @@ -39,12 +39,12 @@ it "should ignore non-monotonic external version updates" do id = "ev2" subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "index", "message" => "foo", "my_version" => 99)]) - r = es.get(generate_common_index_params('logstash-delete', id)) + r = es.get(:index => 'logstash-delete', :id => id, :refresh => true) expect(r['_version']).to eq(99) expect(r['_source']['message']).to eq('foo') subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "delete", "message" => "foo", "my_version" => 98)]) - r2 = es.get(generate_common_index_params('logstash-delete', id)) + r2 = es.get(:index => 'logstash-delete', :id => id, :refresh => true) expect(r2['_version']).to eq(99) expect(r2['_source']['message']).to eq('foo') end @@ -52,12 +52,12 @@ it "should commit monotonic external version updates" do id = "ev3" subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "index", "message" => "foo", "my_version" => 99)]) - r = es.get(generate_common_index_params('logstash-delete', id)) + r = es.get(:index => 'logstash-delete', :id => id, :refresh => true) expect(r['_version']).to eq(99) expect(r['_source']['message']).to eq('foo') subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "delete", "message" => "foo", "my_version" => 100)]) - expect { es.get(generate_common_index_params('logstash-delete', id)) }.to raise_error(get_expected_error_class) + expect { es.get(:index => 'logstash-delete', :id => id, :refresh => true) }.to raise_error(get_expected_error_class) end end end diff --git a/spec/integration/outputs/index_version_spec.rb b/spec/integration/outputs/index_version_spec.rb index 44edce09c..d73872f49 100644 --- a/spec/integration/outputs/index_version_spec.rb +++ b/spec/integration/outputs/index_version_spec.rb @@ -36,11 +36,11 @@ it "should default to ES version" do subject.multi_receive([LogStash::Event.new("my_id" => "123", "message" => "foo")]) - r = es.get(generate_common_index_params('logstash-index', '123')) + r = es.get(:index => 'logstash-index', :id => '123', :refresh => true) expect(r["_version"]).to eq(1) expect(r["_source"]["message"]).to eq('foo') subject.multi_receive([LogStash::Event.new("my_id" => "123", "message" => "foobar")]) - r2 = es.get(generate_common_index_params('logstash-index', '123')) + r2 = es.get(:index => 'logstash-index', :id => '123', :refresh => true) expect(r2["_version"]).to eq(2) expect(r2["_source"]["message"]).to eq('foobar') end @@ -63,7 +63,7 @@ it "should respect the external version" do id = "ev1" subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")]) - r = es.get(generate_common_index_params('logstash-index', id)) + r = es.get(:index => 'logstash-index', :id => id, :refresh => true) expect(r["_version"]).to eq(99) expect(r["_source"]["message"]).to eq('foo') end @@ -71,12 +71,12 @@ it "should ignore non-monotonic external version updates" do id = "ev2" subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")]) - r = es.get(generate_common_index_params('logstash-index', id)) + r = es.get(:index => 'logstash-index', :id => id, :refresh => true) expect(r["_version"]).to eq(99) expect(r["_source"]["message"]).to eq('foo') subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "98", "message" => "foo")]) - r2 = es.get(generate_common_index_params('logstash-index', id)) + r2 = es.get(:index => 'logstash-index', :id => id, :refresh => true) expect(r2["_version"]).to eq(99) expect(r2["_source"]["message"]).to eq('foo') end @@ -84,12 +84,12 @@ it "should commit monotonic external version updates" do id = "ev3" subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")]) - r = es.get(generate_common_index_params('logstash-index', id)) + r = es.get(:index => 'logstash-index', :id => id, :refresh => true) expect(r["_version"]).to eq(99) expect(r["_source"]["message"]).to eq('foo') subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "100", "message" => "foo")]) - r2 = es.get(generate_common_index_params('logstash-index', id)) + r2 = es.get(:index => 'logstash-index', :id => id, :refresh => true) expect(r2["_version"]).to eq(100) expect(r2["_source"]["message"]).to eq('foo') end diff --git a/spec/integration/outputs/painless_update_spec.rb b/spec/integration/outputs/painless_update_spec.rb index cd2763177..0f4eabe06 100644 --- a/spec/integration/outputs/painless_update_spec.rb +++ b/spec/integration/outputs/painless_update_spec.rb @@ -21,9 +21,13 @@ def get_es_output( options={} ) @es.indices.delete_template(:name => "*") # This can fail if there are no indexes, ignore failure. @es.indices.delete(:index => "*") rescue nil - params = generate_common_index_params('logstash-update', '123') - params[:body] = { :message => 'Test', :counter => 1 } - @es.index(params) + @es.index( + { + :index => 'logstash-update', + :id => '123', + :body => { :message => 'Test', :counter => 1 }, + :refresh => true + }) @es.indices.refresh end diff --git a/spec/integration/outputs/unsupported_actions_spec.rb b/spec/integration/outputs/unsupported_actions_spec.rb index fc2ad0266..c3b12f713 100644 --- a/spec/integration/outputs/unsupported_actions_spec.rb +++ b/spec/integration/outputs/unsupported_actions_spec.rb @@ -26,13 +26,23 @@ def get_es_output( options={} ) # This can fail if there are no indexes, ignore failure. @es.indices.delete(:index => "*") rescue nil # index single doc for update purpose - params_index = generate_common_index_params(INDEX, '2') - params_index[:body] = { :message => 'Test to doc indexing', :counter => 1 } - @es.index(params_index) + @es.index( + { + :index => INDEX, + :id => '2', + :body => { :message => 'Test to doc indexing', :counter => 1 }, + :refresh => true + } + ) - params_delete = generate_common_index_params(INDEX, '3') - params_delete[:body] = { :message => 'Test to doc deletion', :counter => 2 } - @es.index(params_delete) + @es.index( + { + :index => INDEX, + :id => '3', + :body => { :message => 'Test to doc deletion', :counter => 2 }, + :refresh => true + } + ) @es.indices.refresh end @@ -58,12 +68,12 @@ def get_es_output( options={} ) rejected_events = events.select { |event| !index_or_update.call(event) } indexed_events.each do |event| - response = @es.get(generate_common_index_params(INDEX, event.get("doc_id"))) + response = @es.get(:index => INDEX, :id => event.get("doc_id"), :refresh => true) expect(response['_source']['message']).to eq(event.get("message")) end rejected_events.each do |event| - expect {@es.get(generate_common_index_params(INDEX, event.get("doc_id")))}.to raise_error(get_expected_error_class) + expect {@es.get(:index => INDEX, :id => event.get("doc_id"), :refresh => true)}.to raise_error(get_expected_error_class) end end end diff --git a/spec/integration/outputs/update_spec.rb b/spec/integration/outputs/update_spec.rb index 38ad6aa2c..1c28f6666 100644 --- a/spec/integration/outputs/update_spec.rb +++ b/spec/integration/outputs/update_spec.rb @@ -21,9 +21,14 @@ def get_es_output( options={} ) @es.indices.delete_template(:name => "*") # This can fail if there are no indexes, ignore failure. @es.indices.delete(:index => "*") rescue nil - params = generate_common_index_params('logstash-update', '123') - params[:body] = { :message => 'Test', :counter => 1 } - @es.index(params) + @es.index( + { + :index => 'logstash-update', + :id => '123', + :body => { :message => 'Test', :counter => 1 }, + :refresh => true + } + ) @es.indices.refresh end @@ -37,14 +42,14 @@ def get_es_output( options={} ) subject = get_es_output({ 'document_id' => "456" } ) subject.register subject.multi_receive([LogStash::Event.new("message" => "sample message here")]) - expect {@es.get(generate_common_index_params('logstash-update', '456'))}.to raise_error(get_expected_error_class) + expect {@es.get(:index => 'logstash-update', :id => '456', :refresh => true)}.to raise_error(get_expected_error_class) end it "should update existing document" do subject = get_es_output({ 'document_id' => "123" }) subject.register subject.multi_receive([LogStash::Event.new("message" => "updated message here")]) - r = @es.get(generate_common_index_params('logstash-update', '123')) + r = @es.get(:index => 'logstash-update', :id => '123', :refresh => true) expect(r["_source"]["message"]).to eq('updated message here') end @@ -54,7 +59,7 @@ def get_es_output( options={} ) subject = get_es_output({ 'document_id' => "123" }) subject.register subject.multi_receive([LogStash::Event.new("data" => "updated message here", "message" => "foo")]) - r = @es.get(generate_common_index_params('logstash-update', '123')) + r = @es.get(:index => 'logstash-update', :id => '123', :refresh => true) expect(r["_source"]["data"]).to eq('updated message here') expect(r["_source"]["message"]).to eq('foo') end @@ -91,7 +96,7 @@ def get_es_output( options={} ) subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' }) subject.register subject.multi_receive([LogStash::Event.new("message" => "sample message here")]) - r = @es.get(generate_common_index_params('logstash-update', '456')) + r = @es.get(:index => 'logstash-update', :id => '456', :refresh => true) expect(r["_source"]["message"]).to eq('upsert message') end @@ -99,7 +104,7 @@ def get_es_output( options={} ) subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true }) subject.register subject.multi_receive([LogStash::Event.new("message" => "sample message here")]) - r = @es.get(generate_common_index_params('logstash-update', '456')) + r = @es.get(:index => 'logstash-update', :id => '456', :refresh => true) expect(r["_source"]["message"]).to eq('sample message here') end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 4d1790ce0..46c97f097 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -15,29 +15,4 @@ def elastic_ruby_v8_client_available? false rescue NameError # NameError: uninitialized constant Elasticsearch::Transport if Elastic Ruby client is not available true -end - -def generate_common_index_params(index, doc_id) - params = {:index => index, :id => doc_id, :refresh => true} - params[:type] = doc_type unless elastic_ruby_v8_client_available? - params -end - - -COMMON_QUERY_PARAMS = [ - :ignore, # Client specific parameters - :format, # Search, Cat, ... - :pretty, # Pretty-print the response - :human, # Return numeric values in human readable format - :filter_path # Filter the JSON response -] - -# This method was removed from elasticsearch-ruby client v8 -# Copied from elasticsearch-ruby v7 client to make it available -# -def __extract_params(arguments, params=[], options={}) - result = arguments.select { |k,v| COMMON_QUERY_PARAMS.include?(k) || params.include?(k) } - result = Hash[result] unless result.is_a?(Hash) # Normalize Ruby 1.8 and Ruby 1.9 Hash#select behaviour - result = Hash[result.map { |k,v| v.is_a?(Array) ? [k, Utils.__listify(v, options)] : [k,v] }] # Listify Arrays - result end \ No newline at end of file From c1685a317772c5b0ab8c8a3c7a1d8cb93bc5e7b2 Mon Sep 17 00:00:00 2001 From: Mashhur Date: Fri, 21 Mar 2025 08:42:02 -0700 Subject: [PATCH 5/6] Put pack removed support package and refine the logics to remove v7 sources easily in the future. --- spec/es_spec_helper.rb | 75 +++++++++++++++---- spec/spec_helper.rb | 2 +- .../api/actions/delete_ilm_policy.rb | 19 +++++ .../elasticsearch/api/actions/get_alias.rb | 18 +++++ .../api/actions/get_ilm_policy.rb | 18 +++++ .../elasticsearch/api/actions/put_alias.rb | 24 ++++++ .../api/actions/put_ilm_policy.rb | 25 +++++++ 7 files changed, 165 insertions(+), 16 deletions(-) create mode 100644 spec/support/elasticsearch/api/actions/delete_ilm_policy.rb create mode 100644 spec/support/elasticsearch/api/actions/get_alias.rb create mode 100644 spec/support/elasticsearch/api/actions/get_ilm_policy.rb create mode 100644 spec/support/elasticsearch/api/actions/put_alias.rb create mode 100644 spec/support/elasticsearch/api/actions/put_ilm_policy.rb diff --git a/spec/es_spec_helper.rb b/spec/es_spec_helper.rb index 791576886..bbab70e73 100644 --- a/spec/es_spec_helper.rb +++ b/spec/es_spec_helper.rb @@ -5,6 +5,16 @@ require 'json' require 'cabin' +# remove this condition and support package once plugin starts consuming elasticsearch-ruby v8 client +# in elasticsearch-ruby v7, ILM APIs were in a separate xpack gem, now directly available +unless elastic_ruby_v8_client_available? + require_relative "support/elasticsearch/api/actions/delete_ilm_policy" + require_relative "support/elasticsearch/api/actions/get_alias" + require_relative "support/elasticsearch/api/actions/put_alias" + require_relative "support/elasticsearch/api/actions/get_ilm_policy" + require_relative "support/elasticsearch/api/actions/put_ilm_policy" +end + module ESHelper def get_host_port if ENV["INTEGRATION"] == "true" @@ -124,24 +134,59 @@ def get_cluster_settings(client) client.cluster.get_settings end - def get_policy(client, policy_name) - client.index_lifecycle_management.get_lifecycle(policy: policy_name) - end + # remove else condition once plugin starts consuming elasticsearch-ruby v8 client + if elastic_ruby_v8_client_available? + def get_policy(client, policy_name) + client.index_lifecycle_management.get_lifecycle(policy: policy_name) + end - def put_policy(client, policy_name, policy) - client.index_lifecycle_management.put_lifecycle({:policy => policy_name, :body=> policy}) - end + def put_policy(client, policy_name, policy) + client.index_lifecycle_management.put_lifecycle({:policy => policy_name, :body=> policy}) + end - def clean_ilm(client) - client.index_lifecycle_management.get_lifecycle.each_key { |key| client.index_lifecycle_management.delete_lifecycle(policy: key) if key =~ /logstash-policy/ } - end + def clean_ilm(client) + client.index_lifecycle_management.get_lifecycle.each_key { |key| client.index_lifecycle_management.delete_lifecycle(policy: key) if key =~ /logstash-policy/ } + end + + def supports_ilm?(client) + begin + client.index_lifecycle_management.get_lifecycle + true + rescue + false + end + end + else + def get_policy(client, policy_name) + client.get_ilm_policy(name: policy_name) + end + + def put_policy(client, policy_name, policy) + client.put_ilm_policy({:name => policy_name, :body=> policy}) + end - def supports_ilm?(client) - begin - client.index_lifecycle_management.get_lifecycle - true - rescue - false + def clean_ilm(client) + client.get_ilm_policy.each_key { |key| client.delete_ilm_policy(name: key) if key =~ /logstash-policy/ } + end + + def supports_ilm?(client) + begin + client.get_ilm_policy + true + rescue + false + end + end + + def put_alias(client, the_alias, index) + body = { + "aliases" => { + index => { + "is_write_index"=> true + } + } + } + client.put_alias({name: the_alias, body: body}) end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 46c97f097..e84fe7617 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -9,7 +9,7 @@ module LogStash::Outputs::ElasticSearch::SpecHelper config.include LogStash::Outputs::ElasticSearch::SpecHelper end - +# remove once plugin starts consuming elasticsearch-ruby v8 client def elastic_ruby_v8_client_available? Elasticsearch::Transport false diff --git a/spec/support/elasticsearch/api/actions/delete_ilm_policy.rb b/spec/support/elasticsearch/api/actions/delete_ilm_policy.rb new file mode 100644 index 000000000..83c66f3c7 --- /dev/null +++ b/spec/support/elasticsearch/api/actions/delete_ilm_policy.rb @@ -0,0 +1,19 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. + +module Elasticsearch + module API + module Actions + + # Update the password of the specified user + def delete_ilm_policy(arguments={}) + method = HTTP_DELETE + path = Utils.__pathify '_ilm/policy/', + Utils.__escape(arguments[:name]) + params = {} + perform_request(method, path, params, nil).body + end + end + end +end diff --git a/spec/support/elasticsearch/api/actions/get_alias.rb b/spec/support/elasticsearch/api/actions/get_alias.rb new file mode 100644 index 000000000..ef4ebbd4f --- /dev/null +++ b/spec/support/elasticsearch/api/actions/get_alias.rb @@ -0,0 +1,18 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. + +module Elasticsearch + module API + module Actions + + # Retrieve the list of index lifecycle management policies + def get_alias(arguments={}) + method = HTTP_GET + path = Utils.__pathify '_alias', Utils.__escape(arguments[:name]) + params = {} + perform_request(method, path, params, nil).body + end + end + end +end \ No newline at end of file diff --git a/spec/support/elasticsearch/api/actions/get_ilm_policy.rb b/spec/support/elasticsearch/api/actions/get_ilm_policy.rb new file mode 100644 index 000000000..accf98466 --- /dev/null +++ b/spec/support/elasticsearch/api/actions/get_ilm_policy.rb @@ -0,0 +1,18 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. + +module Elasticsearch + module API + module Actions + + # Retrieve the list of index lifecycle management policies + def get_ilm_policy(arguments={}) + method = HTTP_GET + path = Utils.__pathify '_ilm/policy', Utils.__escape(arguments[:name]) + params = {} + perform_request(method, path, params, nil).body + end + end + end +end \ No newline at end of file diff --git a/spec/support/elasticsearch/api/actions/put_alias.rb b/spec/support/elasticsearch/api/actions/put_alias.rb new file mode 100644 index 000000000..d0585934f --- /dev/null +++ b/spec/support/elasticsearch/api/actions/put_alias.rb @@ -0,0 +1,24 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. + +module Elasticsearch + module API + module Actions + + # @option arguments [String] :name The name of the alias (*Required*) + # @option arguments [Hash] :The alias definition(*Required*) + + def put_alias(arguments={}) + raise ArgumentError, "Required argument 'name' missing" unless arguments[:name] + raise ArgumentError, "Required argument 'body' missing" unless arguments[:body] + method = HTTP_PUT + path = Utils.__pathify Utils.__escape(arguments[:name]) + + params = Utils.__validate_and_extract_params arguments + body = arguments[:body] + perform_request(method, path, params, body.to_json).body + end + end + end +end diff --git a/spec/support/elasticsearch/api/actions/put_ilm_policy.rb b/spec/support/elasticsearch/api/actions/put_ilm_policy.rb new file mode 100644 index 000000000..e670366b8 --- /dev/null +++ b/spec/support/elasticsearch/api/actions/put_ilm_policy.rb @@ -0,0 +1,25 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. + +module Elasticsearch + module API + module Actions + + # @option arguments [String] :name The name of the policy (*Required*) + # @option arguments [Hash] :body The policy definition (*Required*) + + def put_ilm_policy(arguments={}) + raise ArgumentError, "Required argument 'name' missing" unless arguments[:name] + raise ArgumentError, "Required argument 'body' missing" unless arguments[:body] + method = HTTP_PUT + path = Utils.__pathify '_ilm/policy/', Utils.__escape(arguments[:name]) + + params = Utils.__validate_and_extract_params arguments + + body = arguments[:body] + perform_request(method, path, params, body.to_json).body + end + end + end +end From 57ac75f75f23a11cccfad172f22cc48e66b3ff7d Mon Sep 17 00:00:00 2001 From: Mashhur Date: Fri, 21 Mar 2025 11:06:33 -0700 Subject: [PATCH 6/6] Remove unsused modules. --- spec/es_spec_helper.rb | 67 +++++++------------ .../elasticsearch/api/actions/get_alias.rb | 18 ----- .../elasticsearch/api/actions/put_alias.rb | 24 ------- 3 files changed, 23 insertions(+), 86 deletions(-) delete mode 100644 spec/support/elasticsearch/api/actions/get_alias.rb delete mode 100644 spec/support/elasticsearch/api/actions/put_alias.rb diff --git a/spec/es_spec_helper.rb b/spec/es_spec_helper.rb index bbab70e73..84ebe3ee9 100644 --- a/spec/es_spec_helper.rb +++ b/spec/es_spec_helper.rb @@ -9,8 +9,6 @@ # in elasticsearch-ruby v7, ILM APIs were in a separate xpack gem, now directly available unless elastic_ruby_v8_client_available? require_relative "support/elasticsearch/api/actions/delete_ilm_policy" - require_relative "support/elasticsearch/api/actions/get_alias" - require_relative "support/elasticsearch/api/actions/put_alias" require_relative "support/elasticsearch/api/actions/get_ilm_policy" require_relative "support/elasticsearch/api/actions/put_ilm_policy" end @@ -134,59 +132,40 @@ def get_cluster_settings(client) client.cluster.get_settings end - # remove else condition once plugin starts consuming elasticsearch-ruby v8 client - if elastic_ruby_v8_client_available? - def get_policy(client, policy_name) + def get_policy(client, policy_name) + if elastic_ruby_v8_client_available? client.index_lifecycle_management.get_lifecycle(policy: policy_name) - end - - def put_policy(client, policy_name, policy) - client.index_lifecycle_management.put_lifecycle({:policy => policy_name, :body=> policy}) - end - - def clean_ilm(client) - client.index_lifecycle_management.get_lifecycle.each_key { |key| client.index_lifecycle_management.delete_lifecycle(policy: key) if key =~ /logstash-policy/ } - end - - def supports_ilm?(client) - begin - client.index_lifecycle_management.get_lifecycle - true - rescue - false - end - end - else - def get_policy(client, policy_name) + else client.get_ilm_policy(name: policy_name) end + end - def put_policy(client, policy_name, policy) + def put_policy(client, policy_name, policy) + if elastic_ruby_v8_client_available? + client.index_lifecycle_management.put_lifecycle({:policy => policy_name, :body=> policy}) + else client.put_ilm_policy({:name => policy_name, :body=> policy}) end + end - def clean_ilm(client) - client.get_ilm_policy.each_key { |key| client.delete_ilm_policy(name: key) if key =~ /logstash-policy/ } + def clean_ilm(client) + if elastic_ruby_v8_client_available? + client.index_lifecycle_management.get_lifecycle.each_key { |key| client.index_lifecycle_management.delete_lifecycle(policy: key) if key =~ /logstash-policy/ } + else + client.get_ilm_policy.each_key { |key| client.delete_ilm_policy(name: key) if key =~ /logstash-policy/ } end + end - def supports_ilm?(client) - begin + def supports_ilm?(client) + begin + if elastic_ruby_v8_client_available? + client.index_lifecycle_management.get_lifecycle + else client.get_ilm_policy - true - rescue - false end - end - - def put_alias(client, the_alias, index) - body = { - "aliases" => { - index => { - "is_write_index"=> true - } - } - } - client.put_alias({name: the_alias, body: body}) + true + rescue + false end end diff --git a/spec/support/elasticsearch/api/actions/get_alias.rb b/spec/support/elasticsearch/api/actions/get_alias.rb deleted file mode 100644 index ef4ebbd4f..000000000 --- a/spec/support/elasticsearch/api/actions/get_alias.rb +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License; -# you may not use this file except in compliance with the Elastic License. - -module Elasticsearch - module API - module Actions - - # Retrieve the list of index lifecycle management policies - def get_alias(arguments={}) - method = HTTP_GET - path = Utils.__pathify '_alias', Utils.__escape(arguments[:name]) - params = {} - perform_request(method, path, params, nil).body - end - end - end -end \ No newline at end of file diff --git a/spec/support/elasticsearch/api/actions/put_alias.rb b/spec/support/elasticsearch/api/actions/put_alias.rb deleted file mode 100644 index d0585934f..000000000 --- a/spec/support/elasticsearch/api/actions/put_alias.rb +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License; -# you may not use this file except in compliance with the Elastic License. - -module Elasticsearch - module API - module Actions - - # @option arguments [String] :name The name of the alias (*Required*) - # @option arguments [Hash] :The alias definition(*Required*) - - def put_alias(arguments={}) - raise ArgumentError, "Required argument 'name' missing" unless arguments[:name] - raise ArgumentError, "Required argument 'body' missing" unless arguments[:body] - method = HTTP_PUT - path = Utils.__pathify Utils.__escape(arguments[:name]) - - params = Utils.__validate_and_extract_params arguments - body = arguments[:body] - perform_request(method, path, params, body.to_json).body - end - end - end -end