diff --git a/.ci/docker-run.sh b/.ci/docker-run.sh index 72baaba9..18e331ab 100755 --- a/.ci/docker-run.sh +++ b/.ci/docker-run.sh @@ -7,7 +7,7 @@ cd .ci if [ "$INTEGRATION" == "true" ]; then # remove the `--attach logstash` if you want to see all logs including elasticsearch container logs - docker-compose up --exit-code-from logstash --attach logstash + docker compose up --exit-code-from logstash --attach logstash else - docker-compose up --exit-code-from logstash logstash + docker compose up --exit-code-from logstash logstash fi diff --git a/.ci/docker-setup.sh b/.ci/docker-setup.sh index d63efd90..ddac5139 100755 --- a/.ci/docker-setup.sh +++ b/.ci/docker-setup.sh @@ -81,9 +81,9 @@ cd .ci export BUILDKIT_PROGRESS=plain if [ "$INTEGRATION" == "true" ]; then - docker-compose down - docker-compose build --quiet + docker compose down + docker compose build else - docker-compose down - docker-compose build logstash --quiet + docker compose down + docker compose build logstash fi diff --git a/.ci/logstash-run.sh b/.ci/logstash-run.sh index 0801282b..d746a184 100755 --- a/.ci/logstash-run.sh +++ b/.ci/logstash-run.sh @@ -13,7 +13,7 @@ else fi # CentOS 7 using curl defaults does not enable TLSv1.3 -CURL_OPTS="-k --tlsv1.2 --tls-max 1.3" +CURL_OPTS="-s -u admin:elastic -k --tlsv1.2 --tls-max 1.3" wait_for_es() { count=120 @@ -22,7 +22,7 @@ wait_for_es() { [[ $count -eq 0 ]] && exit 1 sleep 1 done - echo $(curl $CURL_OPTS -vi $ES_URL | jq -r .version.number) + echo $(curl $CURL_OPTS $ES_URL | jq -r .version.number) } if [[ "$INTEGRATION" != "true" ]]; then diff --git a/.travis.yml b/.travis.yml index 112673ac..4e56268b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,7 @@ jobs: - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=8.current - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=8.next - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=8.future + - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=main - stage: "Secure Integration Tests" env: SECURE_INTEGRATION=true INTEGRATION=true LOG_LEVEL=info ELASTIC_STACK_VERSION=8.current SNAPSHOT=true - env: SECURE_INTEGRATION=true INTEGRATION=true LOG_LEVEL=info ELASTIC_STACK_VERSION=7.current diff --git a/Rakefile b/Rakefile index 2ff16dff..d33b2c0a 100644 --- a/Rakefile +++ b/Rakefile @@ -1,7 +1,6 @@ require "logstash/devutils/rake" task :'vendor-ecs-schemata' do - download_ecs_schemata(:v1, elasticsearch_major: 6, ecs_release_tag: 'v1.10.0') # WARNING: v1.11 breaks 6.x (see: https://github.com/elastic/ecs/issues/1649) download_ecs_schemata(:v1, elasticsearch_major: 7, ecs_release_tag: 'v1.12.1') download_ecs_schemata(:v1, elasticsearch_major: 8, ecs_release_tag: 'v1.12.1', generated_for: 7) download_ecs_schemata(:v1, elasticsearch_major: 9, ecs_release_tag: 'v1.12.1', generated_for: 7) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index a4141066..a7dc4ce4 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -612,8 +612,7 @@ Elasticsearch with the same ID. NOTE: This option is deprecated due to the https://www.elastic.co/guide/en/elasticsearch/reference/6.0/removal-of-types.html[removal -of types in Elasticsearch 6.0]. It will be removed in the next major version of -Logstash. +of types in Elasticsearch 6.0]. NOTE: This value is ignored and has no effect for Elasticsearch clusters `8.x`. @@ -622,9 +621,7 @@ similar events to the same 'type'. String expansion `%{foo}` works here. If you don't set a value for this option: - for elasticsearch clusters 8.x: no value will be used; -- for elasticsearch clusters 7.x: the value of '_doc' will be used; -- for elasticsearch clusters 6.x: the value of 'doc' will be used; -- for elasticsearch clusters 5.x and below: the event's 'type' field will be used, if the field is not present the value of 'doc' will be used. +- for elasticsearch clusters 7.x: the value of '_doc' will be used. [id="plugins-{type}s-{plugin}-ecs_compatibility"] ===== `ecs_compatibility` @@ -1039,8 +1036,6 @@ NOTE: Deprecates <>. This setting asks Elasticsearch for the list of all cluster nodes and adds them to the hosts list. -For Elasticsearch 5.x and 6.x any nodes with `http.enabled` (on by default) will -be added to the hosts list, excluding master-only nodes. [id="plugins-{type}s-{plugin}-sniffing_delay"] ===== `sniffing_delay` diff --git a/lib/logstash/outputs/elasticsearch.rb b/lib/logstash/outputs/elasticsearch.rb index e6606fa7..e98f392a 100644 --- a/lib/logstash/outputs/elasticsearch.rb +++ b/lib/logstash/outputs/elasticsearch.rb @@ -14,39 +14,16 @@ # .Compatibility Note # [NOTE] # ================================================================================ -# Starting with Elasticsearch 5.3, there's an {ref}modules-http.html[HTTP setting] -# called `http.content_type.required`. If this option is set to `true`, and you -# are using Logstash 2.4 through 5.2, you need to update the Elasticsearch output -# plugin to version 6.2.5 or higher. -# -# ================================================================================ # # This plugin is the recommended method of storing logs in Elasticsearch. # If you plan on using the Kibana web interface, you'll want to use this output. # -# This output only speaks the HTTP protocol. HTTP is the preferred protocol for interacting with Elasticsearch as of Logstash 2.0. -# We strongly encourage the use of HTTP over the node protocol for a number of reasons. HTTP is only marginally slower, -# yet far easier to administer and work with. When using the HTTP protocol one may upgrade Elasticsearch versions without having -# to upgrade Logstash in lock-step. +# This output only speaks the HTTP protocol. # # You can learn more about Elasticsearch at # -# ==== Template management for Elasticsearch 5.x -# Index template for this version (Logstash 5.0) has been changed to reflect Elasticsearch's mapping changes in version 5.0. -# Most importantly, the subfield for string multi-fields has changed from `.raw` to `.keyword` to match ES default -# behavior. -# -# ** Users installing ES 5.x and LS 5.x ** -# This change will not affect you and you will continue to use the ES defaults. -# -# ** Users upgrading from LS 2.x to LS 5.x with ES 5.x ** -# LS will not force upgrade the template, if `logstash` template already exists. This means you will still use -# `.raw` for sub-fields coming from 2.x. If you choose to use the new template, you will have to reindex your data after -# the new template is installed. -# # ==== Retry Policy # -# The retry policy has changed significantly in the 2.2.0 release. # This plugin uses the Elasticsearch bulk API to optimize its imports into Elasticsearch. These requests may experience # either partial or total failures. # @@ -129,8 +106,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base # - delete: deletes a document by id (An id is required for this action) # - create: indexes a document, fails if a document by that id already exists in the index. # - update: updates a document by id. Update has a special case where you can upsert -- update a - # document if not already present. See the `upsert` option. NOTE: This does not work and is not supported - # in Elasticsearch 1.x. Please upgrade to ES 2.x or greater to use this feature with Logstash! + # document if not already present. See the `upsert` option. # - A sprintf style string to change the action based on the content of the event. The value `%{[foo]}` # would use the foo field for the action # @@ -148,7 +124,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base config :document_type, :validate => :string, - :deprecated => "Document types are being deprecated in Elasticsearch 6.0, and removed entirely in 7.0. You should avoid this feature" + :deprecated => "Document types were deprecated in Elasticsearch 7.0, and no longer configurable since 8.0. You should avoid this feature." # From Logstash 1.3 onwards, a template is applied to Elasticsearch during # Logstash's startup if one with the name `template_name` does not already exist. @@ -483,7 +459,7 @@ def event_action_tuple(event) join_value = event.get(@join_field) parent_value = event.sprintf(@parent) event.set(@join_field, { "name" => join_value, "parent" => parent_value }) - params[routing_field_name] = event.sprintf(@parent) + params[:routing] = event.sprintf(@parent) else params[:parent] = event.sprintf(@parent) end @@ -495,7 +471,7 @@ def event_action_tuple(event) if action == 'update' params[:_upsert] = LogStash::Json.load(event.sprintf(@upsert)) if @upsert != "" params[:_script] = event.sprintf(@script) if @script != "" - params[retry_on_conflict_action_name] = @retry_on_conflict + params[:retry_on_conflict] = @retry_on_conflict end event_control = event.get("[@metadata][_ingest_document]") @@ -552,7 +528,7 @@ def common_event_params(event) params = { :_id => resolve_document_id(event, event_id), :_index => resolve_index!(event, event_index), - routing_field_name => resolve_routing(event, event_routing) + :routing => resolve_routing(event, event_routing) } target_pipeline = resolve_pipeline(event, event_pipeline) @@ -615,16 +591,7 @@ def resolve_pipeline(event, event_pipeline) require "logstash/outputs/elasticsearch/#{name}" end - def retry_on_conflict_action_name - maximum_seen_major_version >= 7 ? :retry_on_conflict : :_retry_on_conflict - end - - def routing_field_name - :routing - end - # Determine the correct value for the 'type' field for the given event - DEFAULT_EVENT_TYPE_ES6 = "doc".freeze DEFAULT_EVENT_TYPE_ES7 = "_doc".freeze def get_event_type(event) @@ -633,9 +600,7 @@ def get_event_type(event) event.sprintf(@document_type) else major_version = maximum_seen_major_version - if major_version == 6 - DEFAULT_EVENT_TYPE_ES6 - elsif major_version == 7 + if major_version == 7 DEFAULT_EVENT_TYPE_ES7 else nil @@ -653,9 +618,9 @@ def get_event_type(event) # @param noop_required_client [nil]: required `nil` for legacy reasons. # @return [Boolean] def use_event_type?(noop_required_client) - # always set type for ES 6 - # for ES 7 only set it if the user defined it - (maximum_seen_major_version < 7) || (maximum_seen_major_version == 7 && @document_type) + # never use event type unless + # ES is 7.x and the user defined it + maximum_seen_major_version == 7 && @document_type end def install_template diff --git a/lib/logstash/outputs/elasticsearch/data_stream_support.rb b/lib/logstash/outputs/elasticsearch/data_stream_support.rb index 6191afee..a137a447 100644 --- a/lib/logstash/outputs/elasticsearch/data_stream_support.rb +++ b/lib/logstash/outputs/elasticsearch/data_stream_support.rb @@ -127,7 +127,6 @@ def invalid_data_stream_params(params) value.to_s == 'true' when 'manage_template' value.to_s == 'false' - when 'ecs_compatibility' then true # required for LS <= 6.x else name.start_with?('data_stream_') || shared_params.include?(name) || diff --git a/lib/logstash/outputs/elasticsearch/http_client/pool.rb b/lib/logstash/outputs/elasticsearch/http_client/pool.rb index 68715066..00675eea 100644 --- a/lib/logstash/outputs/elasticsearch/http_client/pool.rb +++ b/lib/logstash/outputs/elasticsearch/http_client/pool.rb @@ -52,7 +52,6 @@ def initialize(original_error, url) ROOT_URI_PATH = '/'.freeze LICENSE_PATH = '/_license'.freeze - VERSION_6_TO_7 = ::Gem::Requirement.new([">= 6.0.0", "< 7.0.0"]) VERSION_7_TO_7_14 = ::Gem::Requirement.new([">= 7.0.0", "< 7.14.0"]) DEFAULT_OPTIONS = { @@ -550,11 +549,9 @@ def elasticsearch?(response) return false if version_info['version'].nil? version = ::Gem::Version.new(version_info["version"]['number']) - return false if version < ::Gem::Version.new('6.0.0') + return false if version < ::Gem::Version.new('7.0.0') - if VERSION_6_TO_7.satisfied_by?(version) - return valid_tagline?(version_info) - elsif VERSION_7_TO_7_14.satisfied_by?(version) + if VERSION_7_TO_7_14.satisfied_by?(version) build_flavor = version_info["version"]['build_flavor'] return false if build_flavor.nil? || build_flavor != 'default' || !valid_tagline?(version_info) else diff --git a/lib/logstash/outputs/elasticsearch/ilm.rb b/lib/logstash/outputs/elasticsearch/ilm.rb index 6024085b..76dd0bd5 100644 --- a/lib/logstash/outputs/elasticsearch/ilm.rb +++ b/lib/logstash/outputs/elasticsearch/ilm.rb @@ -21,13 +21,7 @@ def ilm_in_use? "Serverless Elasticsearch cluster does not support Index Lifecycle Management.") if @ilm_enabled == 'auto' false elsif @ilm_enabled == 'auto' - if ilm_on_by_default? - ilm_alias_set? - else - @logger.info("ILM auto configuration (`ilm_enabled => auto` or unset) resolved to `false`."\ - " Elasticsearch cluster is before 7.0.0, which is the minimum version required to automatically run Index Lifecycle Management") - false - end + ilm_alias_set? elsif @ilm_enabled.to_s == 'true' ilm_alias_set? else @@ -42,10 +36,6 @@ def ilm_alias_set? default_index?(@index) || !default_rollover_alias?(@ilm_rollover_alias) end - def ilm_on_by_default? - maximum_seen_major_version >= 7 - end - def default_index?(index) index == @default_index end diff --git a/lib/logstash/outputs/elasticsearch/template_manager.rb b/lib/logstash/outputs/elasticsearch/template_manager.rb index 8ebfe273..a6ed2bb6 100644 --- a/lib/logstash/outputs/elasticsearch/template_manager.rb +++ b/lib/logstash/outputs/elasticsearch/template_manager.rb @@ -47,7 +47,7 @@ def self.install(client, template_endpoint, template_name, template, template_ov def self.add_ilm_settings_to_template(plugin, template) # Overwrite any index patterns, and use the rollover alias. Use 'index_patterns' rather than 'template' for pattern # definition - remove any existing definition of 'template' - template.delete('template') if template.include?('template') if plugin.maximum_seen_major_version < 8 + template.delete('template') if template.include?('template') if plugin.maximum_seen_major_version == 7 template['index_patterns'] = "#{plugin.ilm_rollover_alias}-*" settings = resolve_template_settings(plugin, template) if settings && (settings['index.lifecycle.name'] || settings['index.lifecycle.rollover_alias']) diff --git a/lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-6x.json b/lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-6x.json deleted file mode 100644 index 2227906e..00000000 --- a/lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-6x.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "template" : "logstash-*", - "version" : 60001, - "settings" : { - "index.refresh_interval" : "5s" - }, - "mappings" : { - "_default_" : { - "dynamic_templates" : [ { - "message_field" : { - "path_match" : "message", - "match_mapping_type" : "string", - "mapping" : { - "type" : "text", - "norms" : false - } - } - }, { - "string_fields" : { - "match" : "*", - "match_mapping_type" : "string", - "mapping" : { - "type" : "text", "norms" : false, - "fields" : { - "keyword" : { "type": "keyword", "ignore_above": 256 } - } - } - } - } ], - "properties" : { - "@timestamp": { "type": "date"}, - "@version": { "type": "keyword"}, - "geoip" : { - "dynamic": true, - "properties" : { - "ip": { "type": "ip" }, - "location" : { "type" : "geo_point" }, - "latitude" : { "type" : "half_float" }, - "longitude" : { "type" : "half_float" } - } - } - } - } - } -} diff --git a/spec/es_spec_helper.rb b/spec/es_spec_helper.rb index 6981a9c1..9c043afb 100644 --- a/spec/es_spec_helper.rb +++ b/spec/es_spec_helper.rb @@ -30,8 +30,6 @@ def doc_type nil elsif ESHelper.es_version_satisfies?(">=7") "_doc" - else - "doc" end end @@ -70,7 +68,7 @@ def self.es_version end RSpec::Matchers.define :have_hits do |expected| - hits_count_path = ESHelper.es_version_satisfies?(">=7") ? %w(hits total value) : %w(hits total) + hits_count_path = %w(hits total value) match do |actual| @actual_hits_count = actual&.dig(*hits_count_path) @@ -214,8 +212,6 @@ def get_template_mappings(template) template['template']['mappings'] elsif ESHelper.es_version_satisfies?(">=7") template['mappings'] - else - template['mappings']["_default_"] end end end diff --git a/spec/fixtures/_nodes/6x.json b/spec/fixtures/_nodes/6x.json deleted file mode 100644 index 9b519d21..00000000 --- a/spec/fixtures/_nodes/6x.json +++ /dev/null @@ -1,81 +0,0 @@ -{ - "_nodes" : { - "total" : 3, - "successful" : 3, - "failed" : 0 - }, - "cluster_name" : "dev", - "nodes" : { - "Ur_68iBvTlm7Xr1HgSsh6w" : { - "name" : "dev-es-master01", - "transport_address" : "http://localhost:9200", - "host" : "localhost", - "ip" : "127.0.0.1", - "version" : "6.8.10", - "build_hash" : "19c13d0", - "roles" : [ - "master" - ], - "http" : { - "bound_address" : [ - "[::]:9200" - ], - "publish_address" : "127.0.0.1:9200", - "max_content_length_in_bytes" : 104857600 - } - }, - "sari4do3RG-mgh2CIZeHwA" : { - "name" : "dev-es-data01", - "transport_address" : "http://localhost:9201", - "host" : "localhost", - "ip" : "127.0.0.1", - "version" : "6.8.10", - "build_hash" : "19c13d0", - "roles" : [ - "data" - ], - "http" : { - "bound_address" : [ - "[::]:9200" - ], - "publish_address" : "127.0.0.1:9201", - "max_content_length_in_bytes" : 104857600 - } - }, - "Rjy1WL66RHm4fyzXA8PCGQ" : { - "name" : "dev-es-datamaster01", - "transport_address" : "http://localhost:9202", - "host" : "localhost", - "ip" : "127.0.0.1", - "version" : "6.8.10", - "build_hash" : "19c13d0", - "roles" : [ - "data", - "master" - ], - "http" : { - "bound_address" : [ - "[::]:9200" - ], - "publish_address" : "127.0.0.1:9202", - "max_content_length_in_bytes" : 104857600 - } - }, - "OguP_obcT_S9JYNB8SKKgQ" : { - "name" : "dev-es-coordinator01", - "transport_address" : "http://localhost:9203", - "host" : "localhost", - "ip" : "127.0.0.1", - "version" : "6.8.10", - "build_hash" : "19c13d0", - "roles" : [ ], - "http" : { - "bound_address" : [ - "[::]:9200" - ], - "publish_address" : "127.0.0.1:9203", - "max_content_length_in_bytes" : 104857600 - } - } - } -} diff --git a/spec/fixtures/template-with-policy-es6x.json b/spec/fixtures/template-with-policy-es6x.json deleted file mode 100644 index c6b89bec..00000000 --- a/spec/fixtures/template-with-policy-es6x.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "template" : "overwrite-*", - "version" : 60001, - "settings" : { - "index.refresh_interval" : "1s", - "number_of_shards": 1, - "index.lifecycle.name": "overwrite-policy", - "index.lifecycle.rollover_alias": "overwrite" - }, - "mappings" : { - "_default_" : { - "dynamic_templates" : [ { - "message_field" : { - "path_match" : "message", - "match_mapping_type" : "string", - "mapping" : { - "type" : "text", - "norms" : false - } - } - }, { - "string_fields" : { - "match" : "*", - "match_mapping_type" : "string", - "mapping" : { - "type" : "text", "norms" : false, - "fields" : { - "keyword" : { "type": "keyword", "ignore_above": 256 } - } - } - } - } ], - "properties" : { - "@timestamp": { "type": "date"}, - "@version": { "type": "keyword"}, - "geoip" : { - "dynamic": true, - "properties" : { - "ip": { "type": "ip" }, - "location" : { "type" : "geo_point" }, - "latitude" : { "type" : "half_float" }, - "longitude" : { "type" : "half_float" } - } - } - } - } - } -} diff --git a/spec/integration/outputs/compressed_indexing_spec.rb b/spec/integration/outputs/compressed_indexing_spec.rb index 3662f543..c1c30550 100644 --- a/spec/integration/outputs/compressed_indexing_spec.rb +++ b/spec/integration/outputs/compressed_indexing_spec.rb @@ -14,7 +14,7 @@ let(:event_with_invalid_utf_8_bytes) { LogStash::Event.new("message" => "Message from spacecraft which contains \xAC invalid \xD7 byte sequences.", "type" => type) } let(:index) { 10.times.collect { rand(10).to_s }.join("") } - let(:type) { ESHelper.es_version_satisfies?("< 7") ? "doc" : "_doc" } + let(:type) { "_doc" } let(:event_count) { 10000 + rand(500) } # mix the events with valid and invalid UTF-8 payloads let(:events) { event_count.times.map { |i| i%3 == 0 ? event : event_with_invalid_utf_8_bytes }.to_a } @@ -59,10 +59,10 @@ response = http_client.get("#{index_url}/_search?q=*&size=1000") result = LogStash::Json.load(response.body) result["hits"]["hits"].each do |doc| - if ESHelper.es_version_satisfies?("< 8") - expect(doc["_type"]).to eq(type) - else + if ESHelper.es_version_satisfies?(">= 8") expect(doc).not_to include("_type") + else + expect(doc["_type"]).to eq(type) end expect(doc["_index"]).to eq(index) end @@ -78,4 +78,4 @@ it_behaves_like("an indexer") end -end \ No newline at end of file +end diff --git a/spec/integration/outputs/index_spec.rb b/spec/integration/outputs/index_spec.rb index 02f3fb76..35d031c4 100644 --- a/spec/integration/outputs/index_spec.rb +++ b/spec/integration/outputs/index_spec.rb @@ -13,7 +13,7 @@ } } let(:index) { 10.times.collect { rand(10).to_s }.join("") } - let(:type) { ESHelper.es_version_satisfies?("< 7") ? "doc" : "_doc" } + let(:type) { "_doc" } subject { LogStash::Outputs::ElasticSearch.new(config) } @@ -82,7 +82,7 @@ def curl_and_get_json_response(url, method: :get, retrieve_err_payload: false); let(:message) { "Hello from #{__FILE__}" } let(:event) { LogStash::Event.new("message" => message, "type" => type) } let (:index) { "%{[index_name]}_dynamic" } - let(:type) { ESHelper.es_version_satisfies?("< 7") ? "doc" : "_doc" } + let(:type) { "_doc" } let(:event_count) { 1 } let(:user) { "simpleuser" } let(:password) { "abc123" } @@ -151,7 +151,7 @@ def curl_and_get_json_response(url, method: :get, retrieve_err_payload: false); let(:message) { "Hello from #{__FILE__}" } let(:event) { LogStash::Event.new("message" => message, "type" => type) } let(:index) { 10.times.collect { rand(10).to_s }.join("") } - let(:type) { ESHelper.es_version_satisfies?("< 7") ? "doc" : "_doc" } + let(:type) { "_doc" } let(:event_count) { 1 + rand(2) } let(:config) { "not implemented" } let(:events) { event_count.times.map { event }.to_a } @@ -204,10 +204,10 @@ def curl_and_get_json_response(url, method: :get, retrieve_err_payload: false); result["hits"]["hits"].each do |doc| expect(doc["_source"]["message"]).to eq(message) - if ESHelper.es_version_satisfies?("< 8") - expect(doc["_type"]).to eq(type) - else + if ESHelper.es_version_satisfies?(">= 8") expect(doc).not_to include("_type") + else + expect(doc["_type"]).to eq(type) end expect(doc["_index"]).to eq(index) end @@ -346,7 +346,7 @@ def curl_and_get_json_response(url, method: :get, retrieve_err_payload: false); end describe "an indexer with no type value set (default to doc)", :integration => true do - let(:type) { ESHelper.es_version_satisfies?("< 7") ? "doc" : "_doc" } + let(:type) { "_doc" } let(:config) { { "hosts" => get_host_port, diff --git a/spec/integration/outputs/no_es_on_startup_spec.rb b/spec/integration/outputs/no_es_on_startup_spec.rb index f44a956f..1000473e 100644 --- a/spec/integration/outputs/no_es_on_startup_spec.rb +++ b/spec/integration/outputs/no_es_on_startup_spec.rb @@ -74,5 +74,5 @@ expect(r).to have_hits(2) expect(subject.plugin_metadata.get(:cluster_uuid)).not_to be_empty expect(subject.plugin_metadata.get(:cluster_uuid)).not_to eq("_na_") - end if ESHelper.es_version_satisfies?(">=7") + end end diff --git a/spec/integration/outputs/parent_spec.rb b/spec/integration/outputs/parent_spec.rb index cbd803fd..525b810e 100644 --- a/spec/integration/outputs/parent_spec.rb +++ b/spec/integration/outputs/parent_spec.rb @@ -6,7 +6,7 @@ shared_examples "a join field based parent indexer" do let(:index) { 10.times.collect { rand(10).to_s }.join("") } - let(:type) { ESHelper.es_version_satisfies?("< 7") ? "doc" : "_doc" } + let(:type) { "_doc" } let(:event_count) { 10000 + rand(500) } let(:parent) { "not_implemented" } @@ -33,8 +33,7 @@ } } - mapping = ESHelper.es_version_satisfies?('<7') ? { "mappings" => { type => properties } } - : { "mappings" => properties} + mapping = { "mappings" => properties} Manticore.put("#{index_url}", {:body => mapping.to_json, :headers => default_headers}).call pdoc = { "message" => "ohayo", join_field => parent_relation } diff --git a/spec/integration/outputs/retry_spec.rb b/spec/integration/outputs/retry_spec.rb index 5493f44c..027b4d99 100644 --- a/spec/integration/outputs/retry_spec.rb +++ b/spec/integration/outputs/retry_spec.rb @@ -5,19 +5,11 @@ let(:template) { '{"template" : "not important, will be updated by :index"}' } let(:event1) { LogStash::Event.new("somevalue" => 100, "@timestamp" => "2014-11-17T20:37:17.223Z", "@metadata" => {"retry_count" => 0}) } let(:action1) do - if ESHelper.es_version_satisfies?("< 7") - ESHelper.action_for_version(["index", {:_id=>nil, routing_field_name =>nil, :_index=>"logstash-2014.11.17", :_type=> doc_type }, event1.to_hash]) - else - ESHelper.action_for_version(["index", {:_id=>nil, routing_field_name =>nil, :_index=>"logstash-2014.11.17" }, event1.to_hash]) - end + ESHelper.action_for_version(["index", {:_id=>nil, routing_field_name =>nil, :_index=>"logstash-2014.11.17" }, event1.to_hash]) end let(:event2) { LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0] }, "@timestamp" => "2014-11-17T20:37:17.223Z", "@metadata" => {"retry_count" => 0}) } let(:action2) do - if ESHelper.es_version_satisfies?("< 7") - ESHelper.action_for_version(["index", {:_id=>nil, routing_field_name =>nil, :_index=>"logstash-2014.11.17", :_type=> doc_type }, event2.to_hash]) - else - ESHelper.action_for_version(["index", {:_id=>nil, routing_field_name =>nil, :_index=>"logstash-2014.11.17" }, event2.to_hash]) - end + ESHelper.action_for_version(["index", {:_id=>nil, routing_field_name =>nil, :_index=>"logstash-2014.11.17" }, event2.to_hash]) end let(:invalid_event) { LogStash::Event.new("geoip" => { "location" => "notlatlon" }, "@timestamp" => "2014-11-17T20:37:17.223Z") } diff --git a/spec/integration/outputs/sniffer_spec.rb b/spec/integration/outputs/sniffer_spec.rb index cd43c8a4..99ba02ac 100644 --- a/spec/integration/outputs/sniffer_spec.rb +++ b/spec/integration/outputs/sniffer_spec.rb @@ -33,48 +33,13 @@ expect(uris.size).to eq(1) end - - it "should return the correct sniff URL" do - if ESHelper.es_version_satisfies?("<7") - # We do a more thorough check on these versions because we can more reliably guess the ip - uris = subject.check_sniff - - expect(uris).to include(::LogStash::Util::SafeURI.new("//#{es_ip}:#{es_port}")) - else - # ES 1.x (and ES 7.x) returned the public hostname by default. This is hard to approximate - # so for ES1.x and 7.x we don't check the *exact* hostname - skip - end - end end end - if ESHelper.es_version_satisfies?(">= 7") - describe("Complex sniff parsing ES 7x") do - before(:each) do - response_double = double("_nodes/http", body: File.read("spec/fixtures/_nodes/7x.json")) - allow(subject).to receive(:perform_request).and_return([nil, { version: "7.0" }, response_double]) - subject.start - end - - context "with mixed master-only, data-only, and data + master nodes" do - it "should execute a sniff without error" do - expect { subject.check_sniff }.not_to raise_error - end - - it "should return the correct sniff URLs" do - # ie. with the master-only node, and with the node name correctly set. - uris = subject.check_sniff - - expect(uris).to include(::LogStash::Util::SafeURI.new("//dev-masterdata:9201"), ::LogStash::Util::SafeURI.new("//dev-data:9202")) - end - end - end - end - describe("Complex sniff parsing ES") do + describe("Complex sniff parsing") do before(:each) do - response_double = double("_nodes/http", body: File.read("spec/fixtures/_nodes/6x.json")) - allow(subject).to receive(:perform_request).and_return([nil, { version: "6.8" }, response_double]) + response_double = double("_nodes/http", body: File.read("spec/fixtures/_nodes/7x.json")) + allow(subject).to receive(:perform_request).and_return([nil, { version: "7.0" }, response_double]) subject.start end @@ -84,10 +49,10 @@ end it "should return the correct sniff URLs" do - # ie. without the master-only node + # ie. with the master-only node, and with the node name correctly set. uris = subject.check_sniff - expect(uris).to include(::LogStash::Util::SafeURI.new("//127.0.0.1:9201"), ::LogStash::Util::SafeURI.new("//127.0.0.1:9202"), ::LogStash::Util::SafeURI.new("//127.0.0.1:9203")) + expect(uris).to include(::LogStash::Util::SafeURI.new("//dev-masterdata:9201"), ::LogStash::Util::SafeURI.new("//dev-data:9202")) end end end diff --git a/spec/unit/outputs/elasticsearch/data_stream_support_spec.rb b/spec/unit/outputs/elasticsearch/data_stream_support_spec.rb index bbb59f34..ca8ec1f2 100644 --- a/spec/unit/outputs/elasticsearch/data_stream_support_spec.rb +++ b/spec/unit/outputs/elasticsearch/data_stream_support_spec.rb @@ -286,29 +286,6 @@ end end - - context 'non-compatible ES' do - - let(:es_version) { '6.8.11' } - - it "prints an error (from after_successful_connection thread) on LS 7.x" do - change_constant :LOGSTASH_VERSION, '7.12.0' do - expect( subject.logger ).to receive(:error).with(/Elasticsearch version does not support data streams/, - {:es_version=>"6.8.11"}) - stub_plugin_register! - end - end - - it "prints an error (from after_successful_connection thread) on LS 8.0" do - change_constant :LOGSTASH_VERSION, '8.0.5' do - expect( subject.logger ).to receive(:error).with(/Elasticsearch version does not support data streams/, - {:es_version=>"6.8.11"}) - stub_plugin_register! - end - end - - end - end describe "auto routing" do diff --git a/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb b/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb index 1769c699..64061d7b 100644 --- a/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb +++ b/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb @@ -203,29 +203,6 @@ let(:ip_address) { "192.168.1.0"} let(:port) { 9200 } - context 'in Elasticsearch 1.x format' do - context 'with host and ip address' do - let(:publish_address) { "inet[#{host}/#{ip_address}:#{port}]"} - it 'should correctly extract the host' do - expect(subject.address_str_to_uri(publish_address)).to eq (LogStash::Util::SafeURI.new("#{host}:#{port}")) - end - end - context 'with ip address' do - let(:publish_address) { "inet[/#{ip_address}:#{port}]"} - it 'should correctly extract the ip address' do - expect(subject.address_str_to_uri(publish_address)).to eq (LogStash::Util::SafeURI.new("#{ip_address}:#{port}")) - end - end - end - - context 'in Elasticsearch 2.x-6.x format' do - let(:publish_address) { "#{ip_address}:#{port}"} - it 'should correctly extract the ip address' do - expect(subject.address_str_to_uri(publish_address)).to eq (LogStash::Util::SafeURI.new("//#{ip_address}:#{port}")) - end - end - - context 'in Elasticsearch 7.x' context 'with host and ip address' do let(:publish_address) { "#{host}/#{ip_address}:#{port}"} it 'should correctly extract the host' do @@ -367,14 +344,19 @@ def body let(:root_response) { MockResponse.new(200, {"tagline" => "You Know, for Search", "version" => { - "number" => '0.0.0', - "build_flavor" => 'default'} - }) } - let(:root_response2) { MockResponse.new(200, {"tagline" => "You Know, for Search", - "version" => { - "number" => '6.0.0', + "number" => '7.0.0', "build_flavor" => 'default'} }) } + let(:root_response2) { MockResponse.new(200, + { + "tagline" => "You Know, for Search", + "version" => { + "number" => '8.0.0', + "build_flavor" => 'default' + } + }, + { "x-elastic-product" => "Elasticsearch" } + ) } context "if there are nodes with multiple major versions" do before(:each) do @@ -383,7 +365,7 @@ def body end it "picks the largest major version" do - expect(subject.maximum_seen_major_version).to eq(6) + expect(subject.maximum_seen_major_version).to eq(8) end end end @@ -542,30 +524,6 @@ def body end end - context "when connecting to a cluster with version < 6.0.0" do - it "should fail" do - resp = MockResponse.new(200, {"version" => { "number" => "5.0.0" }}) - expect(subject.send(:elasticsearch?, resp)).to be false - end - end - - context "when connecting to a cluster with version in [6.0.0..7.0.0)" do - it "must be successful with valid 'tagline'" do - resp = MockResponse.new(200, {"version" => {"number" => "6.5.0"}, "tagline" => "You Know, for Search"} ) - expect(subject.send(:elasticsearch?, resp)).to be true - end - - it "should fail if invalid 'tagline'" do - resp = MockResponse.new(200, {"version" => {"number" => "6.5.0"}, "tagline" => "You don't know"} ) - expect(subject.send(:elasticsearch?, resp)).to be false - end - - it "should fail if 'tagline' is not present" do - resp = MockResponse.new(200, {"version" => {"number" => "6.5.0"}} ) - expect(subject.send(:elasticsearch?, resp)).to be false - end - end - context "when connecting to a cluster with version in [7.0.0..7.14.0)" do it "must be successful is 'build_flavor' is 'default' and tagline is correct" do resp = MockResponse.new(200, {"version": {"number": "7.5.0", "build_flavor": "default"}, "tagline": "You Know, for Search"} ) diff --git a/spec/unit/outputs/elasticsearch/template_manager_spec.rb b/spec/unit/outputs/elasticsearch/template_manager_spec.rb index a6d7d0f1..dfcb91c8 100644 --- a/spec/unit/outputs/elasticsearch/template_manager_spec.rb +++ b/spec/unit/outputs/elasticsearch/template_manager_spec.rb @@ -4,11 +4,6 @@ describe LogStash::Outputs::ElasticSearch::TemplateManager do describe ".default_template_path" do - context "elasticsearch 6.x" do - it "chooses the 6x template" do - expect(described_class.default_template_path(6)).to end_with("/templates/ecs-disabled/elasticsearch-6x.json") - end - end context "elasticsearch 7.x" do it "chooses the 7x template" do expect(described_class.default_template_path(7)).to end_with("/templates/ecs-disabled/elasticsearch-7x.json") @@ -52,7 +47,7 @@ end end - describe "in version < 8" do + describe "in version 7" do let(:file_path) { described_class.default_template_path(7) } let(:template) { described_class.read_template_file(file_path)} @@ -96,7 +91,7 @@ describe "with `template_api => 'auto'`" do let(:template_api) { "auto" } - describe "with ES < 8 versions" do + describe "with ES 7" do it 'resolves legacy index template API compatible setting' do expect(plugin).to receive(:serverless?).and_return(false) @@ -134,7 +129,7 @@ end end - describe "in version < 8" do + describe "in version 7" do it "should use legacy template API" do expect(plugin).to receive(:serverless?).and_return(false) expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7) diff --git a/spec/unit/outputs/elasticsearch_spec.rb b/spec/unit/outputs/elasticsearch_spec.rb index 95cffa85..105766ed 100644 --- a/spec/unit/outputs/elasticsearch_spec.rb +++ b/spec/unit/outputs/elasticsearch_spec.rb @@ -9,7 +9,7 @@ describe LogStash::Outputs::ElasticSearch do subject(:elasticsearch_output_instance) { described_class.new(options) } let(:options) { {} } - let(:maximum_seen_major_version) { [6,7,8].sample } + let(:maximum_seen_major_version) { [7,8,9].sample } let(:do_register) { true } @@ -216,13 +216,6 @@ expect(subject.send(:get_event_type, LogStash::Event.new("type" => "foo"))).to eql("_doc") end end - - context "for 6.x elasticsearch clusters" do - let(:maximum_seen_major_version) { 6 } - it "should return 'doc'" do - expect(subject.send(:get_event_type, LogStash::Event.new("type" => "foo"))).to eql("doc") - end - end end context "with 'document type set'" do @@ -694,7 +687,7 @@ expect{ subject.register }.to raise_error(LogStash::ConfigurationError, /configured while DLQ is not enabled/) end end - end if LOGSTASH_VERSION > '7.0' + end describe "#multi_receive" do let(:events) { [double("one"), double("two"), double("three")] } @@ -1136,7 +1129,7 @@ it "should not set the retry_on_conflict parameter when creating an event_action_tuple" do allow(subject.client).to receive(:maximum_seen_major_version).and_return(maximum_seen_major_version) action, params, event_data = subject.send(:event_action_tuple, event) - expect(params).not_to include({subject.send(:retry_on_conflict_action_name) => num_retries}) + expect(params).to_not include({:retry_on_conflict => num_retries}) end end @@ -1145,7 +1138,7 @@ it "should set the retry_on_conflict parameter when creating an event_action_tuple" do action, params, event_data = subject.send(:event_action_tuple, event) - expect(params).to include({subject.send(:retry_on_conflict_action_name) => num_retries}) + expect(params).to include({:retry_on_conflict => num_retries}) end end @@ -1154,7 +1147,7 @@ it "should set the retry_on_conflict parameter when creating an event_action_tuple" do action, params, event_data = subject.send(:event_action_tuple, event) - expect(params).to include({subject.send(:retry_on_conflict_action_name) => num_retries}) + expect(params).to include({:retry_on_conflict => num_retries}) expect(action).to eq("update") end end @@ -1318,7 +1311,7 @@ expect { subject.register }.to raise_error LogStash::ConfigurationError, /cloud_id and hosts/ end end - end if LOGSTASH_VERSION > '6.0' + end describe "cloud.auth" do let(:do_register) { false } @@ -1359,7 +1352,7 @@ expect { subject.register }.to raise_error LogStash::ConfigurationError, /Multiple authentication options are specified/ end end - end if LOGSTASH_VERSION > '6.0' + end context 'handling elasticsearch document-level status meant for the DLQ' do let(:es_api_action) { "CUSTOM_ACTION" } @@ -1498,7 +1491,7 @@ include_examples "should write event to DLQ" end - end if LOGSTASH_VERSION > '7.0' + end end describe "custom headers" do @@ -1676,7 +1669,7 @@ subject.send :wait_for_successful_connection expect(logger).to have_received(:error).with(/Unable to retrieve Elasticsearch cluster uuid/i, anything) - end if LOGSTASH_VERSION >= '7.0.0' + end it "logs template install failure" do allow(subject).to receive(:discover_cluster_uuid)