Resolve template ILM settings based on given template and template_api.

Author: mashhurs

CHANGELOG.md

@@ -1,3 +1,6 @@
+## 11.12.5
+ - Fixes the failure when legacy template (`template_api=>'legacy'`) API is used with custom template, the plugin doesn't resolve ILM settings and crash. [#1102](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1102)
+
 ## 11.12.4
  - Changed the `manage_template` default value to `false` when data streams is enabled [#1111](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1111)
    - Added the `manage_template => false` as a valid data stream option

lib/logstash/outputs/elasticsearch/template_manager.rb

@@ -46,15 +46,30 @@ def self.add_ilm_settings_to_template(plugin, template)
       # definition - remove any existing definition of 'template'
       template.delete('template') if template.include?('template') if plugin.maximum_seen_major_version < 8
       template['index_patterns'] = "#{plugin.ilm_rollover_alias}-*"
-      settings = template_settings(plugin, template)
+      settings = resolve_template_settings(plugin, template)
       if settings && (settings['index.lifecycle.name'] || settings['index.lifecycle.rollover_alias'])
         plugin.logger.info("Overwriting index lifecycle name and rollover alias as ILM is enabled")
       end
       settings.update({ 'index.lifecycle.name' => plugin.ilm_policy, 'index.lifecycle.rollover_alias' => plugin.ilm_rollover_alias})
     end
 
-    def self.template_settings(plugin, template)
-      plugin.maximum_seen_major_version < 8 ? template['settings']: template['template']['settings']
+    def self.resolve_template_settings(plugin, template)
+      return composable_index_template_settings(template) if template.key?('template')
+      return legacy_index_template_settings(template) if template.key?('settings')
+      return template_endpoint(plugin) == INDEX_TEMPLATE_ENDPOINT ?
+               composable_index_template_settings(template) :
+               legacy_index_template_settings(template)
+    end
+
+    # Returns (if exists) or creates _template API compatible template settings
+    def self.legacy_index_template_settings(template)
+      template['settings'] ||= {}
+    end
+
+    # Returns (if exists) or creates _index_template API compatible template settings
+    def self.composable_index_template_settings(template)
+      template['template'] ||= {}
+      template['template']['settings'] ||= {}
     end
 
     # Template name - if template_name set, use it

logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.12.4'
+  s.version         = '11.12.5'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

spec/unit/outputs/elasticsearch/template_manager_spec.rb

@@ -1,4 +1,4 @@
-require "logstash/devutils/rspec/spec_helper"
+require_relative "../../../../spec/spec_helper"
 require "logstash/outputs/elasticsearch/template_manager"
 
 describe LogStash::Outputs::ElasticSearch::TemplateManager do
@@ -33,33 +33,83 @@
     end
   end
 
-  describe "index template with ilm settings" do
+  context "index template with ilm settings" do
     let(:plugin_settings) { {"manage_template" => true, "template_overwrite" => true} }
     let(:plugin) { LogStash::Outputs::ElasticSearch.new(plugin_settings) }
 
-    describe "in version 8+" do
-      let(:file_path) { described_class.default_template_path(8) }
-      let(:template) { described_class.read_template_file(file_path)}
+    describe "with custom template" do
 
-      it "should update settings" do
-        expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8)
-        described_class.add_ilm_settings_to_template(plugin, template)
-        expect(template['template']['settings']['index.lifecycle.name']).not_to eq(nil)
-        expect(template['template']['settings']['index.lifecycle.rollover_alias']).not_to eq(nil)
-        expect(template.include?('settings')).to be_falsey
+      describe "in version 8+" do
+        let(:file_path) { described_class.default_template_path(8) }
+        let(:template) { described_class.read_template_file(file_path)}
+
+        it "should update settings" do
+          expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8)
+          described_class.add_ilm_settings_to_template(plugin, template)
+          expect(template['template']['settings']['index.lifecycle.name']).not_to eq(nil)
+          expect(template['template']['settings']['index.lifecycle.rollover_alias']).not_to eq(nil)
+          expect(template.include?('settings')).to be_falsey
+        end
+      end
+
+      describe "in version < 8" do
+        let(:file_path) { described_class.default_template_path(7) }
+        let(:template) { described_class.read_template_file(file_path)}
+
+        it "should update settings" do
+          expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7)
+          described_class.add_ilm_settings_to_template(plugin, template)
+          expect(template['settings']['index.lifecycle.name']).not_to eq(nil)
+          expect(template['settings']['index.lifecycle.rollover_alias']).not_to eq(nil)
+          expect(template.include?('template')).to be_falsey
+        end
       end
     end
 
-    describe "in version < 8" do
-      let(:file_path) { described_class.default_template_path(7) }
-      let(:template) { described_class.read_template_file(file_path)}
+    context "resolve template setting" do
+      let(:plugin_settings) { super().merge({"template_api" => template_api}) }
+
+      describe "with composable template API" do
+        let(:template_api) { "composable" }
 
-      it "should update settings" do
-        expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7)
-        described_class.add_ilm_settings_to_template(plugin, template)
-        expect(template['settings']['index.lifecycle.name']).not_to eq(nil)
-        expect(template['settings']['index.lifecycle.rollover_alias']).not_to eq(nil)
-        expect(template.include?('template')).to be_falsey
+        it 'resolves composable index template API compatible setting' do
+          template = {}
+          described_class.resolve_template_settings(plugin, template)
+          expect(template["template"]["settings"]).not_to eq(nil)
+        end
+      end
+
+      describe "with legacy template API" do
+        let(:template_api) { "legacy" }
+
+        it 'resolves legacy index template API compatible setting' do
+          template = {}
+          described_class.resolve_template_settings(plugin, template)
+          expect(template["settings"]).not_to eq(nil)
+        end
+      end
+
+      describe "with `template_api => 'auto'`" do
+        let(:template_api) { "auto" }
+
+        describe "with ES < 8 versions" do
+
+          it 'resolves legacy index template API compatible setting' do
+            expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7)
+            template = {}
+            described_class.resolve_template_settings(plugin, template)
+            expect(template["settings"]).not_to eq(nil)
+          end
+        end
+
+        describe "with ES >= 8 versions" do
+          it 'resolves composable index template API compatible setting' do
+            expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8)
+            template = {}
+            described_class.resolve_template_settings(plugin, template)
+            expect(template["template"]["settings"]).not_to eq(nil)
+          end
+        end
       end
     end
   end