Skip to content

Commit ebeafcd

Browse files
staaamMichael Gelfand
authored and
Michael Gelfand
committed
Remove empty event action params
In event_action_params() clean params from nil and empty values - when using dynamic fields (like document_id => "%{[@metadata][document_id]}"), empty or nil values could be produced for document_id and parent fields (in es_bulk metadata part), which is not valid es_bulk metadata format, thus those values should be cleaned.
1 parent 2e944ef commit ebeafcd

File tree

3 files changed

+58
-0
lines changed

3 files changed

+58
-0
lines changed

lib/logstash/outputs/elasticsearch.rb

+7
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,13 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
161161
# to see if they have come back to life
162162
config :resurrect_delay, :validate => :number, :default => 5
163163

164+
# Control whether to remove empty valued params from action, specifically parent, _id, _routing
165+
# In event_action_params() clean params from nil and empty values - when using dynamic fields
166+
# (like document_id => "%{[@metadata][document_id]}"), empty or nil values could be produced
167+
# for document_id and parent fields (in es_bulk metadata part), which is not valid es_bulk
168+
# metadata format, thus those values should be cleaned. Default: false
169+
config :remove_empty_action_params, :validate => :boolean, :default => false
170+
164171
def build_client
165172
@client = ::LogStash::Outputs::ElasticSearch::HttpClientBuilder.build(@logger, @hosts, params)
166173
end

lib/logstash/outputs/elasticsearch/common.rb

+4
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,10 @@ def event_action_params(event)
155155
params[:_retry_on_conflict] = @retry_on_conflict
156156
end
157157

158+
if @remove_empty_action_params
159+
params = params.delete_if { |k,v| v.nil? or (v.is_a? String and v.empty?) }
160+
end
161+
158162
params
159163
end
160164

spec/unit/outputs/elasticsearch_spec.rb

+47
Original file line numberDiff line numberDiff line change
@@ -262,6 +262,53 @@
262262
end
263263
end
264264

265+
describe "remove empty params from action params" do
266+
let(:options) { {"parent" => "%{myparent}", "document_id" => "%{myid}"} }
267+
subject(:eso) {LogStash::Outputs::ElasticSearch.new(options)}
268+
269+
let(:event_full) { LogStash::Event.new("message" => "blah", "myid" => "mytestid", "myparent" => "mytestparent") }
270+
let(:event_partial_missing) { LogStash::Event.new("message" => "blah") }
271+
let(:event_partial_empty) { LogStash::Event.new("message" => "blah", "myid" => "", "myparent" => "") }
272+
273+
context "when remove_empty_action_params enabled" do
274+
let(:options) { super.merge("remove_empty_action_params" => true) }
275+
276+
it "should interpolate the requested id and parent values" do
277+
params = eso.event_action_params(event_full)
278+
expect(params).to include(:_id => "mytestid", :parent => "mytestparent")
279+
end
280+
281+
it "should interpolate the requested id and parent values and remove them when they missing" do
282+
params = eso.event_action_params(event_partial_missing)
283+
expect(params).not_to include(:_id, :parent)
284+
end
285+
286+
it "should interpolate the requested id and parent values and remove them when they empty" do
287+
params = eso.event_action_params(event_partial_empty)
288+
expect(params).not_to include(:_id, :parent)
289+
end
290+
end
291+
292+
context "when remove_empty_action_params disabled" do
293+
let(:options) { super.merge("remove_empty_action_params" => false) }
294+
295+
it "should interpolate the requested id and parent values" do
296+
params = eso.event_action_params(event_full)
297+
expect(params).to include({:_id => "mytestid", :parent => "mytestparent"})
298+
end
299+
300+
it "should interpolate the requested id and parent values and not remove them when they missing" do
301+
params = eso.event_action_params(event_partial_missing)
302+
expect(params).to include(:_id => nil, :parent => nil)
303+
end
304+
305+
it "should interpolate the requested id and parent values and not remove them when they empty" do
306+
params = eso.event_action_params(event_partial_missing)
307+
expect(params).to include(:_id => "", :parent => "")
308+
end
309+
end
310+
end
311+
265312
describe "sleep interval calculation" do
266313
let(:retry_max_interval) { 64 }
267314
subject(:eso) { LogStash::Outputs::ElasticSearch.new("retry_max_interval" => retry_max_interval) }

0 commit comments

Comments
 (0)