From 0ee9047583a99dd6847f778bc0963764eb1e7de4 Mon Sep 17 00:00:00 2001 From: Philip Hutchins Date: Fri, 18 Dec 2015 07:42:40 -0500 Subject: [PATCH 1/6] adding configurable sincedb directory and file prefix --- lib/logstash/inputs/file.rb | 36 ++++++++++++++++++++++++++---------- logstash-input-file.gemspec | 2 +- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/lib/logstash/inputs/file.rb b/lib/logstash/inputs/file.rb index 1498176..4216340 100644 --- a/lib/logstash/inputs/file.rb +++ b/lib/logstash/inputs/file.rb @@ -100,6 +100,16 @@ class LogStash::Inputs::File < LogStash::Inputs::Base # NOTE: it must be a file path and not a directory path config :sincedb_path, :validate => :string + # Directory path of where the sincedb file will be written. sincedb_path + # overrides this value so they should not be used together. This has + # the same effect of setting $HOME or $SINCEDB_DIR environment variables. + config :sincedb_dir, :validate => :string + + # Prefix for the sincedb file name. By default the sincedb file is named + # .sincedb*. This allows you to customize the filename but keep the hash + # that is appended to it. + config :sincedb_file_prefix, :validate => :string + # How often (in seconds) to write a since database with the current position of # monitored log files. config :sincedb_write_interval, :validate => :number, :default => 15 @@ -143,21 +153,27 @@ def register end if @sincedb_path.nil? - if ENV["SINCEDB_DIR"].nil? && ENV["HOME"].nil? - @logger.error("No SINCEDB_DIR or HOME environment variable set, I don't know where " \ - "to keep track of the files I'm watching. Either set " \ - "HOME or SINCEDB_DIR in your environment, or set sincedb_path in " \ - "in your Logstash config for the file input with " \ - "path '#{@path.inspect}'") - raise # TODO(sissel): HOW DO I FAIL PROPERLY YO + if @sincedb_dir.nil? + if ENV["SINCEDB_DIR"].nil? && ENV["HOME"].nil? + @logger.error("No SINCEDB_DIR or HOME environment variable set, I don't know where " \ + "to keep track of the files I'm watching. Either set " \ + "HOME or SINCEDB_DIR in your environment, or set sincedb_path in " \ + "in your Logstash config for the file input with " \ + "path '#{@path.inspect}'") + raise # TODO(sissel): HOW DO I FAIL PROPERLY YO + end end - #pick SINCEDB_DIR if available, otherwise use HOME - sincedb_dir = ENV["SINCEDB_DIR"] || ENV["HOME"] + #pick sincedb_path if set, then SINCEDB_DIR if available, otherwise use HOME + sincedb_dir = @sincedb_dir || ENV["SINCEDB_DIR"] || ENV["HOME"] # Join by ',' to make it easy for folks to know their own sincedb # generated path (vs, say, inspecting the @path array) - @sincedb_path = File.join(sincedb_dir, ".sincedb_" + Digest::MD5.hexdigest(@path.join(","))) + path_hex = Digest::MD5.hexdigest(@path.join(",")) + + sincedb_file = @sincedb_file_prefix + path_hex || ".sincedb_" + path_hex + + @sincedb_path = File.join(sincedb_dir, sincedb_file) # Migrate any old .sincedb to the new file (this is for version <=1.1.1 compatibility) old_sincedb = File.join(sincedb_dir, ".sincedb") diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index a7cb459..080ade6 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '2.0.3' + s.version = '2.0.4' s.licenses = ['Apache License (2.0)'] s.summary = "Stream events from files." s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program" From 7fa1acddddc0c0e7d1dd001e1b158512544770ce Mon Sep 17 00:00:00 2001 From: Philip Hutchins Date: Tue, 5 Jan 2016 13:02:22 -0500 Subject: [PATCH 2/6] Updating version and CHANGELOG --- CHANGELOG.md | 3 +++ lib/logstash/inputs/file.rb | 36 ++++++++++++++++++++++++++---------- logstash-input-file.gemspec | 2 +- 3 files changed, 30 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b0c82db..75c6546 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 2.1.4 + - Add configurable path for sincedb file and custom file name prefix + ## 2.1.3 - Use ruby-filewatch 0.7.1, re-enable close after file is modified again diff --git a/lib/logstash/inputs/file.rb b/lib/logstash/inputs/file.rb index bc34d14..500671b 100644 --- a/lib/logstash/inputs/file.rb +++ b/lib/logstash/inputs/file.rb @@ -116,6 +116,16 @@ class LogStash::Inputs::File < LogStash::Inputs::Base # NOTE: it must be a file path and not a directory path config :sincedb_path, :validate => :string + # Directory path of where the sincedb file will be written. sincedb_path + # overrides this value so they should not be used together. This has + # the same effect of setting $HOME or $SINCEDB_DIR environment variables. + config :sincedb_dir, :validate => :string + + # Prefix for the sincedb file name. By default the sincedb file is named + # .sincedb*. This allows you to customize the filename but keep the hash + # that is appended to it. + config :sincedb_file_prefix, :validate => :string + # How often (in seconds) to write a since database with the current position of # monitored log files. config :sincedb_write_interval, :validate => :number, :default => 15 @@ -171,21 +181,27 @@ def register end if @sincedb_path.nil? - if ENV["SINCEDB_DIR"].nil? && ENV["HOME"].nil? - @logger.error("No SINCEDB_DIR or HOME environment variable set, I don't know where " \ - "to keep track of the files I'm watching. Either set " \ - "HOME or SINCEDB_DIR in your environment, or set sincedb_path in " \ - "in your Logstash config for the file input with " \ - "path '#{@path.inspect}'") - raise # TODO(sissel): HOW DO I FAIL PROPERLY YO + if @sincedb_dir.nil? + if ENV["SINCEDB_DIR"].nil? && ENV["HOME"].nil? + @logger.error("No SINCEDB_DIR or HOME environment variable set, I don't know where " \ + "to keep track of the files I'm watching. Either set " \ + "HOME or SINCEDB_DIR in your environment, or set sincedb_path in " \ + "in your Logstash config for the file input with " \ + "path '#{@path.inspect}'") + raise # TODO(sissel): HOW DO I FAIL PROPERLY YO + end end - #pick SINCEDB_DIR if available, otherwise use HOME - sincedb_dir = ENV["SINCEDB_DIR"] || ENV["HOME"] + #pick sincedb_path if set, then SINCEDB_DIR if available, otherwise use HOME + sincedb_dir = @sincedb_dir || ENV["SINCEDB_DIR"] || ENV["HOME"] # Join by ',' to make it easy for folks to know their own sincedb # generated path (vs, say, inspecting the @path array) - @sincedb_path = File.join(sincedb_dir, ".sincedb_" + Digest::MD5.hexdigest(@path.join(","))) + path_hex = Digest::MD5.hexdigest(@path.join(",")) + + sincedb_file = @sincedb_file_prefix + path_hex || ".sincedb_" + path_hex + + @sincedb_path = File.join(sincedb_dir, sincedb_file) # Migrate any old .sincedb to the new file (this is for version <=1.1.1 compatibility) old_sincedb = File.join(sincedb_dir, ".sincedb") diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index 281a9f9..fbaa497 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '2.1.3' + s.version = '2.1.4' s.licenses = ['Apache License (2.0)'] s.summary = "Stream events from files." s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program" From cd0289f2ba40a933e5ef96280113d4aa1e80f292 Mon Sep 17 00:00:00 2001 From: Philip Hutchins Date: Tue, 12 Jan 2016 19:39:46 -0500 Subject: [PATCH 3/6] changes to sincedb_path --- lib/logstash/inputs/file.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/logstash/inputs/file.rb b/lib/logstash/inputs/file.rb index 500671b..470a68f 100644 --- a/lib/logstash/inputs/file.rb +++ b/lib/logstash/inputs/file.rb @@ -124,7 +124,7 @@ class LogStash::Inputs::File < LogStash::Inputs::Base # Prefix for the sincedb file name. By default the sincedb file is named # .sincedb*. This allows you to customize the filename but keep the hash # that is appended to it. - config :sincedb_file_prefix, :validate => :string + config :sincedb_file_prefix, :validate => :string, :default => ".sincedb_" # How often (in seconds) to write a since database with the current position of # monitored log files. @@ -199,7 +199,7 @@ def register # generated path (vs, say, inspecting the @path array) path_hex = Digest::MD5.hexdigest(@path.join(",")) - sincedb_file = @sincedb_file_prefix + path_hex || ".sincedb_" + path_hex + sincedb_file = @sincedb_file_prefix + path_hex @sincedb_path = File.join(sincedb_dir, sincedb_file) From 5097a20ab76176da0211706a3b2539fee33616b3 Mon Sep 17 00:00:00 2001 From: Philip Hutchins Date: Fri, 18 Dec 2015 07:42:40 -0500 Subject: [PATCH 4/6] adding configurable sincedb directory and file prefix --- lib/logstash/inputs/file.rb | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/lib/logstash/inputs/file.rb b/lib/logstash/inputs/file.rb index 39335db..903153c 100644 --- a/lib/logstash/inputs/file.rb +++ b/lib/logstash/inputs/file.rb @@ -120,6 +120,16 @@ class LogStash::Inputs::File < LogStash::Inputs::Base # NOTE: it must be a file path and not a directory path config :sincedb_path, :validate => :string + # Directory path of where the sincedb file will be written. sincedb_path + # overrides this value so they should not be used together. This has + # the same effect of setting $HOME or $SINCEDB_DIR environment variables. + config :sincedb_dir, :validate => :string + + # Prefix for the sincedb file name. By default the sincedb file is named + # .sincedb*. This allows you to customize the filename but keep the hash + # that is appended to it. + config :sincedb_file_prefix, :validate => :string + # How often (in seconds) to write a since database with the current position of # monitored log files. config :sincedb_write_interval, :validate => :number, :default => 15 @@ -175,21 +185,27 @@ def register end if @sincedb_path.nil? - if ENV["SINCEDB_DIR"].nil? && ENV["HOME"].nil? - @logger.error("No SINCEDB_DIR or HOME environment variable set, I don't know where " \ - "to keep track of the files I'm watching. Either set " \ - "HOME or SINCEDB_DIR in your environment, or set sincedb_path in " \ - "in your Logstash config for the file input with " \ - "path '#{@path.inspect}'") - raise # TODO(sissel): HOW DO I FAIL PROPERLY YO + if @sincedb_dir.nil? + if ENV["SINCEDB_DIR"].nil? && ENV["HOME"].nil? + @logger.error("No SINCEDB_DIR or HOME environment variable set, I don't know where " \ + "to keep track of the files I'm watching. Either set " \ + "HOME or SINCEDB_DIR in your environment, or set sincedb_path in " \ + "in your Logstash config for the file input with " \ + "path '#{@path.inspect}'") + raise # TODO(sissel): HOW DO I FAIL PROPERLY YO + end end - #pick SINCEDB_DIR if available, otherwise use HOME - sincedb_dir = ENV["SINCEDB_DIR"] || ENV["HOME"] + #pick sincedb_path if set, then SINCEDB_DIR if available, otherwise use HOME + sincedb_dir = @sincedb_dir || ENV["SINCEDB_DIR"] || ENV["HOME"] # Join by ',' to make it easy for folks to know their own sincedb # generated path (vs, say, inspecting the @path array) - @sincedb_path = File.join(sincedb_dir, ".sincedb_" + Digest::MD5.hexdigest(@path.join(","))) + path_hex = Digest::MD5.hexdigest(@path.join(",")) + + sincedb_file = @sincedb_file_prefix + path_hex || ".sincedb_" + path_hex + + @sincedb_path = File.join(sincedb_dir, sincedb_file) # Migrate any old .sincedb to the new file (this is for version <=1.1.1 compatibility) old_sincedb = File.join(sincedb_dir, ".sincedb") From ff3373ecc854546eff2686f2a8ebb93ad5afb675 Mon Sep 17 00:00:00 2001 From: Philip Hutchins Date: Tue, 5 Jan 2016 13:02:22 -0500 Subject: [PATCH 5/6] Updating version and CHANGELOG --- CHANGELOG.md | 3 +++ logstash-input-file.gemspec | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b0c82db..75c6546 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 2.1.4 + - Add configurable path for sincedb file and custom file name prefix + ## 2.1.3 - Use ruby-filewatch 0.7.1, re-enable close after file is modified again diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index 281a9f9..fbaa497 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '2.1.3' + s.version = '2.1.4' s.licenses = ['Apache License (2.0)'] s.summary = "Stream events from files." s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program" From 10998e1f93c38801bba6a341f216a323005e40ea Mon Sep 17 00:00:00 2001 From: Philip Hutchins Date: Tue, 12 Jan 2016 19:39:46 -0500 Subject: [PATCH 6/6] changes to sincedb_path --- lib/logstash/inputs/file.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/logstash/inputs/file.rb b/lib/logstash/inputs/file.rb index 903153c..38a97a5 100644 --- a/lib/logstash/inputs/file.rb +++ b/lib/logstash/inputs/file.rb @@ -128,7 +128,7 @@ class LogStash::Inputs::File < LogStash::Inputs::Base # Prefix for the sincedb file name. By default the sincedb file is named # .sincedb*. This allows you to customize the filename but keep the hash # that is appended to it. - config :sincedb_file_prefix, :validate => :string + config :sincedb_file_prefix, :validate => :string, :default => ".sincedb_" # How often (in seconds) to write a since database with the current position of # monitored log files. @@ -203,7 +203,7 @@ def register # generated path (vs, say, inspecting the @path array) path_hex = Digest::MD5.hexdigest(@path.join(",")) - sincedb_file = @sincedb_file_prefix + path_hex || ".sincedb_" + path_hex + sincedb_file = @sincedb_file_prefix + path_hex @sincedb_path = File.join(sincedb_dir, sincedb_file)