From 6c793d703b7296110cc5b75c768bf562a6d1ca2d Mon Sep 17 00:00:00 2001 From: Pere Urbon-Bayes Date: Mon, 16 Nov 2015 23:26:55 +0100 Subject: [PATCH 1/7] enhanced response when a file the plugin is handling gets delated. with this change the deleted file can either be recreated or the events appended to the errorlog Conflicts: lib/logstash/outputs/file.rb --- lib/logstash/outputs/file.rb | 39 ++++++++++++++++++++++---- spec/outputs/file_spec.rb | 53 ++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 6 deletions(-) diff --git a/lib/logstash/outputs/file.rb b/lib/logstash/outputs/file.rb index f79d77a..fc58d39 100644 --- a/lib/logstash/outputs/file.rb +++ b/lib/logstash/outputs/file.rb @@ -11,6 +11,8 @@ class LogStash::Outputs::File < LogStash::Outputs::Base config_name "file" + attr_reader :failure_path + # The path to the file to write. Event fields can be used here, # like `/var/log/logstash/%{host}/%{application}` # One may also utilize the path option for date-based log @@ -42,6 +44,11 @@ class LogStash::Outputs::File < LogStash::Outputs::Base # into this file and inside the defined path. config :filename_failure, :validate => :string, :default => '_filepath_failures' + # If the a file is deleted, but an event is comming with the need to be stored + # in such a file, the plugin will created a gain this file. Default => true + config :create_if_deleted, :validate => :boolean, :default => true + + public def register require "fileutils" # For mkdir_p @@ -56,8 +63,10 @@ def register if path_with_field_ref? @file_root = extract_file_root - @failure_path = File.join(@file_root, @filename_failure) + else + @file_root = File.dirname(path) end + @failure_path = File.join(@file_root, @filename_failure) now = Time.now @last_flush_cycle = now @@ -94,6 +103,8 @@ def receive(event) if path_with_field_ref? && !inside_file_root?(file_output_path) @logger.warn("File: the event tried to write outside the files root, writing the event to the failure file", :event => event, :filename => @failure_path) file_output_path = @failure_path + elsif !@create_if_deleted && deleted?(file_output_path) + file_output_path = @failure_path end output = format_message(event) @@ -124,7 +135,6 @@ def inside_file_root?(log_path) def write_event(log_path, event) @logger.debug("File, writing event to file.", :filename => log_path) fd = open(log_path) - # TODO(sissel): Check if we should rotate the file. fd.write(event) @@ -199,9 +209,27 @@ def close_stale_files end private - def open(path) - return @files[path] if @files.include?(path) and not @files[path].nil? + def cached?(path) + @files.include?(path) && !@files[path].nil? + end + private + def deleted?(path) + !File.exist?(path) + end + + private + def open(path) + if !deleted?(path) && cached?(path) + return @files[path] + elsif deleted?(path) + if @create_if_deleted + @logger.debug("Required path was deleted, creating the file again", :path => path) + @files.delete(path) + else + return @files[path] if cached?(path) + end + end @logger.info("Opening file", :path => path) dir = File.dirname(path) @@ -209,13 +237,12 @@ def open(path) @logger.info("Creating directory", :directory => dir) FileUtils.mkdir_p(dir) end - # work around a bug opening fifos (bug JRUBY-6280) stat = File.stat(path) rescue nil if stat && stat.ftype == "fifo" && LogStash::Environment.jruby? fd = java.io.FileWriter.new(java.io.File.new(path)) else - fd = File.new(path, "a") + fd = File.new(path, "a+") end if gzip fd = Zlib::GzipWriter.new(fd) diff --git a/spec/outputs/file_spec.rb b/spec/outputs/file_spec.rb index 507ac1e..b4a0a97 100644 --- a/spec/outputs/file_spec.rb +++ b/spec/outputs/file_spec.rb @@ -6,6 +6,7 @@ require "stud/temporary" require "tempfile" require "uri" +require "fileutils" describe LogStash::Outputs::File do describe "ship lots of events to a file" do @@ -106,6 +107,58 @@ end describe "receiving events" do + + context "when the output file is deleted" do + + let(:temp_file) { Tempfile.new('logstash-spec-output-file_deleted') } + + let(:config) do + { "path" => temp_file.path, "flush_interval" => 0 } + end + + it "should recreate the required file if deleted" do + output = LogStash::Outputs::File.new(config) + output.register + + 10.times do |i| + event = LogStash::Event.new("event_id" => i) + output.receive(event) + end + FileUtils.rm(temp_file) + 10.times do |i| + event = LogStash::Event.new("event_id" => i+10) + output.receive(event) + end + expect(FileTest.size(temp_file.path)).to be > 0 + end + + context "when appending to the error log" do + + let(:config) do + { "path" => temp_file.path, "flush_interval" => 0, "create_if_deleted" => false } + end + + it "should append the events to the filename_failure location" do + output = LogStash::Outputs::File.new(config) + output.register + + 10.times do |i| + event = LogStash::Event.new("event_id" => i) + output.receive(event) + end + FileUtils.rm(temp_file) + 10.times do |i| + event = LogStash::Event.new("event_id" => i+10) + output.receive(event) + end + expect(FileTest.exist?(temp_file.path)).to be_falsey + expect(FileTest.size(output.failure_path)).to be > 0 + end + + end + + end + context "when using an interpolated path" do context "when trying to write outside the files root directory" do let(:bad_event) do From 527e3ef05a3b1c76ab57f772b5f4d799a7dc9e7c Mon Sep 17 00:00:00 2001 From: Pere Urbon-Bayes Date: Thu, 19 Nov 2015 10:37:01 +0100 Subject: [PATCH 2/7] fix gemspec to fetch last available 1.x logstash-core, also enforced the use of concurrent-ruby 0.9.1 --- logstash-output-file.gemspec | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/logstash-output-file.gemspec b/logstash-output-file.gemspec index 8692ff5..5407095 100644 --- a/logstash-output-file.gemspec +++ b/logstash-output-file.gemspec @@ -20,8 +20,9 @@ Gem::Specification.new do |s| s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" } # Gem dependencies - s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0' + s.add_runtime_dependency "logstash-core", '~> 1.4' s.add_runtime_dependency 'logstash-input-generator' + s.add_runtime_dependency 'concurrent-ruby', '0.9.1' s.add_development_dependency 'logstash-devutils' end From 5e71bf80bc74295a5b957a6b572b7bfc7f4835a6 Mon Sep 17 00:00:00 2001 From: Pere Urbon-Bayes Date: Thu, 19 Nov 2015 10:37:51 +0100 Subject: [PATCH 3/7] version 1.1.0 bump --- CHANGELOG.md | 4 ++++ logstash-output-file.gemspec | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e69de29..eec21b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -0,0 +1,4 @@ +## 1.1.0 + - Add create_if_deleted option to create a destination file in case it + was deleted by another agent in the machine. In case of being false + the system will add the incomming messages to the failure file. diff --git a/logstash-output-file.gemspec b/logstash-output-file.gemspec index 5407095..ea5b5c2 100644 --- a/logstash-output-file.gemspec +++ b/logstash-output-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-output-file' - s.version = '1.0.0' + s.version = '1.1.0' s.licenses = ['Apache License (2.0)'] s.summary = "This output will write events to files on disk" s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program" From be576cf9bbfa05519b814a9ddc2e4ab46a746fca Mon Sep 17 00:00:00 2001 From: Pere Urbon-Bayes Date: Wed, 25 Nov 2015 10:18:07 +0100 Subject: [PATCH 4/7] Revert "fix gemspec to fetch last available 1.x logstash-core, also enforced the use of concurrent-ruby 0.9.1" This reverts commit 527e3ef05a3b1c76ab57f772b5f4d799a7dc9e7c. --- logstash-output-file.gemspec | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/logstash-output-file.gemspec b/logstash-output-file.gemspec index ea5b5c2..ba2c35a 100644 --- a/logstash-output-file.gemspec +++ b/logstash-output-file.gemspec @@ -20,9 +20,8 @@ Gem::Specification.new do |s| s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" } # Gem dependencies - s.add_runtime_dependency "logstash-core", '~> 1.4' + s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0' s.add_runtime_dependency 'logstash-input-generator' - s.add_runtime_dependency 'concurrent-ruby', '0.9.1' s.add_development_dependency 'logstash-devutils' end From 6aeec798d7b9a34c1ee7272b7cd816f25dd7ec11 Mon Sep 17 00:00:00 2001 From: Pere Urbon-Bayes Date: Wed, 25 Nov 2015 10:22:18 +0100 Subject: [PATCH 5/7] added version constrains for concurrent-ruby and logstash-core in the Gemfile as requested during review until the right expressions are agreed to be placed in the gemspec file --- Gemfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index d926697..d3dfeab 100644 --- a/Gemfile +++ b/Gemfile @@ -1,2 +1,4 @@ source 'https://rubygems.org' -gemspec \ No newline at end of file +gemspec +gem "concurrent-ruby", "0.9.1" +gem "logstash-core", "~> 1.4" From c3ccb6a3cbda723cf5f01da73af4e508ec8cb8a7 Mon Sep 17 00:00:00 2001 From: Pere Urbon-Bayes Date: Wed, 2 Dec 2015 15:33:10 +0100 Subject: [PATCH 6/7] Ammend the logstash-core dependency requirements as discussed --- logstash-output-file.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logstash-output-file.gemspec b/logstash-output-file.gemspec index ba2c35a..cb5f950 100644 --- a/logstash-output-file.gemspec +++ b/logstash-output-file.gemspec @@ -20,7 +20,7 @@ Gem::Specification.new do |s| s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" } # Gem dependencies - s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0' + s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 1.999.0' s.add_runtime_dependency 'logstash-input-generator' s.add_development_dependency 'logstash-devutils' From b9efa8fa4ac28d5807760a5a601dc0aec51c39ad Mon Sep 17 00:00:00 2001 From: Pere Urbon-Bayes Date: Wed, 2 Dec 2015 15:33:56 +0100 Subject: [PATCH 7/7] updated Gemfile content fixing only the concurrent-ruby --- Gemfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Gemfile b/Gemfile index d3dfeab..85f4d1c 100644 --- a/Gemfile +++ b/Gemfile @@ -1,4 +1,3 @@ source 'https://rubygems.org' gemspec gem "concurrent-ruby", "0.9.1" -gem "logstash-core", "~> 1.4"