Merge pull request #20 from mlasevich/master

Add ability to inject a field based on matching pattern

Author: repeatedly

README.md

@@ -86,6 +86,43 @@ Use `format` instead of `<parse></parse>`.
       </pattern>
     </filter>
 
+### Adding format identity field
+
+Sometimes it may be useful to know which pattern was used. Since pareser usage
+may not support retagging, there is an option to add a format name field and/or
+index field.
+
+Example:
+
+
+    <filter app.**>
+      @type parser
+      key_name message
+      <parse>
+        @type multi_format
+
+        # if set, add this key to record with value being pattern format name
+        # (format_name key)
+        format_key 'format'
+
+        <pattern>
+          format json
+          # set format name for this pattern. If unset, uses format name
+          # followed by index (in this case would be 'json#0')
+          format_name 'json'
+        </pattern>
+        <pattern>
+          format regexp
+          format_name 'MyRefex'
+          expression /...your regexp pattern.../
+        </pattern>
+        <pattern>
+          format none
+          format_name 'unparsed'
+        </pattern>
+      </parse>
+    </filter>
+
 ### NOTE
 
 This plugin doesn't work with `multiline` parsers because parser itself doesn't store previous lines.

VERSION

@@ -1 +1 @@
-1.0.0
+1.1.0

fluent-plugin-multi-format-parser.gemspec

@@ -9,7 +9,7 @@ Gem::Specification.new do |gem|
   gem.version     = File.read("VERSION").strip
   gem.authors     = ["Masahiro Nakagawa"]
   gem.email       = "[email protected]"
-  gem.has_rdoc    = false
+  #gem.has_rdoc    = false
   #gem.platform    = Gem::Platform::RUBY
   gem.license     = 'Apache License (2.0)'
   gem.files       = `git ls-files`.split("\n")

lib/fluent/plugin/parser_multi_format.rb

@@ -4,31 +4,36 @@ module Fluent
   module Plugin
     class MultiFormatParser < Parser
       Plugin.register_parser('multi_format', self)
+      config_param :format_key, :string, default: nil
 
       def initialize
         super
 
         @parsers = []
+        @parser_format_names = []
       end
 
       def configure(conf)
         super
 
-        conf.elements.each { |e|
+        conf.elements.each_with_index { |e, i|
           next unless ['pattern', 'format'].include?(e.name)
           next if e['format'].nil? && (e['@type'] == 'multi_format')
-
+          @parser_format_names << e.delete('format_name') || ""+e['format']+"#"+i.to_s
           parser = Plugin.new_parser(e['format'])
           parser.configure(e)
           @parsers << parser
         }
       end
 
       def parse(text)
-        @parsers.each { |parser|
+        @parsers.each_with_index { |parser, i|
           begin
             parser.parse(text) { |time, record|
               if time && record
+                if @format_key
+                  record[@format_key] = @parser_format_names[i]
+                end
                 yield time, record
                 return
               end