Add address_field config option to handle nested fields

CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.2.0
+  - feature: Add address_field config option to handle nested fields
+
 ## 3.1.3
   - Support string arrays in network setting
 

docs/index.asciidoc

@@ -34,6 +34,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 |=======================================================================
 |Setting |Input type|Required
 | <<plugins-{type}s-{plugin}-address>> |<<array,array>>|No
+| <<plugins-{type}s-{plugin}-address_field>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-network>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-network_path>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-refresh_interval>>| <<number,number>>|No
@@ -61,6 +62,22 @@ The IP address(es) to check with. Example:
       }
     }
 
+[id="plugins-{type}s-{plugin}-address_field"]
+===== `address_field`
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The field containing IP address(es) to check with. Example:
+[source,ruby]
+    filter {
+      cidr {
+        add_tag => [ "testnet" ]
+        address_field => "[host][ip]"
+        network => [ "192.0.2.0/24" ]
+      }
+    }
+
 [id="plugins-{type}s-{plugin}-network"]
 ===== `network`
 

lib/logstash/filters/cidr.rb

@@ -2,6 +2,7 @@
 require "logstash/filters/base"
 require "logstash/namespace"
 require "ipaddr"
+require 'logstash/plugin_mixins/validator_support/field_reference_validation_adapter'
 
 # The CIDR filter is for checking IP addresses in events against a list of
 # network blocks that might contain it. Multiple addresses can be checked
@@ -10,9 +11,10 @@
 java_import 'java.util.concurrent.locks.ReentrantReadWriteLock'
 
 class LogStash::Filters::CIDR < LogStash::Filters::Base
-
   config_name "cidr"
 
+  extend LogStash::PluginMixins::ValidatorSupport::FieldReferenceValidationAdapter
+
   # The IP address(es) to check with. Example:
   # [source,ruby]
   #     filter {
@@ -24,6 +26,17 @@ class LogStash::Filters::CIDR < LogStash::Filters::Base
   #     }
   config :address, :validate => :array, :default => []
 
+  # The field containing IP address(es) to check with. Example:
+  # [source,ruby]
+  #     filter {
+  #       %PLUGIN% {
+  #         add_tag => [ "testnet" ]
+  #         address_field => "[host][ip]"
+  #         network => [ "192.0.2.0/24" ]
+  #       }
+  #     }
+  config :address_field, :validate => :field_reference
+
   # The IP network(s) to check against. Example:
   # [source,ruby]
   #     filter {
@@ -62,7 +75,7 @@ def register
     @read_lock = rw_lock.readLock
     @write_lock = rw_lock.writeLock
 
-    if @network_path && !@network.empty? #checks if both network and network path are defined in configuration options
+    if @network_path && !@network.empty? # checks if both network and network path are defined in configuration options
       raise LogStash::ConfigurationError, I18n.t(
         "logstash.agent.configuration.invalid_plugin_register",
         :plugin => "filter",
@@ -71,6 +84,15 @@ def register
       )
     end
 
+    if @address_field && !@address.empty? # checks if both address and address_field are defined in configuration options
+      raise LogStash::ConfigurationError, I18n.t(
+        "logstash.agent.configuration.invalid_plugin_register",
+        :plugin => "filter",
+        :type => "cidr",
+        :error => "The configuration options 'address' and 'address_field' are mutually exclusive"
+      )
+    end
+
     if @network_path
       @next_refresh = Time.now + @refresh_interval
       lock_for_write { load_file }
@@ -121,15 +143,13 @@ def load_file
 
   public
   def filter(event)
-    address = @address.collect do |a|
+    ip_addresses = get_addresses(event).each_with_object([]) do |a, ips|
       begin
-        IPAddr.new(event.sprintf(a))
+        ips << IPAddr.new(a)
       rescue ArgumentError => e
         @logger.warn("Invalid IP address, skipping", :address => a, :event => event.to_hash)
-        nil
       end
     end
-    address.compact!
 
     if @network_path #in case we are getting networks from an external file
       if needs_refresh?
@@ -166,12 +186,21 @@ def filter(event)
 
     network.compact! #clean nulls
     # Try every combination of address and network, first match wins
-    address.product(network).each do |a, n|
+    ip_addresses.product(network).each do |a, n|
       @logger.debug("Checking IP inclusion", :address => a, :network => n)
       if n.include?(a)
         filter_matched(event)
         return
       end
     end
   end # def filter
+
+  private
+  def get_addresses(event)
+    if @address_field
+      Array(event.get(@address_field))
+    else
+      @address.collect { |a| event.sprintf(a) }
+    end
+  end
 end # class LogStash::Filters::CIDR

logstash-filter-cidr.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-cidr'
-  s.version         = '3.1.3'
+  s.version         = '3.2.0'
   s.platform        = 'java'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Checks IP addresses against a list of network blocks"
@@ -22,6 +22,7 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency 'logstash-mixin-validator_support', '~>1.0'
 
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'insist'

spec/filters/cidr_spec.rb

@@ -192,4 +192,90 @@
       )
     end
   end
+
+  describe "address_field config option" do
+
+    context 'the field is top-level' do
+
+      config <<-CONFIG
+        filter {
+          cidr {
+            address_field => "ip"
+            network => [ "192.168.0.0/24" ]
+            add_tag => [ "matched" ]
+          }
+        }
+      CONFIG
+
+      context 'the input value is a string' do
+        sample({ "ip" => "192.168.0.1" }) do
+          insist { subject.get("tags") }.include?("matched")
+        end
+      end
+
+      context 'the input value is an array' do
+        sample({ "ip" => [ "188.168.0.1", "192.168.0.1" ] }) do
+          insist { subject.get("tags") }.include?("matched")
+        end
+      end
+
+      context 'the input value contains an invalid ip' do
+        sample({ "ip" => [ "invalid", "192.168.0.1" ] }) do
+          insist { subject.get("tags") }.include?("matched")
+        end
+      end
+
+      context 'the input value is not present on the event' do
+        sample({}) do
+          insist { subject.get("tags").nil? }
+        end
+      end
+
+      context 'the input value is of unacceptable shape (integer)' do
+        sample({ "ip" => 377 }) do
+          insist { subject.get("tags").nil? }
+        end
+      end
+
+      context 'the input value is of unacceptable shape (map)' do
+        sample({ "ip" => { "invalid" => "192.168.0.1" } }) do
+          insist { subject.get("tags").nil? }
+        end
+      end
+    end
+
+    context 'the field is nested' do
+
+      config <<-CONFIG
+        filter {
+          cidr {
+            address_field => "[host][ip]"
+            network => [ "192.168.0.0/24" ]
+            add_tag => [ "matched" ]
+          }
+        }
+      CONFIG
+
+      sample({ "host" => { "ip" => [ "188.168.0.1", "192.168.0.1" ] } }) do
+        insist { subject.get("tags") }.include?("matched")
+      end
+    end
+
+    context 'address and address_field are both defined' do
+
+      let(:config) do
+        {
+          "address_field" => "[host][ip]",
+          "address" => [ "%{clientip}" ]
+        }
+      end
+
+      it "raises an exception" do
+        expect { subject.register }.to raise_error(
+          LogStash::ConfigurationError,
+          /The configuration options 'address' and 'address_field' are mutually exclusive/
+        )
+      end
+    end
+  end
 end