Feat: ecs-ize BIND9 query log captures

patterns/ecs-v1/bind

@@ -1,3 +1,13 @@
 BIND9_TIMESTAMP %{MONTHDAY}[-]%{MONTH}[-]%{YEAR} %{TIME}
 
-BIND9 %{BIND9_TIMESTAMP:timestamp} queries: %{LOGLEVEL:loglevel}: client %{IP:clientip}#%{POSINT:clientport} \(%{GREEDYDATA:query}\): query: %{GREEDYDATA:query} IN %{GREEDYDATA:querytype} \(%{IP:dns}\)
+BIND9_DNSTYPE (?:A|AAAA|CAA|CDNSKEY|CDS|CERT|CNAME|CSYNC|DLV|DNAME|DNSKEY|DS|HINFO|LOC|MX|NAPTR|NS|NSEC|NSEC3|OPENPGPKEY|PTR|RRSIG|RP|SIG|SMIMEA|SOA|SRV|TSIG|TXT|URI)
+BIND9_CATEGORY (?:queries)
+
+# dns.question.class is static - only 'IN' is supported by Bind9
+# bind.log.question.name is expected to be a 'duplicate' (same as the dns.question.name capture)
+BIND9_QUERYLOGBASE client(:? @0x(?:[0-9A-Fa-f]+))? %{IP:[client][ip]}#%{POSINT:[client][port]:int} \(%{GREEDYDATA:[bind][log][question][name]}\): query: %{GREEDYDATA:[dns][question][name]} (?<[dns][question][class]>IN) %{BIND9_DNSTYPE:[dns][question][type]}(:? %{DATA:[bind][log][question][flags]})? \(%{IP:[server][ip]}\)
+
+# for query-logging category and severity are always fixed as "queries: info: "
+BIND9_QUERYLOG %{BIND9_TIMESTAMP:timestamp} %{BIND9_CATEGORY:[bing][log][category]}: %{LOGLEVEL:[log][level]}: %{BIND9_QUERYLOGBASE}
+
+BIND9 %{BIND9_QUERYLOG}

patterns/legacy/bind

@@ -1,3 +1,3 @@
 BIND9_TIMESTAMP %{MONTHDAY}[-]%{MONTH}[-]%{YEAR} %{TIME}
 
-BIND9 %{BIND9_TIMESTAMP:timestamp} queries: %{LOGLEVEL:loglevel}: client %{IP:clientip}#%{POSINT:clientport} \(%{GREEDYDATA:query}\): query: %{GREEDYDATA:query} IN %{GREEDYDATA:querytype} \(%{IP:dns}\)
+BIND9 %{BIND9_TIMESTAMP:timestamp} queries: %{LOGLEVEL:loglevel}: client(:? @0x(?:[0-9A-Fa-f]+))? %{IP:clientip}#%{POSINT:clientport} \(%{GREEDYDATA:query}\): query: %{GREEDYDATA:query} IN %{GREEDYDATA:querytype} \(%{IP:dns}\)

spec/patterns/bind_spec.rb

@@ -0,0 +1,78 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/patterns/core"
+
+describe_pattern "BIND9", ['legacy', 'ecs-v1'] do
+
+  let(:message) do # Bind 9.10
+    '17-Feb-2018 23:06:56.326 queries: info: client 172.26.0.1#12345 (test.example.com): query: test.example.com IN A +E(0)K (172.26.0.3)'
+  end
+
+  it 'matches' do
+    should include("timestamp" => "17-Feb-2018 23:06:56.326")
+    if ecs_compatibility?
+      should include("log" => hash_including("level" => "info"))
+      should include("client" => { "ip" => "172.26.0.1", "port" => 12345 })
+      should include("dns" => { "question" => { "name" => "test.example.com", "type" => 'A', "class" => 'IN' }})
+      should include("bind" => { "log" => { "question" => hash_including("flags" => '+E(0)K')}})
+      should include("server" => { "ip" => "172.26.0.3" })
+      # NOTE: duplicate but still captured since we've been doing that before as well :
+      should include("bind" => { "log" => { "question" => hash_including("name" => 'test.example.com')}})
+    else
+      should include("loglevel" => "info")
+      should include("clientip" => "172.26.0.1")
+      should include("clientport" => "12345")
+      should include("query" => ["test.example.com", "test.example.com"])
+      should include("querytype" => "A +E(0)K")
+      should include("dns" => "172.26.0.3")
+    end
+  end
+
+  context 'with client memory address (since Bind 9.11)' do
+    # logging format is the same <= 9.16, but if using a separate query-log all options need to be enabled :
+    #   channel query.log {
+    #     file "/var/log/named/query.log";
+    #     severity debug 3;
+    #     //print-time YES; // @timestamp
+    #     //print-category YES; // queries:
+    #     //print-severity YES; // info:
+    #   };
+
+    let(:message) do # client @0x7f64500020ef - memory address of the data structure representing the client
+      '30-Jun-2018 15:50:00.999 queries: info: client @0x7f64500020ef 192.168.10.48#60061 (91.2.10.170.in-addr.internal): query: 91.2.10.170.in-addr.internal IN PTR + (192.168.2.2)'
+    end
+
+    it 'matches' do
+      should include("timestamp" => "30-Jun-2018 15:50:00.999")
+      if ecs_compatibility?
+        should include("log" => hash_including("level" => "info"))
+        should include("client" => { "ip" => "192.168.10.48", "port" => 60061 })
+        should include("dns" => { "question" => { "name" => "91.2.10.170.in-addr.internal", "type" => 'PTR', "class" => 'IN' }})
+        should include("bind" => { "log" => { "question" => hash_including("flags" => '+')}})
+        should include("server" => { "ip" => "192.168.2.2" })
+      else
+        should include("loglevel" => "info")
+        should include("clientip" => "192.168.10.48")
+        should include("clientport" => "60061")
+        should include("query" => ["91.2.10.170.in-addr.internal", "91.2.10.170.in-addr.internal"])
+        should include("querytype" => "PTR +")
+        should include("dns" => "192.168.2.2")
+      end
+    end
+
+  end
+
+end
+
+describe_pattern "BIND9_QUERYLOGBASE", ['ecs-v1'] do
+  let(:message) do
+    'client @0x7f85b4026ed0 127.0.0.1#42520 (ci.elastic.co): query: ci.elastic.co IN A +E(0)K (35.193.103.164)'
+  end
+
+  it 'matches' do
+    should include("client" => { "ip" => "127.0.0.1", "port" => 42520 })
+    should include("dns" => { "question" => { "name" => "ci.elastic.co", "type" => 'A', "class" => 'IN' }})
+    should include("bind" => { "log" => { "question" => hash_including("flags" => '+E(0)K') }})
+    should include("server" => { "ip" => "35.193.103.164" })
+  end
+end