From 93a8c2e083d30a85a1ba7eec2c4b2ba32d053810 Mon Sep 17 00:00:00 2001 From: Karol Bucek Date: Tue, 4 Aug 2020 17:53:47 +0200 Subject: [PATCH 1/4] Test: spec current BIND9 matching --- spec/patterns/bind_spec.rb | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 spec/patterns/bind_spec.rb diff --git a/spec/patterns/bind_spec.rb b/spec/patterns/bind_spec.rb new file mode 100644 index 00000000..a6636f2f --- /dev/null +++ b/spec/patterns/bind_spec.rb @@ -0,0 +1,21 @@ +# encoding: utf-8 +require "spec_helper" +require "logstash/patterns/core" + +describe_pattern "BIND9" do + + let(:message) do + '17-Feb-2018 23:06:56.326 queries: info: client 172.26.0.1#12345 (test.example.com): query: test.example.com IN A +E(0)K (172.26.0.3)' + end + + it 'matches' do + should include("timestamp" => "17-Feb-2018 23:06:56.326") + should include("loglevel" => "info") + should include("clientip" => "172.26.0.1") + should include("clientport" => "12345") + should include("query" => ["test.example.com", "test.example.com"]) + should include("querytype" => "A +E(0)K") + should include("dns" => "172.26.0.3") + end + +end From 00be5cda23d3b2657d47bb952105cee418c759ef Mon Sep 17 00:00:00 2001 From: Karol Bucek Date: Thu, 6 Aug 2020 08:11:16 +0200 Subject: [PATCH 2/4] Feat: ecs-ize bind query log captures isn't a drop in replacement as querytype field is parsed further --- patterns/ecs-v1/bind | 6 +++++- spec/patterns/bind_spec.rb | 24 +++++++++++++++++------- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/patterns/ecs-v1/bind b/patterns/ecs-v1/bind index 31e4414a..211ef9ac 100644 --- a/patterns/ecs-v1/bind +++ b/patterns/ecs-v1/bind @@ -1,3 +1,7 @@ BIND9_TIMESTAMP %{MONTHDAY}[-]%{MONTH}[-]%{YEAR} %{TIME} -BIND9 %{BIND9_TIMESTAMP:timestamp} queries: %{LOGLEVEL:loglevel}: client %{IP:clientip}#%{POSINT:clientport} \(%{GREEDYDATA:query}\): query: %{GREEDYDATA:query} IN %{GREEDYDATA:querytype} \(%{IP:dns}\) +BIND9_DNSTYPE (?:A|AAAA|CAA|CDNSKEY|CDS|CERT|CNAME|CSYNC|DLV|DNAME|DNSKEY|DS|HINFO|LOC|MX|NAPTR|NS|NSEC|NSEC3|OPENPGPKEY|PTR|RRSIG|RP|SIG|SMIMEA|SOA|SRV|TSIG|TXT|URI) + +# dns.question.class is static - only 'IN' is supported by Bind9 +# bind.log.question.name is expected to be a 'duplicate' (same as the dns.question.name capture) +BIND9 %{BIND9_TIMESTAMP:timestamp} queries: %{LOGLEVEL:[log][level]}: client %{IP:[client][ip]}#%{POSINT:[client][port]:int} \(%{GREEDYDATA:[bind][log][question][name]}\): query: %{GREEDYDATA:[dns][question][name]} (?<[dns][question][class]>IN) %{BIND9_DNSTYPE:[dns][question][type]}(:? %{DATA:[bind][log][question][flags]})? \(%{IP:[server][ip]}\) diff --git a/spec/patterns/bind_spec.rb b/spec/patterns/bind_spec.rb index a6636f2f..099da3c0 100644 --- a/spec/patterns/bind_spec.rb +++ b/spec/patterns/bind_spec.rb @@ -2,7 +2,7 @@ require "spec_helper" require "logstash/patterns/core" -describe_pattern "BIND9" do +describe_pattern "BIND9", ['legacy', 'ecs-v1'] do let(:message) do '17-Feb-2018 23:06:56.326 queries: info: client 172.26.0.1#12345 (test.example.com): query: test.example.com IN A +E(0)K (172.26.0.3)' @@ -10,12 +10,22 @@ it 'matches' do should include("timestamp" => "17-Feb-2018 23:06:56.326") - should include("loglevel" => "info") - should include("clientip" => "172.26.0.1") - should include("clientport" => "12345") - should include("query" => ["test.example.com", "test.example.com"]) - should include("querytype" => "A +E(0)K") - should include("dns" => "172.26.0.3") + if ecs_compatibility? + should include("log" => hash_including("level" => "info")) + should include("client" => { "ip" => "172.26.0.1", "port" => 12345 }) + should include("dns" => { "question" => { "name" => "test.example.com", "type" => 'A', "class" => 'IN' }}) + should include("bind" => { "log" => { "question" => hash_including("flags" => '+E(0)K')}}) + should include("server" => { "ip" => "172.26.0.3" }) + # NOTE: duplicate but still captured since we've been doing that before as well : + should include("bind" => { "log" => { "question" => hash_including("name" => 'test.example.com')}}) + else + should include("loglevel" => "info") + should include("clientip" => "172.26.0.1") + should include("clientport" => "12345") + should include("query" => ["test.example.com", "test.example.com"]) + should include("querytype" => "A +E(0)K") + should include("dns" => "172.26.0.3") + end end end From e11a54fe5590a6a43c193ad25782186efd8755c0 Mon Sep 17 00:00:00 2001 From: Karol Bucek Date: Thu, 6 Aug 2020 10:23:36 +0200 Subject: [PATCH 3/4] support BIND9 query log for 9.11 format --- patterns/ecs-v1/bind | 2 +- patterns/legacy/bind | 2 +- spec/patterns/bind_spec.rb | 28 +++++++++++++++++++++++++++- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/patterns/ecs-v1/bind b/patterns/ecs-v1/bind index 211ef9ac..1703d131 100644 --- a/patterns/ecs-v1/bind +++ b/patterns/ecs-v1/bind @@ -4,4 +4,4 @@ BIND9_DNSTYPE (?:A|AAAA|CAA|CDNSKEY|CDS|CERT|CNAME|CSYNC|DLV|DNAME|DNSKEY|DS|HIN # dns.question.class is static - only 'IN' is supported by Bind9 # bind.log.question.name is expected to be a 'duplicate' (same as the dns.question.name capture) -BIND9 %{BIND9_TIMESTAMP:timestamp} queries: %{LOGLEVEL:[log][level]}: client %{IP:[client][ip]}#%{POSINT:[client][port]:int} \(%{GREEDYDATA:[bind][log][question][name]}\): query: %{GREEDYDATA:[dns][question][name]} (?<[dns][question][class]>IN) %{BIND9_DNSTYPE:[dns][question][type]}(:? %{DATA:[bind][log][question][flags]})? \(%{IP:[server][ip]}\) +BIND9 %{BIND9_TIMESTAMP:timestamp} queries: %{LOGLEVEL:[log][level]}: client(:? @0x(?:[0-9A-Fa-f]+))? %{IP:[client][ip]}#%{POSINT:[client][port]:int} \(%{GREEDYDATA:[bind][log][question][name]}\): query: %{GREEDYDATA:[dns][question][name]} (?<[dns][question][class]>IN) %{BIND9_DNSTYPE:[dns][question][type]}(:? %{DATA:[bind][log][question][flags]})? \(%{IP:[server][ip]}\) diff --git a/patterns/legacy/bind b/patterns/legacy/bind index 31e4414a..683ba360 100644 --- a/patterns/legacy/bind +++ b/patterns/legacy/bind @@ -1,3 +1,3 @@ BIND9_TIMESTAMP %{MONTHDAY}[-]%{MONTH}[-]%{YEAR} %{TIME} -BIND9 %{BIND9_TIMESTAMP:timestamp} queries: %{LOGLEVEL:loglevel}: client %{IP:clientip}#%{POSINT:clientport} \(%{GREEDYDATA:query}\): query: %{GREEDYDATA:query} IN %{GREEDYDATA:querytype} \(%{IP:dns}\) +BIND9 %{BIND9_TIMESTAMP:timestamp} queries: %{LOGLEVEL:loglevel}: client(:? @0x(?:[0-9A-Fa-f]+))? %{IP:clientip}#%{POSINT:clientport} \(%{GREEDYDATA:query}\): query: %{GREEDYDATA:query} IN %{GREEDYDATA:querytype} \(%{IP:dns}\) diff --git a/spec/patterns/bind_spec.rb b/spec/patterns/bind_spec.rb index 099da3c0..a6c6d6e3 100644 --- a/spec/patterns/bind_spec.rb +++ b/spec/patterns/bind_spec.rb @@ -4,7 +4,7 @@ describe_pattern "BIND9", ['legacy', 'ecs-v1'] do - let(:message) do + let(:message) do # Bind 9.10 '17-Feb-2018 23:06:56.326 queries: info: client 172.26.0.1#12345 (test.example.com): query: test.example.com IN A +E(0)K (172.26.0.3)' end @@ -28,4 +28,30 @@ end end + context 'with client memory address (Bind 9.11)' do + + let(:message) do # client @0x7f64500020ef - memory address of the data structure representing the client + '30-Jun-2018 15:50:00.999 queries: info: client @0x7f64500020ef 192.168.10.48#60061 (91.2.10.170.in-addr.internal): query: 91.2.10.170.in-addr.internal IN PTR + (192.168.2.2)' + end + + it 'matches' do + should include("timestamp" => "30-Jun-2018 15:50:00.999") + if ecs_compatibility? + should include("log" => hash_including("level" => "info")) + should include("client" => { "ip" => "192.168.10.48", "port" => 60061 }) + should include("dns" => { "question" => { "name" => "91.2.10.170.in-addr.internal", "type" => 'PTR', "class" => 'IN' }}) + should include("bind" => { "log" => { "question" => hash_including("flags" => '+')}}) + should include("server" => { "ip" => "192.168.2.2" }) + else + should include("loglevel" => "info") + should include("clientip" => "192.168.10.48") + should include("clientport" => "60061") + should include("query" => ["91.2.10.170.in-addr.internal", "91.2.10.170.in-addr.internal"]) + should include("querytype" => "PTR +") + should include("dns" => "192.168.2.2") + end + end + + end + end From 6e6626e267d2f80a693e437b9a1dd5297364fe71 Mon Sep 17 00:00:00 2001 From: Karol Bucek Date: Mon, 10 Aug 2020 16:57:30 +0200 Subject: [PATCH 4/4] Refactor: split BIND9 to a re-usable parts for potential user-extensions of BIND9 query-log parsing --- patterns/ecs-v1/bind | 8 +++++++- spec/patterns/bind_spec.rb | 23 ++++++++++++++++++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/patterns/ecs-v1/bind b/patterns/ecs-v1/bind index 1703d131..9d07e0d3 100644 --- a/patterns/ecs-v1/bind +++ b/patterns/ecs-v1/bind @@ -1,7 +1,13 @@ BIND9_TIMESTAMP %{MONTHDAY}[-]%{MONTH}[-]%{YEAR} %{TIME} BIND9_DNSTYPE (?:A|AAAA|CAA|CDNSKEY|CDS|CERT|CNAME|CSYNC|DLV|DNAME|DNSKEY|DS|HINFO|LOC|MX|NAPTR|NS|NSEC|NSEC3|OPENPGPKEY|PTR|RRSIG|RP|SIG|SMIMEA|SOA|SRV|TSIG|TXT|URI) +BIND9_CATEGORY (?:queries) # dns.question.class is static - only 'IN' is supported by Bind9 # bind.log.question.name is expected to be a 'duplicate' (same as the dns.question.name capture) -BIND9 %{BIND9_TIMESTAMP:timestamp} queries: %{LOGLEVEL:[log][level]}: client(:? @0x(?:[0-9A-Fa-f]+))? %{IP:[client][ip]}#%{POSINT:[client][port]:int} \(%{GREEDYDATA:[bind][log][question][name]}\): query: %{GREEDYDATA:[dns][question][name]} (?<[dns][question][class]>IN) %{BIND9_DNSTYPE:[dns][question][type]}(:? %{DATA:[bind][log][question][flags]})? \(%{IP:[server][ip]}\) +BIND9_QUERYLOGBASE client(:? @0x(?:[0-9A-Fa-f]+))? %{IP:[client][ip]}#%{POSINT:[client][port]:int} \(%{GREEDYDATA:[bind][log][question][name]}\): query: %{GREEDYDATA:[dns][question][name]} (?<[dns][question][class]>IN) %{BIND9_DNSTYPE:[dns][question][type]}(:? %{DATA:[bind][log][question][flags]})? \(%{IP:[server][ip]}\) + +# for query-logging category and severity are always fixed as "queries: info: " +BIND9_QUERYLOG %{BIND9_TIMESTAMP:timestamp} %{BIND9_CATEGORY:[bing][log][category]}: %{LOGLEVEL:[log][level]}: %{BIND9_QUERYLOGBASE} + +BIND9 %{BIND9_QUERYLOG} diff --git a/spec/patterns/bind_spec.rb b/spec/patterns/bind_spec.rb index a6c6d6e3..962e38e1 100644 --- a/spec/patterns/bind_spec.rb +++ b/spec/patterns/bind_spec.rb @@ -28,7 +28,15 @@ end end - context 'with client memory address (Bind 9.11)' do + context 'with client memory address (since Bind 9.11)' do + # logging format is the same <= 9.16, but if using a separate query-log all options need to be enabled : + # channel query.log { + # file "/var/log/named/query.log"; + # severity debug 3; + # //print-time YES; // @timestamp + # //print-category YES; // queries: + # //print-severity YES; // info: + # }; let(:message) do # client @0x7f64500020ef - memory address of the data structure representing the client '30-Jun-2018 15:50:00.999 queries: info: client @0x7f64500020ef 192.168.10.48#60061 (91.2.10.170.in-addr.internal): query: 91.2.10.170.in-addr.internal IN PTR + (192.168.2.2)' @@ -55,3 +63,16 @@ end end + +describe_pattern "BIND9_QUERYLOGBASE", ['ecs-v1'] do + let(:message) do + 'client @0x7f85b4026ed0 127.0.0.1#42520 (ci.elastic.co): query: ci.elastic.co IN A +E(0)K (35.193.103.164)' + end + + it 'matches' do + should include("client" => { "ip" => "127.0.0.1", "port" => 42520 }) + should include("dns" => { "question" => { "name" => "ci.elastic.co", "type" => 'A', "class" => 'IN' }}) + should include("bind" => { "log" => { "question" => hash_including("flags" => '+E(0)K') }}) + should include("server" => { "ip" => "35.193.103.164" }) + end +end