From 7275ea501f3bdfacd365688ccea59755eae4d327 Mon Sep 17 00:00:00 2001 From: "Dale J. Rodriguez" <37818051+dalejrodriguez@users.noreply.github.com> Date: Thu, 24 Jan 2019 10:49:30 -0500 Subject: [PATCH 1/4] Update list_hosts.py Worked with engineering to add support for adding the container count with the host names and also for showing a larger number of hosts. The original script only showed about 10 hosts per environment. --- examples/list_hosts.py | 56 ++++++++++++++---------------------------- 1 file changed, 19 insertions(+), 37 deletions(-) diff --git a/examples/list_hosts.py b/examples/list_hosts.py index 9e4cf605..499f374b 100755 --- a/examples/list_hosts.py +++ b/examples/list_hosts.py @@ -2,15 +2,16 @@ # # This script shows how to leverage sysdig's data query API to obtain the list # of the instrumented hosts that have been seen in the last 5 minutes. +# Updated 1/24/2019 - Dale J. Rodriguez +# This script will also show the container count in addition to the hostnames in a from like this: host.name 12 +# Where 12 is the container count and host.name is the host # - import getopt import json import os import sys sys.path.insert(0, os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), '..')) from sdcclient import SdcClient - # # Parse arguments # @@ -20,57 +21,38 @@ def usage(): print '-j|--json: Print output as json' print 'You can find your token at https://app.sysdigcloud.com/#/settings/user' sys.exit(1) - try: opts, args = getopt.getopt(sys.argv[1:],"jd:",["json", "duration="]) except getopt.GetoptError: usage() - -duration = 600 +duration = 3600 print_json = False for opt, arg in opts: if opt in ("-d", "--duration"): duration = int(arg) elif opt in ("-j", "--json"): print_json = True - -if len(args) != 1: - usage() - sdc_token = args[0] - # # Instantiate the SDC client # sdclient = SdcClient(sdc_token) - -# -# Prepare the query's metrics list. In this case, we have just one metric: -# host.hostName. This is a 'key' metric, and we don't include any number metric. -# Essentially, we create an 'enumeration' of hostnames. -# -metrics = [{"id": "host.hostName"}] - -# -# Fire the query. -# Note: there's no sampling time. This means that we're requesting the result to -# come as a single sample. -# -res = sdclient.get_data(metrics, # metrics list - -duration, # cover the last duration seconds... - 0, # ... ending now... - duration) # ... with just one durations sample - -# -# Show the results! -# +metrics = [{"id": "host.hostName"}, {"id": "container.count","aggregations":{"time":"avg","group":"avg"}}] +res = sdclient.get_data( + metrics, -duration, 0, duration, paging={ + "from": 0, + "to": 999 + }) if res[0]: - data = res[1] + output = [] + data = res[1]['data'] + for i in range(0, len(data)): + sample = data[i] + metrics = sample['d'] + hostName = metrics[0] + count = metrics[1] + output.append('%s\t%d' % (hostName, count)) + print '\n'.join(output) else: print res[1] sys.exit(1) - -if print_json: - print json.dumps(data) -else: - print data From b2986b36bec6947dae4ab594f209be7c0d2057b2 Mon Sep 17 00:00:00 2001 From: "Dale J. Rodriguez" <37818051+dalejrodriguez@users.noreply.github.com> Date: Thu, 24 Jan 2019 14:34:56 -0500 Subject: [PATCH 2/4] Update list_hosts.py --- examples/list_hosts.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/examples/list_hosts.py b/examples/list_hosts.py index 499f374b..27000019 100755 --- a/examples/list_hosts.py +++ b/examples/list_hosts.py @@ -1,9 +1,15 @@ #!/usr/bin/env python # -# This script shows how to leverage sysdig's data query API to obtain the list -# of the instrumented hosts that have been seen in the last 5 minutes. -# Updated 1/24/2019 - Dale J. Rodriguez -# This script will also show the container count in addition to the hostnames in a from like this: host.name 12 +## This script shows how to leverage sysdig's data query API to obtain the list +# of the instrumented hosts that have been seen in the last hour (`duration` variable). +# The output will show the container count (`container.count` metric) in addition to the hostnames (`host.hostName` tag) in a from like this: +# +# host-1 12 +# host-2 4 +# +# host-1 12 +# host-2 4 +#his: host.name 12 # Where 12 is the container count and host.name is the host # import getopt @@ -34,8 +40,13 @@ def usage(): print_json = True sdc_token = args[0] # -# Instantiate the SDC client # +# Prepare the query's metrics list. +# In this case, we have one tag (used for segmentation) and one metric:: +# host.hostName. This is a tag, to identify each item of the output. +# container.count: This is the metric +# +# Instantiate the SDC client sdclient = SdcClient(sdc_token) metrics = [{"id": "host.hostName"}, {"id": "container.count","aggregations":{"time":"avg","group":"avg"}}] res = sdclient.get_data( From d6fc3bd6c517553610f302cdb969e82d54eee77b Mon Sep 17 00:00:00 2001 From: davideschiera Date: Fri, 25 Jan 2019 10:26:49 -0800 Subject: [PATCH 3/4] Add count param, minor refactoring and update docs --- examples/list_hosts.py | 80 +++++++++++++++++++++++++++--------------- 1 file changed, 52 insertions(+), 28 deletions(-) diff --git a/examples/list_hosts.py b/examples/list_hosts.py index 27000019..c3854fef 100755 --- a/examples/list_hosts.py +++ b/examples/list_hosts.py @@ -1,16 +1,15 @@ #!/usr/bin/env python # -## This script shows how to leverage sysdig's data query API to obtain the list -# of the instrumented hosts that have been seen in the last hour (`duration` variable). -# The output will show the container count (`container.count` metric) in addition to the hostnames (`host.hostName` tag) in a from like this: +# This script shows how to leverage Sysdig data query API to obtain the list of the instrumented +# hosts that have been seen in your infrastructure. +# The output will show the container count (`container.count` metric) in addition to the +# hostnames (`host.hostName` tag) in a format like this: # # host-1 12 # host-2 4 # -# host-1 12 -# host-2 4 -#his: host.name 12 -# Where 12 is the container count and host.name is the host +# where the first column is the hostname and the second column is the number of containers running +# in each host. # import getopt import json @@ -18,52 +17,77 @@ import sys sys.path.insert(0, os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), '..')) from sdcclient import SdcClient + # # Parse arguments # def usage(): - print 'usage: %s [-j|--json] [-d|--duration ] ' % sys.argv[0] - print '-d|--duration: List hosts seen in the last seconds' + print 'usage: %s [-j|--json] [-d|--duration ] [-c|--count ] ' % sys.argv[0] + print '-d|--duration: List hosts seen in the last seconds (default: 3600, ie. last hour)' + print '-c|--count: Number of hosts to print (default: 100)' print '-j|--json: Print output as json' print 'You can find your token at https://app.sysdigcloud.com/#/settings/user' sys.exit(1) try: - opts, args = getopt.getopt(sys.argv[1:],"jd:",["json", "duration="]) + opts, args = getopt.getopt(sys.argv[1:],"jd:c:",["json", "duration=", "count="]) except getopt.GetoptError: usage() + duration = 3600 +count = 100 print_json = False for opt, arg in opts: + print opt + print arg if opt in ("-d", "--duration"): duration = int(arg) + elif opt in ("-c", "--count"): + count = int(arg) elif opt in ("-j", "--json"): print_json = True sdc_token = args[0] -# + +# Instantiate the SDC client +sdclient = SdcClient(sdc_token) + # # Prepare the query's metrics list. -# In this case, we have one tag (used for segmentation) and one metric:: -# host.hostName. This is a tag, to identify each item of the output. -# container.count: This is the metric +# In this case, we have one tag (used for segmentation) and one metric: +# - host.hostName. This is a tag, to identify each item of the output +# - container.count: This is the metric # -# Instantiate the SDC client -sdclient = SdcClient(sdc_token) -metrics = [{"id": "host.hostName"}, {"id": "container.count","aggregations":{"time":"avg","group":"avg"}}] +metrics = [ + { "id": "host.hostName" }, + { "id": "container.count", "aggregations": { "time": "avg", "group": "avg" } } +] + res = sdclient.get_data( - metrics, -duration, 0, duration, paging={ + metrics, # list of metrics + -duration, # start time: either a unix timestamp, or a difference from "now" + 0, # end time: either a unix timestamp, or a difference from "now" (0 means you need "last X seconds") + duration, # sampling time, ie. data granularity; + # if equal to the time window span then the result will contain a single sample + paging = { "from": 0, - "to": 999 + "to": count - 1 }) + if res[0]: - output = [] - data = res[1]['data'] - for i in range(0, len(data)): - sample = data[i] - metrics = sample['d'] - hostName = metrics[0] - count = metrics[1] - output.append('%s\t%d' % (hostName, count)) - print '\n'.join(output) + # data fetched successfully + if print_json: + print json.dumps(res[1]) + else: + data = res[1]['data'] + output = [] + for i in range(0, len(data)): + sample = data[i] + metrics = sample['d'] + hostName = metrics[0] + count = metrics[1] + output.append('%s\t%d' % (hostName, count)) + + print '\n'.join(output) else: + # data fetch failed print res[1] sys.exit(1) From b6e142fd83779ba69adc00a2653178fbf9981fa9 Mon Sep 17 00:00:00 2001 From: davideschiera Date: Fri, 25 Jan 2019 10:29:38 -0800 Subject: [PATCH 4/4] Forgot debug lines... --- examples/list_hosts.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/examples/list_hosts.py b/examples/list_hosts.py index c3854fef..85917c62 100755 --- a/examples/list_hosts.py +++ b/examples/list_hosts.py @@ -37,8 +37,6 @@ def usage(): count = 100 print_json = False for opt, arg in opts: - print opt - print arg if opt in ("-d", "--duration"): duration = int(arg) elif opt in ("-c", "--count"):