From 4986fb5ace6905130f2db3e991271834a01f9358 Mon Sep 17 00:00:00 2001 From: Federico Barcelona Date: Mon, 5 Jul 2021 16:29:07 +0200 Subject: [PATCH 1/5] feat: New ActivityAuditClientV1 to query Activity Audit events --- sdcclient/secure/__init__.py | 8 +- sdcclient/secure/_activity_audit_v1.py | 138 +++++++++++++++++++++++++ specs/secure/activitylog_v1_spec.py | 61 +++++++++++ 3 files changed, 204 insertions(+), 3 deletions(-) create mode 100644 sdcclient/secure/_activity_audit_v1.py create mode 100644 specs/secure/activitylog_v1_spec.py diff --git a/sdcclient/secure/__init__.py b/sdcclient/secure/__init__.py index 2af04a75..93e72f7e 100644 --- a/sdcclient/secure/__init__.py +++ b/sdcclient/secure/__init__.py @@ -1,8 +1,10 @@ +from ._activity_audit_v1 import ActivityAuditClientV1, ActivityAuditDataSource from ._falco_rules_files_old import FalcoRulesFilesClientOld from ._policy_events_old import PolicyEventsClientOld from ._policy_events_v1 import PolicyEventsClientV1 -from ._policy_v2 import PolicyClientV2, policy_action_pause, policy_action_stop, policy_action_kill, \ - policy_action_capture +from ._policy_v2 import policy_action_capture, policy_action_kill, policy_action_pause, policy_action_stop, \ + PolicyClientV2 __all__ = ["PolicyEventsClientOld", "PolicyEventsClientV1", "FalcoRulesFilesClientOld", - "PolicyClientV2", "policy_action_pause", "policy_action_stop", "policy_action_kill", "policy_action_capture"] + "PolicyClientV2", "policy_action_pause", "policy_action_stop", "policy_action_kill", "policy_action_capture", + "ActivityAuditClientV1", "ActivityAuditDataSource"] diff --git a/sdcclient/secure/_activity_audit_v1.py b/sdcclient/secure/_activity_audit_v1.py new file mode 100644 index 00000000..89e51722 --- /dev/null +++ b/sdcclient/secure/_activity_audit_v1.py @@ -0,0 +1,138 @@ +from datetime import datetime, timedelta +from typing import List + +from sdcclient._common import _SdcCommon + + +class ActivityAuditDataSource: + CMD = "command" + NET = "connection" + KUBE_EXEC = "kubernetes" + FILE = "fileaccess" + + +_seconds_to_nanoseconds = 10 ** 9 + + +class ActivityAuditClientV1(_SdcCommon): + def __init__(self, token="", sdc_url='https://secure.sysdig.com', ssl_verify=True, custom_headers=None): + super(ActivityAuditClientV1, self).__init__(token, sdc_url, ssl_verify, custom_headers) + self.product = "SDS" + + def list_events(self, from_date=None, to_date=None, scope_filter=None, limit=0, + data_sources=None): + """ + List the events in the Activity Audit. + + Args: + from_date (datetime): the start of the time range from which to get events. The default value is yesterday. + to_date (datetime): the end of the time range from which to get events. The default value is now. + scope_filter (List): a list of Sysdig Monitor-like filter (e.g `processName in ("ubuntu")`). + limit (int): max number of events to retrieve. A limit of 0 or negative will retrieve all events. + data_sources (List): a list of data sources to retrieve events from. None or an empty list retrieves all events. + + Examples: + >>> client = ActivityAuditClientV1(token=SECURE_TOKEN) + >>> + >>> now = datetime.utcnow() + >>> three_days_ago = now - timedelta(days=3) + >>> max_event_number_retrieved = 50 + >>> data_sources = [ActivityAuditDataSource.CMD, ActivityAuditDataSource.KUBE_EXEC] + >>> + >>> ok, events = client.list_events(from_date=three_days_ago, + >>> to_date=now, + >>> limit=max_event_number_retrieved, + >>> data_sources=data_sources) + + Returns: + A list of event objects from the Activity Audit. + """ + number_of_events_per_query = 50 + + if from_date is None: + from_date = datetime.utcnow() - timedelta(days=1) + if to_date is None: + to_date = datetime.utcnow() + + filters = scope_filter if scope_filter else [] + if data_sources: + quoted_data_sources = [f'"{data_source}"' for data_source in data_sources] + data_source_filter = f'type in ({",".join(quoted_data_sources)})' + filters.append(data_source_filter) + + query_params = { + "from": int(from_date.timestamp()) * _seconds_to_nanoseconds, + "to": int(to_date.timestamp()) * _seconds_to_nanoseconds, + "limit": number_of_events_per_query, + "filter": " and ".join(filters), + } + + res = self.http.get(self.url + '/api/v1/activityAudit/events', headers=self.hdrs, verify=self.ssl_verify, + params=query_params) + ok, res = self._request_result(res) + if not ok: + return False, res + + events = [] + + # Pagination required by Secure API + while "page" in res and \ + "total" in res["page"] and \ + res["page"]["total"] > number_of_events_per_query: + events = events + res["data"] + + if 0 < limit < len(events): + events = events[0:limit - 1] + break + + paginated_query_params = { + "limit": number_of_events_per_query, + "filter": " and ".join(filters), + "cursor": res["page"]["prev"] + } + + res = self.http.get(self.url + '/api/v1/activityAudit/events', headers=self.hdrs, verify=self.ssl_verify, + params=paginated_query_params) + ok, res = self._request_result(res) + if not ok: + return False, res + else: + events = events + res["data"] + + return True, events + + def list_trace(self, traceable_event): + """ + Lists the events from an original traceable event. + + Args: + traceable_event(object): an event retrieved from the list_events method. The event must be traceable, + this is, it must have the "traceable" key as true. + + Examples: + >>> client = ActivityAuditClientV1(token=SECURE_TOKEN) + >>> + >>> ok, events = client.list_events() + >>> if not ok: + >>> return + >>> traceable_events = [event for event in events if event["traceable"]] + >>> + >>> ok, trace = client.list_trace(traceable_events[0]) + >>> if not ok: + >>> return + >>> + >>> for event in trace: + >>> print(event) + + Returns: + All the related events that are the trace of the given event. + """ + if not traceable_event or not traceable_event["traceable"]: + return False, "a traceable event must be provided" + + endpoint = f'/api/v1/activityAudit/events/{traceable_event["type"]}/{traceable_event["id"]}/trace' + res = self.http.get(self.url + endpoint, headers=self.hdrs, verify=self.ssl_verify) + ok, res = self._request_result(res) + if not ok: + return False, res + return True, res["data"] diff --git a/specs/secure/activitylog_v1_spec.py b/specs/secure/activitylog_v1_spec.py new file mode 100644 index 00000000..f56a38b0 --- /dev/null +++ b/specs/secure/activitylog_v1_spec.py @@ -0,0 +1,61 @@ +import datetime +import os + +from expects import be_above, be_empty, contain, expect, have_keys, have_len +from mamba import _it, before, context, description, it + +from sdcclient.secure import ActivityAuditClientV1 as ActivityAuditClient, ActivityAuditDataSource +from specs import be_successful_api_call + +with description("Activity Audit v1", "integration") as self: + with before.all: + self.client = ActivityAuditClient(sdc_url=os.getenv("SDC_SECURE_URL", "https://secure.sysdig.com"), + token=os.getenv("SDC_SECURE_TOKEN")) + + with it("is able to list the most recent commands with the default parameters"): + ok, res = self.client.list_events() + + expect((ok, res)).to(be_successful_api_call) + expect(res).to_not(be_empty) + + with context("when listing the most recent commands with a limit of 5"): + with it("retrieves the 5 events"): + ok, res = self.client.list_events(limit=5) + + expect((ok, res)).to(be_successful_api_call) + expect(res).to_not(have_len(5)) + + with context("when listing the events from the last 3 days"): + with it("retrieves all the events"): + three_days_ago = datetime.datetime.utcnow() - datetime.timedelta(days=3) + ok, res = self.client.list_events(from_date=three_days_ago) + + expect((ok, res)).to(be_successful_api_call) + expect(res).to_not(be_empty) + + with context("when listing events from a specific type"): + with it("retrieves the events of this event type only"): + ok, res = self.client.list_events(data_sources=[ActivityAuditDataSource.CMD]) + + expect((ok, res)).to(be_successful_api_call) + expect(res).to(contain(have_keys(type=ActivityAuditDataSource.CMD))) + expect(res).to_not(contain(have_keys(type=ActivityAuditDataSource.KUBE_EXEC))) + expect(res).to_not(contain(have_keys(type=ActivityAuditDataSource.FILE))) + expect(res).to_not(contain(have_keys(type=ActivityAuditDataSource.NET))) + + with context("when retrieving the inner events of a traceable event"): + with _it("retrieves the trace of these events"): + ok, res = self.client.list_events(data_sources=[ActivityAuditDataSource.KUBE_EXEC]) + expect((ok, res)).to(be_successful_api_call) + + # FIXME: Dependiendo del entorno, puede que no existan KUBE_EXEC events, así que aqui necesitaría parar + # este test ya que la lista devuelta, será vacía + + expect(res).to(contain(have_keys(traceable=True))) + + traceable_events = [event for event in res if event["traceable"]] + ok, res = self.client.list_trace(traceable_events[0]) + + expect((ok, res)).to(be_successful_api_call) + expect(res).to(contain(have_keys(type=ActivityAuditDataSource.CMD))) + expect(res).to(have_len(be_above(0))) # Not using be_empty, because we want to ensure this is a list From 22e39b20370abfa11c790fc3226bd7d85d1d087e Mon Sep 17 00:00:00 2001 From: Federico Barcelona Date: Mon, 5 Jul 2021 16:36:32 +0200 Subject: [PATCH 2/5] chore: Deprecate old command audit methods --- sdcclient/_secure.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sdcclient/_secure.py b/sdcclient/_secure.py index 49468e19..47289527 100644 --- a/sdcclient/_secure.py +++ b/sdcclient/_secure.py @@ -496,6 +496,8 @@ def list_commands_audit(self, from_sec=None, to_sec=None, scope_filter=None, com '''**Description** List the commands audit. + **DEPRECATED**: Use sdcclient.secure.ActivityAuditClientV1 instead. This is maintained for old on-prem versions, but will be removed over time. + **Arguments** - from_sec: the start of the timerange for which to get commands audit. - end_sec: the end of the timerange for which to get commands audit. @@ -528,6 +530,8 @@ def get_command_audit(self, id, metrics=[]): '''**Description** Get a command audit. + **DEPRECATED**: Use sdcclient.secure.ActivityAuditClientV1 instead. This is maintained for old on-prem versions, but will be removed over time. + **Arguments** - id: the id of the command audit to get. From f72c3f86bb54d36ecfc5ecbe789954195982ccaf Mon Sep 17 00:00:00 2001 From: Federico Barcelona Date: Mon, 5 Jul 2021 16:38:29 +0200 Subject: [PATCH 3/5] refactor: Import datetime package directly --- sdcclient/secure/_activity_audit_v1.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/sdcclient/secure/_activity_audit_v1.py b/sdcclient/secure/_activity_audit_v1.py index 89e51722..56f28fd8 100644 --- a/sdcclient/secure/_activity_audit_v1.py +++ b/sdcclient/secure/_activity_audit_v1.py @@ -1,5 +1,4 @@ -from datetime import datetime, timedelta -from typing import List +import datetime from sdcclient._common import _SdcCommon @@ -25,8 +24,8 @@ def list_events(self, from_date=None, to_date=None, scope_filter=None, limit=0, List the events in the Activity Audit. Args: - from_date (datetime): the start of the time range from which to get events. The default value is yesterday. - to_date (datetime): the end of the time range from which to get events. The default value is now. + from_date (datetime.datetime): the start of the time range from which to get events. The default value is yesterday. + to_date (datetime.datetime): the end of the time range from which to get events. The default value is now. scope_filter (List): a list of Sysdig Monitor-like filter (e.g `processName in ("ubuntu")`). limit (int): max number of events to retrieve. A limit of 0 or negative will retrieve all events. data_sources (List): a list of data sources to retrieve events from. None or an empty list retrieves all events. @@ -34,8 +33,8 @@ def list_events(self, from_date=None, to_date=None, scope_filter=None, limit=0, Examples: >>> client = ActivityAuditClientV1(token=SECURE_TOKEN) >>> - >>> now = datetime.utcnow() - >>> three_days_ago = now - timedelta(days=3) + >>> now = datetime.datetime.utcnow() + >>> three_days_ago = now - datetime.timedelta(days=3) >>> max_event_number_retrieved = 50 >>> data_sources = [ActivityAuditDataSource.CMD, ActivityAuditDataSource.KUBE_EXEC] >>> @@ -50,9 +49,9 @@ def list_events(self, from_date=None, to_date=None, scope_filter=None, limit=0, number_of_events_per_query = 50 if from_date is None: - from_date = datetime.utcnow() - timedelta(days=1) + from_date = datetime.datetime.utcnow() - datetime.timedelta(days=1) if to_date is None: - to_date = datetime.utcnow() + to_date = datetime.datetime.utcnow() filters = scope_filter if scope_filter else [] if data_sources: From f735af763ede9aeb58d58f6338fd43206f7cafc7 Mon Sep 17 00:00:00 2001 From: Federico Barcelona Date: Mon, 5 Jul 2021 16:42:01 +0200 Subject: [PATCH 4/5] fix(lint): Solve linting problems --- sdcclient/secure/_activity_audit_v1.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdcclient/secure/_activity_audit_v1.py b/sdcclient/secure/_activity_audit_v1.py index 56f28fd8..eae33891 100644 --- a/sdcclient/secure/_activity_audit_v1.py +++ b/sdcclient/secure/_activity_audit_v1.py @@ -22,7 +22,7 @@ def list_events(self, from_date=None, to_date=None, scope_filter=None, limit=0, data_sources=None): """ List the events in the Activity Audit. - + Args: from_date (datetime.datetime): the start of the time range from which to get events. The default value is yesterday. to_date (datetime.datetime): the end of the time range from which to get events. The default value is now. From a0277428697eab911404bc98f05fa4b5ba3faf20 Mon Sep 17 00:00:00 2001 From: Federico Barcelona Date: Mon, 5 Jul 2021 17:16:17 +0200 Subject: [PATCH 5/5] fix(ci): Change the image to scan to quay.io/sysdig/agent --- specs/secure/activitylog_v1_spec.py | 3 --- specs/secure/scanning/policy_evaluation_spec.py | 17 +++++++---------- .../secure/scanning/query_image_content_spec.py | 15 ++++++++------- 3 files changed, 15 insertions(+), 20 deletions(-) diff --git a/specs/secure/activitylog_v1_spec.py b/specs/secure/activitylog_v1_spec.py index f56a38b0..fd4a9299 100644 --- a/specs/secure/activitylog_v1_spec.py +++ b/specs/secure/activitylog_v1_spec.py @@ -48,9 +48,6 @@ ok, res = self.client.list_events(data_sources=[ActivityAuditDataSource.KUBE_EXEC]) expect((ok, res)).to(be_successful_api_call) - # FIXME: Dependiendo del entorno, puede que no existan KUBE_EXEC events, así que aqui necesitaría parar - # este test ya que la lista devuelta, será vacía - expect(res).to(contain(have_keys(traceable=True))) traceable_events = [event for event in res if event["traceable"]] diff --git a/specs/secure/scanning/policy_evaluation_spec.py b/specs/secure/scanning/policy_evaluation_spec.py index c1336953..7524f41a 100644 --- a/specs/secure/scanning/policy_evaluation_spec.py +++ b/specs/secure/scanning/policy_evaluation_spec.py @@ -11,40 +11,38 @@ with before.all: self.client = SdScanningClient(sdc_url=os.getenv("SDC_SECURE_URL", "https://secure.sysdig.com"), token=os.getenv("SDC_SECURE_TOKEN")) + self.image_name = "quay.io/sysdig/agent:latest" with it("is able to retrieve the results for all the policies"): - image_name = "alpine:latest" - ok, res = self.client.get_image_scanning_results(image_name) + ok, res = self.client.get_image_scanning_results(self.image_name) expect((ok, res)).to(be_successful_api_call) expect(res).to( have_keys("image_digest", "image_id", "stop_results", total_warn=be_above_or_equal(0), total_stop=be_above_or_equal(0), last_evaluation=be_an(datetime), - status="pass", image_tag="docker.io/alpine:latest", + status="pass", image_tag=self.image_name, policy_id="*", policy_name="All policies", warn_results=not_(be_empty)) ) with it("is able to retrieve the results for the default policy"): - image_name = "alpine:latest" policy_id = "default" - ok, res = self.client.get_image_scanning_results(image_name, policy_id) + ok, res = self.client.get_image_scanning_results(self.image_name, policy_id) expect((ok, res)).to(be_successful_api_call) expect(res).to( have_keys("image_digest", "image_id", "stop_results", total_warn=be_above_or_equal(0), total_stop=be_above_or_equal(0), last_evaluation=be_an(datetime), - status="pass", image_tag="docker.io/alpine:latest", + status="pass", image_tag=self.image_name, policy_id="default", policy_name="DefaultPolicy", warn_results=not_(be_empty)) ) with context("but the image has not been scanned yet"): with it("returns an error saying that the image has not been found"): - image_name = "unknown_image" - ok, res = self.client.get_image_scanning_results(image_name) + ok, res = self.client.get_image_scanning_results("unknown_image") expect((ok, res)).to_not(be_successful_api_call) expect(res).to(equal("could not retrieve image digest for the given image name, " @@ -52,9 +50,8 @@ with context("but the provided policy id does not exist"): with it("returns an error saying that the policy id is not found"): - image_name = "alpine" policy_id = "unknown_policy_id" - ok, res = self.client.get_image_scanning_results(image_name, policy_id) + ok, res = self.client.get_image_scanning_results(self.image_name, policy_id) expect((ok, res)).to_not(be_successful_api_call) expect(res).to(equal("the specified policy ID doesn't exist")) diff --git a/specs/secure/scanning/query_image_content_spec.py b/specs/secure/scanning/query_image_content_spec.py index 7ef4a3ba..4b224dd1 100644 --- a/specs/secure/scanning/query_image_content_spec.py +++ b/specs/secure/scanning/query_image_content_spec.py @@ -10,40 +10,41 @@ with before.each: self.client = SdScanningClient(sdc_url=os.getenv("SDC_SECURE_URL", "https://secure.sysdig.com"), token=os.getenv("SDC_SECURE_TOKEN")) + self.image_to_scan = "quay.io/sysdig/agent:latest" with it("is able to retrieve the OS contents"): - ok, res = self.client.query_image_content("alpine:latest", "os") + ok, res = self.client.query_image_content(self.image_to_scan, "os") expect((ok, res)).to(be_successful_api_call) expect(res["content"]).to(contain(have_keys("license", "origin", "package", "size", "type", "version"))) expect(res["content_type"]).to(equal("os")) with it("is able to retrieve the npm contents"): - ok, res = self.client.query_image_content("alpine:latest", "npm") + ok, res = self.client.query_image_content(self.image_to_scan, "npm") expect((ok, res)).to(be_successful_api_call) expect(res["content_type"]).to(equal("npm")) with it("is able to retrieve the gem contents"): - ok, res = self.client.query_image_content("alpine:latest", "gem") + ok, res = self.client.query_image_content(self.image_to_scan, "gem") expect((ok, res)).to(be_successful_api_call) expect(res["content_type"]).to(equal("gem")) with it("is able to retrieve the python contents"): - ok, res = self.client.query_image_content("alpine:latest", "python") + ok, res = self.client.query_image_content(self.image_to_scan, "python") expect((ok, res)).to(be_successful_api_call) expect(res["content_type"]).to(equal("python")) with it("is able to retrieve the java contents"): - ok, res = self.client.query_image_content("alpine:latest", "java") + ok, res = self.client.query_image_content(self.image_to_scan, "java") expect((ok, res)).to(be_successful_api_call) expect(res["content_type"]).to(equal("java")) with it("is able to retrieve the files contents"): - ok, res = self.client.query_image_content("alpine:latest", "files") + ok, res = self.client.query_image_content(self.image_to_scan, "files") expect((ok, res)).to(be_successful_api_call) expect(res["content"]).to( @@ -52,7 +53,7 @@ with context("when the type is not in the supported list"): with it("returns an error indicating the type is incorrect"): - ok, res = self.client.query_image_content("alpine:latest", "Unknown") + ok, res = self.client.query_image_content(self.image_to_scan, "Unknown") expect((ok, res)).not_to(be_successful_api_call) expect(res).to(equal(