feat(aws): AWSX-1598 Move source identification to logs-backend (#963)

* feat(aws): AWSX-1598 Move source identification to logs-backend

Signed-off-by: Vincent Boutour <[email protected]>

* fixup! feat(aws): AWSX-1598 Move source identification to logs-backend

Signed-off-by: Vincent Boutour <[email protected]>

---------

Signed-off-by: Vincent Boutour <[email protected]>

Author: ViBiOh

aws/logs_monitoring/steps/enums.py

@@ -3,27 +3,21 @@
 
 class AwsEventSource(Enum):
     AWS = "aws"
-    CARBONBLACK = "carbonblack"
-    CLOUDFRONT = "cloudfront"
     CLOUDTRAIL = "cloudtrail"
     CLOUDWATCH = "cloudwatch"
     ELASTICSEARCH = "elasticsearch"
-    ELB = "elb"
     FARGATE = "fargate"
     GUARDDUTY = "guardduty"
     KINESIS = "kinesis"
     LAMBDA = "lambda"
     MARIADB = "mariadb"
     MSK = "msk"
     MYSQL = "mysql"
-    NETWORKFIREWALL = "network-firewall"
     POSTGRESQL = "postgresql"
     ROUTE53 = "route53"
     S3 = "s3"
     SNS = "sns"
     STEPFUNCTION = "stepfunction"
-    VERIFIED_ACCESS = "verified-access"
-    VPC = "vpc"
     WAF = "waf"
 
     def __str__(self):
@@ -32,15 +26,11 @@ def __str__(self):
     @staticmethod
     def cloudwatch_sources():
         return [
-            AwsEventSource.CLOUDFRONT,
             AwsEventSource.CLOUDTRAIL,
             AwsEventSource.ELASTICSEARCH,
             AwsEventSource.FARGATE,
             AwsEventSource.MSK,
-            AwsEventSource.NETWORKFIREWALL,
             AwsEventSource.ROUTE53,
-            AwsEventSource.VERIFIED_ACCESS,
-            AwsEventSource.VPC,
         ]
 
 
@@ -53,19 +43,11 @@ def __init__(self, string, event_source):
     WAF_0 = ("aws-waf-logs", AwsEventSource.WAF)
     WAF_1 = ("waflogs", AwsEventSource.WAF)
 
-    # e.g. carbon-black-cloud-forwarder/alerts/org_key=*****/year=2021/month=7/day=19/hour=18/minute=15/second=41/8436e850-7e78-40e4-b3cd-6ebbc854d0a2.jsonl.gz
-    CARBONBLACK = ("carbon-black", AwsEventSource.CARBONBLACK)
-    # e.g. AWSLogs/123456779121/elasticloadbalancing/us-east-1/2020/10/02/123456779121_elasticloadbalancing_us-east-1_app.alb.xxxxx.xx.xxx.xxx_x.log.gz
-    ELB = ("elasticloadbalancing", AwsEventSource.ELB)
     GUARDDUTY = ("guardduty", AwsEventSource.GUARDDUTY)
     KINESIS = ("amazon_kinesis", AwsEventSource.KINESIS)
     MSK = ("amazon_msk", AwsEventSource.MSK)
-    NETWORKFIREWALL = ("network-firewall", AwsEventSource.NETWORKFIREWALL)
     # e.g. AWSLogs/123456779121/vpcdnsquerylogs/vpc-********/2021/05/11/vpc-********_vpcdnsquerylogs_********_20210511T0910Z_71584702.log.gz
     ROUTE53 = ("vpcdnsquerylogs", AwsEventSource.ROUTE53)
-    VERIFIED_ACCESS = ("verified-access", AwsEventSource.VERIFIED_ACCESS)
-    # e.g. AWSLogs/123456779121/vpcflowlogs/us-east-1/2020/10/02/123456779121_vpcflowlogs_us-east-1_fl-xxxxx.log.gz
-    VPC = ("vpcflowlogs", AwsEventSource.VPC)
 
     def __str__(self):
         return f"{self.string}"

aws/logs_monitoring/steps/handlers/awslogs_handler.py

@@ -128,8 +128,6 @@ def set_host(self, metadata, aws_attributes):
         match metadata_source:
             case AwsEventSource.CLOUDWATCH:
                 metadata[DD_HOST] = log_group
-            case AwsEventSource.VERIFIED_ACCESS:
-                self.handle_verified_access_source(metadata, aws_attributes)
             case AwsEventSource.STEPFUNCTION:
                 self.handle_step_function_source(metadata, aws_attributes)
 

aws/logs_monitoring/tests/test_parsing.py

@@ -98,28 +98,6 @@ def test_kinesis_event(self):
             str(AwsEventSource.KINESIS),
         )
 
-    def test_vpc_event(self):
-        self.assertEqual(
-            parse_event_source({"awslogs": "logs"}, "abc123_my_vpc_loggroup"),
-            str(AwsEventSource.VPC),
-        )
-        self.assertEqual(
-            parse_event_source(
-                {"Records": ["logs-from-s3"]},
-                "AWSLogs/123456779121/vpcflowlogs/us-east-1/2020/10/02/123456779121_vpcflowlogs_us-east-1_fl-xxxxx.log.gz",
-            ),
-            str(AwsEventSource.VPC),
-        )
-
-    def test_elb_event(self):
-        self.assertEqual(
-            parse_event_source(
-                {"Records": ["logs-from-s3"]},
-                "AWSLogs/123456779121/elasticloadbalancing/us-east-1/2020/10/02/123456779121_elasticloadbalancing_us-east-1_app.alb.xxxxx.xx.xxx.xxx_x.log.gz",
-            ),
-            str(AwsEventSource.ELB),
-        )
-
     def test_waf_event(self):
         self.assertEqual(
             parse_event_source(
@@ -214,15 +192,6 @@ def test_msk_event(self):
             str(AwsEventSource.MSK),
         )
 
-    def test_carbon_black_event(self):
-        self.assertEqual(
-            parse_event_source(
-                {"Records": ["logs-from-s3"]},
-                "carbon-black-cloud-forwarder/alerts/8436e850-7e78-40e4-b3cd-6ebbc854d0a2.jsonl.gz",
-            ),
-            str(AwsEventSource.CARBONBLACK),
-        )
-
     def test_cloudwatch_source_if_none_found(self):
         self.assertEqual(
             parse_event_source({"awslogs": "logs"}, ""), str(AwsEventSource.CLOUDWATCH)