Skip to content

Commit db2e78e

Browse files
authored
feat(firehose-destinations-alpha): support zero buffering (#28716)
support zero buffering ([announcement](https://aws.amazon.com/about-aws/whats-new/2023/12/amazon-kinesis-data-firehose-zero-buffering/), [documentation](https://docs.aws.amazon.com/firehose/latest/dev/buffering-hints.html)) Closes #28714 ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
1 parent c3802c4 commit db2e78e

File tree

11 files changed

+439
-20
lines changed

11 files changed

+439
-20
lines changed

packages/@aws-cdk/aws-kinesisfirehose-alpha/README.md

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -299,6 +299,22 @@ new firehose.DeliveryStream(this, 'Delivery Stream', {
299299
See: [Data Delivery Frequency](https://docs.aws.amazon.com/firehose/latest/dev/basic-deliver.html#frequency)
300300
in the *Kinesis Data Firehose Developer Guide*.
301301

302+
Zero buffering, where Amazon Kinesis Data Firehose stream can be configured to not buffer data before delivery, is supported by
303+
setting the "buffer interval" to 0.
304+
305+
```ts
306+
// Setup zero buffering
307+
declare const bucket: s3.Bucket;
308+
const destination = new destinations.S3Bucket(bucket, {
309+
bufferingInterval: Duration.seconds(0),
310+
});
311+
new firehose.DeliveryStream(this, 'ZeroBufferDeliveryStream', {
312+
destinations: [destination],
313+
});
314+
```
315+
316+
See: [Buffering Hints](https://docs.aws.amazon.com/firehose/latest/dev/buffering-hints.html).
317+
302318
## Destination Encryption
303319

304320
Your data can be automatically encrypted when it is delivered to S3 as a final or an

packages/@aws-cdk/aws-kinesisfirehose-destinations-alpha/lib/common.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ export interface CommonDestinationS3Props {
8686
* The length of time that Firehose buffers incoming data before delivering
8787
* it to the S3 bucket.
8888
*
89-
* Minimum: Duration.seconds(60)
89+
* Minimum: Duration.seconds(0)
9090
* Maximum: Duration.seconds(900)
9191
*
9292
* @default Duration.seconds(300)

packages/@aws-cdk/aws-kinesisfirehose-destinations-alpha/lib/private/helpers.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -87,8 +87,8 @@ export function createBufferingHints(
8787
}
8888

8989
const intervalInSeconds = interval?.toSeconds() ?? 300;
90-
if (intervalInSeconds < 60 || intervalInSeconds > 900) {
91-
throw new Error(`Buffering interval must be between 60 and 900 seconds. Buffering interval provided was ${intervalInSeconds} seconds.`);
90+
if (intervalInSeconds > 900) {
91+
throw new Error(`Buffering interval must be less than 900 seconds. Buffering interval provided was ${intervalInSeconds} seconds.`);
9292
}
9393
const sizeInMBs = size?.toMebibytes() ?? 5;
9494
if (sizeInMBs < 1 || sizeInMBs > 128) {

packages/@aws-cdk/aws-kinesisfirehose-destinations-alpha/test/integ.s3-bucket.lit.js.snapshot/aws-cdk-firehose-delivery-stream-s3-all-properties.assets.json

Lines changed: 3 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

packages/@aws-cdk/aws-kinesisfirehose-destinations-alpha/test/integ.s3-bucket.lit.js.snapshot/aws-cdk-firehose-delivery-stream-s3-all-properties.template.json

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -680,6 +680,146 @@
680680
"DependsOn": [
681681
"DeliveryStreamS3DestinationRoleDefaultPolicy3015D8C7"
682682
]
683+
},
684+
"ZeroBufferingDeliveryStreamS3DestinationRole606BB6E3": {
685+
"Type": "AWS::IAM::Role",
686+
"Properties": {
687+
"AssumeRolePolicyDocument": {
688+
"Statement": [
689+
{
690+
"Action": "sts:AssumeRole",
691+
"Effect": "Allow",
692+
"Principal": {
693+
"Service": "firehose.amazonaws.com"
694+
}
695+
}
696+
],
697+
"Version": "2012-10-17"
698+
}
699+
}
700+
},
701+
"ZeroBufferingDeliveryStreamS3DestinationRoleDefaultPolicyC03FC12E": {
702+
"Type": "AWS::IAM::Policy",
703+
"Properties": {
704+
"PolicyDocument": {
705+
"Statement": [
706+
{
707+
"Action": [
708+
"s3:Abort*",
709+
"s3:DeleteObject*",
710+
"s3:GetBucket*",
711+
"s3:GetObject*",
712+
"s3:List*",
713+
"s3:PutObject",
714+
"s3:PutObjectLegalHold",
715+
"s3:PutObjectRetention",
716+
"s3:PutObjectTagging",
717+
"s3:PutObjectVersionTagging"
718+
],
719+
"Effect": "Allow",
720+
"Resource": [
721+
{
722+
"Fn::GetAtt": [
723+
"Bucket83908E77",
724+
"Arn"
725+
]
726+
},
727+
{
728+
"Fn::Join": [
729+
"",
730+
[
731+
{
732+
"Fn::GetAtt": [
733+
"Bucket83908E77",
734+
"Arn"
735+
]
736+
},
737+
"/*"
738+
]
739+
]
740+
}
741+
]
742+
},
743+
{
744+
"Action": [
745+
"logs:CreateLogStream",
746+
"logs:PutLogEvents"
747+
],
748+
"Effect": "Allow",
749+
"Resource": {
750+
"Fn::GetAtt": [
751+
"ZeroBufferingDeliveryStreamLogGroupFA68202E",
752+
"Arn"
753+
]
754+
}
755+
}
756+
],
757+
"Version": "2012-10-17"
758+
},
759+
"PolicyName": "ZeroBufferingDeliveryStreamS3DestinationRoleDefaultPolicyC03FC12E",
760+
"Roles": [
761+
{
762+
"Ref": "ZeroBufferingDeliveryStreamS3DestinationRole606BB6E3"
763+
}
764+
]
765+
}
766+
},
767+
"ZeroBufferingDeliveryStreamLogGroupFA68202E": {
768+
"Type": "AWS::Logs::LogGroup",
769+
"Properties": {
770+
"RetentionInDays": 731
771+
},
772+
"UpdateReplacePolicy": "Retain",
773+
"DeletionPolicy": "Retain"
774+
},
775+
"ZeroBufferingDeliveryStreamLogGroupS3Destination806CFD96": {
776+
"Type": "AWS::Logs::LogStream",
777+
"Properties": {
778+
"LogGroupName": {
779+
"Ref": "ZeroBufferingDeliveryStreamLogGroupFA68202E"
780+
}
781+
},
782+
"UpdateReplacePolicy": "Retain",
783+
"DeletionPolicy": "Retain"
784+
},
785+
"ZeroBufferingDeliveryStream7C2555C9": {
786+
"Type": "AWS::KinesisFirehose::DeliveryStream",
787+
"Properties": {
788+
"DeliveryStreamType": "DirectPut",
789+
"ExtendedS3DestinationConfiguration": {
790+
"BucketARN": {
791+
"Fn::GetAtt": [
792+
"Bucket83908E77",
793+
"Arn"
794+
]
795+
},
796+
"BufferingHints": {
797+
"IntervalInSeconds": 0,
798+
"SizeInMBs": 5
799+
},
800+
"CloudWatchLoggingOptions": {
801+
"Enabled": true,
802+
"LogGroupName": {
803+
"Ref": "ZeroBufferingDeliveryStreamLogGroupFA68202E"
804+
},
805+
"LogStreamName": {
806+
"Ref": "ZeroBufferingDeliveryStreamLogGroupS3Destination806CFD96"
807+
}
808+
},
809+
"CompressionFormat": "GZIP",
810+
"ErrorOutputPrefix": "errorPrefix",
811+
"Prefix": "regularPrefix",
812+
"RoleARN": {
813+
"Fn::GetAtt": [
814+
"ZeroBufferingDeliveryStreamS3DestinationRole606BB6E3",
815+
"Arn"
816+
]
817+
}
818+
}
819+
},
820+
"DependsOn": [
821+
"ZeroBufferingDeliveryStreamS3DestinationRoleDefaultPolicyC03FC12E"
822+
]
683823
}
684824
},
685825
"Mappings": {

packages/@aws-cdk/aws-kinesisfirehose-destinations-alpha/test/integ.s3-bucket.lit.js.snapshot/cdk.out

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

packages/@aws-cdk/aws-kinesisfirehose-destinations-alpha/test/integ.s3-bucket.lit.js.snapshot/integ.json

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

packages/@aws-cdk/aws-kinesisfirehose-destinations-alpha/test/integ.s3-bucket.lit.js.snapshot/manifest.json

Lines changed: 32 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)