Develop

.gitignore

@@ -10,4 +10,7 @@ ci/.taskcat_overrides.yml
 taskcat_outputs/
 .DS_Store
 .taskcat_overrides.yml
-.taskcat.yml
+.DS_Store
+templates/cribl-single-arm64-new-vpc-logging.template.conditional.yaml
+.DS_Store
+.taskcat/.remote_templates

.taskcat.yml

@@ -0,0 +1,68 @@
+project:
+  name: aws-quickstart-cribl-logstream
+  owner: [email protected]
+  package_lambda: false
+  parameters: 
+      webAccessCidr: 0.0.0.0/0
+      QSS3KeyPrefix: logstream/
+      QSS3BucketRegion: $[taskcat_current_region]
+      QSS3BucketName: aws-quickstart
+tests:
+  cribl-deploy-new-vpc-arm:
+    parameters: 
+      AvailabilityZones: $[taskcat_getaz_2]
+      VPCCIDR: 10.0.0.0/16
+      PublicSubnet1CIDR: 10.0.128.0/20
+      PublicSubnet2CIDR: 10.0.144.0/20
+      PrivateSubnet1CIDR: 10.0.0.0/19
+      PrivateSubnet2CIDR: 10.0.32.0/19
+      VPCTenancy: default
+      QSS3BucketRegion: $[taskcat_current_region]
+    template: templates/cribl-single-arm64-new-vpc.template.yaml
+    regions:
+      - us-west-1
+  cribl-deploy-new-vpc-logging-arm:
+    parameters: 
+      AvailabilityZones: $[taskcat_getaz_2]
+      VPCCIDR: 10.0.0.0/16
+      PublicSubnet1CIDR: 10.0.128.0/20
+      PublicSubnet2CIDR: 10.0.144.0/20
+      PrivateSubnet1CIDR: 10.0.0.0/19
+      PrivateSubnet2CIDR: 10.0.32.0/19
+      VPCTenancy: default
+      QSS3BucketRegion: $[taskcat_current_region]
+      LogFilePrefix: cribl-vpc-logs
+      TrafficType: ALL
+      SQS: cribl-sqs-vpc-logs-arm
+    template: templates/cribl-single-arm64-new-vpc.template.yaml
+    regions:
+      - us-west-2
+  cribl-deploy-new-vpc-x86:
+    parameters: 
+      AvailabilityZones: $[taskcat_getaz_2]
+      VPCCIDR: 10.0.0.0/16
+      PublicSubnet1CIDR: 10.0.128.0/20
+      PublicSubnet2CIDR: 10.0.144.0/20
+      PrivateSubnet1CIDR: 10.0.0.0/19
+      PrivateSubnet2CIDR: 10.0.32.0/19
+      VPCTenancy: default
+      QSS3BucketRegion: $[taskcat_current_region]
+    template: templates/cribl-single-x86-new-vpc.template.yaml
+    regions:
+      - us-east-1
+  cribl-deploy-new-vpc-logging-x86:
+    parameters: 
+      AvailabilityZones: $[taskcat_getaz_2]
+      VPCCIDR: 10.0.0.0/16
+      PublicSubnet1CIDR: 10.0.128.0/20
+      PublicSubnet2CIDR: 10.0.144.0/20
+      PrivateSubnet1CIDR: 10.0.0.0/19
+      PrivateSubnet2CIDR: 10.0.32.0/19
+      VPCTenancy: default
+      QSS3BucketRegion: $[taskcat_current_region]
+      LogFilePrefix: cribl-vpc-logs
+      TrafficType: ALL
+      SQS: cribl-sqs-vpc-logs-x86
+    template: templates/cribl-single-x86-new-vpc.template.yaml
+    regions:
+      - us-east-2

README.md

@@ -11,9 +11,9 @@ This Quick Start deployment builds a new AWS environment consisting of the infra
 
 | VPC | ARM64 | x86_64 |
 | --- | ---- | ---- |
-| Deploy in an existing VPC | [Cribl LogStream ARM64](https://us-west-2.console.aws.amazon.com/cloudformation/home?region=us-west-2#/stacks/create/template?stackName=Cribl-LogStream&templateURL=https://aws-quickstart-cribl-logstream-us-west-2.s3.us-west-2.amazonaws.com/logstream/cribl-single-template-arm64.yaml) | [Cribl LogStream x86_64](https://us-west-2.console.aws.amazon.com/cloudformation/home?region=us-west-2#/stacks/create/template?stackName=Cribl-LogStream&templateURL=https://aws-quickstart-cribl-logstream-us-west-2.s3.us-west-2.amazonaws.com/logstream/cribl-single-template.yaml) |
-| Deploy in a new VPC | [Cribl LogStream ARM64](https://us-west-2.console.aws.amazon.com/cloudformation/home?region=us-west-2#/stacks/create/template?stackName=Cribl-LogStream&templateURL=https://aws-quickstart-cribl-logstream-us-west-1.s3.us-west-1.amazonaws.com/logstream/cribl-single-template-vpc-arm64.yaml) | [Cribl LogStream x86_64](https://us-west-2.console.aws.amazon.com/cloudformation/home?region=us-west-2#/stacks/create/template?stackName=Cribl-LogStream&templateURL=https://aws-quickstart-cribl-logstream-us-west-1.s3.us-west-1.amazonaws.com/logstream/cribl-single-template-vpc.yaml) |
-| Deploy in new VPC with Flow Logs to s3 enabled | [Cribl LogStream ARM64](https://us-west-2.console.aws.amazon.com/cloudformation/home?region=us-west-2#/stacks/create/template?stackName=Cribl-LogStream&templateURL=https://aws-quickstart-cribl-logstream-us-west-2.s3.us-west-2.amazonaws.com/logstream/cribl-single-template-vpc-logging-arm64.yaml) | [Cribl LogStream x86_64](https://us-west-2.console.aws.amazon.com/cloudformation/home?region=us-west-2#/stacks/create/template?stackName=Cribl-LogStream&templateURL=https://aws-quickstart-cribl-logstream-us-west-2.s3.us-west-2.amazonaws.com/logstream/cribl-single-template-vpc-logging.yaml) | 
+| Deploy in an existing VPC | [Cribl LogStream ARM64](https://us-west-2.console.aws.amazon.com/cloudformation/home?region=us-west-2#/stacks/create/template?stackName=Cribl-LogStream&templateURL=https://aws-quickstart-cribl-logstream-us-west-2.s3.us-west-2.amazonaws.com/logstream/cribl-single-arm64.entrypoint.template.yaml) | [Cribl LogStream x86_64](https://us-west-2.console.aws.amazon.com/cloudformation/home?region=us-west-2#/stacks/create/template?stackName=Cribl-LogStream&templateURL=https://aws-quickstart-cribl-logstream-us-west-2.s3.us-west-2.amazonaws.com/logstream/cribl-single-x86.entrypoint.template.yaml) |
+| Deploy in a new VPC | [Cribl LogStream ARM64](https://us-west-2.console.aws.amazon.com/cloudformation/home?region=us-west-2#/stacks/create/template?stackName=Cribl-LogStream&templateURL=https://aws-quickstart-cribl-logstream-us-west-1.s3.us-west-1.amazonaws.com/logstream/cribl-single-arm64-new-vpc.template.yaml) | [Cribl LogStream x86_64](https://us-west-2.console.aws.amazon.com/cloudformation/home?region=us-west-2#/stacks/create/template?stackName=Cribl-LogStream&templateURL=https://aws-quickstart-cribl-logstream-us-west-1.s3.us-west-1.amazonaws.com/logstream/cribl-single-x86-new-vpc.template.yaml) |
+| Deploy in new VPC with Flow Logs to s3 enabled | [Cribl LogStream ARM64](https://us-west-2.console.aws.amazon.com/cloudformation/home?region=us-west-2#/stacks/create/template?stackName=Cribl-LogStream&templateURL=https://aws-quickstart-cribl-logstream-us-west-2.s3.us-west-2.amazonaws.com/logstream/cribl-single-arm64-new-vpc-logging.template.yaml) | [Cribl LogStream x86_64](https://us-west-2.console.aws.amazon.com/cloudformation/home?region=us-west-2#/stacks/create/template?stackName=Cribl-LogStream&templateURL=https://aws-quickstart-cribl-logstream-us-west-2.s3.us-west-2.amazonaws.com/logstream/cribl-single-x86-new-vpc-logging.template.yaml) | 
 
 4. Deploy the stack in your environment, make sure to check the region as this defaults to **Oregon (us-west-2)**. 
 5. Log into Cribl LogStream with the credential supplied in the nested **CriblDeploy** "Outputs" tab on your CloudFormation stack.
@@ -40,9 +40,9 @@ You are responsible for the cost of the AWS services used while running this Qui
 
 ## Architecture
 
-![Architecture](/docs/images/architecture_diagram.png)
+![Architecture](docs/images/Cribl_AWS_Single.png)
 
-To post feedback, submit feature ideas, or report bugs, use the [**Issues**](https://github.com/amiracle/quick-start-cribl/issues) section of this [GitHub repo](https://github.com/amiracle/quick-start-cribl).
+To post feedback, submit feature ideas, or report bugs, use the [**Issues**](https://github.com/amiracle/quick-start-cribl/issues) section of this [GitHub repo](https://github.com/aws-quickstart/quickstart-cribl-logstream).
 
 To submit code for this Quick Start, see the [AWS Quick Start Contributor's Kit](https://aws-quickstart.github.io/).
 
@@ -51,4 +51,8 @@ To submit code for this Quick Start, see the [AWS Quick Start Contributor's Kit]
 - [Cribl Resources](https://cribl.io/resources)
 - [Cribl Docs on Single Instance Deployments](https://docs.cribl.io/docs/deploy-single-instance)
 - [Cribl Docs on Distributed Deployments](https://docs.cribl.io/docs/deploy-distributed)
-- [Cribl Docs on sizing and scaling instances](https://docs.cribl.io/docs/scaling)
+- [Cribl Docs on sizing and scaling instances](https://docs.cribl.io/docs/scaling)
+- [Cribl Docs on AWS Cross-Account Data Collection](https://docs.cribl.io/logstream/usecase-aws-x-account)
+- [Cribl Docs Sources](https://docs.cribl.io/logstream/sources)
+- [Cribl Docs Destinations](https://docs.cribl.io/logstream/destinations)
+- [Cribl Integrations](https://cribl.io/integrations/)

ci/.taskcat.yml

@@ -3,32 +3,22 @@ project:
   owner: [email protected]
   package_lambda: false
   regions:
-    - us-east-1
-    - us-east-2
-    - us-west-1
     - us-west-2
+  parameters: 
+      instanceType: c6g.large
+      webAccessCidr: 0.0.0.0/0
+      QSS3KeyPrefix: logstream/
+      QSS3BucketRegion: us-west-2
+      QSS3BucketName: aws-quickstart
 tests:
-  simple:
-    template: ./templates/cribl-single-template.yaml
-    regions: 
-      - us-west-2
-  newvpcX86:
-    template: ./templates/cribl-single-template-vpc.yaml
-    regions:
-      - us-west-2
-  newvpcloggingX86:
-    template: ./templates/cribl-single-template-vpc-logging.yaml
-    regions:
-      - us-west-2 
-  simple:
-    template: ./templates/cribl-single-template-arm64.yaml
-    regions:
-      - us-west-2
-  newvpcX86:
-    template: ./templates/cribl-single-template-vpc-arm64.yaml
-    regions:
-      - us-west-2
-  newvpcloggingX86:
-    template: ./templates/cribl-single-template-vpc-logging-arm64.yaml
-    regions:
-      - us-west-2
+  cribl-deploy-new-vpc-arm:
+    parameters: 
+      AvailabilityZones: $[taskcat_getaz_2]
+      VPCCIDR: 10.0.0.0/16
+      PublicSubnet1CIDR: 10.0.128.0/20
+      PublicSubnet2CIDR: 10.0.144.0/20
+      PrivateSubnet1CIDR: 10.0.0.0/19
+      PrivateSubnet2CIDR: 10.0.32.0/19
+      VPCTenancy: default
+      QSS3BucketRegion: $[taskcat_current_region]
+    template: templates/cribl-single-arm64-new-vpc.template.yaml

docs/partner_editable/architecture.adoc

@@ -8,7 +8,7 @@ AWS Cloud.
 
 [#architecture1]
 .Quick Start architecture for {partner-product-short-name} on AWS
-image::../images/architecture_diagram.png[Architecture]
+image::../images/Cribl_AWS_Single.png[Architecture]
 
 As shown in <<architecture1>>, the Quick Start sets up the following:
 

docs/partner_editable/faq_troubleshooting.adoc

@@ -19,6 +19,16 @@ For more information, see https://docs.aws.amazon.com/AWSCloudFormation/latest/U
 
 *A.* Yes, you can deploy in other regions that are supported in the regions listed in this document. To change the region, make sure to select the new region in the CloudFormation drop down in the AWS Console. Then, in the CloudFormation template itself, make sure to specify the region that you selected. 
 
+*Q.* Can I modify the EC2 IAM Policy to include other S3 buckets, Kinesis Streams or other AWS resources for my instance to access? 
+
+*A.* Yes, you can modify the EC2 IAM Policy to include other resources you want the Cribl LogStream instance to access. Please refer to https://docs.cribl.io/logstream/sources-s3#s3-and-sqs-permissions for S3 / SQS permissions and https://docs.cribl.io/logstream/sources-kinesis-streams#authentication for Kinesis permissions.
+
+*Q.* Where can I see the outputs when I deploy a New VPC or New VPC + Logging Cloudformation template?
+*A.* When you deploy this stack, you will actually deploy a **VPC** and **CriblDeploy** templates, just go to the **CriblDeploy** template to see the URL for your Cribl Login. You will still need to use the AMI Instance ID as the password.
+
+*Q.* My instance was rebuilt by the autoscale group and I can no longer log into my instance, what's the new password?
+*A.* When your instance gets rebuilt by the AutoScale Group, the password is reset to the new AMI Instance ID. 
+
 == Troubleshooting
 
 Make sure to select TWO Availability Zones (AZ) for your deployment. Selecting one will cause the template to fail.

templates/cribl-single-template-vpc-logging-arm64.yaml → templates/cribl-single-arm64-new-vpc-logging.template.yaml

@@ -1,5 +1,5 @@
 AWSTemplateFormatVersion: 2010-09-09
-Description: Cribl LogStream+VPC QuickStart Deployment arm64 (qs-1s9hhq8an)
+Description: Cribl LogStream+VPC QuickStart Deployment arm64 (qs-1skh1tk4p)
 Metadata:
   QuickStartDocumentation:
     EntrypointName: "Launch into a new VPC with VPC Flow Logs Enabled"
@@ -22,7 +22,6 @@ Metadata:
         Parameters:
           - webAccessCidr
           - instanceType
-          - AdditionalPolicies
       - Label:
           default: Flow Logs Parameters
         Parameters:
@@ -83,10 +82,6 @@ Parameters:
       - m6gd.2xlarge
       - m6gd.4xlarge
     ConstraintDescription: Must contain valid instance type
-  AdditionalPolicies:
-    Default:  "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore,arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy"
-    Description: A comma separated list of Policy ARNs to add to the IAM role used by Logstream instances
-    Type: CommaDelimitedList
   PublicSubnet1CIDR:
     AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/(1[6-9]|2[0-8]))$
     ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/16-28.
@@ -340,15 +335,11 @@ Resources:
     Type: AWS::CloudFormation::Stack
     Properties:
       TemplateURL: !Sub 
-        - https://${S3Bucket}-cribl-logstream-${S3Region}.s3.${S3Region}.${AWS::URLSuffix}/${QSS3KeyPrefix}/free_arm64_template.yaml
+        - https://${S3Bucket}.s3.${QSS3BucketRegion}.${AWS::URLSuffix}/${QSS3KeyPrefix}/cribl-single-arm64.workload.template.yaml
         - S3Bucket: !If
             - UsingDefaultBucket
-            - !Sub 'aws-quickstart'
-            - !Ref 'QSS3BucketName'
-          S3Region: !If
-            - UsingDefaultBucket
-            - !Ref 'AWS::Region'
-            - !Ref 'QSS3BucketRegion'  
+            - !Sub '${QSS3BucketName}-cribl-logstream-${QSS3BucketRegion}'
+            - !Ref QSS3BucketName
       Parameters:
         webAccessCidr: !Ref webAccessCidr
         vpcId: !GetAtt
@@ -358,7 +349,4 @@ Resources:
           - ','
           - - !GetAtt VPCStack.Outputs.PublicSubnet1ID
             - !GetAtt VPCStack.Outputs.PublicSubnet2ID
-        instanceType: !Ref instanceType
-        AdditionalPolicies: !Join 
-          - ','
-          - !Ref AdditionalPolicies
+        instanceType: !Ref instanceType

templates/cribl-single-template-vpc-arm64.yaml → templates/cribl-single-arm64-new-vpc.template.yaml

@@ -1,5 +1,5 @@
 AWSTemplateFormatVersion: 2010-09-09
-Description: Cribl LogStream+VPC QuickStart Deployment arm64 (qs-1s9hhq8an)
+Description: Cribl LogStream+VPC QuickStart Deployment arm64 (qs-1skh1tk4h)
 Metadata:
   QuickStartDocumentation:
     EntrypointName: "Launch into a new VPC"
@@ -21,7 +21,6 @@ Metadata:
           default: Amazon EC2 configuration
         Parameters:
           - instanceType
-          - AdditionalPolicies
           - webAccessCidr
       - Label:
           default: AWS Quick Start configuration
@@ -50,7 +49,6 @@ Metadata:
         default: Quick Start S3 bucket name
       QSS3KeyPrefix:
         default: Quick Start S3 key prefix  
-  cfn-lint: { config: { ignore_checks: [E9007] } }
 Parameters:
   AvailabilityZones:
     Description: List of Availability Zones to use for the subnets in the VPC.
@@ -77,10 +75,6 @@ Parameters:
       - m6gd.2xlarge
       - m6gd.4xlarge
     ConstraintDescription: Must contain valid instance type
-  AdditionalPolicies:
-    Default:  "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore,arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy"
-    Description: A comma separated list of Policy ARNs to add to the IAM role used by Logstream instances
-    Type: CommaDelimitedList
   PrivateSubnet1CIDR:
     AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/(1[6-9]|2[0-8]))$
     ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/16-28.
@@ -203,16 +197,12 @@ Resources:
   CriblDeploy:
     Type: AWS::CloudFormation::Stack
     Properties:
-      TemplateURL: !Sub
-        - https://${S3Bucket}-cribl-logstream-${S3Region}.s3.${S3Region}.${AWS::URLSuffix}/${QSS3KeyPrefix}/free_arm64_template.yaml
+      TemplateURL: !Sub 
+        - https://${S3Bucket}.s3.${QSS3BucketRegion}.${AWS::URLSuffix}/${QSS3KeyPrefix}/cribl-single-arm64.workload.template.yaml
         - S3Bucket: !If
             - UsingDefaultBucket
-            - !Sub 'aws-quickstart'
-            - !Ref 'QSS3BucketName'
-          S3Region: !If
-            - UsingDefaultBucket
-            - !Ref 'AWS::Region'
-            - !Ref 'QSS3BucketRegion'          
+            - !Sub '${QSS3BucketName}-cribl-logstream-${QSS3BucketRegion}'
+            - !Ref QSS3BucketName
       Parameters:
         webAccessCidr: !Ref webAccessCidr
         vpcId: !GetAtt
@@ -222,7 +212,4 @@ Resources:
           - ','
           - - !GetAtt VPCStack.Outputs.PublicSubnet1ID
             - !GetAtt VPCStack.Outputs.PublicSubnet2ID
-        instanceType: !Ref instanceType
-        AdditionalPolicies: !Join 
-          - ','
-          - !Ref AdditionalPolicies
+        instanceType: !Ref instanceType

templates/cribl-single-template-arm64.yaml → templates/cribl-single-arm64.entrypoint.template.yaml

@@ -1,5 +1,5 @@
 AWSTemplateFormatVersion: 2010-09-09
-Description: Cribl LogStream Existing VPC QuickStart Deployment arm64 (qs-1s9hhq8an)
+Description: Cribl LogStream Existing VPC QuickStart Deployment arm64 (qs-1skh1tk21)
 Metadata:
   QuickStartDocumentation:
     EntrypointName: Launch into an existing VPC
@@ -17,7 +17,6 @@ Metadata:
           default: Amazon EC2 configuration
         Parameters:
           - instanceType
-          - AdditionalPolicies
       - Label:
           default: AWS Quick Start configuration
         Parameters:
@@ -37,8 +36,6 @@ Metadata:
         default: Quick Start S3 bucket name
       QSS3KeyPrefix:
         default: Quick Start S3 key prefix  
-#  cfn-lint: { config: { ignore_checks: [E9007] } }
-
 Parameters:
   vpcId:
     Description: "REQUIRED: ID of your existing VPC."
@@ -73,10 +70,6 @@ Parameters:
       - m6gd.2xlarge
       - m6gd.4xlarge
     ConstraintDescription: Must contain valid instance type
-  AdditionalPolicies:
-    Default:  "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore,arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy"
-    Description: A comma separated list of Policy ARNs to add to the IAM role used by Logstream instances
-    Type: CommaDelimitedList
   QSS3BucketName:
     AllowedPattern: ^[0-9a-zA-Z]+([0-9a-zA-Z-]*[0-9a-zA-Z])*$
     ConstraintDescription: The Quick Start bucket name can include numbers, lowercase
@@ -132,11 +125,12 @@ Resources:
   CriblDeploy:
     Type: AWS::CloudFormation::Stack
     Properties:
+      #TemplateURL: https://aws-quickstart-cribl-logstream-us-west-2.s3.us-west-2.amazonaws.com/logstream/cribl-single-arm64.workload.template.yaml
       TemplateURL: !Sub 
-      - https://${S3Bucket}.s3.${QSS3BucketRegion}.${AWS::URLSuffix}/${QSS3KeyPrefix}/free_arm64_template.yaml
+      - https://${S3Bucket}.s3.${QSS3BucketRegion}.${AWS::URLSuffix}/${QSS3KeyPrefix}/cribl-single-arm64.workload.template.yaml
       - S3Bucket: !If
            - UsingDefaultBucket
-           - !Sub 'aws-quickstart-cribl-logstream-${QSS3BucketRegion}'
+           - !Sub '${QSS3BucketName}-${QSS3BucketRegion}'
            - !Ref QSS3BucketName
       Parameters:
         webAccessCidr: !Ref webAccessCidr
@@ -145,6 +139,3 @@ Resources:
           - ','
           - !Ref subnetIds
         instanceType: !Ref instanceType
-        AdditionalPolicies: !Join 
-          - ','
-          - !Ref AdditionalPolicies