feat(new construct): aws-lambda-kinesisfirehose (awslabs#875)

* Initial README.md draft * Minor edits, arch diagram * Typo on arch diagram * First draft - complete implementation * Add dependency to package.json * Suppress autodelete function cfn-nag messages * Updated snapshots, self-review updates * More snapshot updates * One more cfn-nag issue * Response to code review * Address review * Copyright notice updates * Add region to stream name test change * Fix typo * Removing region token from generateName() * Snapshot updates
aws-solutions-constructs-team · Dec 22, 2022 · aef3efa · aef3efa
1 parent 111dcbe
commit aef3efa
Show file tree

Hide file tree

Showing 46 changed files with 5,259 additions and 10 deletions.
diff --git a/...onstructs/aws-eventbridge-kinesisfirehose-s3/test/integ.customLoggingBucket.expected.json b/...onstructs/aws-eventbridge-kinesisfirehose-s3/test/integ.customLoggingBucket.expected.json
@@ -306,6 +306,7 @@
         "DeliveryStreamEncryptionConfigurationInput": {
           "KeyType": "AWS_OWNED_CMK"
         },
+        "DeliveryStreamName": "KinesisFirehosecustomLoggingBuckoses3KinesisFirehoseToS38ABD91BB",
         "ExtendedS3DestinationConfiguration": {
           "BucketARN": {
             "Fn::GetAtt": [

diff --git a/...esisfirehose-s3/test/integ.eventbridge-kinesisfirehose-s3-existing-eventbus.expected.json b/...esisfirehose-s3/test/integ.eventbridge-kinesisfirehose-s3-existing-eventbus.expected.json
@@ -234,6 +234,7 @@
         "DeliveryStreamEncryptionConfigurationInput": {
           "KeyType": "AWS_OWNED_CMK"
         },
+        "DeliveryStreamName": "KinesisFirehoseeventbridgekinesioses3KinesisFirehoseToS316415EBE",
         "ExtendedS3DestinationConfiguration": {
           "BucketARN": {
             "Fn::GetAtt": [

diff --git a/...e-kinesisfirehose-s3/test/integ.eventbridge-kinesisfirehose-s3-new-eventbus.expected.json b/...e-kinesisfirehose-s3/test/integ.eventbridge-kinesisfirehose-s3-new-eventbus.expected.json
@@ -228,6 +228,7 @@
         "DeliveryStreamEncryptionConfigurationInput": {
           "KeyType": "AWS_OWNED_CMK"
         },
+        "DeliveryStreamName": "KinesisFirehoseeventbridgekinesioses3KinesisFirehoseToS3BB825337",
         "ExtendedS3DestinationConfiguration": {
           "BucketARN": {
             "Fn::GetAtt": [

diff --git a/...e-kinesisfirehose-s3/test/integ.eventbridge-kinesisfirehose-s3-no-arguments.expected.json b/...e-kinesisfirehose-s3/test/integ.eventbridge-kinesisfirehose-s3-no-arguments.expected.json
@@ -306,6 +306,7 @@
         "DeliveryStreamEncryptionConfigurationInput": {
           "KeyType": "AWS_OWNED_CMK"
         },
+        "DeliveryStreamName": "KinesisFirehoseeventbridgekinesioses3KinesisFirehoseToS3EFE1A339",
         "ExtendedS3DestinationConfiguration": {
           "BucketARN": {
             "Fn::GetAtt": [

diff --git a/...utions-constructs/aws-iot-kinesisfirehose-s3/test/integ.customLoggingBucket.expected.json b/...utions-constructs/aws-iot-kinesisfirehose-s3/test/integ.customLoggingBucket.expected.json
@@ -306,6 +306,7 @@
         "DeliveryStreamEncryptionConfigurationInput": {
           "KeyType": "AWS_OWNED_CMK"
         },
+        "DeliveryStreamName": "KinesisFirehosecustomLoggingBuckoses3KinesisFirehoseToS356855CD4",
         "ExtendedS3DestinationConfiguration": {
           "BucketARN": {
             "Fn::GetAtt": [

diff --git a/...aws-solutions-constructs/aws-iot-kinesisfirehose-s3/test/integ.no-arguments.expected.json b/...aws-solutions-constructs/aws-iot-kinesisfirehose-s3/test/integ.no-arguments.expected.json
@@ -306,6 +306,7 @@
         "DeliveryStreamEncryptionConfigurationInput": {
           "KeyType": "AWS_OWNED_CMK"
         },
+        "DeliveryStreamName": "KinesisFirehosenoargumentstestiooses3KinesisFirehoseToS300C2E381",
         "ExtendedS3DestinationConfiguration": {
           "BucketARN": {
             "Fn::GetAtt": [

diff --git a/...-solutions-constructs/aws-iot-kinesisfirehose-s3/test/integ.noLoggingBucket.expected.json b/...-solutions-constructs/aws-iot-kinesisfirehose-s3/test/integ.noLoggingBucket.expected.json
@@ -227,6 +227,7 @@
         "DeliveryStreamEncryptionConfigurationInput": {
           "KeyType": "AWS_OWNED_CMK"
         },
+        "DeliveryStreamName": "KinesisFirehosenoLoggingBucketteoses3KinesisFirehoseToS3D5D4755A",
         "ExtendedS3DestinationConfiguration": {
           "BucketARN": {
             "Fn::GetAtt": [

diff --git a/.../aws-kinesisfirehose-s3-and-kinesisanalytics/test/integ.customLoggingBucket.expected.json b/.../aws-kinesisfirehose-s3-and-kinesisanalytics/test/integ.customLoggingBucket.expected.json
@@ -306,6 +306,7 @@
         "DeliveryStreamEncryptionConfigurationInput": {
           "KeyType": "AWS_OWNED_CMK"
         },
+        "DeliveryStreamName": "KinesisFirehosecustomLoggingBuckicss3KinesisFirehoseToS3E91CB2F9",
         "ExtendedS3DestinationConfiguration": {
           "BucketARN": {
             "Fn::GetAtt": [

diff --git a/...structs/aws-kinesisfirehose-s3-and-kinesisanalytics/test/integ.no-arguments.expected.json b/...structs/aws-kinesisfirehose-s3-and-kinesisanalytics/test/integ.no-arguments.expected.json
@@ -305,6 +305,7 @@
         "DeliveryStreamEncryptionConfigurationInput": {
           "KeyType": "AWS_OWNED_CMK"
         },
+        "DeliveryStreamName": "KinesisFirehosenoargumentstestfistackKinesisFirehoseToS38B072B5E",
         "ExtendedS3DestinationConfiguration": {
           "BucketARN": {
             "Fn::GetAtt": [

diff --git a/...ucts/aws-kinesisfirehose-s3-and-kinesisanalytics/test/integ.noLoggingBucket.expected.json b/...ucts/aws-kinesisfirehose-s3-and-kinesisanalytics/test/integ.noLoggingBucket.expected.json
@@ -227,6 +227,7 @@
         "DeliveryStreamEncryptionConfigurationInput": {
           "KeyType": "AWS_OWNED_CMK"
         },
+        "DeliveryStreamName": "KinesisFirehosenoLoggingBucketteicss3KinesisFirehoseToS3EAC1AA77",
         "ExtendedS3DestinationConfiguration": {
           "BucketARN": {
             "Fn::GetAtt": [

diff --git a/source/patterns/@aws-solutions-constructs/aws-kinesisfirehose-s3/lib/index.ts b/source/patterns/@aws-solutions-constructs/aws-kinesisfirehose-s3/lib/index.ts
@@ -91,6 +91,8 @@ export class KinesisFirehoseToS3 extends Construct {
     super(scope, id);
     defaults.CheckProps(props);
 
+    const firehoseId = 'KinesisFirehose';
+
     let bucket: s3.IBucket;
 
     // Setup S3 Bucket
@@ -161,13 +163,19 @@ export class KinesisFirehoseToS3 extends Construct {
       "alias/aws/s3"
     );
 
+    // We need a stream name to set an enviroment variable, as this is an L1 construct
+    // accessing the name as a token doesn't work for environment variable contents, so
+    // we take explicit control of the stream name (but will be overridden by a client provided name)
+    const deliveryStreamName = defaults.generateName(this, firehoseId);
+
     // Setup the default Kinesis Firehose props
     let defaultKinesisFirehoseProps: kinesisfirehose.CfnDeliveryStreamProps = defaults.DefaultCfnDeliveryStreamProps(
       bucket.bucketArn,
       this.kinesisFirehoseRole.roleArn,
       this.kinesisFirehoseLogGroup.logGroupName,
       cwLogStream.logStreamName,
-      awsManagedKey
+      awsManagedKey,
+      deliveryStreamName
     );
 
     // if the client didn't explicity say it was a Kinesis client, then turn on encryption
@@ -189,7 +197,7 @@ export class KinesisFirehoseToS3 extends Construct {
 
     this.kinesisFirehose = new kinesisfirehose.CfnDeliveryStream(
       this,
-      "KinesisFirehose",
+      firehoseId,
       kinesisFirehoseProps
     );
   }

diff --git a/...-solutions-constructs/aws-kinesisfirehose-s3/test/integ.customLoggingBucket.expected.json b/...-solutions-constructs/aws-kinesisfirehose-s3/test/integ.customLoggingBucket.expected.json
@@ -306,6 +306,7 @@
         "DeliveryStreamEncryptionConfigurationInput": {
           "KeyType": "AWS_OWNED_CMK"
         },
+        "DeliveryStreamName": "KinesisFirehosecustomLoggingBuckettestkinesisfirehoses3BC7D2654",
         "ExtendedS3DestinationConfiguration": {
           "BucketARN": {
             "Fn::GetAtt": [

diff --git a/...ns/@aws-solutions-constructs/aws-kinesisfirehose-s3/test/integ.no-arguments.expected.json b/...ns/@aws-solutions-constructs/aws-kinesisfirehose-s3/test/integ.no-arguments.expected.json
@@ -306,6 +306,7 @@
         "DeliveryStreamEncryptionConfigurationInput": {
           "KeyType": "AWS_OWNED_CMK"
         },
+        "DeliveryStreamName": "KinesisFirehosenoargumentstestfirehoses3B5E26CE6",
         "ExtendedS3DestinationConfiguration": {
           "BucketARN": {
             "Fn::GetAtt": [

diff --git a/...@aws-solutions-constructs/aws-kinesisfirehose-s3/test/integ.noLoggingBucket.expected.json b/...@aws-solutions-constructs/aws-kinesisfirehose-s3/test/integ.noLoggingBucket.expected.json
@@ -227,6 +227,7 @@
         "DeliveryStreamEncryptionConfigurationInput": {
           "KeyType": "AWS_OWNED_CMK"
         },
+        "DeliveryStreamName": "KinesisFirehosenoLoggingBuckettestkinesisfirehoses35D36368E",
         "ExtendedS3DestinationConfiguration": {
           "BucketARN": {
             "Fn::GetAtt": [

diff --git a/...-solutions-constructs/aws-kinesisfirehose-s3/test/integ.pre-existing-bucket.expected.json b/...-solutions-constructs/aws-kinesisfirehose-s3/test/integ.pre-existing-bucket.expected.json
@@ -181,6 +181,7 @@
         "DeliveryStreamEncryptionConfigurationInput": {
           "KeyType": "AWS_OWNED_CMK"
         },
+        "DeliveryStreamName": "KinesisFirehosepreexistingbuckets3preexistingbucketstack85DF91B4",
         "ExtendedS3DestinationConfiguration": {
           "BucketARN": {
             "Fn::Join": [

diff --git a/...ns-constructs/aws-kinesisfirehose-s3/test/integ.pre-existing-logging-bucket.expected.json b/...ns-constructs/aws-kinesisfirehose-s3/test/integ.pre-existing-logging-bucket.expected.json
@@ -261,6 +261,7 @@
         "DeliveryStreamEncryptionConfigurationInput": {
           "KeyType": "AWS_OWNED_CMK"
         },
+        "DeliveryStreamName": "KinesisFirehosepreexistinglogginistingloggingbucketstack932A73CD",
         "ExtendedS3DestinationConfiguration": {
           "BucketARN": {
             "Fn::GetAtt": [

diff --git a/...rns/@aws-solutions-constructs/aws-kinesisfirehose-s3/test/test.kinesisfirehose-s3.test.ts b/...rns/@aws-solutions-constructs/aws-kinesisfirehose-s3/test/test.kinesisfirehose-s3.test.ts
@@ -304,4 +304,27 @@ test('s3 bucket with one content bucket and no logging bucket', () => {
   });
 
   expect(stack).toCountResources("AWS::S3::Bucket", 1);
-});
+});
+
+test('check client provided name overrides default DeliveryStreamName', () => {
+  const stack = new cdk.Stack();
+  const testName = 'client-name';
+
+  deploy(stack, {
+    kinesisFirehoseProps: {
+      deliveryStreamName: testName
+    }
+  });
+  expect(stack).toHaveResourceLike("AWS::KinesisFirehose::DeliveryStream", {
+    DeliveryStreamName: testName
+  });
+});
+
+test('check DeliveryStreamName is populated', () => {
+  const stack = new cdk.Stack(undefined, 'test-stack');
+
+  deploy(stack);
+  expect(stack).toHaveResourceLike("AWS::KinesisFirehose::DeliveryStream", {
+    DeliveryStreamName: "KinesisFirehoseteststacktestfirehoses3F50DF0E1"
+  });
+});
diff --git a/...tructs/aws-kinesisstreams-kinesisfirehose-s3/test/integ.customLoggingBucket.expected.json b/...tructs/aws-kinesisstreams-kinesisfirehose-s3/test/integ.customLoggingBucket.expected.json
@@ -317,6 +317,7 @@
     "testkinesisfirehoses3KinesisFirehoseToS3KinesisFirehoseEB65C83D": {
       "Type": "AWS::KinesisFirehose::DeliveryStream",
       "Properties": {
+        "DeliveryStreamName": "KinesisFirehosecustomLoggingBuckoses3KinesisFirehoseToS38ABD91BB",
         "DeliveryStreamType": "KinesisStreamAsSource",
         "ExtendedS3DestinationConfiguration": {
           "BucketARN": {

diff --git a/...constructs/aws-kinesisstreams-kinesisfirehose-s3/test/integ.existing-bucket.expected.json b/...constructs/aws-kinesisstreams-kinesisfirehose-s3/test/integ.existing-bucket.expected.json
@@ -192,6 +192,7 @@
     "testexistingbucketfirehoses3stackKinesisFirehoseToS3KinesisFirehose46614BF6": {
       "Type": "AWS::KinesisFirehose::DeliveryStream",
       "Properties": {
+        "DeliveryStreamName": "KinesisFirehoseexistingbuckettesstackKinesisFirehoseToS3E85D23B4",
         "DeliveryStreamType": "KinesisStreamAsSource",
         "ExtendedS3DestinationConfiguration": {
           "BucketARN": {

diff --git a/...ts/aws-kinesisstreams-kinesisfirehose-s3/test/integ.existing-logging-bucket.expected.json b/...ts/aws-kinesisstreams-kinesisfirehose-s3/test/integ.existing-logging-bucket.expected.json
@@ -272,6 +272,7 @@
     "testexistingloggingbucketstreamsfirehoses3stackKinesisFirehoseToS3KinesisFirehose5F6EDEF6": {
       "Type": "AWS::KinesisFirehose::DeliveryStream",
       "Properties": {
+        "DeliveryStreamName": "KinesisFirehoseexistingloggingbustackKinesisFirehoseToS316B2A519",
         "DeliveryStreamType": "KinesisStreamAsSource",
         "ExtendedS3DestinationConfiguration": {
           "BucketARN": {

diff --git a/...nstructs/aws-kinesisstreams-kinesisfirehose-s3/test/integ.existingStreamObj.expected.json b/...nstructs/aws-kinesisstreams-kinesisfirehose-s3/test/integ.existingStreamObj.expected.json
@@ -553,6 +553,7 @@
     "testexistingstreamfirehoses3stackKinesisFirehoseToS3KinesisFirehose0834A7FB": {
       "Type": "AWS::KinesisFirehose::DeliveryStream",
       "Properties": {
+        "DeliveryStreamName": "KinesisFirehoseexistingStreamObjstackKinesisFirehoseToS36519FC08",
         "DeliveryStreamType": "KinesisStreamAsSource",
         "ExtendedS3DestinationConfiguration": {
           "BucketARN": {

diff --git a/...ns-constructs/aws-kinesisstreams-kinesisfirehose-s3/test/integ.no-arguments.expected.json b/...ns-constructs/aws-kinesisstreams-kinesisfirehose-s3/test/integ.no-arguments.expected.json
@@ -317,6 +317,7 @@
     "teststreamfirehoses3KinesisFirehoseToS3KinesisFirehose7303FF77": {
       "Type": "AWS::KinesisFirehose::DeliveryStream",
       "Properties": {
+        "DeliveryStreamName": "KinesisFirehosenoargumentsteststoses3KinesisFirehoseToS3288D7D36",
         "DeliveryStreamType": "KinesisStreamAsSource",
         "ExtendedS3DestinationConfiguration": {
           "BucketARN": {

diff --git a/...ce/patterns/@aws-solutions-constructs/aws-lambda-elasticachememcached/README.md b/...ce/patterns/@aws-solutions-constructs/aws-lambda-elasticachememcached/README.md
@@ -82,7 +82,7 @@ new LambdaToElasticachememcached(this, "LambdaToCachePattern", new LambdaToElast
 |:-------------|:----------------|-----------------|
 |existingLambdaObj?|[`lambda.Function`](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_lambda.Function.html)|Existing instance of Lambda Function object, providing both this and `lambdaFunctionProps` will cause an error.|
 |lambdaFunctionProps?|[`lambda.FunctionProps`](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_lambda.FunctionProps.html)|Optional user provided props to override the default props for the Lambda function.|
-|existingVpc?|[`ec2.IVpc`](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_ec2.IVpc.html)|An optional, existing VPC into which this pattern should be deployed. When deployed in a VPC, the Lambda function will use ENIs in the VPC to access network resources and an Interface Endpoint will be created in the VPC for Amazon SQS. If an existing VPC is provided, the `deployVpc` property cannot be `true`. This uses `ec2.IVpc` to allow clients to supply VPCs that exist outside the stack using the [`ec2.Vpc.fromLookup()`](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_ec2.Vpc.html#static-fromwbrlookupscope-id-options) method.|
+|existingVpc?|[`ec2.IVpc`](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_ec2.IVpc.html)|An optional, existing VPC into which this pattern should be deployed. When deployed in a VPC, the Lambda function will use ENIs in the VPC to access network resources and an Interface Endpoint will be created in the VPC for Amazon Elasticache. If an existing VPC is provided, the `deployVpc` property cannot be `true`. This uses `ec2.IVpc` to allow clients to supply VPCs that exist outside the stack using the [`ec2.Vpc.fromLookup()`](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_ec2.Vpc.html#static-fromwbrlookupscope-id-options) method.|
 |vpcProps?|[`ec2.VpcProps`](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_ec2.VpcProps.html)|Optional user provided properties to override the default properties for the new VPC. `subnetConfiguration` is set by the pattern, so any values for those properties supplied here will be overrriden. |
 | cacheEndpointEnvironmentVariableName?| string | Optional Name for the Lambda function environment variable set to the cache endpoint. Default: CACHE_ENDPOINT |
 | cacheProps? | [`cache.CfnCacheClusterProps`](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_elasticache.CfnCacheClusterProps.html) | Optional user provided props to override the default props for the Elasticache Cluster. Providing both this and `existingCache` will cause an error. |

diff --git a/source/patterns/@aws-solutions-constructs/aws-lambda-kinesisfirehose/.eslintignore b/source/patterns/@aws-solutions-constructs/aws-lambda-kinesisfirehose/.eslintignore
@@ -0,0 +1,5 @@
+lib/*.js
+test/*.js
+*.d.ts
+coverage
+test/lambda/index.js
diff --git a/source/patterns/@aws-solutions-constructs/aws-lambda-kinesisfirehose/.gitignore b/source/patterns/@aws-solutions-constructs/aws-lambda-kinesisfirehose/.gitignore
@@ -0,0 +1,15 @@
+lib/*.js
+test/*.js
+*.js.map
+*.d.ts
+node_modules
+*.generated.ts
+dist
+.jsii
+
+.LAST_BUILD
+.nyc_output
+coverage
+.nycrc
+.LAST_PACKAGE
+*.snk
diff --git a/source/patterns/@aws-solutions-constructs/aws-lambda-kinesisfirehose/.npmignore b/source/patterns/@aws-solutions-constructs/aws-lambda-kinesisfirehose/.npmignore
@@ -0,0 +1,21 @@
+# Exclude typescript source and config
+*.ts
+tsconfig.json
+coverage
+.nyc_output
+*.tgz
+*.snk
+*.tsbuildinfo
+
+# Include javascript files and typescript declarations
+!*.js
+!*.d.ts
+
+# Exclude jsii outdir
+dist
+
+# Include .jsii
+!.jsii
+
+# Include .jsii
+!.jsii