Fix(Set outputBucket property on aws-kinesisstreams-gluejob): Issue #448

to include S3 bucket for Glue Job that the consturct creates (#452) * update glue max allocation units to 2 * update glue max allocation units to 2 * resolving workerType and maxCapacity config related rules * resolving workerType and maxCapacity config related rules * resolving workerType and maxCapacity config related rules * removing additional lines from the file * fix snapshots for glue helper * fix snapshots for glue helper * fix snapshots for glue helper * fix snapshots for glue helper * fix snapshots for glue helper * fix snapshots for glue helper * documentation rephrase * new construct first commit with README and architecture * adding minimal deployable code to README * adding minimal deployable code to README * fix for github issue #448 Co-authored-by: nihitkas <[email protected]> Co-authored-by: biffgaut <[email protected]>
awslabs · Oct 12, 2021 · c40e1f7 · c40e1f7
1 parent e10394d
commit c40e1f7
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 6 deletions.
diff --git a/source/patterns/@aws-solutions-constructs/aws-kinesisstreams-gluejob/lib/index.ts b/source/patterns/@aws-solutions-constructs/aws-kinesisstreams-gluejob/lib/index.ts
@@ -126,6 +126,11 @@ export class KinesisstreamsToGluejob extends Construct {
   public readonly glueJobRole: IRole;
   public readonly database: glue.CfnDatabase;
   public readonly table: glue.CfnTable;
+  /**
+   * This property is only set if the Glue Job is created by the construct. If an exisiting Glue Job
+   * configuraton is supplied, the construct does not create an S3 bucket and hence the @outputBucket
+   * property is undefined
+   */
   public readonly outputBucket?: [Bucket, (Bucket | undefined)?];
   public readonly cloudwatchAlarms?: cloudwatch.Alarm[];
 
@@ -159,7 +164,7 @@ export class KinesisstreamsToGluejob extends Construct {
       });
     }
 
-    [ this.glueJob, this.glueJobRole ] = defaults.buildGlueJob(this, {
+    [ this.glueJob, this.glueJobRole, this.outputBucket ] = defaults.buildGlueJob(this, {
       existingCfnJob: props.existingGlueJob,
       glueJobProps: props.glueJobProps,
       table: this.table!,

diff --git a/source/patterns/@aws-solutions-constructs/core/lib/glue-job-helper.ts b/source/patterns/@aws-solutions-constructs/core/lib/glue-job-helper.ts
@@ -78,7 +78,7 @@ export interface BuildGlueJobProps {
   readonly outputDataStore?: SinkDataStoreProps
 }
 
-export function buildGlueJob(scope: Construct, props: BuildGlueJobProps): [glue.CfnJob, IRole] {
+export function buildGlueJob(scope: Construct, props: BuildGlueJobProps): [glue.CfnJob, IRole, [Bucket, (Bucket | undefined)?]?] {
   if (!props.existingCfnJob) {
     if (props.glueJobProps) {
       if (props.glueJobProps.glueVersion === '2.0' && props.glueJobProps.maxCapacity) {
@@ -101,7 +101,7 @@ export function buildGlueJob(scope: Construct, props: BuildGlueJobProps): [glue.
 }
 
 export function deployGlueJob(scope: Construct, glueJobProps: glue.CfnJobProps, database: glue.CfnDatabase, table: glue.CfnTable,
-  outputDataStore: SinkDataStoreProps): [glue.CfnJob, IRole] {
+  outputDataStore: SinkDataStoreProps): [glue.CfnJob, IRole, [Bucket, (Bucket | undefined)?]] {
 
   let _glueSecurityConfigName: string;
 
@@ -183,7 +183,7 @@ export function deployGlueJob(scope: Construct, glueJobProps: glue.CfnJobProps,
   _scriptBucketLocation.grantRead(_jobRole);
 
   const _glueJob: glue.CfnJob = new glue.CfnJob(scope, 'KinesisETLJob', _newGlueJobProps);
-  return [_glueJob, _jobRole];
+  return [_glueJob, _jobRole, _outputLocation];
 }
 
 /**

diff --git a/source/patterns/@aws-solutions-constructs/core/test/glue-job-helper.test.ts b/source/patterns/@aws-solutions-constructs/core/test/glue-job-helper.test.ts
@@ -42,7 +42,7 @@ test('Test deployment with role creation', () => {
 
   const _database = defaults.createGlueDatabase(stack, defaults.DefaultGlueDatabaseProps());
 
-  defaults.buildGlueJob(stack, {
+  const _glueJob = defaults.buildGlueJob(stack, {
     glueJobProps: cfnJobProps,
     database: _database,
     table: defaults.createGlueTable(stack, _database, undefined, [{
@@ -52,6 +52,8 @@ test('Test deployment with role creation', () => {
     }], 'kinesis', {STREAM_NAME: 'testStream'})
   });
 
+  expect(_glueJob[2]?.[0]).toBeDefined();
+  expect(_glueJob[2]?.[0]).toBeInstanceOf(Bucket);
   expect(stack).toHaveResourceLike('AWS::Glue::Job', {
     Type: "AWS::Glue::Job",
     Properties: {
@@ -99,7 +101,7 @@ test('Create a Glue Job outside the construct', () => {
 
   const _database = defaults.createGlueDatabase(stack, defaults.DefaultGlueDatabaseProps());
 
-  defaults.buildGlueJob(stack, {
+  const _glueJob = defaults.buildGlueJob(stack, {
     existingCfnJob: _existingCfnJob,
     outputDataStore: {
       datastoreType: defaults.SinkStoreType.S3
@@ -111,6 +113,8 @@ test('Create a Glue Job outside the construct', () => {
       comment: ""
     }], 'kinesis', {STREAM_NAME: 'testStream'})
   });
+
+  expect(_glueJob[2]).not.toBeDefined();
   expect(stack).toHaveResourceLike('AWS::Glue::Job', {
     Type: "AWS::Glue::Job",
     Properties: {