Skip to content

Commit

Permalink
AT-10925: refactor integration tests
Browse files Browse the repository at this point in the history
  • Loading branch information
rddimon committed Dec 13, 2023
1 parent aebdfc1 commit 5463274
Show file tree
Hide file tree
Showing 17 changed files with 690 additions and 746 deletions.
317 changes: 162 additions & 155 deletions package-lock.json

Large diffs are not rendered by default.

8 changes: 6 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -49,11 +49,14 @@
]
},
"devDependencies": {
"@aws-sdk/client-lambda": "^3.465.0",
"@types/js-yaml": "^4.0.9",
"@types/mocha": "^10.0.6",
"@types/node": "^20.10.3",
"@types/randomstring": "^1.1.11",
"@types/shelljs": "^0.8.15",
"@typescript-eslint/eslint-plugin": "^5.62.0",
"@typescript-eslint/parser": "^5.62.0",
"aws-sdk-client-mock": "^3.0.0",
"chai": "^4.3.10",
"chai-spies": "^1.1.0",
"eslint": "^7.32.0",
Expand All @@ -74,6 +77,7 @@
"dependencies": {
"@aws-sdk/client-ec2": "^3.467.0",
"@smithy/smithy-client": "^2.1.18",
"@smithy/util-retry": "^2.0.8"
"@smithy/util-retry": "^2.0.8",
"ts-md5": "^1.3.1"
}
}
46 changes: 42 additions & 4 deletions src/common/lambda-function.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,14 @@ import { VPCDiscovery, VPC } from "../types";
import { isObjectEmpty } from "../utils";
import { validateVPCDiscoveryConfig } from "../validation";
import Logging from "../logging";
import { Md5 } from "ts-md5";

export class LambdaFunction {
public ec2Wrapper: EC2Wrapper;
private readonly basicVPCDiscovery?: VPCDiscovery;
private vpcIdsCache = {};
private subnetsIdsCache = {};
private SGIdsCache = {};

constructor (credentials: any, basicVPCDiscovery: VPCDiscovery) {
this.ec2Wrapper = new EC2Wrapper(credentials);
Expand Down Expand Up @@ -58,14 +62,13 @@ export class LambdaFunction {
*/
private async getVpcConfig (vpcDiscovery: VPCDiscovery): Promise<VPC> {
const vpc: VPC = {};
const vpcId = await this.ec2Wrapper.getVpcId(vpcDiscovery.vpcName);

const vpcId = await this.getVPCId(vpcDiscovery.vpcName);
Logging.logInfo(`Found VPC with id '${vpcId}'`);

if (vpcDiscovery.subnets) {
vpc.subnetIds = [];
for (const subnet of vpcDiscovery.subnets) {
const subnetIds = await this.ec2Wrapper.getSubnetIds(vpcId, subnet.tagKey, subnet.tagValues);
const subnetIds = await this.getVPCSubnets(vpcId, subnet.tagKey, subnet.tagValues);
vpc.subnetIds = vpc.subnetIds.concat(subnetIds);
}
// remove duplicate elements from the array
Expand All @@ -74,12 +77,47 @@ export class LambdaFunction {
if (vpcDiscovery.securityGroups) {
vpc.securityGroupIds = [];
for (const group of vpcDiscovery.securityGroups) {
const groupIds = await this.ec2Wrapper.getSecurityGroupIds(vpcId, group.names, group.tagKey, group.tagValues);
const groupIds = await this.getVPCSecurityGroups(vpcId, group.names, group.tagKey, group.tagValues);
vpc.securityGroupIds = vpc.securityGroupIds.concat(groupIds);
}
// remove duplicate elements from the array
vpc.securityGroupIds = [...new Set(vpc.securityGroupIds)];
}
return vpc;
}

/**
* Get the VPC id from cache or read from AWS
* @returns {Promise<object>}
*/
private async getVPCId (vpcName: string) {
if (this.vpcIdsCache[vpcName] === undefined) {
this.vpcIdsCache[vpcName] = await this.ec2Wrapper.getVpcId(vpcName);
}
return this.vpcIdsCache[vpcName];
}

/**
* Get the subnet ids from cache or read from AWS
* @returns {Promise<object>}
*/
private async getVPCSubnets (vpcId: string, tagKey: string, tagValues: string[]) {
const hash = Md5.hashStr(vpcId + tagKey + tagValues.join());
if (!this.subnetsIdsCache[hash]) {
this.subnetsIdsCache[hash] = await this.ec2Wrapper.getSubnetIds(vpcId, tagKey, tagValues);
}
return this.subnetsIdsCache[hash];
}

/**
* Get the security group ids from cache or read from AWS
* @returns {Promise<object>}
*/
private async getVPCSecurityGroups (vpcId: string, names: string[], tagKey: string, tagValues: string[]) {
const hash = Md5.hashStr(vpcId + (names || []).join() + tagKey + (tagValues || []).join());
if (!this.SGIdsCache[hash]) {
this.SGIdsCache[hash] = await this.ec2Wrapper.getSecurityGroupIds(vpcId, names, tagKey, tagValues);
}
return this.SGIdsCache[hash];
}
}
120 changes: 61 additions & 59 deletions src/types.ts
Original file line number Diff line number Diff line change
@@ -1,86 +1,88 @@
export interface SubnetItem {
tagKey: string;
tagValues: string[];
tagKey: string;
tagValues: string[];
}

export interface SecurityGroupItem {
tagKey?: string;
tagValues?: string[];
names?: string[];
tagKey?: string;
tagValues?: string[];
names?: string[];
}

export interface VPCDiscovery {
vpcName: string;
subnets?: SubnetItem[];
securityGroups?: SecurityGroupItem[];
// for supporting back compatibility
subnetNames?: string[];
securityGroupNames?: string[];
vpcName: string;
subnets?: SubnetItem[];
securityGroups?: SecurityGroupItem[];
// for supporting back compatibility
subnetNames?: string[];
securityGroupNames?: string[];
}

export interface VPC {
subnetIds?: string[];
securityGroupIds?: string[];
subnetIds?: string[];
securityGroupIds?: string[];
}

export interface ServerlessService {
service: string
provider: {
stage: string,
region?: string
vpc: {},
},
functions: {
name: {
vpc: {
subnetIds: {} | undefined,
securityGroupIds: {} | undefined,
},
vpcDiscovery: VPCDiscovery | boolean | undefined
}
},
custom: {
vpcDiscovery: VPCDiscovery | undefined
}
}

export interface ServerlessInstance {
service: {
service: string
provider: {
stage: string,
region?: string
vpc: {},
},
functions: {
name: {
vpc: {
subnetIds: {} | undefined,
securityGroupIds: {} | undefined,
},
vpcDiscovery: VPCDiscovery | boolean | undefined
}
},
custom: {
vpcDiscovery: VPCDiscovery | undefined
},
};
providers: {
aws: {
getCredentials(),
getRegion(),
},
};
configSchemaHandler: {
defineCustomProperties(props: any),
defineFunctionProperties(provider: string, props: any),
};
cli: {
log(str: string, entity?: string),
consoleLog(str: any),
};
service: ServerlessService;
providers: {
aws: {
getCredentials (),
getRegion (),
},
};
configSchemaHandler: {
defineCustomProperties (props: any),
defineFunctionProperties (provider: string, props: any),
};
cli: {
log (str: string, entity?: string),
consoleLog (str: any),
};
}

export interface ServerlessOptions {
stage: string;
region?: string;
stage: string;
region?: string;
}

interface ServerlessProgress {
update(message: string): void
update (message: string): void

remove(): void
remove (): void
}

export interface ServerlessProgressFactory {
get(name: string): ServerlessProgress;
get (name: string): ServerlessProgress;
}

export interface ServerlessUtils {
writeText: (message: string) => void,
log: ((message: string) => void) & {
error(message: string): void
verbose(message: string): void
warning(message: string): void
}
progress: ServerlessProgressFactory
writeText: (message: string) => void,
log: ((message: string) => void) & {
error (message: string): void
verbose (message: string): void
warning (message: string): void
}
progress: ServerlessProgressFactory
}
43 changes: 1 addition & 42 deletions src/utils.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,5 @@
import { Service } from "aws-sdk";
import { Client, Command } from "@smithy/smithy-client";

const RETRYABLE_ERRORS = ["Throttling", "RequestLimitExceeded", "TooManyRequestsException"];

/**
* Iterate through the pages of a AWS SDK response and collect them into a single array
*
Expand All @@ -22,51 +19,14 @@ async function getAWSPagedResults<ClientOutput> (
let results = [];
let response = await client.send(params);
results = results.concat(response[resultsKey] || results);
while (
nextRequestTokenKey in response && response[nextRequestTokenKey]
) {
while (nextRequestTokenKey in response && response[nextRequestTokenKey]) {
params.input[nextTokenKey] = response[nextRequestTokenKey];
response = await client.send(params);
results = results.concat(response[resultsKey]);
}
return results;
}

async function throttledCall (service: Service, funcName: string, params: object): Promise<any> {
const maxTimePassed = 5 * 60;

let timePassed = 0;
let previousInterval = 0;

const minWait = 3;
const maxWait = 60;

while (true) {
try {
return await service[funcName](params).promise();
} catch (ex) {
// rethrow the exception if it is not a type of retryable exception
if (RETRYABLE_ERRORS.indexOf(ex.code) === -1) {
throw ex;
}

// rethrow the exception if we have waited too long
if (timePassed >= maxTimePassed) {
throw ex;
}

// Sleep using the Decorrelated Jitter algorithm recommended by AWS
// https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/
let newInterval = Math.random() * Math.min(maxWait, previousInterval * 3);
newInterval = Math.max(minWait, newInterval);

await sleep(newInterval);
previousInterval = newInterval;
timePassed += previousInterval;
}
}
}

/**
* Stops event thread execution for given number of seconds.
* @param seconds
Expand Down Expand Up @@ -103,7 +63,6 @@ function getValueFromTags (tags, tagKey) {
export {
sleep,
getAWSPagedResults,
throttledCall,
isObjectEmpty,
wildcardMatches,
getValueFromTags
Expand Down
23 changes: 13 additions & 10 deletions test/integration-tests/base.ts
Original file line number Diff line number Diff line change
@@ -1,20 +1,23 @@
import randomstring from "randomstring";

const TEST_VPC_NAME = process.env.TEST_VPC_NAME;

if (!TEST_VPC_NAME) {
throw new Error("TEST_VPC_NAME environment variable not set");
}

function getRandomString (): string {
return randomstring.generate({
capitalization: "lowercase",
charset: "alphanumeric",
length: 5
});
}
const RANDOM_STRING = randomstring.generate({
capitalization: "lowercase",
charset: "alphanumeric",
length: 5
});

// this is important to set as we are using this var in the serverless yml config
process.env.RANDOM_STRING = RANDOM_STRING;

const TEMP_DIR = `~/tmp/vpc-discovery-test-${RANDOM_STRING}`;

export {
getRandomString,
TEST_VPC_NAME
TEST_VPC_NAME,
RANDOM_STRING,
TEMP_DIR
};
22 changes: 13 additions & 9 deletions test/integration-tests/basic/basic-example/serverless.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# basic example
service: test-sls-vpc-dicovery-${env:RANDOM_STRING}
service: test-sls-vpc-discovery-${env:RANDOM_STRING}

provider:
name: aws
Expand All @@ -21,21 +21,25 @@ functions:
# inherit basic subnet ids and use func security group ids
vpcDiscovery:
vpcName: ${env:TEST_VPC_NAME}
securityGroupNames:
- ${env:TEST_VPC_NAME}_maintenance
securityGroups:
- names:
- ${env:TEST_VPC_NAME}_maintenance

plugins:
- serverless-vpc-discovery

custom:
vpcDiscovery:
vpcName: ${env:TEST_VPC_NAME}
subnetNames:
- ${env:TEST_VPC_NAME}_intranet_us-west-2a
- ${env:TEST_VPC_NAME}_intranet_us-west-2b
- ${env:TEST_VPC_NAME}_intranet_us-west-2c
securityGroupNames:
- ${env:TEST_VPC_NAME}_intranet
subnets:
- tagKey: Name
tagValues:
- ${env:TEST_VPC_NAME}_intranet_us-west-2a
- ${env:TEST_VPC_NAME}_intranet_us-west-2b
- ${env:TEST_VPC_NAME}_intranet_us-west-2c
securityGroups:
- names:
- ${env:TEST_VPC_NAME}_intranet

package:
patterns:
Expand Down
Loading

0 comments on commit 5463274

Please sign in to comment.