From 9efc6a07027faf80dafd97ac36bbece46a780aae Mon Sep 17 00:00:00 2001 From: Huafu Gandon Date: Sat, 22 Sep 2018 16:12:21 +0200 Subject: [PATCH] ci(e2e): allows a diff dir for each config --- .travis.yml | 1 + appveyor.yml | 1 + .../test-case/__hooks-source__.js.hbs | 4 +- e2e/__helpers__/test-case/run-descriptor.ts | 17 +- e2e/__helpers__/test-case/run-result.ts | 12 +- e2e/__helpers__/test-case/runtime.ts | 175 +++++++++++------- e2e/__helpers__/test-case/types.ts | 3 - e2e/__helpers__/test-case/utils.ts | 4 + .../__snapshots__/coverage.test.ts.snap | 10 +- scripts/clean.js | 1 + scripts/e2e.js | 17 +- 11 files changed, 159 insertions(+), 86 deletions(-) diff --git a/.travis.yml b/.travis.yml index 40686a9023..963abf9b47 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,7 @@ language: node_js env: - TS_JEST_E2E_WORKDIR=/tmp/ts-jest-e2e-workdir + - TS_JEST_E2E_OPTIMIZATIONS=1 cache: npm: true diff --git a/appveyor.yml b/appveyor.yml index 7610f4e0f9..0aaae05f7e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -24,6 +24,7 @@ install: # - set CI=true # Our E2E work dir - set TS_JEST_E2E_WORKDIR=%APPDATA%\ts-jest-e2e + - set TS_JEST_E2E_OPTIMIZATIONS=1 - npm ci --ignore-scripts - npm run clean -- --when-ci-commit-message diff --git a/e2e/__helpers__/test-case/__hooks-source__.js.hbs b/e2e/__helpers__/test-case/__hooks-source__.js.hbs index 286ef5fc94..25e8c3af12 100644 --- a/e2e/__helpers__/test-case/__hooks-source__.js.hbs +++ b/e2e/__helpers__/test-case/__hooks-source__.js.hbs @@ -3,13 +3,13 @@ const fs = require('fs') const path = require('path') const root = __dirname -const writeProcessIoTo = {{writeProcessIoTo}} +const writeProcessIoTo = path.resolve(root, {{writeProcessIoTo}}) exports.afterProcess = function (args, result) { // const source = args[0] const filePath = args[1] const relPath = path.relative(root, filePath) - if (writeProcessIoTo && filePath.startsWith(`${root}${path.sep}`)) { + if (filePath.startsWith(`${root}${path.sep}`)) { const dest = `${path.join(writeProcessIoTo, relPath)}.json` const segments = relPath.split(path.sep) segments.pop() diff --git a/e2e/__helpers__/test-case/run-descriptor.ts b/e2e/__helpers__/test-case/run-descriptor.ts index 8c40e86e32..c76e0a7cea 100644 --- a/e2e/__helpers__/test-case/run-descriptor.ts +++ b/e2e/__helpers__/test-case/run-descriptor.ts @@ -26,7 +26,10 @@ export default class RunDescriptor { } get sourcePackageJson() { - return this._sourcePackageJson || (this._sourcePackageJson = require(join(this.sourceDir, 'package.json'))) + try { + return this._sourcePackageJson || (this._sourcePackageJson = require(join(this.sourceDir, 'package.json'))) + } catch (err) {} + return {} } get templateName(): string { @@ -45,8 +48,18 @@ export default class RunDescriptor { if (logUnlessStatus != null && logUnlessStatus !== result.status) { // tslint:disable-next-line:no-console console.log( - `Output of test run in "${this.name}" using template "${this.templateName}" (exit code: ${result.status}):\n\n`, + '='.repeat(70), + '\n', + `Test exited with unexpected status in "${this.name}" using template "${this.templateName}" (exit code: ${ + result.status + }):\n`, + result.context.cmd, + result.context.args.join(' '), + '\n\n', result.output.trim(), + '\n', + '='.repeat(70), + '\n', ) } return result diff --git a/e2e/__helpers__/test-case/run-result.ts b/e2e/__helpers__/test-case/run-result.ts index ca80a640e5..cce67b1da4 100644 --- a/e2e/__helpers__/test-case/run-result.ts +++ b/e2e/__helpers__/test-case/run-result.ts @@ -20,6 +20,7 @@ export default class RunResult { args: string[] env: { [key: string]: string } config: jest.InitialOptions + digest: string }>, ) {} get logFilePath() { @@ -59,11 +60,14 @@ export default class RunResult { return normalizeJestOutput(this.stdout) } get cmdLine() { - return this.normalize( - [this.context.cmd, ...this.context.args] - .filter(a => !['-u', '--updateSnapshot', '--runInBand', '--'].includes(a)) - .join(' '), + const args = [this.context.cmd, ...this.context.args].filter( + a => !['-u', '--updateSnapshot', '--runInBand', '--'].includes(a), ) + const configIndex = args.indexOf('--config') + if (configIndex !== -1) { + args.splice(configIndex, 2) + } + return this.normalize(args.join(' ')) } ioFor(relFilePath: string): ProcessedFileIo { diff --git a/e2e/__helpers__/test-case/runtime.ts b/e2e/__helpers__/test-case/runtime.ts index c4eb2b5684..41a542f037 100644 --- a/e2e/__helpers__/test-case/runtime.ts +++ b/e2e/__helpers__/test-case/runtime.ts @@ -1,4 +1,6 @@ import { sync as spawnSync } from 'cross-spawn' +import { createHash } from 'crypto' +import stringifyJson from 'fast-json-stable-stringify' import { copySync, ensureSymlinkSync, @@ -11,10 +13,12 @@ import { readdirSync, realpathSync, removeSync, + renameSync, statSync, symlinkSync, writeFileSync, } from 'fs-extra' +import { stringify as stringifyJson5 } from 'json5' import merge from 'lodash.merge' import { join, relative, resolve, sep } from 'path' @@ -22,7 +26,7 @@ import * as Paths from '../../../scripts/lib/paths' import RunResult from './run-result' import { PreparedTest, RunTestOptions } from './types' -import { templateNameForPath } from './utils' +import { enableOptimizations, templateNameForPath } from './utils' const TEMPLATE_EXCLUDED_ITEMS = ['node_modules', 'package-lock.json'] @@ -49,25 +53,18 @@ function hooksSourceWith(vars: Record): string { } export function run(name: string, options: RunTestOptions = {}): RunResult { - const { - args = [], - env = {}, - template, - inject, - writeIo, - noCache, - jestConfigPath: configFile = 'jest.config.js', - } = options - const { workdir: dir, sourceDir, hooksFile, ioDir } = prepareTest( + const { env = {}, template, inject, writeIo, noCache, jestConfigPath: configFile = 'jest.config.js' } = options + const { workdir: dir, sourceDir } = prepareTest( name, template || templateNameForPath(join(Paths.e2eSourceDir, name)), options, ) const pkg = readJsonSync(join(dir, 'package.json')) + const jestConfigPath = (path: string = dir) => resolve(path, configFile) + // grab base configuration - const jestConfigPath = resolve(dir, configFile) - let baseConfig: jest.InitialOptions = require(jestConfigPath) + let baseConfig: jest.InitialOptions = require(jestConfigPath()) if (configFile === 'package.json') baseConfig = (baseConfig as any).jest const extraConfig = {} as jest.InitialOptions @@ -81,25 +78,14 @@ export function run(name: string, options: RunTestOptions = {}): RunResult { if (process.argv.find(v => ['--updateSnapshot', '-u'].includes(v))) { cmdArgs.push('-u') } - cmdArgs.push(...args) if (!inject && pkg.scripts && pkg.scripts.test) { - if (cmdArgs.length) { - cmdArgs.unshift('--') - } - cmdArgs = ['npm', '-s', 'run', 'test', ...cmdArgs] + cmdArgs = ['npm', '-s', 'run', 'test', '--', ...cmdArgs] shortCmd = 'npm' } else { - cmdArgs.unshift(join(dir, 'node_modules', '.bin', 'jest')) + cmdArgs.unshift(join('node_modules', '.bin', 'jest')) shortCmd = 'jest' } - // check/merge config - if (cmdArgs.includes('--config')) { - throw new Error(`Extend config using tsJestConfig and jestConfig options, not thru args.`) - } - if (cmdArgs.includes('--no-cache')) { - throw new Error(`Use the noCache option to disable cache, not thru args.`) - } // extends config if (options.jestConfig) { merge(extraConfig, options.jestConfig) @@ -110,21 +96,39 @@ export function run(name: string, options: RunTestOptions = {}): RunResult { merge(tsJestConfig, options.tsJestConfig) } + // cache dir if (noCache || writeIo) { cmdArgs.push('--no-cache') + extraConfig.cacheDirectory = undefined } else if (!(baseConfig.cacheDirectory || extraConfig.cacheDirectory)) { // force the cache directory if not set extraConfig.cacheDirectory = join(Paths.cacheDir, `e2e-${template}`) } - // write final config + // build final config and create dir suffix based on it const finalConfig = merge({}, baseConfig, extraConfig) - if (Object.keys(extraConfig).length !== 0) { + const digest = createHash('sha1') + .update(stringifyJson(finalConfig)) + .digest('hex') + // this must be in the same path hierarchy as dir + const nextDirPrefix = `${dir}-${digest.substr(0, 7)}.` + let index = 1 + while (existsSync(`${nextDirPrefix}${index}`)) index++ + const nextDir = `${nextDirPrefix}${index}` + + // move the directory related to config digest + renameSync(dir, nextDir) + + // write final config + // FIXME: sounds like the json fail to be encoded as an arg + if (false /* enableOptimizations() */) { + cmdArgs.push('--config', JSON.stringify(finalConfig)) + } else if (Object.keys(extraConfig).length !== 0) { if (configFile === 'package.json') { pkg.jest = finalConfig - outputJsonSync(jestConfigPath, pkg) + outputJsonSync(jestConfigPath(nextDir), pkg) } else { - outputFileSync(jestConfigPath, `module.exports = ${JSON.stringify(finalConfig, null, 2)}`, 'utf8') + outputFileSync(jestConfigPath(nextDir), `module.exports = ${JSON.stringify(finalConfig, null, 2)}`, 'utf8') } } @@ -134,48 +138,74 @@ export function run(name: string, options: RunTestOptions = {}): RunResult { } const cmd = cmdArgs.shift() as string + if (cmdArgs[cmdArgs.length - 1] === '--') cmdArgs.pop() - // Add both process.env which is the standard and custom env variables - const mergedEnv: any = { - ...process.env, - ...env, - } + // extend env + const localEnv: any = { ...env } if (inject) { const injected = typeof inject === 'function' ? `(${inject.toString()}).apply(this);` : inject - mergedEnv.__TS_JEST_EVAL = injected + localEnv.__TS_JEST_EVAL = injected } if (writeIo) { - mergedEnv.TS_JEST_HOOKS = hooksFile + localEnv.TS_JEST_HOOKS = defaultHooksFile('.') } - const result = spawnSync(cmd, cmdArgs, { - cwd: dir, - env: mergedEnv, - }) + // arguments to give to spawn + const spawnOptions: { env: Record; cwd: string } = { env: localEnv } as any + + // create started script for debugging + if (!enableOptimizations()) { + outputFileSync( + join(nextDir, '__launch.js'), + ` +const { execFile } = require('child_process') +const cmd = ${stringifyJson5(cmd, null, 2)} +const args = ${stringifyJson5(cmdArgs, null, 2)} +const options = ${stringifyJson5(spawnOptions, null, 2)} +options.env = Object.assign({}, process.env, options.env) +execFile(cmd, args, options) +`, + 'utf8', + ) + } + + // extend env with our env + spawnOptions.env = { ...process.env, ...localEnv } + spawnOptions.cwd = nextDir + + // run jest + const result = spawnSync(cmd, cmdArgs, spawnOptions) // we need to copy each snapshot which does NOT exists in the source dir - readdirSync(dir).forEach(item => { - if (item === 'node_modules' || !statSync(join(dir, item)).isDirectory()) { - return - } - const srcDir = join(sourceDir, item) - const wrkDir = join(dir, item) - copySync(wrkDir, srcDir, { - overwrite: false, - filter: from => { - return relative(sourceDir, from) - .split(sep) - .includes('__snapshots__') - }, + if (!enableOptimizations()) { + readdirSync(nextDir).forEach(item => { + if (item === 'node_modules' || !statSync(join(nextDir, item)).isDirectory()) { + return + } + const srcDir = join(sourceDir, item) + const wrkDir = join(nextDir, item) + + // do not try to copy a linked root snapshots + if (item === '__snapshots__' && existsSync(srcDir)) return + + copySync(wrkDir, srcDir, { + overwrite: false, + filter: from => { + return relative(sourceDir, from) + .split(sep) + .includes('__snapshots__') + }, + }) }) - }) + } - return new RunResult(realpathSync(dir), result, { + return new RunResult(realpathSync(nextDir), result, { cmd: shortCmd, args: cmdArgs, - env: mergedEnv, - ioDir: writeIo ? ioDir : undefined, + env: localEnv, + ioDir: writeIo ? ioDirForPath(nextDir) : undefined, config: finalConfig, + digest, }) } @@ -247,18 +277,19 @@ export function prepareTest(name: string, template: string, options: RunTestOpti // create the special files outputFileSync(join(caseWorkdir, '__eval.ts'), EVAL_SOURCE, 'utf8') let ioDir!: string + // hooks if (options.writeIo) { - ioDir = join(caseWorkdir, '__io__') + ioDir = ioDirForPath(caseWorkdir) mkdirpSync(ioDir) + const hooksFile = defaultHooksFile(caseWorkdir) + outputFileSync( + hooksFile, + hooksSourceWith({ + writeProcessIoTo: ioDirForPath('.') || false, + }), + 'utf8', + ) } - const hooksFile = join(caseWorkdir, '__hooks.js') - outputFileSync( - hooksFile, - hooksSourceWith({ - writeProcessIoTo: ioDir || false, - }), - 'utf8', - ) // create a package.json if it does not exists, and/or enforce the package name const pkgFile = join(caseWorkdir, 'package.json') @@ -268,5 +299,13 @@ export function prepareTest(name: string, template: string, options: RunTestOpti pkg.version = `0.0.0-mock0` outputJsonSync(pkgFile, pkg, { spaces: 2 }) - return { workdir: caseWorkdir, templateDir, sourceDir, hooksFile, ioDir } + return { workdir: caseWorkdir, templateDir, sourceDir } +} + +function ioDirForPath(path: string) { + return join(path, '__io__') +} + +function defaultHooksFile(path: string) { + return join(path, '__hooks.js') } diff --git a/e2e/__helpers__/test-case/types.ts b/e2e/__helpers__/test-case/types.ts index 3df848b707..459ada4fe5 100644 --- a/e2e/__helpers__/test-case/types.ts +++ b/e2e/__helpers__/test-case/types.ts @@ -5,7 +5,6 @@ import RunResult from './run-result' export interface RunTestOptions { template?: string env?: {} - args?: string[] inject?: (() => any) | string writeIo?: boolean jestConfig?: jest.ProjectConfig | any @@ -30,6 +29,4 @@ export interface PreparedTest { workdir: string templateDir: string sourceDir: string - ioDir: string - hooksFile: string } diff --git a/e2e/__helpers__/test-case/utils.ts b/e2e/__helpers__/test-case/utils.ts index 5a6b446edd..2a87c9160b 100644 --- a/e2e/__helpers__/test-case/utils.ts +++ b/e2e/__helpers__/test-case/utils.ts @@ -47,3 +47,7 @@ export function normalizeJestOutput(output: string): string { export function escapeRegex(s: string) { return s.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&') } + +export function enableOptimizations() { + return !!process.env.TS_JEST_E2E_OPTIMIZATIONS +} diff --git a/e2e/__tests__/__snapshots__/coverage.test.ts.snap b/e2e/__tests__/__snapshots__/coverage.test.ts.snap index dbb6679cc7..4dc8a2504f 100644 --- a/e2e/__tests__/__snapshots__/coverage.test.ts.snap +++ b/e2e/__tests__/__snapshots__/coverage.test.ts.snap @@ -1,7 +1,7 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP exports[`using template "default" should report coverages 1`] = ` - √ jest --coverage + √ jest ↳ exit code: 0 ===[ STDOUT ]=================================================================== ----------|----------|----------|----------|----------|-------------------| @@ -24,7 +24,7 @@ exports[`using template "default" should report coverages 1`] = ` `; exports[`using template "with-babel-6" should report coverages 1`] = ` - √ jest --coverage + √ jest ↳ exit code: 0 ===[ STDOUT ]=================================================================== ----------|----------|----------|----------|----------|-------------------| @@ -47,7 +47,7 @@ exports[`using template "with-babel-6" should report coverages 1`] = ` `; exports[`using template "with-babel-7" should report coverages 1`] = ` - √ jest --coverage + √ jest ↳ exit code: 0 ===[ STDOUT ]=================================================================== ----------|----------|----------|----------|----------|-------------------| @@ -70,7 +70,7 @@ exports[`using template "with-babel-7" should report coverages 1`] = ` `; exports[`using template "with-jest-22" should report coverages 1`] = ` - √ jest --coverage + √ jest ↳ exit code: 0 ===[ STDOUT ]=================================================================== ----------|----------|----------|----------|----------|-------------------| @@ -93,7 +93,7 @@ exports[`using template "with-jest-22" should report coverages 1`] = ` `; exports[`using template "with-typescript-2-7" should report coverages 1`] = ` - √ jest --coverage + √ jest ↳ exit code: 0 ===[ STDOUT ]=================================================================== ----------|----------|----------|----------|----------|-------------------| diff --git a/scripts/clean.js b/scripts/clean.js index 152df5a9ce..90b6ee88ab 100755 --- a/scripts/clean.js +++ b/scripts/clean.js @@ -22,5 +22,6 @@ removeSync(join(Paths.testsRootDir, '*', 'debug.txt')) removeSync(join(Paths.testsRootDir, '*', 'node_modules')) removeSync(join(Paths.e2eSourceDir, '*', 'node_modules')) removeSync(join(Paths.e2eTemplatesDir, '*', 'node_modules')) +removeSync(Paths.cacheDir) removeSync(Paths.e2eWorkDir) removeSync(Paths.e2eWotkDirLink) diff --git a/scripts/e2e.js b/scripts/e2e.js index dcc9fcb2ac..3e2ad46765 100755 --- a/scripts/e2e.js +++ b/scripts/e2e.js @@ -23,8 +23,13 @@ if (parentArgs.includes('--coverage')) { if (!parentArgs.includes('--runInBand')) parentArgs.push('--runInBand') const prepareOnly = parentArgs.includes('--prepareOnly') +// eslint-disable-next-line no-unused-vars +function enableOptimizations() { + return !!process.env.TS_JEST_E2E_OPTIMIZATIONS +} + function getDirectories(rootDir) { - return fs.readdirSync(rootDir).filter(function(file) { + return fs.readdirSync(rootDir).filter(function (file) { return fs.statSync(path.join(rootDir, file)).isDirectory() }) } @@ -64,10 +69,18 @@ function setupE2e() { ) } + // cleanup files related to old test run + getDirectories(Paths.e2eWorkDir).forEach(name => { + const dir = path.join(Paths.e2eWorkDir, name) + if (dir === Paths.e2eWorkTemplatesDir) return + log('cleaning old artifacts in', name) + fs.removeSync(dir) + }) + // install with `npm ci` in each template, this is the fastest but needs a package lock file, // that is why we end with the npm install of our bundle getDirectories(Paths.e2eTemplatesDir).forEach(name => { - log('checking template ', name) + log('checking template', name) const sourceDir = path.join(Paths.e2eTemplatesDir, name) const dir = path.join(Paths.e2eWorkTemplatesDir, name) const nodeModulesDir = path.join(dir, 'node_modules')