ci(e2e): allows a diff dir for each config

.travis.yml

@@ -2,6 +2,7 @@ language: node_js
 
 env:
   - TS_JEST_E2E_WORKDIR=/tmp/ts-jest-e2e-workdir
+  - TS_JEST_E2E_OPTIMIZATIONS=1
 
 cache:
   npm: true

appveyor.yml

@@ -24,6 +24,7 @@ install:
   # - set CI=true
   # Our E2E work dir
   - set TS_JEST_E2E_WORKDIR=%APPDATA%\ts-jest-e2e
+  - set TS_JEST_E2E_OPTIMIZATIONS=1
   - npm ci --ignore-scripts
   - npm run clean -- --when-ci-commit-message
 

e2e/__helpers__/test-case/__hooks-source__.js.hbs

@@ -3,13 +3,13 @@
 const fs = require('fs')
 const path = require('path')
 const root = __dirname
-const writeProcessIoTo = {{writeProcessIoTo}}
+const writeProcessIoTo = path.resolve(root, {{writeProcessIoTo}})
 
 exports.afterProcess = function (args, result) {
   // const source = args[0]
   const filePath = args[1]
   const relPath = path.relative(root, filePath)
-  if (writeProcessIoTo && filePath.startsWith(`${root}${path.sep}`)) {
+  if (filePath.startsWith(`${root}${path.sep}`)) {
     const dest = `${path.join(writeProcessIoTo, relPath)}.json`
     const segments = relPath.split(path.sep)
     segments.pop()

e2e/__helpers__/test-case/run-descriptor.ts

@@ -26,7 +26,10 @@ export default class RunDescriptor {
   }
 
   get sourcePackageJson() {
-    return this._sourcePackageJson || (this._sourcePackageJson = require(join(this.sourceDir, 'package.json')))
+    try {
+      return this._sourcePackageJson || (this._sourcePackageJson = require(join(this.sourceDir, 'package.json')))
+    } catch (err) {}
+    return {}
   }
 
   get templateName(): string {
@@ -45,8 +48,18 @@ export default class RunDescriptor {
     if (logUnlessStatus != null && logUnlessStatus !== result.status) {
       // tslint:disable-next-line:no-console
       console.log(
-        `Output of test run in "${this.name}" using template "${this.templateName}" (exit code: ${result.status}):\n\n`,
+        '='.repeat(70),
+        '\n',
+        `Test exited with unexpected status in "${this.name}" using template "${this.templateName}" (exit code: ${
+          result.status
+        }):\n`,
+        result.context.cmd,
+        result.context.args.join(' '),
+        '\n\n',
         result.output.trim(),
+        '\n',
+        '='.repeat(70),
+        '\n',
       )
     }
     return result

e2e/__helpers__/test-case/run-result.ts

@@ -20,6 +20,7 @@ export default class RunResult {
       args: string[]
       env: { [key: string]: string }
       config: jest.InitialOptions
+      digest: string
     }>,
   ) {}
   get logFilePath() {
@@ -59,11 +60,14 @@ export default class RunResult {
     return normalizeJestOutput(this.stdout)
   }
   get cmdLine() {
-    return this.normalize(
-      [this.context.cmd, ...this.context.args]
-        .filter(a => !['-u', '--updateSnapshot', '--runInBand', '--'].includes(a))
-        .join(' '),
+    const args = [this.context.cmd, ...this.context.args].filter(
+      a => !['-u', '--updateSnapshot', '--runInBand', '--'].includes(a),
     )
+    const configIndex = args.indexOf('--config')
+    if (configIndex !== -1) {
+      args.splice(configIndex, 2)
+    }
+    return this.normalize(args.join(' '))
   }
 
   ioFor(relFilePath: string): ProcessedFileIo {

e2e/__helpers__/test-case/runtime.ts

@@ -1,4 +1,6 @@
 import { sync as spawnSync } from 'cross-spawn'
+import { createHash } from 'crypto'
+import stringifyJson from 'fast-json-stable-stringify'
 import {
   copySync,
   ensureSymlinkSync,
@@ -11,18 +13,20 @@ import {
   readdirSync,
   realpathSync,
   removeSync,
+  renameSync,
   statSync,
   symlinkSync,
   writeFileSync,
 } from 'fs-extra'
+import { stringify as stringifyJson5 } from 'json5'
 import merge from 'lodash.merge'
 import { join, relative, resolve, sep } from 'path'
 
 import * as Paths from '../../../scripts/lib/paths'
 
 import RunResult from './run-result'
 import { PreparedTest, RunTestOptions } from './types'
-import { templateNameForPath } from './utils'
+import { enableOptimizations, templateNameForPath } from './utils'
 
 const TEMPLATE_EXCLUDED_ITEMS = ['node_modules', 'package-lock.json']
 
@@ -49,25 +53,18 @@ function hooksSourceWith(vars: Record<string, any>): string {
 }
 
 export function run(name: string, options: RunTestOptions = {}): RunResult {
-  const {
-    args = [],
-    env = {},
-    template,
-    inject,
-    writeIo,
-    noCache,
-    jestConfigPath: configFile = 'jest.config.js',
-  } = options
-  const { workdir: dir, sourceDir, hooksFile, ioDir } = prepareTest(
+  const { env = {}, template, inject, writeIo, noCache, jestConfigPath: configFile = 'jest.config.js' } = options
+  const { workdir: dir, sourceDir } = prepareTest(
     name,
     template || templateNameForPath(join(Paths.e2eSourceDir, name)),
     options,
   )
   const pkg = readJsonSync(join(dir, 'package.json'))
 
+  const jestConfigPath = (path: string = dir) => resolve(path, configFile)
+
   // grab base configuration
-  const jestConfigPath = resolve(dir, configFile)
-  let baseConfig: jest.InitialOptions = require(jestConfigPath)
+  let baseConfig: jest.InitialOptions = require(jestConfigPath())
   if (configFile === 'package.json') baseConfig = (baseConfig as any).jest
 
   const extraConfig = {} as jest.InitialOptions
@@ -81,25 +78,14 @@ export function run(name: string, options: RunTestOptions = {}): RunResult {
   if (process.argv.find(v => ['--updateSnapshot', '-u'].includes(v))) {
     cmdArgs.push('-u')
   }
-  cmdArgs.push(...args)
   if (!inject && pkg.scripts && pkg.scripts.test) {
-    if (cmdArgs.length) {
-      cmdArgs.unshift('--')
-    }
-    cmdArgs = ['npm', '-s', 'run', 'test', ...cmdArgs]
+    cmdArgs = ['npm', '-s', 'run', 'test', '--', ...cmdArgs]
     shortCmd = 'npm'
   } else {
-    cmdArgs.unshift(join(dir, 'node_modules', '.bin', 'jest'))
+    cmdArgs.unshift(join('node_modules', '.bin', 'jest'))
     shortCmd = 'jest'
   }
 
-  // check/merge config
-  if (cmdArgs.includes('--config')) {
-    throw new Error(`Extend config using tsJestConfig and jestConfig options, not thru args.`)
-  }
-  if (cmdArgs.includes('--no-cache')) {
-    throw new Error(`Use the noCache option to disable cache, not thru args.`)
-  }
   // extends config
   if (options.jestConfig) {
     merge(extraConfig, options.jestConfig)
@@ -110,21 +96,39 @@ export function run(name: string, options: RunTestOptions = {}): RunResult {
     merge(tsJestConfig, options.tsJestConfig)
   }
 
+  // cache dir
   if (noCache || writeIo) {
     cmdArgs.push('--no-cache')
+    extraConfig.cacheDirectory = undefined
   } else if (!(baseConfig.cacheDirectory || extraConfig.cacheDirectory)) {
     // force the cache directory if not set
     extraConfig.cacheDirectory = join(Paths.cacheDir, `e2e-${template}`)
   }
 
-  // write final config
+  // build final config and create dir suffix based on it
   const finalConfig = merge({}, baseConfig, extraConfig)
-  if (Object.keys(extraConfig).length !== 0) {
+  const digest = createHash('sha1')
+    .update(stringifyJson(finalConfig))
+    .digest('hex')
+  // this must be in the same path hierarchy as dir
+  const nextDirPrefix = `${dir}-${digest.substr(0, 7)}.`
+  let index = 1
+  while (existsSync(`${nextDirPrefix}${index}`)) index++
+  const nextDir = `${nextDirPrefix}${index}`
+
+  // move the directory related to config digest
+  renameSync(dir, nextDir)
+
+  // write final config
+  // FIXME: sounds like the json fail to be encoded as an arg
+  if (false /* enableOptimizations() */) {
+    cmdArgs.push('--config', JSON.stringify(finalConfig))
+  } else if (Object.keys(extraConfig).length !== 0) {
     if (configFile === 'package.json') {
       pkg.jest = finalConfig
-      outputJsonSync(jestConfigPath, pkg)
+      outputJsonSync(jestConfigPath(nextDir), pkg)
     } else {
-      outputFileSync(jestConfigPath, `module.exports = ${JSON.stringify(finalConfig, null, 2)}`, 'utf8')
+      outputFileSync(jestConfigPath(nextDir), `module.exports = ${JSON.stringify(finalConfig, null, 2)}`, 'utf8')
     }
   }
 
@@ -134,48 +138,74 @@ export function run(name: string, options: RunTestOptions = {}): RunResult {
   }
 
   const cmd = cmdArgs.shift() as string
+  if (cmdArgs[cmdArgs.length - 1] === '--') cmdArgs.pop()
 
-  // Add both process.env which is the standard and custom env variables
-  const mergedEnv: any = {
-    ...process.env,
-    ...env,
-  }
+  // extend env
+  const localEnv: any = { ...env }
   if (inject) {
     const injected = typeof inject === 'function' ? `(${inject.toString()}).apply(this);` : inject
-    mergedEnv.__TS_JEST_EVAL = injected
+    localEnv.__TS_JEST_EVAL = injected
   }
   if (writeIo) {
-    mergedEnv.TS_JEST_HOOKS = hooksFile
+    localEnv.TS_JEST_HOOKS = defaultHooksFile('.')
   }
 
-  const result = spawnSync(cmd, cmdArgs, {
-    cwd: dir,
-    env: mergedEnv,
-  })
+  // arguments to give to spawn
+  const spawnOptions: { env: Record<string, string>; cwd: string } = { env: localEnv } as any
+
+  // create started script for debugging
+  if (!enableOptimizations()) {
+    outputFileSync(
+      join(nextDir, '__launch.js'),
+      `
+const { execFile } = require('child_process')
+const cmd = ${stringifyJson5(cmd, null, 2)}
+const args = ${stringifyJson5(cmdArgs, null, 2)}
+const options = ${stringifyJson5(spawnOptions, null, 2)}
+options.env = Object.assign({}, process.env, options.env)
+execFile(cmd, args, options)
+`,
+      'utf8',
+    )
+  }
+
+  // extend env with our env
+  spawnOptions.env = { ...process.env, ...localEnv }
+  spawnOptions.cwd = nextDir
+
+  // run jest
+  const result = spawnSync(cmd, cmdArgs, spawnOptions)
 
   // we need to copy each snapshot which does NOT exists in the source dir
-  readdirSync(dir).forEach(item => {
-    if (item === 'node_modules' || !statSync(join(dir, item)).isDirectory()) {
-      return
-    }
-    const srcDir = join(sourceDir, item)
-    const wrkDir = join(dir, item)
-    copySync(wrkDir, srcDir, {
-      overwrite: false,
-      filter: from => {
-        return relative(sourceDir, from)
-          .split(sep)
-          .includes('__snapshots__')
-      },
+  if (!enableOptimizations()) {
+    readdirSync(nextDir).forEach(item => {
+      if (item === 'node_modules' || !statSync(join(nextDir, item)).isDirectory()) {
+        return
+      }
+      const srcDir = join(sourceDir, item)
+      const wrkDir = join(nextDir, item)
+
+      // do not try to copy a linked root snapshots
+      if (item === '__snapshots__' && existsSync(srcDir)) return
+
+      copySync(wrkDir, srcDir, {
+        overwrite: false,
+        filter: from => {
+          return relative(sourceDir, from)
+            .split(sep)
+            .includes('__snapshots__')
+        },
+      })
     })
-  })
+  }
 
-  return new RunResult(realpathSync(dir), result, {
+  return new RunResult(realpathSync(nextDir), result, {
     cmd: shortCmd,
     args: cmdArgs,
-    env: mergedEnv,
-    ioDir: writeIo ? ioDir : undefined,
+    env: localEnv,
+    ioDir: writeIo ? ioDirForPath(nextDir) : undefined,
     config: finalConfig,
+    digest,
   })
 }
 
@@ -247,18 +277,19 @@ export function prepareTest(name: string, template: string, options: RunTestOpti
   // create the special files
   outputFileSync(join(caseWorkdir, '__eval.ts'), EVAL_SOURCE, 'utf8')
   let ioDir!: string
+  // hooks
   if (options.writeIo) {
-    ioDir = join(caseWorkdir, '__io__')
+    ioDir = ioDirForPath(caseWorkdir)
     mkdirpSync(ioDir)
+    const hooksFile = defaultHooksFile(caseWorkdir)
+    outputFileSync(
+      hooksFile,
+      hooksSourceWith({
+        writeProcessIoTo: ioDirForPath('.') || false,
+      }),
+      'utf8',
+    )
   }
-  const hooksFile = join(caseWorkdir, '__hooks.js')
-  outputFileSync(
-    hooksFile,
-    hooksSourceWith({
-      writeProcessIoTo: ioDir || false,
-    }),
-    'utf8',
-  )
 
   // create a package.json if it does not exists, and/or enforce the package name
   const pkgFile = join(caseWorkdir, 'package.json')
@@ -268,5 +299,13 @@ export function prepareTest(name: string, template: string, options: RunTestOpti
   pkg.version = `0.0.0-mock0`
   outputJsonSync(pkgFile, pkg, { spaces: 2 })
 
-  return { workdir: caseWorkdir, templateDir, sourceDir, hooksFile, ioDir }
+  return { workdir: caseWorkdir, templateDir, sourceDir }
+}
+
+function ioDirForPath(path: string) {
+  return join(path, '__io__')
+}
+
+function defaultHooksFile(path: string) {
+  return join(path, '__hooks.js')
 }

e2e/__helpers__/test-case/types.ts

@@ -5,7 +5,6 @@ import RunResult from './run-result'
 export interface RunTestOptions {
   template?: string
   env?: {}
-  args?: string[]
   inject?: (() => any) | string
   writeIo?: boolean
   jestConfig?: jest.ProjectConfig | any
@@ -30,6 +29,4 @@ export interface PreparedTest {
   workdir: string
   templateDir: string
   sourceDir: string
-  ioDir: string
-  hooksFile: string
 }

e2e/__helpers__/test-case/utils.ts

@@ -47,3 +47,7 @@ export function normalizeJestOutput(output: string): string {
 export function escapeRegex(s: string) {
   return s.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&')
 }
+
+export function enableOptimizations() {
+  return !!process.env.TS_JEST_E2E_OPTIMIZATIONS
+}