test: transpile using esbuild (#5581)

**What's the problem this PR addresses?**

We use esbuild to build and run Yarn from sources but not for our test
files.

Follow-up to #5180

**How did you fix it?**

Use esbuild to transpile files in our testing setup.

**Checklist**
- [x] I have read the [Contributing
Guide](https://yarnpkg.com/advanced/contributing).
- [x] I have set the packages that need to be released for my changes to
be effective.
- [x] I will check that all automated PR checks pass before the PR gets
reviewed.

jest.config.js

@@ -8,4 +8,7 @@ module.exports = {
   ],
   setupFiles: [require.resolve(`@yarnpkg/cli/polyfills`)],
   testTimeout: 50000,
+  transform: {
+    "\\.[jt]sx?$": require.resolve(`./scripts/setup-ts-jest.js`),
+  },
 };

package.json

@@ -7,10 +7,6 @@
   ],
   "devDependencies": {
     "@arcanis/sherlock": "^2.0.3",
-    "@babel/core": "^7.18.10",
-    "@babel/preset-env": "^7.18.10",
-    "@babel/preset-react": "^7.18.6",
-    "@babel/preset-typescript": "^7.18.6",
     "@types/jest": "^28.1.6",
     "@types/micromatch": "^4.0.1",
     "@types/node": "^18.17.15",

scripts/extract-hooks.ts

@@ -75,7 +75,7 @@ async function processFile(file: ts.SourceFile) {
   return hooks;
 }
 
-async function execute(files: Array<string>) {
+export async function execute(files: Array<string>) {
   const allHooks = new Map<string, HookDefinition>();
 
   for (const relativePath of files) {
@@ -115,8 +115,6 @@ async function execute(files: Array<string>) {
   return allHooksArray;
 }
 
-exports.execute = execute;
-
 if (require.main === module) {
   Cli.from([
     class extends Command {

scripts/setup-ts-cache.js

@@ -0,0 +1,100 @@
+const crypto = require(`crypto`);
+const esbuild = require(`esbuild-wasm`);
+const fs = require(`fs`);
+const path = require(`path`);
+const v8 = require(`v8`);
+const zlib = require(`zlib`);
+
+// Needed by the worker spawned by esbuild
+if (process.versions.pnp)
+  // Unquoted because Yarn doesn't support it quoted yet
+  // TODO: make Yarn support quoted PnP requires in NODE_OPTIONS
+  process.env.NODE_OPTIONS = `${process.env.NODE_OPTIONS || ``} --require ${require.resolve(`pnpapi`)}`;
+
+const resolveVirtual = process.versions.pnp
+  ? require(`pnpapi`).resolveVirtual
+  : undefined;
+
+// esbuild only supports major.minor.patch, no pre-release (nightly) specifier is allowed
+// so we reduce the version down to major.minor
+const NODE_VERSION = process.versions.node.split(`.`, 2).join(`.`);
+
+const cache = {
+  version: `1\0${esbuild.version}\0${NODE_VERSION}`,
+  files: new Map(),
+  isDirty: false,
+};
+
+const cachePath = path.join(__dirname, `../node_modules/.cache/yarn/esbuild-transpile-cache.bin`);
+try {
+  const cacheData = v8.deserialize(zlib.brotliDecompressSync(fs.readFileSync(cachePath)));
+  if (cacheData.version === cache.version) {
+    cache.files = cacheData.files;
+  }
+} catch {}
+
+function persistCache() {
+  if (!cache.isDirty)
+    return;
+
+  cache.isDirty = false;
+
+  const data = v8.serialize({
+    version: cache.version,
+    files: cache.files,
+  });
+
+  fs.mkdirSync(path.dirname(cachePath), {recursive: true});
+
+  const tmpPath = cachePath + crypto.randomBytes(8).toString(`hex`);
+  fs.writeFileSync(tmpPath, zlib.brotliCompressSync(data, {
+    params: {
+      [zlib.constants.BROTLI_PARAM_QUALITY]: 4,
+    },
+  }));
+
+  try {
+    fs.renameSync(tmpPath, cachePath);
+  } catch {
+    fs.unlinkSync(tmpPath);
+  }
+}
+
+process.once(`exit`, persistCache);
+process.nextTick(persistCache);
+
+process.setSourceMapsEnabled?.(true);
+
+function compileFile(sourceCode, filename) {
+  filename = resolveVirtual?.(filename) ?? filename;
+
+  const cacheEntry = cache.files.get(filename);
+  if (cacheEntry?.source === sourceCode)
+    return {code: cacheEntry.code, map: cacheEntry.map};
+
+  const res = esbuild.transformSync(sourceCode, {
+    target: `node${NODE_VERSION}`,
+    loader: path.extname(filename).slice(1),
+    sourcefile: filename,
+    sourcemap: `both`,
+    platform: `node`,
+    format: `cjs`,
+    supported: {
+      'dynamic-import': false,
+    },
+  });
+
+  cache.isDirty = true;
+  cache.files.set(filename, {
+    source: sourceCode,
+    code: res.code,
+    map: res.map,
+  });
+
+  return {code: res.code, map: res.map};
+}
+
+module.exports = {
+  compileFile,
+  version: cache.version,
+};

scripts/setup-ts-execution.js

@@ -1,103 +1,11 @@
-const crypto = require(`crypto`);
-const esbuild = require(`esbuild-wasm`);
-const fs = require(`fs`);
-const path = require(`path`);
 const pirates = require(`pirates`);
-const v8 = require(`v8`);
-const zlib = require(`zlib`);
+const {compileFile} = require(`./setup-ts-cache.js`);
 
-// Needed by the worker spawned by esbuild
-if (process.versions.pnp)
-  // Unquoted because Yarn doesn't support it quoted yet
-  // TODO: make Yarn support quoted PnP requires in NODE_OPTIONS
-  process.env.NODE_OPTIONS = `${process.env.NODE_OPTIONS || ``} --require ${require.resolve(`pnpapi`)}`;
-
-const resolveVirtual = process.versions.pnp
-  ? require(`pnpapi`).resolveVirtual
-  : undefined;
-
-// esbuild only supports major.minor.patch, no pre-release (nightly) specifier is allowed
-// so we reduce the version down to major.minor
-const NODE_VERSION = process.versions.node.split(`.`, 2).join(`.`);
-
-const cache = {
-  version: `${esbuild.version}\0${NODE_VERSION}`,
-  files: new Map(),
-  isDirty: false,
-};
-
-const cachePath = path.join(__dirname, `../node_modules/.cache/yarn/esbuild-transpile-cache.bin`);
-try {
-  const cacheData = v8.deserialize(zlib.brotliDecompressSync(fs.readFileSync(cachePath)));
-  if (cacheData.version === cache.version) {
-    cache.files = cacheData.files;
-  }
-} catch {}
-
-function persistCache() {
-  if (!cache.isDirty)
-    return;
-
-  cache.isDirty = false;
-
-  const data = v8.serialize({
-    version: cache.version,
-    files: cache.files,
-  });
-
-  fs.mkdirSync(path.dirname(cachePath), {recursive: true});
-
-  const tmpPath = cachePath + crypto.randomBytes(8).toString(`hex`);
-  fs.writeFileSync(tmpPath, zlib.brotliCompressSync(data, {
-    params: {
-      [zlib.constants.BROTLI_PARAM_QUALITY]: 4,
-    },
-  }));
-
-  fs.renameSync(tmpPath, cachePath);
-}
-
-process.once(`exit`, persistCache);
-process.nextTick(persistCache);
-
-process.setSourceMapsEnabled?.(true);
-
-function compileFile(sourceCode, filename) {
-  filename = resolveVirtual?.(filename) ?? filename;
-
-  const cacheEntry = cache.files.get(filename);
-  if (cacheEntry?.source === sourceCode)
-    return cacheEntry.code;
-
-  const res = esbuild.transformSync(sourceCode, {
-    target: `node${NODE_VERSION}`,
-    loader: path.extname(filename).slice(1),
-    sourcefile: filename,
-    sourcemap: `inline`,
-    platform: `node`,
-    format: `cjs`,
-    supported: {
-      'dynamic-import': false,
-    },
-  });
-
-  cache.isDirty = true;
-  cache.files.set(filename, {
-    source: sourceCode,
-    code: res.code,
-  });
-
-  return res.code;
-}
-
-pirates.addHook(compileFile, {
-  extensions: [`.tsx`, `.ts`, `.js`],
-  matcher(p) {
-    if (p?.endsWith(`.js`)) {
-      const normalizedP = p.replace(/\\/g, `/`);
-      return normalizedP.includes(`packages/yarnpkg-pnp/sources/node`) || normalizedP.endsWith(`packages/yarnpkg-pnp/sources/loader/node-options.js`);
-    }
-
-    return true;
+pirates.addHook(
+  (code, filename) => {
+    return compileFile(code, filename).code;
+  },
+  {
+    extensions: [`.tsx`, `.ts`, `.js`],
   },
-});
+);

scripts/setup-ts-jest.js

@@ -0,0 +1,22 @@
+const {createHash} = require(`node:crypto`);
+const {compileFile, version} = require(`./setup-ts-cache.js`);
+
+module.exports = {
+  getCacheKey(sourceText, sourcePath, transformOptions) {
+    return createHash(`sha1`)
+      .update(sourceText)
+      .update(`\0`)
+      .update(version)
+      .digest(`hex`)
+      .substring(0, 32);
+  },
+  process(sourceText, sourcePath, options) {
+    if (/[\\/]node_modules[\\/]/.test(sourcePath)) {
+      return {
+        code: sourceText,
+      };
+    }
+
+    return compileFile(sourceText, sourcePath);
+  },
+};

yarn.lock

@@ -285,7 +285,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@babel/core@npm:^7.11.6, @babel/core@npm:^7.12.3, @babel/core@npm:^7.18.10, @babel/core@npm:^7.19.6, @babel/core@npm:^7.23.3":
+"@babel/core@npm:^7.11.6, @babel/core@npm:^7.12.3, @babel/core@npm:^7.19.6, @babel/core@npm:^7.23.3":
   version: 7.23.9
   resolution: "@babel/core@npm:7.23.9"
   dependencies:
@@ -1536,7 +1536,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@babel/preset-env@npm:^7.18.10, @babel/preset-env@npm:^7.19.4, @babel/preset-env@npm:^7.22.9":
+"@babel/preset-env@npm:^7.19.4, @babel/preset-env@npm:^7.22.9":
   version: 7.23.9
   resolution: "@babel/preset-env@npm:7.23.9"
   dependencies:
@@ -5449,10 +5449,6 @@ __metadata:
   resolution: "@yarnpkg/monorepo@workspace:."
   dependencies:
     "@arcanis/sherlock": "npm:^2.0.3"
-    "@babel/core": "npm:^7.18.10"
-    "@babel/preset-env": "npm:^7.18.10"
-    "@babel/preset-react": "npm:^7.18.6"
-    "@babel/preset-typescript": "npm:^7.18.6"
     "@iarna/toml": "npm:^2.2.5"
     "@types/jest": "npm:^28.1.6"
     "@types/micromatch": "npm:^4.0.1"