feat(pnp): support consolidated ESM loader hooks (#3603)

Author: merceyz

.yarn/versions/cf74cfe1.yml

@@ -0,0 +1,28 @@
+releases:
+  "@yarnpkg/cli": minor
+  "@yarnpkg/plugin-pnp": minor
+  "@yarnpkg/pnp": minor
+
+declined:
+  - "@yarnpkg/esbuild-plugin-pnp"
+  - "@yarnpkg/plugin-compat"
+  - "@yarnpkg/plugin-constraints"
+  - "@yarnpkg/plugin-dlx"
+  - "@yarnpkg/plugin-essentials"
+  - "@yarnpkg/plugin-init"
+  - "@yarnpkg/plugin-interactive-tools"
+  - "@yarnpkg/plugin-nm"
+  - "@yarnpkg/plugin-npm-cli"
+  - "@yarnpkg/plugin-pack"
+  - "@yarnpkg/plugin-patch"
+  - "@yarnpkg/plugin-pnpm"
+  - "@yarnpkg/plugin-stage"
+  - "@yarnpkg/plugin-typescript"
+  - "@yarnpkg/plugin-version"
+  - "@yarnpkg/plugin-workspace-tools"
+  - "@yarnpkg/builder"
+  - "@yarnpkg/core"
+  - "@yarnpkg/doctor"
+  - "@yarnpkg/nm"
+  - "@yarnpkg/pnpify"
+  - "@yarnpkg/sdks"

packages/plugin-pnp/sources/PnpLinker.ts

@@ -348,7 +348,7 @@ export class PnpInstaller implements Installer {
     }
 
     if (this.isEsmEnabled()) {
-      this.opts.report.reportWarning(MessageName.UNNAMED, `ESM support for PnP uses the experimental loader API and is therefor experimental`);
+      this.opts.report.reportWarning(MessageName.UNNAMED, `ESM support for PnP uses the experimental loader API and is therefore experimental`);
       await xfs.changeFilePromise(pnpPath.esmLoader, getESMLoaderTemplate(), {
         automaticNewlines: true,
         mode: 0o644,

packages/yarnpkg-pnp/sources/esm-loader/built-loader.js

@@ -0,0 +1,57 @@
+import fs from 'fs';
+
+//#region ESM to CJS support
+/*
+  In order to import CJS files from ESM Node does some translating
+  internally[1]. This translator calls an unpatched `readFileSync`[2]
+  which itself calls an internal `tryStatSync`[3] which calls
+  `binding.fstat`[4]. A PR[5] has been made to use the monkey-patchable
+  `fs.readFileSync` but assuming that wont be merged this region of code
+  patches that final `binding.fstat` call.
+
+  1: https://github.com/nodejs/node/blob/d872aaf1cf20d5b6f56a699e2e3a64300e034269/lib/internal/modules/esm/translators.js#L177-L277
+  2: https://github.com/nodejs/node/blob/d872aaf1cf20d5b6f56a699e2e3a64300e034269/lib/internal/modules/esm/translators.js#L240
+  3: https://github.com/nodejs/node/blob/1317252dfe8824fd9cfee125d2aaa94004db2f3b/lib/fs.js#L452
+  4: https://github.com/nodejs/node/blob/1317252dfe8824fd9cfee125d2aaa94004db2f3b/lib/fs.js#L403
+  5: https://github.com/nodejs/node/pull/39513
+*/
+
+const binding = (process as any).binding(`fs`) as {
+  fstat: (fd: number, useBigint: false, req: any, ctx: object) => Float64Array
+};
+const originalfstat = binding.fstat;
+
+const ZIP_FD = 0x80000000;
+binding.fstat = function(...args) {
+  const [fd, useBigint, req] = args;
+  if ((fd & ZIP_FD) !== 0 && useBigint === false && req === undefined) {
+    try {
+      const stats = fs.fstatSync(fd);
+      // The reverse of this internal util
+      // https://github.com/nodejs/node/blob/8886b63cf66c29d453fdc1ece2e489dace97ae9d/lib/internal/fs/utils.js#L542-L551
+      return new Float64Array([
+        stats.dev,
+        stats.mode,
+        stats.nlink,
+        stats.uid,
+        stats.gid,
+        stats.rdev,
+        stats.blksize,
+        stats.ino,
+        stats.size,
+        stats.blocks,
+        // atime sec
+        // atime ns
+        // mtime sec
+        // mtime ns
+        // ctime sec
+        // ctime ns
+        // birthtime sec
+        // birthtime ns
+      ]);
+    } catch {}
+  }
+
+  return originalfstat.apply(this, args);
+};
+//#endregion

packages/yarnpkg-pnp/sources/esm-loader/fspatch.ts

@@ -0,0 +1,23 @@
+import {fileURLToPath}  from 'url';
+
+import * as loaderUtils from '../loaderUtils';
+
+// The default `getFormat` doesn't support reading from zip files
+export async function getFormat(
+  resolved: string,
+  context: object,
+  defaultGetFormat: typeof getFormat,
+): Promise<{ format: string }> {
+  const url = loaderUtils.tryParseURL(resolved);
+  if (url?.protocol !== `file:`)
+    return defaultGetFormat(resolved, context, defaultGetFormat);
+
+  const format = loaderUtils.getFileFormat(fileURLToPath(url));
+  if (format) {
+    return {
+      format,
+    };
+  }
+
+  return defaultGetFormat(resolved, context, defaultGetFormat);
+}

packages/yarnpkg-pnp/sources/esm-loader/hooks/getFormat.ts

@@ -0,0 +1,19 @@
+import fs               from 'fs';
+import {fileURLToPath}  from 'url';
+
+import * as loaderUtils from '../loaderUtils';
+
+// The default `getSource` doesn't support reading from zip files
+export async function getSource(
+  urlString: string,
+  context: { format: string },
+  defaultGetSource: typeof getSource,
+): Promise<{ source: string }> {
+  const url = loaderUtils.tryParseURL(urlString);
+  if (url?.protocol !== `file:`)
+    return defaultGetSource(urlString, context, defaultGetSource);
+
+  return {
+    source: await fs.promises.readFile(fileURLToPath(url), `utf8`),
+  };
+}

packages/yarnpkg-pnp/sources/esm-loader/hooks/getSource.ts

@@ -0,0 +1,26 @@
+import fs               from 'fs';
+import {fileURLToPath}  from 'url';
+
+import * as loaderUtils from '../loaderUtils';
+
+// The default `load` doesn't support reading from zip files
+export async function load(
+  urlString: string,
+  context: { format: string | null | undefined },
+  defaultLoad: typeof load,
+): Promise<{ format: string; source: string }> {
+  const url = loaderUtils.tryParseURL(urlString);
+  if (url?.protocol !== `file:`)
+    return defaultLoad(urlString, context, defaultLoad);
+
+  const filePath = fileURLToPath(url);
+
+  const format = loaderUtils.getFileFormat(filePath);
+  if (!format)
+    return defaultLoad(urlString, context, defaultLoad);
+
+  return {
+    format,
+    source: await fs.promises.readFile(filePath, `utf8`),
+  };
+}

packages/yarnpkg-pnp/sources/esm-loader/hooks/load.ts

@@ -0,0 +1,74 @@
+import {NativePath, PortablePath}     from '@yarnpkg/fslib';
+import moduleExports                  from 'module';
+import {fileURLToPath, pathToFileURL} from 'url';
+
+import {PnpApi}                       from '../../types';
+import * as loaderUtils               from '../loaderUtils';
+
+const builtins = new Set([...moduleExports.builtinModules]);
+
+const pathRegExp = /^(?![a-zA-Z]:[\\/]|\\\\|\.{0,2}(?:\/|$))((?:node:)?(?:@[^/]+\/)?[^/]+)\/*(.*|)$/;
+
+export async function resolve(
+  originalSpecifier: string,
+  context: { conditions: Array<string>; parentURL: string | undefined },
+  defaultResolver: typeof resolve,
+): Promise<{ url: string }> {
+  const {findPnpApi} = (moduleExports as unknown) as { findPnpApi?: (path: NativePath) => null | PnpApi };
+  if (!findPnpApi || builtins.has(originalSpecifier))
+    return defaultResolver(originalSpecifier, context, defaultResolver);
+
+  let specifier = originalSpecifier;
+  const url = loaderUtils.tryParseURL(specifier);
+  if (url) {
+    if (url.protocol !== `file:`)
+      return defaultResolver(originalSpecifier, context, defaultResolver);
+
+    specifier = fileURLToPath(specifier);
+  }
+
+  const {parentURL, conditions = []} = context;
+
+  const issuer = parentURL ? fileURLToPath(parentURL) : process.cwd();
+
+  // Get the pnpapi of either the issuer or the specifier.
+  // The latter is required when the specifier is an absolute path to a
+  // zip file and the issuer doesn't belong to a pnpapi
+  const pnpapi = findPnpApi(issuer) ?? (url ? findPnpApi(specifier) : null);
+  if (!pnpapi)
+    return defaultResolver(originalSpecifier, context, defaultResolver);
+
+  const dependencyNameMatch = specifier.match(pathRegExp);
+
+  let allowLegacyResolve = false;
+
+  if (dependencyNameMatch) {
+    const [, dependencyName, subPath] = dependencyNameMatch as [unknown, string, PortablePath];
+
+    // If the package.json doesn't list an `exports` field, Node will tolerate omitting the extension
+    // https://github.com/nodejs/node/blob/0996eb71edbd47d9f9ec6153331255993fd6f0d1/lib/internal/modules/esm/resolve.js#L686-L691
+    if (subPath === ``) {
+      const resolved = pnpapi.resolveToUnqualified(`${dependencyName}/package.json`, issuer);
+      if (resolved) {
+        const content = await loaderUtils.tryReadFile(resolved);
+        if (content) {
+          const pkg = JSON.parse(content);
+          allowLegacyResolve = pkg.exports == null;
+        }
+      }
+    }
+  }
+
+  const result = pnpapi.resolveRequest(specifier, issuer, {
+    conditions: new Set(conditions),
+    // TODO: Handle --experimental-specifier-resolution=node
+    extensions: allowLegacyResolve ? undefined : [],
+  });
+
+  if (!result)
+    throw new Error(`Resolving '${specifier}' from '${issuer}' failed`);
+
+  return {
+    url: pathToFileURL(result).href,
+  };
+}