Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{
"changes": [
{
"packageName": "@microsoft/rush",
"comment": "rush-resolver-cache-plugin: add pnpm 10 / lockfile v9 compatibility",
"type": "none"
}
],
"packageName": "@microsoft/rush"
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,11 @@ import {
computeResolverCacheFromLockfileAsync,
type IPlatformInfo
} from './computeResolverCacheFromLockfileAsync';
import {
type PnpmMajorVersion,
type IPnpmVersionHelpers,
getPnpmVersionHelpersAsync
} from './pnpm/pnpmVersionHelpers';
import type { IResolverContext } from './types';

/**
Expand Down Expand Up @@ -79,10 +84,26 @@ export async function afterInstallAsync(

const lockFilePath: string = subspace.getCommittedShrinkwrapFilePath(variant);

const pnpmStoreDir: string = `${rushConfiguration.pnpmOptions.pnpmStorePath}/v3/files/`;
const pnpmStorePath: string = rushConfiguration.pnpmOptions.pnpmStorePath;

const pnpmMajorVersion: PnpmMajorVersion = (() => {
const major: number = parseInt(rushConfiguration.packageManagerToolVersion, 10);
switch (major) {
case 10:
return 10;
case 9:
return 9;
case 8:
return 8;
default:
throw new Error(`Unsupported pnpm major version: ${major}`);
}
})();

const pnpmHelpers: IPnpmVersionHelpers = await getPnpmVersionHelpersAsync(pnpmMajorVersion);

terminal.writeLine(`Using pnpm-lock from: ${lockFilePath}`);
terminal.writeLine(`Using pnpm store folder: ${pnpmStoreDir}`);
terminal.writeLine(`Using pnpm ${pnpmMajorVersion} store at: ${pnpmStorePath}`);

const workspaceRoot: string = subspace.getSubspaceTempFolderPath();
const cacheFilePath: string = `${workspaceRoot}/resolver-cache.json`;
Expand Down Expand Up @@ -166,10 +187,7 @@ export async function afterInstallAsync(
const prefixIndex: number = descriptionFileHash.indexOf('-');
const hash: string = Buffer.from(descriptionFileHash.slice(prefixIndex + 1), 'base64').toString('hex');

// The pnpm store directory has index files of package contents at paths:
// <store>/v3/files/<hash (0-2)>/<hash (2-)>-index.json
// See https://github.com/pnpm/pnpm/blob/f394cfccda7bc519ceee8c33fc9b68a0f4235532/store/cafs/src/getFilePathInCafs.ts#L33
const indexPath: string = `${pnpmStoreDir}${hash.slice(0, 2)}/${hash.slice(2)}-index.json`;
const indexPath: string = pnpmHelpers.getStoreIndexPath(pnpmStorePath, context, hash);

try {
const indexContent: string = await FileSystem.readFileAsync(indexPath);
Expand Down Expand Up @@ -254,6 +272,7 @@ export async function afterInstallAsync(
platformInfo: getPlatformInfo(),
projectByImporterPath,
lockfile: lockFile,
pnpmVersion: pnpmMajorVersion,
afterExternalPackagesAsync
});

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,17 @@ import type {
} from '@rushstack/webpack-workspace-resolve-plugin';

import type { PnpmShrinkwrapFile } from './externals';
import { getDescriptionFileRootFromKey, resolveDependencies, createContextSerializer } from './helpers';
import {
getDescriptionFileRootFromKey,
resolveDependencies,
createContextSerializer,
extractNameAndVersionFromKey
} from './helpers';
import {
type PnpmMajorVersion,
type IPnpmVersionHelpers,
getPnpmVersionHelpersAsync
} from './pnpm/pnpmVersionHelpers';
import type { IResolverContext } from './types';

/**
Expand Down Expand Up @@ -105,6 +115,9 @@ function extractBundledDependencies(
}
}

// Re-export for downstream consumers
export type { PnpmMajorVersion, IPnpmVersionHelpers } from './pnpm/pnpmVersionHelpers';

/**
* Options for computing the resolver cache from a lockfile.
*/
Expand All @@ -129,6 +142,11 @@ export interface IComputeResolverCacheFromLockfileOptions {
* The lockfile to compute the cache from
*/
lockfile: PnpmShrinkwrapFile;
/**
* The major version of pnpm configured in rush.json (e.g. `"10.27.0"` → 10).
* Used to select the correct dep-path hashing algorithm and store layout.
*/
pnpmVersion: PnpmMajorVersion;
/**
* A callback to process external packages after they have been enumerated.
* Broken out as a separate function to facilitate testing without hitting the disk.
Expand All @@ -143,13 +161,15 @@ export interface IComputeResolverCacheFromLockfileOptions {
) => Promise<void>;
}

const BACKSLASH_REGEX: RegExp = /\\/g;

/**
* Copied from `@rushstack/node-core-library/src/Path.ts` to avoid expensive dependency
* @param path - Path using backslashes as path separators
* @returns The same string using forward slashes as path separators
*/
function convertToSlashes(path: string): string {
return path.replace(/\\/g, '/');
return path.replace(BACKSLASH_REGEX, '/');
}

/**
Expand All @@ -169,10 +189,19 @@ export async function computeResolverCacheFromLockfileAsync(
const contexts: Map<string, IResolverContext> = new Map();
const missingOptionalDependencies: Set<string> = new Set();

const helpers: IPnpmVersionHelpers = await getPnpmVersionHelpersAsync(params.pnpmVersion);

const { packages } = lockfile;

// Enumerate external dependencies first, to simplify looping over them for store data
for (const [key, pack] of lockfile.packages) {
for (const [key, pack] of packages) {
let name: string | undefined = pack.name;
const descriptionFileRoot: string = getDescriptionFileRootFromKey(workspaceRoot, key, name);
const descriptionFileRoot: string = getDescriptionFileRootFromKey(
workspaceRoot,
key,
helpers.depPathToFilename,
name
);

// Skip optional dependencies that are incompatible with the current environment
if (pack.optional && !isPackageCompatible(pack, platformInfo)) {
Expand All @@ -182,9 +211,10 @@ export async function computeResolverCacheFromLockfileAsync(

const integrity: string | undefined = pack.resolution?.integrity;

if (!name && key.startsWith('/')) {
const versionIndex: number = key.indexOf('@', 2);
name = key.slice(1, versionIndex);
// Extract name and version from the key if not already provided
const parsed: { name: string; version: string } | undefined = extractNameAndVersionFromKey(key);
if (parsed) {
name ||= parsed.name;
}

if (!name) {
Expand All @@ -196,6 +226,7 @@ export async function computeResolverCacheFromLockfileAsync(
descriptionFileHash: integrity,
isProject: false,
name,
version: parsed?.version,
deps: new Map(),
ordinal: -1,
optional: pack.optional
Expand All @@ -204,10 +235,10 @@ export async function computeResolverCacheFromLockfileAsync(
contexts.set(descriptionFileRoot, context);

if (pack.dependencies) {
resolveDependencies(workspaceRoot, pack.dependencies, context);
resolveDependencies(workspaceRoot, pack.dependencies, context, helpers, packages);
}
if (pack.optionalDependencies) {
resolveDependencies(workspaceRoot, pack.optionalDependencies, context);
resolveDependencies(workspaceRoot, pack.optionalDependencies, context, helpers, packages);
}
}

Expand Down Expand Up @@ -248,13 +279,13 @@ export async function computeResolverCacheFromLockfileAsync(
contexts.set(descriptionFileRoot, context);

if (importer.dependencies) {
resolveDependencies(workspaceRoot, importer.dependencies, context);
resolveDependencies(workspaceRoot, importer.dependencies, context, helpers, packages);
}
if (importer.devDependencies) {
resolveDependencies(workspaceRoot, importer.devDependencies, context);
resolveDependencies(workspaceRoot, importer.devDependencies, context, helpers, packages);
}
if (importer.optionalDependencies) {
resolveDependencies(workspaceRoot, importer.optionalDependencies, context);
resolveDependencies(workspaceRoot, importer.optionalDependencies, context, helpers, packages);
}
}

Expand Down
125 changes: 52 additions & 73 deletions rush-plugins/rush-resolver-cache-plugin/src/helpers.ts
Original file line number Diff line number Diff line change
@@ -1,98 +1,62 @@
// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
// See LICENSE in the project root for license information.

import { createHash } from 'node:crypto';
import * as path from 'node:path';

import type { ISerializedResolveContext } from '@rushstack/webpack-workspace-resolve-plugin';

import type { IDependencyEntry, IResolverContext } from './types';

const MAX_LENGTH_WITHOUT_HASH: number = 120 - 26 - 1;
const BASE32: string[] = 'abcdefghijklmnopqrstuvwxyz234567'.split('');

// https://github.com/swansontec/rfc4648.js/blob/ead9c9b4b68e5d4a529f32925da02c02984e772c/src/codec.ts#L82-L118
export function createBase32Hash(input: string): string {
const data: Buffer = createHash('md5').update(input).digest();

const mask: 0x1f = 0x1f;
let out: string = '';

let bits: number = 0; // Number of bits currently in the buffer
let buffer: number = 0; // Bits waiting to be written out, MSB first
for (let i: number = 0; i < data.length; ++i) {
// eslint-disable-next-line no-bitwise
buffer = (buffer << 8) | (0xff & data[i]);
bits += 8;

// Write out as much as we can:
while (bits > 5) {
bits -= 5;
// eslint-disable-next-line no-bitwise
out += BASE32[mask & (buffer >> bits)];
}
}

// Partial character:
if (bits) {
// eslint-disable-next-line no-bitwise
out += BASE32[mask & (buffer << (5 - bits))];
}

return out;
}

// https://github.com/pnpm/pnpm/blob/f394cfccda7bc519ceee8c33fc9b68a0f4235532/packages/dependency-path/src/index.ts#L167-L189
export function depPathToFilename(depPath: string): string {
let filename: string = depPathToFilenameUnescaped(depPath).replace(/[\\/:*?"<>|]/g, '+');
if (filename.includes('(')) {
filename = filename.replace(/(\)\()|\(/g, '_').replace(/\)$/, '');
}
if (filename.length > 120 || (filename !== filename.toLowerCase() && !filename.startsWith('file+'))) {
return `${filename.substring(0, MAX_LENGTH_WITHOUT_HASH)}_${createBase32Hash(filename)}`;
}
return filename;
}
import type { IPnpmVersionHelpers } from './pnpm/pnpmVersionHelpers';

/**
* Computes the root folder for a dependency from a reference to it in another package
* @param lockfileFolder - The folder that contains the lockfile
* @param key - The key of the dependency
* @param specifier - The specifier in the lockfile for the dependency
* @param context - The owning package
* @param helpers - Version-specific pnpm helpers
* @returns The identifier for the dependency
*/
export function resolveDependencyKey(
lockfileFolder: string,
key: string,
specifier: string,
context: IResolverContext
context: IResolverContext,
helpers: IPnpmVersionHelpers,
packageKeys?: { has(key: string): boolean }
): string {
if (specifier.startsWith('/')) {
return getDescriptionFileRootFromKey(lockfileFolder, specifier);
} else if (specifier.startsWith('link:')) {
if (context.isProject) {
return path.posix.join(context.descriptionFileRoot, specifier.slice(5));
} else {
return path.posix.join(lockfileFolder, specifier.slice(5));
}
if (specifier.startsWith('link:')) {
return path.posix.join(
context.isProject ? context.descriptionFileRoot : lockfileFolder,
specifier.slice(5)
);
} else if (specifier.startsWith('file:')) {
return getDescriptionFileRootFromKey(lockfileFolder, specifier, key);
return getDescriptionFileRootFromKey(lockfileFolder, specifier, helpers.depPathToFilename, key);
} else {
return getDescriptionFileRootFromKey(lockfileFolder, `/${key}@${specifier}`);
const resolvedKey: string = packageKeys?.has(specifier)
? specifier
: helpers.buildDependencyKey(key, specifier);
return getDescriptionFileRootFromKey(lockfileFolder, resolvedKey, helpers.depPathToFilename);
}
}

/**
* Computes the physical path to a dependency based on its entry
* @param lockfileFolder - The folder that contains the lockfile during installation
* @param key - The key of the dependency
* @param depPathToFilename - Version-specific function to convert dep paths to filenames
* @param name - The name of the dependency, if provided
* @returns The physical path to the dependency
*/
export function getDescriptionFileRootFromKey(lockfileFolder: string, key: string, name?: string): string {
if (!key.startsWith('file:')) {
name = key.slice(1, key.indexOf('@', 2));
export function getDescriptionFileRootFromKey(
lockfileFolder: string,
key: string,
depPathToFilename: (depPath: string) => string,
name?: string
): string {
if (!key.startsWith('file:') && !name) {
const offset: number = key.startsWith('/') ? 1 : 0;
name = key.slice(offset, key.indexOf('@', offset + 1));
}
if (!name) {
throw new Error(`Missing package name for ${key}`);
Expand All @@ -106,29 +70,44 @@ export function getDescriptionFileRootFromKey(lockfileFolder: string, key: strin
export function resolveDependencies(
lockfileFolder: string,
collection: Record<string, IDependencyEntry>,
context: IResolverContext
context: IResolverContext,
helpers: IPnpmVersionHelpers,
packageKeys?: { has(key: string): boolean }
): void {
for (const [key, value] of Object.entries(collection)) {
const version: string = typeof value === 'string' ? value : value.version;
const resolved: string = resolveDependencyKey(lockfileFolder, key, version, context);
const resolved: string = resolveDependencyKey(
lockfileFolder,
key,
version,
context,
helpers,
packageKeys
);

context.deps.set(key, resolved);
}
}

/**
*
* @param depPath - The path to the dependency
* @returns The folder name for the dependency
* Extracts the package name and version from a lockfile package key.
* @param key - The lockfile package key (e.g. '/autoprefixer\@9.8.8', '\@scope/name\@1.0.0(peer\@2.0.0)')
* @returns The extracted name and version, or undefined for file: keys
*/
export function depPathToFilenameUnescaped(depPath: string): string {
if (depPath.indexOf('file:') !== 0) {
if (depPath.startsWith('/')) {
depPath = depPath.slice(1);
}
return depPath;
export function extractNameAndVersionFromKey(key: string): { name: string; version: string } | undefined {
if (key.startsWith('file:')) {
return undefined;
}
const offset: number = key.startsWith('/') ? 1 : 0;
const versionAtIndex: number = key.indexOf('@', offset + 1);
if (versionAtIndex === -1) {
return undefined;
}
return depPath.replace(':', '+');
const name: string = key.slice(offset, versionAtIndex);
const parenIndex: number = key.indexOf('(', versionAtIndex);
const version: string =
parenIndex !== -1 ? key.slice(versionAtIndex + 1, parenIndex) : key.slice(versionAtIndex + 1);
return { name, version };
}

/**
Expand Down
Loading
Loading