Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
{
"changes": [
{
"comment": "Allow plugin-defined outputless operations to use the build cache without requiring rush-project.json.",
"type": "none",
"packageName": "@microsoft/rush"
}
],
"packageName": "@microsoft/rush",
"email": "iclanton@users.noreply.github.com"
}
2 changes: 2 additions & 0 deletions common/reviews/api/rush-lib.api.md
Original file line number Diff line number Diff line change
Expand Up @@ -2087,6 +2087,8 @@ export class RushProjectConfiguration {
trackedFileNames: Iterable<string>;
phaseName: string;
isNoOp: boolean;
project?: RushConfigurationProject;
operationSettings?: Readonly<IOperationSettings>;
}): string | undefined;
// @internal (undocumented)
_getJsonForFingerprint(): string;
Expand Down
88 changes: 69 additions & 19 deletions libraries/rush-lib/src/api/RushProjectConfiguration.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,14 @@
import * as path from 'node:path';
import * as fs from 'node:fs';

import { AlreadyReportedError, Async, FileSystem, Import, JsonFile, Path } from '@rushstack/node-core-library';
import {
AlreadyReportedError,
Async,
FileSystem,
Import,
JsonFile,
Path
} from '@rushstack/node-core-library';
import type { ITerminal } from '@rushstack/terminal';
import { ProjectConfigurationFile, InheritanceType } from '@rushstack/heft-config-file';
import {
Expand Down Expand Up @@ -146,6 +153,11 @@ export interface IOperationSettings {
* root folder.
*
* These folders should not be tracked by Git. They must not contain symlinks.
*
* @remarks
* A plugin-defined operation for a project without a `rush-project.json` file can use the build cache
* if this property is explicitly set to an empty array and the operation does not use the
* `dependsOnEnvVars`, `dependsOnNodeVersion`, or `dependsOnAdditionalFiles` settings.
*/
outputFolderNames?: string[];

Expand Down Expand Up @@ -377,6 +389,18 @@ interface IProjectConfigurationCacheEntry {
const _currentConfigurationCache: WeakMap<RushConfigurationProject, IProjectConfigurationCacheEntry> =
new WeakMap();

function getHotlinkCacheDisabledReason(project: RushConfigurationProject): string | undefined {
const rushConfiguration: RushConfiguration | undefined = project.rushConfiguration;
if (rushConfiguration) {
const hotlinkManager: HotlinkManager = HotlinkManager.loadFromRushConfiguration(rushConfiguration);
if (hotlinkManager.hasAnyHotlinksInSubspace(project.subspace.subspaceName)) {
return 'Caching has been disabled for this project because it is in a subspace with hotlinked dependencies.';
}
}

return undefined;
}

/**
* Use this class to load the "config/rush-project.json" config file.
*
Expand Down Expand Up @@ -520,12 +544,9 @@ export class RushProjectConfiguration {
phaseName: string,
isNoOp: boolean
): string | undefined {
const rushConfiguration: RushConfiguration | undefined = this.project.rushConfiguration;
if (rushConfiguration) {
const hotlinkManager: HotlinkManager = HotlinkManager.loadFromRushConfiguration(rushConfiguration);
if (hotlinkManager.hasAnyHotlinksInSubspace(this.project.subspace.subspaceName)) {
return 'Caching has been disabled for this project because it is in a subspace with hotlinked dependencies.';
}
const hotlinkReason: string | undefined = getHotlinkCacheDisabledReason(this.project);
if (hotlinkReason) {
return hotlinkReason;
}

// Skip no-op operations as they won't have any output/cacheable things.
Expand Down Expand Up @@ -575,28 +596,49 @@ export class RushProjectConfiguration {

/**
* Source of truth for whether a project is unable to use the build cache for a given phase.
* As some operations may not have a rush-project.json file defined at all, but may be no-op operations
* we'll want to ignore those completely.
* No-op operations and explicitly outputless plugin operations do not require a rush-project.json file.
*/
public static getCacheDisabledReasonForProject(options: {
projectConfiguration: RushProjectConfiguration | undefined;
trackedFileNames: Iterable<string>;
phaseName: string;
isNoOp: boolean;
project?: RushConfigurationProject;
operationSettings?: Readonly<IOperationSettings>;
}): string | undefined {
const { projectConfiguration, trackedFileNames, phaseName, isNoOp } = options;
const { projectConfiguration, trackedFileNames, phaseName, isNoOp, project, operationSettings } = options;
if (isNoOp) {
return undefined;
}

if (!projectConfiguration) {
return (
`Project does not have a ${RushConstants.rushProjectConfigFilename} configuration file, ` +
'or one provided by a rig, so it does not support caching.'
);
if (projectConfiguration) {
return projectConfiguration.getCacheDisabledReason(trackedFileNames, phaseName, false);
}

if (
project &&
operationSettings?.operationName === phaseName &&
operationSettings.outputFolderNames?.length === 0 &&
!operationSettings.dependsOnEnvVars?.length &&
!operationSettings.dependsOnNodeVersion &&
!operationSettings.dependsOnAdditionalFiles?.length
) {
const hotlinkReason: string | undefined = getHotlinkCacheDisabledReason(project);
if (hotlinkReason) {
return hotlinkReason;
}

if (operationSettings.disableBuildCacheForOperation) {
return `Caching has been disabled for this project's "${phaseName}" command.`;
}

return undefined;
}

return projectConfiguration.getCacheDisabledReason(trackedFileNames, phaseName, isNoOp);
return (
`Project does not have a ${RushConstants.rushProjectConfigFilename} configuration file, ` +
'or one provided by a rig, so it does not support caching.'
);
}

/**
Expand Down Expand Up @@ -946,7 +988,10 @@ class ConfigurationInputView {
let result: IConfigurationFileStat | undefined;
try {
// statSync follows links, so dev and ino identify the file whose content is loaded.
const stat: fs.BigIntStats | undefined = fs.statSync(filePath, { bigint: true, throwIfNoEntry: false });
const stat: fs.BigIntStats | undefined = fs.statSync(filePath, {
bigint: true,
throwIfNoEntry: false
});
if (!stat) {
result = { stamp: MISSING_FILE_STAMP, settled: true };
} else if (stat.isFile()) {
Expand Down Expand Up @@ -994,7 +1039,11 @@ class ConfigurationInputView {
if (ownFileStamp !== MISSING_FILE_STAMP) {
filePath = getOwnConfigurationFilePath(project);
} else if (rigConfig instanceof RealProfileFolderRigConfig) {
const rigFolderPath: string = path.join(project.projectFolder, 'node_modules', rigConfig.rigPackageName);
const rigFolderPath: string = path.join(
project.projectFolder,
'node_modules',
rigConfig.rigPackageName
);
rigProfile = {
packageJsonPath: path.join(rigFolderPath, 'package.json'),
profileFolderPath: path.join(rigFolderPath, rigConfig.relativeProfileFolderPath),
Expand Down Expand Up @@ -1091,7 +1140,8 @@ class ConfigurationInputView {
files.push({
filePath,
stamp: stat.stamp,
parent: parentPath !== undefined ? { specifier: specifier as string, filePath: parentPath } : undefined
parent:
parentPath !== undefined ? { specifier: specifier as string, filePath: parentPath } : undefined
});
filePath = parentPath;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import type { CommandLineParameter } from '@rushstack/ts-command-line';

import type { IPhase } from '../CommandLineConfiguration';
import type { RushConfigurationProject } from '../RushConfigurationProject';
import { RushProjectConfiguration } from '../RushProjectConfiguration';
import { RushProjectConfiguration, type IOperationSettings } from '../RushProjectConfiguration';
import { PhasedCommandEngineProjectConfigurationError } from '../PhasedCommandEngineProjectConfigurationError';

// Several cases write, link and stat many files, which can take more than the default 5 s on a busy
Expand Down Expand Up @@ -242,23 +242,24 @@ describe(RushProjectConfiguration.name, () => {
readFileAsync.mockRestore();
dateNow.mockRestore();
}
expect(readPaths.filter((p) => p.endsWith('/profiles/default/config/rush-project.json'))).toHaveLength(1);
expect(readPaths.filter((p) => p.endsWith('/profiles/default/config/rush-project.json'))).toHaveLength(
1
);
expect(readPaths.filter((p) => p.endsWith('/shared/rush-project.json'))).toHaveLength(1);

for (const linked of projects.slice(0, 3)) {
expect(getOutputFolderNames(configurations.get(linked))).toEqual(['from-rig']);
expect([
...configurations.get(linked)!.operationSettingsByOperationName.get('_phase:test')!.outputFolderNames!
...configurations.get(linked)!.operationSettingsByOperationName.get('_phase:test')!
.outputFolderNames!
]).toEqual(['from-shared']);
}
expect(getOutputFolderNames(configurations.get(projects[3]))).toEqual(['from-project']);

const terminal: Terminal = new Terminal(new StringBufferTerminalProvider());
for (const linked of projects) {
const native: RushProjectConfiguration | undefined = await RushProjectConfiguration.tryLoadForProjectAsync(
linked,
terminal
);
const native: RushProjectConfiguration | undefined =
await RushProjectConfiguration.tryLoadForProjectAsync(linked, terminal);
expect(configurations.get(linked)!._getJsonForFingerprint()).toBe(native!._getJsonForFingerprint());
}
});
Expand Down Expand Up @@ -726,4 +727,60 @@ describe(RushProjectConfiguration.name, () => {
expect(reason).toMatchSnapshot();
});
});

describe(RushProjectConfiguration.getCacheDisabledReasonForProject.name, () => {
const project: RushConfigurationProject = {
packageName: 'plugin-project',
projectFolder: '/repo/plugin-project',
projectRelativeFolder: 'plugin-project'
} as RushConfigurationProject;
const phaseName: string = '_phase:plugin';
const getReason = (operationSettings?: Readonly<IOperationSettings>): string | undefined =>
RushProjectConfiguration.getCacheDisabledReasonForProject({
projectConfiguration: undefined,
trackedFileNames: [],
phaseName,
isNoOp: false,
project,
operationSettings
});

it('allows an explicitly outputless plugin operation without project configuration', () => {
expect(
getReason({
operationName: phaseName,
outputFolderNames: []
})
).toBeUndefined();
});

it('requires project configuration unless the operation is explicitly outputless', () => {
expect(getReason()).toContain('does not have a rush-project.json');
expect(getReason({ operationName: phaseName })).toContain('does not have a rush-project.json');
expect(getReason({ operationName: phaseName, outputFolderNames: ['lib'] })).toContain(
'does not have a rush-project.json'
);
});

it('requires project configuration for operation-specific cache inputs', () => {
const settings: IOperationSettings[] = [
{ operationName: phaseName, outputFolderNames: [], dependsOnEnvVars: ['ENV_VAR'] },
{ operationName: phaseName, outputFolderNames: [], dependsOnNodeVersion: true },
{ operationName: phaseName, outputFolderNames: [], dependsOnAdditionalFiles: ['config.json'] }
];
for (const operationSettings of settings) {
expect(getReason(operationSettings)).toContain('does not have a rush-project.json');
}
});

it('respects an explicit cache disable', () => {
expect(
getReason({
operationName: phaseName,
outputFolderNames: [],
disableBuildCacheForOperation: true
})
).toContain(`Caching has been disabled for this project's "${phaseName}" command.`);
});
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,9 @@ export class BuildPlanPlugin implements IPhasedCommandPlugin {
projectConfiguration,
trackedFileNames: fileHashes.keys(),
isNoOp: operation.isNoOp,
phaseName: associatedPhase.name
phaseName: associatedPhase.name,
project: associatedProject,
operationSettings: operation.settings
});
buildCacheByOperation.set(operation, { cacheDisabledReason });
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import {
formatMegabytes
} from '../buildCache/DeferredCacheEntryWrites';
import { RushConstants } from '../RushConstants';
import type { RushProjectConfiguration } from '../../api/RushProjectConfiguration';
import { RushProjectConfiguration } from '../../api/RushProjectConfiguration';
import {
initializeProjectLogFilesAsync,
getProjectLogFilePaths,
Expand Down Expand Up @@ -1230,12 +1230,14 @@ function readCacheInputs(
let isCacheDisabledReasonComputed: boolean = false;
const getCacheDisabledReason = (): string | undefined => {
if (!isCacheDisabledReasonComputed) {
cacheDisabledReason = getCacheDisabledReasonForOperation(
cacheDisabledReason = RushProjectConfiguration.getCacheDisabledReasonForProject({
projectConfiguration,
fileHashes,
trackedFileNames: fileHashes.keys(),
phaseName,
operation.isNoOp
);
isNoOp: operation.isNoOp,
project: operation.associatedProject,
operationSettings: operation.settings
});
isCacheDisabledReasonComputed = true;
}
return cacheDisabledReason;
Expand All @@ -1248,15 +1250,3 @@ function readCacheInputs(
inputFileHashes: fileHashes
};
}

function getCacheDisabledReasonForOperation(
projectConfiguration: RushProjectConfiguration | undefined,
fileHashes: ReadonlyMap<string, string>,
phaseName: string,
isNoOp: boolean
): string | undefined {
return projectConfiguration
? projectConfiguration.getCacheDisabledReason(fileHashes.keys(), phaseName, isNoOp)
: `Project does not have a ${RushConstants.rushProjectConfigFilename} configuration file, ` +
'or one provided by a rig, so it does not support caching.';
}
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,7 @@ interface ITestGraph {
cacheDisabledReasons: Map<string, string>;
// The names of the operations whose reason that caching is disabled was computed, in order
cacheDisabledReasonComputations: string[];
projectConfigurations: Map<RushConfigurationProject, RushProjectConfiguration>;
executions: string[];
cacheWrites: string[];
// The names of the operations that are restored from the build cache
Expand Down Expand Up @@ -280,6 +281,7 @@ async function createTestGraphAsync(
trackedFileHashes,
cacheDisabledReasons,
cacheDisabledReasonComputations,
projectConfigurations,
executions,
cacheWrites,
cacheHits,
Expand Down Expand Up @@ -982,6 +984,39 @@ describe(CacheableOperationPlugin.name, () => {
describe('reason that caching is disabled', () => {
const cacheDisabledReason: string = 'Caching has been disabled for this project.';

it('allows an explicitly outputless operation without project configuration', async () => {
const testGraph: ITestGraph = await createTestGraphAsync(['a']);
const operation: Operation = testGraph.operations.get('a')!;
operation.settings = {
operationName: mockPhase.name,
outputFolderNames: []
};
testGraph.projectConfigurations.delete(operation.associatedProject);

const result: IExecutionResult = await testGraph.executeAsync();

expect(result.status).toBe(OperationStatus.Success);
expect(testGraph.executions).toEqual(['a']);
expect(testGraph.cacheWrites).toEqual(['a']);
});

it('requires project configuration for operation-specific cache inputs', async () => {
const testGraph: ITestGraph = await createTestGraphAsync(['a']);
const operation: Operation = testGraph.operations.get('a')!;
operation.settings = {
operationName: mockPhase.name,
outputFolderNames: [],
dependsOnEnvVars: ['TEST_ENV_VAR']
};
testGraph.projectConfigurations.delete(operation.associatedProject);

const result: IExecutionResult = await testGraph.executeAsync();

expect(result.status).toBe(OperationStatus.Success);
expect(testGraph.executions).toEqual(['a']);
expect(testGraph.cacheWrites).toEqual([]);
});

it('is only computed for the operations that execute', async () => {
const testGraph: ITestGraph = await createTestGraphAsync(['a', 'b', 'c']);
await testGraph.executeAsync();
Expand Down