Skip to content

Compact the tsbuild info by encoding info differently for some of the situations #58641

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
May 24, 2024
Merged

Compact the tsbuild info by encoding info differently for some of the situations #58641

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
8df8f4b
Dont record file name in diagnostic if its for same file to reduce bu…
sheetalkamat May 24, 2024
666f49f
Change recording of semantic diagnsotics to reduce size in normal sce…
sheetalkamat May 24, 2024
8fc7268
Dont record empty reference map
sheetalkamat May 24, 2024
9bd3d58
Better readable types
sheetalkamat May 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
127 changes: 84 additions & 43 deletions src/compiler/builder.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ import {
isNumber,
isString,
map,
mapDefinedIterator,
maybeBind,
noop,
notImplemented,
Expand Down Expand Up @@ -102,7 +103,7 @@ export interface ReusableDiagnostic extends ReusableDiagnosticRelatedInformation
export interface ReusableDiagnosticRelatedInformation {
category: DiagnosticCategory;
code: number;
file: string | undefined;
file: string | undefined | false;
start: number | undefined;
length: number | undefined;
messageText: string | ReusableDiagnosticMessageChain;
Expand Down Expand Up @@ -384,7 +385,7 @@ function createBuilderProgramState(newProgram: Program, oldState: Readonly<Reusa
(state.emitDiagnosticsPerFile ??= new Map()).set(
sourceFilePath,
oldState!.hasReusableDiagnostic ?
convertToDiagnostics(emitDiagnostics as readonly ReusableDiagnostic[], newProgram) :
convertToDiagnostics(emitDiagnostics as readonly ReusableDiagnostic[], sourceFilePath, newProgram) :
repopulateDiagnostics(emitDiagnostics as readonly Diagnostic[], newProgram),
);
}
Expand All @@ -399,7 +400,7 @@ function createBuilderProgramState(newProgram: Program, oldState: Readonly<Reusa
state.semanticDiagnosticsPerFile!.set(
sourceFilePath,
oldState!.hasReusableDiagnostic ?
convertToDiagnostics(diagnostics as readonly ReusableDiagnostic[], newProgram) :
convertToDiagnostics(diagnostics as readonly ReusableDiagnostic[], sourceFilePath, newProgram) :
repopulateDiagnostics(diagnostics as readonly Diagnostic[], newProgram),
);
(state.semanticDiagnosticsFromOldState ??= new Set()).add(sourceFilePath);
Expand Down Expand Up @@ -516,19 +517,19 @@ function convertOrRepopulateDiagnosticMessageChainArray<T extends DiagnosticMess
return sameMap(array, chain => convertOrRepopulateDiagnosticMessageChain(chain, sourceFile, newProgram, repopulateInfo));
}

function convertToDiagnostics(diagnostics: readonly ReusableDiagnostic[], newProgram: Program): readonly Diagnostic[] {
function convertToDiagnostics(diagnostics: readonly ReusableDiagnostic[], diagnosticFilePath: Path, newProgram: Program): readonly Diagnostic[] {
if (!diagnostics.length) return emptyArray;
let buildInfoDirectory: string | undefined;
return diagnostics.map(diagnostic => {
const result: Diagnostic = convertToDiagnosticRelatedInformation(diagnostic, newProgram, toPathInBuildInfoDirectory);
const result: Diagnostic = convertToDiagnosticRelatedInformation(diagnostic, diagnosticFilePath, newProgram, toPathInBuildInfoDirectory);
result.reportsUnnecessary = diagnostic.reportsUnnecessary;
result.reportsDeprecated = diagnostic.reportDeprecated;
result.source = diagnostic.source;
result.skippedOn = diagnostic.skippedOn;
const { relatedInformation } = diagnostic;
result.relatedInformation = relatedInformation ?
relatedInformation.length ?
relatedInformation.map(r => convertToDiagnosticRelatedInformation(r, newProgram, toPathInBuildInfoDirectory)) :
relatedInformation.map(r => convertToDiagnosticRelatedInformation(r, diagnosticFilePath, newProgram, toPathInBuildInfoDirectory)) :
[] :
undefined;
return result;
Expand All @@ -540,9 +541,11 @@ function convertToDiagnostics(diagnostics: readonly ReusableDiagnostic[], newPro
}
}

function convertToDiagnosticRelatedInformation(diagnostic: ReusableDiagnosticRelatedInformation, newProgram: Program, toPath: (path: string) => Path): DiagnosticRelatedInformation {
function convertToDiagnosticRelatedInformation(diagnostic: ReusableDiagnosticRelatedInformation, diagnosticFilePath: Path, newProgram: Program, toPath: (path: string) => Path): DiagnosticRelatedInformation {
const { file } = diagnostic;
const sourceFile = file ? newProgram.getSourceFileByPath(toPath(file)) : undefined;
const sourceFile = file !== false ?
newProgram.getSourceFileByPath(file ? toPath(file) : diagnosticFilePath) :
undefined;
return {
...diagnostic,
file: sourceFile,
Expand Down Expand Up @@ -981,7 +984,14 @@ export type ProgramBuildInfoFileId = number & { __programBuildInfoFileIdBrand: a
/** @internal */
export type ProgramBuildInfoFileIdListId = number & { __programBuildInfoFileIdListIdBrand: any; };
/** @internal */
export type ProgramBuildInfoDiagnostic = ProgramBuildInfoFileId | [fileId: ProgramBuildInfoFileId, diagnostics: readonly ReusableDiagnostic[]];
export type ProgramBuildInfoDiagnosticOfFile = [fileId: ProgramBuildInfoFileId, diagnostics: readonly ReusableDiagnostic[]];
/** @internal */
export type ProgramBuildInfoDiagnostic =
| ProgramBuildInfoFileId // File is not in changedSet and still doesnt have cached diagnostics
| ProgramBuildInfoDiagnosticOfFile; // Diagnostics for file
/** @internal */
export type ProgramBuildInfoEmitDiagnostic = ProgramBuildInfoDiagnosticOfFile; // Diagnostics for the file

/**
* fileId if pending emit is same as what compilerOptions suggest
* [fileId] if pending emit is only dts file emit
Expand Down Expand Up @@ -1034,7 +1044,7 @@ export interface ProgramMultiFileEmitBuildInfo {
fileIdsList: readonly (readonly ProgramBuildInfoFileId[])[] | undefined;
referencedMap: ProgramBuildInfoReferencedMap | undefined;
semanticDiagnosticsPerFile: ProgramBuildInfoDiagnostic[] | undefined;
emitDiagnosticsPerFile: ProgramBuildInfoDiagnostic[] | undefined;
emitDiagnosticsPerFile: ProgramBuildInfoEmitDiagnostic[] | undefined;
affectedFilesPendingEmit: ProgramBuilderInfoFilePendingEmit[] | undefined;
changeFileSet: readonly ProgramBuildInfoFileId[] | undefined;
emitSignatures: readonly ProgramBuildInfoEmitSignature[] | undefined;
Expand Down Expand Up @@ -1156,14 +1166,14 @@ function getBuildInfo(state: BuilderProgramState): BuildInfo {
});

let referencedMap: ProgramBuildInfoReferencedMap | undefined;
if (state.referencedMap) {
if (state.referencedMap?.size()) {
referencedMap = arrayFrom(state.referencedMap.keys()).sort(compareStringsCaseSensitive).map(key => [
toFileId(key),
toFileIdListId(state.referencedMap!.getValues(key)!),
]);
}

const semanticDiagnosticsPerFile = convertToProgramBuildInfoDiagnostics(state.semanticDiagnosticsPerFile);
const semanticDiagnosticsPerFile = convertToProgramBuildInfoDiagnostics();
let affectedFilesPendingEmit: ProgramBuilderInfoFilePendingEmit[] | undefined;
if (state.affectedFilesPendingEmit?.size) {
const fullEmitForOptions = getBuilderFileEmit(state.compilerOptions);
Expand Down Expand Up @@ -1191,7 +1201,7 @@ function getBuildInfo(state: BuilderProgramState): BuildInfo {
changeFileSet = append(changeFileSet, toFileId(path));
}
}
const emitDiagnosticsPerFile = convertToProgramBuildInfoDiagnostics(state.emitDiagnosticsPerFile);
const emitDiagnosticsPerFile = convertToProgramBuildInfoEmitDiagnostics();
const program: ProgramMultiFileEmitBuildInfo = {
fileNames,
fileInfos,
Expand Down Expand Up @@ -1301,48 +1311,63 @@ function getBuildInfo(state: BuilderProgramState): BuildInfo {
return value;
}

function convertToProgramBuildInfoDiagnostics(diagnostics: Map<Path, readonly Diagnostic[]> | undefined) {
function convertToProgramBuildInfoDiagnostics() {
let result: ProgramBuildInfoDiagnostic[] | undefined;
if (diagnostics) {
for (const key of arrayFrom(diagnostics.keys()).sort(compareStringsCaseSensitive)) {
const value = diagnostics.get(key)!;
result = append(
result,
value.length ?
[
toFileId(key),
convertToReusableDiagnostics(value),
] :
toFileId(key),
);
state.fileInfos.forEach((_value, key) => {
const value = state.semanticDiagnosticsPerFile?.get(key);
if (!value) {
if (!state.changedFilesSet.has(key)) result = append(result, toFileId(key));
}
else if (value.length) {
result = append(result, [
toFileId(key),
convertToReusableDiagnostics(value, key),
]);
}
});
return result;
}

function convertToProgramBuildInfoEmitDiagnostics() {
let result: ProgramBuildInfoEmitDiagnostic[] | undefined;
if (!state.emitDiagnosticsPerFile?.size) return result;
for (const key of arrayFrom(state.emitDiagnosticsPerFile.keys()).sort(compareStringsCaseSensitive)) {
const value = state.emitDiagnosticsPerFile.get(key)!;
result = append(result, [
toFileId(key),
convertToReusableDiagnostics(value, key),
]);
}
return result;
}

function convertToReusableDiagnostics(diagnostics: readonly Diagnostic[]): readonly ReusableDiagnostic[] {
function convertToReusableDiagnostics(diagnostics: readonly Diagnostic[], diagnosticFilePath: Path): readonly ReusableDiagnostic[] {
Debug.assert(!!diagnostics.length);
return diagnostics.map(diagnostic => {
const result: ReusableDiagnostic = convertToReusableDiagnosticRelatedInformation(diagnostic);
const result: ReusableDiagnostic = convertToReusableDiagnosticRelatedInformation(diagnostic, diagnosticFilePath);
result.reportsUnnecessary = diagnostic.reportsUnnecessary;
result.reportDeprecated = diagnostic.reportsDeprecated;
result.source = diagnostic.source;
result.skippedOn = diagnostic.skippedOn;
const { relatedInformation } = diagnostic;
result.relatedInformation = relatedInformation ?
relatedInformation.length ?
relatedInformation.map(r => convertToReusableDiagnosticRelatedInformation(r)) :
relatedInformation.map(r => convertToReusableDiagnosticRelatedInformation(r, diagnosticFilePath)) :
[] :
undefined;
return result;
});
}

function convertToReusableDiagnosticRelatedInformation(diagnostic: DiagnosticRelatedInformation): ReusableDiagnosticRelatedInformation {
function convertToReusableDiagnosticRelatedInformation(diagnostic: DiagnosticRelatedInformation, diagnosticFilePath: Path): ReusableDiagnosticRelatedInformation {
const { file } = diagnostic;
return {
...diagnostic,
file: file ? relativeToBuildInfo(file.resolvedPath) : undefined,
file: file ?
file.resolvedPath === diagnosticFilePath ?
undefined :
relativeToBuildInfo(file.resolvedPath) :
false,
messageText: isString(diagnostic.messageText) ? diagnostic.messageText : convertToReusableDiagnosticMessageChain(diagnostic.messageText),
};
}
Expand Down Expand Up @@ -1881,16 +1906,17 @@ export function createBuilderProgramUsingProgramBuildInfo(buildInfo: BuildInfo,
);
}
});
const changedFilesSet = new Set(map(program.changeFileSet, toFilePath));
const fullEmitForOptions = program.affectedFilesPendingEmit ? getBuilderFileEmit(program.options || {}) : undefined;
state = {
fileInfos,
compilerOptions: program.options ? convertToOptionsWithAbsolutePaths(program.options, toAbsolutePath) : {},
referencedMap: toManyToManyPathMap(program.referencedMap),
semanticDiagnosticsPerFile: toPerFileDiagnostics(program.semanticDiagnosticsPerFile),
emitDiagnosticsPerFile: toPerFileDiagnostics(program.emitDiagnosticsPerFile),
referencedMap: toManyToManyPathMap(program.referencedMap, program.options ?? {}),
semanticDiagnosticsPerFile: toPerFileSemanticDiagnostics(program.semanticDiagnosticsPerFile, fileInfos, changedFilesSet),
emitDiagnosticsPerFile: toPerFileEmitDiagnostics(program.emitDiagnosticsPerFile),
hasReusableDiagnostic: true,
affectedFilesPendingEmit: program.affectedFilesPendingEmit && arrayToMap(program.affectedFilesPendingEmit, value => toFilePath(isNumber(value) ? value : value[0]), value => toBuilderFileEmit(value, fullEmitForOptions!)),
changedFilesSet: new Set(map(program.changeFileSet, toFilePath)),
changedFilesSet,
latestChangedDtsFile,
emitSignatures: emitSignatures?.size ? emitSignatures : undefined,
};
Expand Down Expand Up @@ -1938,18 +1964,33 @@ export function createBuilderProgramUsingProgramBuildInfo(buildInfo: BuildInfo,
return filePathsSetList![fileIdsListId - 1];
}

function toManyToManyPathMap(referenceMap: ProgramBuildInfoReferencedMap | undefined): BuilderState.ManyToManyPathMap | undefined {
if (!referenceMap) {
return undefined;
}

const map = BuilderState.createManyToManyPathMap();
function toManyToManyPathMap(referenceMap: ProgramBuildInfoReferencedMap | undefined, options: CompilerOptions): BuilderState.ManyToManyPathMap | undefined {
const map = BuilderState.createReferencedMap(options);
if (!map || !referenceMap) return map;
referenceMap.forEach(([fileId, fileIdListId]) => map.set(toFilePath(fileId), toFilePathsSet(fileIdListId)));
return map;
}

function toPerFileDiagnostics(diagnostics: readonly ProgramBuildInfoDiagnostic[] | undefined): Map<Path, readonly ReusableDiagnostic[]> | undefined {
return diagnostics && arrayToMap(diagnostics, value => toFilePath(isNumber(value) ? value : value[0]), value => isNumber(value) ? emptyArray : value[1]);
function toPerFileSemanticDiagnostics(
diagnostics: readonly ProgramBuildInfoDiagnostic[] | undefined,
fileInfos: Map<Path, BuilderState.FileInfo>,
changedFilesSet: Set<Path>,
): Map<Path, readonly ReusableDiagnostic[]> | undefined {
const semanticDiagnostics = new Map<Path, readonly ReusableDiagnostic[]>(
mapDefinedIterator(
fileInfos.keys(),
key => !changedFilesSet.has(key) ? [key, emptyArray] : undefined,
),
);
diagnostics?.forEach(value => {
if (isNumber(value)) semanticDiagnostics.delete(toFilePath(value));
else semanticDiagnostics.set(toFilePath(value[0]), value[1]);
});
return semanticDiagnostics.size ? semanticDiagnostics : undefined;
}

function toPerFileEmitDiagnostics(diagnostics: readonly ProgramBuildInfoEmitDiagnostic[] | undefined): Map<Path, readonly ReusableDiagnostic[]> | undefined {
return diagnostics && arrayToMap(diagnostics, value => toFilePath(value[0]), value => value[1]);
}
}

Expand Down
15 changes: 11 additions & 4 deletions src/compiler/builderState.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import {
arrayFrom,
CancellationToken,
CompilerOptions,
computeSignatureWithDiagnostics,
CustomTransformers,
Debug,
Expand Down Expand Up @@ -110,6 +111,7 @@ export namespace BuilderState {
getKeys(v: Path): ReadonlySet<Path> | undefined;
getValues(k: Path): ReadonlySet<Path> | undefined;
keys(): IterableIterator<Path>;
size(): number;
}

export interface ManyToManyPathMap extends ReadonlyManyToManyPathMap {
Expand All @@ -123,6 +125,7 @@ export namespace BuilderState {
getKeys: v => reverse.get(v),
getValues: k => forward.get(k),
keys: () => forward.keys(),
size: () => forward.size,

deleteKey: k => {
(deleted ||= new Set<Path>()).add(k);
Expand Down Expand Up @@ -292,15 +295,19 @@ export namespace BuilderState {
return oldState && !oldState.referencedMap === !newReferencedMap;
}

export function createReferencedMap(options: CompilerOptions) {
return options.module !== ModuleKind.None && !options.outFile ?
createManyToManyPathMap() :
undefined;
}

/**
* Creates the state of file references and signature for the new program from oldState if it is safe
*/
export function create(newProgram: Program, oldState: Readonly<BuilderState> | undefined, disableUseFileVersionAsSignature: boolean): BuilderState {
const fileInfos = new Map<Path, FileInfo>();
const options = newProgram.getCompilerOptions();
const isOutFile = options.outFile;
const referencedMap = options.module !== ModuleKind.None && !isOutFile ?
createManyToManyPathMap() : undefined;
const referencedMap = createReferencedMap(options);
const useOldState = canReuseOldState(referencedMap, oldState);

// Ensure source files have parent pointers set
Expand All @@ -323,7 +330,7 @@ export namespace BuilderState {
version,
signature,
// No need to calculate affectsGlobalScope with --out since its not used at all
affectsGlobalScope: !isOutFile ? isFileAffectingGlobalScope(sourceFile) || undefined : undefined,
affectsGlobalScope: !options.outFile ? isFileAffectingGlobalScope(sourceFile) || undefined : undefined,
impliedFormat: sourceFile.impliedNodeFormat,
});
}
Expand Down
4 changes: 1 addition & 3 deletions src/compiler/tsbuildPublic.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,6 @@ import {
getWatchErrorSummaryDiagnosticMessage,
hasProperty,
identity,
isArray,
isIgnoredFileFromWildCardWatching,
isIncrementalCompilation,
isPackageJsonInfo,
Expand Down Expand Up @@ -105,7 +104,6 @@ import {
SemanticDiagnosticsBuilderProgram,
setGetSourceFileAsHashVersioned,
SharedExtendedConfigFileWatcher,
some,
SourceFile,
Status,
sys,
Expand Down Expand Up @@ -1698,7 +1696,7 @@ function getUpToDateStatusWorker<T extends BuilderProgram>(state: SolutionBuilde
(!project.options.noEmit ?
(buildInfo.program as ProgramMultiFileEmitBuildInfo).affectedFilesPendingEmit?.length ||
(buildInfo.program as ProgramMultiFileEmitBuildInfo).emitDiagnosticsPerFile?.length :
some((buildInfo.program as ProgramMultiFileEmitBuildInfo).semanticDiagnosticsPerFile, isArray))
(buildInfo.program as ProgramMultiFileEmitBuildInfo).semanticDiagnosticsPerFile?.length)
) {
return {
type: UpToDateStatusType.OutOfDateBuildInfo,
Expand Down
Loading