Skip to content

Commit db5368d

Browse files
Orta TheroxsandersnDanielRosenwassersheetalkamat
authored
Import the semantic highlighter from typescript-vscode-sh-plugin (#39119)
* Initial import of the vscode semantic highlight code * Adds the ability to test modern semantic classification via strings instead of numbers * Adds existing tests * Port over the semantic classification tests * Update baselines * Update src/harness/fourslashImpl.ts Co-authored-by: Nathan Shively-Sanders <[email protected]> * Handle feedback from #39119 * Consistent formatting in the 2020 classifier * Update baselines * Apply suggestions from code review Co-authored-by: Daniel Rosenwasser <[email protected]> * Update src/harness/fourslashImpl.ts Co-authored-by: Daniel Rosenwasser <[email protected]> * Reafactor after comments * Use 2020 everywhere * Handle feedback * WIP - don't provide a breaking change * Fix all build errors * Update baselines * Update src/services/classifier2020.ts Co-authored-by: Sheetal Nandi <[email protected]> * Addresses Ron's feedback Co-authored-by: Nathan Shively-Sanders <[email protected]> Co-authored-by: Daniel Rosenwasser <[email protected]> Co-authored-by: Sheetal Nandi <[email protected]>
1 parent a36f17c commit db5368d

File tree

81 files changed

+1169
-194
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

81 files changed

+1169
-194
lines changed

‎src/harness/client.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -747,7 +747,7 @@ namespace ts.server {
747747
return notImplemented();
748748
}
749749

750-
getEncodedSemanticClassifications(_fileName: string, _span: TextSpan): Classifications {
750+
getEncodedSemanticClassifications(_fileName: string, _span: TextSpan, _format?: SemanticClassificationFormat): Classifications {
751751
return notImplemented();
752752
}
753753

‎src/harness/fourslashImpl.ts

Lines changed: 70 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2492,7 +2492,49 @@ namespace FourSlash {
24922492
Harness.IO.log(this.spanInfoToString(this.getNameOrDottedNameSpan(pos)!, "**"));
24932493
}
24942494

2495-
private verifyClassifications(expected: { classificationType: string; text: string; textSpan?: TextSpan }[], actual: ts.ClassifiedSpan[], sourceFileText: string) {
2495+
private classificationToIdentifier(classification: number){
2496+
2497+
const tokenTypes: string[] = [];
2498+
tokenTypes[ts.classifier.v2020.TokenType.class] = "class";
2499+
tokenTypes[ts.classifier.v2020.TokenType.enum] = "enum";
2500+
tokenTypes[ts.classifier.v2020.TokenType.interface] = "interface";
2501+
tokenTypes[ts.classifier.v2020.TokenType.namespace] = "namespace";
2502+
tokenTypes[ts.classifier.v2020.TokenType.typeParameter] = "typeParameter";
2503+
tokenTypes[ts.classifier.v2020.TokenType.type] = "type";
2504+
tokenTypes[ts.classifier.v2020.TokenType.parameter] = "parameter";
2505+
tokenTypes[ts.classifier.v2020.TokenType.variable] = "variable";
2506+
tokenTypes[ts.classifier.v2020.TokenType.enumMember] = "enumMember";
2507+
tokenTypes[ts.classifier.v2020.TokenType.property] = "property";
2508+
tokenTypes[ts.classifier.v2020.TokenType.function] = "function";
2509+
tokenTypes[ts.classifier.v2020.TokenType.member] = "member";
2510+
2511+
const tokenModifiers: string[] = [];
2512+
tokenModifiers[ts.classifier.v2020.TokenModifier.async] = "async";
2513+
tokenModifiers[ts.classifier.v2020.TokenModifier.declaration] = "declaration";
2514+
tokenModifiers[ts.classifier.v2020.TokenModifier.readonly] = "readonly";
2515+
tokenModifiers[ts.classifier.v2020.TokenModifier.static] = "static";
2516+
tokenModifiers[ts.classifier.v2020.TokenModifier.local] = "local";
2517+
tokenModifiers[ts.classifier.v2020.TokenModifier.defaultLibrary] = "defaultLibrary";
2518+
2519+
2520+
function getTokenTypeFromClassification(tsClassification: number): number | undefined {
2521+
if (tsClassification > ts.classifier.v2020.TokenEncodingConsts.modifierMask) {
2522+
return (tsClassification >> ts.classifier.v2020.TokenEncodingConsts.typeOffset) - 1;
2523+
}
2524+
return undefined;
2525+
}
2526+
2527+
function getTokenModifierFromClassification(tsClassification: number) {
2528+
return tsClassification & ts.classifier.v2020.TokenEncodingConsts.modifierMask;
2529+
}
2530+
2531+
const typeIdx = getTokenTypeFromClassification(classification) || 0;
2532+
const modSet = getTokenModifierFromClassification(classification);
2533+
2534+
return [tokenTypes[typeIdx], ...tokenModifiers.filter((_, i) => modSet & 1 << i)].join(".");
2535+
}
2536+
2537+
private verifyClassifications(expected: { classificationType: string | number, text?: string; textSpan?: TextSpan }[], actual: (ts.ClassifiedSpan | ts.ClassifiedSpan2020)[] , sourceFileText: string) {
24962538
if (actual.length !== expected.length) {
24972539
this.raiseError("verifyClassifications failed - expected total classifications to be " + expected.length +
24982540
", but was " + actual.length +
@@ -2501,10 +2543,12 @@ namespace FourSlash {
25012543

25022544
ts.zipWith(expected, actual, (expectedClassification, actualClassification) => {
25032545
const expectedType = expectedClassification.classificationType;
2504-
if (expectedType !== actualClassification.classificationType) {
2546+
const actualType = typeof actualClassification.classificationType === "number" ? this.classificationToIdentifier(actualClassification.classificationType) : actualClassification.classificationType;
2547+
2548+
if (expectedType !== actualType) {
25052549
this.raiseError("verifyClassifications failed - expected classifications type to be " +
25062550
expectedType + ", but was " +
2507-
actualClassification.classificationType +
2551+
actualType +
25082552
jsonMismatchString());
25092553
}
25102554

@@ -2555,10 +2599,30 @@ namespace FourSlash {
25552599
}
25562600
}
25572601

2558-
public verifySemanticClassifications(expected: { classificationType: string; text: string }[]) {
2602+
public replaceWithSemanticClassifications(format: ts.SemanticClassificationFormat.TwentyTwenty) {
25592603
const actual = this.languageService.getSemanticClassifications(this.activeFile.fileName,
2560-
ts.createTextSpan(0, this.activeFile.content.length));
2604+
ts.createTextSpan(0, this.activeFile.content.length), format);
2605+
const replacement = [`const c2 = classification("2020");`,`verify.semanticClassificationsAre("2020",`];
2606+
for (const a of actual) {
2607+
const identifier = this.classificationToIdentifier(a.classificationType as number);
2608+
const text = this.activeFile.content.slice(a.textSpan.start, a.textSpan.start + a.textSpan.length);
2609+
replacement.push(` c2.semanticToken("${identifier}", "${text}"), `);
2610+
};
2611+
replacement.push(");");
25612612

2613+
throw new Error("You need to change the source code of fourslash test to use replaceWithSemanticClassifications");
2614+
2615+
// const fs = require("fs");
2616+
// const testfilePath = this.originalInputFileName.slice(1);
2617+
// const testfile = fs.readFileSync(testfilePath, "utf8");
2618+
// const newfile = testfile.replace("verify.replaceWithSemanticClassifications(\"2020\")", replacement.join("\n"));
2619+
// fs.writeFileSync(testfilePath, newfile);
2620+
}
2621+
2622+
2623+
public verifySemanticClassifications(format: ts.SemanticClassificationFormat, expected: { classificationType: string | number; text?: string }[]) {
2624+
const actual = this.languageService.getSemanticClassifications(this.activeFile.fileName,
2625+
ts.createTextSpan(0, this.activeFile.content.length), format);
25622626
this.verifyClassifications(expected, actual, this.activeFile.content);
25632627
}
25642628

@@ -3859,7 +3923,7 @@ namespace FourSlash {
38593923
const cancellation = new FourSlashInterface.Cancellation(state);
38603924
// eslint-disable-next-line no-eval
38613925
const f = eval(wrappedCode);
3862-
f(test, goTo, plugins, verify, edit, debug, format, cancellation, FourSlashInterface.Classification, FourSlashInterface.Completion, verifyOperationIsCancelled);
3926+
f(test, goTo, plugins, verify, edit, debug, format, cancellation, FourSlashInterface.classification, FourSlashInterface.Completion, verifyOperationIsCancelled);
38633927
}
38643928
catch (err) {
38653929
// ensure 'source-map-support' is triggered while we still have the handler attached by accessing `error.stack`.

‎src/harness/fourslashInterfaceImpl.ts

Lines changed: 85 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -524,8 +524,12 @@ namespace FourSlashInterface {
524524
/**
525525
* This method *requires* an ordered stream of classifications for a file, and spans are highly recommended.
526526
*/
527-
public semanticClassificationsAre(...classifications: Classification[]) {
528-
this.state.verifySemanticClassifications(classifications);
527+
public semanticClassificationsAre(format: ts.SemanticClassificationFormat, ...classifications: Classification[]) {
528+
this.state.verifySemanticClassifications(format, classifications);
529+
}
530+
531+
public replaceWithSemanticClassifications(format: ts.SemanticClassificationFormat.TwentyTwenty) {
532+
this.state.replaceWithSemanticClassifications(format);
529533
}
530534

531535
public renameInfoSucceeded(displayName?: string, fullDisplayName?: string, kind?: string, kindModifiers?: string, fileToRename?: string, expectedRange?: FourSlash.Range, options?: ts.RenameInfoOptions) {
@@ -768,109 +772,163 @@ namespace FourSlashInterface {
768772
}
769773
}
770774

771-
interface Classification {
775+
interface OlderClassification {
772776
classificationType: ts.ClassificationTypeNames;
773777
text: string;
774778
textSpan?: FourSlash.TextSpan;
775779
}
776-
export namespace Classification {
777-
export function comment(text: string, position?: number): Classification {
780+
781+
// The VS Code LSP
782+
interface ModernClassification {
783+
classificationType: string;
784+
text?: string;
785+
textSpan?: FourSlash.TextSpan;
786+
}
787+
788+
type Classification = OlderClassification | ModernClassification;
789+
790+
export function classification(format: ts.SemanticClassificationFormat) {
791+
792+
function semanticToken(identifier: string, text: string, _position: number): Classification {
793+
return {
794+
classificationType: identifier,
795+
text
796+
};
797+
}
798+
799+
if (format === ts.SemanticClassificationFormat.TwentyTwenty) {
800+
return {
801+
semanticToken
802+
};
803+
}
804+
805+
// Defaults to the previous semantic classifier factory functions
806+
807+
function comment(text: string, position?: number): Classification {
778808
return getClassification(ts.ClassificationTypeNames.comment, text, position);
779809
}
780810

781-
export function identifier(text: string, position?: number): Classification {
811+
function identifier(text: string, position?: number): Classification {
782812
return getClassification(ts.ClassificationTypeNames.identifier, text, position);
783813
}
784814

785-
export function keyword(text: string, position?: number): Classification {
815+
function keyword(text: string, position?: number): Classification {
786816
return getClassification(ts.ClassificationTypeNames.keyword, text, position);
787817
}
788818

789-
export function numericLiteral(text: string, position?: number): Classification {
819+
function numericLiteral(text: string, position?: number): Classification {
790820
return getClassification(ts.ClassificationTypeNames.numericLiteral, text, position);
791821
}
792822

793-
export function operator(text: string, position?: number): Classification {
823+
function operator(text: string, position?: number): Classification {
794824
return getClassification(ts.ClassificationTypeNames.operator, text, position);
795825
}
796826

797-
export function stringLiteral(text: string, position?: number): Classification {
827+
function stringLiteral(text: string, position?: number): Classification {
798828
return getClassification(ts.ClassificationTypeNames.stringLiteral, text, position);
799829
}
800830

801-
export function whiteSpace(text: string, position?: number): Classification {
831+
function whiteSpace(text: string, position?: number): Classification {
802832
return getClassification(ts.ClassificationTypeNames.whiteSpace, text, position);
803833
}
804834

805-
export function text(text: string, position?: number): Classification {
835+
function text(text: string, position?: number): Classification {
806836
return getClassification(ts.ClassificationTypeNames.text, text, position);
807837
}
808838

809-
export function punctuation(text: string, position?: number): Classification {
839+
function punctuation(text: string, position?: number): Classification {
810840
return getClassification(ts.ClassificationTypeNames.punctuation, text, position);
811841
}
812842

813-
export function docCommentTagName(text: string, position?: number): Classification {
843+
function docCommentTagName(text: string, position?: number): Classification {
814844
return getClassification(ts.ClassificationTypeNames.docCommentTagName, text, position);
815845
}
816846

817-
export function className(text: string, position?: number): Classification {
847+
function className(text: string, position?: number): Classification {
818848
return getClassification(ts.ClassificationTypeNames.className, text, position);
819849
}
820850

821-
export function enumName(text: string, position?: number): Classification {
851+
function enumName(text: string, position?: number): Classification {
822852
return getClassification(ts.ClassificationTypeNames.enumName, text, position);
823853
}
824854

825-
export function interfaceName(text: string, position?: number): Classification {
855+
function interfaceName(text: string, position?: number): Classification {
826856
return getClassification(ts.ClassificationTypeNames.interfaceName, text, position);
827857
}
828858

829-
export function moduleName(text: string, position?: number): Classification {
859+
function moduleName(text: string, position?: number): Classification {
830860
return getClassification(ts.ClassificationTypeNames.moduleName, text, position);
831861
}
832862

833-
export function typeParameterName(text: string, position?: number): Classification {
863+
function typeParameterName(text: string, position?: number): Classification {
834864
return getClassification(ts.ClassificationTypeNames.typeParameterName, text, position);
835865
}
836866

837-
export function parameterName(text: string, position?: number): Classification {
867+
function parameterName(text: string, position?: number): Classification {
838868
return getClassification(ts.ClassificationTypeNames.parameterName, text, position);
839869
}
840870

841-
export function typeAliasName(text: string, position?: number): Classification {
871+
function typeAliasName(text: string, position?: number): Classification {
842872
return getClassification(ts.ClassificationTypeNames.typeAliasName, text, position);
843873
}
844874

845-
export function jsxOpenTagName(text: string, position?: number): Classification {
875+
function jsxOpenTagName(text: string, position?: number): Classification {
846876
return getClassification(ts.ClassificationTypeNames.jsxOpenTagName, text, position);
847877
}
848878

849-
export function jsxCloseTagName(text: string, position?: number): Classification {
879+
function jsxCloseTagName(text: string, position?: number): Classification {
850880
return getClassification(ts.ClassificationTypeNames.jsxCloseTagName, text, position);
851881
}
852882

853-
export function jsxSelfClosingTagName(text: string, position?: number): Classification {
883+
function jsxSelfClosingTagName(text: string, position?: number): Classification {
854884
return getClassification(ts.ClassificationTypeNames.jsxSelfClosingTagName, text, position);
855885
}
856886

857-
export function jsxAttribute(text: string, position?: number): Classification {
887+
function jsxAttribute(text: string, position?: number): Classification {
858888
return getClassification(ts.ClassificationTypeNames.jsxAttribute, text, position);
859889
}
860890

861-
export function jsxText(text: string, position?: number): Classification {
891+
function jsxText(text: string, position?: number): Classification {
862892
return getClassification(ts.ClassificationTypeNames.jsxText, text, position);
863893
}
864894

865-
export function jsxAttributeStringLiteralValue(text: string, position?: number): Classification {
895+
function jsxAttributeStringLiteralValue(text: string, position?: number): Classification {
866896
return getClassification(ts.ClassificationTypeNames.jsxAttributeStringLiteralValue, text, position);
867897
}
868898

869899
function getClassification(classificationType: ts.ClassificationTypeNames, text: string, position?: number): Classification {
870900
const textSpan = position === undefined ? undefined : { start: position, end: position + text.length };
871901
return { classificationType, text, textSpan };
872902
}
903+
904+
return {
905+
comment,
906+
identifier,
907+
keyword,
908+
numericLiteral,
909+
operator,
910+
stringLiteral,
911+
whiteSpace,
912+
text,
913+
punctuation,
914+
docCommentTagName,
915+
className,
916+
enumName,
917+
interfaceName,
918+
moduleName,
919+
typeParameterName,
920+
parameterName,
921+
typeAliasName,
922+
jsxOpenTagName,
923+
jsxCloseTagName,
924+
jsxSelfClosingTagName,
925+
jsxAttribute,
926+
jsxText,
927+
jsxAttributeStringLiteralValue,
928+
getClassification
929+
};
873930
}
931+
874932
export namespace Completion {
875933
export import SortText = ts.Completions.SortText;
876934
export import CompletionSource = ts.Completions.CompletionSource;

‎src/harness/harnessLanguageService.ts

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -461,14 +461,15 @@ namespace Harness.LanguageService {
461461
getSyntacticClassifications(fileName: string, span: ts.TextSpan): ts.ClassifiedSpan[] {
462462
return unwrapJSONCallResult(this.shim.getSyntacticClassifications(fileName, span.start, span.length));
463463
}
464-
getSemanticClassifications(fileName: string, span: ts.TextSpan): ts.ClassifiedSpan[] {
465-
return unwrapJSONCallResult(this.shim.getSemanticClassifications(fileName, span.start, span.length));
464+
getSemanticClassifications(fileName: string, span: ts.TextSpan, format?: ts.SemanticClassificationFormat): ts.ClassifiedSpan[] {
465+
return unwrapJSONCallResult(this.shim.getSemanticClassifications(fileName, span.start, span.length, format));
466466
}
467467
getEncodedSyntacticClassifications(fileName: string, span: ts.TextSpan): ts.Classifications {
468468
return unwrapJSONCallResult(this.shim.getEncodedSyntacticClassifications(fileName, span.start, span.length));
469469
}
470-
getEncodedSemanticClassifications(fileName: string, span: ts.TextSpan): ts.Classifications {
471-
return unwrapJSONCallResult(this.shim.getEncodedSemanticClassifications(fileName, span.start, span.length));
470+
getEncodedSemanticClassifications(fileName: string, span: ts.TextSpan, format?: ts.SemanticClassificationFormat): ts.Classifications {
471+
const responseFormat = format || ts.SemanticClassificationFormat.Original;
472+
return unwrapJSONCallResult(this.shim.getEncodedSemanticClassifications(fileName, span.start, span.length, responseFormat));
472473
}
473474
getCompletionsAtPosition(fileName: string, position: number, preferences: ts.UserPreferences | undefined): ts.CompletionInfo {
474475
return unwrapJSONCallResult(this.shim.getCompletionsAtPosition(fileName, position, preferences));

0 commit comments

Comments
 (0)