Skip to content
This repository was archived by the owner on May 28, 2025. It is now read-only.

Commit b069eb7

Browse files
borsbors
committed
Auto merge of rust-lang#14777 - rust-lang:highlight, r=Veykril
Add basic support for `augmentsSyntaxTokens` and non-standard semantic token config cc rust-lang#12783 Closes rust-lang/rust-analyzer#13066
2 parents b7cdd93 + 91d5a68 commit b069eb7

File tree

7 files changed

+140
-42
lines changed

7 files changed

+140
-42
lines changed

‎crates/ide/src/syntax_highlighting.rs

Lines changed: 26 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -469,25 +469,8 @@ fn traverse(
469469
}
470470

471471
// apply config filtering
472-
match &mut highlight.tag {
473-
HlTag::StringLiteral if !config.strings => continue,
474-
// If punctuation is disabled, make the macro bang part of the macro call again.
475-
tag @ HlTag::Punctuation(HlPunct::MacroBang) => {
476-
if !config.macro_bang {
477-
*tag = HlTag::Symbol(SymbolKind::Macro);
478-
} else if !config.specialize_punctuation {
479-
*tag = HlTag::Punctuation(HlPunct::Other);
480-
}
481-
}
482-
HlTag::Punctuation(_) if !config.punctuation => continue,
483-
tag @ HlTag::Punctuation(_) if !config.specialize_punctuation => {
484-
*tag = HlTag::Punctuation(HlPunct::Other);
485-
}
486-
HlTag::Operator(_) if !config.operator && highlight.mods.is_empty() => continue,
487-
tag @ HlTag::Operator(_) if !config.specialize_operator => {
488-
*tag = HlTag::Operator(HlOperator::Other);
489-
}
490-
_ => (),
472+
if !filter_by_config(&mut highlight, config) {
473+
continue;
491474
}
492475

493476
if inside_attribute {
@@ -498,3 +481,27 @@ fn traverse(
498481
}
499482
}
500483
}
484+
485+
fn filter_by_config(highlight: &mut Highlight, config: HighlightConfig) -> bool {
486+
match &mut highlight.tag {
487+
HlTag::StringLiteral if !config.strings => return false,
488+
// If punctuation is disabled, make the macro bang part of the macro call again.
489+
tag @ HlTag::Punctuation(HlPunct::MacroBang) => {
490+
if !config.macro_bang {
491+
*tag = HlTag::Symbol(SymbolKind::Macro);
492+
} else if !config.specialize_punctuation {
493+
*tag = HlTag::Punctuation(HlPunct::Other);
494+
}
495+
}
496+
HlTag::Punctuation(_) if !config.punctuation => return false,
497+
tag @ HlTag::Punctuation(_) if !config.specialize_punctuation => {
498+
*tag = HlTag::Punctuation(HlPunct::Other);
499+
}
500+
HlTag::Operator(_) if !config.operator && highlight.mods.is_empty() => return false,
501+
tag @ HlTag::Operator(_) if !config.specialize_operator => {
502+
*tag = HlTag::Operator(HlOperator::Other);
503+
}
504+
_ => (),
505+
}
506+
true
507+
}

‎crates/rust-analyzer/src/config.rs

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -483,6 +483,8 @@ config_data! {
483483
/// When enabled, rust-analyzer will highlight rust source in doc comments as well as intra
484484
/// doc links.
485485
semanticHighlighting_doc_comment_inject_enable: bool = "true",
486+
/// Whether the server is allowed to emit non-standard tokens and modifiers.
487+
semanticHighlighting_nonStandardTokens: bool = "true",
486488
/// Use semantic tokens for operators.
487489
///
488490
/// When disabled, rust-analyzer will emit semantic tokens only for operator tokens when
@@ -1028,6 +1030,11 @@ impl Config {
10281030
.is_some()
10291031
}
10301032

1033+
pub fn semantics_tokens_augments_syntax_tokens(&self) -> bool {
1034+
try_!(self.caps.text_document.as_ref()?.semantic_tokens.as_ref()?.augments_syntax_tokens?)
1035+
.unwrap_or(false)
1036+
}
1037+
10311038
pub fn position_encoding(&self) -> PositionEncoding {
10321039
negotiated_encoding(&self.caps)
10331040
}
@@ -1459,6 +1466,10 @@ impl Config {
14591466
}
14601467
}
14611468

1469+
pub fn highlighting_non_standard_tokens(&self) -> bool {
1470+
self.data.semanticHighlighting_nonStandardTokens
1471+
}
1472+
14621473
pub fn highlighting_config(&self) -> HighlightConfig {
14631474
HighlightConfig {
14641475
strings: self.data.semanticHighlighting_strings_enable,

‎crates/rust-analyzer/src/handlers/request.rs

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1472,7 +1472,13 @@ pub(crate) fn handle_semantic_tokens_full(
14721472
snap.workspaces.is_empty() || !snap.proc_macros_loaded;
14731473

14741474
let highlights = snap.analysis.highlight(highlight_config, file_id)?;
1475-
let semantic_tokens = to_proto::semantic_tokens(&text, &line_index, highlights);
1475+
let semantic_tokens = to_proto::semantic_tokens(
1476+
&text,
1477+
&line_index,
1478+
highlights,
1479+
snap.config.semantics_tokens_augments_syntax_tokens(),
1480+
snap.config.highlighting_non_standard_tokens(),
1481+
);
14761482

14771483
// Unconditionally cache the tokens
14781484
snap.semantic_tokens_cache.lock().insert(params.text_document.uri, semantic_tokens.clone());
@@ -1496,7 +1502,13 @@ pub(crate) fn handle_semantic_tokens_full_delta(
14961502
snap.workspaces.is_empty() || !snap.proc_macros_loaded;
14971503

14981504
let highlights = snap.analysis.highlight(highlight_config, file_id)?;
1499-
let semantic_tokens = to_proto::semantic_tokens(&text, &line_index, highlights);
1505+
let semantic_tokens = to_proto::semantic_tokens(
1506+
&text,
1507+
&line_index,
1508+
highlights,
1509+
snap.config.semantics_tokens_augments_syntax_tokens(),
1510+
snap.config.highlighting_non_standard_tokens(),
1511+
);
15001512

15011513
let mut cache = snap.semantic_tokens_cache.lock();
15021514
let cached_tokens = cache.entry(params.text_document.uri).or_default();
@@ -1530,7 +1542,13 @@ pub(crate) fn handle_semantic_tokens_range(
15301542
snap.workspaces.is_empty() || !snap.proc_macros_loaded;
15311543

15321544
let highlights = snap.analysis.highlight_range(highlight_config, frange)?;
1533-
let semantic_tokens = to_proto::semantic_tokens(&text, &line_index, highlights);
1545+
let semantic_tokens = to_proto::semantic_tokens(
1546+
&text,
1547+
&line_index,
1548+
highlights,
1549+
snap.config.semantics_tokens_augments_syntax_tokens(),
1550+
snap.config.highlighting_non_standard_tokens(),
1551+
);
15341552
Ok(Some(semantic_tokens.into()))
15351553
}
15361554

‎crates/rust-analyzer/src/semantic_tokens.rs

Lines changed: 40 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ macro_rules! define_semantic_token_types {
1313
$($standard:ident),*$(,)?
1414
}
1515
custom {
16-
$(($custom:ident, $string:literal)),*$(,)?
16+
$(($custom:ident, $string:literal) $(=> $fallback:ident)?),*$(,)?
1717
}
1818

1919
) => {
@@ -24,6 +24,15 @@ macro_rules! define_semantic_token_types {
2424
$(SemanticTokenType::$standard,)*
2525
$($custom),*
2626
];
27+
28+
pub(crate) fn standard_fallback_type(token: SemanticTokenType) -> Option<SemanticTokenType> {
29+
$(
30+
if token == $custom {
31+
None $(.or(Some(SemanticTokenType::$fallback)))?
32+
} else
33+
)*
34+
{ Some(token )}
35+
}
2736
};
2837
}
2938

@@ -51,42 +60,46 @@ define_semantic_token_types![
5160

5261
custom {
5362
(ANGLE, "angle"),
54-
(ARITHMETIC, "arithmetic"),
55-
(ATTRIBUTE, "attribute"),
56-
(ATTRIBUTE_BRACKET, "attributeBracket"),
57-
(BITWISE, "bitwise"),
63+
(ARITHMETIC, "arithmetic") => OPERATOR,
64+
(ATTRIBUTE, "attribute") => DECORATOR,
65+
(ATTRIBUTE_BRACKET, "attributeBracket") => DECORATOR,
66+
(BITWISE, "bitwise") => OPERATOR,
5867
(BOOLEAN, "boolean"),
5968
(BRACE, "brace"),
6069
(BRACKET, "bracket"),
61-
(BUILTIN_ATTRIBUTE, "builtinAttribute"),
70+
(BUILTIN_ATTRIBUTE, "builtinAttribute") => DECORATOR,
6271
(BUILTIN_TYPE, "builtinType"),
63-
(CHAR, "character"),
72+
(CHAR, "character") => STRING,
6473
(COLON, "colon"),
6574
(COMMA, "comma"),
66-
(COMPARISON, "comparison"),
75+
(COMPARISON, "comparison") => OPERATOR,
6776
(CONST_PARAMETER, "constParameter"),
68-
(DERIVE, "derive"),
69-
(DERIVE_HELPER, "deriveHelper"),
77+
(DERIVE, "derive") => DECORATOR,
78+
(DERIVE_HELPER, "deriveHelper") => DECORATOR,
7079
(DOT, "dot"),
71-
(ESCAPE_SEQUENCE, "escapeSequence"),
72-
(FORMAT_SPECIFIER, "formatSpecifier"),
73-
(GENERIC, "generic"),
80+
(ESCAPE_SEQUENCE, "escapeSequence") => STRING,
81+
(FORMAT_SPECIFIER, "formatSpecifier") => STRING,
82+
(GENERIC, "generic") => TYPE_PARAMETER,
7483
(LABEL, "label"),
7584
(LIFETIME, "lifetime"),
76-
(LOGICAL, "logical"),
77-
(MACRO_BANG, "macroBang"),
85+
(LOGICAL, "logical") => OPERATOR,
86+
(MACRO_BANG, "macroBang") => MACRO,
7887
(PARENTHESIS, "parenthesis"),
7988
(PUNCTUATION, "punctuation"),
80-
(SELF_KEYWORD, "selfKeyword"),
81-
(SELF_TYPE_KEYWORD, "selfTypeKeyword"),
89+
(SELF_KEYWORD, "selfKeyword") => KEYWORD,
90+
(SELF_TYPE_KEYWORD, "selfTypeKeyword") => KEYWORD,
8291
(SEMICOLON, "semicolon"),
8392
(TYPE_ALIAS, "typeAlias"),
84-
(TOOL_MODULE, "toolModule"),
93+
(TOOL_MODULE, "toolModule") => DECORATOR,
8594
(UNION, "union"),
8695
(UNRESOLVED_REFERENCE, "unresolvedReference"),
8796
}
8897
];
8998

99+
macro_rules! count_tts {
100+
() => {0usize};
101+
($_head:tt $($tail:tt)*) => {1usize + count_tts!($($tail)*)};
102+
}
90103
macro_rules! define_semantic_token_modifiers {
91104
(
92105
standard {
@@ -105,6 +118,8 @@ macro_rules! define_semantic_token_modifiers {
105118
$(SemanticTokenModifier::$standard,)*
106119
$($custom),*
107120
];
121+
122+
const LAST_STANDARD_MOD: usize = count_tts!($($standard)*);
108123
};
109124
}
110125

@@ -137,6 +152,13 @@ define_semantic_token_modifiers![
137152
#[derive(Default)]
138153
pub(crate) struct ModifierSet(pub(crate) u32);
139154

155+
impl ModifierSet {
156+
pub(crate) fn standard_fallback(&mut self) {
157+
// Remove all non standard modifiers
158+
self.0 = self.0 & !(!0u32 << LAST_STANDARD_MOD)
159+
}
160+
}
161+
140162
impl ops::BitOrAssign<SemanticTokenModifier> for ModifierSet {
141163
fn bitor_assign(&mut self, rhs: SemanticTokenModifier) {
142164
let idx = SUPPORTED_MODIFIERS.iter().position(|it| it == &rhs).unwrap();

‎crates/rust-analyzer/src/to_proto.rs

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ use crate::{
2424
line_index::{LineEndings, LineIndex, PositionEncoding},
2525
lsp_ext,
2626
lsp_utils::invalid_params_error,
27-
semantic_tokens,
27+
semantic_tokens::{self, standard_fallback_type},
2828
};
2929

3030
pub(crate) fn position(line_index: &LineIndex, offset: TextSize) -> lsp_types::Position {
@@ -586,6 +586,8 @@ pub(crate) fn semantic_tokens(
586586
text: &str,
587587
line_index: &LineIndex,
588588
highlights: Vec<HlRange>,
589+
semantics_tokens_augments_syntax_tokens: bool,
590+
non_standard_tokens: bool,
589591
) -> lsp_types::SemanticTokens {
590592
let id = TOKEN_RESULT_COUNTER.fetch_add(1, Ordering::SeqCst).to_string();
591593
let mut builder = semantic_tokens::SemanticTokensBuilder::new(id);
@@ -595,7 +597,35 @@ pub(crate) fn semantic_tokens(
595597
continue;
596598
}
597599

598-
let (ty, mods) = semantic_token_type_and_modifiers(highlight_range.highlight);
600+
if semantics_tokens_augments_syntax_tokens {
601+
match highlight_range.highlight.tag {
602+
HlTag::BoolLiteral
603+
| HlTag::ByteLiteral
604+
| HlTag::CharLiteral
605+
| HlTag::Comment
606+
| HlTag::Keyword
607+
| HlTag::NumericLiteral
608+
| HlTag::Operator(_)
609+
| HlTag::Punctuation(_)
610+
| HlTag::StringLiteral
611+
| HlTag::None
612+
if highlight_range.highlight.mods.is_empty() =>
613+
{
614+
continue
615+
}
616+
_ => (),
617+
}
618+
}
619+
620+
let (mut ty, mut mods) = semantic_token_type_and_modifiers(highlight_range.highlight);
621+
622+
if !non_standard_tokens {
623+
ty = match standard_fallback_type(ty) {
624+
Some(ty) => ty,
625+
None => continue,
626+
};
627+
mods.standard_fallback();
628+
}
599629
let token_index = semantic_tokens::type_index(ty);
600630
let modifier_bitset = mods.0;
601631

‎docs/user/generated_config.adoc

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -753,6 +753,11 @@ Inject additional highlighting into doc comments.
753753
When enabled, rust-analyzer will highlight rust source in doc comments as well as intra
754754
doc links.
755755
--
756+
[[rust-analyzer.semanticHighlighting.nonStandardTokens]]rust-analyzer.semanticHighlighting.nonStandardTokens (default: `true`)::
757+
+
758+
--
759+
Whether the server is allowed to emit non-standard tokens and modifiers.
760+
--
756761
[[rust-analyzer.semanticHighlighting.operator.enable]]rust-analyzer.semanticHighlighting.operator.enable (default: `true`)::
757762
+
758763
--

‎editors/code/package.json

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1395,6 +1395,11 @@
13951395
"default": true,
13961396
"type": "boolean"
13971397
},
1398+
"rust-analyzer.semanticHighlighting.nonStandardTokens": {
1399+
"markdownDescription": "Whether the server is allowed to emit non-standard tokens and modifiers.",
1400+
"default": true,
1401+
"type": "boolean"
1402+
},
13981403
"rust-analyzer.semanticHighlighting.operator.enable": {
13991404
"markdownDescription": "Use semantic tokens for operators.\n\nWhen disabled, rust-analyzer will emit semantic tokens only for operator tokens when\nthey are tagged with modifiers.",
14001405
"default": true,

0 commit comments

Comments
 (0)