Skip to content

Commit 42f0795

Browse files
committed
Move jointness info from TokenStream to Token
1 parent b7820b2 commit 42f0795

File tree

13 files changed

+124
-121
lines changed

13 files changed

+124
-121
lines changed

src/librustc/ich/impls_syntax.rs

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -349,9 +349,18 @@ impl<'a> HashStable<StableHashingContext<'a>> for token::TokenKind {
349349
}
350350
}
351351

352+
impl<'a> HashStable<StableHashingContext<'a>> for token::IsJoint {
353+
fn hash_stable<W: StableHasherResult>(&self,
354+
hcx: &mut StableHashingContext<'a>,
355+
hasher: &mut StableHasher<W>) {
356+
mem::discriminant(self).hash_stable(hcx, hasher);
357+
}
358+
}
359+
352360
impl_stable_hash_for!(struct token::Token {
353361
kind,
354-
span
362+
span,
363+
joint
355364
});
356365

357366
impl_stable_hash_for!(enum ::syntax::ast::NestedMetaItem {

src/libsyntax/attr/mod.rs

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -478,12 +478,12 @@ impl MetaItem {
478478
let mod_sep_span = Span::new(last_pos,
479479
segment.ident.span.lo(),
480480
segment.ident.span.ctxt());
481-
idents.push(TokenTree::token(token::ModSep, mod_sep_span).into());
481+
idents.push(TokenTree::token(token::ModSep, mod_sep_span));
482482
}
483-
idents.push(TokenTree::Token(Token::from_ast_ident(segment.ident)).into());
483+
idents.push(TokenTree::Token(Token::from_ast_ident(segment.ident)));
484484
last_pos = segment.ident.span.hi();
485485
}
486-
self.node.tokens(self.span).append_to_tree_and_joint_vec(&mut idents);
486+
self.node.tokens(self.span).append_to(&mut idents);
487487
TokenStream::new(idents)
488488
}
489489

@@ -492,8 +492,8 @@ impl MetaItem {
492492
{
493493
// FIXME: Share code with `parse_path`.
494494
let path = match tokens.next() {
495-
Some(TokenTree::Token(Token { kind: kind @ token::Ident(..), span })) |
496-
Some(TokenTree::Token(Token { kind: kind @ token::ModSep, span })) => 'arm: {
495+
Some(TokenTree::Token(Token { kind: kind @ token::Ident(..), span, .. })) |
496+
Some(TokenTree::Token(Token { kind: kind @ token::ModSep, span, .. })) => 'arm: {
497497
let mut segments = if let token::Ident(name, _) = kind {
498498
if let Some(TokenTree::Token(Token { kind: token::ModSep, .. }))
499499
= tokens.peek() {
@@ -506,7 +506,7 @@ impl MetaItem {
506506
vec![PathSegment::path_root(span)]
507507
};
508508
loop {
509-
if let Some(TokenTree::Token(Token { kind: token::Ident(name, _), span }))
509+
if let Some(TokenTree::Token(Token { kind: token::Ident(name, _), span, .. }))
510510
= tokens.next() {
511511
segments.push(PathSegment::from_ident(Ident::new(name, span)));
512512
} else {
@@ -547,17 +547,17 @@ impl MetaItemKind {
547547
match *self {
548548
MetaItemKind::Word => TokenStream::empty(),
549549
MetaItemKind::NameValue(ref lit) => {
550-
let mut vec = vec![TokenTree::token(token::Eq, span).into()];
551-
lit.tokens().append_to_tree_and_joint_vec(&mut vec);
550+
let mut vec = vec![TokenTree::token(token::Eq, span)];
551+
lit.tokens().append_to(&mut vec);
552552
TokenStream::new(vec)
553553
}
554554
MetaItemKind::List(ref list) => {
555555
let mut tokens = Vec::new();
556556
for (i, item) in list.iter().enumerate() {
557557
if i > 0 {
558-
tokens.push(TokenTree::token(token::Comma, span).into());
558+
tokens.push(TokenTree::token(token::Comma, span));
559559
}
560-
item.tokens().append_to_tree_and_joint_vec(&mut tokens);
560+
item.tokens().append_to(&mut tokens);
561561
}
562562
TokenTree::Delimited(
563563
DelimSpan::from_single(span),

src/libsyntax/ext/mbe/quoted.rs

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ pub(super) fn parse(
6666
match tree {
6767
TokenTree::MetaVar(start_sp, ident) if expect_matchers => {
6868
let span = match trees.next() {
69-
Some(tokenstream::TokenTree::Token(Token { kind: token::Colon, span })) => {
69+
Some(tokenstream::TokenTree::Token(Token { kind: token::Colon, span, .. })) => {
7070
match trees.next() {
7171
Some(tokenstream::TokenTree::Token(token)) => match token.ident() {
7272
Some((kind, _)) => {
@@ -120,7 +120,9 @@ fn parse_tree(
120120
// Depending on what `tree` is, we could be parsing different parts of a macro
121121
match tree {
122122
// `tree` is a `$` token. Look at the next token in `trees`
123-
tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }) => match trees.next() {
123+
tokenstream::TokenTree::Token(Token {
124+
kind: token::Dollar, span, ..
125+
}) => match trees.next() {
124126
// `tree` is followed by a delimited set of token trees. This indicates the beginning
125127
// of a repetition sequence in the macro (e.g. `$(pat)*`).
126128
Some(tokenstream::TokenTree::Delimited(span, delim, tts)) => {

src/libsyntax/ext/mbe/transcribe.rs

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ use crate::ext::mbe;
44
use crate::ext::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, NamedMatch};
55
use crate::mut_visit::{self, MutVisitor};
66
use crate::parse::token::{self, NtTT, Token};
7-
use crate::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint};
7+
use crate::tokenstream::{DelimSpan, TokenStream, TokenTree};
88

99
use smallvec::{smallvec, SmallVec};
1010

@@ -118,7 +118,7 @@ pub(super) fn transcribe(
118118
//
119119
// Thus, if we try to pop the `result_stack` and it is empty, we have reached the top-level
120120
// again, and we are done transcribing.
121-
let mut result: Vec<TreeAndJoint> = Vec::new();
121+
let mut result: Vec<TokenTree> = Vec::new();
122122
let mut result_stack = Vec::new();
123123
let mut marker = Marker(cx.current_expansion.id, transparency);
124124

@@ -138,7 +138,7 @@ pub(super) fn transcribe(
138138
if repeat_idx < repeat_len {
139139
*idx = 0;
140140
if let Some(sep) = sep {
141-
result.push(TokenTree::Token(sep.clone()).into());
141+
result.push(TokenTree::Token(sep.clone()));
142142
}
143143
continue;
144144
}
@@ -241,11 +241,11 @@ pub(super) fn transcribe(
241241
// (e.g. `$x:tt`), but not when we are matching any other type of token
242242
// tree?
243243
if let NtTT(ref tt) = **nt {
244-
result.push(tt.clone().into());
244+
result.push(tt.clone());
245245
} else {
246246
marker.visit_span(&mut sp);
247247
let token = TokenTree::token(token::Interpolated(nt.clone()), sp);
248-
result.push(token.into());
248+
result.push(token);
249249
}
250250
} else {
251251
// We were unable to descend far enough. This is an error.
@@ -259,8 +259,8 @@ pub(super) fn transcribe(
259259
// with modified syntax context. (I believe this supports nested macros).
260260
marker.visit_span(&mut sp);
261261
marker.visit_ident(&mut ident);
262-
result.push(TokenTree::token(token::Dollar, sp).into());
263-
result.push(TokenTree::Token(Token::from_ast_ident(ident)).into());
262+
result.push(TokenTree::token(token::Dollar, sp));
263+
result.push(TokenTree::Token(Token::from_ast_ident(ident)));
264264
}
265265
}
266266

@@ -280,7 +280,7 @@ pub(super) fn transcribe(
280280
mbe::TokenTree::Token(token) => {
281281
let mut tt = TokenTree::Token(token);
282282
marker.visit_tt(&mut tt);
283-
result.push(tt.into());
283+
result.push(tt);
284284
}
285285

286286
// There should be no meta-var declarations in the invocation of a macro.

src/libsyntax/ext/proc_macro_server.rs

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ use crate::ast;
22
use crate::ext::base::ExtCtxt;
33
use crate::parse::{self, token, ParseSess};
44
use crate::parse::lexer::comments;
5-
use crate::tokenstream::{self, DelimSpan, IsJoint::*, TokenStream, TreeAndJoint};
5+
use crate::tokenstream::{self, DelimSpan, TokenStream};
66

77
use errors::Diagnostic;
88
use rustc_data_structures::sync::Lrc;
@@ -44,15 +44,14 @@ impl ToInternal<token::DelimToken> for Delimiter {
4444
}
4545
}
4646

47-
impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec<Self>)>
47+
impl FromInternal<(tokenstream::TokenTree, &'_ ParseSess, &'_ mut Vec<Self>)>
4848
for TokenTree<Group, Punct, Ident, Literal>
4949
{
50-
fn from_internal(((tree, is_joint), sess, stack): (TreeAndJoint, &ParseSess, &mut Vec<Self>))
50+
fn from_internal((tree, sess, stack): (tokenstream::TokenTree, &ParseSess, &mut Vec<Self>))
5151
-> Self {
5252
use crate::parse::token::*;
5353

54-
let joint = is_joint == Joint;
55-
let Token { kind, span } = match tree {
54+
let Token { kind, span, joint } = match tree {
5655
tokenstream::TokenTree::Delimited(span, delim, tts) => {
5756
let delimiter = Delimiter::from_internal(delim);
5857
return TokenTree::Group(Group {
@@ -63,6 +62,7 @@ impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec<Self>)>
6362
}
6463
tokenstream::TokenTree::Token(token) => token,
6564
};
65+
let joint = joint == Joint;
6666

6767
macro_rules! tt {
6868
($ty:ident { $($field:ident $(: $value:expr)*),+ $(,)? }) => (
@@ -262,8 +262,11 @@ impl ToInternal<TokenStream> for TokenTree<Group, Punct, Ident, Literal> {
262262
_ => unreachable!(),
263263
};
264264

265-
let tree = tokenstream::TokenTree::token(kind, span);
266-
TokenStream::new(vec![(tree, if joint { Joint } else { NonJoint })])
265+
let token = Token::new(kind, span)
266+
.with_joint(if joint { Joint } else { NonJoint });
267+
let tree = tokenstream::TokenTree::Token(token);
268+
269+
TokenStream::new(vec![(tree)])
267270
}
268271
}
269272

@@ -440,7 +443,7 @@ impl server::TokenStreamIter for Rustc<'_> {
440443
) -> Option<TokenTree<Self::Group, Self::Punct, Self::Ident, Self::Literal>> {
441444
loop {
442445
let tree = iter.stack.pop().or_else(|| {
443-
let next = iter.cursor.next_with_joint()?;
446+
let next = iter.cursor.next()?;
444447
Some(TokenTree::from_internal((next, self.sess, &mut iter.stack)))
445448
})?;
446449
// HACK: The condition "dummy span + group with empty delimiter" represents an AST

src/libsyntax/mut_visit.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -611,15 +611,15 @@ pub fn noop_visit_tt<T: MutVisitor>(tt: &mut TokenTree, vis: &mut T) {
611611
pub fn noop_visit_tts<T: MutVisitor>(TokenStream(tts): &mut TokenStream, vis: &mut T) {
612612
visit_opt(tts, |tts| {
613613
let tts = Lrc::make_mut(tts);
614-
visit_vec(tts, |(tree, _is_joint)| vis.visit_tt(tree));
614+
visit_vec(tts, |tree| vis.visit_tt(tree));
615615
})
616616
}
617617

618618
// Applies ident visitor if it's an ident; applies other visits to interpolated nodes.
619619
// In practice the ident part is not actually used by specific visitors right now,
620620
// but there's a test below checking that it works.
621621
pub fn noop_visit_token<T: MutVisitor>(t: &mut Token, vis: &mut T) {
622-
let Token { kind, span } = t;
622+
let Token { kind, span, .. } = t;
623623
match kind {
624624
token::Ident(name, _) | token::Lifetime(name) => {
625625
let mut ident = Ident::new(*name, *span);

src/libsyntax/parse/lexer/tokentrees.rs

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,9 @@ use syntax_pos::Span;
22

33
use crate::print::pprust::token_to_string;
44
use crate::parse::lexer::{StringReader, UnmatchedBrace};
5-
use crate::parse::token::{self, Token};
5+
use crate::parse::token::{self, Token, IsJoint::{self, *}};
66
use crate::parse::PResult;
7-
use crate::tokenstream::{DelimSpan, IsJoint::{self, *}, TokenStream, TokenTree, TreeAndJoint};
7+
use crate::tokenstream::{DelimSpan, TokenStream, TokenTree};
88

99
impl<'a> StringReader<'a> {
1010
crate fn into_token_trees(self) -> (PResult<'a, TokenStream>, Vec<UnmatchedBrace>) {
@@ -67,7 +67,7 @@ impl<'a> TokenTreesReader<'a> {
6767
}
6868
}
6969

70-
fn parse_token_tree(&mut self) -> PResult<'a, TreeAndJoint> {
70+
fn parse_token_tree(&mut self) -> PResult<'a, TokenTree> {
7171
let sm = self.string_reader.sess.source_map();
7272
match self.token.kind {
7373
token::Eof => {
@@ -191,7 +191,7 @@ impl<'a> TokenTreesReader<'a> {
191191
delim_span,
192192
delim,
193193
tts.into()
194-
).into())
194+
))
195195
},
196196
token::CloseDelim(_) => {
197197
// An unexpected closing delimiter (i.e., there is no
@@ -204,10 +204,10 @@ impl<'a> TokenTreesReader<'a> {
204204
Err(err)
205205
},
206206
_ => {
207-
let tt = TokenTree::Token(self.token.take());
207+
let token = self.token.take();
208208
self.real_token();
209-
let is_joint = self.joint_to_prev == Joint && self.token.is_op();
210-
Ok((tt, if is_joint { Joint } else { NonJoint }))
209+
let is_joint = self.joint_to_prev == Joint && token.is_op() && self.token.is_op();
210+
Ok(TokenTree::Token(token.with_joint(if is_joint { Joint } else { NonJoint })))
211211
}
212212
}
213213
}
@@ -231,21 +231,21 @@ impl<'a> TokenTreesReader<'a> {
231231

232232
#[derive(Default)]
233233
struct TokenStreamBuilder {
234-
buf: Vec<TreeAndJoint>,
234+
buf: Vec<TokenTree>,
235235
}
236236

237237
impl TokenStreamBuilder {
238-
fn push(&mut self, (tree, joint): TreeAndJoint) {
239-
if let Some((TokenTree::Token(prev_token), Joint)) = self.buf.last() {
238+
fn push(&mut self, tree: TokenTree) {
239+
if let Some(TokenTree::Token(prev_token)) = self.buf.last() {
240240
if let TokenTree::Token(token) = &tree {
241241
if let Some(glued) = prev_token.glue(token) {
242242
self.buf.pop();
243-
self.buf.push((TokenTree::Token(glued), joint));
243+
self.buf.push(TokenTree::Token(glued));
244244
return;
245245
}
246246
}
247247
}
248-
self.buf.push((tree, joint))
248+
self.buf.push(tree)
249249
}
250250

251251
fn into_token_stream(self) -> TokenStream {

src/libsyntax/parse/parser.rs

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ use crate::print::pprust;
2424
use crate::ptr::P;
2525
use crate::source_map::{self, respan};
2626
use crate::symbol::{kw, sym, Symbol};
27-
use crate::tokenstream::{self, DelimSpan, TokenTree, TokenStream, TreeAndJoint};
27+
use crate::tokenstream::{self, DelimSpan, TokenTree, TokenStream};
2828
use crate::ThinVec;
2929

3030
use errors::{Applicability, DiagnosticId, FatalError};
@@ -195,8 +195,8 @@ struct TokenCursorFrame {
195195
/// on the parser.
196196
#[derive(Clone)]
197197
crate enum LastToken {
198-
Collecting(Vec<TreeAndJoint>),
199-
Was(Option<TreeAndJoint>),
198+
Collecting(Vec<TokenTree>),
199+
Was(Option<TokenTree>),
200200
}
201201

202202
impl TokenCursorFrame {
@@ -231,8 +231,8 @@ impl TokenCursor {
231231
};
232232

233233
match self.frame.last_token {
234-
LastToken::Collecting(ref mut v) => v.push(tree.clone().into()),
235-
LastToken::Was(ref mut t) => *t = Some(tree.clone().into()),
234+
LastToken::Collecting(ref mut v) => v.push(tree.clone()),
235+
LastToken::Was(ref mut t) => *t = Some(tree.clone()),
236236
}
237237

238238
match tree {
@@ -247,7 +247,7 @@ impl TokenCursor {
247247

248248
fn next_desugared(&mut self) -> Token {
249249
let (name, sp) = match self.next() {
250-
Token { kind: token::DocComment(name), span } => (name, span),
250+
Token { kind: token::DocComment(name), span, .. } => (name, span),
251251
tok => return tok,
252252
};
253253

@@ -1173,7 +1173,7 @@ impl<'a> Parser<'a> {
11731173
loop {
11741174
match self.token.kind {
11751175
token::Eof | token::CloseDelim(..) => break,
1176-
_ => result.push(self.parse_token_tree().into()),
1176+
_ => result.push(self.parse_token_tree()),
11771177
}
11781178
}
11791179
TokenStream::new(result)

0 commit comments

Comments
 (0)