diff --git a/src/libproc_macro_plugin/qquote.rs b/src/libproc_macro_plugin/qquote.rs index 1ae906e0aa4e0..03873b20c18cb 100644 --- a/src/libproc_macro_plugin/qquote.rs +++ b/src/libproc_macro_plugin/qquote.rs @@ -60,14 +60,14 @@ pub fn qquote<'cx>(cx: &'cx mut ExtCtxt, sp: Span, tts: &[TokenTree]) struct QDelimited { delim: token::DelimToken, open_span: Span, - tts: Vec, + tts: Vec, close_span: Span, } #[derive(Debug)] -enum QTT { +enum Qtt { TT(TokenTree), - QDL(QDelimited), + Delimited(QDelimited), QIdent(TokenTree), } @@ -103,10 +103,10 @@ fn qquoter<'cx>(cx: &'cx mut ExtCtxt, ts: TokenStream) -> TokenStream { } } -fn qquote_iter<'cx>(cx: &'cx mut ExtCtxt, depth: i64, ts: TokenStream) -> (Bindings, Vec) { +fn qquote_iter<'cx>(cx: &'cx mut ExtCtxt, depth: i64, ts: TokenStream) -> (Bindings, Vec) { let mut depth = depth; let mut bindings: Bindings = Vec::new(); - let mut output: Vec = Vec::new(); + let mut output: Vec = Vec::new(); let mut iter = ts.iter(); @@ -133,10 +133,10 @@ fn qquote_iter<'cx>(cx: &'cx mut ExtCtxt, depth: i64, ts: TokenStream) -> (Bindi for b in bindings.clone() { debug!("{:?} = {}", b.0, pprust::tts_to_string(&b.1.to_tts()[..])); } - output.push(QTT::QIdent(as_tt(Token::Ident(new_id.clone())))); + output.push(Qtt::QIdent(as_tt(Token::Ident(new_id.clone())))); } else { depth = depth - 1; - output.push(QTT::TT(next.clone())); + output.push(Qtt::TT(next.clone())); } } TokenTree::Token(_, Token::Ident(id)) if is_qquote(id) => { @@ -144,21 +144,21 @@ fn qquote_iter<'cx>(cx: &'cx mut ExtCtxt, depth: i64, ts: TokenStream) -> (Bindi } TokenTree::Delimited(_, ref dl) => { let br = qquote_iter(cx, depth, TokenStream::from_tts(dl.tts.clone().to_owned())); - let mut bind_ = br.0; - let res_ = br.1; - bindings.append(&mut bind_); + let mut nested_bindings = br.0; + let nested = br.1; + bindings.append(&mut nested_bindings); let new_dl = QDelimited { delim: dl.delim, open_span: dl.open_span, - tts: res_, + tts: nested, close_span: dl.close_span, }; - output.push(QTT::QDL(new_dl)); + output.push(Qtt::Delimited(new_dl)); } t => { - output.push(QTT::TT(t)); + output.push(Qtt::TT(t)); } } } @@ -188,9 +188,9 @@ fn unravel_concats(tss: Vec) -> TokenStream { output } -/// This converts the vector of QTTs into a seet of Bindings for construction and the main +/// This converts the vector of Qtts into a set of Bindings for construction and the main /// body as a TokenStream. -fn convert_complex_tts<'cx>(cx: &'cx mut ExtCtxt, tts: Vec) -> (Bindings, TokenStream) { +fn convert_complex_tts<'cx>(cx: &'cx mut ExtCtxt, tts: Vec) -> (Bindings, TokenStream) { let mut pushes: Vec = Vec::new(); let mut bindings: Bindings = Vec::new(); @@ -203,28 +203,37 @@ fn convert_complex_tts<'cx>(cx: &'cx mut ExtCtxt, tts: Vec) -> (Bindings, T } let next = next.unwrap(); match next { - QTT::TT(TokenTree::Token(_, t)) => { + Qtt::TT(TokenTree::Token(_, t)) => { let token_out = emit_token(t); pushes.push(token_out); } // FIXME handle sequence repetition tokens - QTT::QDL(qdl) => { - debug!(" QDL: {:?} ", qdl.tts); - let new_id = Ident::with_empty_ctxt(Symbol::gensym("qdl_tmp")); - let mut cct_rec = convert_complex_tts(cx, qdl.tts); - bindings.append(&mut cct_rec.0); - bindings.push((new_id, cct_rec.1)); - - let sep = build_delim_tok(qdl.delim); - - pushes.push(build_mod_call( - vec![Ident::from_str("proc_macro_tokens"), - Ident::from_str("build"), - Ident::from_str("build_delimited")], - concat(from_tokens(vec![Token::Ident(new_id)]), concat(lex(","), sep)), - )); + Qtt::Delimited(qdl) => { + debug!(" Delimited: {:?} ", qdl.tts); + let fresh_id = Ident::with_empty_ctxt(Symbol::gensym("qdl_tmp")); + let (mut nested_bindings, nested_toks) = convert_complex_tts(cx, qdl.tts); + + let body = if nested_toks.is_empty() { + assert!(nested_bindings.is_empty()); + build_mod_call(vec![Ident::from_str("TokenStream"), + Ident::from_str("mk_empty")], + TokenStream::mk_empty()) + } else { + bindings.append(&mut nested_bindings); + bindings.push((fresh_id, nested_toks)); + TokenStream::from_tokens(vec![Token::Ident(fresh_id)]) + }; + + let delimitiers = build_delim_tok(qdl.delim); + + pushes.push(build_mod_call(vec![Ident::from_str("proc_macro_tokens"), + Ident::from_str("build"), + Ident::from_str("build_delimited")], + flatten(vec![body, + lex(","), + delimitiers].into_iter()))); } - QTT::QIdent(t) => { + Qtt::QIdent(t) => { pushes.push(TokenStream::from_tts(vec![t])); pushes.push(TokenStream::mk_empty()); } @@ -240,14 +249,8 @@ fn convert_complex_tts<'cx>(cx: &'cx mut ExtCtxt, tts: Vec) -> (Bindings, T // Utilities /// Unravels Bindings into a TokenStream of `let` declarations. -fn unravel(binds: Bindings) -> TokenStream { - let mut output = TokenStream::mk_empty(); - - for b in binds { - output = concat(output, build_let(b.0, b.1)); - } - - output +fn unravel(bindings: Bindings) -> TokenStream { + flatten(bindings.into_iter().map(|(a, b)| build_let(a, b))) } /// Checks if the Ident is `unquote`. diff --git a/src/libproc_macro_tokens/build.rs b/src/libproc_macro_tokens/build.rs index d39aba0aa7787..89c84b6bc221a 100644 --- a/src/libproc_macro_tokens/build.rs +++ b/src/libproc_macro_tokens/build.rs @@ -24,6 +24,19 @@ pub fn concat(ts1: TokenStream, ts2: TokenStream) -> TokenStream { TokenStream::concat(ts1, ts2) } +/// Flatten a sequence of TokenStreams into a single TokenStream. +pub fn flatten>(mut iter: T) -> TokenStream { + match iter.next() { + Some(mut ts) => { + for next in iter { + ts = TokenStream::concat(ts, next); + } + ts + } + None => TokenStream::mk_empty() + } +} + /// Checks if two identifiers have the same name, disregarding context. This allows us to /// fake 'reserved' keywords. // FIXME We really want `free-identifier-=?` (a la Dybvig 1993). von Tander 2007 is diff --git a/src/test/run-pass-fulldeps/auxiliary/hello_macro.rs b/src/test/run-pass-fulldeps/auxiliary/hello_macro.rs new file mode 100644 index 0000000000000..a2dbb0a96d967 --- /dev/null +++ b/src/test/run-pass-fulldeps/auxiliary/hello_macro.rs @@ -0,0 +1,35 @@ +// Copyright 2017 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +#![feature(plugin)] +#![feature(plugin_registrar)] +#![feature(rustc_private)] +#![plugin(proc_macro_plugin)] + +extern crate rustc_plugin; +extern crate proc_macro_tokens; +extern crate syntax; + +use syntax::ext::proc_macro_shim::prelude::*; +use proc_macro_tokens::prelude::*; + +use rustc_plugin::Registry; + +#[plugin_registrar] +pub fn plugin_registrar(reg: &mut Registry) { + reg.register_macro("hello", hello); +} + +// This macro is not very interesting, but it does contain delimited tokens with +// no content - `()` and `{}` - which has caused problems in the past. +fn hello<'cx>(cx: &'cx mut ExtCtxt, sp: Span, tts: &[TokenTree]) -> Box { + let output = qquote!({ fn hello() {} hello(); }); + build_block_emitter(cx, sp, output) +} diff --git a/src/test/run-pass-fulldeps/macro-quote-empty-delims.rs b/src/test/run-pass-fulldeps/macro-quote-empty-delims.rs new file mode 100644 index 0000000000000..bdbea8a419416 --- /dev/null +++ b/src/test/run-pass-fulldeps/macro-quote-empty-delims.rs @@ -0,0 +1,22 @@ +// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// Test that a macro can emit delimiters with nothing inside - `()`, `{}` + +// aux-build:hello_macro.rs +// ignore-stage1 + +#![feature(plugin)] +#![feature(rustc_private)] +#![plugin(hello_macro)] + +fn main() { + hello!(); +}