Auto merge of #74846 - Aaron1011:fix/pat-token-capture, r=petrochenkov

Capture tokens for Pat used in macro_rules! argument

This extends PR #73293 to handle patterns (Pat). Unlike expressions,
patterns do not support custom attributes, so we only need to capture
tokens during macro_rules! argument parsing.

src/librustc_ast/ast.rs

@@ -550,6 +550,7 @@ pub struct Pat {
     pub id: NodeId,
     pub kind: PatKind,
     pub span: Span,
+    pub tokens: Option<TokenStream>,
 }
 
 impl Pat {
@@ -2138,6 +2139,7 @@ pub fn from_self(attrs: AttrVec, eself: ExplicitSelf, eself_ident: Ident) -> Par
                 id: DUMMY_NODE_ID,
                 kind: PatKind::Ident(BindingMode::ByValue(mutbl), eself_ident, None),
                 span,
+                tokens: None,
             }),
             span,
             ty,

src/librustc_ast/mut_visit.rs

@@ -1053,7 +1053,7 @@ pub fn noop_flat_map_foreign_item<T: MutVisitor>(
 }
 
 pub fn noop_visit_pat<T: MutVisitor>(pat: &mut P<Pat>, vis: &mut T) {
-    let Pat { id, kind, span } = pat.deref_mut();
+    let Pat { id, kind, span, tokens: _ } = pat.deref_mut();
     vis.visit_id(id);
     match kind {
         PatKind::Wild | PatKind::Rest => {}

src/librustc_expand/base.rs

@@ -551,6 +551,7 @@ fn make_pat(self: Box<Self>) -> Option<P<ast::Pat>> {
                     id: ast::DUMMY_NODE_ID,
                     span: e.span,
                     kind: PatKind::Lit(e),
+                    tokens: None,
                 }));
             }
         }
@@ -597,7 +598,7 @@ pub fn raw_expr(sp: Span, is_error: bool) -> P<ast::Expr> {
 
     /// A plain dummy pattern.
     pub fn raw_pat(sp: Span) -> ast::Pat {
-        ast::Pat { id: ast::DUMMY_NODE_ID, kind: PatKind::Wild, span: sp }
+        ast::Pat { id: ast::DUMMY_NODE_ID, kind: PatKind::Wild, span: sp, tokens: None }
     }
 
     /// A plain dummy type.

src/librustc_expand/build.rs

@@ -392,7 +392,7 @@ pub fn expr_try(&self, sp: Span, head: P<ast::Expr>) -> P<ast::Expr> {
     }
 
     pub fn pat(&self, span: Span, kind: PatKind) -> P<ast::Pat> {
-        P(ast::Pat { id: ast::DUMMY_NODE_ID, kind, span })
+        P(ast::Pat { id: ast::DUMMY_NODE_ID, kind, span, tokens: None })
     }
     pub fn pat_wild(&self, span: Span) -> P<ast::Pat> {
         self.pat(span, PatKind::Wild)

src/librustc_expand/placeholders.rs

@@ -38,7 +38,8 @@ fn mac_placeholder() -> ast::MacCall {
         })
     };
     let ty = || P(ast::Ty { id, kind: ast::TyKind::MacCall(mac_placeholder()), span });
-    let pat = || P(ast::Pat { id, kind: ast::PatKind::MacCall(mac_placeholder()), span });
+    let pat =
+        || P(ast::Pat { id, kind: ast::PatKind::MacCall(mac_placeholder()), span, tokens: None });
 
     match kind {
         AstFragmentKind::Expr => AstFragment::Expr(expr_placeholder()),
@@ -85,6 +86,7 @@ fn mac_placeholder() -> ast::MacCall {
             id,
             span,
             kind: ast::PatKind::MacCall(mac_placeholder()),
+            tokens: None,
         })),
         AstFragmentKind::Ty => {
             AstFragment::Ty(P(ast::Ty { id, span, kind: ast::TyKind::MacCall(mac_placeholder()) }))

src/librustc_parse/lib.rs

@@ -263,6 +263,7 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke
         Nonterminal::NtItem(ref item) => {
             prepend_attrs(sess, &item.attrs, item.tokens.as_ref(), span)
         }
+        Nonterminal::NtPat(ref pat) => pat.tokens.clone(),
         Nonterminal::NtIdent(ident, is_raw) => {
             Some(tokenstream::TokenTree::token(token::Ident(ident.name, is_raw), ident.span).into())
         }

src/librustc_parse/parser/diagnostics.rs

@@ -26,6 +26,7 @@ pub(super) fn dummy_arg(ident: Ident) -> Param {
         id: ast::DUMMY_NODE_ID,
         kind: PatKind::Ident(BindingMode::ByValue(Mutability::Not), ident, None),
         span: ident.span,
+        tokens: None,
     });
     let ty = Ty { kind: TyKind::Err, span: ident.span, id: ast::DUMMY_NODE_ID };
     Param {
@@ -83,7 +84,12 @@ fn to_ty(&self) -> Option<P<Ty>> {
         self.to_ty()
     }
     fn recovered(qself: Option<QSelf>, path: ast::Path) -> Self {
-        Self { span: path.span, kind: PatKind::Path(qself, path), id: ast::DUMMY_NODE_ID }
+        Self {
+            span: path.span,
+            kind: PatKind::Path(qself, path),
+            id: ast::DUMMY_NODE_ID,
+            tokens: None,
+        }
     }
 }
 
@@ -1526,7 +1532,8 @@ pub(super) fn recover_arg_parse(&mut self) -> PResult<'a, (P<ast::Pat>, P<ast::T
         .emit();
 
         // Pretend the pattern is `_`, to avoid duplicate errors from AST validation.
-        let pat = P(Pat { kind: PatKind::Wild, span: pat.span, id: ast::DUMMY_NODE_ID });
+        let pat =
+            P(Pat { kind: PatKind::Wild, span: pat.span, id: ast::DUMMY_NODE_ID, tokens: None });
         Ok((pat, ty))
     }
 

src/librustc_parse/parser/nonterminal.rs

@@ -116,7 +116,14 @@ pub fn parse_nonterminal(&mut self, kind: NonterminalKind) -> PResult<'a, Nonter
                 Some(s) => token::NtStmt(s),
                 None => return Err(self.struct_span_err(self.token.span, "expected a statement")),
             },
-            NonterminalKind::Pat => token::NtPat(self.parse_pat(None)?),
+            NonterminalKind::Pat => {
+                let (mut pat, tokens) = self.collect_tokens(|this| this.parse_pat(None))?;
+                // We have have eaten an NtPat, which could already have tokens
+                if pat.tokens.is_none() {
+                    pat.tokens = Some(tokens);
+                }
+                token::NtPat(pat)
+            }
             NonterminalKind::Expr => {
                 let (mut expr, tokens) = self.collect_tokens(|this| this.parse_expr())?;
                 // If we captured tokens during parsing (due to outer attributes),

src/librustc_parse/parser/pat.rs

@@ -1007,6 +1007,6 @@ pub(super) fn mk_pat_ident(&self, span: Span, bm: BindingMode, ident: Ident) ->
     }
 
     fn mk_pat(&self, span: Span, kind: PatKind) -> P<Pat> {
-        P(Pat { kind, span, id: ast::DUMMY_NODE_ID })
+        P(Pat { kind, span, id: ast::DUMMY_NODE_ID, tokens: None })
     }
 }

src/test/ui-fulldeps/pprust-expr-roundtrip.rs

@@ -164,6 +164,7 @@ fn iter_exprs(depth: usize, f: &mut dyn FnMut(P<Expr>)) {
                     id: DUMMY_NODE_ID,
                     kind: PatKind::Wild,
                     span: DUMMY_SP,
+                    tokens: None,
                 });
                 iter_exprs(depth - 1, &mut |e| g(ExprKind::Let(pat.clone(), e)))
             },

src/test/ui/proc-macro/capture-macro-rules-invoke.rs

 // aux-build:test-macros.rs
 // check-pass
+// compile-flags: -Z span-debug
+// normalize-stdout-test "#\d+" -> "#CTXT"
 
 extern crate test_macros;
-use test_macros::recollect;
+use test_macros::print_bang;
 
 macro_rules! use_expr {
     ($expr:expr) => {
-        recollect!($expr)
+        print_bang!($expr)
+    }
+}
+
+macro_rules! use_pat {
+    ($pat:pat) => {
+        print_bang!($pat)
     }
 }
 
@@ -17,6 +25,10 @@ impl Foo {
     fn use_self(self) {
         drop(use_expr!(self));
     }
+
+    fn with_pat(use_pat!((a, b)): (u32, u32)) {
+        println!("Args: {} {}", a, b);
+    }
 }
 
 fn main() {}

src/test/ui/proc-macro/capture-macro-rules-invoke.stdout

+PRINT-BANG INPUT (DISPLAY): self
+PRINT-BANG INPUT (DEBUG): TokenStream [
+    Group {
+        delimiter: None,
+        stream: TokenStream [
+            Ident {
+                ident: "self",
+                span: $DIR/capture-macro-rules-invoke.rs:26:24: 26:28 (#CTXT),
+            },
+        ],
+        span: $DIR/capture-macro-rules-invoke.rs:11:21: 11:26 (#CTXT),
+    },
+]
+PRINT-BANG INPUT (DISPLAY): (a, b)
+PRINT-BANG INPUT (DEBUG): TokenStream [
+    Group {
+        delimiter: None,
+        stream: TokenStream [
+            Group {
+                delimiter: Parenthesis,
+                stream: TokenStream [
+                    Ident {
+                        ident: "a",
+                        span: $DIR/capture-macro-rules-invoke.rs:29:27: 29:28 (#CTXT),
+                    },
+                    Punct {
+                        ch: ',',
+                        spacing: Alone,
+                        span: $DIR/capture-macro-rules-invoke.rs:29:28: 29:29 (#CTXT),
+                    },
+                    Ident {
+                        ident: "b",
+                        span: $DIR/capture-macro-rules-invoke.rs:29:30: 29:31 (#CTXT),
+                    },
+                ],
+                span: $DIR/capture-macro-rules-invoke.rs:29:26: 29:32 (#CTXT),
+            },
+        ],
+        span: $DIR/capture-macro-rules-invoke.rs:17:21: 17:25 (#CTXT),
+    },
+]

src/tools/clippy/clippy_lints/src/unnested_or_patterns.rs

@@ -340,6 +340,7 @@ fn take_pat(from: &mut Pat) -> Pat {
         id: DUMMY_NODE_ID,
         kind: Wild,
         span: DUMMY_SP,
+        tokens: None
     };
     mem::replace(from, dummy)
 }