Auto merge of rust-lang#118958 - c410-f3r:concat-again, r=petrochenkov

Add a new concat metavar expr

Revival of rust-lang#111930

Giving it another try now that rust-lang#117050 was merged.

With the new rules, meta-variable expressions must be referenced with a dollar sign (`$`) and this can cause misunderstands with `$concat`.

```rust
macro_rules! foo {
    ( $bar:ident ) => {
        const ${concat(VAR, bar)}: i32 = 1;
    };
}

// Will produce `VARbar` instead of `VAR_123`
foo!(_123);
```

In other words, forgetting the dollar symbol can produce undesired outputs.

cc rust-lang#29599
cc rust-lang#124225

Author: bors

compiler/rustc_expand/src/mbe/metavar_expr.rs

@@ -8,9 +8,14 @@ use rustc_session::parse::ParseSess;
 use rustc_span::symbol::Ident;
 use rustc_span::Span;
 
+pub(crate) const RAW_IDENT_ERR: &str = "`${concat(..)}` currently does not support raw identifiers";
+
 /// A meta-variable expression, for expansions based on properties of meta-variables.
-#[derive(Debug, Clone, PartialEq, Encodable, Decodable)]
+#[derive(Debug, PartialEq, Encodable, Decodable)]
 pub(crate) enum MetaVarExpr {
+    /// Unification of two or more identifiers.
+    Concat(Box<[MetaVarExprConcatElem]>),
+
     /// The number of repetitions of an identifier.
     Count(Ident, usize),
 
@@ -42,6 +47,31 @@ impl MetaVarExpr {
         check_trailing_token(&mut tts, psess)?;
         let mut iter = args.trees();
         let rslt = match ident.as_str() {
+            "concat" => {
+                let mut result = Vec::new();
+                loop {
+                    let is_var = try_eat_dollar(&mut iter);
+                    let element_ident = parse_ident(&mut iter, psess, outer_span)?;
+                    let element = if is_var {
+                        MetaVarExprConcatElem::Var(element_ident)
+                    } else {
+                        MetaVarExprConcatElem::Ident(element_ident)
+                    };
+                    result.push(element);
+                    if iter.look_ahead(0).is_none() {
+                        break;
+                    }
+                    if !try_eat_comma(&mut iter) {
+                        return Err(psess.dcx.struct_span_err(outer_span, "expected comma"));
+                    }
+                }
+                if result.len() < 2 {
+                    return Err(psess
+                        .dcx
+                        .struct_span_err(ident.span, "`concat` must have at least two elements"));
+                }
+                MetaVarExpr::Concat(result.into())
+            }
             "count" => parse_count(&mut iter, psess, ident.span)?,
             "ignore" => {
                 eat_dollar(&mut iter, psess, ident.span)?;
@@ -68,11 +98,21 @@ impl MetaVarExpr {
     pub(crate) fn ident(&self) -> Option<Ident> {
         match *self {
             MetaVarExpr::Count(ident, _) | MetaVarExpr::Ignore(ident) => Some(ident),
-            MetaVarExpr::Index(..) | MetaVarExpr::Len(..) => None,
+            MetaVarExpr::Concat { .. } | MetaVarExpr::Index(..) | MetaVarExpr::Len(..) => None,
         }
     }
 }
 
+#[derive(Debug, Decodable, Encodable, PartialEq)]
+pub(crate) enum MetaVarExprConcatElem {
+    /// There is NO preceding dollar sign, which means that this identifier should be interpreted
+    /// as a literal.
+    Ident(Ident),
+    /// There is a preceding dollar sign, which means that this identifier should be expanded
+    /// and interpreted as a variable.
+    Var(Ident),
+}
+
 // Checks if there are any remaining tokens. For example, `${ignore(ident ... a b c ...)}`
 fn check_trailing_token<'psess>(
     iter: &mut RefTokenTreeCursor<'_>,
@@ -138,26 +178,30 @@ fn parse_depth<'psess>(
 fn parse_ident<'psess>(
     iter: &mut RefTokenTreeCursor<'_>,
     psess: &'psess ParseSess,
-    span: Span,
+    fallback_span: Span,
 ) -> PResult<'psess, Ident> {
-    if let Some(tt) = iter.next()
-        && let TokenTree::Token(token, _) = tt
-    {
-        if let Some((elem, IdentIsRaw::No)) = token.ident() {
-            return Ok(elem);
+    let Some(tt) = iter.next() else {
+        return Err(psess.dcx.struct_span_err(fallback_span, "expected identifier"));
+    };
+    let TokenTree::Token(token, _) = tt else {
+        return Err(psess.dcx.struct_span_err(tt.span(), "expected identifier"));
+    };
+    if let Some((elem, is_raw)) = token.ident() {
+        if let IdentIsRaw::Yes = is_raw {
+            return Err(psess.dcx.struct_span_err(elem.span, RAW_IDENT_ERR));
         }
-        let token_str = pprust::token_to_string(token);
-        let mut err =
-            psess.dcx.struct_span_err(span, format!("expected identifier, found `{}`", &token_str));
-        err.span_suggestion(
-            token.span,
-            format!("try removing `{}`", &token_str),
-            "",
-            Applicability::MaybeIncorrect,
-        );
-        return Err(err);
+        return Ok(elem);
     }
-    Err(psess.dcx.struct_span_err(span, "expected identifier"))
+    let token_str = pprust::token_to_string(token);
+    let mut err =
+        psess.dcx.struct_span_err(token.span, format!("expected identifier, found `{token_str}`"));
+    err.span_suggestion(
+        token.span,
+        format!("try removing `{token_str}`"),
+        "",
+        Applicability::MaybeIncorrect,
+    );
+    Err(err)
 }
 
 /// Tries to move the iterator forward returning `true` if there is a comma. If not, then the
@@ -170,6 +214,17 @@ fn try_eat_comma(iter: &mut RefTokenTreeCursor<'_>) -> bool {
     false
 }
 
+/// Tries to move the iterator forward returning `true` if there is a dollar sign. If not, then the
+/// iterator is not modified and the result is `false`.
+fn try_eat_dollar(iter: &mut RefTokenTreeCursor<'_>) -> bool {
+    if let Some(TokenTree::Token(token::Token { kind: token::Dollar, .. }, _)) = iter.look_ahead(0)
+    {
+        let _ = iter.next();
+        return true;
+    }
+    false
+}
+
 /// Expects that the next item is a dollar sign.
 fn eat_dollar<'psess>(
     iter: &mut RefTokenTreeCursor<'_>,

compiler/rustc_expand/src/mbe/quoted.rs

@@ -155,6 +155,13 @@ fn maybe_emit_macro_metavar_expr_feature(features: &Features, sess: &Session, sp
     }
 }
 
+fn maybe_emit_macro_metavar_expr_concat_feature(features: &Features, sess: &Session, span: Span) {
+    if !features.macro_metavar_expr_concat {
+        let msg = "the `concat` meta-variable expression is unstable";
+        feature_err(sess, sym::macro_metavar_expr_concat, span, msg).emit();
+    }
+}
+
 /// Takes a `tokenstream::TokenTree` and returns a `self::TokenTree`. Specifically, this takes a
 /// generic `TokenTree`, such as is used in the rest of the compiler, and returns a `TokenTree`
 /// for use in parsing a macro.
@@ -217,11 +224,19 @@ fn parse_tree<'a>(
                                         return TokenTree::token(token::Dollar, dollar_span);
                                     }
                                     Ok(elem) => {
-                                        maybe_emit_macro_metavar_expr_feature(
-                                            features,
-                                            sess,
-                                            delim_span.entire(),
-                                        );
+                                        if let MetaVarExpr::Concat(_) = elem {
+                                            maybe_emit_macro_metavar_expr_concat_feature(
+                                                features,
+                                                sess,
+                                                delim_span.entire(),
+                                            );
+                                        } else {
+                                            maybe_emit_macro_metavar_expr_feature(
+                                                features,
+                                                sess,
+                                                delim_span.entire(),
+                                            );
+                                        }
                                         return TokenTree::MetaVarExpr(delim_span, elem);
                                     }
                                 }

compiler/rustc_expand/src/mbe/transcribe.rs

@@ -3,18 +3,19 @@ use crate::errors::{
     NoSyntaxVarsExprRepeat, VarStillRepeating,
 };
 use crate::mbe::macro_parser::{NamedMatch, NamedMatch::*};
+use crate::mbe::metavar_expr::{MetaVarExprConcatElem, RAW_IDENT_ERR};
 use crate::mbe::{self, KleeneOp, MetaVarExpr};
 use rustc_ast::mut_visit::{self, MutVisitor};
+use rustc_ast::token::IdentIsRaw;
 use rustc_ast::token::{self, Delimiter, Token, TokenKind};
 use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_errors::{pluralize, Diag, DiagCtxt, PResult};
 use rustc_parse::parser::ParseNtResult;
+use rustc_session::parse::ParseSess;
 use rustc_span::hygiene::{LocalExpnId, Transparency};
 use rustc_span::symbol::{sym, Ident, MacroRulesNormalizedIdent};
-use rustc_span::{with_metavar_spans, Span, SyntaxContext};
-
-use rustc_session::parse::ParseSess;
+use rustc_span::{with_metavar_spans, Span, Symbol, SyntaxContext};
 use smallvec::{smallvec, SmallVec};
 use std::mem;
 
@@ -675,6 +676,23 @@ fn transcribe_metavar_expr<'a>(
         span
     };
     match *expr {
+        MetaVarExpr::Concat(ref elements) => {
+            let mut concatenated = String::new();
+            for element in elements.into_iter() {
+                let string = match element {
+                    MetaVarExprConcatElem::Ident(ident) => ident.to_string(),
+                    MetaVarExprConcatElem::Var(ident) => extract_ident(dcx, *ident, interp)?,
+                };
+                concatenated.push_str(&string);
+            }
+            // The current implementation marks the span as coming from the macro regardless of
+            // contexts of the concatenated identifiers but this behavior may change in the
+            // future.
+            result.push(TokenTree::Token(
+                Token::from_ast_ident(Ident::new(Symbol::intern(&concatenated), visited_span())),
+                Spacing::Alone,
+            ));
+        }
         MetaVarExpr::Count(original_ident, depth) => {
             let matched = matched_from_ident(dcx, original_ident, interp)?;
             let count = count_repetitions(dcx, depth, matched, repeats, sp)?;
@@ -709,3 +727,33 @@ fn transcribe_metavar_expr<'a>(
     }
     Ok(())
 }
+
+/// Extracts an identifier that can be originated from a `$var:ident` variable or from a token tree.
+fn extract_ident<'a>(
+    dcx: &'a DiagCtxt,
+    ident: Ident,
+    interp: &FxHashMap<MacroRulesNormalizedIdent, NamedMatch>,
+) -> PResult<'a, String> {
+    if let NamedMatch::MatchedSingle(pnr) = matched_from_ident(dcx, ident, interp)? {
+        if let ParseNtResult::Ident(nt_ident, is_raw) = pnr {
+            if let IdentIsRaw::Yes = is_raw {
+                return Err(dcx.struct_span_err(ident.span, RAW_IDENT_ERR));
+            }
+            return Ok(nt_ident.to_string());
+        }
+        if let ParseNtResult::Tt(TokenTree::Token(
+            Token { kind: TokenKind::Ident(token_ident, is_raw), .. },
+            _,
+        )) = pnr
+        {
+            if let IdentIsRaw::Yes = is_raw {
+                return Err(dcx.struct_span_err(ident.span, RAW_IDENT_ERR));
+            }
+            return Ok(token_ident.to_string());
+        }
+    }
+    Err(dcx.struct_span_err(
+        ident.span,
+        "`${concat(..)}` currently only accepts identifiers or meta-variables as parameters",
+    ))
+}

compiler/rustc_feature/src/unstable.rs

@@ -516,6 +516,8 @@ declare_features! (
     (unstable, lint_reasons, "1.31.0", Some(54503)),
     /// Give access to additional metadata about declarative macro meta-variables.
     (unstable, macro_metavar_expr, "1.61.0", Some(83527)),
+    /// Provides a way to concatenate identifiers using metavariable expressions.
+    (unstable, macro_metavar_expr_concat, "CURRENT_RUSTC_VERSION", Some(124225)),
     /// Allows `#[marker]` on certain traits allowing overlapping implementations.
     (unstable, marker_trait_attr, "1.30.0", Some(29864)),
     /// Allows exhaustive pattern matching on types that contain uninhabited types in cases that are

compiler/rustc_span/src/symbol.rs

@@ -1118,6 +1118,7 @@ symbols! {
         macro_lifetime_matcher,
         macro_literal_matcher,
         macro_metavar_expr,
+        macro_metavar_expr_concat,
         macro_reexport,
         macro_use,
         macro_vis_matcher,

tests/ui/feature-gates/feature-gate-macro-metavar-expr-concat.rs

@@ -0,0 +1,9 @@
+macro_rules! join {
+    ($lhs:ident, $rhs:ident) => {
+        let ${concat($lhs, $rhs)}: () = ();
+        //~^ ERROR the `concat` meta-variable expression is unstable
+    };
+}
+
+fn main() {
+}

tests/ui/feature-gates/feature-gate-macro-metavar-expr-concat.stderr

@@ -0,0 +1,13 @@
+error[E0658]: the `concat` meta-variable expression is unstable
+  --> $DIR/feature-gate-macro-metavar-expr-concat.rs:3:14
+   |
+LL |         let ${concat($lhs, $rhs)}: () = ();
+   |              ^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: see issue #124225 <https://github.com/rust-lang/rust/issues/124225> for more information
+   = help: add `#![feature(macro_metavar_expr_concat)]` to the crate attributes to enable
+   = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0658`.

tests/ui/macros/macro-metavar-expr-concat/allowed-operations.rs

@@ -0,0 +1,58 @@
+//@ run-pass
+
+#![allow(dead_code, non_camel_case_types, non_upper_case_globals)]
+#![feature(macro_metavar_expr_concat)]
+
+macro_rules! create_things {
+    ($lhs:ident) => {
+        struct ${concat($lhs, _separated_idents_in_a_struct)} {
+            foo: i32,
+            ${concat($lhs, _separated_idents_in_a_field)}: i32,
+        }
+
+        mod ${concat($lhs, _separated_idents_in_a_module)} {
+            pub const FOO: () = ();
+        }
+
+        fn ${concat($lhs, _separated_idents_in_a_fn)}() {}
+    };
+}
+
+macro_rules! many_idents {
+    ($a:ident, $c:ident) => {
+        const ${concat($a, B, $c, D)}: i32 = 1;
+    };
+}
+
+macro_rules! valid_tts {
+    ($_0:tt, $_1:tt) => {
+        const ${concat($_0, $_1)}: i32 = 1;
+    }
+}
+
+macro_rules! without_dollar_sign_is_an_ident {
+    ($ident:ident) => {
+        const ${concat(VAR, ident)}: i32 = 1;
+        const ${concat(VAR, $ident)}: i32 = 2;
+    };
+}
+
+fn main() {
+    create_things!(behold);
+    behold_separated_idents_in_a_fn();
+    let _ = behold_separated_idents_in_a_module::FOO;
+    let _ = behold_separated_idents_in_a_struct {
+        foo: 1,
+        behold_separated_idents_in_a_field: 2,
+    };
+
+    many_idents!(A, C);
+    assert_eq!(ABCD, 1);
+
+    valid_tts!(X, YZ);
+    assert_eq!(XYZ, 1);
+
+    without_dollar_sign_is_an_ident!(_123);
+    assert_eq!(VARident, 1);
+    assert_eq!(VAR_123, 2);
+}

tests/ui/macros/macro-metavar-expr-concat/hygiene.rs

@@ -0,0 +1,13 @@
+#![feature(macro_metavar_expr_concat)]
+
+macro_rules! join {
+    ($lhs:ident, $rhs:ident) => {
+        ${concat($lhs, $rhs)}
+        //~^ ERROR cannot find value `abcdef` in this scope
+    };
+}
+
+fn main() {
+    let abcdef = 1;
+    let _another = join!(abc, def);
+}

tests/ui/macros/macro-metavar-expr-concat/hygiene.stderr

@@ -0,0 +1,14 @@
+error[E0425]: cannot find value `abcdef` in this scope
+  --> $DIR/hygiene.rs:5:10
+   |
+LL |         ${concat($lhs, $rhs)}
+   |          ^^^^^^^^^^^^^^^^^^^^ not found in this scope
+...
+LL |     let _another = join!(abc, def);
+   |                    --------------- in this macro invocation
+   |
+   = note: this error originates in the macro `join` (in Nightly builds, run with -Z macro-backtrace for more info)
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0425`.