Token nonterminals
As explained above, each of the token nonterminals defined below is an ordered choice of token-kind nonterminals.
Grammar
TOKENS_2015 = { TOKEN_2015 * }
TOKENS_2021 = { TOKEN_2021 * }
TOKENS_2024 = { TOKEN_2024 * }
TOKEN_2015 = {
Whitespace |
Line_comment |
Block_comment |
Unterminated_block_comment |
Character_literal |
Byte_literal |
String_literal |
Byte_string_literal |
Raw_string_literal |
Raw_byte_string_literal |
Unterminated_literal_2015 |
Reserved_single_quoted_literal_2015 |
Float_literal |
Reserved_float |
Integer_literal |
Lifetime_or_label |
Raw_ident |
Reserved_prefix_2015 |
Ident |
Punctuation
}
TOKEN_2021 = {
Whitespace |
Line_comment |
Block_comment |
Unterminated_block_comment |
Character_literal |
Byte_literal |
String_literal |
Byte_string_literal |
C_string_literal |
Raw_string_literal |
Raw_byte_string_literal |
Raw_c_string_literal |
Reserved_literal_2021 |
Reserved_single_quoted_literal_2021 |
Float_literal |
Reserved_float |
Integer_literal |
Raw_lifetime_or_label |
Reserved_lifetime_or_label_prefix |
Lifetime_or_label |
Raw_ident |
Reserved_prefix_2021 |
Ident |
Punctuation
}
TOKEN_2024 = {
Whitespace |
Line_comment |
Block_comment |
Unterminated_block_comment |
Character_literal |
Byte_literal |
String_literal |
Byte_string_literal |
C_string_literal |
Raw_string_literal |
Raw_byte_string_literal |
Raw_c_string_literal |
Reserved_literal_2021 |
Reserved_single_quoted_literal_2021 |
Reserved_guard |
Float_literal |
Reserved_float |
Integer_literal |
Raw_lifetime_or_label |
Reserved_lifetime_or_label_prefix |
Lifetime_or_label |
Raw_ident |
Reserved_prefix_2021 |
Ident |
Punctuation
}