Merge pull request #1283 from epage/email

fix(token): Improve email detection by prefering that over base64
This commit is contained in:
Ed Page authored and GitHub committed 2025-04-23 14:09:26 -05:00
commit aacbdb39ec
1 file changed
+7 -7
+7 -7
View File
@@ -188,15 +188,15 @@ mod parser {
// - Update `is_ignore_char` to make sure `sep1` doesn't eat it all up
// - Make sure you always consume it
terminated(uuid_literal, peek(sep1)),
terminated(hash_literal, peek(sep1)),
terminated(email_literal, peek(sep1)),
terminated(url_literal, peek(sep1)),
terminated(jwt, peek(sep1)),
terminated(base64_literal, peek(sep1)), // base64 should be quoted or something
terminated(hash_literal, peek(sep1)),
terminated(ordinal_literal, peek(sep1)),
terminated(hex_literal, peek(sep1)),
terminated(dec_literal, peek(sep1)), // Allow digit-prefixed words
terminated(email_literal, peek(sep1)),
terminated(url_literal, peek(sep1)),
terminated(css_color, peek(sep1)),
terminated(jwt, peek(sep1)),
c_escape,
printf,
other,
@@ -1653,7 +1653,7 @@ mod test {
fn tokenize_ignore_email() {
let parser = TokenizerBuilder::new().build();
let input = "Good example@example.com Bye";
let input = "Good example@example.com if=@nam iff=@nam Bye";
let actual: Vec<_> = parser.parse_bytes(input.as_bytes()).collect();
assert_data_eq!(
actual.to_debug(),
@@ -1667,7 +1667,7 @@ mod test {
Identifier {
token: "Bye",
case: None,
offset: 25,
offset: 42,
},
]
@@ -1686,7 +1686,7 @@ mod test {
Identifier {
token: "Bye",
case: None,
offset: 25,
offset: 42,
},
]