From 8c4898131f8c5e79aa3600386def478447ad7cc1 Mon Sep 17 00:00:00 2001
From: Ed Page <eopage@gmail.com>
Date: Wed, 23 Apr 2025 13:57:16 -0500
Subject: [PATCH 1/2] docs(token): Fix a bad comment

---
 crates/typos/src/tokens.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/crates/typos/src/tokens.rs b/crates/typos/src/tokens.rs
index db0bc578..c48ed431 100644
--- a/crates/typos/src/tokens.rs
+++ b/crates/typos/src/tokens.rs
@@ -185,7 +185,7 @@ mod parser {
             "ignore",
             take_many0(alt((
                 // CAUTION: If adding an ignorable literal, if it doesn't start with `is_xid_continue`,
-                // - Update `is_ignore_char` to make sure `sep1` doesn't eat it all up
+                // - Update `is_ignore_char` to make sure `other` doesn't eat it all up
                 // - Make sure you always consume it
                 terminated(uuid_literal, peek(sep1)),
                 terminated(email_literal, peek(sep1)),
@@ -647,11 +647,11 @@ mod parser {
     fn is_ignore_char(i: impl AsChar + Copy) -> bool {
         let c = i.as_char();
         !unicode_xid::UnicodeXID::is_xid_continue(c) &&
-            // See c_escape
+            // See `c_escape`
             c != '\\' &&
-            // See printf
+            // See `printf`
             c != '%' &&
-            // See css_color
+            // See `css_color`
             c != '#'
     }
 

From 400cd2fad6f58670d806c37e85739b203ee51670 Mon Sep 17 00:00:00 2001
From: Ed Page <eopage@gmail.com>
Date: Wed, 23 Apr 2025 14:09:18 -0500
Subject: [PATCH 2/2] perf(token): Reduce splitting overhead by about 10%

---
 crates/typos/src/tokens.rs | 37 ++++++++++++++++++++++++-------------
 1 file changed, 24 insertions(+), 13 deletions(-)

diff --git a/crates/typos/src/tokens.rs b/crates/typos/src/tokens.rs
index c48ed431..520dfdeb 100644
--- a/crates/typos/src/tokens.rs
+++ b/crates/typos/src/tokens.rs
@@ -187,18 +187,18 @@ mod parser {
                 // CAUTION: If adding an ignorable literal, if it doesn't start with `is_xid_continue`,
                 // - Update `is_ignore_char` to make sure `other` doesn't eat it all up
                 // - Make sure you always consume it
-                terminated(uuid_literal, peek(sep1)),
-                terminated(email_literal, peek(sep1)),
-                terminated(url_literal, peek(sep1)),
-                terminated(jwt, peek(sep1)),
-                terminated(base64_literal, peek(sep1)), // base64 should be quoted or something
-                terminated(hash_literal, peek(sep1)),
-                terminated(ordinal_literal, peek(sep1)),
-                terminated(hex_literal, peek(sep1)),
-                terminated(dec_literal, peek(sep1)), // Allow digit-prefixed words
-                terminated(css_color, peek(sep1)),
-                c_escape,
-                printf,
+                terminated(uuid_literal, sep1),
+                terminated(email_literal, sep1),
+                terminated(url_literal, sep1),
+                terminated(jwt, sep1),
+                terminated(base64_literal, sep1), // base64 should be quoted or something
+                terminated(hash_literal, sep1),
+                terminated(ordinal_literal, sep1),
+                terminated(hex_literal, sep1),
+                terminated(dec_literal, sep1), // Allow digit-prefixed words
+                terminated(css_color, sep1),
+                terminated(c_escape, sep1),
+                terminated(printf, sep1),
                 other,
             ))),
         )
@@ -212,7 +212,8 @@ mod parser {
         <T as Stream>::Token: AsChar + Copy,
     {
         alt((
-            one_of(|c| !is_xid_continue(c)).take(),
+            take_while(1.., is_ignore_char),
+            peek(one_of(|c| !is_xid_continue(c))).take(),
             eof.map(|_| <T as Stream>::Slice::default()),
         ))
         .parse_next(input)
@@ -1439,6 +1440,11 @@ mod test {
         case: None,
         offset: 25,
     },
+    Identifier {
+        token: "src",
+        case: None,
+        offset: 77,
+    },
     Identifier {
         token: "rs",
         case: None,
@@ -1458,6 +1464,11 @@ mod test {
         case: None,
         offset: 25,
     },
+    Identifier {
+        token: "src",
+        case: None,
+        offset: 77,
+    },
     Identifier {
         token: "rs",
         case: None,