From 3fc6089660ec51cab1de9ccb4296803cddc30863 Mon Sep 17 00:00:00 2001
From: "Neubauer, Sebastian" <Sebastian.Neubauer@amd.com>
Date: Mon, 15 Nov 2021 11:31:40 +0100
Subject: [PATCH] fix: Fix multiple escape sequences

If escape sequences follow straight after each other, there is no
delimiter in-between.
In such a case, parsing previously stopped and did not find any
typos further in the file.
---
 crates/typos/src/tokens.rs      | 38 +++++++++++++++++++++++++++++++--
 tests/cmd/double-escaped.stdin  |  2 ++
 tests/cmd/double-escaped.stdout |  2 ++
 tests/cmd/double-escaped.toml   |  3 +++
 4 files changed, 43 insertions(+), 2 deletions(-)
 create mode 100644 tests/cmd/double-escaped.stdin
 create mode 100644 tests/cmd/double-escaped.stdout
 create mode 100644 tests/cmd/double-escaped.toml

diff --git a/crates/typos/src/tokens.rs b/crates/typos/src/tokens.rs
index 426c2486..20750fdd 100644
--- a/crates/typos/src/tokens.rs
+++ b/crates/typos/src/tokens.rs
@@ -190,8 +190,8 @@ mod parser {
             terminated(base64_literal, sep1),
             terminated(email_literal, sep1),
             terminated(url_literal, sep1),
-            terminated(c_escape, sep1),
-            terminated(printf, sep1),
+            c_escape,
+            printf,
             sep1,
         )))(input)
     }
@@ -410,6 +410,10 @@ mod parser {
         <T as nom::InputTakeAtPosition>::Item: AsChar + Copy,
         <T as nom::InputIter>::Item: AsChar + Copy,
     {
+        // We don't know whether the string we are parsing is a literal string (no escaping) or
+        // regular string that does escaping. The escaped letter might be part of a word, or it
+        // might not be. Rather than guess and be wrong part of the time and correct people's words
+        // incorrectly, we opt for just not evaluating it at all.
         preceded(take_while1(is_escape), take_while(is_xid_continue))(input)
     }
 
@@ -1103,6 +1107,36 @@ mod test {
         assert_eq!(expected, actual);
     }
 
+    #[test]
+    fn tokenize_double_escape() {
+        let parser = TokenizerBuilder::new().build();
+
+        let input = "Hello \\n\\n World";
+        let expected: Vec<Identifier> = vec![
+            Identifier::new_unchecked("Hello", Case::None, 0),
+            Identifier::new_unchecked("World", Case::None, 11),
+        ];
+        let actual: Vec<_> = parser.parse_bytes(input.as_bytes()).collect();
+        assert_eq!(expected, actual);
+        let actual: Vec<_> = parser.parse_str(input).collect();
+        assert_eq!(expected, actual);
+    }
+
+    #[test]
+    fn tokenize_ignore_escape() {
+        let parser = TokenizerBuilder::new().build();
+
+        let input = "Hello \\nanana\\nanana World";
+        let expected: Vec<Identifier> = vec![
+            Identifier::new_unchecked("Hello", Case::None, 0),
+            Identifier::new_unchecked("World", Case::None, 21),
+        ];
+        let actual: Vec<_> = parser.parse_bytes(input.as_bytes()).collect();
+        assert_eq!(expected, actual);
+        let actual: Vec<_> = parser.parse_str(input).collect();
+        assert_eq!(expected, actual);
+    }
+
     #[test]
     fn tokenize_printf() {
         let parser = TokenizerBuilder::new().build();
diff --git a/tests/cmd/double-escaped.stdin b/tests/cmd/double-escaped.stdin
new file mode 100644
index 00000000..74091245
--- /dev/null
+++ b/tests/cmd/double-escaped.stdin
@@ -0,0 +1,2 @@
+\n\n
+Destory
diff --git a/tests/cmd/double-escaped.stdout b/tests/cmd/double-escaped.stdout
new file mode 100644
index 00000000..35ace038
--- /dev/null
+++ b/tests/cmd/double-escaped.stdout
@@ -0,0 +1,2 @@
+\n\n
+Destroy
diff --git a/tests/cmd/double-escaped.toml b/tests/cmd/double-escaped.toml
new file mode 100644
index 00000000..6c0caf96
--- /dev/null
+++ b/tests/cmd/double-escaped.toml
@@ -0,0 +1,3 @@
+bin.name = "typos"
+args = "--write-changes -"
+status.code = 0