From ad89736832865aa5c2cab01ec55b252648dcb2aa Mon Sep 17 00:00:00 2001 From: Ed Page Date: Tue, 10 May 2022 12:55:55 -0500 Subject: [PATCH 1/6] refactor(parser): Clarify precedence levels --- crates/typos/src/tokens.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/typos/src/tokens.rs b/crates/typos/src/tokens.rs index 734f3a5b..9dc7eff2 100644 --- a/crates/typos/src/tokens.rs +++ b/crates/typos/src/tokens.rs @@ -183,10 +183,10 @@ mod parser { // - Make sure you always consume it terminated(uuid_literal, sep1), terminated(hash_literal, sep1), - terminated(hex_literal, sep1), - terminated(dec_literal, sep1), + terminated(base64_literal, sep1), // base64 should be quoted or something terminated(ordinal_literal, sep1), - terminated(base64_literal, sep1), + terminated(hex_literal, sep1), + terminated(dec_literal, sep1), // Allow digit-prefixed words terminated(email_literal, sep1), terminated(url_literal, sep1), terminated(css_color, sep1), From 4869764f7bb367aee8977bc35754a610956aa865 Mon Sep 17 00:00:00 2001 From: Ed Page Date: Tue, 10 May 2022 12:56:48 -0500 Subject: [PATCH 2/6] test(parser): Remove unclear test case Unsure why this case is here and it causes difficulties --- crates/typos/src/tokens.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/crates/typos/src/tokens.rs b/crates/typos/src/tokens.rs index 9dc7eff2..964cfc31 100644 --- a/crates/typos/src/tokens.rs +++ b/crates/typos/src/tokens.rs @@ -1132,12 +1132,6 @@ mod test { ("D41D8CD98F00B204E9800998ECF8427E", true), // A 31-character hexadecimal string: too short to be a hash. ("D41D8CD98F00B204E9800998ECF8427", false), - // A 40-character string, but with non-hex characters (in - // several positions.) - ("Z85865fd0412e40d041e861506bb3ac11a3a91e3", false), - ("485865fd04Z2e40d041e861506bb3ac11a3a91e3", false), - ("485865fd0412e40d041e8Z1506bb3ac11a3a91e3", false), - ("485865fd0412e40d041e861506bb3ac11a3a91eZ", false), ] { let input = format!("Hello {} World", hashlike); let mut expected: Vec = vec![ From 7e15afe81f024d948869c69381511fb3675c1b6a Mon Sep 17 00:00:00 2001 From: Ed Page Date: Tue, 10 May 2022 12:57:19 -0500 Subject: [PATCH 3/6] test(parser): Add reproduction of #481 --- crates/typos/src/tokens.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/crates/typos/src/tokens.rs b/crates/typos/src/tokens.rs index 964cfc31..ee93233f 100644 --- a/crates/typos/src/tokens.rs +++ b/crates/typos/src/tokens.rs @@ -1148,6 +1148,25 @@ mod test { } } + #[test] + fn tokenize_hash_in_mixed_path() { + let parser = TokenizerBuilder::new().build(); + + let input = " /// at /rustc/c7087fe00d2ba919df1d813c040a5d47e43b0fe7\\/src\\libstd\\rt.rs:51"; + let expected: Vec = vec![ + Identifier::new_unchecked("at", Case::None, 25), + Identifier::new_unchecked("rustc", Case::None, 29), + Identifier::new_unchecked("c7087fe00d2ba919df1d813c040a5d47e43b0fe7", Case::None, 35), // BUG: This shouldn't be here + Identifier::new_unchecked("src", Case::None, 77), + Identifier::new_unchecked("rs", Case::None, 91), + Identifier::new_unchecked("51", Case::None, 94), + ]; + let actual: Vec<_> = parser.parse_bytes(input.as_bytes()).collect(); + assert_eq!(expected, actual); + let actual: Vec<_> = parser.parse_str(input).collect(); + assert_eq!(expected, actual); + } + #[test] fn tokenize_ignore_base64_case_1() { let parser = TokenizerBuilder::new().build(); From 1720e7d65e13623d00e30b3108084304f245cf0c Mon Sep 17 00:00:00 2001 From: Ed Page Date: Tue, 10 May 2022 13:38:03 -0500 Subject: [PATCH 4/6] fix(parser): Ignore items at end of input --- crates/typos/src/tokens.rs | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/crates/typos/src/tokens.rs b/crates/typos/src/tokens.rs index ee93233f..8a6e5969 100644 --- a/crates/typos/src/tokens.rs +++ b/crates/typos/src/tokens.rs @@ -142,6 +142,7 @@ mod parser { + nom::Slice> + nom::Offset + Clone + + Default + PartialEq + std::fmt::Debug, ::Item: AsChar + Copy, @@ -172,6 +173,7 @@ mod parser { + nom::Slice> + nom::Offset + Clone + + Default + PartialEq + std::fmt::Debug, ::Item: AsChar + Copy, @@ -198,10 +200,20 @@ mod parser { fn sep1(input: T) -> IResult where - T: nom::InputTakeAtPosition + std::fmt::Debug, + T: nom::InputTakeAtPosition + + nom::InputTake + + nom::InputIter + + nom::InputLength + + nom::Slice> + + nom::Slice> + + nom::Offset + + Clone + + Default + + PartialEq + + std::fmt::Debug, ::Item: AsChar + Copy, { - take_while1(is_ignore_char)(input) + alt((take_while1(is_ignore_char), map(eof, |_| T::default())))(input) } fn other(input: T) -> IResult @@ -1159,7 +1171,6 @@ mod test { Identifier::new_unchecked("c7087fe00d2ba919df1d813c040a5d47e43b0fe7", Case::None, 35), // BUG: This shouldn't be here Identifier::new_unchecked("src", Case::None, 77), Identifier::new_unchecked("rs", Case::None, 91), - Identifier::new_unchecked("51", Case::None, 94), ]; let actual: Vec<_> = parser.parse_bytes(input.as_bytes()).collect(); assert_eq!(expected, actual); From bd5048def52fb840f403cae9a9c91c66b6f1ae2c Mon Sep 17 00:00:00 2001 From: Ed Page Date: Tue, 10 May 2022 13:50:59 -0500 Subject: [PATCH 5/6] fix(parser): Allow backslashes after ignore items To allow `\\` to start a token, we couldn't let it end a token. By switching the termiantor to a peek, we can now make it end a token **and** start a token, allowing us to work better with windows paths. Fixes #481 --- crates/typos/src/tokens.rs | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/crates/typos/src/tokens.rs b/crates/typos/src/tokens.rs index 8a6e5969..860d5f36 100644 --- a/crates/typos/src/tokens.rs +++ b/crates/typos/src/tokens.rs @@ -183,15 +183,15 @@ mod parser { // CAUTION: If adding an ignorable literal, if it doesn't start with `is_xid_continue`, // - Update `is_ignore_char` to make sure `sep1` doesn't eat it all up // - Make sure you always consume it - terminated(uuid_literal, sep1), - terminated(hash_literal, sep1), - terminated(base64_literal, sep1), // base64 should be quoted or something - terminated(ordinal_literal, sep1), - terminated(hex_literal, sep1), - terminated(dec_literal, sep1), // Allow digit-prefixed words - terminated(email_literal, sep1), - terminated(url_literal, sep1), - terminated(css_color, sep1), + terminated(uuid_literal, peek(sep1)), + terminated(hash_literal, peek(sep1)), + terminated(base64_literal, peek(sep1)), // base64 should be quoted or something + terminated(ordinal_literal, peek(sep1)), + terminated(hex_literal, peek(sep1)), + terminated(dec_literal, peek(sep1)), // Allow digit-prefixed words + terminated(email_literal, peek(sep1)), + terminated(url_literal, peek(sep1)), + terminated(css_color, peek(sep1)), c_escape, printf, other, @@ -212,8 +212,12 @@ mod parser { + PartialEq + std::fmt::Debug, ::Item: AsChar + Copy, + ::Item: AsChar + Copy, { - alt((take_while1(is_ignore_char), map(eof, |_| T::default())))(input) + alt(( + recognize(satisfy(|c| !is_xid_continue(c))), + map(eof, |_| T::default()), + ))(input) } fn other(input: T) -> IResult @@ -1167,9 +1171,7 @@ mod test { let input = " /// at /rustc/c7087fe00d2ba919df1d813c040a5d47e43b0fe7\\/src\\libstd\\rt.rs:51"; let expected: Vec = vec![ Identifier::new_unchecked("at", Case::None, 25), - Identifier::new_unchecked("rustc", Case::None, 29), - Identifier::new_unchecked("c7087fe00d2ba919df1d813c040a5d47e43b0fe7", Case::None, 35), // BUG: This shouldn't be here - Identifier::new_unchecked("src", Case::None, 77), + // `rustc...` looks like the start of a URL Identifier::new_unchecked("rs", Case::None, 91), ]; let actual: Vec<_> = parser.parse_bytes(input.as_bytes()).collect(); From fd5398316fd152fb16261da1750f230d20e215f3 Mon Sep 17 00:00:00 2001 From: Ed Page Date: Tue, 10 May 2022 13:57:54 -0500 Subject: [PATCH 6/6] fix(parser): Better short base64 detection Previously, we bailed out if the string is too short (<90) and there weren't non-alpha-base64 bytes present. What we ignored were the padding bytes. We key off of padding bytes to detect that a string is in fact base64 encoded. Like the other cases, there can be false positives but those strings should show up elsewhere or the compiler will fail. This was called out in #485 --- crates/typos/src/tokens.rs | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/crates/typos/src/tokens.rs b/crates/typos/src/tokens.rs index 860d5f36..acc4a873 100644 --- a/crates/typos/src/tokens.rs +++ b/crates/typos/src/tokens.rs @@ -407,7 +407,16 @@ mod parser { ::Item: AsChar + Copy, { let (padding, captured) = take_while1(is_base64_digit)(input.clone())?; + + const CHUNK: usize = 4; + let padding_offset = input.offset(&padding); + let mut padding_len = CHUNK - padding_offset % CHUNK; + if padding_len == CHUNK { + padding_len = 0; + } + if captured.input_len() < 90 + && padding_len == 0 && captured .iter_elements() .all(|c| !['/', '+'].contains(&c.as_char())) @@ -418,14 +427,8 @@ mod parser { ))); } - const CHUNK: usize = 4; - let padding_offset = input.offset(&padding); - let mut padding_len = CHUNK - padding_offset % CHUNK; - if padding_len == CHUNK { - padding_len = 0; - } - let (after, _) = take_while_m_n(padding_len, padding_len, is_base64_padding)(padding)?; + let after_offset = input.offset(&after); Ok(input.take_split(after_offset)) } @@ -1207,6 +1210,21 @@ mod test { assert_eq!(expected, actual); } + #[test] + fn tokenize_ignore_base64_case_3() { + let parser = TokenizerBuilder::new().build(); + + let input = r#" "integrity": "sha512-hCmlUAIlUiav8Xdqw3Io4LcpA1DOt7h3LSTAC4G6JGHFFaWzI6qvFt9oilvl8BmkbBRX1IhM90ZAmpk68zccQA==","#; + let expected: Vec = vec![ + Identifier::new_unchecked("integrity", Case::None, 8), + Identifier::new_unchecked("sha512", Case::None, 21), + ]; + let actual: Vec<_> = parser.parse_bytes(input.as_bytes()).collect(); + assert_eq!(expected, actual); + let actual: Vec<_> = parser.parse_str(input).collect(); + assert_eq!(expected, actual); + } + #[test] fn tokenize_ignore_email() { let parser = TokenizerBuilder::new().build();