From a8629916b473e1104920661e68425642471337aa Mon Sep 17 00:00:00 2001 From: Ed Page Date: Fri, 25 Oct 2019 14:52:37 -0600 Subject: [PATCH 1/6] refactor: Delegate to rust for IO --- typos/src/checks.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/typos/src/checks.rs b/typos/src/checks.rs index 591fdae7..02d8f19b 100644 --- a/typos/src/checks.rs +++ b/typos/src/checks.rs @@ -1,6 +1,3 @@ -use std::fs::File; -use std::io::Read; - use bstr::ByteSlice; use crate::report; @@ -122,8 +119,7 @@ impl<'d, 'p> Checks<'d, 'p> { return Ok(typos_found); } - let mut buffer = Vec::new(); - File::open(path)?.read_to_end(&mut buffer)?; + let buffer = std::fs::read(path)?; if !explicit && !self.binary && buffer.find_byte(b'\0').is_some() { let msg = report::BinaryFile { path, From ff8fce5fb62bc5cc63594cae18c38717a256497b Mon Sep 17 00:00:00 2001 From: Ed Page Date: Fri, 25 Oct 2019 14:58:08 -0600 Subject: [PATCH 2/6] fix: Don't check words if ident gets a hit --- typos/src/checks.rs | 54 +++++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/typos/src/checks.rs b/typos/src/checks.rs index 02d8f19b..618ab2f3 100644 --- a/typos/src/checks.rs +++ b/typos/src/checks.rs @@ -88,17 +88,18 @@ impl<'d, 'p> Checks<'d, 'p> { }; report(msg.into()); typos_found = true; - } - for word in ident.split() { - if let Some(correction) = self.dictionary.correct_word(word) { - let msg = report::FilenameCorrection { - path, - typo: word.token(), - correction, - non_exhaustive: (), - }; - report(msg.into()); - typos_found = true; + } else { + for word in ident.split() { + if let Some(correction) = self.dictionary.correct_word(word) { + let msg = report::FilenameCorrection { + path, + typo: word.token(), + correction, + non_exhaustive: (), + }; + report(msg.into()); + typos_found = true; + } } } } @@ -145,21 +146,22 @@ impl<'d, 'p> Checks<'d, 'p> { }; typos_found = true; report(msg.into()); - } - for word in ident.split() { - if let Some(correction) = self.dictionary.correct_word(word) { - let col_num = word.offset(); - let msg = report::Correction { - path, - line, - line_num, - col_num, - typo: word.token(), - correction, - non_exhaustive: (), - }; - typos_found = true; - report(msg.into()); + } else { + for word in ident.split() { + if let Some(correction) = self.dictionary.correct_word(word) { + let col_num = word.offset(); + let msg = report::Correction { + path, + line, + line_num, + col_num, + typo: word.token(), + correction, + non_exhaustive: (), + }; + typos_found = true; + report(msg.into()); + } } } } From c20e8f6880d398dc31759ebc3293586e23905b15 Mon Sep 17 00:00:00 2001 From: Ed Page Date: Fri, 25 Oct 2019 15:05:34 -0600 Subject: [PATCH 3/6] perf: Speed up detection of text files We reduce how much of the buffer we walk twice which should speed up large files. We still load the entire file into memory which will still hurt binary files. This is part of #34. --- typos/src/checks.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/typos/src/checks.rs b/typos/src/checks.rs index 618ab2f3..f6bc4c27 100644 --- a/typos/src/checks.rs +++ b/typos/src/checks.rs @@ -121,7 +121,8 @@ impl<'d, 'p> Checks<'d, 'p> { } let buffer = std::fs::read(path)?; - if !explicit && !self.binary && buffer.find_byte(b'\0').is_some() { + let null_max = std::cmp::min(buffer.len(), 1024); + if !explicit && !self.binary && buffer[0..null_max].find_byte(b'\0').is_some() { let msg = report::BinaryFile { path, non_exhaustive: (), From 3bbd9b1c726b082ec62fc421dbb0c3711868a767 Mon Sep 17 00:00:00 2001 From: Ed Page Date: Fri, 25 Oct 2019 15:14:03 -0600 Subject: [PATCH 4/6] refactor: Update naming --- benches/tokenize.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/benches/tokenize.rs b/benches/tokenize.rs index dea743a9..32e6a747 100644 --- a/benches/tokenize.rs +++ b/benches/tokenize.rs @@ -5,19 +5,19 @@ extern crate test; mod data; #[bench] -fn symbol_parse_empty(b: &mut test::Bencher) { +fn ident_parse_empty(b: &mut test::Bencher) { let parser = typos::tokens::Parser::new(); b.iter(|| parser.parse_bytes(data::EMPTY.as_bytes()).last()); } #[bench] -fn symbol_parse_no_tokens(b: &mut test::Bencher) { +fn ident_parse_no_tokens(b: &mut test::Bencher) { let parser = typos::tokens::Parser::new(); b.iter(|| parser.parse_bytes(data::NO_TOKENS.as_bytes()).last()); } #[bench] -fn symbol_parse_single_token(b: &mut test::Bencher) { +fn ident_parse_single_token(b: &mut test::Bencher) { let parser = typos::tokens::Parser::new(); b.iter(|| { parser.parse_bytes(data::SINGLE_TOKEN.as_bytes()).last(); @@ -25,46 +25,46 @@ fn symbol_parse_single_token(b: &mut test::Bencher) { } #[bench] -fn symbol_parse_sherlock(b: &mut test::Bencher) { +fn ident_parse_sherlock(b: &mut test::Bencher) { let parser = typos::tokens::Parser::new(); b.iter(|| parser.parse_bytes(data::SHERLOCK.as_bytes()).last()); } #[bench] -fn symbol_parse_code(b: &mut test::Bencher) { +fn ident_parse_code(b: &mut test::Bencher) { let parser = typos::tokens::Parser::new(); b.iter(|| parser.parse_bytes(data::CODE.as_bytes()).last()); } #[bench] -fn symbol_parse_corpus(b: &mut test::Bencher) { +fn ident_parse_corpus(b: &mut test::Bencher) { let parser = typos::tokens::Parser::new(); b.iter(|| parser.parse_bytes(data::CORPUS.as_bytes()).last()); } #[bench] -fn symbol_split_lowercase_short(b: &mut test::Bencher) { +fn ident_split_lowercase_short(b: &mut test::Bencher) { let input = "abcabcabcabc"; let symbol = typos::tokens::Identifier::new_unchecked(input, 0); b.iter(|| symbol.split().last()); } #[bench] -fn symbol_split_lowercase_long(b: &mut test::Bencher) { +fn ident_split_lowercase_long(b: &mut test::Bencher) { let input = "abcabcabcabc".repeat(90); let symbol = typos::tokens::Identifier::new_unchecked(&input, 0); b.iter(|| symbol.split().last()); } #[bench] -fn symbol_split_mixed_short(b: &mut test::Bencher) { +fn ident_split_mixed_short(b: &mut test::Bencher) { let input = "abcABCAbc123"; let symbol = typos::tokens::Identifier::new_unchecked(input, 0); b.iter(|| symbol.split().last()); } #[bench] -fn symbol_split_mixed_long(b: &mut test::Bencher) { +fn ident_split_mixed_long(b: &mut test::Bencher) { let input = "abcABCAbc123".repeat(90); let symbol = typos::tokens::Identifier::new_unchecked(&input, 0); b.iter(|| symbol.split().last()); From 09513fdc13da2916077500723cea34881fd69a48 Mon Sep 17 00:00:00 2001 From: Ed Page Date: Fri, 25 Oct 2019 15:17:24 -0600 Subject: [PATCH 5/6] refactor: Update naming --- benches/file.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/benches/file.rs b/benches/file.rs index c6530043..ccf3ff22 100644 --- a/benches/file.rs +++ b/benches/file.rs @@ -7,7 +7,7 @@ mod data; pub use assert_fs::prelude::*; #[bench] -fn process_empty(b: &mut test::Bencher) { +fn check_file_empty(b: &mut test::Bencher) { let temp = assert_fs::TempDir::new().unwrap(); let sample_path = temp.child("sample"); sample_path.write_str(data::EMPTY).unwrap(); @@ -21,7 +21,7 @@ fn process_empty(b: &mut test::Bencher) { } #[bench] -fn process_no_tokens(b: &mut test::Bencher) { +fn check_file_no_tokens(b: &mut test::Bencher) { let temp = assert_fs::TempDir::new().unwrap(); let sample_path = temp.child("sample"); sample_path.write_str(data::NO_TOKENS).unwrap(); @@ -35,7 +35,7 @@ fn process_no_tokens(b: &mut test::Bencher) { } #[bench] -fn process_single_token(b: &mut test::Bencher) { +fn check_file_single_token(b: &mut test::Bencher) { let temp = assert_fs::TempDir::new().unwrap(); let sample_path = temp.child("sample"); sample_path.write_str(data::SINGLE_TOKEN).unwrap(); @@ -49,7 +49,7 @@ fn process_single_token(b: &mut test::Bencher) { } #[bench] -fn process_sherlock(b: &mut test::Bencher) { +fn check_file_sherlock(b: &mut test::Bencher) { let temp = assert_fs::TempDir::new().unwrap(); let sample_path = temp.child("sample"); sample_path.write_str(data::SHERLOCK).unwrap(); @@ -63,7 +63,7 @@ fn process_sherlock(b: &mut test::Bencher) { } #[bench] -fn process_code(b: &mut test::Bencher) { +fn check_file_code(b: &mut test::Bencher) { let temp = assert_fs::TempDir::new().unwrap(); let sample_path = temp.child("sample"); sample_path.write_str(data::CODE).unwrap(); @@ -77,7 +77,7 @@ fn process_code(b: &mut test::Bencher) { } #[bench] -fn process_corpus(b: &mut test::Bencher) { +fn check_file_corpus(b: &mut test::Bencher) { let temp = assert_fs::TempDir::new().unwrap(); let sample_path = temp.child("sample"); sample_path.write_str(data::CORPUS).unwrap(); From a60ab52c5684f4ef5738e0da44c4dfc9efb589e1 Mon Sep 17 00:00:00 2001 From: Ed Page Date: Fri, 25 Oct 2019 15:48:39 -0600 Subject: [PATCH 6/6] test: Add benchmarks for real-word processing --- Cargo.lock | 1 + Cargo.toml | 1 + benches/file.rs | 291 +++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 292 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index a62f41ae..afe752a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -758,6 +758,7 @@ name = "typos-cli" version = "0.1.1" dependencies = [ "assert_fs 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)", + "bstr 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", "clap-verbosity-flag 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/Cargo.toml b/Cargo.toml index b78815ae..8733acde 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,3 +37,4 @@ env_logger = "0.6" [dev-dependencies] assert_fs = "0.11" +bstr = "0.2" diff --git a/benches/file.rs b/benches/file.rs index ccf3ff22..fc941e24 100644 --- a/benches/file.rs +++ b/benches/file.rs @@ -4,7 +4,8 @@ extern crate test; mod data; -pub use assert_fs::prelude::*; +use assert_fs::prelude::*; +use bstr::ByteSlice; #[bench] fn check_file_empty(b: &mut test::Bencher) { @@ -89,3 +90,291 @@ fn check_file_corpus(b: &mut test::Bencher) { temp.close().unwrap(); } + +#[bench] +fn read_empty(b: &mut test::Bencher) { + let temp = assert_fs::TempDir::new().unwrap(); + let sample_path = temp.child("sample"); + sample_path.write_str(data::EMPTY).unwrap(); + + b.iter(|| std::fs::read(sample_path.path())); + + temp.close().unwrap(); +} + +#[bench] +fn read_no_tokens(b: &mut test::Bencher) { + let temp = assert_fs::TempDir::new().unwrap(); + let sample_path = temp.child("sample"); + sample_path.write_str(data::NO_TOKENS).unwrap(); + + b.iter(|| std::fs::read(sample_path.path())); + + temp.close().unwrap(); +} + +#[bench] +fn read_single_token(b: &mut test::Bencher) { + let temp = assert_fs::TempDir::new().unwrap(); + let sample_path = temp.child("sample"); + sample_path.write_str(data::SINGLE_TOKEN).unwrap(); + + b.iter(|| std::fs::read(sample_path.path())); + + temp.close().unwrap(); +} + +#[bench] +fn read_sherlock(b: &mut test::Bencher) { + let temp = assert_fs::TempDir::new().unwrap(); + let sample_path = temp.child("sample"); + sample_path.write_str(data::SHERLOCK).unwrap(); + + b.iter(|| std::fs::read(sample_path.path())); + + temp.close().unwrap(); +} + +#[bench] +fn read_code(b: &mut test::Bencher) { + let temp = assert_fs::TempDir::new().unwrap(); + let sample_path = temp.child("sample"); + sample_path.write_str(data::CODE).unwrap(); + + b.iter(|| std::fs::read(sample_path.path())); + + temp.close().unwrap(); +} + +#[bench] +fn read_corpus(b: &mut test::Bencher) { + let temp = assert_fs::TempDir::new().unwrap(); + let sample_path = temp.child("sample"); + sample_path.write_str(data::CORPUS).unwrap(); + + b.iter(|| std::fs::read(sample_path.path())); + + temp.close().unwrap(); +} + +#[bench] +fn split_lines_empty(b: &mut test::Bencher) { + b.iter(|| data::EMPTY.as_bytes().lines().enumerate().last()); +} + +#[bench] +fn split_lines_no_tokens(b: &mut test::Bencher) { + b.iter(|| data::NO_TOKENS.as_bytes().lines().enumerate().last()); +} + +#[bench] +fn split_lines_single_token(b: &mut test::Bencher) { + b.iter(|| data::SINGLE_TOKEN.as_bytes().lines().enumerate().last()); +} + +#[bench] +fn split_lines_sherlock(b: &mut test::Bencher) { + b.iter(|| data::SHERLOCK.as_bytes().lines().enumerate().last()); +} + +#[bench] +fn split_lines_code(b: &mut test::Bencher) { + b.iter(|| data::CODE.as_bytes().lines().enumerate().last()); +} + +#[bench] +fn split_lines_corpus(b: &mut test::Bencher) { + b.iter(|| data::CORPUS.as_bytes().lines().enumerate().last()); +} + +#[bench] +fn parse_empty(b: &mut test::Bencher) { + let parser = typos::tokens::Parser::new(); + b.iter(|| { + data::EMPTY + .as_bytes() + .lines() + .enumerate() + .for_each(|(_idx, l)| { + parser.parse_bytes(l).last(); + () + }) + }); +} + +#[bench] +fn parse_no_tokens(b: &mut test::Bencher) { + let parser = typos::tokens::Parser::new(); + b.iter(|| { + data::NO_TOKENS + .as_bytes() + .lines() + .enumerate() + .for_each(|(_idx, l)| { + parser.parse_bytes(l).last(); + () + }) + }); +} + +#[bench] +fn parse_single_token(b: &mut test::Bencher) { + let parser = typos::tokens::Parser::new(); + b.iter(|| { + data::SINGLE_TOKEN + .as_bytes() + .lines() + .enumerate() + .for_each(|(_idx, l)| { + parser.parse_bytes(l).last(); + () + }) + }); +} + +#[bench] +fn parse_sherlock(b: &mut test::Bencher) { + let parser = typos::tokens::Parser::new(); + b.iter(|| { + data::SHERLOCK + .as_bytes() + .lines() + .enumerate() + .for_each(|(_idx, l)| { + parser.parse_bytes(l).last(); + () + }) + }); +} + +#[bench] +fn parse_code(b: &mut test::Bencher) { + let parser = typos::tokens::Parser::new(); + b.iter(|| { + data::CODE + .as_bytes() + .lines() + .enumerate() + .for_each(|(_idx, l)| { + parser.parse_bytes(l).last(); + () + }) + }); +} + +#[bench] +fn parse_corpus(b: &mut test::Bencher) { + let parser = typos::tokens::Parser::new(); + b.iter(|| { + data::CORPUS + .as_bytes() + .lines() + .enumerate() + .for_each(|(_idx, l)| { + parser.parse_bytes(l).last(); + () + }) + }); +} + +#[bench] +fn split_empty(b: &mut test::Bencher) { + let parser = typos::tokens::Parser::new(); + b.iter(|| { + data::EMPTY + .as_bytes() + .lines() + .enumerate() + .for_each(|(_idx, l)| { + parser.parse_bytes(l).for_each(|l| { + l.split().last(); + () + }) + }) + }); +} + +#[bench] +fn split_no_tokens(b: &mut test::Bencher) { + let parser = typos::tokens::Parser::new(); + b.iter(|| { + data::NO_TOKENS + .as_bytes() + .lines() + .enumerate() + .for_each(|(_idx, l)| { + parser.parse_bytes(l).for_each(|l| { + l.split().last(); + () + }) + }) + }); +} + +#[bench] +fn split_single_token(b: &mut test::Bencher) { + let parser = typos::tokens::Parser::new(); + b.iter(|| { + data::SINGLE_TOKEN + .as_bytes() + .lines() + .enumerate() + .for_each(|(_idx, l)| { + parser.parse_bytes(l).for_each(|l| { + l.split().last(); + () + }) + }) + }); +} + +#[bench] +fn split_sherlock(b: &mut test::Bencher) { + let parser = typos::tokens::Parser::new(); + b.iter(|| { + data::SHERLOCK + .as_bytes() + .lines() + .enumerate() + .for_each(|(_idx, l)| { + parser.parse_bytes(l).for_each(|l| { + l.split().last(); + () + }) + }) + }); +} + +#[bench] +fn split_code(b: &mut test::Bencher) { + let parser = typos::tokens::Parser::new(); + b.iter(|| { + data::CODE + .as_bytes() + .lines() + .enumerate() + .for_each(|(_idx, l)| { + parser.parse_bytes(l).for_each(|l| { + l.split().last(); + () + }) + }) + }); +} + +#[bench] +fn split_corpus(b: &mut test::Bencher) { + let parser = typos::tokens::Parser::new(); + b.iter(|| { + data::CORPUS + .as_bytes() + .lines() + .enumerate() + .for_each(|(_idx, l)| { + parser.parse_bytes(l).for_each(|l| { + l.split().last(); + () + }) + }) + }); +}