Refactor
This commit is contained in:
Dominik Werder
173
commonio/src/commonio.rs
Normal file
173
commonio/src/commonio.rs
Normal file
@@ -0,0 +1,173 @@
|
||||
pub mod ringbuf;
|
||||
|
||||
use async_channel::Sender;
|
||||
use err::Error;
|
||||
use futures_util::StreamExt;
|
||||
use items::eventsitem::EventsItem;
|
||||
use items::{Sitemty, StatsItem, StreamItem};
|
||||
use netpod::log::*;
|
||||
use netpod::{DiskStats, OpenStats, ReadExactStats, ReadStats, SeekStats};
|
||||
use std::fmt;
|
||||
use std::io::{self, SeekFrom};
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::Instant;
|
||||
use tokio::fs::{File, OpenOptions};
|
||||
use tokio::io::{AsyncReadExt, AsyncSeekExt};
|
||||
|
||||
const LOG_IO: bool = true;
|
||||
const STATS_IO: bool = true;
|
||||
|
||||
pub struct StatsChannel {
|
||||
chn: Sender<Sitemty<EventsItem>>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for StatsChannel {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
f.debug_struct("StatsChannel").finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl StatsChannel {
|
||||
pub fn new(chn: Sender<Sitemty<EventsItem>>) -> Self {
|
||||
Self { chn }
|
||||
}
|
||||
|
||||
pub fn dummy() -> Self {
|
||||
let (tx, rx) = async_channel::bounded(2);
|
||||
taskrun::spawn(async move {
|
||||
let mut rx = rx;
|
||||
while let Some(_) = rx.next().await {}
|
||||
});
|
||||
Self::new(tx)
|
||||
}
|
||||
|
||||
pub async fn send(&self, item: StatsItem) -> Result<(), Error> {
|
||||
Ok(self.chn.send(Ok(StreamItem::Stats(item))).await?)
|
||||
}
|
||||
}
|
||||
|
||||
impl Clone for StatsChannel {
|
||||
fn clone(&self) -> Self {
|
||||
Self { chn: self.chn.clone() }
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
struct ReadExactWrap<'a> {
|
||||
fut: &'a mut dyn Future<Output = io::Result<usize>>,
|
||||
}
|
||||
|
||||
trait TimedIo {
|
||||
fn read_exact<'a, F>(&'a mut self, buf: &'a mut [u8]) -> ReadExactWrap
|
||||
where
|
||||
Self: Unpin;
|
||||
}
|
||||
|
||||
impl TimedIo for File {
|
||||
fn read_exact<'a, F>(&'a mut self, buf: &'a mut [u8]) -> ReadExactWrap
|
||||
where
|
||||
Self: Unpin,
|
||||
{
|
||||
let fut = tokio::io::AsyncReadExt::read_exact(self, buf);
|
||||
ReadExactWrap { fut: Box::pin(fut) }
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
static CHANNEL_SEND_ERROR: AtomicUsize = AtomicUsize::new(0);
|
||||
|
||||
fn channel_send_error() {
|
||||
let c = CHANNEL_SEND_ERROR.fetch_add(1, Ordering::AcqRel);
|
||||
if c < 10 {
|
||||
error!("CHANNEL_SEND_ERROR {}", c);
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn open_read(path: PathBuf, stats: &StatsChannel) -> io::Result<File> {
|
||||
let ts1 = Instant::now();
|
||||
let res = OpenOptions::new().read(true).open(path).await;
|
||||
let ts2 = Instant::now();
|
||||
let dt = ts2.duration_since(ts1);
|
||||
if LOG_IO {
|
||||
let dt = dt.as_secs_f64() * 1e3;
|
||||
debug!("timed open_read dt: {:.3} ms", dt);
|
||||
}
|
||||
if STATS_IO {
|
||||
if let Err(_) = stats
|
||||
.send(StatsItem::DiskStats(DiskStats::OpenStats(OpenStats::new(
|
||||
ts2.duration_since(ts1),
|
||||
))))
|
||||
.await
|
||||
{
|
||||
channel_send_error();
|
||||
}
|
||||
}
|
||||
res
|
||||
}
|
||||
|
||||
pub async fn seek(file: &mut File, pos: SeekFrom, stats: &StatsChannel) -> io::Result<u64> {
|
||||
let ts1 = Instant::now();
|
||||
let res = file.seek(pos).await;
|
||||
let ts2 = Instant::now();
|
||||
let dt = ts2.duration_since(ts1);
|
||||
if LOG_IO {
|
||||
let dt = dt.as_secs_f64() * 1e3;
|
||||
debug!("timed seek dt: {:.3} ms", dt);
|
||||
}
|
||||
if STATS_IO {
|
||||
if let Err(_) = stats
|
||||
.send(StatsItem::DiskStats(DiskStats::SeekStats(SeekStats::new(
|
||||
ts2.duration_since(ts1),
|
||||
))))
|
||||
.await
|
||||
{
|
||||
channel_send_error();
|
||||
}
|
||||
}
|
||||
res
|
||||
}
|
||||
|
||||
pub async fn read(file: &mut File, buf: &mut [u8], stats: &StatsChannel) -> io::Result<usize> {
|
||||
let ts1 = Instant::now();
|
||||
let res = file.read(buf).await;
|
||||
let ts2 = Instant::now();
|
||||
let dt = ts2.duration_since(ts1);
|
||||
if LOG_IO {
|
||||
let dt = dt.as_secs_f64() * 1e3;
|
||||
debug!("timed read dt: {:.3} ms res: {:?}", dt, res);
|
||||
}
|
||||
if STATS_IO {
|
||||
if let Err(_) = stats
|
||||
.send(StatsItem::DiskStats(DiskStats::ReadStats(ReadStats::new(
|
||||
ts2.duration_since(ts1),
|
||||
))))
|
||||
.await
|
||||
{
|
||||
channel_send_error();
|
||||
}
|
||||
}
|
||||
res
|
||||
}
|
||||
|
||||
pub async fn read_exact(file: &mut File, buf: &mut [u8], stats: &StatsChannel) -> io::Result<usize> {
|
||||
let ts1 = Instant::now();
|
||||
let res = file.read_exact(buf).await;
|
||||
let ts2 = Instant::now();
|
||||
let dt = ts2.duration_since(ts1);
|
||||
if LOG_IO {
|
||||
let dt = dt.as_secs_f64() * 1e3;
|
||||
debug!("timed read_exact dt: {:.3} ms res: {:?}", dt, res);
|
||||
}
|
||||
if STATS_IO {
|
||||
if let Err(_) = stats
|
||||
.send(StatsItem::DiskStats(DiskStats::ReadExactStats(ReadExactStats::new(
|
||||
ts2.duration_since(ts1),
|
||||
))))
|
||||
.await
|
||||
{
|
||||
channel_send_error();
|
||||
};
|
||||
}
|
||||
res
|
||||
}
|
||||
154
commonio/src/ringbuf.rs
Normal file
154
commonio/src/ringbuf.rs
Normal file
@@ -0,0 +1,154 @@
|
||||
use crate::{read, seek, StatsChannel};
|
||||
use err::Error;
|
||||
use netpod::log::*;
|
||||
use std::fmt;
|
||||
use std::{borrow::BorrowMut, io::SeekFrom};
|
||||
use tokio::fs::File;
|
||||
|
||||
pub struct RingBuf<F> {
|
||||
file: Option<F>,
|
||||
buf: Vec<u8>,
|
||||
abs: usize,
|
||||
wp: usize,
|
||||
rp: usize,
|
||||
stats: StatsChannel,
|
||||
seek_request: u64,
|
||||
seek_done: u64,
|
||||
read_done: u64,
|
||||
small_pos: u64,
|
||||
small_neg: u64,
|
||||
bytes_read: u64,
|
||||
}
|
||||
|
||||
impl<F> RingBuf<F>
|
||||
where
|
||||
F: BorrowMut<File>,
|
||||
{
|
||||
pub async fn new(file: F, pos: u64, stats: StatsChannel) -> Result<Self, Error> {
|
||||
let mut ret = Self {
|
||||
file: Some(file),
|
||||
buf: vec![0; 1024 * 1024],
|
||||
abs: usize::MAX,
|
||||
wp: 0,
|
||||
rp: 0,
|
||||
stats,
|
||||
seek_request: 0,
|
||||
seek_done: 0,
|
||||
read_done: 0,
|
||||
small_pos: 0,
|
||||
small_neg: 0,
|
||||
bytes_read: 0,
|
||||
};
|
||||
ret.seek(pos).await?;
|
||||
Ok(ret)
|
||||
}
|
||||
|
||||
pub fn into_file(mut self) -> F {
|
||||
self.file.take().unwrap()
|
||||
}
|
||||
|
||||
pub fn len(&self) -> usize {
|
||||
self.wp - self.rp
|
||||
}
|
||||
|
||||
pub fn adv(&mut self, n: usize) {
|
||||
self.rp += n;
|
||||
}
|
||||
|
||||
pub fn data(&self) -> &[u8] {
|
||||
&self.buf[self.rp..self.wp]
|
||||
}
|
||||
|
||||
async fn fill(&mut self) -> Result<usize, Error> {
|
||||
if self.rp == self.wp {
|
||||
if self.rp != 0 {
|
||||
self.wp = 0;
|
||||
self.rp = 0;
|
||||
}
|
||||
} else {
|
||||
unsafe {
|
||||
std::ptr::copy::<u8>(&self.buf[self.rp], &mut self.buf[0], self.len());
|
||||
self.wp -= self.rp;
|
||||
self.rp = 0;
|
||||
}
|
||||
}
|
||||
let max = (self.buf.len() - self.wp).min(1024 * 8) + self.wp;
|
||||
let n = read(
|
||||
self.file.as_mut().unwrap().borrow_mut(),
|
||||
&mut self.buf[self.wp..max],
|
||||
&self.stats,
|
||||
)
|
||||
.await?;
|
||||
self.wp += n;
|
||||
self.read_done += 1;
|
||||
self.bytes_read += n as u64;
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
pub async fn fill_min(&mut self, min: usize) -> Result<usize, Error> {
|
||||
let len = self.len();
|
||||
while self.len() < min {
|
||||
let n = self.fill().await?;
|
||||
if n == 0 {
|
||||
return Err(Error::with_msg_no_trace(format!("fill_min can not read min {}", min)));
|
||||
}
|
||||
}
|
||||
Ok(self.len() - len)
|
||||
}
|
||||
|
||||
pub async fn seek(&mut self, pos: u64) -> Result<u64, Error> {
|
||||
let dp = pos as i64 - self.rp_abs() as i64;
|
||||
if dp < 0 && dp > -2048 {
|
||||
debug!("small NEG seek {}", dp);
|
||||
} else if dp == 0 {
|
||||
// TODO check callsites, some cases could be eliminated.
|
||||
//debug!("zero seek");
|
||||
return Ok(pos);
|
||||
} else if dp > 0 && dp < 2048 {
|
||||
debug!("small POS seek {}", dp);
|
||||
}
|
||||
self.abs = pos as usize;
|
||||
self.rp = 0;
|
||||
self.wp = 0;
|
||||
let ret = seek(
|
||||
self.file.as_mut().unwrap().borrow_mut(),
|
||||
SeekFrom::Start(pos),
|
||||
&self.stats,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| Error::from(e))?;
|
||||
self.seek_request += 1;
|
||||
self.seek_done += 1;
|
||||
Ok(ret)
|
||||
}
|
||||
|
||||
pub fn rp_abs(&self) -> u64 {
|
||||
self.abs as u64 + self.rp as u64
|
||||
}
|
||||
|
||||
pub fn bytes_read(&self) -> u64 {
|
||||
self.bytes_read
|
||||
}
|
||||
}
|
||||
|
||||
impl<F> fmt::Debug for RingBuf<F> {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
f.debug_struct("RingBuf")
|
||||
.field("abs", &self.abs)
|
||||
.field("wp", &self.wp)
|
||||
.field("rp", &self.rp)
|
||||
.field("seek_request", &self.seek_request)
|
||||
.field("seek_done", &self.seek_done)
|
||||
.field("read_done", &self.read_done)
|
||||
.field("small_pos", &self.small_pos)
|
||||
.field("small_neg", &self.small_neg)
|
||||
.field("bytes_read", &self.bytes_read)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl<F> Drop for RingBuf<F> {
|
||||
fn drop(&mut self) {
|
||||
trace!("Drop {:?}", self);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user