Refactor and add CacheUsage to query params

This commit is contained in:
Dominik Werder
2021-05-03 11:47:48 +02:00
parent a47ffc843d
commit c5b5986a28
7 changed files with 218 additions and 146 deletions
+13 -1
View File
@@ -1,5 +1,6 @@
use crate::agg::scalarbinbatch::MinMaxAvgScalarBinBatch; use crate::agg::scalarbinbatch::MinMaxAvgScalarBinBatch;
use crate::cache::pbvfs::{PreBinnedItem, PreBinnedValueFetchedStream}; use crate::cache::pbvfs::{PreBinnedItem, PreBinnedValueFetchedStream};
use crate::cache::{CacheUsage, PreBinnedQuery};
use err::Error; use err::Error;
use futures_core::Stream; use futures_core::Stream;
use futures_util::StreamExt; use futures_util::StreamExt;
@@ -20,6 +21,7 @@ impl BinnedStream {
channel: Channel, channel: Channel,
range: BinnedRange, range: BinnedRange,
agg_kind: AggKind, agg_kind: AggKind,
cache_usage: CacheUsage,
node_config: &NodeConfigCached, node_config: &NodeConfigCached,
) -> Self { ) -> Self {
let patches: Vec<_> = patch_it.collect(); let patches: Vec<_> = patch_it.collect();
@@ -31,7 +33,17 @@ impl BinnedStream {
let inp = futures_util::stream::iter(patches.into_iter()) let inp = futures_util::stream::iter(patches.into_iter())
.map({ .map({
let node_config = node_config.clone(); let node_config = node_config.clone();
move |coord| PreBinnedValueFetchedStream::new(coord, channel.clone(), agg_kind.clone(), &node_config) move |patch| {
let query = PreBinnedQuery::new(patch, channel.clone(), agg_kind.clone(), cache_usage.clone());
PreBinnedValueFetchedStream::new(&query, &node_config)
}
})
.filter_map(|k| match k {
Ok(k) => ready(Some(k)),
Err(e) => {
error!("{:?}", e);
ready(None)
}
}) })
.flatten() .flatten()
.filter_map({ .filter_map({
+116 -61
View File
@@ -15,7 +15,7 @@ use netpod::{
AggKind, BinnedRange, Channel, Cluster, NanoRange, NodeConfigCached, PreBinnedPatchCoord, PreBinnedPatchIterator, AggKind, BinnedRange, Channel, Cluster, NanoRange, NodeConfigCached, PreBinnedPatchCoord, PreBinnedPatchIterator,
PreBinnedPatchRange, ToNanos, PreBinnedPatchRange, ToNanos,
}; };
use serde::{Deserialize, Serialize}; use std::collections::BTreeMap;
use std::future::Future; use std::future::Future;
use std::pin::Pin; use std::pin::Pin;
use std::task::{Context, Poll}; use std::task::{Context, Poll};
@@ -27,54 +27,139 @@ use tracing::{debug, error, info, trace, warn};
pub mod pbv; pub mod pbv;
pub mod pbvfs; pub mod pbvfs;
#[derive(Clone, Debug, Serialize, Deserialize)] #[derive(Clone, Debug)]
pub struct Query { pub enum CacheUsage {
range: NanoRange, Use,
count: u64, Ignore,
agg_kind: AggKind, Recreate,
channel: Channel,
} }
impl Query { #[derive(Clone, Debug)]
pub struct BinnedQuery {
range: NanoRange,
bin_count: u64,
agg_kind: AggKind,
channel: Channel,
cache_usage: CacheUsage,
}
impl BinnedQuery {
pub fn from_request(req: &http::request::Parts) -> Result<Self, Error> { pub fn from_request(req: &http::request::Parts) -> Result<Self, Error> {
let params = netpod::query_params(req.uri.query()); let params = netpod::query_params(req.uri.query());
let beg_date = params let beg_date = params.get("beg_date").ok_or(Error::with_msg("missing beg_date"))?;
.get("beg_date") let end_date = params.get("end_date").ok_or(Error::with_msg("missing end_date"))?;
.ok_or_else(|| Error::with_msg("missing beg_date"))?; let ret = BinnedQuery {
let end_date = params
.get("end_date")
.ok_or_else(|| Error::with_msg("missing end_date"))?;
let ret = Query {
range: NanoRange { range: NanoRange {
beg: beg_date.parse::<DateTime<Utc>>()?.to_nanos(), beg: beg_date.parse::<DateTime<Utc>>()?.to_nanos(),
end: end_date.parse::<DateTime<Utc>>()?.to_nanos(), end: end_date.parse::<DateTime<Utc>>()?.to_nanos(),
}, },
count: params bin_count: params
.get("bin_count") .get("bin_count")
.ok_or_else(|| Error::with_msg("missing beg_date"))? .ok_or(Error::with_msg("missing bin_count"))?
.parse() .parse()
.unwrap(), .map_err(|e| Error::with_msg(format!("can not parse bin_count {:?}", e)))?,
agg_kind: AggKind::DimXBins1, agg_kind: AggKind::DimXBins1,
channel: Channel { channel: channel_from_params(&params)?,
backend: params.get("channel_backend").unwrap().into(), cache_usage: cache_usage_from_params(&params)?,
name: params.get("channel_name").unwrap().into(),
},
}; };
info!("BinnedQuery::from_request {:?}", ret);
Ok(ret) Ok(ret)
} }
} }
#[derive(Clone, Debug)]
pub struct PreBinnedQuery {
patch: PreBinnedPatchCoord,
agg_kind: AggKind,
channel: Channel,
cache_usage: CacheUsage,
}
impl PreBinnedQuery {
pub fn new(patch: PreBinnedPatchCoord, channel: Channel, agg_kind: AggKind, cache_usage: CacheUsage) -> Self {
Self {
patch,
agg_kind,
channel,
cache_usage,
}
}
pub fn from_request(req: &http::request::Parts) -> Result<Self, Error> {
let params = netpod::query_params(req.uri.query());
let patch_ix = params
.get("patch_ix")
.ok_or(Error::with_msg("missing patch_ix"))?
.parse()?;
let bin_t_len = params
.get("bin_t_len")
.ok_or(Error::with_msg("missing bin_t_len"))?
.parse()?;
let ret = PreBinnedQuery {
patch: PreBinnedPatchCoord::new(bin_t_len, patch_ix),
agg_kind: AggKind::DimXBins1,
channel: channel_from_params(&params)?,
cache_usage: cache_usage_from_params(&params)?,
};
info!("PreBinnedQuery::from_request {:?}", ret);
Ok(ret)
}
pub fn make_query_string(&self) -> String {
format!(
"{}&channel_backend={}&channel_name={}&agg_kind={:?}",
self.patch.to_url_params_strings(),
self.channel.backend,
self.channel.name,
self.agg_kind
)
}
pub fn patch(&self) -> &PreBinnedPatchCoord {
&self.patch
}
}
fn channel_from_params(params: &BTreeMap<String, String>) -> Result<Channel, Error> {
let ret = Channel {
backend: params
.get("channel_backend")
.ok_or(Error::with_msg("missing channel_backend"))?
.into(),
name: params
.get("channel_name")
.ok_or(Error::with_msg("missing channel_name"))?
.into(),
};
Ok(ret)
}
fn cache_usage_from_params(params: &BTreeMap<String, String>) -> Result<CacheUsage, Error> {
let ret = params.get("cache_usage").map_or(Ok::<_, Error>(CacheUsage::Use), |k| {
if k == "use" {
Ok(CacheUsage::Use)
} else if k == "ignore" {
Ok(CacheUsage::Ignore)
} else if k == "recreate" {
Ok(CacheUsage::Recreate)
} else {
Err(Error::with_msg(format!("unexpected cache_usage {:?}", k)))?
}
})?;
Ok(ret)
}
pub async fn binned_bytes_for_http( pub async fn binned_bytes_for_http(
node_config: &NodeConfigCached, node_config: &NodeConfigCached,
query: &Query, query: &BinnedQuery,
) -> Result<BinnedBytesForHttpStream, Error> { ) -> Result<BinnedBytesForHttpStream, Error> {
let range = &query.range; let range = &query.range;
let channel_config = read_local_config(&query.channel, &node_config.node).await?; let channel_config = read_local_config(&query.channel, &node_config.node).await?;
let entry = extract_matching_config_entry(range, &channel_config); let entry = extract_matching_config_entry(range, &channel_config);
info!("found config entry {:?}", entry); info!("found config entry {:?}", entry);
let range = BinnedRange::covering_range(range.clone(), query.count) let range = BinnedRange::covering_range(range.clone(), query.bin_count)
.ok_or(Error::with_msg(format!("BinnedRange::covering_range returned None")))?; .ok_or(Error::with_msg(format!("BinnedRange::covering_range returned None")))?;
match PreBinnedPatchRange::covering_range(query.range.clone(), query.count) { match PreBinnedPatchRange::covering_range(query.range.clone(), query.bin_count) {
Some(pre_range) => { Some(pre_range) => {
info!("Found pre_range: {:?}", pre_range); info!("Found pre_range: {:?}", pre_range);
if range.grid_spec.bin_t_len() < pre_range.grid_spec.bin_t_len() { if range.grid_spec.bin_t_len() < pre_range.grid_spec.bin_t_len() {
@@ -90,6 +175,7 @@ pub async fn binned_bytes_for_http(
query.channel.clone(), query.channel.clone(),
range, range,
query.agg_kind.clone(), query.agg_kind.clone(),
query.cache_usage.clone(),
node_config, node_config,
); );
let ret = BinnedBytesForHttpStream::new(s1); let ret = BinnedBytesForHttpStream::new(s1);
@@ -150,31 +236,6 @@ impl Stream for BinnedBytesForHttpStream {
} }
} }
#[derive(Clone, Debug)]
pub struct PreBinnedQuery {
pub patch: PreBinnedPatchCoord,
agg_kind: AggKind,
channel: Channel,
}
impl PreBinnedQuery {
pub fn from_request(req: &http::request::Parts) -> Result<Self, Error> {
let params = netpod::query_params(req.uri.query());
let patch_ix = params.get("patch_ix").unwrap().parse().unwrap();
let bin_t_len = params.get("bin_t_len").unwrap().parse().unwrap();
let ret = PreBinnedQuery {
patch: PreBinnedPatchCoord::new(bin_t_len, patch_ix),
agg_kind: AggKind::DimXBins1,
channel: Channel {
backend: params.get("channel_backend").unwrap().into(),
name: params.get("channel_name").unwrap().into(),
},
};
info!("PreBinnedQuery::from_request {:?}", ret);
Ok(ret)
}
}
// NOTE This answers a request for a single valid pre-binned patch. // NOTE This answers a request for a single valid pre-binned patch.
// A user must first make sure that the grid spec is valid, and that this node is responsible for it. // A user must first make sure that the grid spec is valid, and that this node is responsible for it.
// Otherwise it is an error. // Otherwise it is an error.
@@ -183,12 +244,7 @@ pub fn pre_binned_bytes_for_http(
query: &PreBinnedQuery, query: &PreBinnedQuery,
) -> Result<PreBinnedValueByteStream, Error> { ) -> Result<PreBinnedValueByteStream, Error> {
info!("pre_binned_bytes_for_http {:?} {:?}", query, node_config.node); info!("pre_binned_bytes_for_http {:?} {:?}", query, node_config.node);
let ret = super::cache::pbv::pre_binned_value_byte_stream_new( let ret = super::cache::pbv::pre_binned_value_byte_stream_new(query, node_config);
query.patch.clone(),
query.channel.clone(),
query.agg_kind.clone(),
node_config,
);
Ok(ret) Ok(ret)
} }
@@ -288,10 +344,9 @@ impl Stream for MergedFromRemotes {
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> { fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
use Poll::*; use Poll::*;
if self.completed { if self.completed {
panic!("MergedFromRemotes ✗✗✗✗✗✗✗✗✗✗✗✗✗✗✗✗✗✗✗✗✗✗✗✗✗✗✗✗✗✗✗✗✗✗✗✗✗✗✗✗ poll_next on completed"); panic!("MergedFromRemotes poll_next on completed");
} }
if self.errored { if self.errored {
warn!("MergedFromRemotes return None after Err");
self.completed = true; self.completed = true;
return Ready(None); return Ready(None);
} }
@@ -355,10 +410,10 @@ impl Stream for MergedFromRemotes {
} }
} }
pub struct SomeReturnThing {} pub struct BytesWrap {}
impl From<SomeReturnThing> for Bytes { impl From<BytesWrap> for Bytes {
fn from(_k: SomeReturnThing) -> Self { fn from(_k: BytesWrap) -> Self {
error!("TODO convert result to octets"); error!("TODO convert result to octets");
todo!("TODO convert result to octets") todo!("TODO convert result to octets")
} }
+61 -61
View File
@@ -1,6 +1,6 @@
use crate::agg::binnedt::IntoBinnedT; use crate::agg::binnedt::IntoBinnedT;
use crate::cache::pbvfs::{PreBinnedFrame, PreBinnedItem, PreBinnedValueFetchedStream}; use crate::cache::pbvfs::{PreBinnedFrame, PreBinnedItem, PreBinnedValueFetchedStream};
use crate::cache::{node_ix_for_patch, MergedFromRemotes}; use crate::cache::{node_ix_for_patch, MergedFromRemotes, PreBinnedQuery};
use crate::frame::makeframe::make_frame; use crate::frame::makeframe::make_frame;
use crate::raw::EventsQuery; use crate::raw::EventsQuery;
use bytes::Bytes; use bytes::Bytes;
@@ -9,11 +9,8 @@ use futures_core::Stream;
use futures_util::{FutureExt, StreamExt}; use futures_util::{FutureExt, StreamExt};
use netpod::log::*; use netpod::log::*;
use netpod::streamext::SCC; use netpod::streamext::SCC;
use netpod::{ use netpod::{BinnedRange, NodeConfigCached, PreBinnedPatchIterator, PreBinnedPatchRange};
AggKind, BinnedRange, Channel, NanoRange, NodeConfigCached, PreBinnedPatchCoord, PreBinnedPatchIterator, use std::future::{ready, Future};
PreBinnedPatchRange,
};
use std::future::Future;
use std::pin::Pin; use std::pin::Pin;
use std::task::{Context, Poll}; use std::task::{Context, Poll};
@@ -24,12 +21,10 @@ pub struct PreBinnedValueByteStreamInner {
} }
pub fn pre_binned_value_byte_stream_new( pub fn pre_binned_value_byte_stream_new(
patch: PreBinnedPatchCoord, query: &PreBinnedQuery,
channel: Channel,
agg_kind: AggKind,
node_config: &NodeConfigCached, node_config: &NodeConfigCached,
) -> PreBinnedValueByteStream { ) -> PreBinnedValueByteStream {
let s1 = PreBinnedValueStream::new(patch, channel, agg_kind, node_config); let s1 = PreBinnedValueStream::new(query.clone(), node_config);
let s2 = PreBinnedValueByteStreamInner { inp: s1 }; let s2 = PreBinnedValueByteStreamInner { inp: s1 };
SCC::new(s2) SCC::new(s2)
} }
@@ -51,9 +46,7 @@ impl Stream for PreBinnedValueByteStreamInner {
} }
pub struct PreBinnedValueStream { pub struct PreBinnedValueStream {
patch_coord: PreBinnedPatchCoord, query: PreBinnedQuery,
channel: Channel,
agg_kind: AggKind,
node_config: NodeConfigCached, node_config: NodeConfigCached,
open_check_local_file: Option<Pin<Box<dyn Future<Output = Result<tokio::fs::File, std::io::Error>> + Send>>>, open_check_local_file: Option<Pin<Box<dyn Future<Output = Result<tokio::fs::File, std::io::Error>> + Send>>>,
fut2: Option<Pin<Box<dyn Stream<Item = PreBinnedFrame> + Send>>>, fut2: Option<Pin<Box<dyn Stream<Item = PreBinnedFrame> + Send>>>,
@@ -62,18 +55,11 @@ pub struct PreBinnedValueStream {
} }
impl PreBinnedValueStream { impl PreBinnedValueStream {
pub fn new( pub fn new(query: PreBinnedQuery, node_config: &NodeConfigCached) -> Self {
patch_coord: PreBinnedPatchCoord,
channel: Channel,
agg_kind: AggKind,
node_config: &NodeConfigCached,
) -> Self {
// TODO check that we are the correct node. // TODO check that we are the correct node.
let _node_ix = node_ix_for_patch(&patch_coord, &channel, &node_config.node_config.cluster); let _node_ix = node_ix_for_patch(&query.patch, &query.channel, &node_config.node_config.cluster);
Self { Self {
patch_coord, query,
channel,
agg_kind,
node_config: node_config.clone(), node_config: node_config.clone(),
open_check_local_file: None, open_check_local_file: None,
fut2: None, fut2: None,
@@ -83,33 +69,53 @@ impl PreBinnedValueStream {
} }
fn try_setup_fetch_prebinned_higher_res(&mut self) { fn try_setup_fetch_prebinned_higher_res(&mut self) {
info!("try to find a next better granularity for {:?}", self.patch_coord); info!("try_setup_fetch_prebinned_higher_res for {:?}", self.query.patch);
let g = self.patch_coord.bin_t_len(); let g = self.query.patch.bin_t_len();
let range = NanoRange { let range = self.query.patch.patch_range();
beg: self.patch_coord.patch_beg(), match PreBinnedPatchRange::covering_range(range, self.query.patch.bin_count() + 1) {
end: self.patch_coord.patch_end(),
};
match PreBinnedPatchRange::covering_range(range, self.patch_coord.bin_count() + 1) {
Some(range) => { Some(range) => {
let h = range.grid_spec.bin_t_len(); let h = range.grid_spec.bin_t_len();
info!( info!(
"FOUND NEXT GRAN g {} h {} ratio {} mod {} {:?}", "try_setup_fetch_prebinned_higher_res found g {} h {} ratio {} mod {} {:?}",
g, g,
h, h,
g / h, g / h,
g % h, g % h,
range range,
); );
assert!(g / h > 1); if g / h <= 1 {
assert!(g / h < 200); error!("try_setup_fetch_prebinned_higher_res g {} h {}", g, h);
assert!(g % h == 0); return;
let channel = self.channel.clone(); }
let agg_kind = self.agg_kind.clone(); if g / h > 200 {
error!("try_setup_fetch_prebinned_higher_res g {} h {}", g, h);
return;
}
if g % h != 0 {
error!("try_setup_fetch_prebinned_higher_res g {} h {}", g, h);
return;
}
let node_config = self.node_config.clone(); let node_config = self.node_config.clone();
let patch_it = PreBinnedPatchIterator::from_range(range); let patch_it = PreBinnedPatchIterator::from_range(range);
let s = futures_util::stream::iter(patch_it) let s = futures_util::stream::iter(patch_it)
.map(move |coord| { .map({
PreBinnedValueFetchedStream::new(coord, channel.clone(), agg_kind.clone(), &node_config) let q2 = self.query.clone();
move |patch| {
let query = PreBinnedQuery {
patch,
channel: q2.channel.clone(),
agg_kind: q2.agg_kind.clone(),
cache_usage: q2.cache_usage.clone(),
};
PreBinnedValueFetchedStream::new(&query, &node_config)
}
})
.filter_map(|k| match k {
Ok(k) => ready(Some(k)),
Err(e) => {
error!("{:?}", e);
ready(None)
}
}) })
.flatten(); .flatten();
self.fut2 = Some(Box::pin(s)); self.fut2 = Some(Box::pin(s));
@@ -117,32 +123,26 @@ impl PreBinnedValueStream {
None => { None => {
warn!("no better resolution found for g {}", g); warn!("no better resolution found for g {}", g);
let evq = EventsQuery { let evq = EventsQuery {
channel: self.channel.clone(), channel: self.query.channel.clone(),
range: NanoRange { range: self.query.patch.patch_range(),
beg: self.patch_coord.patch_beg(), agg_kind: self.query.agg_kind.clone(),
end: self.patch_coord.patch_end(),
},
agg_kind: self.agg_kind.clone(),
}; };
assert!(self.patch_coord.patch_t_len() % self.patch_coord.bin_t_len() == 0); if self.query.patch.patch_t_len() % self.query.patch.bin_t_len() != 0 {
error!(
"Patch length inconsistency {} {}",
self.query.patch.patch_t_len(),
self.query.patch.bin_t_len()
);
return;
}
error!("try_setup_fetch_prebinned_higher_res apply all requested transformations and T-binning"); error!("try_setup_fetch_prebinned_higher_res apply all requested transformations and T-binning");
let count = self.patch_coord.patch_t_len() / self.patch_coord.bin_t_len(); let count = self.query.patch.patch_t_len() / self.query.patch.bin_t_len();
let range = BinnedRange::covering_range(evq.range.clone(), count).unwrap(); let range = BinnedRange::covering_range(evq.range.clone(), count).unwrap();
let s1 = MergedFromRemotes::new(evq, self.node_config.node_config.cluster.clone()); let s1 = MergedFromRemotes::new(evq, self.node_config.node_config.cluster.clone());
let s2 = s1 let s2 = s1.into_binned_t(range).map(|k| match k {
.map(|k| { Ok(k) => PreBinnedFrame(Ok(PreBinnedItem::Batch(k))),
if k.is_err() { Err(e) => PreBinnedFrame(Err(e)),
error!("\n\n\n.................. try_setup_fetch_prebinned_higher_res got ERROR"); });
} else {
trace!("try_setup_fetch_prebinned_higher_res got some item from MergedFromRemotes");
}
k
})
.into_binned_t(range)
.map(|k| match k {
Ok(k) => PreBinnedFrame(Ok(PreBinnedItem::Batch(k))),
Err(e) => PreBinnedFrame(Err(e)),
});
self.fut2 = Some(Box::pin(s2)); self.fut2 = Some(Box::pin(s2));
} }
} }
+10 -19
View File
@@ -1,5 +1,5 @@
use crate::agg::scalarbinbatch::MinMaxAvgScalarBinBatch; use crate::agg::scalarbinbatch::MinMaxAvgScalarBinBatch;
use crate::cache::{node_ix_for_patch, HttpBodyAsAsyncRead}; use crate::cache::{node_ix_for_patch, HttpBodyAsAsyncRead, PreBinnedQuery};
use crate::frame::inmem::InMemoryFrameAsyncReadStream; use crate::frame::inmem::InMemoryFrameAsyncReadStream;
use crate::frame::makeframe::decode_frame; use crate::frame::makeframe::decode_frame;
use err::Error; use err::Error;
@@ -7,7 +7,7 @@ use futures_core::Stream;
use futures_util::{pin_mut, FutureExt}; use futures_util::{pin_mut, FutureExt};
#[allow(unused_imports)] #[allow(unused_imports)]
use netpod::log::*; use netpod::log::*;
use netpod::{AggKind, Channel, NodeConfigCached, PreBinnedPatchCoord}; use netpod::NodeConfigCached;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::pin::Pin; use std::pin::Pin;
use std::task::{Context, Poll}; use std::task::{Context, Poll};
@@ -21,35 +21,26 @@ pub struct PreBinnedValueFetchedStream {
} }
impl PreBinnedValueFetchedStream { impl PreBinnedValueFetchedStream {
pub fn new( pub fn new(query: &PreBinnedQuery, node_config: &NodeConfigCached) -> Result<Self, Error> {
patch_coord: PreBinnedPatchCoord, let nodeix = node_ix_for_patch(&query.patch, &query.channel, &node_config.node_config.cluster);
channel: Channel,
agg_kind: AggKind,
node_config: &NodeConfigCached,
) -> Self {
let nodeix = node_ix_for_patch(&patch_coord, &channel, &node_config.node_config.cluster);
let node = &node_config.node_config.cluster.nodes[nodeix as usize]; let node = &node_config.node_config.cluster.nodes[nodeix as usize];
warn!("TODO defining property of a PreBinnedPatchCoord? patchlen + ix? binsize + patchix? binsize + patchsize + patchix?"); warn!("TODO defining property of a PreBinnedPatchCoord? patchlen + ix? binsize + patchix? binsize + patchsize + patchix?");
// TODO encapsulate uri creation, how to express aggregation kind?
let uri: hyper::Uri = format!( let uri: hyper::Uri = format!(
"http://{}:{}/api/1/prebinned?{}&channel_backend={}&channel_name={}&agg_kind={:?}", "http://{}:{}/api/1/prebinned?{}",
node.host, node.host,
node.port, node.port,
patch_coord.to_url_params_strings(), query.make_query_string()
channel.backend,
channel.name,
agg_kind,
) )
.parse() .parse()?;
.unwrap();
info!("PreBinnedValueFetchedStream open uri {}", uri); info!("PreBinnedValueFetchedStream open uri {}", uri);
Self { let ret = Self {
uri, uri,
resfut: None, resfut: None,
res: None, res: None,
errored: false, errored: false,
completed: false, completed: false,
} };
Ok(ret)
} }
} }
+7
View File
@@ -2,6 +2,7 @@
Error handling and reporting. Error handling and reporting.
*/ */
use http::uri::InvalidUri;
use nom::error::ErrorKind; use nom::error::ErrorKind;
use serde::{Deserialize, Serialize, Serializer}; use serde::{Deserialize, Serialize, Serializer};
use std::fmt::Debug; use std::fmt::Debug;
@@ -198,6 +199,12 @@ impl<T> From<async_channel::SendError<T>> for Error {
} }
} }
impl From<InvalidUri> for Error {
fn from(k: InvalidUri) -> Self {
Self::with_msg(k.to_string())
}
}
pub fn todoval<T>() -> T { pub fn todoval<T>() -> T {
todo!("TODO todoval") todo!("TODO todoval")
} }
+4 -4
View File
@@ -220,7 +220,7 @@ where
async fn binned(req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> { async fn binned(req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
let (head, _body) = req.into_parts(); let (head, _body) = req.into_parts();
let query = disk::cache::Query::from_request(&head)?; let query = disk::cache::BinnedQuery::from_request(&head)?;
let ret = match disk::cache::binned_bytes_for_http(node_config, &query).await { let ret = match disk::cache::binned_bytes_for_http(node_config, &query).await {
Ok(s) => response(StatusCode::OK).body(BodyStream::wrapped(s, format!("desc-BINNED")))?, Ok(s) => response(StatusCode::OK).body(BodyStream::wrapped(s, format!("desc-BINNED")))?,
Err(e) => { Err(e) => {
@@ -234,7 +234,7 @@ async fn binned(req: Request<Body>, node_config: &NodeConfigCached) -> Result<Re
async fn prebinned(req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> { async fn prebinned(req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
let (head, _body) = req.into_parts(); let (head, _body) = req.into_parts();
let q = PreBinnedQuery::from_request(&head)?; let q = PreBinnedQuery::from_request(&head)?;
let desc = format!("pre-b-{}", q.patch.bin_t_len() / 1000000000); let desc = format!("pre-b-{}", q.patch().bin_t_len() / 1000000000);
let span1 = span!(Level::INFO, "httpret::prebinned", desc = &desc.as_str()); let span1 = span!(Level::INFO, "httpret::prebinned", desc = &desc.as_str());
span1.in_scope(|| { span1.in_scope(|| {
trace!("prebinned"); trace!("prebinned");
@@ -243,8 +243,8 @@ async fn prebinned(req: Request<Body>, node_config: &NodeConfigCached) -> Result
s, s,
format!( format!(
"pre-b-{}-p-{}", "pre-b-{}-p-{}",
q.patch.bin_t_len() / 1000000000, q.patch().bin_t_len() / 1000000000,
q.patch.patch_beg() / 1000000000, q.patch().patch_beg() / 1000000000,
), ),
))?, ))?,
Err(e) => { Err(e) => {
+7
View File
@@ -397,6 +397,13 @@ impl PreBinnedPatchCoord {
self.spec.patch_t_len() * (self.ix + 1) self.spec.patch_t_len() * (self.ix + 1)
} }
pub fn patch_range(&self) -> NanoRange {
NanoRange {
beg: self.patch_beg(),
end: self.patch_end(),
}
}
pub fn bin_count(&self) -> u64 { pub fn bin_count(&self) -> u64 {
self.patch_t_len() / self.spec.bin_t_len self.patch_t_len() / self.spec.bin_t_len
} }