daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor

Browse Source
This commit is contained in:
Dominik Werder
2023-03-16 10:10:16 +01:00
parent b0f71b2143
commit 6a86ac1063
46 changed files with 549 additions and 1871 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
httpret/src/api1.rs
View File

@@ -6,6 +6,8 @@ use crate::BodyStream;
use crate::ReqCtx;
use bytes::BufMut;
use bytes::BytesMut;
use disk::merge::mergedblobsfromremotes::MergedBlobsFromRemotes;
use disk::raw::conn::make_local_event_blobs_stream;
use futures_util::FutureExt;
use futures_util::Stream;
use futures_util::StreamExt;
@@ -820,13 +822,7 @@ impl Stream for DataApiPython3DataStream {
};
let channel = self.channels[self.chan_ix - 1].clone();
debug!("found channel_config for {}: {:?}", channel.name, entry);
let evq = PlainEventsQuery::new(
channel,
self.range.clone(),
Some(netpod::AggKind::EventBlobs),
Some(Duration::from_millis(600000)),
None,
);
let evq = PlainEventsQuery::new(channel, self.range.clone()).for_event_blobs();
info!("query for event blobs retrieval: evq {evq:?}");
warn!("fix magic inmem_bufcap");
let perf_opts = PerfOpts::default();
@@ -835,8 +831,8 @@ impl Stream for DataApiPython3DataStream {
info!("Set up central storage stream");
// TODO pull up this config
let event_chunker_conf = EventChunkerConf::new(ByteSize::kb(1024));
let s = disk::raw::conn::make_local_event_blobs_stream(
evq.range().clone(),
let s = make_local_event_blobs_stream(
evq.range().try_into()?,
evq.channel().clone(),
&entry,
evq.one_before_range(),
@@ -853,7 +849,7 @@ impl Stream for DataApiPython3DataStream {
}
}
debug!("Set up merged remote stream");
let s = disk::merge::mergedblobsfromremotes::MergedBlobsFromRemotes::new(
let s = MergedBlobsFromRemotes::new(
evq,
perf_opts,
self.node_config.node_config.cluster.clone(),

4
httpret/src/api4/binned.rs
View File

@@ -31,8 +31,8 @@ async fn binned_json(url: Url, req: Request<Body>, node_config: &NodeConfigCache
let span1 = span!(
Level::INFO,
"httpret::binned",
beg = query.range().beg / SEC,
end = query.range().end / SEC,
beg = query.range().beg_u64() / SEC,
end = query.range().end_u64() / SEC,
ch = query.channel().name(),
);
span1.in_scope(|| {

4
httpret/src/channelconfig.rs
View File

@@ -35,7 +35,7 @@ use std::collections::BTreeMap;
use url::Url;
pub async fn chconf_from_events_v1(q: &PlainEventsQuery, ncc: &NodeConfigCached) -> Result<ChConf, Error> {
let ret = nodenet::channelconfig::channel_config(q.range().clone(), q.channel().clone(), ncc).await?;
let ret = nodenet::channelconfig::channel_config(q.range().try_into()?, q.channel().clone(), ncc).await?;
Ok(ret)
}
@@ -54,7 +54,7 @@ pub async fn chconf_from_prebinned(q: &PreBinnedQuery, _ncc: &NodeConfigCached)
}
pub async fn chconf_from_binned(q: &BinnedQuery, ncc: &NodeConfigCached) -> Result<ChConf, Error> {
let ret = nodenet::channelconfig::channel_config(q.range().clone(), q.channel().clone(), ncc).await?;
let ret = nodenet::channelconfig::channel_config(q.range().try_into()?, q.channel().clone(), ncc).await?;
Ok(ret)
}

9
httpret/src/httpret.rs
View File

@@ -32,7 +32,6 @@ use hyper::Response;
use net::SocketAddr;
use netpod::log::*;
use netpod::query::prebinned::PreBinnedQuery;
use netpod::timeunits::SEC;
use netpod::NodeConfigCached;
use netpod::ProxyConfig;
use netpod::APP_JSON;
@@ -464,15 +463,11 @@ async fn prebinned_inner(
let (head, _body) = req.into_parts();
let url: url::Url = format!("dummy://{}", head.uri).parse()?;
let query = PreBinnedQuery::from_url(&url)?;
let desc = format!(
"pre-W-{}-B-{}",
query.patch().bin_t_len() / SEC,
query.patch().patch_beg() / SEC
);
let span1 = span!(Level::INFO, "httpret::prebinned", desc = &desc.as_str());
let span1 = span!(Level::INFO, "httpret::prebinned", desc = &query.patch().span_desc());
span1.in_scope(|| {
debug!("begin");
});
error!("TODO hhtpret prebinned_inner");
//let fut = disk::binned::prebinned::pre_binned_bytes_for_http(node_config, &query).instrument(span1);
todo!()
}