daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP refactor channel config usage

Browse Source
This commit is contained in:
Dominik Werder
2023-06-19 10:04:56 +02:00
parent 44dd43240b
commit 0daa66a800
12 changed files with 303 additions and 269 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
httpret/src/api1.rs
View File

@@ -640,9 +640,13 @@ impl Api1ChannelHeader {
}
}
async fn find_ch_conf(channel: SfDbChannel, ncc: NodeConfigCached) -> Result<SfChFetchInfo, Error> {
//find_sf_channel_config_basics_quorum()
todo!()
async fn find_ch_conf(
range: NanoRange,
channel: SfDbChannel,
ncc: NodeConfigCached,
) -> Result<ChannelTypeConfigGen, Error> {
let ret = nodenet::channelconfig::channel_config(range, channel, &ncc).await?;
Ok(ret)
}
pub struct DataApiPython3DataStream {
@@ -651,8 +655,8 @@ pub struct DataApiPython3DataStream {
current_channel: Option<SfDbChannel>,
node_config: NodeConfigCached,
chan_stream: Option<Pin<Box<dyn Stream<Item = Result<BytesMut, Error>> + Send>>>,
config_fut: Option<Pin<Box<dyn Future<Output = Result<SfChFetchInfo, Error>> + Send>>>,
ch_conf: Option<SfChFetchInfo>,
config_fut: Option<Pin<Box<dyn Future<Output = Result<ChannelTypeConfigGen, Error>> + Send>>>,
ch_conf: Option<ChannelTypeConfigGen>,
disk_io_tune: DiskIoTune,
do_decompress: bool,
#[allow(unused)]
@@ -882,14 +886,22 @@ impl Stream for DataApiPython3DataStream {
}
} else if let Some(fut) = &mut self.config_fut {
match fut.poll_unpin(cx) {
Ready(Ok(k)) => match self.handle_config_fut_ready(k) {
Ok(()) => continue,
Err(e) => {
self.config_fut = None;
Ready(Ok(k)) => match k {
ChannelTypeConfigGen::Scylla(_) => {
let e = Error::with_msg_no_trace("scylla");
error!("{e}");
self.data_done = true;
error!("api1_binary_events error {:?}", e);
Ready(Some(Err(e)))
}
ChannelTypeConfigGen::SfDatabuffer(k) => match self.handle_config_fut_ready(k) {
Ok(()) => continue,
Err(e) => {
self.config_fut = None;
self.data_done = true;
error!("api1_binary_events error {:?}", e);
Ready(Some(Err(e)))
}
},
},
Ready(Err(e)) => {
self.data_done = true;
@@ -900,7 +912,11 @@ impl Stream for DataApiPython3DataStream {
} else {
if let Some(channel) = self.channels.pop_front() {
self.current_channel = Some(channel.clone());
self.config_fut = Some(Box::pin(find_ch_conf(channel, self.node_config.clone())));
self.config_fut = Some(Box::pin(find_ch_conf(
self.range.clone(),
channel,
self.node_config.clone(),
)));
continue;
} else {
self.data_done = true;

29
httpret/src/channel_status.rs
View File

@@ -12,6 +12,7 @@ use items_2::channelevents::ChannelStatusEvent;
use items_2::channelevents::ConnStatusEvent;
use netpod::log::*;
use netpod::query::ChannelStateEventsQuery;
use netpod::ChannelTypeConfigGen;
use netpod::FromUrl;
use netpod::NodeConfigCached;
use netpod::ACCEPT_ALL;
@@ -77,7 +78,6 @@ impl ConnectionStatusEvents {
let _scy = scyllaconn::create_scy_session(scyco).await?;
let chconf =
nodenet::channelconfig::channel_config(q.range().clone(), q.channel().clone(), node_config).await?;
let _series = chconf.series;
let _do_one_before_range = true;
let ret = Vec::new();
if true {
@@ -153,17 +153,22 @@ impl ChannelStatusEvents {
let chconf =
nodenet::channelconfig::channel_config(q.range().clone(), q.channel().clone(), node_config).await?;
let do_one_before_range = true;
let mut stream = scyllaconn::status::StatusStreamScylla::new(
chconf.try_series().context("channel_status")?,
q.range().clone(),
do_one_before_range,
scy,
);
let mut ret = Vec::new();
while let Some(item) = stream.next().await {
let item = item?;
ret.push(item);
match chconf {
ChannelTypeConfigGen::Scylla(ch_conf) => {
let mut stream = scyllaconn::status::StatusStreamScylla::new(
ch_conf.series(),
q.range().clone(),
do_one_before_range,
scy,
);
let mut ret = Vec::new();
while let Some(item) = stream.next().await {
let item = item?;
ret.push(item);
}
Ok(ret)
}
ChannelTypeConfigGen::SfDatabuffer(k) => todo!(),
}
Ok(ret)
}
}

50
httpret/src/channelconfig.rs
View File

@@ -12,7 +12,6 @@ use netpod::get_url_query_pairs;
use netpod::log::*;
use netpod::query::prebinned::PreBinnedQuery;
use netpod::timeunits::*;
use netpod::ChConf;
use netpod::ChannelConfigQuery;
use netpod::ChannelConfigResponse;
use netpod::ChannelTypeConfigGen;
@@ -38,24 +37,16 @@ pub async fn chconf_from_events_v1(
q: &PlainEventsQuery,
ncc: &NodeConfigCached,
) -> Result<ChannelTypeConfigGen, Error> {
// let ret = nodenet::channelconfig::channel_config(q.range().try_into()?, q.channel().clone(), ncc).await?;
let ret = nodenet::configquorum::find_config_basics_quorum(q.channel(), ncc).await?;
Ok(ret)
}
pub async fn chconf_from_prebinned(q: &PreBinnedQuery, _ncc: &NodeConfigCached) -> Result<ChConf, Error> {
let ret = ChConf {
backend: q.channel().backend().into(),
series: q.channel().series().clone(),
name: q.channel().name().into(),
scalar_type: q.scalar_type().clone(),
shape: q.shape().clone(),
};
pub async fn chconf_from_prebinned(q: &PreBinnedQuery, ncc: &NodeConfigCached) -> Result<ChannelTypeConfigGen, Error> {
let ret = nodenet::configquorum::find_config_basics_quorum(q.channel(), ncc).await?;
Ok(ret)
}
pub async fn ch_conf_from_binned(q: &BinnedQuery, ncc: &NodeConfigCached) -> Result<ChannelTypeConfigGen, Error> {
// let ret = nodenet::channelconfig::channel_config(q.range().try_into()?, q.channel().clone(), ncc).await?;
let ret = nodenet::configquorum::find_config_basics_quorum(q.channel(), ncc).await?;
Ok(ret)
}
@@ -103,24 +94,11 @@ impl ChannelConfigHandler {
let url = Url::parse(&format!("dummy:{}", req.uri()))?;
let q = ChannelConfigQuery::from_url(&url)?;
info!("channel_config for q {q:?}");
let conf = if let Some(_scyco) = &node_config.node_config.cluster.scylla {
let c = nodenet::channelconfig::channel_config(q.range.clone(), q.channel.clone(), node_config).await?;
ChannelConfigResponse {
channel: SfDbChannel::from_full(q.channel.backend(), c.series.clone(), &c.name),
scalar_type: c.scalar_type,
byte_order: None,
shape: c.shape,
}
} else if let Some(_) = &node_config.node.channel_archiver {
return Err(Error::with_msg_no_trace("channel archiver not supported"));
} else if let Some(_) = &node_config.node.archiver_appliance {
return Err(Error::with_msg_no_trace("archiver appliance not supported"));
} else {
parse::channelconfig::channel_config(&q, node_config).await?
};
let conf = nodenet::channelconfig::channel_config(q.range.clone(), q.channel.clone(), node_config).await?;
let res: ChannelConfigResponse = conf.into();
let ret = response(StatusCode::OK)
.header(http::header::CONTENT_TYPE, APP_JSON)
.body(Body::from(serde_json::to_string(&conf)?))?;
.body(Body::from(serde_json::to_string(&res)?))?;
Ok(ret)
}
}
@@ -159,27 +137,15 @@ impl ChannelConfigsHandler {
}
}
async fn channel_configs(
&self,
req: Request<Body>,
node_config: &NodeConfigCached,
) -> Result<Response<Body>, Error> {
async fn channel_configs(&self, req: Request<Body>, ncc: &NodeConfigCached) -> Result<Response<Body>, Error> {
info!("channel_configs");
let url = Url::parse(&format!("dummy:{}", req.uri()))?;
let q = ChannelConfigQuery::from_url(&url)?;
info!("channel_configs for q {q:?}");
let conf = if let Some(_) = &node_config.node_config.cluster.scylla {
return Err(Error::with_msg_no_trace("TODO"));
} else if let Some(_) = &node_config.node.channel_archiver {
return Err(Error::with_msg_no_trace("TODO"));
} else if let Some(_) = &node_config.node.archiver_appliance {
return Err(Error::with_msg_no_trace("TODO"));
} else {
disk::channelconfig::configs(q.channel, node_config).await?
};
let ch_conf = nodenet::channelconfig::channel_config(q.range.clone(), q.channel, ncc).await?;
let ret = response(StatusCode::OK)
.header(http::header::CONTENT_TYPE, APP_JSON)
.body(Body::from(serde_json::to_string(&conf)?))?;
.body(Body::from(serde_json::to_string(&ch_conf)?))?;
Ok(ret)
}
}