Move workspace crates into subfolder
This commit is contained in:
Dominik Werder
@@ -0,0 +1,520 @@
|
||||
use crate::errconv::ErrConv;
|
||||
use crate::events::EventsStreamScylla;
|
||||
use err::Error;
|
||||
use futures_util::Future;
|
||||
use futures_util::Stream;
|
||||
use futures_util::StreamExt;
|
||||
use items_0::timebin::TimeBinned;
|
||||
use items_2::binsdim0::BinsDim0;
|
||||
use items_2::channelevents::ChannelEvents;
|
||||
use netpod::log::*;
|
||||
use netpod::query::CacheUsage;
|
||||
use netpod::timeunits::*;
|
||||
use netpod::AggKind;
|
||||
use netpod::ChannelTyped;
|
||||
use netpod::Dim0Kind;
|
||||
use netpod::PreBinnedPatchCoord;
|
||||
use netpod::PreBinnedPatchCoordEnum;
|
||||
use netpod::PreBinnedPatchRange;
|
||||
use netpod::PreBinnedPatchRangeEnum;
|
||||
use netpod::ScalarType;
|
||||
use netpod::Shape;
|
||||
use query::transform::TransformQuery;
|
||||
use scylla::Session as ScySession;
|
||||
use std::collections::VecDeque;
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::task::Context;
|
||||
use std::task::Poll;
|
||||
use std::time::Duration;
|
||||
use std::time::Instant;
|
||||
|
||||
pub async fn read_cached_scylla(
|
||||
series: u64,
|
||||
chn: &ChannelTyped,
|
||||
coord: &PreBinnedPatchCoordEnum,
|
||||
scy: &ScySession,
|
||||
) -> Result<Option<Box<dyn TimeBinned>>, Error> {
|
||||
/*let vals = (
|
||||
series as i64,
|
||||
(coord.bin_t_len() / SEC) as i32,
|
||||
(coord.patch_t_len() / SEC) as i32,
|
||||
coord.ix() as i64,
|
||||
);*/
|
||||
todo!();
|
||||
let vals: (i64, i32, i32, i64) = todo!();
|
||||
let res = scy
|
||||
.query_iter(
|
||||
"select counts, avgs, mins, maxs from binned_scalar_f32 where series = ? and bin_len_sec = ? and patch_len_sec = ? and agg_kind = 'dummy-agg-kind' and offset = ?",
|
||||
vals,
|
||||
)
|
||||
.await;
|
||||
let mut res = res.err_conv().map_err(|e| {
|
||||
error!("can not read from cache");
|
||||
e
|
||||
})?;
|
||||
while let Some(item) = res.next().await {
|
||||
let row = item.err_conv()?;
|
||||
// let edges = coord.edges();
|
||||
let edges: Vec<u64> = todo!();
|
||||
let (counts, avgs, mins, maxs): (Vec<i64>, Vec<f32>, Vec<f32>, Vec<f32>) = row.into_typed().err_conv()?;
|
||||
let mut counts_mismatch = false;
|
||||
if edges.len() != counts.len() + 1 {
|
||||
counts_mismatch = true;
|
||||
}
|
||||
if counts.len() != avgs.len() {
|
||||
counts_mismatch = true;
|
||||
}
|
||||
let ts1s: VecDeque<_> = edges[..(edges.len() - 1).min(edges.len())].iter().map(|&x| x).collect();
|
||||
let ts2s: VecDeque<_> = edges[1.min(edges.len())..].iter().map(|&x| x).collect();
|
||||
if ts1s.len() != ts2s.len() {
|
||||
error!("ts1s vs ts2s mismatch");
|
||||
counts_mismatch = true;
|
||||
}
|
||||
if ts1s.len() != counts.len() {
|
||||
counts_mismatch = true;
|
||||
}
|
||||
let avgs: VecDeque<_> = avgs.into_iter().map(|x| x).collect();
|
||||
let mins: VecDeque<_> = mins.into_iter().map(|x| x as _).collect();
|
||||
let maxs: VecDeque<_> = maxs.into_iter().map(|x| x as _).collect();
|
||||
if counts_mismatch {
|
||||
error!(
|
||||
"mismatch: edges {} ts1s {} ts2s {} counts {} avgs {} mins {} maxs {}",
|
||||
edges.len(),
|
||||
ts1s.len(),
|
||||
ts2s.len(),
|
||||
counts.len(),
|
||||
avgs.len(),
|
||||
mins.len(),
|
||||
maxs.len(),
|
||||
);
|
||||
}
|
||||
let counts: VecDeque<_> = counts.into_iter().map(|x| x as u64).collect();
|
||||
// TODO construct a dyn TimeBinned using the scalar type and shape information.
|
||||
// TODO place the values with little copying into the TimeBinned.
|
||||
use ScalarType::*;
|
||||
use Shape::*;
|
||||
match &chn.shape {
|
||||
Scalar => match &chn.scalar_type {
|
||||
F64 => {
|
||||
let ret = BinsDim0::<f64> {
|
||||
ts1s,
|
||||
ts2s,
|
||||
counts,
|
||||
avgs,
|
||||
mins,
|
||||
maxs,
|
||||
// TODO:
|
||||
dim0kind: Some(Dim0Kind::Time),
|
||||
};
|
||||
return Ok(Some(Box::new(ret)));
|
||||
}
|
||||
_ => {
|
||||
error!("TODO can not yet restore {:?} {:?}", chn.scalar_type, chn.shape);
|
||||
err::todoval()
|
||||
}
|
||||
},
|
||||
_ => {
|
||||
error!("TODO can not yet restore {:?} {:?}", chn.scalar_type, chn.shape);
|
||||
err::todoval()
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
struct WriteFut<'a> {
|
||||
chn: &'a ChannelTyped,
|
||||
coord: &'a PreBinnedPatchCoordEnum,
|
||||
data: &'a dyn TimeBinned,
|
||||
scy: &'a ScySession,
|
||||
}
|
||||
|
||||
impl<'a> WriteFut<'a> {
|
||||
#[allow(unused)]
|
||||
fn new(
|
||||
chn: &'a ChannelTyped,
|
||||
coord: &'a PreBinnedPatchCoordEnum,
|
||||
data: &'a dyn TimeBinned,
|
||||
scy: &'a ScySession,
|
||||
) -> Self {
|
||||
Self { chn, coord, data, scy }
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> Future for WriteFut<'a> {
|
||||
type Output = Result<(), Error>;
|
||||
|
||||
fn poll(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Self::Output> {
|
||||
let _ = cx;
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn write_cached_scylla<'a>(
|
||||
series: u64,
|
||||
_chn: &'a ChannelTyped,
|
||||
coord: &'a PreBinnedPatchCoordEnum,
|
||||
//data: &'a dyn TimeBinned,
|
||||
data: Box<dyn TimeBinned>,
|
||||
//scy: &'a ScySession,
|
||||
_scy: Arc<ScySession>,
|
||||
) -> Pin<Box<dyn Future<Output = Result<(), Error>> + Send + 'a>> {
|
||||
//let _chn = unsafe { &*(chn as *const ChannelTyped) };
|
||||
//let data_ptr = data as *const dyn TimeBinned as usize;
|
||||
//let scy_ptr = scy as *const ScySession as usize;
|
||||
let fut = async move {
|
||||
//let data = unsafe { &*(data_ptr as *const dyn TimeBinned) };
|
||||
//let scy = unsafe { &*(scy_ptr as *const ScySession) };
|
||||
todo!();
|
||||
/*let bin_len_sec = (coord.bin_t_len() / SEC) as i32;
|
||||
let patch_len_sec = (coord.patch_t_len() / SEC) as i32;
|
||||
let offset = coord.ix();*/
|
||||
let bin_len_sec = 0i32;
|
||||
let patch_len_sec = 0i32;
|
||||
let offset = 0i32;
|
||||
warn!(
|
||||
"write_cached_scylla len {} where series = {} and bin_len_sec = {} and patch_len_sec = {} and agg_kind = 'dummy-agg-kind' and offset = {}",
|
||||
data.counts().len(),
|
||||
series,
|
||||
bin_len_sec,
|
||||
patch_len_sec,
|
||||
offset,
|
||||
);
|
||||
let _data2 = data.counts().iter().map(|x| *x as i64).collect::<Vec<_>>();
|
||||
/*
|
||||
let stmt = scy.prepare("insert into binned_scalar_f32 (series, bin_len_sec, patch_len_sec, agg_kind, offset, counts, avgs, mins, maxs) values (?, ?, ?, 'dummy-agg-kind', ?, ?, ?, ?, ?)").await.err_conv()?;
|
||||
scy.execute(
|
||||
&stmt,
|
||||
(
|
||||
series as i64,
|
||||
bin_len_sec,
|
||||
patch_len_sec,
|
||||
offset as i64,
|
||||
data2,
|
||||
data.avgs(),
|
||||
data.mins(),
|
||||
data.maxs(),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.err_conv()
|
||||
.map_err(|e| {
|
||||
error!("can not write to cache");
|
||||
e
|
||||
})?;
|
||||
*/
|
||||
Ok(())
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
|
||||
pub async fn fetch_uncached_data(
|
||||
series: u64,
|
||||
chn: ChannelTyped,
|
||||
coord: PreBinnedPatchCoordEnum,
|
||||
one_before_range: bool,
|
||||
transform: TransformQuery,
|
||||
cache_usage: CacheUsage,
|
||||
scy: Arc<ScySession>,
|
||||
) -> Result<Option<(Box<dyn TimeBinned>, bool)>, Error> {
|
||||
/*info!("fetch_uncached_data {coord:?}");
|
||||
// Try to find a higher resolution pre-binned grid which covers the requested patch.
|
||||
let (bin, complete) = match PreBinnedPatchRange::covering_range(coord.patch_range(), coord.bin_count() + 1) {
|
||||
Ok(Some(range)) => {
|
||||
if coord.patch_range() != range.range() {
|
||||
error!(
|
||||
"The chosen covering range does not exactly cover the requested patch {:?} vs {:?}",
|
||||
coord.patch_range(),
|
||||
range.range()
|
||||
);
|
||||
}
|
||||
fetch_uncached_higher_res_prebinned(
|
||||
series,
|
||||
&chn,
|
||||
coord.clone(),
|
||||
range,
|
||||
one_before_range,
|
||||
transform,
|
||||
cache_usage.clone(),
|
||||
scy.clone(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
Ok(None) => {
|
||||
fetch_uncached_binned_events(series, &chn, coord.clone(), one_before_range, transform, scy.clone()).await
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
}?;
|
||||
if true || complete {
|
||||
let edges = coord.edges();
|
||||
if edges.len() < bin.len() + 1 {
|
||||
error!(
|
||||
"attempt to write overfull bin to cache edges {} bin {}",
|
||||
edges.len(),
|
||||
bin.len()
|
||||
);
|
||||
return Err(Error::with_msg_no_trace(format!(
|
||||
"attempt to write overfull bin to cache"
|
||||
)));
|
||||
} else if edges.len() > bin.len() + 1 {
|
||||
let missing = edges.len() - bin.len() - 1;
|
||||
error!("attempt to write incomplete bin to cache missing {missing}");
|
||||
}
|
||||
if let CacheUsage::Use | CacheUsage::Recreate = &cache_usage {
|
||||
// TODO pass data in safe way.
|
||||
let _data = bin.as_ref();
|
||||
//let fut = WriteFut::new(&chn, &coord, err::todoval(), &scy);
|
||||
//fut.await?;
|
||||
//write_cached_scylla(series, &chn, &coord, bin.as_ref(), &scy).await?;
|
||||
}
|
||||
}
|
||||
Ok(Some((bin, complete)))*/
|
||||
todo!()
|
||||
}
|
||||
|
||||
pub fn fetch_uncached_data_box(
|
||||
series: u64,
|
||||
chn: &ChannelTyped,
|
||||
coord: &PreBinnedPatchCoordEnum,
|
||||
one_before_range: bool,
|
||||
transform: TransformQuery,
|
||||
cache_usage: CacheUsage,
|
||||
scy: Arc<ScySession>,
|
||||
) -> Pin<Box<dyn Future<Output = Result<Option<(Box<dyn TimeBinned>, bool)>, Error>> + Send>> {
|
||||
Box::pin(fetch_uncached_data(
|
||||
series,
|
||||
chn.clone(),
|
||||
coord.clone(),
|
||||
one_before_range,
|
||||
transform,
|
||||
cache_usage,
|
||||
scy,
|
||||
))
|
||||
}
|
||||
|
||||
pub async fn fetch_uncached_higher_res_prebinned(
|
||||
series: u64,
|
||||
chn: &ChannelTyped,
|
||||
coord: PreBinnedPatchCoordEnum,
|
||||
range: PreBinnedPatchRangeEnum,
|
||||
one_before_range: bool,
|
||||
transform: TransformQuery,
|
||||
cache_usage: CacheUsage,
|
||||
scy: Arc<ScySession>,
|
||||
) -> Result<(Box<dyn TimeBinned>, bool), Error> {
|
||||
/*let edges = coord.edges();
|
||||
// TODO refine the AggKind scheme or introduce a new BinningOpts type and get time-weight from there.
|
||||
let do_time_weight = true;
|
||||
// We must produce some result with correct types even if upstream delivers nothing at all.
|
||||
//let bin0 = empty_binned_dyn_tb(&chn.scalar_type, &chn.shape, &transform);
|
||||
let bin0 = err::todoval();
|
||||
let mut time_binner = bin0.time_binner_new(edges.clone(), do_time_weight);
|
||||
let mut complete = true;
|
||||
//let patch_it = PreBinnedPatchIterator::from_range(range.clone());
|
||||
let patches_dummy: Vec<PreBinnedPatchCoordEnum> = Vec::new();
|
||||
let mut patch_it = patches_dummy.into_iter();
|
||||
for patch_coord in patch_it {
|
||||
// We request data here for a Coord, meaning that we expect to receive multiple bins.
|
||||
// The expectation is that we receive a single TimeBinned which contains all bins of that PatchCoord.
|
||||
//let patch_coord = PreBinnedPatchCoord::new(patch.bin_t_len(), patch.patch_t_len(), patch.ix());
|
||||
let (bin, comp) = pre_binned_value_stream_with_scy(
|
||||
series,
|
||||
chn,
|
||||
&patch_coord,
|
||||
one_before_range,
|
||||
transform.clone(),
|
||||
cache_usage.clone(),
|
||||
scy.clone(),
|
||||
)
|
||||
.await?;
|
||||
if let Err(msg) = bin.validate() {
|
||||
error!(
|
||||
"pre-binned intermediate issue {} coord {:?} patch_coord {:?}",
|
||||
msg, coord, patch_coord
|
||||
);
|
||||
}
|
||||
complete = complete && comp;
|
||||
time_binner.ingest(bin.as_time_binnable_dyn());
|
||||
}
|
||||
// Fixed limit to defend against a malformed implementation:
|
||||
let mut i = 0;
|
||||
while i < 80000 && time_binner.bins_ready_count() < coord.bin_count() as usize {
|
||||
let n1 = time_binner.bins_ready_count();
|
||||
if false {
|
||||
trace!(
|
||||
"pre-binned extra cycle {} {} {}",
|
||||
i,
|
||||
time_binner.bins_ready_count(),
|
||||
coord.bin_count()
|
||||
);
|
||||
}
|
||||
time_binner.cycle();
|
||||
i += 1;
|
||||
if time_binner.bins_ready_count() <= n1 {
|
||||
warn!("pre-binned cycle did not add another bin, break");
|
||||
break;
|
||||
}
|
||||
}
|
||||
if time_binner.bins_ready_count() < coord.bin_count() as usize {
|
||||
return Err(Error::with_msg_no_trace(format!(
|
||||
"pre-binned unable to produce all bins for the patch bins_ready {} coord.bin_count {} edges.len {}",
|
||||
time_binner.bins_ready_count(),
|
||||
coord.bin_count(),
|
||||
edges.len(),
|
||||
)));
|
||||
}
|
||||
let ready = time_binner
|
||||
.bins_ready()
|
||||
.ok_or_else(|| Error::with_msg_no_trace(format!("unable to produce any bins for the patch range")))?;
|
||||
if let Err(msg) = ready.validate() {
|
||||
error!("pre-binned final issue {} coord {:?}", msg, coord);
|
||||
}
|
||||
Ok((ready, complete))*/
|
||||
todo!()
|
||||
}
|
||||
|
||||
pub async fn fetch_uncached_binned_events(
|
||||
series: u64,
|
||||
chn: &ChannelTyped,
|
||||
coord: PreBinnedPatchCoordEnum,
|
||||
one_before_range: bool,
|
||||
transform: TransformQuery,
|
||||
scy: Arc<ScySession>,
|
||||
) -> Result<(Box<dyn TimeBinned>, bool), Error> {
|
||||
/*let edges = coord.edges();
|
||||
// TODO refine the AggKind scheme or introduce a new BinningOpts type and get time-weight from there.
|
||||
let do_time_weight = true;
|
||||
// We must produce some result with correct types even if upstream delivers nothing at all.
|
||||
//let bin0 = empty_events_dyn_tb(&chn.scalar_type, &chn.shape, &agg_kind);
|
||||
//let mut time_binner = bin0.time_binner_new(edges.clone(), do_time_weight);
|
||||
let mut time_binner = items_2::empty::empty_events_dyn_ev(&chn.scalar_type, &chn.shape)?
|
||||
.as_time_binnable()
|
||||
.time_binner_new(edges.clone(), do_time_weight);
|
||||
// TODO handle deadline better
|
||||
let deadline = Instant::now();
|
||||
// TODO take timeout from query
|
||||
let deadline = deadline
|
||||
.checked_add(Duration::from_millis(6000))
|
||||
.ok_or_else(|| Error::with_msg_no_trace(format!("deadline overflow")))?;
|
||||
let evq = PlainEventsQuery::new(chn.channel.clone(), coord.patch_range());
|
||||
let mut events_dyn = EventsStreamScylla::new(
|
||||
series,
|
||||
evq.range().clone(),
|
||||
one_before_range,
|
||||
chn.scalar_type.clone(),
|
||||
chn.shape.clone(),
|
||||
true,
|
||||
scy,
|
||||
false,
|
||||
);
|
||||
let mut complete = false;
|
||||
loop {
|
||||
let item = tokio::time::timeout_at(deadline.into(), events_dyn.next()).await;
|
||||
let item = match item {
|
||||
Ok(Some(k)) => k,
|
||||
Ok(None) => break,
|
||||
Err(_) => {
|
||||
error!("fetch_uncached_binned_events timeout");
|
||||
return Err(Error::with_msg_no_trace(format!(
|
||||
"TODO handle fetch_uncached_binned_events timeout"
|
||||
)));
|
||||
}
|
||||
};
|
||||
if false {
|
||||
// TODO as soon we encounter RangeComplete we just:
|
||||
complete = true;
|
||||
}
|
||||
match item {
|
||||
Ok(ChannelEvents::Events(item)) => {
|
||||
time_binner.ingest(item.as_time_binnable());
|
||||
// TODO could also ask the binner here whether we are "complete" to stop sending useless data.
|
||||
}
|
||||
Ok(ChannelEvents::Status(_)) => {
|
||||
// TODO flag, should not happen.
|
||||
return Err(Error::with_msg_no_trace(format!(
|
||||
"unexpected read of channel status events"
|
||||
)));
|
||||
}
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
}
|
||||
// Fixed limit to defend against a malformed implementation:
|
||||
let mut i = 0;
|
||||
while i < 80000 && time_binner.bins_ready_count() < coord.bin_count() as usize {
|
||||
let n1 = time_binner.bins_ready_count();
|
||||
if false {
|
||||
trace!(
|
||||
"events extra cycle {} {} {}",
|
||||
i,
|
||||
time_binner.bins_ready_count(),
|
||||
coord.bin_count()
|
||||
);
|
||||
}
|
||||
time_binner.cycle();
|
||||
i += 1;
|
||||
if time_binner.bins_ready_count() <= n1 {
|
||||
warn!("events cycle did not add another bin, break");
|
||||
break;
|
||||
}
|
||||
}
|
||||
if time_binner.bins_ready_count() < coord.bin_count() as usize {
|
||||
return Err(Error::with_msg_no_trace(format!(
|
||||
"events unable to produce all bins for the patch bins_ready {} coord.bin_count {} edges.len {}",
|
||||
time_binner.bins_ready_count(),
|
||||
coord.bin_count(),
|
||||
edges.len(),
|
||||
)));
|
||||
}
|
||||
let ready = time_binner
|
||||
.bins_ready()
|
||||
.ok_or_else(|| Error::with_msg_no_trace(format!("unable to produce any bins for the patch")))?;
|
||||
if let Err(msg) = ready.validate() {
|
||||
error!("time binned invalid {} coord {:?}", msg, coord);
|
||||
}
|
||||
Ok((ready, complete))*/
|
||||
todo!()
|
||||
}
|
||||
|
||||
pub async fn pre_binned_value_stream_with_scy(
|
||||
series: u64,
|
||||
chn: &ChannelTyped,
|
||||
coord: &PreBinnedPatchCoordEnum,
|
||||
one_before_range: bool,
|
||||
transform: TransformQuery,
|
||||
cache_usage: CacheUsage,
|
||||
scy: Arc<ScySession>,
|
||||
) -> Result<(Box<dyn TimeBinned>, bool), Error> {
|
||||
trace!("pre_binned_value_stream_with_scy {chn:?} {coord:?}");
|
||||
if let (Some(item), CacheUsage::Use) = (read_cached_scylla(series, chn, coord, &scy).await?, &cache_usage) {
|
||||
info!("+++++++++++++ GOOD READ");
|
||||
Ok((item, true))
|
||||
} else {
|
||||
if let CacheUsage::Use = &cache_usage {
|
||||
warn!("--+--+--+--+--+--+ NOT YET CACHED");
|
||||
}
|
||||
let res = fetch_uncached_data_box(series, chn, coord, one_before_range, transform, cache_usage, scy).await?;
|
||||
let (bin, complete) =
|
||||
res.ok_or_else(|| Error::with_msg_no_trace(format!("pre_binned_value_stream_with_scy got None bin")))?;
|
||||
Ok((bin, complete))
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn pre_binned_value_stream(
|
||||
series: u64,
|
||||
chn: &ChannelTyped,
|
||||
coord: &PreBinnedPatchCoordEnum,
|
||||
one_before_range: bool,
|
||||
transform: TransformQuery,
|
||||
agg_kind: AggKind,
|
||||
cache_usage: CacheUsage,
|
||||
scy: Arc<ScySession>,
|
||||
) -> Result<Pin<Box<dyn Stream<Item = Result<Box<dyn TimeBinned>, Error>> + Send>>, Error> {
|
||||
trace!("pre_binned_value_stream series {series} {chn:?} {coord:?}");
|
||||
let res =
|
||||
pre_binned_value_stream_with_scy(series, chn, coord, one_before_range, transform, cache_usage, scy).await?;
|
||||
error!("TODO pre_binned_value_stream");
|
||||
err::todo();
|
||||
Ok(Box::pin(futures_util::stream::iter([Ok(res.0)])))
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
use err::Error;
|
||||
use scylla::frame::response::cql_to_rust::FromRowError as ScyFromRowError;
|
||||
use scylla::transport::errors::NewSessionError as ScyNewSessionError;
|
||||
use scylla::transport::errors::QueryError as ScyQueryError;
|
||||
use scylla::transport::query_result::RowsExpectedError;
|
||||
|
||||
pub trait ErrConv<T> {
|
||||
fn err_conv(self) -> Result<T, Error>;
|
||||
}
|
||||
|
||||
impl<T> ErrConv<T> for Result<T, tokio_postgres::Error> {
|
||||
fn err_conv(self) -> Result<T, Error> {
|
||||
match self {
|
||||
Ok(k) => Ok(k),
|
||||
Err(e) => Err(Error::with_msg(e.to_string())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T, A> ErrConv<T> for Result<T, async_channel::SendError<A>> {
|
||||
fn err_conv(self) -> Result<T, Error> {
|
||||
match self {
|
||||
Ok(k) => Ok(k),
|
||||
Err(e) => Err(Error::with_msg(e.to_string())),
|
||||
}
|
||||
}
|
||||
}
|
||||
impl<T> ErrConv<T> for Result<T, ScyQueryError> {
|
||||
fn err_conv(self) -> Result<T, Error> {
|
||||
match self {
|
||||
Ok(k) => Ok(k),
|
||||
Err(e) => Err(Error::with_msg_no_trace(format!("{e:?}"))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> ErrConv<T> for Result<T, ScyNewSessionError> {
|
||||
fn err_conv(self) -> Result<T, Error> {
|
||||
match self {
|
||||
Ok(k) => Ok(k),
|
||||
Err(e) => Err(Error::with_msg_no_trace(format!("{e:?}"))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> ErrConv<T> for Result<T, ScyFromRowError> {
|
||||
fn err_conv(self) -> Result<T, Error> {
|
||||
match self {
|
||||
Ok(k) => Ok(k),
|
||||
Err(e) => Err(Error::with_msg_no_trace(format!("{e:?}"))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> ErrConv<T> for Result<T, RowsExpectedError> {
|
||||
fn err_conv(self) -> Result<T, Error> {
|
||||
match self {
|
||||
Ok(k) => Ok(k),
|
||||
Err(e) => Err(Error::with_msg_no_trace(format!("{e:?}"))),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,649 @@
|
||||
use crate::errconv::ErrConv;
|
||||
use crate::ScyllaSeriesRange;
|
||||
use err::Error;
|
||||
use futures_util::Future;
|
||||
use futures_util::FutureExt;
|
||||
use futures_util::Stream;
|
||||
use items_0::scalar_ops::ScalarOps;
|
||||
use items_0::Appendable;
|
||||
use items_0::Empty;
|
||||
use items_0::Events;
|
||||
use items_0::WithLen;
|
||||
use items_2::channelevents::ChannelEvents;
|
||||
use items_2::eventsdim0::EventsDim0;
|
||||
use items_2::eventsdim1::EventsDim1;
|
||||
use netpod::log::*;
|
||||
use netpod::ScalarType;
|
||||
use netpod::Shape;
|
||||
use scylla::Session as ScySession;
|
||||
use std::collections::VecDeque;
|
||||
use std::mem;
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::task::Context;
|
||||
use std::task::Poll;
|
||||
|
||||
async fn find_ts_msp(
|
||||
series: u64,
|
||||
range: ScyllaSeriesRange,
|
||||
scy: Arc<ScySession>,
|
||||
) -> Result<(VecDeque<u64>, VecDeque<u64>), Error> {
|
||||
trace!("find_ts_msp series {} {:?}", series, range);
|
||||
let mut ret1 = VecDeque::new();
|
||||
let mut ret2 = VecDeque::new();
|
||||
// TODO use prepared statements
|
||||
let cql = "select ts_msp from ts_msp where series = ? and ts_msp < ? order by ts_msp desc limit 2";
|
||||
let res = scy.query(cql, (series as i64, range.beg as i64)).await.err_conv()?;
|
||||
for row in res.rows_typed_or_empty::<(i64,)>() {
|
||||
let row = row.err_conv()?;
|
||||
ret1.push_front(row.0 as u64);
|
||||
}
|
||||
let cql = "select ts_msp from ts_msp where series = ? and ts_msp >= ? and ts_msp < ?";
|
||||
let res = scy
|
||||
.query(cql, (series as i64, range.beg as i64, range.end as i64))
|
||||
.await
|
||||
.err_conv()?;
|
||||
for row in res.rows_typed_or_empty::<(i64,)>() {
|
||||
let row = row.err_conv()?;
|
||||
ret2.push_back(row.0 as u64);
|
||||
}
|
||||
let cql = "select ts_msp from ts_msp where series = ? and ts_msp >= ? limit 1";
|
||||
let res = scy.query(cql, (series as i64, range.end as i64)).await.err_conv()?;
|
||||
for row in res.rows_typed_or_empty::<(i64,)>() {
|
||||
let row = row.err_conv()?;
|
||||
ret2.push_back(row.0 as u64);
|
||||
}
|
||||
trace!("find_ts_msp n1 {} n2 {}", ret1.len(), ret2.len());
|
||||
Ok((ret1, ret2))
|
||||
}
|
||||
|
||||
trait ValTy: Sized {
|
||||
type ScaTy: ScalarOps + std::default::Default;
|
||||
type ScyTy: scylla::cql_to_rust::FromCqlVal<scylla::frame::response::result::CqlValue>;
|
||||
type Container: Events + Appendable<Self>;
|
||||
fn from_scyty(inp: Self::ScyTy) -> Self;
|
||||
fn table_name() -> &'static str;
|
||||
fn default() -> Self;
|
||||
}
|
||||
|
||||
macro_rules! impl_scaty_scalar {
|
||||
($st:ty, $st_scy:ty, $table_name:expr) => {
|
||||
impl ValTy for $st {
|
||||
type ScaTy = $st;
|
||||
type ScyTy = $st_scy;
|
||||
type Container = EventsDim0<Self::ScaTy>;
|
||||
fn from_scyty(inp: Self::ScyTy) -> Self {
|
||||
inp as Self
|
||||
}
|
||||
fn table_name() -> &'static str {
|
||||
$table_name
|
||||
}
|
||||
fn default() -> Self {
|
||||
<Self as std::default::Default>::default()
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
macro_rules! impl_scaty_array {
|
||||
($vt:ty, $st:ty, $st_scy:ty, $table_name:expr) => {
|
||||
impl ValTy for $vt {
|
||||
type ScaTy = $st;
|
||||
type ScyTy = $st_scy;
|
||||
type Container = EventsDim1<Self::ScaTy>;
|
||||
fn from_scyty(inp: Self::ScyTy) -> Self {
|
||||
inp.into_iter().map(|x| x as Self::ScaTy).collect()
|
||||
}
|
||||
fn table_name() -> &'static str {
|
||||
$table_name
|
||||
}
|
||||
fn default() -> Self {
|
||||
Vec::new()
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
impl_scaty_scalar!(u8, i8, "events_scalar_u8");
|
||||
impl_scaty_scalar!(u16, i16, "events_scalar_u16");
|
||||
impl_scaty_scalar!(u32, i32, "events_scalar_u32");
|
||||
impl_scaty_scalar!(u64, i64, "events_scalar_u64");
|
||||
impl_scaty_scalar!(i8, i8, "events_scalar_i8");
|
||||
impl_scaty_scalar!(i16, i16, "events_scalar_i16");
|
||||
impl_scaty_scalar!(i32, i32, "events_scalar_i32");
|
||||
impl_scaty_scalar!(i64, i64, "events_scalar_i64");
|
||||
impl_scaty_scalar!(f32, f32, "events_scalar_f32");
|
||||
impl_scaty_scalar!(f64, f64, "events_scalar_f64");
|
||||
impl_scaty_scalar!(bool, bool, "events_scalar_bool");
|
||||
|
||||
impl_scaty_array!(Vec<u8>, u8, Vec<i8>, "events_array_u8");
|
||||
impl_scaty_array!(Vec<u16>, u16, Vec<i16>, "events_array_u16");
|
||||
impl_scaty_array!(Vec<u32>, u32, Vec<i32>, "events_array_u32");
|
||||
impl_scaty_array!(Vec<u64>, u64, Vec<i64>, "events_array_u64");
|
||||
impl_scaty_array!(Vec<i8>, i8, Vec<i8>, "events_array_i8");
|
||||
impl_scaty_array!(Vec<i16>, i16, Vec<i16>, "events_array_i16");
|
||||
impl_scaty_array!(Vec<i32>, i32, Vec<i32>, "events_array_i32");
|
||||
impl_scaty_array!(Vec<i64>, i64, Vec<i64>, "events_array_i64");
|
||||
impl_scaty_array!(Vec<f32>, f32, Vec<f32>, "events_array_f32");
|
||||
impl_scaty_array!(Vec<f64>, f64, Vec<f64>, "events_array_f64");
|
||||
impl_scaty_array!(Vec<bool>, bool, Vec<bool>, "events_array_bool");
|
||||
|
||||
struct ReadNextValuesOpts {
|
||||
series: u64,
|
||||
ts_msp: u64,
|
||||
range: ScyllaSeriesRange,
|
||||
fwd: bool,
|
||||
with_values: bool,
|
||||
scy: Arc<ScySession>,
|
||||
}
|
||||
|
||||
async fn read_next_values<ST>(opts: ReadNextValuesOpts) -> Result<Box<dyn Events>, Error>
|
||||
where
|
||||
ST: ValTy,
|
||||
{
|
||||
let series = opts.series;
|
||||
let ts_msp = opts.ts_msp;
|
||||
let range = opts.range;
|
||||
let fwd = opts.fwd;
|
||||
let scy = opts.scy;
|
||||
let table_name = ST::table_name();
|
||||
if range.end > i64::MAX as u64 {
|
||||
return Err(Error::with_msg_no_trace(format!("range.end overflows i64")));
|
||||
}
|
||||
let cql_fields = if opts.with_values {
|
||||
"ts_lsp, pulse, value"
|
||||
} else {
|
||||
"ts_lsp, pulse"
|
||||
};
|
||||
let ret = if fwd {
|
||||
let ts_lsp_min = if ts_msp < range.beg { range.beg - ts_msp } else { 0 };
|
||||
let ts_lsp_max = if ts_msp < range.end { range.end - ts_msp } else { 0 };
|
||||
trace!(
|
||||
"FWD ts_msp {} ts_lsp_min {} ts_lsp_max {} beg {} end {} {}",
|
||||
ts_msp,
|
||||
ts_lsp_min,
|
||||
ts_lsp_max,
|
||||
range.beg,
|
||||
range.end,
|
||||
table_name,
|
||||
);
|
||||
// TODO use prepared!
|
||||
let cql = format!(
|
||||
concat!(
|
||||
"select {} from {}",
|
||||
" where series = ? and ts_msp = ? and ts_lsp >= ? and ts_lsp < ?"
|
||||
),
|
||||
cql_fields, table_name,
|
||||
);
|
||||
let res = scy
|
||||
.query(
|
||||
cql,
|
||||
(series as i64, ts_msp as i64, ts_lsp_min as i64, ts_lsp_max as i64),
|
||||
)
|
||||
.await
|
||||
.err_conv()?;
|
||||
let mut last_before = None;
|
||||
let mut ret = ST::Container::empty();
|
||||
for row in res.rows().err_conv()? {
|
||||
let (ts, pulse, value) = if opts.with_values {
|
||||
let row: (i64, i64, ST::ScyTy) = row.into_typed().err_conv()?;
|
||||
let ts = ts_msp + row.0 as u64;
|
||||
let pulse = row.1 as u64;
|
||||
let value = ValTy::from_scyty(row.2);
|
||||
(ts, pulse, value)
|
||||
} else {
|
||||
let row: (i64, i64) = row.into_typed().err_conv()?;
|
||||
let ts = ts_msp + row.0 as u64;
|
||||
let pulse = row.1 as u64;
|
||||
let value = ValTy::default();
|
||||
(ts, pulse, value)
|
||||
};
|
||||
if ts >= range.end {
|
||||
// TODO count as logic error
|
||||
error!("ts >= range.end");
|
||||
} else if ts >= range.beg {
|
||||
if pulse % 27 != 3618 {
|
||||
ret.push(ts, pulse, value);
|
||||
}
|
||||
} else {
|
||||
if last_before.is_none() {
|
||||
warn!("encounter event before range in forward read {ts}");
|
||||
}
|
||||
last_before = Some((ts, pulse, value));
|
||||
}
|
||||
}
|
||||
ret
|
||||
} else {
|
||||
let ts_lsp_max = if ts_msp < range.beg { range.beg - ts_msp } else { 0 };
|
||||
trace!(
|
||||
"BCK ts_msp {} ts_lsp_max {} beg {} end {} {}",
|
||||
ts_msp,
|
||||
ts_lsp_max,
|
||||
range.beg,
|
||||
range.end,
|
||||
table_name,
|
||||
);
|
||||
// TODO use prepared!
|
||||
let cql = format!(
|
||||
concat!(
|
||||
"select {} from {}",
|
||||
" where series = ? and ts_msp = ? and ts_lsp < ? order by ts_lsp desc limit 1"
|
||||
),
|
||||
cql_fields, table_name,
|
||||
);
|
||||
let res = scy
|
||||
.query(cql, (series as i64, ts_msp as i64, ts_lsp_max as i64))
|
||||
.await
|
||||
.err_conv()?;
|
||||
let mut seen_before = false;
|
||||
let mut ret = ST::Container::empty();
|
||||
for row in res.rows().err_conv()? {
|
||||
let (ts, pulse, value) = if opts.with_values {
|
||||
let row: (i64, i64, ST::ScyTy) = row.into_typed().err_conv()?;
|
||||
let ts = ts_msp + row.0 as u64;
|
||||
let pulse = row.1 as u64;
|
||||
let value = ValTy::from_scyty(row.2);
|
||||
(ts, pulse, value)
|
||||
} else {
|
||||
let row: (i64, i64) = row.into_typed().err_conv()?;
|
||||
let ts = ts_msp + row.0 as u64;
|
||||
let pulse = row.1 as u64;
|
||||
let value = ValTy::default();
|
||||
(ts, pulse, value)
|
||||
};
|
||||
if ts >= range.beg {
|
||||
// TODO count as logic error
|
||||
error!("ts >= range.beg");
|
||||
} else if ts < range.beg {
|
||||
if pulse % 27 != 3618 {
|
||||
ret.push(ts, pulse, value);
|
||||
}
|
||||
} else {
|
||||
seen_before = true;
|
||||
}
|
||||
}
|
||||
let _ = seen_before;
|
||||
if ret.len() > 1 {
|
||||
error!("multiple events in backwards search {}", ret.len());
|
||||
}
|
||||
ret
|
||||
};
|
||||
trace!("read ts_msp {} len {}", ts_msp, ret.len());
|
||||
let ret = Box::new(ret);
|
||||
Ok(ret)
|
||||
}
|
||||
|
||||
struct ReadValues {
|
||||
series: u64,
|
||||
scalar_type: ScalarType,
|
||||
shape: Shape,
|
||||
range: ScyllaSeriesRange,
|
||||
ts_msps: VecDeque<u64>,
|
||||
fwd: bool,
|
||||
with_values: bool,
|
||||
fut: Pin<Box<dyn Future<Output = Result<Box<dyn Events>, Error>> + Send>>,
|
||||
fut_done: bool,
|
||||
scy: Arc<ScySession>,
|
||||
}
|
||||
|
||||
impl ReadValues {
|
||||
fn new(
|
||||
series: u64,
|
||||
scalar_type: ScalarType,
|
||||
shape: Shape,
|
||||
range: ScyllaSeriesRange,
|
||||
ts_msps: VecDeque<u64>,
|
||||
fwd: bool,
|
||||
with_values: bool,
|
||||
scy: Arc<ScySession>,
|
||||
) -> Self {
|
||||
let mut ret = Self {
|
||||
series,
|
||||
scalar_type,
|
||||
shape,
|
||||
range,
|
||||
ts_msps,
|
||||
fwd,
|
||||
with_values,
|
||||
fut: Box::pin(futures_util::future::ready(Err(Error::with_msg_no_trace(
|
||||
"future not initialized",
|
||||
)))),
|
||||
fut_done: false,
|
||||
scy,
|
||||
};
|
||||
ret.next();
|
||||
ret
|
||||
}
|
||||
|
||||
fn next(&mut self) -> bool {
|
||||
if let Some(ts_msp) = self.ts_msps.pop_front() {
|
||||
self.fut = self.make_fut(ts_msp);
|
||||
self.fut_done = false;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
fn make_fut(&mut self, ts_msp: u64) -> Pin<Box<dyn Future<Output = Result<Box<dyn Events>, Error>> + Send>> {
|
||||
let opts = ReadNextValuesOpts {
|
||||
series: self.series.clone(),
|
||||
ts_msp,
|
||||
range: self.range.clone(),
|
||||
fwd: self.fwd,
|
||||
with_values: self.with_values,
|
||||
scy: self.scy.clone(),
|
||||
};
|
||||
let scalar_type = self.scalar_type.clone();
|
||||
let shape = self.shape.clone();
|
||||
let fut = async move {
|
||||
match &shape {
|
||||
Shape::Scalar => match &scalar_type {
|
||||
ScalarType::U8 => read_next_values::<u8>(opts).await,
|
||||
ScalarType::U16 => read_next_values::<u16>(opts).await,
|
||||
ScalarType::U32 => read_next_values::<u32>(opts).await,
|
||||
ScalarType::U64 => read_next_values::<u64>(opts).await,
|
||||
ScalarType::I8 => read_next_values::<i8>(opts).await,
|
||||
ScalarType::I16 => read_next_values::<i16>(opts).await,
|
||||
ScalarType::I32 => read_next_values::<i32>(opts).await,
|
||||
ScalarType::I64 => read_next_values::<i64>(opts).await,
|
||||
ScalarType::F32 => read_next_values::<f32>(opts).await,
|
||||
ScalarType::F64 => read_next_values::<f64>(opts).await,
|
||||
ScalarType::BOOL => read_next_values::<bool>(opts).await,
|
||||
_ => {
|
||||
error!("TODO ReadValues add more types");
|
||||
err::todoval()
|
||||
}
|
||||
},
|
||||
Shape::Wave(_) => match &scalar_type {
|
||||
ScalarType::U8 => read_next_values::<Vec<u8>>(opts).await,
|
||||
ScalarType::U16 => read_next_values::<Vec<u16>>(opts).await,
|
||||
ScalarType::U32 => read_next_values::<Vec<u32>>(opts).await,
|
||||
ScalarType::U64 => read_next_values::<Vec<u64>>(opts).await,
|
||||
ScalarType::I8 => read_next_values::<Vec<i8>>(opts).await,
|
||||
ScalarType::I16 => read_next_values::<Vec<i16>>(opts).await,
|
||||
ScalarType::I32 => read_next_values::<Vec<i32>>(opts).await,
|
||||
ScalarType::I64 => read_next_values::<Vec<i64>>(opts).await,
|
||||
ScalarType::F32 => read_next_values::<Vec<f32>>(opts).await,
|
||||
ScalarType::F64 => read_next_values::<Vec<f64>>(opts).await,
|
||||
ScalarType::BOOL => read_next_values::<Vec<bool>>(opts).await,
|
||||
_ => {
|
||||
error!("TODO ReadValues add more types");
|
||||
err::todoval()
|
||||
}
|
||||
},
|
||||
_ => {
|
||||
error!("TODO ReadValues add more types");
|
||||
err::todoval()
|
||||
}
|
||||
}
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
}
|
||||
|
||||
enum FrState {
|
||||
New,
|
||||
FindMsp(Pin<Box<dyn Future<Output = Result<(VecDeque<u64>, VecDeque<u64>), Error>> + Send>>),
|
||||
ReadBack1(ReadValues),
|
||||
ReadBack2(ReadValues),
|
||||
ReadValues(ReadValues),
|
||||
DataDone,
|
||||
Done,
|
||||
}
|
||||
|
||||
pub struct EventsStreamScylla {
|
||||
state: FrState,
|
||||
series: u64,
|
||||
scalar_type: ScalarType,
|
||||
shape: Shape,
|
||||
range: ScyllaSeriesRange,
|
||||
do_one_before_range: bool,
|
||||
ts_msp_bck: VecDeque<u64>,
|
||||
ts_msp_fwd: VecDeque<u64>,
|
||||
scy: Arc<ScySession>,
|
||||
do_test_stream_error: bool,
|
||||
found_one_after: bool,
|
||||
with_values: bool,
|
||||
outqueue: VecDeque<Box<dyn Events>>,
|
||||
}
|
||||
|
||||
impl EventsStreamScylla {
|
||||
pub fn new(
|
||||
series: u64,
|
||||
range: ScyllaSeriesRange,
|
||||
do_one_before_range: bool,
|
||||
scalar_type: ScalarType,
|
||||
shape: Shape,
|
||||
with_values: bool,
|
||||
scy: Arc<ScySession>,
|
||||
do_test_stream_error: bool,
|
||||
) -> Self {
|
||||
Self {
|
||||
state: FrState::New,
|
||||
series,
|
||||
scalar_type,
|
||||
shape,
|
||||
range,
|
||||
do_one_before_range,
|
||||
ts_msp_bck: VecDeque::new(),
|
||||
ts_msp_fwd: VecDeque::new(),
|
||||
scy,
|
||||
do_test_stream_error,
|
||||
found_one_after: false,
|
||||
with_values,
|
||||
outqueue: VecDeque::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn ts_msps_found(&mut self, msps1: VecDeque<u64>, msps2: VecDeque<u64>) {
|
||||
trace!("ts_msps_found msps1 {msps1:?} msps2 {msps2:?}");
|
||||
self.ts_msp_bck = msps1;
|
||||
self.ts_msp_fwd = msps2;
|
||||
for x in self.ts_msp_bck.iter().rev() {
|
||||
let x = x.clone();
|
||||
if x >= self.range.end {
|
||||
info!("FOUND one-after because of MSP");
|
||||
self.found_one_after = true;
|
||||
}
|
||||
self.ts_msp_fwd.push_front(x);
|
||||
}
|
||||
trace!("ts_msp_bck {:?}", self.ts_msp_bck);
|
||||
trace!("ts_msp_fwd {:?}", self.ts_msp_fwd);
|
||||
if let Some(msp) = self.ts_msp_bck.pop_back() {
|
||||
trace!("Try ReadBack1");
|
||||
let st = ReadValues::new(
|
||||
self.series,
|
||||
self.scalar_type.clone(),
|
||||
self.shape.clone(),
|
||||
self.range.clone(),
|
||||
[msp].into(),
|
||||
false,
|
||||
self.with_values,
|
||||
self.scy.clone(),
|
||||
);
|
||||
self.state = FrState::ReadBack1(st);
|
||||
} else if self.ts_msp_fwd.len() > 0 {
|
||||
trace!("Go straight for forward read");
|
||||
let st = ReadValues::new(
|
||||
self.series,
|
||||
self.scalar_type.clone(),
|
||||
self.shape.clone(),
|
||||
self.range.clone(),
|
||||
mem::replace(&mut self.ts_msp_fwd, VecDeque::new()),
|
||||
true,
|
||||
self.with_values,
|
||||
self.scy.clone(),
|
||||
);
|
||||
self.state = FrState::ReadValues(st);
|
||||
} else {
|
||||
self.state = FrState::DataDone;
|
||||
}
|
||||
}
|
||||
|
||||
fn back_1_done(&mut self, item: Box<dyn Events>) {
|
||||
trace!("back_1_done item len {}", item.len());
|
||||
if item.len() > 0 {
|
||||
self.outqueue.push_back(item);
|
||||
if self.ts_msp_fwd.len() > 0 {
|
||||
let st = ReadValues::new(
|
||||
self.series,
|
||||
self.scalar_type.clone(),
|
||||
self.shape.clone(),
|
||||
self.range.clone(),
|
||||
mem::replace(&mut self.ts_msp_fwd, VecDeque::new()),
|
||||
true,
|
||||
self.with_values,
|
||||
self.scy.clone(),
|
||||
);
|
||||
self.state = FrState::ReadValues(st);
|
||||
} else {
|
||||
self.state = FrState::DataDone;
|
||||
}
|
||||
} else {
|
||||
if let Some(msp) = self.ts_msp_bck.pop_back() {
|
||||
trace!("Try ReadBack2");
|
||||
let st = ReadValues::new(
|
||||
self.series,
|
||||
self.scalar_type.clone(),
|
||||
self.shape.clone(),
|
||||
self.range.clone(),
|
||||
[msp].into(),
|
||||
false,
|
||||
self.with_values,
|
||||
self.scy.clone(),
|
||||
);
|
||||
self.state = FrState::ReadBack2(st);
|
||||
} else if self.ts_msp_fwd.len() > 0 {
|
||||
trace!("No 2nd back MSP, go for forward read");
|
||||
let st = ReadValues::new(
|
||||
self.series,
|
||||
self.scalar_type.clone(),
|
||||
self.shape.clone(),
|
||||
self.range.clone(),
|
||||
mem::replace(&mut self.ts_msp_fwd, VecDeque::new()),
|
||||
true,
|
||||
self.with_values,
|
||||
self.scy.clone(),
|
||||
);
|
||||
self.state = FrState::ReadValues(st);
|
||||
} else {
|
||||
trace!("No 2nd back MSP, but also nothing to go forward");
|
||||
self.state = FrState::DataDone;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn back_2_done(&mut self, item: Box<dyn Events>) {
|
||||
trace!("back_1_done item len {}", item.len());
|
||||
if item.len() > 0 {
|
||||
self.outqueue.push_back(item);
|
||||
}
|
||||
if self.ts_msp_fwd.len() > 0 {
|
||||
let st = ReadValues::new(
|
||||
self.series,
|
||||
self.scalar_type.clone(),
|
||||
self.shape.clone(),
|
||||
self.range.clone(),
|
||||
mem::replace(&mut self.ts_msp_fwd, VecDeque::new()),
|
||||
true,
|
||||
self.with_values,
|
||||
self.scy.clone(),
|
||||
);
|
||||
self.state = FrState::ReadValues(st);
|
||||
} else {
|
||||
self.state = FrState::DataDone;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Stream for EventsStreamScylla {
|
||||
type Item = Result<ChannelEvents, Error>;
|
||||
|
||||
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
|
||||
use Poll::*;
|
||||
if self.do_test_stream_error {
|
||||
let e = Error::with_msg(format!("Test PRIVATE STREAM error."))
|
||||
.add_public_msg(format!("Test PUBLIC STREAM error."));
|
||||
return Ready(Some(Err(e)));
|
||||
}
|
||||
loop {
|
||||
if let Some(item) = self.outqueue.pop_front() {
|
||||
item.verify();
|
||||
item.output_info();
|
||||
break Ready(Some(Ok(ChannelEvents::Events(item))));
|
||||
}
|
||||
break match self.state {
|
||||
FrState::New => {
|
||||
let fut = find_ts_msp(self.series, self.range.clone(), self.scy.clone());
|
||||
let fut = Box::pin(fut);
|
||||
self.state = FrState::FindMsp(fut);
|
||||
continue;
|
||||
}
|
||||
FrState::FindMsp(ref mut fut) => match fut.poll_unpin(cx) {
|
||||
Ready(Ok((msps1, msps2))) => {
|
||||
self.ts_msps_found(msps1, msps2);
|
||||
continue;
|
||||
}
|
||||
Ready(Err(e)) => {
|
||||
self.state = FrState::DataDone;
|
||||
Ready(Some(Err(e)))
|
||||
}
|
||||
Pending => Pending,
|
||||
},
|
||||
FrState::ReadBack1(ref mut st) => match st.fut.poll_unpin(cx) {
|
||||
Ready(Ok(item)) => {
|
||||
st.fut_done = true;
|
||||
self.back_1_done(item);
|
||||
continue;
|
||||
}
|
||||
Ready(Err(e)) => {
|
||||
st.fut_done = true;
|
||||
self.state = FrState::DataDone;
|
||||
Ready(Some(Err(e)))
|
||||
}
|
||||
Pending => Pending,
|
||||
},
|
||||
FrState::ReadBack2(ref mut st) => match st.fut.poll_unpin(cx) {
|
||||
Ready(Ok(item)) => {
|
||||
st.fut_done = true;
|
||||
self.back_2_done(item);
|
||||
continue;
|
||||
}
|
||||
Ready(Err(e)) => {
|
||||
st.fut_done = true;
|
||||
self.state = FrState::DataDone;
|
||||
Ready(Some(Err(e)))
|
||||
}
|
||||
Pending => Pending,
|
||||
},
|
||||
FrState::ReadValues(ref mut st) => match st.fut.poll_unpin(cx) {
|
||||
Ready(Ok(item)) => {
|
||||
st.fut_done = true;
|
||||
if !st.next() {
|
||||
trace!("ReadValues exhausted");
|
||||
self.state = FrState::DataDone;
|
||||
}
|
||||
if item.len() > 0 {
|
||||
self.outqueue.push_back(item);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
Ready(Err(e)) => {
|
||||
st.fut_done = true;
|
||||
Ready(Some(Err(e)))
|
||||
}
|
||||
Pending => Pending,
|
||||
},
|
||||
FrState::DataDone => {
|
||||
if self.found_one_after {
|
||||
// TODO emit RangeComplete
|
||||
}
|
||||
self.state = FrState::Done;
|
||||
continue;
|
||||
}
|
||||
FrState::Done => Ready(None),
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
pub mod bincache;
|
||||
pub mod errconv;
|
||||
pub mod events;
|
||||
pub mod status;
|
||||
|
||||
use err::Error;
|
||||
use errconv::ErrConv;
|
||||
use netpod::range::evrange::SeriesRange;
|
||||
use netpod::ScyllaConfig;
|
||||
use scylla::execution_profile::ExecutionProfileBuilder;
|
||||
use scylla::statement::Consistency;
|
||||
use scylla::Session as ScySession;
|
||||
use std::sync::Arc;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ScyllaSeriesRange {
|
||||
beg: u64,
|
||||
end: u64,
|
||||
}
|
||||
|
||||
impl From<&SeriesRange> for ScyllaSeriesRange {
|
||||
fn from(value: &SeriesRange) -> Self {
|
||||
match value {
|
||||
SeriesRange::TimeRange(k) => Self { beg: k.beg, end: k.end },
|
||||
SeriesRange::PulseRange(k) => Self { beg: k.beg, end: k.end },
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn create_scy_session(scyconf: &ScyllaConfig) -> Result<Arc<ScySession>, Error> {
|
||||
let scy = scylla::SessionBuilder::new()
|
||||
.known_nodes(&scyconf.hosts)
|
||||
.use_keyspace(&scyconf.keyspace, true)
|
||||
.default_execution_profile_handle(
|
||||
ExecutionProfileBuilder::default()
|
||||
.consistency(Consistency::LocalOne)
|
||||
.build()
|
||||
.into_handle(),
|
||||
)
|
||||
.build()
|
||||
.await
|
||||
.err_conv()?;
|
||||
let ret = Arc::new(scy);
|
||||
Ok(ret)
|
||||
}
|
||||
@@ -0,0 +1,246 @@
|
||||
use crate::errconv::ErrConv;
|
||||
use err::Error;
|
||||
use futures_util::Future;
|
||||
use futures_util::FutureExt;
|
||||
use futures_util::Stream;
|
||||
use items_2::channelevents::ChannelStatus;
|
||||
use items_2::channelevents::ChannelStatusEvent;
|
||||
use netpod::log::*;
|
||||
use netpod::range::evrange::NanoRange;
|
||||
use netpod::range::evrange::SeriesRange;
|
||||
use netpod::CONNECTION_STATUS_DIV;
|
||||
use scylla::Session as ScySession;
|
||||
use std::collections::VecDeque;
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::task::Context;
|
||||
use std::task::Poll;
|
||||
use std::time::Duration;
|
||||
use std::time::SystemTime;
|
||||
|
||||
async fn read_next_status_events(
|
||||
series: u64,
|
||||
ts_msp: u64,
|
||||
range: NanoRange,
|
||||
fwd: bool,
|
||||
do_one_before: bool,
|
||||
scy: Arc<ScySession>,
|
||||
) -> Result<VecDeque<ChannelStatusEvent>, Error> {
|
||||
if ts_msp >= range.end {
|
||||
warn!(
|
||||
"given ts_msp {} >= range.end {} not necessary to read this",
|
||||
ts_msp, range.end
|
||||
);
|
||||
}
|
||||
if range.end > i64::MAX as u64 {
|
||||
return Err(Error::with_msg_no_trace(format!("range.end overflows i64")));
|
||||
}
|
||||
let res = if fwd {
|
||||
let ts_lsp_min = if ts_msp < range.beg { range.beg - ts_msp } else { 0 };
|
||||
let ts_lsp_max = if ts_msp < range.end { range.end - ts_msp } else { 0 };
|
||||
trace!(
|
||||
"FWD ts_msp {} ts_lsp_min {} ts_lsp_max {} beg {} end {}",
|
||||
ts_msp,
|
||||
ts_lsp_min,
|
||||
ts_lsp_max,
|
||||
range.beg,
|
||||
range.end
|
||||
);
|
||||
// TODO use prepared!
|
||||
let cql = concat!(
|
||||
"select ts_lsp, kind from channel_status where series = ? and ts_msp = ? and ts_lsp >= ? and ts_lsp < ?"
|
||||
);
|
||||
scy.query(
|
||||
cql,
|
||||
(series as i64, ts_msp as i64, ts_lsp_min as i64, ts_lsp_max as i64),
|
||||
)
|
||||
.await
|
||||
.err_conv()?
|
||||
} else {
|
||||
let ts_lsp_max = if ts_msp < range.beg { range.beg - ts_msp } else { 0 };
|
||||
debug!(
|
||||
"BCK ts_msp {} ts_lsp_max {} beg {} end {} {}",
|
||||
ts_msp,
|
||||
ts_lsp_max,
|
||||
range.beg,
|
||||
range.end,
|
||||
stringify!($fname)
|
||||
);
|
||||
// TODO use prepared!
|
||||
let cql = concat!(
|
||||
"select ts_lsp, kind from channel_status where series = ? and ts_msp = ? and ts_lsp < ? order by ts_lsp desc limit 1"
|
||||
);
|
||||
scy.query(cql, (series as i64, ts_msp as i64, ts_lsp_max as i64))
|
||||
.await
|
||||
.err_conv()?
|
||||
};
|
||||
let mut last_before = None;
|
||||
let mut ret = VecDeque::new();
|
||||
for row in res.rows_typed_or_empty::<(i64, i32)>() {
|
||||
let row = row.err_conv()?;
|
||||
let ts = ts_msp + row.0 as u64;
|
||||
let kind = row.1 as u32;
|
||||
let ev = ChannelStatusEvent {
|
||||
ts,
|
||||
datetime: SystemTime::UNIX_EPOCH + Duration::from_millis(ts / 1000000),
|
||||
status: ChannelStatus::from_ca_ingest_status_kind(kind),
|
||||
};
|
||||
if ts >= range.end {
|
||||
} else if ts >= range.beg {
|
||||
ret.push_back(ev);
|
||||
} else {
|
||||
last_before = Some(ev);
|
||||
}
|
||||
}
|
||||
if do_one_before {
|
||||
if let Some(ev) = last_before {
|
||||
debug!("PREPENDING THE LAST BEFORE {ev:?}");
|
||||
ret.push_front(ev);
|
||||
}
|
||||
}
|
||||
trace!("found in total {} events ts_msp {}", ret.len(), ts_msp);
|
||||
Ok(ret)
|
||||
}
|
||||
|
||||
struct ReadValues {
|
||||
series: u64,
|
||||
range: NanoRange,
|
||||
ts_msps: VecDeque<u64>,
|
||||
fwd: bool,
|
||||
do_one_before_range: bool,
|
||||
fut: Pin<Box<dyn Future<Output = Result<VecDeque<ChannelStatusEvent>, Error>> + Send>>,
|
||||
scy: Arc<ScySession>,
|
||||
}
|
||||
|
||||
impl ReadValues {
|
||||
fn new(
|
||||
series: u64,
|
||||
range: NanoRange,
|
||||
ts_msps: VecDeque<u64>,
|
||||
fwd: bool,
|
||||
do_one_before_range: bool,
|
||||
scy: Arc<ScySession>,
|
||||
) -> Self {
|
||||
let mut ret = Self {
|
||||
series,
|
||||
range,
|
||||
ts_msps,
|
||||
fwd,
|
||||
do_one_before_range,
|
||||
fut: Box::pin(futures_util::future::ready(Err(Error::with_msg_no_trace(
|
||||
"future not initialized",
|
||||
)))),
|
||||
scy,
|
||||
};
|
||||
ret.next();
|
||||
ret
|
||||
}
|
||||
|
||||
fn next(&mut self) -> bool {
|
||||
if let Some(ts_msp) = self.ts_msps.pop_front() {
|
||||
self.fut = self.make_fut(ts_msp);
|
||||
true
|
||||
} else {
|
||||
info!("no more msp");
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
fn make_fut(
|
||||
&mut self,
|
||||
ts_msp: u64,
|
||||
) -> Pin<Box<dyn Future<Output = Result<VecDeque<ChannelStatusEvent>, Error>> + Send>> {
|
||||
info!("make fut for {ts_msp}");
|
||||
let fut = read_next_status_events(
|
||||
self.series,
|
||||
ts_msp,
|
||||
self.range.clone(),
|
||||
self.fwd,
|
||||
self.do_one_before_range,
|
||||
self.scy.clone(),
|
||||
);
|
||||
Box::pin(fut)
|
||||
}
|
||||
}
|
||||
|
||||
enum FrState {
|
||||
New,
|
||||
ReadValues(ReadValues),
|
||||
Done,
|
||||
}
|
||||
|
||||
pub struct StatusStreamScylla {
|
||||
state: FrState,
|
||||
series: u64,
|
||||
range: NanoRange,
|
||||
do_one_before_range: bool,
|
||||
scy: Arc<ScySession>,
|
||||
outbuf: VecDeque<ChannelStatusEvent>,
|
||||
}
|
||||
|
||||
impl StatusStreamScylla {
|
||||
pub fn new(series: u64, range: NanoRange, do_one_before_range: bool, scy: Arc<ScySession>) -> Self {
|
||||
Self {
|
||||
state: FrState::New,
|
||||
series,
|
||||
range,
|
||||
do_one_before_range,
|
||||
scy,
|
||||
outbuf: VecDeque::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Stream for StatusStreamScylla {
|
||||
type Item = Result<ChannelStatusEvent, Error>;
|
||||
|
||||
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
|
||||
use Poll::*;
|
||||
let span = tracing::span!(tracing::Level::TRACE, "poll_next");
|
||||
let _spg = span.enter();
|
||||
loop {
|
||||
if let Some(x) = self.outbuf.pop_front() {
|
||||
break Ready(Some(Ok(x)));
|
||||
}
|
||||
break match self.state {
|
||||
FrState::New => {
|
||||
let mut ts_msps = VecDeque::new();
|
||||
let mut ts = self.range.beg / CONNECTION_STATUS_DIV * CONNECTION_STATUS_DIV;
|
||||
while ts < self.range.end {
|
||||
info!("Use ts {ts}");
|
||||
ts_msps.push_back(ts);
|
||||
ts += CONNECTION_STATUS_DIV;
|
||||
}
|
||||
let st = ReadValues::new(
|
||||
self.series,
|
||||
self.range.clone(),
|
||||
ts_msps,
|
||||
true,
|
||||
self.do_one_before_range,
|
||||
self.scy.clone(),
|
||||
);
|
||||
self.state = FrState::ReadValues(st);
|
||||
continue;
|
||||
}
|
||||
FrState::ReadValues(ref mut st) => match st.fut.poll_unpin(cx) {
|
||||
Ready(Ok(item)) => {
|
||||
if !st.next() {
|
||||
debug!("ReadValues exhausted");
|
||||
self.state = FrState::Done;
|
||||
}
|
||||
for x in item {
|
||||
self.outbuf.push_back(x);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
Ready(Err(e)) => {
|
||||
error!("{e}");
|
||||
Ready(Some(Err(e)))
|
||||
}
|
||||
Pending => Pending,
|
||||
},
|
||||
FrState::Done => Ready(None),
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user