daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove RawEventsQuery

Browse Source
This commit is contained in:
Dominik Werder
2022-12-09 20:46:23 +01:00
parent f1292a5b32
commit b9607f27d2
21 changed files with 542 additions and 431 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
httpret/src/api1.rs
View File

@@ -9,7 +9,7 @@ use items::eventfull::EventFull;
use items::{RangeCompletableItem, Sitemty, StreamItem};
use itertools::Itertools;
use netpod::query::api1::Api1Query;
use netpod::query::RawEventsQuery;
use netpod::query::PlainEventsQuery;
use netpod::timeunits::SEC;
use netpod::{log::*, ScalarType};
use netpod::{ByteSize, Channel, DiskIoTune, NanoRange, NodeConfigCached, PerfOpts, Shape};
@@ -773,7 +773,14 @@ impl Stream for DataApiPython3DataStream {
};
let channel = self.channels[self.chan_ix - 1].clone();
debug!("found channel_config for {}: {:?}", channel.name, entry);
let evq = RawEventsQuery::new(channel, self.range.clone(), netpod::AggKind::EventBlobs);
let evq = PlainEventsQuery::new(
channel,
self.range.clone(),
netpod::AggKind::EventBlobs,
Duration::from_millis(10000),
None,
true,
);
let perf_opts = PerfOpts { inmem_bufcap: 1024 * 4 };
// TODO is this a good to place decide this?
let s = if self.node_config.node_config.cluster.is_central_storage {
@@ -781,11 +788,11 @@ impl Stream for DataApiPython3DataStream {
// TODO pull up this config
let event_chunker_conf = EventChunkerConf::new(ByteSize::kb(1024));
let s = disk::raw::conn::make_local_event_blobs_stream(
evq.range.clone(),
evq.channel.clone(),
evq.range().clone(),
evq.channel().clone(),
&entry,
evq.agg_kind.need_expand(),
evq.do_decompress,
evq.agg_kind().need_expand(),
true,
event_chunker_conf,
self.disk_io_tune.clone(),
&self.node_config,

139
httpret/src/channelconfig.rs
View File

@@ -1,5 +1,6 @@
use crate::err::Error;
use crate::{response, ToPublicResponse};
use dbconn::channelconfig::{chconf_from_database, ChConf};
use dbconn::{create_connection, create_scylla_connection};
use futures_util::StreamExt;
use http::{Method, Request, Response, StatusCode};
@@ -21,144 +22,12 @@ use std::convert::TryInto;
use std::time::{Duration, Instant};
use url::Url;
pub struct ChConf {
pub series: u64,
pub scalar_type: ScalarType,
pub shape: Shape,
}
/// It is an unsolved question as to how we want to uniquely address channels.
/// Currently, the usual (backend, channelname) works in 99% of the cases, but the edge-cases
/// are not solved. At the same time, it is desirable to avoid to complicate things for users.
/// Current state:
/// If the series id is given, we take that.
/// Otherwise we try to uniquely identify the series id from the given information.
/// In the future, we can even try to involve time range information for that, but backends like
/// old archivers and sf databuffer do not support such lookup.
pub async fn chconf_from_database(channel: &Channel, ncc: &NodeConfigCached) -> Result<ChConf, Error> {
if channel.backend != ncc.node_config.cluster.backend {
warn!(
"mismatched backend {} vs {}",
channel.backend, ncc.node_config.cluster.backend
);
}
if channel.backend() == "test-inmem" {
let ret = if channel.name() == "inmem-d0-i32" {
let ret = ChConf {
series: 1,
scalar_type: ScalarType::I32,
shape: Shape::Scalar,
};
Ok(ret)
} else {
error!("no test information");
Err(Error::with_msg_no_trace(format!("no test information"))
.add_public_msg("No channel config for test channel {:?}"))
};
return ret;
}
if channel.backend() == "test-disk-databuffer" {
// TODO the series-ids here are just random. Need to integrate with better test setup.
let ret = if channel.name() == "scalar-i32-be" {
let ret = ChConf {
series: 1,
scalar_type: ScalarType::I32,
shape: Shape::Scalar,
};
Ok(ret)
} else if channel.name() == "wave-f64-be-n21" {
let ret = ChConf {
series: 2,
scalar_type: ScalarType::F64,
shape: Shape::Wave(21),
};
Ok(ret)
} else if channel.name() == "const-regular-scalar-i32-be" {
let ret = ChConf {
series: 3,
scalar_type: ScalarType::I32,
shape: Shape::Scalar,
};
Ok(ret)
} else {
error!("no test information");
Err(Error::with_msg_no_trace(format!("no test information"))
.add_public_msg("No channel config for test channel {:?}"))
};
return ret;
}
// TODO use a common already running worker pool for these queries:
let dbconf = &ncc.node_config.cluster.database;
let dburl = format!(
"postgresql://{}:{}@{}:{}/{}",
dbconf.user, dbconf.pass, dbconf.host, dbconf.port, dbconf.name
);
let (pgclient, pgconn) = tokio_postgres::connect(&dburl, tokio_postgres::NoTls)
.await
.err_conv()?;
tokio::spawn(pgconn);
if let Some(series) = channel.series() {
let res = pgclient
.query(
"select scalar_type, shape_dims from series_by_channel where series = $1",
&[&(series as i64)],
)
.await
.err_conv()?;
if res.len() < 1 {
warn!("can not find channel information for series {series} given through {channel:?}");
let e = Error::with_public_msg_no_trace(format!("can not find channel information for {channel:?}"));
Err(e)
} else {
let row = res.first().unwrap();
let scalar_type = ScalarType::from_dtype_index(row.get::<_, i32>(0) as u8)?;
// TODO can I get a slice from psql driver?
let shape = Shape::from_scylla_shape_dims(&row.get::<_, Vec<i32>>(1))?;
let ret = ChConf {
series,
scalar_type,
shape,
};
Ok(ret)
}
} else {
let res = pgclient
.query(
"select series, scalar_type, shape_dims from series_by_channel where facility = $1 and channel = $2",
&[&channel.backend(), &channel.name()],
)
.await
.err_conv()?;
if res.len() < 1 {
warn!("can not find channel information for {channel:?}");
let e = Error::with_public_msg_no_trace(format!("can not find channel information for {channel:?}"));
Err(e)
} else if res.len() > 1 {
warn!("ambigious channel {channel:?}");
let e = Error::with_public_msg_no_trace(format!("ambigious channel {channel:?}"));
Err(e)
} else {
let row = res.first().unwrap();
let series = row.get::<_, i64>(0) as u64;
let scalar_type = ScalarType::from_dtype_index(row.get::<_, i32>(1) as u8)?;
// TODO can I get a slice from psql driver?
let shape = Shape::from_scylla_shape_dims(&row.get::<_, Vec<i32>>(2))?;
let ret = ChConf {
series,
scalar_type,
shape,
};
Ok(ret)
}
}
}
pub async fn chconf_from_events_binary(q: &PlainEventsQuery, ncc: &NodeConfigCached) -> Result<ChConf, Error> {
chconf_from_database(q.channel(), ncc).await
chconf_from_database(q.channel(), ncc).await.map_err(Into::into)
}
pub async fn chconf_from_events_json(q: &PlainEventsQuery, ncc: &NodeConfigCached) -> Result<ChConf, Error> {
chconf_from_database(q.channel(), ncc).await
chconf_from_database(q.channel(), ncc).await.map_err(Into::into)
}
pub async fn chconf_from_prebinned(q: &PreBinnedQuery, _ncc: &NodeConfigCached) -> Result<ChConf, Error> {
@@ -174,7 +43,7 @@ pub async fn chconf_from_prebinned(q: &PreBinnedQuery, _ncc: &NodeConfigCached)
}
pub async fn chconf_from_binned(q: &BinnedQuery, ncc: &NodeConfigCached) -> Result<ChConf, Error> {
chconf_from_database(q.channel(), ncc).await
chconf_from_database(q.channel(), ncc).await.map_err(Into::into)
}
pub struct ChannelConfigHandler {}

32
httpret/src/events.rs
View File

@@ -8,15 +8,15 @@ use items_2::channelevents::ChannelEvents;
use items_2::merger_cev::ChannelEventsMerger;
use items_2::{binned_collected, empty_events_dyn, empty_events_dyn_2};
use netpod::log::*;
use netpod::query::{BinnedQuery, ChannelStateEventsQuery, PlainEventsQuery, RawEventsQuery};
use netpod::query::{BinnedQuery, ChannelStateEventsQuery, PlainEventsQuery};
use netpod::{AggKind, BinnedRange, FromUrl, NodeConfigCached};
use netpod::{ACCEPT_ALL, APP_JSON, APP_OCTET};
use scyllaconn::create_scy_session;
use scyllaconn::errconv::ErrConv;
use scyllaconn::events::{channel_state_events, find_series, make_scylla_stream};
use scyllaconn::events::{channel_state_events, make_scylla_stream};
use std::pin::Pin;
use std::sync::Arc;
use std::time::Instant;
use std::time::{Duration, Instant};
use url::Url;
pub struct EventsHandler {}
@@ -103,8 +103,15 @@ async fn plain_events_json(
query.set_series_id(chconf.series);
let query = query;
// ---
let query = RawEventsQuery::new(query.channel().clone(), query.range().clone(), AggKind::Plain);
let item = streams::plaineventsjson::plain_events_json(query, &node_config.node_config.cluster).await?;
//let query = RawEventsQuery::new(query.channel().clone(), query.range().clone(), AggKind::Plain);
let item = streams::plaineventsjson::plain_events_json(&query, &node_config.node_config.cluster).await;
let item = match item {
Ok(item) => item,
Err(e) => {
error!("got error from streams::plaineventsjson::plain_events_json {e:?}");
return Err(e.into());
}
};
let buf = serde_json::to_vec(&item)?;
let ret = response(StatusCode::OK).body(Body::from(buf))?;
Ok(ret)
@@ -188,8 +195,8 @@ impl EventsHandlerScylla {
return Err(Error::with_public_msg(format!("no scylla configured")));
};
let scy = create_scy_session(scyco).await?;
let do_one_before_range = evq.do_one_before_range();
let (series, scalar_type, shape) = find_series(evq.channel(), pgclient.clone()).await?;
let do_one_before_range = evq.agg_kind().need_expand();
let (series, scalar_type, shape) = dbconn::find_series(evq.channel(), pgclient.clone()).await?;
let empty_item = empty_events_dyn_2(&scalar_type, &shape, &AggKind::TimeWeightedScalar);
let empty_stream =
futures_util::stream::once(futures_util::future::ready(Ok(ChannelEvents::Events(empty_item))));
@@ -342,10 +349,17 @@ impl BinnedHandlerScylla {
let scy = create_scy_session(scyco).await?;
let covering = BinnedRange::covering_range(evq.range().clone(), evq.bin_count())?;
let range = covering.full_range();
let mut query2 = PlainEventsQuery::new(evq.channel().clone(), range.clone(), 0, None, false);
let mut query2 = PlainEventsQuery::new(
evq.channel().clone(),
range.clone(),
evq.agg_kind().clone(),
Duration::from_millis(6000),
None,
false,
);
query2.set_timeout(evq.timeout());
let query2 = query2;
let (series, scalar_type, shape) = find_series(evq.channel(), pgclient.clone()).await?;
let (series, scalar_type, shape) = dbconn::find_series(evq.channel(), pgclient.clone()).await?;
let stream = make_scylla_stream(
&query2,
do_one_before_range,