daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Reduce pg conn create, move lookup into worker mod

Browse Source
This commit is contained in:
Dominik Werder
2024-07-24 14:00:28 +02:00
parent a4228d03a4
commit 3889d8bf37
7 changed files with 81 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/httpret/src/api4/events.rs
+10 -16
View File
@@ -24,6 +24,7 @@ use httpclient::StreamResponse;
use httpclient::ToJsonBody;
use netpod::log::*;
use netpod::req_uri_to_url;
use netpod::ChannelTypeConfigGen;
use netpod::FromUrl;
use netpod::NodeConfigCached;
use netpod::ReqCtx;
@@ -87,12 +88,15 @@ async fn plain_events(
pgqueue: &PgQueue,
ncc: &NodeConfigCached,
) -> Result<StreamResponse, Error> {
let ch_conf = chconf_from_events_quorum(&evq, ctx, pgqueue, ncc)
.await?
.ok_or_else(|| Error::with_msg_no_trace("channel not found"))?;
if accepts_cbor_framed(req.headers()) {
Ok(plain_events_cbor_framed(req, evq, ctx, pgqueue, ncc).await?)
Ok(plain_events_cbor_framed(req, evq, ch_conf, ctx, ncc).await?)
} else if accepts_json_framed(req.headers()) {
Ok(plain_events_json_framed(req, evq, ctx, pgqueue, ncc).await?)
Ok(plain_events_json_framed(req, evq, ch_conf, ctx, ncc).await?)
} else if accepts_json_or_all(req.headers()) {
Ok(plain_events_json(req, evq, ctx, pgqueue, ncc).await?)
Ok(plain_events_json(req, evq, ch_conf, ctx, ncc).await?)
} else {
let ret = response_err_msg(StatusCode::NOT_ACCEPTABLE, format!("unsupported accept {:?}", req))?;
Ok(ret)
@@ -102,13 +106,10 @@ async fn plain_events(
async fn plain_events_cbor_framed(
req: Requ,
evq: PlainEventsQuery,
ch_conf: ChannelTypeConfigGen,
ctx: &ReqCtx,
pgqueue: &PgQueue,
ncc: &NodeConfigCached,
) -> Result<StreamResponse, Error> {
let ch_conf = chconf_from_events_quorum(&evq, ctx, pgqueue, ncc)
.await?
.ok_or_else(|| Error::with_msg_no_trace("channel not found"))?;
debug!("plain_events_cbor_framed chconf_from_events_quorum: {ch_conf:?} {req:?}");
let open_bytes = OpenBoxedBytesViaHttp::new(ncc.node_config.cluster.clone());
let stream = streams::plaineventscbor::plain_events_cbor_stream(&evq, ch_conf, ctx, Box::pin(open_bytes)).await?;
@@ -130,13 +131,10 @@ async fn plain_events_cbor_framed(
async fn plain_events_json_framed(
req: Requ,
evq: PlainEventsQuery,
ch_conf: ChannelTypeConfigGen,
ctx: &ReqCtx,
pgqueue: &PgQueue,
ncc: &NodeConfigCached,
) -> Result<StreamResponse, Error> {
let ch_conf = chconf_from_events_quorum(&evq, ctx, pgqueue, ncc)
.await?
.ok_or_else(|| Error::with_msg_no_trace("channel not found"))?;
debug!("plain_events_json_framed chconf_from_events_quorum: {ch_conf:?} {req:?}");
let open_bytes = OpenBoxedBytesViaHttp::new(ncc.node_config.cluster.clone());
let stream = streams::plaineventsjson::plain_events_json_stream(&evq, ch_conf, ctx, Box::pin(open_bytes)).await?;
@@ -148,18 +146,14 @@ async fn plain_events_json_framed(
async fn plain_events_json(
req: Requ,
evq: PlainEventsQuery,
ch_conf: ChannelTypeConfigGen,
ctx: &ReqCtx,
pgqueue: &PgQueue,
ncc: &NodeConfigCached,
) -> Result<StreamResponse, Error> {
let self_name = "plain_events_json";
debug!("{self_name} req: {:?}", req);
let (_head, _body) = req.into_parts();
// TODO handle None case better and return 404
let ch_conf = chconf_from_events_quorum(&evq, ctx, pgqueue, ncc)
.await
.map_err(Error::from)?
.ok_or_else(|| Error::with_msg_no_trace("channel not found"))?;
debug!("{self_name} chconf_from_events_quorum: {ch_conf:?}");
let open_bytes = OpenBoxedBytesViaHttp::new(ncc.node_config.cluster.clone());
let item =
crates/httpret/src/channelconfig.rs
+5 -12
View File
@@ -36,7 +36,6 @@ use nodenet::configquorum::find_config_basics_quorum;
use query::api4::binned::BinnedQuery;
use query::api4::events::PlainEventsQuery;
use scyllaconn::errconv::ErrConv;
use scyllaconn::range::ScyllaSeriesRange;
use serde::Deserialize;
use serde::Serialize;
use std::collections::BTreeMap;
@@ -524,6 +523,8 @@ impl IocForChannel {
&[&q.backend, &q.name],
)
.await?;
drop(pg_client);
pgjh.await??;
if let Some(row) = rows.first() {
let ioc_addr = row.get(0);
let ret = IocForChannelRes { ioc_addr };
@@ -626,7 +627,7 @@ impl ScyllaSeriesTsMsp {
};
let chconf = shared_res
.pgqueue
.chconf_best_matching_name_range_job(backend, name, nano_range)
.chconf_best_matching_name_range(backend, name, nano_range)
.await
.map_err(|e| Error::with_msg_no_trace(format!("error from pg worker: {e}")))?
.recv()
@@ -734,6 +735,8 @@ impl AmbigiousChannelNames {
&[],
)
.await?;
drop(pg_client);
pgjh.await??;
let mut ret = AmbigiousChannelNamesResponse { ambigious: Vec::new() };
for row in rows {
let g = AmbigiousChannel {
@@ -748,16 +751,6 @@ impl AmbigiousChannelNames {
}
}
struct TestData01Iter {}
impl Iterator for TestData01Iter {
type Item = f64;
fn next(&mut self) -> Option<Self::Item> {
None
}
}
struct Msps(Vec<u64>);
struct Lsps(Vec<u64>);
struct Pulses(Vec<u64>);