daqbuf/daqingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Factor out ioc finder to db crate

Browse Source
This commit is contained in:
Dominik Werder
2023-08-28 22:58:47 +02:00
parent 837265a7b3
commit e05970ef56
29 changed files with 617 additions and 633 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
netfetch/src/batcher.rs
View File

@@ -1,55 +0,0 @@
use async_channel::Receiver;
use netpod::log::*;
use std::time::Duration;
use taskrun::tokio;
pub fn batch<T>(
batch_limit: usize,
timeout: Duration,
outcap: usize,
rx: Receiver<T>,
) -> (Receiver<Vec<T>>, tokio::task::JoinHandle<()>)
where
T: Send + 'static,
{
let (batch_tx, batch_rx) = async_channel::bounded(outcap);
let fut2 = async move {
let mut all = Vec::new();
let mut do_emit = false;
loop {
if do_emit {
do_emit = false;
let batch = std::mem::replace(&mut all, Vec::new());
match batch_tx.send(batch).await {
Ok(_) => {}
Err(e) => {
error!("can not send batch");
all = e.0;
}
}
}
match tokio::time::timeout(timeout, rx.recv()).await {
Ok(k) => match k {
Ok(item) => {
all.push(item);
if all.len() >= batch_limit {
do_emit = true;
}
}
Err(e) => {
error!("error in batcher, no more input {e}");
break;
}
},
Err(e) => {
let _e: tokio::time::error::Elapsed = e;
if all.len() > 0 {
do_emit = true;
}
}
}
}
warn!("-------- batcher is done --------------");
};
(batch_rx, tokio::spawn(fut2))
}

1
netfetch/src/batchquery.rs
View File

@@ -1 +0,0 @@
pub mod series_by_channel;

306
netfetch/src/batchquery/series_by_channel.rs
View File

@@ -1,306 +0,0 @@
use crate::batcher;
use crate::dbpg::make_pg_client;
use crate::errconv::ErrConv;
use crate::series::Existence;
use crate::series::SeriesId;
use async_channel::Receiver;
use async_channel::Sender;
use err::Error;
use futures_util::StreamExt;
use md5::Digest;
use netpod::log::*;
use netpod::Database;
use std::time::Duration;
use std::time::Instant;
use taskrun::tokio;
use tokio::task::JoinHandle;
use tokio_postgres::Client as PgClient;
use tokio_postgres::Statement as PgStatement;
pub struct ChannelInfoQuery {
pub backend: String,
pub channel: String,
pub scalar_type: i32,
pub shape_dims: Vec<i32>,
pub tx: Sender<Result<Existence<SeriesId>, Error>>,
}
impl ChannelInfoQuery {
pub fn dummy(&self) -> Self {
Self {
backend: String::new(),
channel: String::new(),
scalar_type: -1,
shape_dims: Vec::new(),
tx: self.tx.clone(),
}
}
}
struct ChannelInfoResult {
series: Existence<SeriesId>,
tx: Sender<Result<Existence<SeriesId>, Error>>,
}
struct PgRes {
pgc: PgClient,
st: PgStatement,
}
async fn prepare_pgcs(sql: &str, pgcn: usize, db: &Database) -> Result<(Sender<PgRes>, Receiver<PgRes>), Error> {
let (pgc_tx, pgc_rx) = async_channel::bounded(pgcn);
for _ in 0..pgcn {
let pgc = make_pg_client(&db).await?;
let st = pgc.prepare(sql).await.map_err(|e| Error::from(e.to_string()))?;
let k = PgRes { pgc, st };
match pgc_tx.send(k).await {
Ok(_) => {}
Err(e) => {
error!("can not enqueue pgc {e}");
}
}
}
Ok((pgc_tx, pgc_rx))
}
async fn select(
batch: Vec<ChannelInfoQuery>,
pgres: PgRes,
) -> Result<(Vec<ChannelInfoResult>, Vec<ChannelInfoQuery>, PgRes), Error> {
let mut backend = Vec::new();
let mut channel = Vec::new();
let mut scalar_type = Vec::new();
let mut shape_dims = Vec::new();
let mut shape_dims_str: Vec<String> = Vec::new();
let mut rid = Vec::new();
let mut tx = Vec::new();
for (i, e) in batch.into_iter().enumerate() {
backend.push(e.backend);
channel.push(e.channel);
scalar_type.push(e.scalar_type);
let mut dims = String::with_capacity(32);
dims.push('{');
for (i, &v) in e.shape_dims.iter().enumerate() {
if i > 0 {
dims.push(',');
}
use std::fmt::Write;
write!(dims, "{}", v).unwrap();
}
dims.push('}');
shape_dims_str.push(dims);
shape_dims.push(e.shape_dims);
rid.push(i as i32);
tx.push((i as u32, e.tx));
}
match pgres
.pgc
.query(&pgres.st, &[&backend, &channel, &scalar_type, &shape_dims_str, &rid])
.await
.map_err(|e| {
error!("{e}");
Error::from(e.to_string())
}) {
Ok(rows) => {
let mut result = Vec::new();
let mut missing = Vec::new();
let mut it1 = rows.into_iter();
let mut e1 = it1.next();
for (qrid, tx) in tx {
if let Some(row) = &e1 {
let rid: i32 = row.get(1);
if rid as u32 == qrid {
let series: i64 = row.get(0);
let series = SeriesId::new(series as _);
let res = ChannelInfoResult {
series: Existence::Existing(series),
tx,
};
result.push(res);
}
e1 = it1.next();
} else {
let i = qrid as usize;
let k = ChannelInfoQuery {
backend: backend[i].clone(),
channel: channel[i].clone(),
scalar_type: scalar_type[i].clone(),
shape_dims: shape_dims[i].clone(),
tx,
};
missing.push(k);
}
}
Ok((result, missing, pgres))
}
Err(e) => {
error!("error in pg query {e}");
tokio::time::sleep(Duration::from_millis(2000)).await;
Err(e)
}
}
}
async fn insert_missing(batch: &Vec<ChannelInfoQuery>, pgres: PgRes) -> Result<((), PgRes), Error> {
let tsbeg = Instant::now();
let mut backends = Vec::new();
let mut channels = Vec::new();
let mut scalar_types = Vec::new();
let mut shape_dimss = Vec::new();
let mut shape_dims_strs: Vec<String> = Vec::new();
let mut hashers = Vec::new();
for e in batch.into_iter() {
{
let mut h = md5::Md5::new();
h.update(e.backend.as_bytes());
h.update(e.channel.as_bytes());
h.update(format!("{:?}", e.scalar_type).as_bytes());
h.update(format!("{:?}", e.shape_dims).as_bytes());
hashers.push(h);
}
backends.push(&e.backend);
channels.push(&e.channel);
scalar_types.push(e.scalar_type);
let mut dims = String::with_capacity(32);
dims.push('{');
for (i, &v) in e.shape_dims.iter().enumerate() {
if i > 0 {
dims.push(',');
}
use std::fmt::Write;
write!(dims, "{}", v).unwrap();
}
dims.push('}');
shape_dims_strs.push(dims);
shape_dimss.push(&e.shape_dims);
}
let mut i1 = 0;
loop {
i1 += 1;
if i1 >= 200 {
return Err(Error::with_msg_no_trace("not able to generate series information"));
}
let mut seriess = Vec::with_capacity(hashers.len());
let mut all_good = true;
for h in &mut hashers {
let mut good = false;
for _ in 0..50 {
h.update(tsbeg.elapsed().subsec_nanos().to_ne_bytes());
let f = h.clone().finalize();
let series = u64::from_le_bytes(f.as_slice()[0..8].try_into().unwrap());
if series >= 100000000000000000 && series <= i64::MAX as u64 {
seriess.push(series as i64);
good = true;
break;
}
}
if !good {
all_good = false;
break;
}
}
if !all_good {
continue;
}
let sql = concat!(
"with q1 as (select * from unnest($1::text[], $2::text[], $3::int[], $4::text[], $5::bigint[])",
" as inp (backend, channel, scalar_type, shape_dims, series))",
" insert into series_by_channel (series, facility, channel, scalar_type, shape_dims, agg_kind)",
" select series, backend, channel, scalar_type, shape_dims::int[], 0 from q1",
" on conflict do nothing"
);
pgres
.pgc
.execute(sql, &[&backends, &channels, &scalar_types, &shape_dims_strs, &seriess])
.await
.err_conv()?;
break;
}
Ok(((), pgres))
}
async fn fetch_data(batch: Vec<ChannelInfoQuery>, pgres: PgRes) -> Result<(Vec<ChannelInfoResult>, PgRes), Error> {
let (res1, missing, pgres) = select(batch, pgres).await?;
if missing.len() > 0 {
let ((), pgres) = insert_missing(&missing, pgres).await?;
let (res2, missing2, pgres) = select(missing, pgres).await?;
if missing2.len() > 0 {
Err(Error::with_msg_no_trace("some series not found even after write"))
} else {
Ok((res2, pgres))
}
} else {
Ok((res1, pgres))
}
}
async fn run_queries(
npg: usize,
batch_rx: Receiver<Vec<ChannelInfoQuery>>,
pgc_rx: Receiver<PgRes>,
pgc_tx: Sender<PgRes>,
) -> Result<(), Error> {
let mut stream = batch_rx
.map(|batch| {
debug!("see batch of {}", batch.len());
let pgc_rx = pgc_rx.clone();
let pgc_tx = pgc_tx.clone();
async move {
if let Ok(pgres) = pgc_rx.recv().await {
let (res, pgres) = fetch_data(batch, pgres).await?;
if let Err(_) = pgc_tx.send(pgres).await {
error!("can not hand back pgres");
Err(Error::with_msg_no_trace("can not hand back pgres"))
} else {
Ok(res)
}
} else {
error!("can not get pgc");
Err(Error::with_msg_no_trace("no more pgres"))
}
}
})
.buffer_unordered(npg);
while let Some(item) = stream.next().await {
match item {
Ok(res) => {
for r in res {
match r.tx.send(Ok(r.series)).await {
Ok(_) => {}
Err(e) => {
// TODO count cases, but no log. Client may no longer be interested in this result.
error!("{e}");
}
}
}
}
Err(e) => {
error!("{e}");
tokio::time::sleep(Duration::from_millis(1000)).await;
}
}
}
info!("run_queries done");
Ok(())
}
pub async fn start_task(db: &Database) -> Result<Sender<ChannelInfoQuery>, Error> {
let sql = concat!(
"with q1 as (select * from unnest($1::text[], $2::text[], $3::int[], $4::text[], $5::int[])",
" as inp (backend, channel, scalar_type, shape_dims, rid))",
" select t.series, q1.rid from series_by_channel t",
" join q1 on t.facility = q1.backend and t.channel = q1.channel",
" and t.scalar_type = q1.scalar_type and t.shape_dims = q1.shape_dims::int[]",
" and t.agg_kind = 0",
" order by q1.rid",
);
let inp_cap = 128;
let batch_out_cap = 4;
let pgcn = 4;
let timeout = Duration::from_millis(200);
let (pgc_tx, pgc_rx) = prepare_pgcs(sql, pgcn, db).await?;
let (query_tx, query_rx) = async_channel::bounded(inp_cap);
let (batch_rx, _batch_jh) = batcher::batch(inp_cap, timeout, batch_out_cap, query_rx);
let _queries_jh: JoinHandle<_> = tokio::task::spawn(run_queries(pgcn, batch_rx, pgc_rx, pgc_tx));
Ok(query_tx)
}

6
netfetch/src/bsreadclient.rs
View File

@@ -1,12 +1,8 @@
use crate::batchquery::series_by_channel::ChannelInfoQuery;
use crate::bsread::BsreadMessage;
use crate::bsread::ChannelDescDecoded;
use crate::bsread::HeadB;
use crate::bsread::Parser;
use crate::ca::proto::CaDataArrayValue;
use crate::ca::proto::CaDataValue;
use crate::ca::IngestCommons;
use crate::series::SeriesId;
use crate::zmtp::zmtpproto;
use crate::zmtp::zmtpproto::SocketType;
use crate::zmtp::zmtpproto::Zmtp;
@@ -15,6 +11,7 @@ use crate::zmtp::zmtpproto::ZmtpMessage;
use crate::zmtp::ZmtpClientOpts;
use crate::zmtp::ZmtpEvent;
use async_channel::Sender;
use dbpg::seriesbychannel::ChannelInfoQuery;
use err::thiserror;
use err::ThisError;
use futures_util::StreamExt;
@@ -31,6 +28,7 @@ use scywr::iteminsertqueue::DataValue;
use scywr::iteminsertqueue::InsertItem;
use scywr::iteminsertqueue::QueryItem;
use scywr::session::ScySession;
use series::SeriesId;
use stats::CheckEvery;
use std::io;
use std::net::SocketAddr;

98
netfetch/src/ca.rs
View File

@@ -5,7 +5,6 @@ pub mod proto;
pub mod search;
use crate::ca::connset::CaConnSet;
use crate::errconv::ErrConv;
use crate::metrics::ExtraInsertsConf;
use crate::rt::TokMx;
use err::Error;
@@ -19,15 +18,15 @@ use scywr::store::DataStore;
use stats::CaConnStatsAgg;
use std::net::SocketAddrV4;
use std::pin::Pin;
use std::sync::atomic;
use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
use std::sync::atomic::AtomicU32;
use std::sync::atomic::AtomicU64;
use std::sync::atomic::Ordering;
use std::sync::Arc;
use std::sync::Mutex;
use std::task::Poll;
use std::time::Duration;
use std::time::Instant;
use taskrun::tokio;
use tokio_postgres::Client as PgClient;
pub static SIGINT: AtomicU32 = AtomicU32::new(0);
@@ -145,97 +144,6 @@ where
}
}
pub async fn find_channel_addr(
backend: String,
name: String,
pgconf: &Database,
) -> Result<Option<SocketAddrV4>, Error> {
// TODO also here, provide a db pool.
let d = pgconf;
let (pg_client, pg_conn) = tokio_postgres::connect(
&format!("postgresql://{}:{}@{}:{}/{}", d.user, d.pass, d.host, d.port, d.name),
tokio_postgres::tls::NoTls,
)
.await
.unwrap();
// TODO allow clean shutdown on ctrl-c and join the pg_conn in the end:
tokio::spawn(async {
pg_conn.await.unwrap();
info!("drop pg conn after find_channel_addr");
});
let pg_client = Arc::new(pg_client);
let qu_find_addr = pg_client
.prepare(
"select t1.facility, t1.channel, t1.addr from ioc_by_channel_log t1 where t1.facility = $1 and t1.channel = $2 and addr is not null order by tsmod desc limit 1",
)
.await
.err_conv()?;
let rows = pg_client.query(&qu_find_addr, &[&backend, &name]).await.err_conv()?;
if rows.is_empty() {
error!("can not find any addresses of channels {:?}", name);
Err(Error::with_msg_no_trace(format!("no address for channel {}", name)))
} else {
for row in rows {
match row.try_get::<_, &str>(2) {
Ok(addr) => match addr.parse::<SocketAddrV4>() {
Ok(addr) => return Ok(Some(addr)),
Err(e) => {
error!("can not parse {e:?}");
return Err(Error::with_msg_no_trace(format!("no address for channel {}", name)));
}
},
Err(e) => {
error!("can not find addr for {name} {e:?}");
}
}
}
Ok(None)
}
}
#[allow(unused)]
async fn query_addr_multiple(pg_client: &PgClient) -> Result<(), Error> {
let backend: &String = err::todoval();
// TODO factor the find loop into a separate Stream.
let qu_find_addr = pg_client
.prepare("with q1 as (select t1.facility, t1.channel, t1.addr from ioc_by_channel_log t1 where t1.facility = $1 and t1.channel in ($2, $3, $4, $5, $6, $7, $8, $9) and t1.addr is not null order by t1.tsmod desc) select distinct on (q1.facility, q1.channel) q1.facility, q1.channel, q1.addr from q1")
.await
.map_err(|e| Error::with_msg_no_trace(format!("{e:?}")))?;
let mut chns_todo: &[String] = err::todoval();
let mut chstmp = ["__NONE__"; 8];
for (s1, s2) in chns_todo.iter().zip(chstmp.iter_mut()) {
*s2 = s1;
}
chns_todo = &chns_todo[chstmp.len().min(chns_todo.len())..];
let rows = pg_client
.query(
&qu_find_addr,
&[
&backend, &chstmp[0], &chstmp[1], &chstmp[2], &chstmp[3], &chstmp[4], &chstmp[5], &chstmp[6],
&chstmp[7],
],
)
.await
.map_err(|e| Error::with_msg_no_trace(format!("pg lookup error: {e:?}")))?;
for row in rows {
let ch: &str = row.get(1);
let addr: &str = row.get(2);
if addr == "" {
// TODO the address was searched before but could not be found.
} else {
let addr: SocketAddrV4 = match addr.parse() {
Ok(k) => k,
Err(e) => {
error!("can not parse {addr:?} for channel {ch:?} {e:?}");
continue;
}
};
let _ = addr;
}
}
Ok(())
}
fn handler_sigaction(_a: libc::c_int, _b: *const libc::siginfo_t, _c: *const libc::c_void) {
crate::ca::SIGINT.store(1, Ordering::Release);
let _ = crate::linuxhelper::unset_signal_handler(libc::SIGINT);

18
netfetch/src/ca/conn.rs
View File

@@ -4,15 +4,12 @@ use super::proto::CaMsg;
use super::proto::CaMsgTy;
use super::proto::CaProto;
use super::ExtraInsertsConf;
use crate::batchquery::series_by_channel::ChannelInfoQuery;
use crate::bsread::ChannelDescDecoded;
use crate::ca::proto::CreateChan;
use crate::ca::proto::EventAdd;
use crate::series::ChannelStatusSeriesId;
use crate::series::Existence;
use crate::series::SeriesId;
use crate::timebin::ConnTimeBin;
use async_channel::Sender;
use dbpg::seriesbychannel::ChannelInfoQuery;
use err::Error;
use futures_util::stream::FuturesUnordered;
use futures_util::Future;
@@ -39,6 +36,9 @@ use scywriiq::IvlItem;
use scywriiq::MuteItem;
use scywriiq::QueryItem;
use serde::Serialize;
use series::series::Existence;
use series::ChannelStatusSeriesId;
use series::SeriesId;
use stats::CaConnStats;
use stats::IntervalEma;
use std::collections::BTreeMap;
@@ -764,7 +764,7 @@ impl CaConn {
ChannelState::Created(series, ..) => {
let item = QueryItem::ChannelStatus(ChannelStatusItem {
ts: SystemTime::now(),
series: series.into(),
series: series.clone(),
status: ChannelStatus::Closed(channel_reason.clone()),
});
self.insert_item_queue.push_back(item);
@@ -897,7 +897,7 @@ impl CaConn {
st.info_store_msp_last = msp;
let item = QueryItem::ChannelInfo(ChannelInfoItem {
ts_msp: msp,
series: series.into(),
series: series.clone(),
ivl: st.item_recv_ivl_ema.ema().ema(),
interest: 0.,
evsize: 0,
@@ -1277,7 +1277,7 @@ impl CaConn {
st.insert_recv_ivl_last = tsnow;
let ema = st.insert_item_ivl_ema.ema();
let item = IvlItem {
series: (&series).into(),
series: series.clone(),
ts,
ema: ema.ema(),
emd: ema.emv().sqrt(),
@@ -1287,7 +1287,7 @@ impl CaConn {
if false && st.muted_before == 0 {
let ema = st.insert_item_ivl_ema.ema();
let item = MuteItem {
series: series.into(),
series: series.clone(),
ts,
ema: ema.ema(),
emd: ema.emv().sqrt(),
@@ -1496,7 +1496,7 @@ impl CaConn {
match rx.recv().await {
Ok(item) => match item {
Ok(item) => Ok((cid, sid, k.data_type, k.data_count, item)),
Err(e) => Err(e),
Err(e) => Err(Error::with_msg_no_trace(e.to_string())),
},
Err(e) => {
// TODO count only

4
netfetch/src/ca/connset.rs
View File

@@ -2,15 +2,14 @@ use super::conn::CaConnEvent;
use super::conn::ChannelSetOps;
use super::conn::ConnCommand;
use super::SlowWarnable;
use crate::batchquery::series_by_channel::ChannelInfoQuery;
use crate::ca::conn::CaConn;
use crate::ca::conn::CaConnEventValue;
use crate::errconv::ErrConv;
use crate::rt::JoinHandle;
use crate::rt::TokMx;
use crate::series::ChannelStatusSeriesId;
use async_channel::Receiver;
use async_channel::Sender;
use dbpg::seriesbychannel::ChannelInfoQuery;
use err::Error;
use futures_util::FutureExt;
use futures_util::StreamExt;
@@ -18,6 +17,7 @@ use netpod::log::*;
use scywr::iteminsertqueue::CommonInsertItemQueue;
use scywr::iteminsertqueue::CommonInsertItemQueueSender;
use scywr::store::DataStore;
use series::ChannelStatusSeriesId;
use stats::CaConnStats;
use std::collections::BTreeMap;
use std::collections::VecDeque;

9
netfetch/src/errconv.rs
View File

@@ -23,12 +23,3 @@ impl<T> ErrConv<T> for Result<T, RecvError> {
}
}
}
impl<T> ErrConv<T> for Result<T, tokio_postgres::Error> {
fn err_conv(self) -> Result<T, Error> {
match self {
Ok(k) => Ok(k),
Err(e) => Err(Error::with_msg_no_trace(format!("{e:?}"))),
}
}
}

3
netfetch/src/lib.rs
View File

@@ -1,5 +1,3 @@
pub mod batcher;
pub mod batchquery;
pub mod bsread;
pub mod bsreadclient;
pub mod ca;
@@ -12,7 +10,6 @@ pub mod metrics;
pub mod netbuf;
pub mod patchcollect;
pub mod rt;
pub mod series;
#[cfg(test)]
pub mod test;
pub mod timebin;

145
netfetch/src/series.rs
View File

@@ -1,145 +0,0 @@
use crate::bsread::ChannelDescDecoded;
use crate::errconv::ErrConv;
use err::Error;
use log::*;
use serde::Serialize;
use std::time::{Duration, Instant};
use taskrun::tokio;
use tokio_postgres::Client as PgClient;
#[derive(Clone, Debug)]
pub enum Existence<T> {
Created(T),
Existing(T),
}
impl<T> Existence<T> {
pub fn into_inner(self) -> T {
use Existence::*;
match self {
Created(x) => x,
Existing(x) => x,
}
}
}
#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug, Serialize)]
pub struct SeriesId(u64);
impl SeriesId {
pub fn new(id: u64) -> Self {
Self(id)
}
pub fn id(&self) -> u64 {
self.0
}
}
impl From<&SeriesId> for scywr::iteminsertqueue::SeriesId {
fn from(value: &SeriesId) -> Self {
Self::new(value.id())
}
}
impl From<&mut SeriesId> for scywr::iteminsertqueue::SeriesId {
fn from(value: &mut SeriesId) -> Self {
Self::new(value.id())
}
}
impl From<SeriesId> for scywr::iteminsertqueue::SeriesId {
fn from(value: SeriesId) -> Self {
Self::new(value.id())
}
}
#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug, Serialize)]
pub struct ChannelStatusSeriesId(u64);
impl ChannelStatusSeriesId {
pub fn new(id: u64) -> Self {
Self(id)
}
pub fn id(&self) -> u64 {
self.0
}
}
// TODO don't need byte_order or compression from ChannelDescDecoded for channel registration.
pub async fn get_series_id(
pg_client: &PgClient,
cd: &ChannelDescDecoded,
backend: String,
) -> Result<Existence<SeriesId>, Error> {
let channel_name = &cd.name;
let scalar_type = cd.scalar_type.to_scylla_i32();
let shape = cd.shape.to_scylla_vec();
let res = pg_client
.query(
"select series from series_by_channel where facility = $1 and channel = $2 and scalar_type = $3 and shape_dims = $4 and agg_kind = 0",
&[&backend, channel_name, &scalar_type, &shape],
)
.await
.err_conv()?;
let mut all = vec![];
for row in res {
let series: i64 = row.get(0);
let series = series as u64;
all.push(series);
}
let rn = all.len();
let tsbeg = Instant::now();
if rn == 0 {
use md5::Digest;
let mut h = md5::Md5::new();
h.update(backend.as_bytes());
h.update(channel_name.as_bytes());
h.update(format!("{:?}", scalar_type).as_bytes());
h.update(format!("{:?}", shape).as_bytes());
for _ in 0..200 {
h.update(tsbeg.elapsed().subsec_nanos().to_ne_bytes());
let f = h.clone().finalize();
let series = u64::from_le_bytes(f.as_slice()[0..8].try_into().unwrap());
if series > i64::MAX as u64 {
continue;
}
if series == 0 {
continue;
}
if series <= 0 || series > i64::MAX as u64 {
return Err(Error::with_msg_no_trace(format!(
"attempt to insert bad series id {series}"
)));
}
let res = pg_client
.execute(
concat!(
"insert into series_by_channel",
" (series, facility, channel, scalar_type, shape_dims, agg_kind)",
" values ($1, $2, $3, $4, $5, 0) on conflict do nothing"
),
&[&(series as i64), &backend, channel_name, &scalar_type, &shape],
)
.await
.unwrap();
if res == 1 {
let series = Existence::Created(SeriesId(series));
return Ok(series);
} else {
warn!(
"tried to insert {series:?} for {backend:?} {channel_name:?} {scalar_type:?} {shape:?} trying again..."
);
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
error!("tried to insert new series id for {backend:?} {channel_name:?} {scalar_type:?} {shape:?} but failed");
Err(Error::with_msg_no_trace(format!("get_series_id can not create and insert series id {backend:?} {channel_name:?} {scalar_type:?} {shape:?}")))
} else {
let series = all[0] as u64;
let series = Existence::Existing(SeriesId(series));
debug!("get_series_id {backend:?} {channel_name:?} {scalar_type:?} {shape:?} {series:?}");
Ok(series)
}
}

7
netfetch/src/timebin.rs
View File

@@ -2,7 +2,6 @@ use crate::ca::proto;
use crate::ca::proto::CaDataValue;
use crate::ca::proto::CaEventValue;
use crate::patchcollect::PatchCollect;
use crate::series::SeriesId;
use err::Error;
use items_0::scalar_ops::ScalarOps;
use items_0::timebin::TimeBinner;
@@ -21,6 +20,7 @@ use netpod::Shape;
use netpod::TsNano;
use scywr::iteminsertqueue::QueryItem;
use scywr::iteminsertqueue::TimeBinPatchSimpleF32;
use series::SeriesId;
use std::any;
use std::any::Any;
use std::collections::VecDeque;
@@ -187,8 +187,7 @@ fn store_patch(series: SeriesId, pc: &mut PatchCollect, iiq: &mut VecDeque<Query
let off_msp = off / 1000;
let off_lsp = off % 1000;
let item = TimeBinPatchSimpleF32 {
// TODO use the same SeriesId type
series: (&series).into(),
series: series.clone(),
bin_len_sec: (pc.bin_len().ns() / SEC) as u32,
bin_count: pc.bin_count() as u32,
off_msp: off_msp as u32,
@@ -264,7 +263,7 @@ where
// TODO
//let off_msp =
let item = TimeBinPatchSimpleF32 {
series: (&params.series).into(),
series: params.series.clone(),
bin_len_sec: (pc.bin_len().ns() / SEC) as u32,
bin_count: pc.bin_count() as u32,
off_msp: 0,