Move workspace crates into subfolder

crates/dbconn/src/channelconfig.rs

@@ -0,0 +1,82 @@
+use crate::ErrConv;
+use err::Error;
+use netpod::log::*;
+use netpod::ChConf;
+use netpod::NodeConfigCached;
+use netpod::ScalarType;
+use netpod::SfDbChannel;
+use netpod::Shape;
+
+/// It is an unsolved question as to how we want to uniquely address channels.
+/// Currently, the usual (backend, channelname) works in 99% of the cases, but the edge-cases
+/// are not solved. At the same time, it is desirable to avoid to complicate things for users.
+/// Current state:
+/// If the series id is given, we take that.
+/// Otherwise we try to uniquely identify the series id from the given information.
+/// In the future, we can even try to involve time range information for that, but backends like
+/// old archivers and sf databuffer do not support such lookup.
+pub async fn chconf_from_scylla_type_backend(channel: &SfDbChannel, ncc: &NodeConfigCached) -> Result<ChConf, Error> {
+    if channel.backend() != ncc.node_config.cluster.backend {
+        warn!(
+            "mismatched backend  {}  vs  {}",
+            channel.backend(),
+            ncc.node_config.cluster.backend
+        );
+    }
+    let backend = channel.backend();
+    let dbconf = &ncc.node_config.cluster.database;
+    let pgclient = crate::create_connection(dbconf).await?;
+    if let Some(series) = channel.series() {
+        let res = pgclient
+            .query(
+                "select channel, scalar_type, shape_dims from series_by_channel where facility = $1 and series = $2",
+                &[&channel.backend(), &(series as i64)],
+            )
+            .await
+            .err_conv()?;
+        if res.len() < 1 {
+            warn!("can not find channel information for series {series} given through {channel:?}");
+            let e = Error::with_public_msg_no_trace(format!("can not find channel information for {channel:?}"));
+            Err(e)
+        } else {
+            let row = res.first().unwrap();
+            let name: String = row.get(0);
+            let scalar_type = ScalarType::from_dtype_index(row.get::<_, i32>(1) as u8)?;
+            // TODO can I get a slice from psql driver?
+            let shape = Shape::from_scylla_shape_dims(&row.get::<_, Vec<i32>>(2))?;
+            let ret = ChConf::new(backend, series, scalar_type, shape, name);
+            Ok(ret)
+        }
+    } else {
+        if ncc.node_config.cluster.scylla.is_some() {
+            let res = pgclient
+            .query(
+                "select channel, series, scalar_type, shape_dims from series_by_channel where facility = $1 and channel = $2",
+                &[&channel.backend(), &channel.name()],
+            )
+            .await
+            .err_conv()?;
+            if res.len() < 1 {
+                warn!("can not find channel information for {channel:?}");
+                let e = Error::with_public_msg_no_trace(format!("can not find channel information for {channel:?}"));
+                Err(e)
+            } else if res.len() > 1 {
+                warn!("ambigious channel {channel:?}");
+                let e = Error::with_public_msg_no_trace(format!("ambigious channel {channel:?}"));
+                Err(e)
+            } else {
+                let row = res.first().unwrap();
+                let name: String = row.get(0);
+                let series = row.get::<_, i64>(1) as u64;
+                let scalar_type = ScalarType::from_dtype_index(row.get::<_, i32>(2) as u8)?;
+                // TODO can I get a slice from psql driver?
+                let shape = Shape::from_scylla_shape_dims(&row.get::<_, Vec<i32>>(3))?;
+                let ret = ChConf::new(backend, series, scalar_type, shape, name);
+                Ok(ret)
+            }
+        } else {
+            error!("TODO xm89ur8932cr");
+            Err(Error::with_msg_no_trace("TODO xm89ur8932cr"))
+        }
+    }
+}

crates/dbconn/src/dbconn.rs

@@ -0,0 +1,220 @@
+pub mod channelconfig;
+pub mod query;
+pub mod scan;
+pub mod search;
+
+pub mod pg {
+    pub use tokio_postgres::{Client, Error, NoTls};
+}
+
+use err::anyhow;
+use err::Error;
+use err::Res2;
+use netpod::log::*;
+use netpod::TableSizes;
+use netpod::{Database, NodeConfigCached, SfDbChannel};
+use netpod::{ScalarType, Shape};
+use pg::{Client as PgClient, NoTls};
+use std::sync::Arc;
+use std::time::Duration;
+
+trait ErrConv<T> {
+    fn err_conv(self) -> Result<T, Error>;
+}
+
+impl<T> ErrConv<T> for Result<T, tokio_postgres::Error> {
+    fn err_conv(self) -> Result<T, Error> {
+        match self {
+            Ok(k) => Ok(k),
+            Err(e) => Err(Error::with_msg(e.to_string())),
+        }
+    }
+}
+
+impl<T, A> ErrConv<T> for Result<T, async_channel::SendError<A>> {
+    fn err_conv(self) -> Result<T, Error> {
+        match self {
+            Ok(k) => Ok(k),
+            Err(e) => Err(Error::with_msg(e.to_string())),
+        }
+    }
+}
+
+pub async fn delay_us(mu: u64) {
+    tokio::time::sleep(Duration::from_micros(mu)).await;
+}
+
+pub async fn delay_io_short() {
+    delay_us(1000).await;
+}
+
+pub async fn delay_io_medium() {
+    delay_us(2000).await;
+}
+
+pub async fn create_connection(db_config: &Database) -> Result<PgClient, Error> {
+    // TODO use a common already running worker pool for these queries:
+    let d = db_config;
+    let uri = format!("postgresql://{}:{}@{}:{}/{}", d.user, d.pass, d.host, d.port, d.name);
+    let (cl, conn) = tokio_postgres::connect(&uri, NoTls)
+        .await
+        .map_err(|e| format!("Can not connect to database: {e:?}"))
+        //.errconv()
+        ?;
+    // TODO monitor connection drop.
+    let _cjh = tokio::spawn(async move {
+        if let Err(e) = conn.await {
+            error!("connection error: {}", e);
+        }
+        Ok::<_, Error>(())
+    });
+    Ok(cl)
+}
+
+pub async fn channel_exists(channel_name: &str, node_config: &NodeConfigCached) -> Result<bool, Error> {
+    let cl = create_connection(&node_config.node_config.cluster.database).await?;
+    let rows = cl
+        .query("select rowid from channels where name = $1::text", &[&channel_name])
+        .await
+        .err_conv()?;
+    debug!("channel_exists  {} rows", rows.len());
+    for row in rows {
+        debug!(
+            "  db on channel search: {:?}  {:?}  {:?}",
+            row,
+            row.columns(),
+            row.get::<_, i64>(0)
+        );
+    }
+    Ok(true)
+}
+
+pub async fn database_size(node_config: &NodeConfigCached) -> Result<u64, Error> {
+    let cl = create_connection(&node_config.node_config.cluster.database).await?;
+    let rows = cl
+        .query(
+            "select pg_database_size($1::text)",
+            &[&node_config.node_config.cluster.database.name],
+        )
+        .await
+        .err_conv()?;
+    if rows.len() == 0 {
+        Err(Error::with_msg("could not get database size"))?;
+    }
+    let size: i64 = rows[0].get(0);
+    let size = size as u64;
+    Ok(size)
+}
+
+pub async fn table_sizes(node_config: &NodeConfigCached) -> Result<TableSizes, Error> {
+    let sql = format!(
+        "{} {} {} {} {} {} {}",
+        "SELECT nspname || '.' || relname AS relation, pg_size_pretty(pg_total_relation_size(C.oid)) AS total_size",
+        "FROM pg_class C",
+        "LEFT JOIN pg_namespace N ON (N.oid = C.relnamespace)",
+        "WHERE nspname NOT IN ('pg_catalog', 'information_schema')",
+        "AND C.relkind <> 'i'",
+        "AND nspname !~ '^pg_toast'",
+        "ORDER BY pg_total_relation_size(C.oid) DESC LIMIT 20",
+    );
+    let sql = sql.as_str();
+    let cl = create_connection(&node_config.node_config.cluster.database).await?;
+    let rows = cl.query(sql, &[]).await.err_conv()?;
+    let mut sizes = TableSizes { sizes: Vec::new() };
+    sizes.sizes.push((format!("table"), format!("size")));
+    for row in rows {
+        sizes.sizes.push((row.get(0), row.get(1)));
+    }
+    Ok(sizes)
+}
+
+pub async fn random_channel(node_config: &NodeConfigCached) -> Result<String, Error> {
+    let sql = "select name from channels order by rowid limit 1 offset (random() * (select count(rowid) from channels))::bigint";
+    let cl = create_connection(&node_config.node_config.cluster.database).await?;
+    let rows = cl.query(sql, &[]).await.err_conv()?;
+    if rows.len() == 0 {
+        Err(Error::with_msg("can not get random channel"))?;
+    }
+    Ok(rows[0].get(0))
+}
+
+pub async fn insert_channel(name: String, facility: i64, dbc: &PgClient) -> Result<(), Error> {
+    let rows = dbc
+        .query(
+            "select count(rowid) from channels where facility = $1 and name = $2",
+            &[&facility, &name],
+        )
+        .await
+        .err_conv()?;
+    if rows[0].get::<_, i64>(0) == 0 {
+        let sql =
+            concat!("insert into channels (facility, name) values ($1, $2) on conflict (facility, name) do nothing");
+        dbc.query(sql, &[&facility, &name]).await.err_conv()?;
+    }
+    Ok(())
+}
+
+// Currently only for scylla type backends
+pub async fn find_series(channel: &SfDbChannel, pgclient: Arc<PgClient>) -> Result<(u64, ScalarType, Shape), Error> {
+    info!("find_series  channel {:?}", channel);
+    let rows = if let Some(series) = channel.series() {
+        let q = "select series, facility, channel, scalar_type, shape_dims from series_by_channel where series = $1";
+        pgclient.query(q, &[&(series as i64)]).await.err_conv()?
+    } else {
+        let q = "select series, facility, channel, scalar_type, shape_dims from series_by_channel where facility = $1 and channel = $2";
+        pgclient
+            .query(q, &[&channel.backend(), &channel.name()])
+            .await
+            .err_conv()?
+    };
+    if rows.len() < 1 {
+        return Err(Error::with_public_msg_no_trace(format!(
+            "No series found for {channel:?}"
+        )));
+    }
+    if rows.len() > 1 {
+        error!("Multiple series found for {channel:?}");
+        return Err(Error::with_public_msg_no_trace(
+            "Multiple series found for channel, can not return data for ambiguous series",
+        ));
+    }
+    let row = rows
+        .into_iter()
+        .next()
+        .ok_or_else(|| Error::with_public_msg_no_trace(format!("can not find series for channel")))?;
+    let series = row.get::<_, i64>(0) as u64;
+    let _facility: String = row.get(1);
+    let _channel: String = row.get(2);
+    let a: i32 = row.get(3);
+    let scalar_type = ScalarType::from_scylla_i32(a)?;
+    let a: Vec<i32> = row.get(4);
+    let shape = Shape::from_scylla_shape_dims(&a)?;
+    Ok((series, scalar_type, shape))
+}
+
+// Currently only for sf-databuffer type backends
+// Note: we currently treat the channels primary key as series-id for sf-databuffer type backends.
+pub async fn find_series_sf_databuffer(channel: &SfDbChannel, pgclient: Arc<PgClient>) -> Res2<u64> {
+    info!("find_series  channel {:?}", channel);
+    let sql = "select rowid from facilities where name = $1";
+    let rows = pgclient.query(sql, &[&channel.backend()]).await.err_conv()?;
+    let row = rows
+        .into_iter()
+        .next()
+        .ok_or_else(|| anyhow::anyhow!("no backend for {channel:?}"))?;
+    let backend_id: i64 = row.get(0);
+    let sql = "select rowid from channels where facility = $1 and name = $2";
+    let rows = pgclient.query(sql, &[&backend_id, &channel.name()]).await.err_conv()?;
+    if rows.len() < 1 {
+        return Err(anyhow::anyhow!("No series found for {channel:?}"));
+    }
+    if rows.len() > 1 {
+        return Err(anyhow::anyhow!("Multiple series found for {channel:?}"));
+    }
+    let row = rows
+        .into_iter()
+        .next()
+        .ok_or_else(|| anyhow::anyhow!("No series found for {channel:?}"))?;
+    let series = row.get::<_, i64>(0) as u64;
+    Ok(series)
+}

crates/dbconn/src/query.rs

@@ -0,0 +1,49 @@
+use crate::create_connection;
+use crate::ErrConv;
+use err::Error;
+use netpod::log::*;
+use netpod::NodeConfigCached;
+use netpod::SfDbChannel;
+
+// For sf-databuffer backend, given a Channel, try to complete the information if only id is given.
+#[allow(unused)]
+async fn sf_databuffer_fetch_channel_by_series(
+    channel: SfDbChannel,
+    ncc: &NodeConfigCached,
+) -> Result<SfDbChannel, Error> {
+    let me = "sf_databuffer_fetch_channel_by_series";
+    info!("{me}");
+    // TODO should not be needed at some point.
+    if channel.backend().is_empty() || channel.name().is_empty() {
+        if let Some(series) = channel.series() {
+            if series < 1 {
+                error!("{me}  bad input: {channel:?}");
+                Err(Error::with_msg_no_trace(format!("{me}  bad input: {channel:?}")))
+            } else {
+                info!("{me} do the lookup");
+                let series = channel
+                    .series()
+                    .ok_or_else(|| Error::with_msg_no_trace("no series id given"))? as i64;
+                let pgcon = create_connection(&ncc.node_config.cluster.database).await?;
+                let mut rows = pgcon
+                    .query("select name from channels where rowid = $1", &[&series])
+                    .await
+                    .err_conv()?;
+                if let Some(row) = rows.pop() {
+                    info!("{me} got a row {row:?}");
+                    let name: String = row.get(0);
+                    let channel = SfDbChannel::from_full(&ncc.node_config.cluster.backend, channel.series(), name);
+                    info!("{me} return {channel:?}");
+                    Ok(channel)
+                } else {
+                    info!("{me} nothing found");
+                    Err(Error::with_msg_no_trace("can not find series"))
+                }
+            }
+        } else {
+            Err(Error::with_msg_no_trace(format!("{me}  bad input: {channel:?}")))
+        }
+    } else {
+        Ok(channel)
+    }
+}

crates/dbconn/src/scan.rs

@@ -0,0 +1,765 @@
+use crate::create_connection;
+use crate::delay_io_medium;
+use crate::delay_io_short;
+use crate::ErrConv;
+use async_channel::bounded;
+use async_channel::Receiver;
+use chrono::DateTime;
+use chrono::Utc;
+use err::Error;
+use futures_util::FutureExt;
+use futures_util::Stream;
+use netpod::log::*;
+use netpod::Database;
+use netpod::NodeConfigCached;
+use parse::channelconfig::NErr;
+use pin_project::pin_project;
+use serde::Deserialize;
+use serde::Serialize;
+use std::future::Future;
+use std::io::ErrorKind;
+use std::os::unix::ffi::OsStringExt;
+use std::path::Path;
+use std::path::PathBuf;
+use std::pin::Pin;
+use std::sync::Arc;
+use std::sync::RwLock;
+use std::task::Context;
+use std::task::Poll;
+use tokio::fs::DirEntry;
+use tokio::fs::ReadDir;
+use tokio_postgres::Client;
+
+mod updatechannelnames;
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct NodeDiskIdent {
+    pub rowid: i64,
+    pub facility: i64,
+    pub split: i32,
+    pub hostname: String,
+}
+
+impl NodeDiskIdent {
+    pub fn rowid(&self) -> i64 {
+        self.rowid
+    }
+    pub fn facility(&self) -> i64 {
+        self.facility
+    }
+}
+
+fn _get_hostname() -> Result<String, Error> {
+    let out = std::process::Command::new("hostname").output()?;
+    Ok(String::from_utf8(out.stdout[..out.stdout.len() - 1].to_vec())?)
+}
+
+pub async fn get_node_disk_ident(node_config: &NodeConfigCached, dbc: &Client) -> Result<NodeDiskIdent, Error> {
+    let sql = "select nodes.rowid, facility, split, hostname from nodes, facilities where facilities.name = $1 and facility = facilities.rowid and hostname = $2";
+    let rows = dbc
+        .query(sql, &[&node_config.node_config.cluster.backend, &node_config.node.host])
+        .await
+        .err_conv()?;
+    if rows.len() != 1 {
+        return Err(Error::with_msg(format!(
+            "get_node can't find unique entry for {} {}",
+            node_config.node.host, node_config.node_config.cluster.backend
+        )));
+    }
+    let row = &rows[0];
+    Ok(NodeDiskIdent {
+        rowid: row.get(0),
+        facility: row.get(1),
+        split: row.get(2),
+        hostname: row.get(3),
+    })
+}
+
+pub async fn get_node_disk_ident_2(
+    node_config: Pin<&NodeConfigCached>,
+    dbc: Pin<&Client>,
+) -> Result<NodeDiskIdent, Error> {
+    let sql = "select nodes.rowid, facility, split, hostname from nodes, facilities where facilities.name = $1 and facility = facilities.rowid and hostname = $2";
+    let rows = dbc
+        .query(sql, &[&node_config.node_config.cluster.backend, &node_config.node.host])
+        .await
+        .err_conv()?;
+    if rows.len() != 1 {
+        return Err(Error::with_msg(format!(
+            "get_node can't find unique entry for {} {}",
+            node_config.node.host, node_config.node_config.cluster.backend
+        )));
+    }
+    let row = &rows[0];
+    Ok(NodeDiskIdent {
+        rowid: row.get(0),
+        facility: row.get(1),
+        split: row.get(2),
+        hostname: row.get(3),
+    })
+}
+
+#[pin_project]
+pub struct FindChannelNamesFromConfigReadDir {
+    #[pin]
+    read_dir_fut: Option<Pin<Box<dyn Future<Output = std::io::Result<ReadDir>> + Send>>>,
+    #[pin]
+    read_dir: Option<Pin<Box<ReadDir>>>,
+    #[pin]
+    done: bool,
+}
+
+impl FindChannelNamesFromConfigReadDir {
+    pub fn new(base_dir: impl AsRef<Path>) -> Self {
+        Self {
+            read_dir_fut: Some(Box::pin(tokio::fs::read_dir(base_dir.as_ref().join("config")))),
+            read_dir: None,
+            done: false,
+        }
+    }
+}
+
+impl Stream for FindChannelNamesFromConfigReadDir {
+    type Item = Result<DirEntry, Error>;
+
+    fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
+        use Poll::*;
+        let span = span!(Level::INFO, "FindChNameCfgDir");
+        let _spg = span.enter();
+        let mut pself = self.project();
+        loop {
+            break if *pself.done {
+                Ready(None)
+            } else if let Some(mut fut) = pself.read_dir.as_mut().as_pin_mut() {
+                match fut.poll_next_entry(cx) {
+                    Ready(Ok(Some(item))) => Ready(Some(Ok(item))),
+                    Ready(Ok(None)) => {
+                        *pself.done = true;
+                        Ready(None)
+                    }
+                    Ready(Err(e)) => {
+                        *pself.done = true;
+                        Ready(Some(Err(e.into())))
+                    }
+                    Pending => Pending,
+                }
+            } else if let Some(fut) = pself.read_dir_fut.as_mut().as_pin_mut() {
+                match fut.poll(cx) {
+                    Ready(Ok(item)) => {
+                        *pself.read_dir_fut = None;
+                        *pself.read_dir = Some(Box::pin(item));
+                        continue;
+                    }
+                    Ready(Err(e)) => {
+                        *pself.done = true;
+                        Ready(Some(Err(e.into())))
+                    }
+                    Pending => Pending,
+                }
+            } else {
+                Pending
+            };
+        }
+    }
+}
+
+async fn find_channel_names_from_config(base_dir: impl AsRef<Path>) -> Result<Vec<String>, Error> {
+    let mut ret = Vec::new();
+    let path2: PathBuf = base_dir.as_ref().join("config");
+    let mut rd = tokio::fs::read_dir(&path2).await?;
+    while let Ok(Some(entry)) = rd.next_entry().await {
+        let fname = String::from_utf8(entry.file_name().into_vec())?;
+        ret.push(fname);
+    }
+    Ok(ret)
+}
+
+#[allow(unused)]
+async fn update_db_with_channel_name_list(list: Vec<String>, backend: i64, dbc: &Client) -> Result<(), Error> {
+    delay_io_short().await;
+    dbc.query("begin", &[]).await.err_conv()?;
+    for ch in list {
+        dbc.query(
+            "insert into channels (facility, name) values ($1, $2) on conflict do nothing",
+            &[&backend, &ch],
+        )
+        .await
+        .err_conv()?;
+    }
+    dbc.query("commit", &[]).await.err_conv()?;
+    Ok(())
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct UpdatedDbWithChannelNames {
+    msg: String,
+    count: u32,
+}
+
+async fn update_db_with_channel_names_inner(
+    tx: async_channel::Sender<Result<UpdatedDbWithChannelNames, Error>>,
+    node_config: NodeConfigCached,
+    db_config: Database,
+) -> Result<(), Error> {
+    let dbc = create_connection(&db_config).await?;
+    info!("update_db_with_channel_names connection done");
+    let node_disk_ident = get_node_disk_ident(&node_config, &dbc).await?;
+    info!("update_db_with_channel_names get_node_disk_ident done");
+    let insert_sql = concat!(
+        "insert into channels (facility, name) select facility, name from (values ($1::bigint, $2::text)) v1 (facility, name)",
+        " where not exists (select 1 from channels t1 where t1.facility = v1.facility and t1.name = v1.name)",
+        " on conflict do nothing",
+    );
+    if false {
+        let fac: i64 = 1;
+        let ch = format!("tmp_dummy_04");
+        let ret = dbc
+            .query(insert_sql, &[&fac, &ch])
+            .await
+            .err_conv()
+            .map_err(|e| format!("in channel name insert: {e}"));
+        info!("DUMMY INSERT ATTEMPT: {ret:?}");
+    }
+    let mut c1 = 0;
+    dbc.query("begin", &[]).await.err_conv()?;
+    let dbc = Arc::new(dbc);
+    let tx = Arc::new(tx);
+    let base_path = &node_config
+        .node
+        .sf_databuffer
+        .as_ref()
+        .ok_or_else(|| Error::with_msg(format!("missing sf databuffer config in node")))?
+        .data_base_path;
+    let channel_names = find_channel_names_from_config(base_path).await?;
+    for ch in channel_names {
+        let fac = node_disk_ident.facility;
+        crate::delay_io_short().await;
+        let ret = dbc
+            .query(insert_sql, &[&fac, &ch])
+            .await
+            .err_conv()
+            .map_err(|e| format!("in channel name insert: {e}"));
+        let ret = match ret {
+            Ok(x) => x,
+            Err(e) => {
+                error!("failed insert attempt {e}");
+                Err(e)?
+            }
+        };
+        {
+            let n = ret.len();
+            if n > 0 {
+                info!("insert n {n}");
+            }
+        }
+        c1 += 1;
+        if c1 % 200 == 0 {
+            dbc.query("commit", &[]).await.err_conv()?;
+            let ret = UpdatedDbWithChannelNames {
+                msg: format!("current {}", ch),
+                count: c1,
+            };
+            tx.send(Ok(ret)).await.err_conv()?;
+            delay_io_medium().await;
+            dbc.query("begin", &[]).await.err_conv()?;
+        }
+    }
+    dbc.query("commit", &[]).await.err_conv()?;
+    let ret = UpdatedDbWithChannelNames {
+        msg: format!("all done"),
+        count: c1,
+    };
+    tx.send(Ok(ret)).await.err_conv()?;
+    Ok::<_, Error>(())
+}
+
+pub async fn update_db_with_channel_names(
+    node_config: NodeConfigCached,
+    db_config: &Database,
+) -> Result<Receiver<Result<UpdatedDbWithChannelNames, Error>>, Error> {
+    info!("update_db_with_channel_names");
+    let (tx, rx) = bounded(16);
+    let tx2 = tx.clone();
+    let db_config = db_config.clone();
+    let block1 = update_db_with_channel_names_inner(tx, node_config.clone(), db_config.clone());
+    let block2 = async move {
+        match block1.await {
+            Ok(_) => {}
+            Err(e) => match tx2.send(Err(e)).await {
+                Ok(_) => {}
+                Err(e) => {
+                    error!("can not report error through channel: {:?}", e);
+                }
+            },
+        }
+    };
+    tokio::spawn(block2);
+    Ok(rx)
+}
+
+pub fn update_db_with_channel_names_3(
+    node_config: &NodeConfigCached,
+) -> impl Stream<Item = Result<UpdatedDbWithChannelNames, Error>> + 'static {
+    let base_path = &node_config
+        .node
+        .sf_databuffer
+        .as_ref()
+        .ok_or_else(|| Error::with_msg(format!("missing sf databuffer config in node")))
+        .unwrap()
+        .data_base_path;
+    futures_util::future::ready(base_path.clone())
+        .then(|path| tokio::fs::read_dir(path))
+        .map(Result::unwrap)
+        .map(|rd| {
+            futures_util::stream::unfold(rd, move |rd| {
+                //let fut = rd.next_entry();
+                futures_util::future::ready(Ok(None)).map(move |item: Result<Option<u32>, Error>| match item {
+                    Ok(Some(item)) => Some((item, rd)),
+                    Ok(None) => None,
+                    Err(_e) => None,
+                })
+            })
+        })
+        .map(|_conf| Err(Error::with_msg("TODO")))
+        .into_stream()
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct UpdatedDbWithAllChannelConfigs {
+    msg: String,
+    count: u32,
+}
+
+async fn update_db_with_all_channel_configs_inner(
+    tx: async_channel::Sender<Result<UpdatedDbWithAllChannelConfigs, Error>>,
+    node_config: NodeConfigCached,
+) -> Result<(), Error> {
+    let node_config = &node_config;
+    let dbc = create_connection(&node_config.node_config.cluster.database).await?;
+    let dbc = Arc::new(dbc);
+    let node_disk_ident = &get_node_disk_ident(node_config, &dbc).await?;
+    let rows = dbc
+        .query(
+            "select rowid, facility, name from channels where facility = $1 order by facility, name",
+            &[&node_disk_ident.facility],
+        )
+        .await
+        .err_conv()?;
+    let mut c1 = 0;
+    dbc.query("begin", &[]).await.err_conv()?;
+    let mut count_inserted = 0;
+    let mut count_updated = 0;
+    let mut count_config_not_found = 0;
+    for row in rows {
+        let rowid: i64 = row.try_get(0).err_conv()?;
+        let _facility: i64 = row.try_get(1).err_conv()?;
+        let channel: String = row.try_get(2).err_conv()?;
+        match update_db_with_channel_config(
+            node_config,
+            node_disk_ident,
+            rowid,
+            &channel,
+            dbc.clone(),
+            &mut count_inserted,
+            &mut count_updated,
+        )
+        .await
+        {
+            Err(e) => {
+                error!("{:?}", e);
+                delay_io_medium().await;
+                // TODO recover, open new transaction, test recovery.
+                return Err(e);
+            }
+            Ok(UpdateChannelConfigResult::NotFound) => {
+                //warn!("can not find channel config {}", channel);
+                count_config_not_found += 1;
+                delay_io_short().await;
+            }
+            Ok(UpdateChannelConfigResult::Done) => {
+                c1 += 1;
+                if c1 % 200 == 0 {
+                    dbc.query("commit", &[]).await.err_conv()?;
+                    let msg = format!(
+                        "channel no {:6}  inserted {:6}  updated {:6}",
+                        c1, count_inserted, count_updated
+                    );
+                    let ret = UpdatedDbWithAllChannelConfigs { msg, count: c1 };
+                    tx.send(Ok(ret)).await.err_conv()?;
+                    dbc.query("begin", &[]).await.err_conv()?;
+                }
+                delay_io_short().await;
+            }
+        }
+    }
+    dbc.query("commit", &[]).await.err_conv()?;
+    let msg = format!(
+        "ALL DONE  channel no {:6}  inserted {:6}  updated {:6}  not_found {:6}",
+        c1, count_inserted, count_updated, count_config_not_found,
+    );
+    let ret = UpdatedDbWithAllChannelConfigs { msg, count: c1 };
+    tx.send(Ok(ret)).await.err_conv()?;
+    Ok::<_, Error>(())
+}
+
+pub async fn update_db_with_all_channel_configs(
+    node_config: NodeConfigCached,
+) -> Result<Receiver<Result<UpdatedDbWithAllChannelConfigs, Error>>, Error> {
+    let (tx, rx) = bounded(16);
+    let tx2 = tx.clone();
+    let tx3 = tx.clone();
+    let block1 = update_db_with_all_channel_configs_inner(tx, node_config).then({
+        |item| async move {
+            match item {
+                Ok(_) => {}
+                Err(e) => {
+                    let msg = format!("Seeing error: {:?}", e);
+                    let ret = UpdatedDbWithAllChannelConfigs { msg, count: 0 };
+                    tx2.send(Ok(ret)).await.err_conv()?;
+                }
+            }
+            Ok::<_, Error>(())
+        }
+    });
+    let block2 = async move {
+        match block1.await {
+            Ok(_) => {}
+            Err(e) => match tx3.send(Err(e)).await {
+                Ok(_) => {}
+                Err(e) => {
+                    error!("can not deliver error through channel: {:?}", e);
+                }
+            },
+        }
+    };
+    tokio::spawn(block2);
+    Ok(rx)
+}
+
+pub async fn update_search_cache(node_config: &NodeConfigCached) -> Result<bool, Error> {
+    let dbc = create_connection(&node_config.node_config.cluster.database).await?;
+    dbc.query("select update_cache()", &[])
+        .await
+        .err_conv()
+        .map_err(|e| format!("error update_search_cache: {e}"))?;
+    Ok(true)
+}
+
+pub enum UpdateChannelConfigResult {
+    NotFound,
+    Done,
+}
+
+/**
+Parse the config of the given channel and update database.
+*/
+async fn update_db_with_channel_config(
+    node_config: &NodeConfigCached,
+    node_disk_ident: &NodeDiskIdent,
+    channel_id: i64,
+    channel: &str,
+    dbc: Arc<Client>,
+    count_inserted: &mut usize,
+    count_updated: &mut usize,
+) -> Result<UpdateChannelConfigResult, Error> {
+    let base_path = &node_config
+        .node
+        .sf_databuffer
+        .as_ref()
+        .ok_or_else(|| Error::with_msg(format!("missing sf databuffer config in node")))?
+        .data_base_path;
+    let path = base_path
+        .join("config")
+        .join(channel)
+        .join("latest")
+        .join("00000_Config");
+    let meta = if let Ok(k) = tokio::fs::metadata(&path).await {
+        k
+    } else {
+        return Ok(UpdateChannelConfigResult::NotFound);
+    };
+    if meta.len() > 40 * 1024 * 1024 {
+        return Err(Error::with_msg("meta data too long {meta:?}"));
+    }
+    let rows = dbc
+        .query(
+            "select rowid, fileSize, parsedUntil, channel from configs where node = $1 and channel = $2",
+            &[&node_disk_ident.rowid(), &channel_id],
+        )
+        .await
+        .err_conv()?;
+    if rows.len() > 1 {
+        return Err(Error::with_msg("more than one row"));
+    }
+    let (config_id, do_parse) = if let Some(row) = rows.first() {
+        let rowid: i64 = row.get(0);
+        let file_size: u32 = row.get::<_, i64>(1) as u32;
+        let parsed_until: u32 = row.get::<_, i64>(2) as u32;
+        let _channel_id = row.get::<_, i64>(2) as i64;
+        if meta.len() < file_size as u64 || meta.len() < parsed_until as u64 {
+            let sql = concat!(
+                "insert into configs_history (rowid_original, node, channel, fileSize, parsedUntil, config, tsinsert) ",
+                "select rowid as rowid_original, node, channel, fileSize, parsedUntil, config, now() from configs where rowid = $1"
+            );
+            dbc.query(sql, &[&rowid])
+                .await
+                .err_conv()
+                .map_err(|e| format!("on config history insert {e}"))?;
+        }
+        //ensure!(meta.len() >= parsed_until as u64, ConfigFileOnDiskShrunk{path});
+        (Some(rowid), true)
+    } else {
+        (None, true)
+    };
+    if do_parse {
+        let buf = tokio::fs::read(&path).await?;
+        let config = parse::channelconfig::parse_config(&buf).map_err(NErr::from)?.1;
+        match config_id {
+            None => {
+                dbc.query(
+                    "insert into configs (node, channel, fileSize, parsedUntil, config) values ($1, $2, $3, $4, $5) on conflict (node, channel) do update set fileSize = $3, parsedUntil = $4, config = $5",
+                    &[
+                        &node_disk_ident.rowid(),
+                        &channel_id,
+                        &(meta.len() as i64),
+                        &(buf.len() as i64),
+                        &serde_json::to_value(config)?,
+                    ],
+                ).await.err_conv().map_err(|e| format!("on config insert {e}"))?;
+                *count_inserted += 1;
+            }
+            Some(_config_id) => {
+                dbc.query(
+                    "update configs set fileSize = $3, parsedUntil = $4, config = $5 where node = $1 and channel = $2",
+                    &[
+                        &node_disk_ident.rowid(),
+                        &channel_id,
+                        &(meta.len() as i64),
+                        &(buf.len() as i64),
+                        &serde_json::to_value(config)?,
+                    ],
+                )
+                .await
+                .err_conv()
+                .map_err(|e| format!("on config update {e}"))?;
+                *count_updated += 1;
+            }
+        }
+    }
+    Ok(UpdateChannelConfigResult::Done)
+}
+
+pub async fn update_db_with_all_channel_datafiles(
+    node_config: &NodeConfigCached,
+    node_disk_ident: &NodeDiskIdent,
+    ks_prefix: &str,
+) -> Result<(), Error> {
+    let dbc = Arc::new(create_connection(&node_config.node_config.cluster.database).await?);
+    let rows = dbc
+        .query(
+            "select rowid, facility, name from channels where facility = $1 order by facility, name",
+            &[&node_disk_ident.facility()],
+        )
+        .await
+        .err_conv()?;
+    let mut c1 = 0;
+    dbc.query("begin", &[]).await.err_conv()?;
+    for row in rows {
+        let rowid: i64 = row.try_get(0).err_conv()?;
+        let _facility: i64 = row.try_get(1).err_conv()?;
+        let channel: String = row.try_get(2).err_conv()?;
+        update_db_with_channel_datafiles(node_config, node_disk_ident, ks_prefix, rowid, &channel, dbc.clone()).await?;
+        c1 += 1;
+        if c1 % 40 == 0 {
+            trace!("import datafiles  {}  {}", c1, channel);
+            dbc.query("commit", &[]).await.err_conv()?;
+            dbc.query("begin", &[]).await.err_conv()?;
+        }
+        if false && c1 >= 30 {
+            break;
+        }
+    }
+    dbc.query("commit", &[]).await.err_conv()?;
+    Ok(())
+}
+
+struct DatafileDbWriter {
+    channel_id: i64,
+    node_id: i64,
+    dbc: Arc<Client>,
+    c1: Arc<RwLock<u32>>,
+}
+
+#[derive(Debug, Serialize)]
+pub struct ChannelDesc {
+    name: String,
+}
+
+#[derive(Debug, Serialize)]
+pub struct ChannelDatafileDesc {
+    channel: ChannelDesc,
+    ks: u32,
+    tb: u32,
+    sp: u32,
+    bs: u32,
+    fs: u64,
+    mt: DateTime<Utc>,
+    ix_fs: Option<u64>,
+    ix_mt: Option<DateTime<Utc>>,
+}
+
+impl ChannelDatafileDesc {
+    pub fn timebin(&self) -> u32 {
+        self.tb
+    }
+    pub fn binsize(&self) -> u32 {
+        self.bs
+    }
+    pub fn keyspace(&self) -> u32 {
+        self.ks
+    }
+    pub fn split(&self) -> u32 {
+        self.sp
+    }
+}
+
+pub trait ChannelDatafileDescSink {
+    fn sink(&self, k: ChannelDatafileDesc) -> Pin<Box<dyn Future<Output = Result<(), Error>>>>;
+}
+
+impl ChannelDatafileDescSink for DatafileDbWriter {
+    fn sink(&self, k: ChannelDatafileDesc) -> Pin<Box<dyn Future<Output = Result<(), Error>>>> {
+        let dbc = self.dbc.clone();
+        let c1 = self.c1.clone();
+        let channel_id = self.channel_id;
+        let node_id = self.node_id;
+        Box::pin(async move {
+            dbc.query(
+                "insert into datafiles (node, channel, tsbeg, tsend, props) values ($1, $2, $3, $4, $5) on conflict do nothing",
+                &[
+                    &node_id,
+                    &channel_id,
+                    &(k.timebin() as i64 * k.binsize() as i64),
+                    &((k.timebin() + 1) as i64 * k.binsize() as i64),
+                    &serde_json::to_value(k)?,
+                ]
+            ).await.err_conv()?;
+            *c1.write()? += 1;
+            Ok(())
+        })
+    }
+}
+
+pub async fn find_channel_datafiles_in_ks(
+    base_dir: impl AsRef<Path>,
+    ks_prefix: &str,
+    ks: u32,
+    channel: &str,
+    cb: &dyn ChannelDatafileDescSink,
+) -> Result<Option<()>, Error> {
+    let data_dir_path: PathBuf = base_dir
+        .as_ref()
+        .join(format!("{}_{}", ks_prefix, ks))
+        .join("byTime")
+        .join(channel);
+    let re1 = regex::Regex::new(r"^\d{19}$")?;
+    let re2 = regex::Regex::new(r"^\d{10}$")?;
+    let re4 = regex::Regex::new(r"^(\d{19})_0{5}_Data$")?;
+    let mut rd = match tokio::fs::read_dir(&data_dir_path).await {
+        Ok(k) => k,
+        Err(e) => match e.kind() {
+            ErrorKind::NotFound => return Ok(None),
+            _ => Err(e)?,
+        },
+    };
+    while let Ok(Some(entry)) = rd.next_entry().await {
+        let fname = String::from_utf8(entry.file_name().into_vec())?;
+        if !re1.is_match(&fname) {
+            warn!("unexpected file  {}", fname);
+            continue;
+        }
+        let timebin: u32 = fname.parse()?;
+        let path = data_dir_path.join(fname);
+        let mut rd = tokio::fs::read_dir(&path).await?;
+        while let Ok(Some(entry)) = rd.next_entry().await {
+            let fname = String::from_utf8(entry.file_name().into_vec())?;
+            if !re2.is_match(&fname) {
+                warn!("unexpected file  {}", fname);
+                continue;
+            }
+            let split: u32 = fname.parse()?;
+            let path = path.join(fname);
+            let mut rd = tokio::fs::read_dir(&path).await?;
+            while let Ok(Some(entry)) = rd.next_entry().await {
+                let fname = String::from_utf8(entry.file_name().into_vec())?;
+                if let Some(m) = re4.captures(&fname) {
+                    let binsize: u32 = m.get(1).unwrap().as_str().parse()?;
+                    let data_path = path.join(&fname);
+                    let meta = tokio::fs::metadata(&data_path).await?;
+                    let index_path = path.join(format!("{}_Index", fname));
+                    let (ix_size, ix_tmod) = if let Ok(meta) = tokio::fs::metadata(&index_path).await {
+                        (Some(meta.len()), Some(meta.modified().unwrap().into()))
+                    } else {
+                        (None, None)
+                    };
+                    cb.sink(ChannelDatafileDesc {
+                        channel: ChannelDesc { name: channel.into() },
+                        ks: ks,
+                        tb: timebin,
+                        sp: split,
+                        bs: binsize,
+                        fs: meta.len(),
+                        ix_fs: ix_size,
+                        ix_mt: ix_tmod,
+                        mt: meta.modified().unwrap().into(),
+                    })
+                    .await?;
+                }
+            }
+        }
+    }
+    Ok(Some(()))
+}
+
+pub async fn update_db_with_channel_datafiles(
+    node_config: &NodeConfigCached,
+    node_disk_ident: &NodeDiskIdent,
+    ks_prefix: &str,
+    channel_id: i64,
+    channel: &str,
+    dbc: Arc<Client>,
+) -> Result<(), Error> {
+    let base_path = &node_config
+        .node
+        .sf_databuffer
+        .as_ref()
+        .ok_or_else(|| Error::with_msg(format!("missing sf databuffer config in node")))?
+        .data_base_path;
+    let writer = DatafileDbWriter {
+        node_id: node_disk_ident.rowid(),
+        channel_id: channel_id,
+        dbc: dbc.clone(),
+        c1: Arc::new(RwLock::new(0)),
+    };
+    let mut n_nothing = 0;
+    for ks in &[2, 3, 4] {
+        match find_channel_datafiles_in_ks(base_path, ks_prefix, *ks, channel, &writer).await {
+            /*Err(Error::ChannelDatadirNotFound { .. }) => {
+                n_nothing += 1;
+            }*/
+            Ok(None) => {
+                n_nothing += 1;
+            }
+            x => {
+                x?;
+            }
+        };
+        if false && *writer.c1.read()? >= 10 {
+            break;
+        }
+    }
+    if n_nothing >= 3 {
+        //warn!("No datafile directories in any keyspace  writer got {:5}  n_nothing {}  channel {}", writer.c1.borrow(), n_nothing, channel);
+    }
+    Ok(())
+}

crates/dbconn/src/scan/updatechannelnames.rs

@@ -0,0 +1,171 @@
+use super::get_node_disk_ident;
+use super::update_db_with_channel_name_list;
+use super::FindChannelNamesFromConfigReadDir;
+use super::NodeDiskIdent;
+use super::UpdatedDbWithChannelNames;
+use crate::create_connection;
+use crate::pg::Client as PgClient;
+use err::Error;
+use futures_util::Future;
+use futures_util::Stream;
+use netpod::NodeConfigCached;
+use pin_project::pin_project;
+use std::os::unix::prelude::OsStringExt;
+use std::pin::Pin;
+use std::task::Context;
+use std::task::Poll;
+
+#[pin_project]
+struct UpdatedDbWithChannelNamesStream {
+    errored: bool,
+    data_complete: bool,
+    #[allow(dead_code)]
+    node_config: Pin<Box<NodeConfigCached>>,
+    // TODO can we pass a Pin to the async fn instead of creating static ref?
+    node_config_ref: &'static NodeConfigCached,
+    #[pin]
+    client_fut: Option<Pin<Box<dyn Future<Output = Result<PgClient, Error>> + Send>>>,
+    #[pin]
+    client: Option<PgClient>,
+    client_ref: Option<&'static PgClient>,
+    #[pin]
+    ident_fut: Option<Pin<Box<dyn Future<Output = Result<NodeDiskIdent, Error>> + Send>>>,
+    ident: Option<NodeDiskIdent>,
+    #[pin]
+    find: Option<FindChannelNamesFromConfigReadDir>,
+    #[pin]
+    update_batch: Option<Pin<Box<dyn Future<Output = Result<(), Error>> + Send>>>,
+    channel_inp_done: bool,
+    clist: Vec<String>,
+}
+
+impl UpdatedDbWithChannelNamesStream {
+    #[allow(unused)]
+    fn new(node_config: NodeConfigCached) -> Result<Self, Error> {
+        let node_config = Box::pin(node_config.clone());
+        let node_config_ref = unsafe { &*(&node_config as &NodeConfigCached as *const _) };
+        let mut ret = Self {
+            errored: false,
+            data_complete: false,
+            node_config,
+            node_config_ref,
+            client_fut: None,
+            client: None,
+            client_ref: None,
+            ident_fut: None,
+            ident: None,
+            find: None,
+            update_batch: None,
+            channel_inp_done: false,
+            clist: Vec::new(),
+        };
+        ret.client_fut = Some(Box::pin(create_connection(
+            &ret.node_config_ref.node_config.cluster.database,
+        )));
+        Ok(ret)
+    }
+}
+
+impl Stream for UpdatedDbWithChannelNamesStream {
+    type Item = Result<UpdatedDbWithChannelNames, Error>;
+
+    fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
+        use Poll::*;
+        let mut pself = self.project();
+        loop {
+            break if *pself.errored {
+                Ready(None)
+            } else if *pself.data_complete {
+                Ready(None)
+            } else if let Some(fut) = pself.find.as_mut().as_pin_mut() {
+                match fut.poll_next(cx) {
+                    Ready(Some(Ok(item))) => {
+                        pself
+                            .clist
+                            .push(String::from_utf8(item.file_name().into_vec()).unwrap());
+                        continue;
+                    }
+                    Ready(Some(Err(e))) => {
+                        *pself.errored = true;
+                        Ready(Some(Err(e)))
+                    }
+                    Ready(None) => {
+                        *pself.channel_inp_done = true;
+                        // Work through the collected items
+                        let l = std::mem::replace(pself.clist, Vec::new());
+                        let fut = update_db_with_channel_name_list(
+                            l,
+                            pself.ident.as_ref().unwrap().facility,
+                            pself.client.as_ref().get_ref().as_ref().unwrap(),
+                        );
+                        // TODO
+                        //pself.update_batch.replace(Box::pin(fut));
+                        let _ = fut;
+                        continue;
+                    }
+                    Pending => Pending,
+                }
+            } else if let Some(fut) = pself.ident_fut.as_mut().as_pin_mut() {
+                match fut.poll(cx) {
+                    Ready(Ok(item)) => {
+                        *pself.ident_fut = None;
+                        *pself.ident = Some(item);
+                        let ret = UpdatedDbWithChannelNames {
+                            msg: format!("Got ident {:?}", pself.ident),
+                            count: 43,
+                        };
+                        let base_path = &pself
+                            .node_config
+                            .node
+                            .sf_databuffer
+                            .as_ref()
+                            .ok_or_else(|| Error::with_msg(format!("missing sf databuffer config in node")))?
+                            .data_base_path;
+                        let s = FindChannelNamesFromConfigReadDir::new(base_path);
+                        *pself.find = Some(s);
+                        Ready(Some(Ok(ret)))
+                    }
+                    Ready(Err(e)) => {
+                        *pself.errored = true;
+                        Ready(Some(Err(e)))
+                    }
+                    Pending => Pending,
+                }
+            } else if let Some(fut) = pself.client_fut.as_mut().as_pin_mut() {
+                match fut.poll(cx) {
+                    Ready(Ok(item)) => {
+                        *pself.client_fut = None;
+                        //*pself.client = Some(Box::pin(item));
+                        //*pself.client_ref = Some(unsafe { &*(&pself.client.as_ref().unwrap() as &Client as *const _) });
+                        *pself.client = Some(item);
+                        let c2: &PgClient = pself.client.as_ref().get_ref().as_ref().unwrap();
+                        *pself.client_ref = Some(unsafe { &*(c2 as *const _) });
+
+                        //() == pself.node_config.as_ref();
+                        //() == pself.client.as_ref().as_pin_ref().unwrap();
+                        /* *pself.ident_fut = Some(Box::pin(get_node_disk_ident_2(
+                            pself.node_config.as_ref(),
+                            pself.client.as_ref().as_pin_ref().unwrap(),
+                        )));*/
+                        *pself.ident_fut = Some(Box::pin(get_node_disk_ident(
+                            pself.node_config_ref,
+                            pself.client_ref.as_ref().unwrap(),
+                        )));
+                        let ret = UpdatedDbWithChannelNames {
+                            msg: format!("Client opened connection"),
+                            count: 42,
+                        };
+                        Ready(Some(Ok(ret)))
+                    }
+                    Ready(Err(e)) => {
+                        *pself.errored = true;
+                        Ready(Some(Err(e)))
+                    }
+                    Pending => Pending,
+                }
+            } else {
+                Ready(None)
+            };
+        }
+    }
+}

crates/dbconn/src/search.rs

@@ -0,0 +1,262 @@
+use crate::create_connection;
+use crate::ErrConv;
+use err::Error;
+use netpod::ChannelArchiver;
+use netpod::ChannelSearchQuery;
+use netpod::ChannelSearchResult;
+use netpod::ChannelSearchSingleResult;
+use netpod::Database;
+use netpod::NodeConfigCached;
+use netpod::ScalarType;
+use netpod::ScyllaConfig;
+use netpod::Shape;
+use serde_json::Value as JsVal;
+
+pub async fn search_channel_databuffer(
+    query: ChannelSearchQuery,
+    node_config: &NodeConfigCached,
+) -> Result<ChannelSearchResult, Error> {
+    let empty = if !query.name_regex.is_empty() {
+        false
+    } else if !query.source_regex.is_empty() {
+        false
+    } else if !query.description_regex.is_empty() {
+        false
+    } else {
+        true
+    };
+    if empty {
+        let ret = ChannelSearchResult { channels: Vec::new() };
+        return Ok(ret);
+    }
+    let sql = format!(concat!(
+        "select ",
+        "channel_id, channel_name, source_name, dtype, shape, unit, description, channel_backend",
+        " from searchext($1, $2, $3, $4)",
+    ));
+    let cl = create_connection(&node_config.node_config.cluster.database).await?;
+    let rows = cl
+        .query(
+            sql.as_str(),
+            &[&query.name_regex, &query.source_regex, &query.description_regex, &"asc"],
+        )
+        .await
+        .err_conv()?;
+    let mut res = Vec::new();
+    for row in rows {
+        let shapedb: Option<serde_json::Value> = row.get(4);
+        let shape = match &shapedb {
+            Some(top) => match top {
+                serde_json::Value::Null => Vec::new(),
+                serde_json::Value::Array(items) => {
+                    let mut a = Vec::new();
+                    for item in items {
+                        match item {
+                            serde_json::Value::Number(n) => match n.as_i64() {
+                                Some(n) => {
+                                    a.push(n as u32);
+                                }
+                                None => return Err(Error::with_msg(format!("can not understand shape {:?}", shapedb))),
+                            },
+                            _ => return Err(Error::with_msg(format!("can not understand shape {:?}", shapedb))),
+                        }
+                    }
+                    a
+                }
+                _ => return Err(Error::with_msg(format!("can not understand shape {:?}", shapedb))),
+            },
+            None => Vec::new(),
+        };
+        let ty: String = row.get(3);
+        let k = ChannelSearchSingleResult {
+            backend: row.get(7),
+            name: row.get(1),
+            series: row.get::<_, i64>(0) as u64,
+            source: row.get(2),
+            ty,
+            shape: shape,
+            unit: row.get(5),
+            description: row.get(6),
+            is_api_0: None,
+        };
+        res.push(k);
+    }
+    let ret = ChannelSearchResult { channels: res };
+    Ok(ret)
+}
+
+pub async fn search_channel_scylla(
+    query: ChannelSearchQuery,
+    _scyconf: &ScyllaConfig,
+    pgconf: &Database,
+) -> Result<ChannelSearchResult, Error> {
+    let empty = if !query.name_regex.is_empty() { false } else { true };
+    if empty {
+        let ret = ChannelSearchResult { channels: Vec::new() };
+        return Ok(ret);
+    }
+    let sql = format!(concat!(
+        "select",
+        " series, facility, channel, scalar_type, shape_dims",
+        " from series_by_channel",
+        " where channel ~* $1",
+        " and scalar_type != -2147483647",
+        " limit 400000",
+    ));
+    let pgclient = crate::create_connection(pgconf).await?;
+    let rows = pgclient.query(sql.as_str(), &[&query.name_regex]).await.err_conv()?;
+    let mut res = Vec::new();
+    for row in rows {
+        let series: i64 = row.get(0);
+        let series = series as u64;
+        let backend: String = row.get(1);
+        let channel: String = row.get(2);
+        let a: i32 = row.get(3);
+        let scalar_type = ScalarType::from_scylla_i32(a)?;
+        let a: Vec<i32> = row.get(4);
+        let shape = Shape::from_scylla_shape_dims(&a)?;
+        let k = ChannelSearchSingleResult {
+            backend,
+            name: channel,
+            series,
+            source: "".into(),
+            ty: scalar_type.to_variant_str().into(),
+            shape: shape.to_scylla_vec().into_iter().map(|x| x as u32).collect(),
+            unit: "".into(),
+            description: "".into(),
+            is_api_0: None,
+        };
+        res.push(k);
+    }
+    let ret = ChannelSearchResult { channels: res };
+    Ok(ret)
+}
+
+pub async fn search_channel_archeng(
+    query: ChannelSearchQuery,
+    backend: String,
+    _conf: &ChannelArchiver,
+    database: &Database,
+) -> Result<ChannelSearchResult, Error> {
+    // Channel archiver provides only channel name. Also, search criteria are currently ANDed.
+    // Therefore search only if user only provides a name criterion.
+    let empty = if !query.source_regex.is_empty() {
+        true
+    } else if !query.description_regex.is_empty() {
+        true
+    } else if query.name_regex.is_empty() {
+        true
+    } else {
+        false
+    };
+    if empty {
+        let ret = ChannelSearchResult { channels: Vec::new() };
+        return Ok(ret);
+    }
+    let sql = format!(concat!(
+        "select c.name, c.config",
+        " from channels c",
+        " where c.name ~* $1",
+        " order by c.name",
+        " limit 100"
+    ));
+    let cl = create_connection(database).await?;
+    let rows = cl.query(sql.as_str(), &[&query.name_regex]).await.err_conv()?;
+    let mut res = Vec::new();
+    for row in rows {
+        let name: String = row.get(0);
+        let config: JsVal = row.get(1);
+        let st = match config.get("scalarType") {
+            Some(k) => match k {
+                JsVal::String(k) => match k.as_str() {
+                    "U8" => "Uint8",
+                    "U16" => "Uint16",
+                    "U32" => "Uint32",
+                    "U64" => "Uint64",
+                    "I8" => "Int8",
+                    "I16" => "Int16",
+                    "I32" => "Int32",
+                    "I64" => "Int64",
+                    "F32" => "Float32",
+                    "F64" => "Float64",
+                    _ => k,
+                }
+                .into(),
+                _ => "",
+            },
+            None => "",
+        };
+        let shape = match config.get("shape") {
+            Some(k) => match k {
+                JsVal::String(k) => {
+                    if k == "Scalar" {
+                        Vec::new()
+                    } else {
+                        return Err(Error::with_msg_no_trace(format!(
+                            "search_channel_archeng can not understand {:?}",
+                            config
+                        )));
+                    }
+                }
+                JsVal::Object(k) => match k.get("Wave") {
+                    Some(k) => match k {
+                        JsVal::Number(k) => {
+                            vec![k.as_i64().unwrap_or(u32::MAX as i64) as u32]
+                        }
+                        _ => {
+                            return Err(Error::with_msg_no_trace(format!(
+                                "search_channel_archeng can not understand {:?}",
+                                config
+                            )));
+                        }
+                    },
+                    None => {
+                        return Err(Error::with_msg_no_trace(format!(
+                            "search_channel_archeng can not understand {:?}",
+                            config
+                        )));
+                    }
+                },
+                _ => {
+                    return Err(Error::with_msg_no_trace(format!(
+                        "search_channel_archeng can not understand {:?}",
+                        config
+                    )));
+                }
+            },
+            None => Vec::new(),
+        };
+        let k = ChannelSearchSingleResult {
+            backend: backend.clone(),
+            name,
+            // TODO provide a unique id also within this backend:
+            series: 0,
+            source: String::new(),
+            ty: st.into(),
+            shape,
+            unit: String::new(),
+            description: String::new(),
+            is_api_0: None,
+        };
+        res.push(k);
+    }
+    let ret = ChannelSearchResult { channels: res };
+    Ok(ret)
+}
+
+pub async fn search_channel(
+    query: ChannelSearchQuery,
+    node_config: &NodeConfigCached,
+) -> Result<ChannelSearchResult, Error> {
+    let pgconf = &node_config.node_config.cluster.database;
+    if let Some(scyconf) = node_config.node_config.cluster.scylla.as_ref() {
+        search_channel_scylla(query, scyconf, pgconf).await
+    } else if let Some(conf) = node_config.node.channel_archiver.as_ref() {
+        search_channel_archeng(query, node_config.node_config.cluster.backend.clone(), conf, pgconf).await
+    } else if let Some(_conf) = node_config.node.archiver_appliance.as_ref() {
+        // TODO
+        err::todoval()
+    } else {
+        search_channel_databuffer(query, node_config).await
+    }
+}