#![allow(unused_imports)]
use async_channel::{bounded, Receiver};
use chrono::{DateTime, Utc};
use err::Error;
use futures_core::Stream;
use futures_util::{pin_mut, FutureExt, StreamExt};
use netpod::log::*;
use netpod::{Database, NodeConfigCached};
use pin_project::pin_project;
use serde::{Deserialize, Serialize};
use std::future::Future;
use std::io::ErrorKind;
use std::marker::PhantomPinned;
use std::os::unix::ffi::OsStringExt;
use std::path::{Path, PathBuf};
use std::pin::Pin;
use std::sync::{Arc, RwLock};
use std::task::{Context, Poll};
use tokio::fs::{DirEntry, ReadDir};
use tokio_postgres::Client;

#[derive(Debug, Serialize, Deserialize)]
pub struct NodeDiskIdent {
    pub rowid: i64,
    pub facility: i64,
    pub split: i32,
    pub hostname: String,
}

impl NodeDiskIdent {
    pub fn rowid(&self) -> i64 {
        self.rowid
    }
    pub fn facility(&self) -> i64 {
        self.facility
    }
}

fn _get_hostname() -> Result<String, Error> {
    let out = std::process::Command::new("hostname").output()?;
    Ok(String::from_utf8(out.stdout[..out.stdout.len() - 1].to_vec())?)
}

pub async fn get_node_disk_ident(node_config: &NodeConfigCached, dbc: &Client) -> Result<NodeDiskIdent, Error> {
    let sql = "select nodes.rowid, facility, split, hostname from nodes, facilities where facilities.name = $1 and facility = facilities.rowid and hostname = $2";
    let rows = dbc
        .query(sql, &[&node_config.node.backend, &node_config.node.host])
        .await?;
    if rows.len() != 1 {
        return Err(Error::with_msg(format!(
            "get_node can't find unique entry for {} {}",
            node_config.node.host, node_config.node.backend
        )));
    }
    let row = &rows[0];
    Ok(NodeDiskIdent {
        rowid: row.get(0),
        facility: row.get(1),
        split: row.get(2),
        hostname: row.get(3),
    })
}

pub async fn get_node_disk_ident_2(
    node_config: Pin<&NodeConfigCached>,
    dbc: Pin<&Client>,
) -> Result<NodeDiskIdent, Error> {
    let sql = "select nodes.rowid, facility, split, hostname from nodes, facilities where facilities.name = $1 and facility = facilities.rowid and hostname = $2";
    let rows = dbc
        .query(sql, &[&node_config.node.backend, &node_config.node.host])
        .await?;
    if rows.len() != 1 {
        return Err(Error::with_msg(format!(
            "get_node can't find unique entry for {} {}",
            node_config.node.host, node_config.node.backend
        )));
    }
    let row = &rows[0];
    Ok(NodeDiskIdent {
        rowid: row.get(0),
        facility: row.get(1),
        split: row.get(2),
        hostname: row.get(3),
    })
}

#[pin_project]
pub struct FindChannelNamesFromConfigReadDir {
    #[pin]
    read_dir_fut: Option<Pin<Box<dyn Future<Output = std::io::Result<ReadDir>> + Send>>>,
    read_dir: Option<ReadDir>,
    #[pin]
    dir_entry_fut: Option<Pin<Box<dyn Future<Output = std::io::Result<Option<DirEntry>>> + Send>>>,
}

impl FindChannelNamesFromConfigReadDir {
    pub fn new(base_dir: impl AsRef<Path>) -> Self {
        Self {
            read_dir_fut: Some(Box::pin(tokio::fs::read_dir(base_dir.as_ref().join("config")))),
            read_dir: None,
            dir_entry_fut: None,
        }
    }
}

impl Stream for FindChannelNamesFromConfigReadDir {
    type Item = Result<DirEntry, Error>;

    fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
        use Poll::*;
        let mut pself = self.project();
        loop {
            break if let Some(fut) = pself.dir_entry_fut.as_mut().as_pin_mut() {
                match fut.poll(cx) {
                    Ready(Ok(Some(item))) => {
                        let g = unsafe { &mut *(pself.read_dir.as_mut().unwrap() as *mut ReadDir) };
                        let fut = g.next_entry();
                        *pself.dir_entry_fut = Some(Box::pin(fut));
                        Ready(Some(Ok(item)))
                    }
                    Ready(Ok(None)) => {
                        *pself.dir_entry_fut = None;
                        Ready(None)
                    }
                    Ready(Err(e)) => Ready(Some(Err(e.into()))),
                    Pending => Pending,
                }
            } else if let Some(fut) = pself.read_dir_fut.as_mut().as_pin_mut() {
                match fut.poll(cx) {
                    Ready(Ok(item)) => {
                        *pself.read_dir_fut = None;
                        *pself.read_dir = Some(item);
                        //let fut = pself.read_dir.as_mut().unwrap().next_entry();
                        let g = unsafe { &mut *(pself.read_dir.as_mut().unwrap() as *mut ReadDir) };
                        let fut = g.next_entry();
                        *pself.dir_entry_fut = Some(Box::pin(fut));
                        continue;
                    }
                    Ready(Err(e)) => Ready(Some(Err(e.into()))),
                    Pending => Pending,
                }
            } else {
                Pending
            };
        }
    }
}

async fn find_channel_names_from_config<F, Fut>(base_dir: impl AsRef<Path>, mut cb: F) -> Result<(), Error>
where
    F: FnMut(&str) -> Fut,
    Fut: Future<Output = Result<(), Error>>,
{
    let path2: PathBuf = base_dir.as_ref().join("config");
    let mut rd = tokio::fs::read_dir(&path2).await?;
    while let Ok(Some(entry)) = rd.next_entry().await {
        let fname = String::from_utf8(entry.file_name().into_vec())?;
        cb(&fname).await?;
    }
    Ok(())
}

#[derive(Debug, Serialize, Deserialize)]
pub struct UpdatedDbWithChannelNames {
    msg: String,
    count: u32,
}

#[pin_project]
pub struct UpdatedDbWithChannelNamesStream {
    errored: bool,
    data_complete: bool,
    #[allow(dead_code)]
    node_config: Pin<Box<NodeConfigCached>>,
    // TODO can we pass a Pin to the async fn instead of creating static ref?
    node_config_ref: &'static NodeConfigCached,
    #[pin]
    client_fut: Option<Pin<Box<dyn Future<Output = Result<Client, Error>> + Send>>>,
    #[pin]
    client: Option<Client>,
    client_ref: Option<&'static Client>,
    #[pin]
    ident_fut: Option<Pin<Box<dyn Future<Output = Result<NodeDiskIdent, Error>> + Send>>>,
    ident: Option<NodeDiskIdent>,
    #[pin]
    find: Option<FindChannelNamesFromConfigReadDir>,
    #[pin]
    update_batch: Option<Pin<Box<dyn Future<Output = Result<(), Error>> + Send>>>,
    channel_inp_done: bool,
    clist: Vec<String>,
}

impl UpdatedDbWithChannelNamesStream {
    pub fn new(node_config: NodeConfigCached) -> Result<Self, Error> {
        let node_config = Box::pin(node_config.clone());
        let node_config_ref = unsafe { &*(&node_config as &NodeConfigCached as *const _) };
        let mut ret = Self {
            errored: false,
            data_complete: false,
            node_config,
            node_config_ref,
            client_fut: None,
            client: None,
            client_ref: None,
            ident_fut: None,
            ident: None,
            find: None,
            update_batch: None,
            channel_inp_done: false,
            clist: vec![],
        };
        ret.client_fut = Some(Box::pin(crate::create_connection(
            &ret.node_config_ref.node_config.cluster.database,
        )));
        Ok(ret)
    }
}

impl Stream for UpdatedDbWithChannelNamesStream {
    type Item = Result<UpdatedDbWithChannelNames, Error>;

    fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
        use Poll::*;
        let mut pself = self.project();
        loop {
            break if *pself.errored {
                Ready(None)
            } else if *pself.data_complete {
                Ready(None)
            } else if let Some(fut) = pself.find.as_mut().as_pin_mut() {
                match fut.poll_next(cx) {
                    Ready(Some(Ok(item))) => {
                        pself
                            .clist
                            .push(String::from_utf8(item.file_name().into_vec()).unwrap());
                        continue;
                    }
                    Ready(Some(Err(e))) => {
                        *pself.errored = true;
                        Ready(Some(Err(e)))
                    }
                    Ready(None) => {
                        *pself.channel_inp_done = true;
                        // Work through the collected items
                        let l = std::mem::replace(pself.clist, vec![]);
                        let fut = update_db_with_channel_name_list(
                            l,
                            pself.ident.as_ref().unwrap().facility,
                            pself.client.as_ref().get_ref().as_ref().unwrap(),
                        );
                        // TODO
                        //pself.update_batch.replace(Box::pin(fut));
                        let _ = fut;
                        continue;
                    }
                    Pending => Pending,
                }
            } else if let Some(fut) = pself.ident_fut.as_mut().as_pin_mut() {
                match fut.poll(cx) {
                    Ready(Ok(item)) => {
                        *pself.ident_fut = None;
                        *pself.ident = Some(item);
                        let ret = UpdatedDbWithChannelNames {
                            msg: format!("Got ident {:?}", pself.ident),
                            count: 43,
                        };
                        let s = FindChannelNamesFromConfigReadDir::new(&pself.node_config.node.data_base_path);
                        *pself.find = Some(s);
                        Ready(Some(Ok(ret)))
                    }
                    Ready(Err(e)) => {
                        *pself.errored = true;
                        Ready(Some(Err(e)))
                    }
                    Pending => Pending,
                }
            } else if let Some(fut) = pself.client_fut.as_mut().as_pin_mut() {
                match fut.poll(cx) {
                    Ready(Ok(item)) => {
                        *pself.client_fut = None;
                        //*pself.client = Some(Box::pin(item));
                        //*pself.client_ref = Some(unsafe { &*(&pself.client.as_ref().unwrap() as &Client as *const _) });
                        *pself.client = Some(item);
                        let c2: &Client = pself.client.as_ref().get_ref().as_ref().unwrap();
                        *pself.client_ref = Some(unsafe { &*(c2 as *const _) });

                        //() == pself.node_config.as_ref();
                        //() == pself.client.as_ref().as_pin_ref().unwrap();
                        /* *pself.ident_fut = Some(Box::pin(get_node_disk_ident_2(
                            pself.node_config.as_ref(),
                            pself.client.as_ref().as_pin_ref().unwrap(),
                        )));*/
                        *pself.ident_fut = Some(Box::pin(get_node_disk_ident(
                            pself.node_config_ref,
                            pself.client_ref.as_ref().unwrap(),
                        )));
                        let ret = UpdatedDbWithChannelNames {
                            msg: format!("Client opened connection"),
                            count: 42,
                        };
                        Ready(Some(Ok(ret)))
                    }
                    Ready(Err(e)) => {
                        *pself.errored = true;
                        Ready(Some(Err(e)))
                    }
                    Pending => Pending,
                }
            } else {
                Ready(None)
            };
        }
    }
}

async fn update_db_with_channel_name_list(list: Vec<String>, backend: i64, dbc: &Client) -> Result<(), Error> {
    crate::delay_io_short().await;
    dbc.query("begin", &[]).await?;
    for ch in list {
        dbc.query(
            "insert into channels (facility, name) values ($1, $2) on conflict do nothing",
            &[&backend, &ch],
        )
        .await?;
    }
    dbc.query("commit", &[]).await?;
    Ok(())
}

pub async fn update_db_with_channel_names(
    node_config: NodeConfigCached,
    db_config: &Database,
) -> Result<Receiver<Result<UpdatedDbWithChannelNames, Error>>, Error> {
    let (tx, rx) = bounded(16);
    let tx2 = tx.clone();
    let db_config = db_config.clone();
    let block1 = async move {
        //return Err(Error::with_msg("some test error1"));
        //tx.send(Err(Error::with_msg("some test error2"))).await?;
        let dbc = crate::create_connection(&db_config).await?;
        let node_disk_ident = get_node_disk_ident(&node_config, &dbc).await?;
        let c1 = Arc::new(RwLock::new(0u32));
        dbc.query("begin", &[]).await?;
        let dbc = Arc::new(dbc);
        let tx = Arc::new(tx);
        find_channel_names_from_config(&node_config.node.data_base_path, |ch| {
            let ch = ch.to_owned();
            let dbc = dbc.clone();
            let c1 = c1.clone();
            let tx = tx.clone();
            let fac = node_disk_ident.facility;
            async move {
                crate::delay_io_short().await;
                dbc.query(
                    "insert into channels (facility, name) values ($1, $2) on conflict do nothing",
                    &[&fac, &ch],
                )
                .await?;
                let c2 = {
                    let mut g = c1.write()?;
                    *g += 1;
                    *g
                };
                if c2 % 200 == 0 {
                    dbc.query("commit", &[]).await?;
                    let ret = UpdatedDbWithChannelNames {
                        msg: format!("current {}", ch),
                        count: c2,
                    };
                    tx.send(Ok(ret)).await?;
                    crate::delay_io_medium().await;
                    dbc.query("begin", &[]).await?;
                }
                Ok(())
            }
        })
        .await?;
        dbc.query("commit", &[]).await?;
        let c2 = *c1.read()?;
        let ret = UpdatedDbWithChannelNames {
            msg: format!("all done"),
            count: c2,
        };
        tx.send(Ok(ret)).await?;
        Ok::<_, Error>(())
    };
    let block2 = async move {
        match block1.await {
            Ok(_) => {}
            Err(e) => match tx2.send(Err(e)).await {
                Ok(_) => {}
                Err(e) => {
                    error!("can not report error through channel: {:?}", e);
                }
            },
        }
    };
    tokio::spawn(block2);
    Ok(rx)
}

pub fn update_db_with_channel_names_3<'a>(
    node_config: &'a NodeConfigCached,
) -> impl Stream<Item = Result<UpdatedDbWithChannelNames, Error>> + 'static {
    futures_util::future::ready(node_config.node.data_base_path.clone())
        .then(|path| tokio::fs::read_dir(path))
        .map(Result::unwrap)
        .map(|rd| {
            futures_util::stream::unfold(rd, move |rd| {
                //let fut = rd.next_entry();
                futures_util::future::ready(Ok(None)).map(move |item: Result<Option<u32>, Error>| match item {
                    Ok(Some(item)) => Some((item, rd)),
                    Ok(None) => None,
                    Err(_e) => None,
                })
            })
        })
        .map(|_conf| Err(Error::with_msg("TODO")))
        .into_stream()
}

#[derive(Debug, Serialize, Deserialize)]
pub struct UpdatedDbWithAllChannelConfigs {
    msg: String,
    count: u32,
}

pub async fn update_db_with_all_channel_configs(
    node_config: NodeConfigCached,
) -> Result<Receiver<Result<UpdatedDbWithAllChannelConfigs, Error>>, Error> {
    let (tx, rx) = bounded(16);
    let tx = Arc::new(tx);
    let tx2 = tx.clone();
    let tx3 = tx.clone();
    let block1 = async move {
        let node_config = &node_config;
        let dbc = crate::create_connection(&node_config.node_config.cluster.database).await?;
        let dbc = Arc::new(dbc);
        let node_disk_ident = &get_node_disk_ident(node_config, &dbc).await?;
        let rows = dbc
            .query(
                "select rowid, facility, name from channels where facility = $1 order by facility, name",
                &[&node_disk_ident.facility],
            )
            .await?;
        let mut c1 = 0;
        dbc.query("begin", &[]).await?;
        let mut count_inserted = 0;
        let mut count_updated = 0;
        for row in rows {
            let rowid: i64 = row.try_get(0)?;
            let _facility: i64 = row.try_get(1)?;
            let channel: String = row.try_get(2)?;
            match update_db_with_channel_config(
                node_config,
                node_disk_ident,
                rowid,
                &channel,
                dbc.clone(),
                &mut count_inserted,
                &mut count_updated,
            )
            .await
            {
                Err(e) => {
                    error!("{:?}", e);
                    crate::delay_io_medium().await;
                }
                Ok(UpdateChannelConfigResult::NotFound) => {
                    warn!("can not find channel config {}", channel);
                    crate::delay_io_medium().await;
                }
                Ok(UpdateChannelConfigResult::Done) => {
                    c1 += 1;
                    if c1 % 200 == 0 {
                        dbc.query("commit", &[]).await?;
                        let msg = format!(
                            "channel no {:6}  inserted {:6}  updated {:6}",
                            c1, count_inserted, count_updated
                        );
                        let ret = UpdatedDbWithAllChannelConfigs { msg, count: c1 };
                        tx.send(Ok(ret)).await?;
                        dbc.query("begin", &[]).await?;
                    }
                    crate::delay_io_short().await;
                }
            }
        }
        dbc.query("commit", &[]).await?;
        let msg = format!(
            "ALL DONE  channel no {:6}  inserted {:6}  updated {:6}",
            c1, count_inserted, count_updated
        );
        let ret = UpdatedDbWithAllChannelConfigs { msg, count: c1 };
        tx.send(Ok(ret)).await?;
        Ok::<_, Error>(())
    }
    .then({
        |item| async move {
            match item {
                Ok(_) => {}
                Err(e) => {
                    let msg = format!("Seeing error: {:?}", e);
                    let ret = UpdatedDbWithAllChannelConfigs { msg, count: 0 };
                    tx2.send(Ok(ret)).await?;
                }
            }
            Ok::<_, Error>(())
        }
    });
    let block2 = async move {
        match block1.await {
            Ok(_) => {}
            Err(e) => match tx3.send(Err(e)).await {
                Ok(_) => {}
                Err(e) => {
                    error!("can not deliver error through channel: {:?}", e);
                }
            },
        }
    };
    tokio::spawn(block2);
    Ok(rx)
}

pub async fn update_search_cache(node_config: &NodeConfigCached) -> Result<(), Error> {
    let dbc = crate::create_connection(&node_config.node_config.cluster.database).await?;
    dbc.query("select update_cache()", &[]).await?;
    Ok(())
}

pub enum UpdateChannelConfigResult {
    NotFound,
    Done,
}

/**
Parse the config of the given channel and update database.
*/
pub async fn update_db_with_channel_config(
    node_config: &NodeConfigCached,
    node_disk_ident: &NodeDiskIdent,
    channel_id: i64,
    channel: &str,
    dbc: Arc<Client>,
    count_inserted: &mut usize,
    count_updated: &mut usize,
) -> Result<UpdateChannelConfigResult, Error> {
    let path = node_config
        .node
        .data_base_path
        .join("config")
        .join(channel)
        .join("latest")
        .join("00000_Config");
    let meta = if let Ok(k) = tokio::fs::metadata(&path).await {
        k
    } else {
        return Ok(UpdateChannelConfigResult::NotFound);
    };
    if meta.len() > 8 * 1024 * 1024 {
        return Err(Error::with_msg("meta data too long"));
    }
    let rows = dbc
        .query(
            "select rowid, fileSize, parsedUntil, channel from configs where node = $1 and channel = $2",
            &[&node_disk_ident.rowid(), &channel_id],
        )
        .await?;
    if rows.len() > 1 {
        return Err(Error::with_msg("more than one row"));
    }
    let (config_id, do_parse) = if rows.len() > 0 {
        let row = &rows[0];
        let rowid: i64 = row.get(0);
        let file_size: u32 = row.get::<_, i64>(1) as u32;
        let parsed_until: u32 = row.get::<_, i64>(2) as u32;
        let _channel_id = row.get::<_, i64>(2) as i64;
        if meta.len() < file_size as u64 || meta.len() < parsed_until as u64 {
            let sql = concat!(
            "insert into configs_history (rowid_original, node, channel, fileSize, parsedUntil, config, tsinsert) ",
            "select rowid as rowid_original, node, channel, fileSize, parsedUntil, config, now() from configs where rowid = $1"
            );
            dbc.query(sql, &[&rowid]).await?;
        }
        //ensure!(meta.len() >= parsed_until as u64, ConfigFileOnDiskShrunk{path});
        (Some(rowid), true)
    } else {
        (None, true)
    };
    if do_parse {
        let buf = tokio::fs::read(&path).await?;
        let config = parse::channelconfig::parse_config(&buf)?.1;
        match config_id {
            None => {
                dbc.query(
                    "insert into configs (node, channel, fileSize, parsedUntil, config) values ($1, $2, $3, $4, $5)",
                    &[
                        &node_disk_ident.rowid(),
                        &channel_id,
                        &(meta.len() as i64),
                        &(buf.len() as i64),
                        &serde_json::to_value(config)?,
                    ],
                )
                .await?;
                *count_inserted += 1;
            }
            Some(_config_id_2) => {
                dbc.query(
                    "insert into configs (node, channel, fileSize, parsedUntil, config) values ($1, $2, $3, $4, $5) on conflict (node, channel) do update set fileSize = $3, parsedUntil = $4, config = $5",
                    &[&node_disk_ident.rowid(), &channel_id, &(meta.len() as i64), &(buf.len() as i64), &serde_json::to_value(config)?],
                ).await?;
                *count_updated += 1;
            }
        }
    }
    Ok(UpdateChannelConfigResult::Done)
}

pub async fn update_db_with_all_channel_datafiles(
    node_config: &NodeConfigCached,
    node_disk_ident: &NodeDiskIdent,
    ks_prefix: &str,
) -> Result<(), Error> {
    let dbc = Arc::new(crate::create_connection(&node_config.node_config.cluster.database).await?);
    let rows = dbc
        .query(
            "select rowid, facility, name from channels where facility = $1 order by facility, name",
            &[&node_disk_ident.facility()],
        )
        .await?;
    let mut c1 = 0;
    dbc.query("begin", &[]).await?;
    for row in rows {
        let rowid: i64 = row.try_get(0)?;
        let _facility: i64 = row.try_get(1)?;
        let channel: String = row.try_get(2)?;
        update_db_with_channel_datafiles(node_config, node_disk_ident, ks_prefix, rowid, &channel, dbc.clone()).await?;
        c1 += 1;
        if c1 % 40 == 0 {
            trace!("import datafiles  {}  {}", c1, channel);
            dbc.query("commit", &[]).await?;
            dbc.query("begin", &[]).await?;
        }
        if false && c1 >= 30 {
            break;
        }
    }
    dbc.query("commit", &[]).await?;
    Ok(())
}

struct DatafileDbWriter {
    channel_id: i64,
    node_id: i64,
    dbc: Arc<Client>,
    c1: Arc<RwLock<u32>>,
}

#[derive(Debug, Serialize)]
pub struct ChannelDesc {
    name: String,
}

#[derive(Debug, Serialize)]
pub struct ChannelDatafileDesc {
    channel: ChannelDesc,
    ks: u32,
    tb: u32,
    sp: u32,
    bs: u32,
    fs: u64,
    mt: DateTime<Utc>,
    ix_fs: Option<u64>,
    ix_mt: Option<DateTime<Utc>>,
}

impl ChannelDatafileDesc {
    pub fn timebin(&self) -> u32 {
        self.tb
    }
    pub fn binsize(&self) -> u32 {
        self.bs
    }
    pub fn keyspace(&self) -> u32 {
        self.ks
    }
    pub fn split(&self) -> u32 {
        self.sp
    }
}

pub trait ChannelDatafileDescSink {
    fn sink(&self, k: ChannelDatafileDesc) -> Pin<Box<dyn Future<Output = Result<(), Error>>>>;
}

impl ChannelDatafileDescSink for DatafileDbWriter {
    fn sink(&self, k: ChannelDatafileDesc) -> Pin<Box<dyn Future<Output = Result<(), Error>>>> {
        let dbc = self.dbc.clone();
        let c1 = self.c1.clone();
        let channel_id = self.channel_id;
        let node_id = self.node_id;
        Box::pin(async move {
            dbc.query(
                "insert into datafiles (node, channel, tsbeg, tsend, props) values ($1, $2, $3, $4, $5) on conflict do nothing",
                &[
                    &node_id,
                    &channel_id,
                    &(k.timebin() as i64 * k.binsize() as i64),
                    &((k.timebin() + 1) as i64 * k.binsize() as i64),
                    &serde_json::to_value(k)?,
                ]
            ).await?;
            *c1.write()? += 1;
            Ok(())
        })
    }
}

pub async fn find_channel_datafiles_in_ks(
    base_dir: impl AsRef<Path>,
    ks_prefix: &str,
    ks: u32,
    channel: &str,
    cb: &dyn ChannelDatafileDescSink,
) -> Result<Option<()>, Error> {
    let data_dir_path: PathBuf = base_dir
        .as_ref()
        .join(format!("{}_{}", ks_prefix, ks))
        .join("byTime")
        .join(channel);
    let re1 = regex::Regex::new(r"^\d{19}$")?;
    let re2 = regex::Regex::new(r"^\d{10}$")?;
    let re4 = regex::Regex::new(r"^(\d{19})_0{5}_Data$")?;
    let mut rd = match tokio::fs::read_dir(&data_dir_path).await {
        Ok(k) => k,
        Err(e) => match e.kind() {
            ErrorKind::NotFound => return Ok(None),
            _ => Err(e)?,
        },
    };
    while let Ok(Some(entry)) = rd.next_entry().await {
        let fname = String::from_utf8(entry.file_name().into_vec())?;
        if !re1.is_match(&fname) {
            warn!("unexpected file  {}", fname);
            continue;
        }
        let timebin: u32 = fname.parse()?;
        let path = data_dir_path.join(fname);
        let mut rd = tokio::fs::read_dir(&path).await?;
        while let Ok(Some(entry)) = rd.next_entry().await {
            let fname = String::from_utf8(entry.file_name().into_vec())?;
            if !re2.is_match(&fname) {
                warn!("unexpected file  {}", fname);
                continue;
            }
            let split: u32 = fname.parse()?;
            let path = path.join(fname);
            let mut rd = tokio::fs::read_dir(&path).await?;
            while let Ok(Some(entry)) = rd.next_entry().await {
                let fname = String::from_utf8(entry.file_name().into_vec())?;
                if let Some(m) = re4.captures(&fname) {
                    let binsize: u32 = m.get(1).unwrap().as_str().parse()?;
                    let data_path = path.join(&fname);
                    let meta = tokio::fs::metadata(&data_path).await?;
                    let index_path = path.join(format!("{}_Index", fname));
                    let (ix_size, ix_tmod) = if let Ok(meta) = tokio::fs::metadata(&index_path).await {
                        (Some(meta.len()), Some(meta.modified().unwrap().into()))
                    } else {
                        (None, None)
                    };
                    cb.sink(ChannelDatafileDesc {
                        channel: ChannelDesc { name: channel.into() },
                        ks: ks,
                        tb: timebin,
                        sp: split,
                        bs: binsize,
                        fs: meta.len(),
                        ix_fs: ix_size,
                        ix_mt: ix_tmod,
                        mt: meta.modified().unwrap().into(),
                    })
                    .await?;
                }
            }
        }
    }
    Ok(Some(()))
}

pub async fn update_db_with_channel_datafiles(
    node_config: &NodeConfigCached,
    node_disk_ident: &NodeDiskIdent,
    ks_prefix: &str,
    channel_id: i64,
    channel: &str,
    dbc: Arc<Client>,
) -> Result<(), Error> {
    let writer = DatafileDbWriter {
        node_id: node_disk_ident.rowid(),
        channel_id: channel_id,
        dbc: dbc.clone(),
        c1: Arc::new(RwLock::new(0)),
    };
    let mut n_nothing = 0;
    for ks in &[2, 3, 4] {
        match find_channel_datafiles_in_ks(&node_config.node.data_base_path, ks_prefix, *ks, channel, &writer).await {
            /*Err(Error::ChannelDatadirNotFound { .. }) => {
                n_nothing += 1;
            }*/
            Ok(None) => {
                n_nothing += 1;
            }
            x => {
                x?;
            }
        };
        if false && *writer.c1.read()? >= 10 {
            break;
        }
    }
    if n_nothing >= 3 {
        //warn!("No datafile directories in any keyspace  writer got {:5}  n_nothing {}  channel {}", writer.c1.borrow(), n_nothing, channel);
    }
    Ok(())
}