Update deps, wip on bin read index

crates/scyllaconn/src/accounting/totals.rs

@@ -9,10 +9,10 @@ use items_0::Empty;
 use items_0::Extendable;
 use items_0::WithLen;
 use items_2::accounting::AccountingEvents;
+use netpod::EMIT_ACCOUNTING_SNAP;
 use netpod::log::*;
 use netpod::range::evrange::NanoRange;
 use netpod::timeunits;
-use netpod::EMIT_ACCOUNTING_SNAP;
 use scylla::statement::prepared::PreparedStatement;
 use std::collections::VecDeque;
 use std::pin::Pin;
@@ -215,7 +215,7 @@ impl Stream for AccountingStreamScylla {
                         continue;
                     }
                 }
-                FrState::ReadValues(ref mut st) => match st.fut.poll_unpin(cx) {
+                FrState::ReadValues(st) => match st.fut.poll_unpin(cx) {
                     Ready(Ok(mut item)) => {
                         if !st.next() {
                             self.state = FrState::Done;

crates/scyllaconn/src/binned2.rs

@@ -0,0 +1,4 @@
+pub mod binnedrtbinlen;
+pub mod binnedrtmsplsps;
+pub mod intraday;
+pub mod msplspiter;

crates/scyllaconn/src/binned2/binnedrtbinlen.rs

@@ -0,0 +1,47 @@
+use crate::worker::ScyllaQueue;
+use daqbuf_series::SeriesId;
+use daqbuf_series::msp::PrebinnedPartitioning;
+use netpod::BinnedRange;
+use netpod::TsNano;
+use netpod::ttl::RetentionTime;
+
+/*
+Given RT, PBP and range, loop over all the bins to retrieve.
+
+When there is no content, skip over.
+*/
+
+pub struct BinnedRtBinlenStream {
+    series: SeriesId,
+    rt: RetentionTime,
+    pbp: PrebinnedPartitioning,
+    range: BinnedRange<TsNano>,
+    scyqueue: ScyllaQueue,
+}
+
+impl BinnedRtBinlenStream {
+    pub fn new(
+        series: SeriesId,
+        rt: RetentionTime,
+        pbp: PrebinnedPartitioning,
+        range: BinnedRange<TsNano>,
+        scyqueue: ScyllaQueue,
+    ) -> Self {
+        Self {
+            series,
+            rt,
+            pbp,
+            range,
+            scyqueue,
+        }
+    }
+
+    fn make_next_fut(&mut self) -> Option<()> {
+        let series = self.series.clone();
+        let rt = self.rt.clone();
+        let msp = todo!();
+        let binlen = todo!();
+        let lsps = todo!();
+        super::binnedrtmsplsps::BinnedRtMspLsps::new(series, rt, msp, binlen, lsps, self.scyqueue.clone());
+    }
+}

crates/scyllaconn/src/binned2/binnedrtmsplsps.rs

@@ -0,0 +1,82 @@
+/*
+Fetches the bins for a given RT, binlen and MSP.
+Issues the scylla commands.
+Assembles the results.
+Does basic sanity checks.
+May re-chunk the result if too large.
+*/
+
+use crate::worker::ScyllaQueue;
+use daqbuf_series::SeriesId;
+use daqbuf_series::msp::LspU32;
+use daqbuf_series::msp::MspU32;
+use items_2::binning::container_bins::ContainerBins;
+use netpod::DtMs;
+use netpod::ttl::RetentionTime;
+use std::fmt;
+use std::pin::Pin;
+
+macro_rules! info { ($($arg:expr),*) => ( if true { log::info!($($arg),*); } ); }
+macro_rules! debug { ($($arg:expr),*) => ( if true { log::debug!($($arg),*); } ); }
+
+autoerr::create_error_v1!(
+    name(Error, "BinnedRtMsp"),
+    enum variants {
+        ReadJob(#[from] streams::timebin::cached::reader::Error),
+    },
+);
+
+type Fut =
+    Pin<Box<dyn Future<Output = Result<ContainerBins<f32, f32>, streams::timebin::cached::reader::Error>> + Send>>;
+
+struct FutW(Fut);
+
+impl fmt::Debug for FutW {
+    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
+        fmt.debug_tuple("Fut").finish()
+    }
+}
+
+pub struct BinnedRtMspLsps {
+    series: SeriesId,
+    rt: RetentionTime,
+    msp: MspU32,
+    lsps: (LspU32, LspU32),
+    binlen: DtMs,
+    scyqueue: ScyllaQueue,
+    fut: Option<FutW>,
+}
+
+impl BinnedRtMspLsps {
+    pub fn new(
+        series: SeriesId,
+        rt: RetentionTime,
+        msp: MspU32,
+        binlen: DtMs,
+        lsps: (LspU32, LspU32),
+        scyqueue: ScyllaQueue,
+    ) -> Self {
+        Self {
+            series,
+            rt,
+            msp,
+            lsps,
+            binlen,
+            scyqueue,
+            fut: None,
+        }
+    }
+
+    fn make_next_fut(&mut self) -> Option<Fut> {
+        let rt = self.rt.clone();
+        let series = self.series.id();
+        let binlen = self.binlen.clone();
+        let msp = self.msp.to_u64();
+        let offs = self.lsps.0.to_u32()..self.lsps.1.to_u32();
+        // SAFETY we only use scyqueue while we self are alive.
+        let scyqueue = unsafe { &mut *(&mut self.scyqueue as *mut ScyllaQueue) };
+        let fut = scyqueue.read_prebinned_f32(rt, series, binlen, msp, offs);
+        let fut = Box::pin(fut);
+        Some(fut)
+    }
+}

crates/scyllaconn/src/binned2/intraday.rs

@@ -0,0 +1 @@
+

crates/scyllaconn/src/binned2/msplspiter.rs

@@ -0,0 +1,51 @@
+use daqbuf_series::msp::LspU32;
+use daqbuf_series::msp::MspU32;
+use daqbuf_series::msp::PrebinnedPartitioning;
+use netpod::TsMs;
+use netpod::range::evrange::NanoRange;
+
+#[derive(Debug, Clone)]
+pub struct MspLspItem {
+    pub msp: MspU32,
+    pub lsp: LspU32,
+}
+
+#[derive(Debug)]
+pub struct MspLspIter {
+    range: NanoRange,
+    pbp: PrebinnedPartitioning,
+    ts: TsMs,
+}
+
+impl MspLspIter {
+    pub fn new(range: NanoRange, pbp: PrebinnedPartitioning) -> Self {
+        let ts = range.beg_ts().to_ts_ms();
+        Self { range, pbp, ts }
+    }
+}
+
+impl Iterator for MspLspIter {
+    type Item = (MspU32, LspU32);
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.ts >= self.range.end_ts().to_ts_ms() {
+            None
+        } else {
+            let x = self.pbp.msp_lsp(self.ts);
+            let msp = MspU32(x.0);
+            let lsp = LspU32(x.1);
+            self.ts = self.ts.add_dt_ms(self.pbp.bin_len());
+            Some((msp, lsp))
+        }
+    }
+}
+
+#[test]
+fn test_iter_00() {
+    let range = NanoRange::from_strings("", "").unwrap();
+    let pbp = PrebinnedPartitioning::Sec1;
+    let mut it = MspLspIter::new(range, pbp);
+    for x in it {
+        eprintln!("{:?}", x);
+    }
+}

crates/scyllaconn/src/binwriteindex.rs

@@ -1,4 +1,5 @@
 pub mod bwxcmb;
+pub mod read_all_coarse;
 
 use crate::worker::ScyllaQueue;
 use daqbuf_series::msp::MspU32;
@@ -10,6 +11,7 @@ use futures_util::Stream;
 use netpod::log;
 use netpod::range::evrange::NanoRange;
 use netpod::ttl::RetentionTime;
+use netpod::DtMs;
 use std::collections::VecDeque;
 use std::fmt;
 use std::pin::Pin;
@@ -45,7 +47,6 @@ pub struct BinWriteIndexEntry {
 
 #[derive(Debug)]
 pub struct BinWriteIndexSet {
-    pub rt: RetentionTime,
     pub msp: MspU32,
     pub entries: VecDeque<BinWriteIndexEntry>,
 }
@@ -78,7 +79,12 @@ impl BinWriteIndexRtStream {
         info!("{}::new  INFO/DEBUG test", Self::type_name());
         debug!("{}::new", Self::type_name());
         let (msp_beg, lsp_beg) = pbp.msp_lsp(range.beg_ts().to_ts_ms());
-        let (msp_end, lsp_end) = pbp.msp_lsp(range.end_ts().add_dt_nano(pbp.bin_len().dt_ns()).to_ts_ms());
+        let (msp_end, lsp_end) = pbp.msp_lsp(
+            range
+                .end_ts()
+                .add_dt_nano(DtMs::from_ms_u64(pbp.bin_len().ms() - 1).dt_ns())
+                .to_ts_ms(),
+        );
         BinWriteIndexRtStream {
             rt1,
             series,
@@ -148,7 +154,6 @@ impl Stream for BinWriteIndexRtStream {
                     Ready(Ok(x)) => {
                         self.fut1 = None;
                         let item = BinWriteIndexSet {
-                            rt: self.rt1.clone(),
                             msp: MspU32(x.0),
                             entries: x.3,
                         };

crates/scyllaconn/src/binwriteindex/read_all_coarse.rs

@@ -0,0 +1,54 @@
+use super::BinWriteIndexRtStream;
+use crate::worker::ScyllaQueue;
+use daqbuf_series::msp::MspU32;
+use daqbuf_series::msp::PrebinnedPartitioning;
+use daqbuf_series::SeriesId;
+use futures_util::TryStreamExt;
+use netpod::log;
+use netpod::range::evrange::NanoRange;
+use netpod::ttl::RetentionTime;
+use netpod::DtMs;
+use std::collections::VecDeque;
+
+macro_rules! info { ($($arg:expr),*) => ( if true { log::info!($($arg),*); } ); }
+macro_rules! debug { ($($arg:expr),*) => ( if true { log::debug!($($arg),*); } ); }
+
+autoerr::create_error_v1!(
+    name(Error, "BinIndexReadAllCoarse"),
+    enum variants {
+        Worker(#[from] crate::worker::Error),
+        BinWriteIndexRead(#[from] super::Error),
+    },
+);
+
+pub async fn read_all_coarse(
+    series: SeriesId,
+    range: NanoRange,
+    scyqueue: &ScyllaQueue,
+) -> Result<VecDeque<(RetentionTime, MspU32, u32, DtMs)>, Error> {
+    let rts = {
+        use RetentionTime::*;
+        [Long, Medium, Short]
+    };
+    let mut ret = VecDeque::new();
+    for rt in rts {
+        let pbp = PrebinnedPartitioning::Day1;
+        let mut stream = BinWriteIndexRtStream::new(rt.clone(), series, pbp, range.clone(), scyqueue.clone());
+        while let Some(x) = stream.try_next().await? {
+            for e in x.entries {
+                let binlen = DtMs::from_ms_u64(e.binlen as u64);
+                let item = (rt.clone(), x.msp.clone(), e.lsp, binlen);
+                ret.push_back(item);
+            }
+        }
+    }
+    Ok(ret)
+}
+
+pub fn select_potential_binlen(options: VecDeque<(RetentionTime, MspU32, u32, DtMs)>) -> Result<(), Error> {
+    // Check first if there are common binlen over all the range.
+    // If not, filter out the options which could build content from finer resolution.
+    // Then heuristically select the best match.
+    // PrebinnedPartitioning::Day1.msp_lsp(val)
+    todo!()
+}

crates/scyllaconn/src/conn.rs

@@ -1,19 +1,23 @@
-use crate::errconv::ErrConv;
-use daqbuf_err as err;
-use err::Error;
 use netpod::log::*;
 use netpod::ScyllaConfig;
 use scylla::client::execution_profile::ExecutionProfileBuilder;
 use scylla::client::session::Session;
 use scylla::client::session_builder::SessionBuilder;
+use scylla::errors::NewSessionError;
 use scylla::statement::Consistency;
 use std::sync::Arc;
 
+autoerr::create_error_v1!(
+    name(Error, "ScyllaSessionCreate"),
+    enum variants {
+        ScyllaSessionNew(#[from] NewSessionError),
+        ScyllaUseKeyspace(#[from] scylla::errors::UseKeyspaceError),
+    },
+);
+
 pub async fn create_scy_session(scyconf: &ScyllaConfig) -> Result<Arc<Session>, Error> {
     let scy = create_scy_session_no_ks(scyconf).await?;
-    scy.use_keyspace(&scyconf.keyspace, true)
-        .await
-        .map_err(Error::from_string)?;
+    scy.use_keyspace(&scyconf.keyspace, true).await?;
     let ret = Arc::new(scy);
     Ok(ret)
 }
@@ -24,12 +28,11 @@ pub async fn create_scy_session_no_ks(scyconf: &ScyllaConfig) -> Result<Session,
         .known_nodes(&scyconf.hosts)
         .default_execution_profile_handle(
             ExecutionProfileBuilder::default()
-                .consistency(Consistency::LocalOne)
+                .consistency(Consistency::Quorum)
                 .build()
                 .into_handle(),
         )
         .build()
-        .await
-        .err_conv()?;
+        .await?;
     Ok(scy)
 }

crates/scyllaconn/src/events2/events.rs

@@ -9,18 +9,16 @@ use futures_util::FutureExt;
 use futures_util::Stream;
 use futures_util::StreamExt;
 use futures_util::TryStreamExt;
+use items_0::Appendable;
+use items_0::Empty;
+use items_0::WithLen;
 use items_0::container::ByteEstimate;
 use items_0::merge::DrainIntoNewDynResult;
 use items_0::merge::MergeableDyn;
 use items_0::scalar_ops::ScalarOps;
 use items_0::timebin::BinningggContainerEventsDyn;
-use items_0::Appendable;
-use items_0::Empty;
-use items_0::WithLen;
 use items_2::binning::container_events::ContainerEvents;
 use items_2::channelevents::ChannelEvents;
-use netpod::log;
-use netpod::ttl::RetentionTime;
 use netpod::ChConf;
 use netpod::DtNano;
 use netpod::EnumVariant;
@@ -29,6 +27,8 @@ use netpod::Shape;
 use netpod::TsMs;
 use netpod::TsMsVecFmt;
 use netpod::TsNano;
+use netpod::log;
+use netpod::ttl::RetentionTime;
 use scylla::client::session::Session;
 use std::collections::VecDeque;
 use std::fmt;
@@ -70,7 +70,7 @@ impl EventReadOpts {
         Self {
             one_before,
             with_values,
-            qucap: qucap.unwrap_or(3),
+            qucap: qucap.unwrap_or(6),
         }
     }
 

crates/scyllaconn/src/lib.rs

@@ -1,5 +1,6 @@
 pub mod accounting;
 pub mod bincache;
+pub mod binned2;
 pub mod binwriteindex;
 pub mod conn;
 pub mod errconv;

crates/scyllaconn/src/worker.rs

@@ -5,7 +5,6 @@ use crate::events2::prepare::StmtsEvents;
 use crate::range::ScyllaSeriesRange;
 use async_channel::Receiver;
 use async_channel::Sender;
-use daqbuf_err as err;
 use daqbuf_series::msp::MspU32;
 use daqbuf_series::msp::PrebinnedPartitioning;
 use daqbuf_series::SeriesId;
@@ -34,7 +33,7 @@ const SCYLLA_WORKER_QUEUE_LEN: usize = 200;
 autoerr::create_error_v1!(
     name(Error, "ScyllaWorker"),
     enum variants {
-        ScyllaConnection(err::Error),
+        ScyllaConnection(#[from] crate::conn::Error),
         Prepare(#[from] crate::events2::prepare::Error),
         Events(#[from] crate::events2::events::Error),
         Msp(#[from] crate::events2::msp::Error),
@@ -326,9 +325,7 @@ impl ScyllaWorker {
     }
 
     pub async fn work(self) -> Result<(), Error> {
-        let scy = create_scy_session_no_ks(&self.scyconf_st)
-            .await
-            .map_err(Error::ScyllaConnection)?;
+        let scy = create_scy_session_no_ks(&self.scyconf_st).await?;
         let scy = Arc::new(scy);
         let kss = [
             self.scyconf_st.keyspace.as_str(),