Seems like I can write and read cached data

disk/src/agg/binnedt.rs

@@ -129,6 +129,9 @@ where
                             return Ready(None);
                         }
                     }
+                } else {
+                    self.completed = true;
+                    return Ready(None);
                 }
             }
             let cur = if let Some(k) = self.left.take() {
@@ -163,7 +166,6 @@ where
                             //info!("ENDS BEFORE");
                             continue 'outer;
                         } else if ag.starts_after(&k) {
-                            //info!("STARTS AFTER");
                             self.left = Some(Ready(Some(Ok(k))));
                             self.curbin += 1;
                             let range = self.spec.get_range(self.curbin);
@@ -172,17 +174,16 @@ where
                                 .replace(I::aggregator_new_static(range.beg, range.end))
                                 .unwrap()
                                 .result();
-                            //Ready(Some(Ok(ret)))
                             self.tmp_agg_results = ret.into();
+                            if self.curbin as u64 >= self.spec.count {
+                                self.data_completed = true;
+                            }
                             continue 'outer;
                         } else {
-                            //info!("INGEST");
                             let mut k = k;
                             ag.ingest(&mut k);
-                            // if this input contains also data after the current bin, then I need to keep
+                            let k = k;
-                            // it for the next round.
                             if ag.ends_after(&k) {
-                                //info!("ENDS AFTER");
                                 self.left = Some(Ready(Some(Ok(k))));
                                 self.curbin += 1;
                                 let range = self.spec.get_range(self.curbin);
@@ -191,11 +192,12 @@ where
                                     .replace(I::aggregator_new_static(range.beg, range.end))
                                     .unwrap()
                                     .result();
-                                //Ready(Some(Ok(ret)))
                                 self.tmp_agg_results = ret.into();
+                                if self.curbin as u64 >= self.spec.count {
+                                    self.data_completed = true;
+                                }
                                 continue 'outer;
                             } else {
-                                //info!("ENDS WITHIN");
                                 continue 'outer;
                             }
                         }
@@ -208,7 +210,6 @@ where
                 Ready(None) => {
                     self.inp_completed = true;
                     if self.curbin as u64 >= self.spec.count {
-                        warn!("IntoBinnedTDefaultStream  curbin out of spec, END");
                         self.data_completed = true;
                         continue 'outer;
                     } else {

disk/src/cache.rs

@@ -2,6 +2,7 @@ use crate::agg::eventbatch::MinMaxAvgScalarEventBatchStreamItem;
 use crate::agg::scalarbinbatch::MinMaxAvgScalarBinBatch;
 use crate::binnedstream::BinnedStream;
 use crate::cache::pbv::PreBinnedValueByteStream;
+use crate::cache::pbvfs::PreBinnedItem;
 use crate::channelconfig::{extract_matching_config_entry, read_local_config};
 use crate::frame::makeframe::make_frame;
 use crate::merge::MergedMinMaxAvgScalarStream;
@@ -23,8 +24,8 @@ use std::path::PathBuf;
 use std::pin::Pin;
 use std::task::{Context, Poll};
 use tiny_keccak::Hasher;
-use tokio::fs::OpenOptions;
+use tokio::fs::File;
-use tokio::io::{AsyncRead, AsyncWriteExt, ReadBuf};
+use tokio::io::{AsyncRead, AsyncReadExt, ReadBuf};
 #[allow(unused_imports)]
 use tracing::{debug, error, info, trace, warn};
 
@@ -512,12 +513,31 @@ pub async fn write_pb_cache_min_max_avg_scalar(
     let enc = serde_cbor::to_vec(&values)?;
     info!("Encoded size: {}", enc.len());
     tokio::fs::create_dir_all(path.parent().unwrap()).await?;
-    let mut f = OpenOptions::new()
+    tokio::task::spawn_blocking({
-        .truncate(true)
+        let path = path.clone();
-        .create(true)
+        move || {
-        .write(true)
+            use fs2::FileExt;
-        .open(&path)
+            use std::io::Write;
-        .await?;
+            let mut f = std::fs::OpenOptions::new()
-    f.write_all(&enc).await?;
+                .create(true)
+                .truncate(true)
+                .write(true)
+                .open(&path)?;
+            f.lock_exclusive()?;
+            f.write_all(&enc)?;
+            f.unlock()?;
+            Ok::<_, Error>(())
+        }
+    })
+    .await??;
     Ok(())
 }
+
+pub async fn read_pbv(mut file: File) -> Result<PreBinnedItem, Error> {
+    let mut buf = vec![];
+    file.read_to_end(&mut buf).await?;
+    info!("Read cached file  len {}", buf.len());
+    let dec: MinMaxAvgScalarBinBatch = serde_cbor::from_slice(&buf)?;
+    info!("Decoded cached file");
+    Ok(PreBinnedItem::Batch(dec))
+}

disk/src/cache/pbv.rs

@@ -1,7 +1,7 @@
 use crate::agg::binnedt::IntoBinnedT;
 use crate::agg::scalarbinbatch::{MinMaxAvgScalarBinBatch, MinMaxAvgScalarBinBatchStreamItem};
 use crate::cache::pbvfs::{PreBinnedItem, PreBinnedValueFetchedStream};
-use crate::cache::{MergedFromRemotes, PreBinnedQuery};
+use crate::cache::{CacheFileDesc, MergedFromRemotes, PreBinnedQuery};
 use crate::frame::makeframe::make_frame;
 use crate::raw::EventsQuery;
 use crate::streamlog::Streamlog;
@@ -61,6 +61,7 @@ pub struct PreBinnedValueStream {
     streamlog: Streamlog,
     values: MinMaxAvgScalarBinBatch,
     write_fut: Option<Pin<Box<dyn Future<Output = Result<(), Error>> + Send>>>,
+    read_cache_fut: Option<Pin<Box<dyn Future<Output = Result<PreBinnedItem, Error>> + Send>>>,
 }
 
 impl PreBinnedValueStream {
@@ -78,6 +79,7 @@ impl PreBinnedValueStream {
             streamlog: Streamlog::new(node_config.ix as u32),
             values: MinMaxAvgScalarBinBatch::empty(),
             write_fut: None,
+            read_cache_fut: None,
         }
     }
 
@@ -196,14 +198,35 @@ impl Stream for PreBinnedValueStream {
             } else if let Some(fut) = &mut self.write_fut {
                 pin_mut!(fut);
                 match fut.poll(cx) {
-                    Ready(Ok(())) => {
+                    Ready(item) => {
                         self.write_fut = None;
-                        self.streamlog.append(Level::INFO, format!("cache file written"));
+                        match item {
-                        continue 'outer;
+                            Ok(()) => {
+                                self.streamlog.append(Level::INFO, format!("cache file written"));
+                                continue 'outer;
+                            }
+                            Err(e) => {
+                                self.errored = true;
+                                Ready(Some(Err(e)))
+                            }
+                        }
                     }
-                    Ready(Err(e)) => {
+                    Pending => Pending,
-                        self.errored = true;
+                }
-                        Ready(Some(Err(e)))
+            } else if let Some(fut) = &mut self.read_cache_fut {
+                match fut.poll_unpin(cx) {
+                    Ready(item) => {
+                        self.read_cache_fut = None;
+                        match item {
+                            Ok(item) => {
+                                self.data_complete = true;
+                                Ready(Some(Ok(item)))
+                            }
+                            Err(e) => {
+                                self.errored = true;
+                                Ready(Some(Err(e)))
+                            }
+                        }
                     }
                     Pending => Pending,
                 }
@@ -268,29 +291,35 @@ impl Stream for PreBinnedValueStream {
                 }
             } else if let Some(fut) = self.open_check_local_file.as_mut() {
                 match fut.poll_unpin(cx) {
-                    Ready(Ok(_file)) => {
+                    Ready(item) => {
-                        let e = Err(Error::with_msg(format!("TODO use the cached data from file")));
+                        self.open_check_local_file = None;
-                        self.errored = true;
+                        match item {
-                        Ready(Some(e))
+                            Ok(file) => {
-                    }
+                                let fut = super::read_pbv(file);
-                    Ready(Err(e)) => match e.kind() {
+                                self.read_cache_fut = Some(Box::pin(fut));
-                        std::io::ErrorKind::NotFound => {
-                            error!("TODO LOCAL CACHE FILE NOT FOUND");
-                            self.try_setup_fetch_prebinned_higher_res();
-                            if self.fut2.is_none() {
-                                let e = Err(Error::with_msg(format!("try_setup_fetch_prebinned_higher_res  failed")));
-                                self.errored = true;
-                                Ready(Some(e))
-                            } else {
                                 continue 'outer;
                             }
+                            Err(e) => match e.kind() {
+                                std::io::ErrorKind::NotFound => {
+                                    self.try_setup_fetch_prebinned_higher_res();
+                                    if self.fut2.is_none() {
+                                        let e = Err(Error::with_msg(format!(
+                                            "try_setup_fetch_prebinned_higher_res  failed"
+                                        )));
+                                        self.errored = true;
+                                        Ready(Some(e))
+                                    } else {
+                                        continue 'outer;
+                                    }
+                                }
+                                _ => {
+                                    error!("File I/O error: {:?}", e);
+                                    self.errored = true;
+                                    Ready(Some(Err(e.into())))
+                                }
+                            },
                         }
-                        _ => {
+                    }
-                            error!("File I/O error: {:?}", e);
-                            self.errored = true;
-                            Ready(Some(Err(e.into())))
-                        }
-                    },
                     Pending => Pending,
                 }
             } else {
@@ -298,7 +327,13 @@ impl Stream for PreBinnedValueStream {
                 use std::os::unix::fs::OpenOptionsExt;
                 let mut opts = std::fs::OpenOptions::new();
                 opts.read(true);
-                let fut = async { tokio::fs::OpenOptions::from(opts).open("/DOESNOTEXIST").await };
+                let cfd = CacheFileDesc {
+                    channel: self.query.channel.clone(),
+                    patch: self.query.patch.clone(),
+                    agg_kind: self.query.agg_kind.clone(),
+                };
+                let path = cfd.path(&self.node_config);
+                let fut = async { tokio::fs::OpenOptions::from(opts).open(path).await };
                 self.open_check_local_file = Some(Box::pin(fut));
                 continue 'outer;
             };

retrieval/src/test.rs

@@ -151,7 +151,7 @@ where
                                     Some(Ok(MinMaxAvgScalarBinBatchStreamItem::Log(item)))
                                 }
                                 item => {
-                                    info!("TEST GOT ITEM {:?}\n", item);
+                                    info!("TEST GOT ITEM {:?}", item);
                                     Some(Ok(item))
                                 }
                             },