WIP on client and clean up the binned range data structures

retrieval/src/bin/retrieval.rs

@@ -17,7 +17,7 @@ pub fn main() {
 
 async fn go() -> Result<(), Error> {
     use clap::Clap;
-    use retrieval::cli::{Opts, SubCmd};
+    use retrieval::cli::{ClientType, Opts, SubCmd};
     let opts = Opts::parse();
     match opts.subcmd {
         SubCmd::Retrieval(subcmd) => {
@@ -33,6 +33,13 @@ async fn go() -> Result<(), Error> {
                 .ok_or(Error::with_msg(format!("nodeid config error")))?;
             retrieval::run_node(node_config.clone(), node.clone()).await?;
         }
+        SubCmd::Client(client) => match client.client_type {
+            ClientType::Binned(opts) => {
+                let beg = opts.beg.parse()?;
+                let end = opts.end.parse()?;
+                retrieval::client::get_binned(opts.host, opts.port, opts.channel, beg, end, opts.bins).await?;
+            }
+        },
     }
     Ok(())
 }

retrieval/src/cli.rs

@@ -12,6 +12,7 @@ pub struct Opts {
 #[derive(Debug, Clap)]
 pub enum SubCmd {
     Retrieval(Retrieval),
+    Client(Client),
 }
 
 #[derive(Debug, Clap)]
@@ -19,3 +20,30 @@ pub struct Retrieval {
     #[clap(long)]
     pub config: String,
 }
+
+#[derive(Debug, Clap)]
+pub struct Client {
+    #[clap(subcommand)]
+    pub client_type: ClientType,
+}
+
+#[derive(Debug, Clap)]
+pub enum ClientType {
+    Binned(BinnedClient),
+}
+
+#[derive(Debug, Clap)]
+pub struct BinnedClient {
+    #[clap(long)]
+    pub host: String,
+    #[clap(long)]
+    pub port: u16,
+    #[clap(long)]
+    pub channel: String,
+    #[clap(long)]
+    pub beg: String,
+    #[clap(long)]
+    pub end: String,
+    #[clap(long)]
+    pub bins: u32,
+}

retrieval/src/client.rs

@@ -0,0 +1,86 @@
+use chrono::{DateTime, Utc};
+use disk::frame::inmem::InMemoryFrameAsyncReadStream;
+use err::Error;
+use futures_util::TryStreamExt;
+use hyper::Body;
+use netpod::log::*;
+
+pub async fn get_binned(
+    host: String,
+    port: u16,
+    channel_name: String,
+    beg_date: DateTime<Utc>,
+    end_date: DateTime<Utc>,
+    bin_count: u32,
+) -> Result<(), Error> {
+    let t1 = Utc::now();
+    let channel_backend = "NOBACKEND";
+    let date_fmt = "%Y-%m-%dT%H:%M:%S.%3fZ";
+    let uri = format!(
+        "http://{}:{}/api/1/binned?channel_backend={}&channel_name={}&beg_date={}&end_date={}&bin_count={}",
+        host,
+        port,
+        channel_backend,
+        channel_name,
+        beg_date.format(date_fmt),
+        end_date.format(date_fmt),
+        bin_count,
+    );
+    info!("URI {:?}", uri);
+    let req = hyper::Request::builder()
+        .method(http::Method::GET)
+        .uri(uri)
+        .body(Body::empty())?;
+    info!("Request for {:?}", req);
+    let client = hyper::Client::new();
+    let res = client.request(req).await?;
+    info!("client response {:?}", res);
+    //let (res_head, mut res_body) = res.into_parts();
+    let s1 = disk::cache::HttpBodyAsAsyncRead::new(res);
+    let s2 = InMemoryFrameAsyncReadStream::new(s1);
+    use futures_util::StreamExt;
+    use std::future::ready;
+    let mut bin_count = 0;
+    let s3 = s2
+        .map_err(|e| error!("{:?}", e))
+        .filter_map(|item| {
+            let g = match item {
+                Ok(frame) => {
+                    type ExpectedType = disk::cache::BinnedBytesForHttpStreamFrame;
+                    info!("frame  len {}", frame.buf().len());
+                    match bincode::deserialize::<ExpectedType>(frame.buf()) {
+                        Ok(item) => match item {
+                            Ok(item) => {
+                                info!("item: {:?}", item);
+                                bin_count += 1;
+                                Some(Ok(item))
+                            }
+                            Err(e) => {
+                                error!("error frame: {:?}", e);
+                                Some(Err(e))
+                            }
+                        },
+                        Err(e) => {
+                            error!("bincode error: {:?}", e);
+                            Some(Err(e.into()))
+                        }
+                    }
+                }
+                Err(e) => Some(Err(Error::with_msg(format!("{:?}", e)))),
+            };
+            ready(g)
+        })
+        .for_each(|_| ready(()));
+    s3.await;
+    let t2 = chrono::Utc::now();
+    let ntot = 0;
+    let ms = t2.signed_duration_since(t1).num_milliseconds() as u64;
+    let throughput = ntot / 1024 * 1000 / ms;
+    info!(
+        "get_cached_0 DONE  total download {} MB   throughput {:5} kB/s  bin_count {}",
+        ntot / 1024 / 1024,
+        throughput,
+        bin_count,
+    );
+    Ok(())
+}

retrieval/src/lib.rs

@@ -5,6 +5,7 @@ use tokio::task::JoinHandle;
 use tracing::{debug, error, info, trace, warn};
 
 pub mod cli;
+pub mod client;
 #[cfg(test)]
 pub mod test;
 

retrieval/src/test.rs

@@ -1,6 +1,6 @@
 use crate::spawn_test_hosts;
 use bytes::BytesMut;
-use chrono::Utc;
+use chrono::{DateTime, Utc};
 use disk::frame::inmem::InMemoryFrameAsyncReadStream;
 use err::Error;
 use futures_util::TryStreamExt;
@@ -40,12 +40,12 @@ fn get_cached_0() {
 }
 
 async fn get_cached_0_inner() -> Result<(), Error> {
-    let t1 = chrono::Utc::now();
+    let t1 = Utc::now();
     let cluster = test_cluster();
     let node0 = &cluster.nodes[0];
     let hosts = spawn_test_hosts(cluster.clone());
-    let beg_date: chrono::DateTime<Utc> = "1970-01-01T00:20:10.000Z".parse()?;
-    let end_date: chrono::DateTime<Utc> = "1970-01-01T00:20:51.000Z".parse()?;
+    let beg_date: DateTime<Utc> = "1970-01-01T00:20:10.000Z".parse()?;
+    let end_date: DateTime<Utc> = "1970-01-01T00:20:51.000Z".parse()?;
     let channel_backend = "back";
     let channel_name = "wave1";
     let bin_count = 4;