Extract support for all numeric types

Cargo.toml

@@ -3,6 +3,6 @@ members = ["retrieval"]
 
 [profile.release]
 debug = 1
-opt-level = 1
+opt-level = 2
 #overflow-checks = true
 #debug-assertions = true

disk/src/agg.rs

@@ -356,10 +356,25 @@ macro_rules! make_get_values {
     };
 }
 
+make_get_values!(get_values_u8_le, u8, from_le_bytes, 1);
 make_get_values!(get_values_u16_le, u16, from_le_bytes, 2);
+make_get_values!(get_values_u32_le, u32, from_le_bytes, 4);
+make_get_values!(get_values_u64_le, u64, from_le_bytes, 8);
+make_get_values!(get_values_i8_le, i8, from_le_bytes, 1);
+make_get_values!(get_values_i16_le, i16, from_le_bytes, 2);
+make_get_values!(get_values_i32_le, i32, from_le_bytes, 4);
+make_get_values!(get_values_i64_le, i64, from_le_bytes, 8);
 make_get_values!(get_values_f32_le, f32, from_le_bytes, 4);
 make_get_values!(get_values_f64_le, f64, from_le_bytes, 8);
+
+make_get_values!(get_values_u8_be, u8, from_be_bytes, 1);
 make_get_values!(get_values_u16_be, u16, from_be_bytes, 2);
+make_get_values!(get_values_u32_be, u32, from_be_bytes, 4);
+make_get_values!(get_values_u64_be, u64, from_be_bytes, 8);
+make_get_values!(get_values_i8_be, i8, from_be_bytes, 1);
+make_get_values!(get_values_i16_be, i16, from_be_bytes, 2);
+make_get_values!(get_values_i32_be, i32, from_be_bytes, 4);
+make_get_values!(get_values_i64_be, i64, from_be_bytes, 8);
 make_get_values!(get_values_f32_be, f32, from_be_bytes, 4);
 make_get_values!(get_values_f64_be, f64, from_be_bytes, 8);
 
@@ -381,41 +396,96 @@ impl<S> Dim1F32Stream<S> {
             let be = k.be[i1];
             let decomp = k.decomps[i1].as_ref().unwrap();
             match ty {
-                U16 if be => {
+                U8 => {
-                    let value = get_values_u16_be(decomp, ty)?;
+                    let value = if be {
+                        get_values_u8_be(decomp, ty)?
+                    } else {
+                        get_values_u8_le(decomp, ty)?
+                    };
                     ret.tss.push(k.tss[i1]);
                     ret.values.push(value);
                 }
                 U16 => {
-                    let value = get_values_u16_le(decomp, ty)?;
+                    let value = if be {
+                        get_values_u16_be(decomp, ty)?
+                    } else {
+                        get_values_u16_le(decomp, ty)?
+                    };
                     ret.tss.push(k.tss[i1]);
                     ret.values.push(value);
                 }
-                F32 if be => {
+                U32 => {
-                    let value = get_values_f32_be(decomp, ty)?;
+                    let value = if be {
+                        get_values_u32_be(decomp, ty)?
+                    } else {
+                        get_values_u32_le(decomp, ty)?
+                    };
+                    ret.tss.push(k.tss[i1]);
+                    ret.values.push(value);
+                }
+                U64 => {
+                    let value = if be {
+                        get_values_u64_be(decomp, ty)?
+                    } else {
+                        get_values_u64_le(decomp, ty)?
+                    };
+                    ret.tss.push(k.tss[i1]);
+                    ret.values.push(value);
+                }
+                I8 => {
+                    let value = if be {
+                        get_values_i8_be(decomp, ty)?
+                    } else {
+                        get_values_i8_le(decomp, ty)?
+                    };
+                    ret.tss.push(k.tss[i1]);
+                    ret.values.push(value);
+                }
+                I16 => {
+                    let value = if be {
+                        get_values_i16_be(decomp, ty)?
+                    } else {
+                        get_values_i16_le(decomp, ty)?
+                    };
+                    ret.tss.push(k.tss[i1]);
+                    ret.values.push(value);
+                }
+                I32 => {
+                    let value = if be {
+                        get_values_i32_be(decomp, ty)?
+                    } else {
+                        get_values_i32_le(decomp, ty)?
+                    };
+                    ret.tss.push(k.tss[i1]);
+                    ret.values.push(value);
+                }
+                I64 => {
+                    let value = if be {
+                        get_values_i64_be(decomp, ty)?
+                    } else {
+                        get_values_i64_le(decomp, ty)?
+                    };
                     ret.tss.push(k.tss[i1]);
                     ret.values.push(value);
                 }
                 F32 => {
-                    let value = get_values_f32_le(decomp, ty)?;
+                    let value = if be {
-                    ret.tss.push(k.tss[i1]);
+                        get_values_f32_be(decomp, ty)?
-                    ret.values.push(value);
+                    } else {
-                }
+                        get_values_f32_le(decomp, ty)?
-                F64 if be => {
+                    };
-                    let value = get_values_f64_be(decomp, ty)?;
                     ret.tss.push(k.tss[i1]);
                     ret.values.push(value);
                 }
                 F64 => {
-                    let value = get_values_f64_le(decomp, ty)?;
+                    let value = if be {
+                        get_values_f64_be(decomp, ty)?
+                    } else {
+                        get_values_f64_le(decomp, ty)?
+                    };
                     ret.tss.push(k.tss[i1]);
                     ret.values.push(value);
                 }
-                _ => {
-                    let e = Error::with_msg(format!("Dim1F32Stream  unhandled scalar type: {:?}", ty));
-                    self.errored = true;
-                    return Err(e);
-                }
             }
         }
         Ok(ret)

disk/src/eventchunker.rs

@@ -182,7 +182,6 @@ impl EventChunker {
                             //debug!("event  len {}  ts {}  is_compressed {}  shape_dim {}  len-dim-0 {}  value_bytes {}  block_size {}", len, ts, is_compressed, shape_dim, shape_lens[0], value_bytes, block_size);
                             assert!(value_bytes < 1024 * 256);
                             assert!(block_size < 1024 * 32);
-                            //let value_bytes = value_bytes;
                             let type_size = scalar_type.bytes() as u32;
                             let ele_count = value_bytes / type_size as u64;
                             let ele_size = type_size;
@@ -208,7 +207,6 @@ impl EventChunker {
                             unsafe {
                                 decomp.set_len(decomp_bytes);
                             }
-                            //debug!("try decompress  value_bytes {}  ele_size {}  ele_count {}  type_index {}", value_bytes, ele_size, ele_count, type_index);
                             match bitshuffle_decompress(
                                 &buf.as_ref()[p1 as usize..],
                                 &mut decomp,
@@ -218,7 +216,6 @@ impl EventChunker {
                             ) {
                                 Ok(c1) => {
                                     assert!(c1 as u32 == k1);
-                                    //trace!("decompress result  c1 {}  k1 {}", c1, k1);
                                     if ts < self.range.beg {
                                     } else if ts >= self.range.end {
                                         Err(Error::with_msg(format!("event after range  {}", ts / SEC)))?;
@@ -238,13 +235,11 @@ impl EventChunker {
                             };
                         } else {
                             let p1 = sl.position();
-                            //info!("len: {}  p1: {}", len, p1);
                             if len < p1 as u32 + 4 {
                                 let msg = format!("uncomp  len: {}  p1: {}", len, p1);
                                 Err(Error::with_msg(msg))?;
                             }
                             let vlen = len - p1 as u32 - 4;
-                            //info!("vlen: {}", vlen);
                             let decomp = BytesMut::from(&buf[p1 as usize..(p1 as u32 + vlen) as usize]);
                             ret.add_event(
                                 ts,
@@ -315,8 +310,7 @@ impl Stream for EventChunker {
                 Ready(None)
             } else if self.parsed_bytes >= self.stats_conf.disk_stats_every.bytes() as u64 {
                 let item = EventDataReadStats {
-                    //parsed_bytes: self.parsed_bytes,
+                    parsed_bytes: self.parsed_bytes,
-                    parsed_bytes: 1000,
                 };
                 self.parsed_bytes = 0;
                 let ret = EventChunkerItem::EventDataReadStats(item);

disk/src/index.rs

@@ -154,37 +154,28 @@ pub async fn position_file(mut file: File, beg: u64) -> Result<File, Error> {
         //
     }
     let hres = parse_channel_header(&buf)?;
-    info!("hres: {:?}", hres);
     let headoff = 2 + hres.0 as u64;
     let ev = parse_event(&buf[headoff as usize..])?;
-    info!("ev: {:?}", ev);
     let evlen = ev.0 as u64;
-    info!("flen: {}  flen - headoff mod evlen: {}", flen, (flen - headoff) % evlen);
     let mut j = headoff;
     let mut k = ((flen - headoff) / evlen - 1) * evlen + headoff;
-    info!("j {}  k {}", j, k);
     let x = ev.1.ns;
     let y = read_event_at(k, &mut file).await?.1.ns;
-    info!("x {}  y {}", x, y);
     if x >= beg {
-        info!("found A");
         file.seek(SeekFrom::Start(j)).await?;
         return Ok(file);
     }
     if y < beg {
-        info!("found B");
         file.seek(SeekFrom::Start(j)).await?;
         return Ok(file);
     }
     loop {
         if k - j < 2 * evlen {
-            info!("found C");
             file.seek(SeekFrom::Start(k)).await?;
             return Ok(file);
         }
         let m = j + (k - j) / 2 / evlen * evlen;
         let x = read_event_at(m, &mut file).await?.1.ns;
-        info!("event at m: {}  ts: {}", m, x);
         if x < beg {
             j = m;
         } else {

disk/src/raw/conn.rs

@@ -133,7 +133,6 @@ async fn raw_conn_handler_inner_try(
         Err(e) => return Err((e, netout))?,
     };
     debug!("found config entry {:?}", entry);
-
     let shape = match &entry.shape {
         Some(lens) => {
             if lens.len() == 1 {
@@ -147,29 +146,24 @@ async fn raw_conn_handler_inner_try(
         }
         None => Shape::Scalar,
     };
-    let query = netpod::AggQuerySingleChannel {
+    let channel_config = netpod::ChannelConfig {
-        channel_config: netpod::ChannelConfig {
+        channel: evq.channel.clone(),
-            channel: evq.channel.clone(),
+        keyspace: entry.ks as u8,
-            keyspace: entry.ks as u8,
+        time_bin_size: entry.bs,
-            time_bin_size: entry.bs,
+        shape: shape,
-            shape: shape,
+        scalar_type: entry.scalar_type.clone(),
-            scalar_type: entry.scalar_type.clone(),
+        big_endian: entry.is_big_endian,
-            big_endian: entry.is_big_endian,
+        array: entry.is_array,
-            array: entry.is_array,
+        compression: entry.is_compressed,
-            compression: entry.is_compressed,
-        },
-        // TODO use a NanoRange and search for matching files
-        timebin: 0,
-        tb_file_count: 1,
-        // TODO use the requested buffer size
-        buffer_size: 1024 * 4,
     };
+    // TODO use a requested buffer size
+    let buffer_size = 1024 * 4;
     let event_chunker_conf = EventChunkerConf::new(ByteSize::kb(1024));
     let mut s1 = EventBlobsComplete::new(
         range.clone(),
-        query.channel_config.clone(),
+        channel_config.clone(),
         node_config.node.clone(),
-        query.buffer_size as usize,
+        buffer_size,
         event_chunker_conf,
     )
     .into_dim_1_f32_stream()

netpod/src/lib.rs

@@ -248,9 +248,9 @@ pub mod timeunits {
     pub const DAY: u64 = HOUR * 24;
 }
 
-const BIN_T_LEN_OPTIONS: [u64; 3] = [SEC, MIN * 10, HOUR * 2];
+const BIN_T_LEN_OPTIONS: [u64; 4] = [SEC, MIN * 10, HOUR * 2, DAY];
 
-const PATCH_T_LEN_OPTIONS: [u64; 3] = [MIN * 20, HOUR * 12, DAY * 16];
+const PATCH_T_LEN_OPTIONS: [u64; 4] = [MIN * 10, HOUR * 2, DAY * 4, DAY * 32];
 
 const BIN_THRESHOLDS: [u64; 31] = [
     2,