Check for repeated value already in RtWriter

serieswriter/src/binwriter.rs

@@ -485,7 +485,7 @@ impl BinWriter {
                             series: series.id() as i64,
                             pbp: pbp_ix.db_ix() as i16,
                             msp: msp as i32,
-                            rt: rt.index_db_i32() as i16,
+                            rt: rt.to_index_db_i32() as i16,
                             lsp: lsp as i32,
                             binlen: pbp.bin_len().ms() as i32,
                         };
@@ -513,7 +513,7 @@ impl BinWriter {
                             series: series.id() as i64,
                             pbp: pbp_ix.db_ix() as i16,
                             msp: msp as i32,
-                            rt: rt.index_db_i32() as i16,
+                            rt: rt.to_index_db_i32() as i16,
                             lsp: lsp as i32,
                             binlen: pbp.bin_len().ms() as i32,
                         };

serieswriter/src/ratelimitwriter.rs

@@ -1,9 +1,9 @@
 use crate::writer::EmittableType;
 use crate::writer::SeriesWriter;
 use core::fmt;
-use netpod::log::*;
 use netpod::DtNano;
 use netpod::TsNano;
+use netpod::log;
 use scywr::iteminsertqueue::QueryItem;
 use series::SeriesId;
 use std::collections::VecDeque;
@@ -11,7 +11,9 @@ use std::marker::PhantomData;
 use std::time::Duration;
 use std::time::Instant;
 
-macro_rules! trace_rt_decision { ($det:expr, $($arg:tt)*) => { if $det { trace!($($arg)*); } }; }
+macro_rules! debug { ($($arg:expr),*) => ( if true { log::debug!($($arg),*); } ); }
+macro_rules! trace { ($($arg:expr),*) => ( if true { log::trace!($($arg),*); } ); }
+macro_rules! trace_rt_decision { ($det:expr, $($arg:expr),*) => ( if $det { log::trace!($($arg),*); } ); }
 
 autoerr::create_error_v1!(
     name(Error, "RateLimitWriter"),
@@ -92,10 +94,13 @@ where
         if false {
             trace_rt_decision!(
                 det,
-                "{dbgname}  {sid}  min_quiet {min_quiet:?}  ts1 {ts1:?}  ts2 {ts2:?}  item {item:?}",
-                ts1 = ts.ms(),
-                ts2 = tsl.ms(),
-                item = item,
+                "{}  {}  min_quiet {:?}  ts1 {:?}  ts2 {:?}  item {:?}",
+                dbgname,
+                sid,
+                min_quiet,
+                ts.ms(),
+                tsl.ms(),
+                item
             );
         }
         let do_write = {
@@ -105,7 +110,11 @@ where
             } else if ts < tsl {
                 trace_rt_decision!(
                     det,
-                    "{dbgname}  {sid}  ignore, because ts_local  rewind  {ts:?}  {tsl:?}",
+                    "{}  {}  ignore, because ts_local  rewind  {:?}  {:?}",
+                    dbgname,
+                    sid,
+                    ts,
+                    tsl
                 );
                 false
             } else if !self.is_polled && ts.ms() < tsl.ms() + min_quiet {
@@ -120,20 +129,8 @@ where
             } else if ts < tsl.add_dt_nano(DtNano::from_ms(5)) {
                 trace_rt_decision!(det, "{dbgname}  {sid}  ignore, because store rate cap");
                 false
-            } else if self
-                .last_insert_val
-                .as_ref()
-                .map(|k| !item.has_change(k))
-                .unwrap_or(false)
-            {
-                trace_rt_decision!(det, "{dbgname}  {sid}  ignore, because value did not change");
-                false
             } else {
                 trace_rt_decision!(det, "{dbgname}  {sid}  accept");
-                if true {
-                    self.last_insert_val = Some(item.clone());
-                }
-                self.last_insert_ts = ts.clone();
                 true
             }
         };

serieswriter/src/rtwriter.rs

@@ -12,6 +12,7 @@ use std::time::Duration;
 use std::time::Instant;
 
 macro_rules! trace_emit { ($det:expr, $($arg:expr),*) => ( if $det { log::trace!($($arg),*); } ); }
+macro_rules! trace_rt_decision { ($det:expr, $($arg:expr),*) => ( if $det { log::trace!($($arg),*); } ); }
 
 autoerr::create_error_v1!(
     name(Error, "SerieswriterRtwriter"),
@@ -48,6 +49,10 @@ impl WriteRes {
     pub fn nstatus(&self) -> u8 {
         self.st.status + self.mt.status + self.lt.status
     }
+
+    pub fn accept_any(&self) -> bool {
+        self.lt.accept || self.mt.accept || self.st.accept
+    }
 }
 
 #[derive(Debug)]
@@ -81,6 +86,8 @@ where
     min_quiets: MinQuiets,
     do_trace_detail: bool,
     do_st_rf1: bool,
+    last_insert_ts: TsNano,
+    last_insert_val: Option<ET>,
 }
 
 impl<ET> RtWriter<ET>
@@ -119,6 +126,8 @@ where
             min_quiets,
             do_trace_detail: netpod::TRACE_SERIES_ID.contains(&series.id()),
             do_st_rf1,
+            last_insert_ts: TsNano::from_ns(0),
+            last_insert_val: None,
         };
         Ok(ret)
     }
@@ -152,9 +161,15 @@ where
         // Optimize for the common case that we only write into one of the stores.
         // Make the decision first, based on ref, then clone only as required.
         let res_lt;
-        let mut res_mt = WriteRtRes::default();
-        let mut res_st = WriteRtRes::default();
+        let res_mt;
+        let res_st;
+        if self
+            .last_insert_val
+            .as_ref()
+            .map(|k| item.has_change(k))
+            .unwrap_or(true)
         {
+            // TODO filter duplicate values already here
             res_lt = Self::write_inner(&mut self.state_lt, item.clone(), ts_net, tsev, &mut iqdqs.lt_rf3_qu)?;
             if !res_lt.accept {
                 res_mt = Self::write_inner(&mut self.state_mt, item.clone(), ts_net, tsev, &mut iqdqs.mt_rf3_qu)?;
@@ -166,14 +181,28 @@ where
                         res_st =
                             Self::write_inner(&mut self.state_st, item.clone(), ts_net, tsev, &mut iqdqs.st_rf3_qu)?;
                     }
+                } else {
+                    res_st = WriteRtRes::default();
                 }
+            } else {
+                res_mt = WriteRtRes::default();
+                res_st = WriteRtRes::default();
             }
+        } else {
+            trace_rt_decision!(det, "{}  ignore, because value did not change", self.series);
+            res_lt = WriteRtRes::default();
+            res_mt = WriteRtRes::default();
+            res_st = WriteRtRes::default();
         }
         let ret = WriteRes {
             st: res_st,
             mt: res_mt,
             lt: res_lt,
         };
+        if ret.accept_any() {
+            self.last_insert_ts = tsev.clone();
+            self.last_insert_val = Some(item.clone());
+        }
         Ok(ret)
     }