use crate::collect::collect; use crate::generators::GenerateI32; use crate::test::runfut; use chrono::DateTime; use chrono::Utc; use err::Error; use futures_util::stream; use futures_util::StreamExt; use items_0::on_sitemty_data; use items_0::streamitem::sitem_data; use items_0::streamitem::RangeCompletableItem; use items_0::streamitem::StreamItem; use items_0::timebin::TimeBinnable; use items_0::Empty; use items_2::binsdim0::BinsDim0; use items_2::channelevents::ChannelEvents; use items_2::channelevents::ConnStatus; use items_2::channelevents::ConnStatusEvent; use items_2::testgen::make_some_boxed_d0_f32; use netpod::range::evrange::NanoRange; use netpod::range::evrange::SeriesRange; use netpod::timeunits::MS; use netpod::timeunits::SEC; use netpod::BinnedRangeEnum; use std::collections::VecDeque; use std::time::Duration; use std::time::Instant; #[test] fn time_bin_00() { let fut = async { let range = nano_range_from_str("1970-01-01T00:00:00Z", "1970-01-01T00:00:08Z")?; let range = SeriesRange::TimeRange(range); let min_bin_count = 8; let binned_range = BinnedRangeEnum::covering_range(range, min_bin_count)?; let evs0 = make_some_boxed_d0_f32(10, SEC * 1, MS * 500, 0, 1846713782); let v0 = ChannelEvents::Events(evs0); let v2 = ChannelEvents::Status(Some(ConnStatusEvent::new(MS * 100, ConnStatus::Connect))); let v4 = ChannelEvents::Status(Some(ConnStatusEvent::new(MS * 6000, ConnStatus::Disconnect))); let stream0 = Box::pin(stream::iter(vec![ // sitem_data(v2), sitem_data(v0), sitem_data(v4), ])); let mut exps = { let mut d = VecDeque::new(); let mut bins = BinsDim0::empty(); bins.push(SEC * 0, SEC * 1, 0, 0.0, 0.0, 0.0); bins.push(SEC * 1, SEC * 2, 2, 0.0535830, 100.0589, 50.05624); bins.push(SEC * 2, SEC * 3, 2, 200.06143, 300.07645, 250.06894); bins.push(SEC * 3, SEC * 4, 2, 400.08554, 500.05222, 450.06888); bins.push(SEC * 4, SEC * 5, 2, 600.0025, 700.09094, 650.04675); d.push_back(bins); let mut bins = BinsDim0::empty(); bins.push(SEC * 5, SEC * 6, 2, 800.0619, 900.02844, 850.04517); d.push_back(bins); d }; let deadline = Instant::now() + Duration::from_millis(2000000); let mut binned_stream = crate::timebin::TimeBinnedStream::new(stream0, binned_range, true, deadline); while let Some(item) = binned_stream.next().await { //eprintln!("{item:?}"); match item { Ok(item) => match item { StreamItem::DataItem(item) => match item { RangeCompletableItem::Data(item) => { if let Some(item) = item.as_any_ref().downcast_ref::>() { let exp = exps.pop_front().unwrap(); if !item.equal_slack(&exp) { return Err(Error::with_msg_no_trace(format!("bad, content not equal"))); } } else { return Err(Error::with_msg_no_trace(format!("bad, got item with unexpected type"))); } } RangeCompletableItem::RangeComplete => {} }, StreamItem::Log(_) => {} StreamItem::Stats(_) => {} }, Err(e) => Err(e).unwrap(), } } Ok(()) }; runfut(fut).unwrap() } #[test] fn time_bin_01() { let fut = async { let range = nano_range_from_str("1970-01-01T00:00:00Z", "1970-01-01T00:00:08Z")?; let range = SeriesRange::TimeRange(range); let min_bin_count = 8; let binned_range = BinnedRangeEnum::covering_range(range, min_bin_count)?; let evs0 = make_some_boxed_d0_f32(10, SEC * 1, MS * 500, 0, 1846713782); let evs1 = make_some_boxed_d0_f32(10, SEC * 6, MS * 500, 0, 1846713781); let v0 = ChannelEvents::Events(evs0); let v1 = ChannelEvents::Events(evs1); let stream0 = stream::iter(vec![ // sitem_data(v0), sitem_data(v1), ]); let stream0 = stream0.then({ let mut i = 0; move |x| { let delay = if i == 1 { 2000 } else { 0 }; i += 1; let dur = Duration::from_millis(delay); async move { tokio::time::sleep(dur).await; x } } }); let stream0 = Box::pin(stream0); let deadline = Instant::now() + Duration::from_millis(200); let mut binned_stream = crate::timebin::TimeBinnedStream::new(stream0, binned_range, true, deadline); while let Some(item) = binned_stream.next().await { if true { eprintln!("{item:?}"); } match item { Ok(item) => match item { StreamItem::DataItem(item) => match item { RangeCompletableItem::Data(item) => { if let Some(_) = item.as_any_ref().downcast_ref::>() { } else { return Err(Error::with_msg_no_trace(format!("bad, got item with unexpected type"))); } } RangeCompletableItem::RangeComplete => {} }, StreamItem::Log(_) => {} StreamItem::Stats(_) => {} }, Err(e) => Err(e).unwrap(), } } // TODO assert that we get the bins which are sure to be ready. // TODO assert correct numbers. // TODO assert that we don't get bins which may be still changing. // TODO add similar test case with a RangeComplete event at different places before the timeout. Ok(()) }; runfut(fut).unwrap() } fn nano_range_from_str(beg_date: &str, end_date: &str) -> Result { let beg_date = beg_date.parse()?; let end_date = end_date.parse()?; let range = NanoRange::from_date_time(beg_date, end_date); Ok(range) } #[test] fn time_bin_02() -> Result<(), Error> { let fut = async { let do_time_weight = true; let deadline = Instant::now() + Duration::from_millis(4000); let range = nano_range_from_str("1970-01-01T00:20:04Z", "1970-01-01T00:21:10Z")?; let range = SeriesRange::TimeRange(range); let min_bin_count = 10; let binned_range = BinnedRangeEnum::covering_range(range.clone(), min_bin_count)?; eprintln!("binned_range: {:?}", binned_range); for i in 0.. { if let Some(r) = binned_range.range_at(i) { eprintln!("Series Range to cover: {r:?}"); } else { break; } } let event_range = binned_range.binned_range_time().full_range(); let series_range = SeriesRange::TimeRange(event_range); // TODO the test stream must be able to generate also one-before (on demand) and RangeComplete (by default). let stream = GenerateI32::new(0, 1, series_range); // TODO apply first some box dyn EventTransform which later is provided by TransformQuery. // Then the Merge will happen always by default for backends where this is needed. // TODO then apply the transform chain for the after-merged-stream. let stream = stream.map(|x| { let x = on_sitemty_data!(x, |x| Ok(StreamItem::DataItem(RangeCompletableItem::Data( Box::new(x) as Box )))); x }); let stream = Box::pin(stream); let mut binned_stream = crate::timebin::TimeBinnedStream::new(stream, binned_range.clone(), do_time_weight, deadline); // From there on it should no longer be neccessary to distinguish whether its still events or time bins. // Then, optionally collect for output type like json, or stream as batches. // TODO the timebinner should already provide batches to make this efficient. if false { while let Some(e) = binned_stream.next().await { eprintln!("see item {e:?}"); let x = on_sitemty_data!(e, |e| { // Ok(StreamItem::DataItem(RangeCompletableItem::Data(e))) }); } } else { let res = collect(binned_stream, deadline, 200, None, Some(binned_range)).await?; let d = res.to_json_result()?.to_json_bytes()?; let s = String::from_utf8_lossy(&d); eprintln!("{s}"); } Ok(()) }; runfut(fut) } // TODO add test case to observe RangeComplete after binning.