Cleanup and load test

2023-07-21 23:06:15 +02:00
parent 9314c58a9b
commit 7c26b72537
16 changed files with 171 additions and 118 deletions
@@ -32,6 +32,7 @@ use netpod::ChannelSearchQuery;
 use netpod::ChannelSearchResult;
 use netpod::ChannelTypeConfigGen;
 use netpod::DiskIoTune;
+use netpod::FromUrl;
 use netpod::NodeConfigCached;
 use netpod::ProxyConfig;
 use netpod::ReqCtxArc;
@@ -544,6 +545,7 @@ impl DataApiPython3DataStream {
        reqctx: ReqCtxArc,
        node_config: NodeConfigCached,
    ) -> Self {
+        debug!("DataApiPython3DataStream::new  settings {settings:?}  disk_io_tune {disk_io_tune:?}");
        Self {
            range,
            channels: channels.into_iter().collect(),
@@ -910,17 +912,33 @@ impl Api1EventsBinaryHandler {
        } else {
            tracing::Span::none()
        };
+        let url = {
+            let s1 = format!("dummy:{}", head.uri);
+            Url::parse(&s1)
+                .map_err(Error::from)
+                .map_err(|e| e.add_public_msg(format!("Can not parse query url")))?
+        };
+        let disk_tune = DiskIoTune::from_url(&url)?;
        let reqidspan = tracing::info_span!("api1query", reqid = reqctx.reqid());
-        self.handle_for_query(qu, accept, &reqctx, span.clone(), reqidspan.clone(), node_config)
-            .instrument(span)
-            .instrument(reqidspan)
-            .await
+        self.handle_for_query(
+            qu,
+            accept,
+            disk_tune,
+            &reqctx,
+            span.clone(),
+            reqidspan.clone(),
+            node_config,
+        )
+        .instrument(span)
+        .instrument(reqidspan)
+        .await
    }

    pub async fn handle_for_query(
        &self,
        qu: Api1Query,
        accept: String,
+        disk_io_tune: DiskIoTune,
        reqctx: &ReqCtxArc,
        span: tracing::Span,
        reqidspan: tracing::Span,
@@ -981,7 +999,7 @@ impl Api1EventsBinaryHandler {
                chans,
                // TODO carry those settings from the query again
                settings,
-                DiskIoTune::default(),
+                disk_io_tune,
                qu.decompress()
                    .unwrap_or_else(|| ncc.node_config.cluster.decompress_default()),
                qu.events_max().unwrap_or(u64::MAX),
@@ -27,20 +27,17 @@ impl FromUrl for DownloadQuery {
    fn from_pairs(pairs: &std::collections::BTreeMap<String, String>) -> Result<Self, err::Error> {
        let read_sys = pairs
            .get("ReadSys")
-            .map(|x| x as &str)
-            .unwrap_or("TokioAsyncRead")
-            .into();
+            .map(|x| x.as_str().into())
+            .unwrap_or_else(|| netpod::ReadSys::default());
        let read_buffer_len = pairs
            .get("ReadBufferLen")
-            .map(|x| x as &str)
-            .unwrap_or("xx")
-            .parse()
+            .map(|x| x.parse().map_or(None, Some))
+            .unwrap_or(None)
            .unwrap_or(1024 * 4);
        let read_queue_len = pairs
            .get("ReadQueueLen")
-            .map(|x| x as &str)
-            .unwrap_or("xx")
-            .parse()
+            .map(|x| x.parse().map_or(None, Some))
+            .unwrap_or(None)
            .unwrap_or(8);
        let disk_io_tune = DiskIoTune {
            read_sys,
@@ -67,10 +64,10 @@ impl DownloadHandler {
        }
    }

-    pub async fn get(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
+    pub async fn get(&self, req: Request<Body>, ncc: &NodeConfigCached) -> Result<Response<Body>, Error> {
        let (head, _body) = req.into_parts();
        let p2 = &head.uri.path()[Self::path_prefix().len()..];
-        let base = match &node_config.node.sf_databuffer {
+        let base = match &ncc.node.sf_databuffer {
            Some(k) => k.data_base_path.clone(),
            None => "/UNDEFINED".into(),
        };
@@ -251,6 +251,7 @@ impl Stream for FileStream {
        let mut rb = ReadBuf::new(&mut buf);
        let f = &mut self.file;
        pin_mut!(f);
+        trace!("poll_read for proxy distri");
        match f.poll_read(cx, &mut rb) {
            Ready(k) => match k {
                Ok(_) => {