daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP

Browse Source
This commit is contained in:
Dominik Werder
2023-12-05 15:44:11 +01:00
parent a5d3350747
commit 1b3e9ebd2a
35 changed files with 1180 additions and 948 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
crates/httpclient/Cargo.toml
View File

@@ -8,12 +8,15 @@ edition = "2021"
futures-util = "0.3.25"
serde = { version = "1.0.147", features = ["derive"] }
serde_json = "1.0.89"
http = "0.2.8"
url = "2.3.1"
tokio = { version = "1.22.0", features = ["rt-multi-thread", "io-util", "net", "time", "sync", "fs"] }
tracing = "0.1.37"
hyper = { version = "0.14.23", features = ["http1", "http2", "client", "server", "tcp", "stream"] }
http = "1.0.0"
http-body = "1.0.0"
http-body-util = "0.1.0"
hyper = { version = "1.0.1", features = ["http1", "http2", "client", "server"] }
hyper-tls = { version = "0.5.0" }
hyper-util = { version = "0.1.1", features = ["full"] }
bytes = "1.3.0"
async-channel = "1.8.0"
err = { path = "../err" }

298
crates/httpclient/src/httpclient.rs
View File

@@ -1,44 +1,95 @@
pub use hyper_util;
pub use http_body_util;
pub use http_body_util::Full;
use bytes::Bytes;
use err::Error;
use bytes::BytesMut;
use err::PublicError;
use futures_util::pin_mut;
use http::header;
use http::Request;
use http::Response;
use http::StatusCode;
use hyper::body::HttpBody;
use hyper::Body;
use http_body_util::combinators::BoxBody;
use hyper::body::Body;
use hyper::client::conn::http2::SendRequest;
use hyper::Method;
use netpod::log::*;
use netpod::AppendToUrl;
use netpod::ChannelConfigQuery;
use netpod::ChannelConfigResponse;
use netpod::NodeConfigCached;
use netpod::APP_JSON;
use std::fmt;
use std::pin::Pin;
use std::task::Context;
use std::task::Poll;
use tokio::io;
use tokio::io::AsyncRead;
use tokio::io::ReadBuf;
use tokio::net::TcpStream;
use url::Url;
pub trait ErrConv<T> {
fn ec(self) -> Result<T, ::err::Error>;
pub type BodyBox = BoxBody<Bytes, BodyError>;
pub type RespBox = Response<BodyBox>;
#[derive(Debug)]
pub enum BodyError {
Bad,
}
pub trait Convable: ToString {}
impl<T, E: Convable> ErrConv<T> for Result<T, E> {
fn ec(self) -> Result<T, ::err::Error> {
match self {
Ok(x) => Ok(x),
Err(e) => Err(::err::Error::from_string(e.to_string())),
}
impl fmt::Display for BodyError {
fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
fmt.write_str("Bad")
}
}
impl Convable for http::Error {}
impl Convable for hyper::Error {}
impl std::error::Error for BodyError {}
impl From<std::convert::Infallible> for BodyError {
fn from(_value: std::convert::Infallible) -> Self {
BodyError::Bad
}
}
#[derive(Debug)]
pub enum Error {
BadUrl,
Connection,
IO,
Http,
}
impl From<std::io::Error> for Error {
fn from(value: std::io::Error) -> Self {
Self::IO
}
}
impl From<http::Error> for Error {
fn from(value: http::Error) -> Self {
Self::Http
}
}
impl From<hyper::Error> for Error {
fn from(value: hyper::Error) -> Self {
Self::Http
}
}
impl fmt::Display for Error {
fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
write!(fmt, "{self:?}")
}
}
impl err::ToErr for Error {
fn to_err(self) -> err::Error {
err::Error::with_msg_no_trace(format!("self"))
}
}
pub struct HttpResponse {
pub head: http::response::Parts,
@@ -49,143 +100,130 @@ pub async fn http_get(url: Url, accept: &str) -> Result<HttpResponse, Error> {
let req = Request::builder()
.method(http::Method::GET)
.uri(url.to_string())
.header(header::HOST, url.host_str().ok_or_else(|| Error::BadUrl)?)
.header(header::ACCEPT, accept)
.body(Body::empty())
.ec()?;
let client = hyper::Client::new();
let res = client.request(req).await.ec()?;
let (head, body) = res.into_parts();
.body(Full::new(Bytes::new()))?;
let mut send_req = connect_client(req.uri()).await?;
let res = send_req.send_request(req).await?;
let (head, mut body) = res.into_parts();
debug!("http_get head {head:?}");
let body = hyper::body::to_bytes(body).await.ec()?;
let ret = HttpResponse { head, body };
use bytes::BufMut;
use http_body_util::BodyExt;
let mut buf = BytesMut::new();
while let Some(x) = body.frame().await {
match x {
Ok(mut x) => {
if let Some(x) = x.data_mut() {
buf.put(x);
}
}
Err(e) => return Err(e.into()),
}
}
let ret = HttpResponse {
head,
body: buf.freeze(),
};
Ok(ret)
}
pub async fn http_post(url: Url, accept: &str, body: String) -> Result<Bytes, Error> {
let body = Bytes::from(body.as_bytes().to_vec());
let req = Request::builder()
.method(http::Method::POST)
.uri(url.to_string())
.header(header::HOST, url.host_str().ok_or_else(|| Error::BadUrl)?)
.header(header::CONTENT_TYPE, APP_JSON)
.header(header::ACCEPT, accept)
.body(Body::from(body))
.ec()?;
let client = hyper::Client::new();
let res = client.request(req).await.ec()?;
.body(Full::new(body))?;
let mut send_req = connect_client(req.uri()).await?;
let res = send_req.send_request(req).await?;
if res.status() != StatusCode::OK {
error!("Server error {:?}", res);
let (head, body) = res.into_parts();
let buf = hyper::body::to_bytes(body).await.ec()?;
let (_head, body) = res.into_parts();
let buf = read_body_bytes(body).await?;
let s = String::from_utf8_lossy(&buf);
return Err(Error::with_msg(format!(
concat!(
"Server error {:?}\n",
"---------------------- message from http body:\n",
"{}\n",
"---------------------- end of http body",
),
head, s
)));
return Err(Error::Http);
}
let body = hyper::body::to_bytes(res.into_body()).await.ec()?;
Ok(body)
}
// TODO move to a better fitting module:
pub struct HttpBodyAsAsyncRead {
inp: Response<Body>,
left: Bytes,
rp: usize,
}
impl HttpBodyAsAsyncRead {
pub fn new(inp: Response<Body>) -> Self {
Self {
inp,
left: Bytes::new(),
rp: 0,
let (head, mut body) = res.into_parts();
debug!("http_get head {head:?}");
use bytes::BufMut;
use http_body_util::BodyExt;
let mut buf = BytesMut::new();
while let Some(x) = body.frame().await {
match x {
Ok(mut x) => {
if let Some(x) = x.data_mut() {
buf.put(x);
}
}
Err(e) => return Err(e.into()),
}
}
let buf = read_body_bytes(body).await?;
Ok(buf)
}
impl AsyncRead for HttpBodyAsAsyncRead {
fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context, buf: &mut ReadBuf) -> Poll<io::Result<()>> {
trace!("impl AsyncRead for HttpBodyAsAsyncRead");
use Poll::*;
if self.left.len() != 0 {
let n1 = buf.remaining();
let n2 = self.left.len() - self.rp;
if n2 <= n1 {
buf.put_slice(self.left[self.rp..].as_ref());
self.left = Bytes::new();
self.rp = 0;
Ready(Ok(()))
} else {
buf.put_slice(self.left[self.rp..(self.rp + n2)].as_ref());
self.rp += n2;
Ready(Ok(()))
pub async fn connect_client(uri: &http::Uri) -> Result<SendRequest<Full<Bytes>>, Error> {
let host = uri.host().ok_or_else(|| Error::BadUrl)?;
let port = uri.port_u16().ok_or_else(|| Error::BadUrl)?;
let stream = TcpStream::connect(format!("{host}:{port}")).await?;
let executor = hyper_util::rt::TokioExecutor::new();
let (send_req, conn) = hyper::client::conn::http2::Builder::new(executor)
.handshake(hyper_util::rt::TokioIo::new(stream))
.await?;
// TODO would need to take greater care of this task to catch connection-level errors.
tokio::spawn(conn);
Ok(send_req)
}
pub async fn read_body_bytes(mut body: hyper::body::Incoming) -> Result<Bytes, Error> {
use bytes::BufMut;
use http_body_util::BodyExt;
let mut buf = BytesMut::new();
while let Some(x) = body.frame().await {
match x {
Ok(mut x) => {
if let Some(x) = x.data_mut() {
buf.put(x);
}
}
} else {
let f = &mut self.inp;
pin_mut!(f);
match f.poll_data(cx) {
Ready(Some(Ok(k))) => {
let n1 = buf.remaining();
if k.len() <= n1 {
buf.put_slice(k.as_ref());
Ready(Ok(()))
Err(e) => return Err(e.into()),
}
}
Ok(buf.freeze())
}
pub struct IncomingStream {
inp: hyper::body::Incoming,
}
impl IncomingStream {
pub fn new(inp: hyper::body::Incoming) -> Self {
Self { inp }
}
}
impl futures_util::Stream for IncomingStream {
type Item = Result<Bytes, err::Error>;
fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
use Poll::*;
let j = &mut self.get_mut().inp;
let k = Pin::new(j);
match hyper::body::Body::poll_frame(k, cx) {
Ready(Some(x)) => match x {
Ok(x) => {
if let Ok(x) = x.into_data() {
Ready(Some(Ok(x)))
} else {
buf.put_slice(k[..n1].as_ref());
self.left = k;
self.rp = n1;
Ready(Ok(()))
Ready(Some(Ok(Bytes::new())))
}
}
Ready(Some(Err(e))) => Ready(Err(io::Error::new(
io::ErrorKind::Other,
Error::with_msg(format!("Received by HttpBodyAsAsyncRead: {:?}", e)),
))),
Ready(None) => Ready(Ok(())),
Pending => Pending,
}
}
}
}
pub async fn get_channel_config(
q: &ChannelConfigQuery,
node_config: &NodeConfigCached,
) -> Result<ChannelConfigResponse, Error> {
let mut url = Url::parse(&format!(
"http://{}:{}/api/4/channel/config",
node_config.node.host, node_config.node.port
))?;
q.append_to_url(&mut url);
let req = hyper::Request::builder()
.method(Method::GET)
.uri(url.as_str())
.body(Body::empty())
.map_err(Error::from_string)?;
let client = hyper::Client::new();
let res = client
.request(req)
.await
.map_err(|e| Error::with_msg(format!("get_channel_config request error: {e:?}")))?;
if res.status().is_success() {
let buf = hyper::body::to_bytes(res.into_body())
.await
.map_err(|e| Error::with_msg(format!("can not read response: {e:?}")))?;
let ret: ChannelConfigResponse = serde_json::from_slice(&buf)
.map_err(|e| Error::with_msg(format!("can not parse the channel config response json: {e:?}")))?;
Ok(ret)
} else {
let buf = hyper::body::to_bytes(res.into_body())
.await
.map_err(|e| Error::with_msg(format!("can not read response: {e:?}")))?;
match serde_json::from_slice::<PublicError>(&buf) {
Ok(e) => Err(e.into()),
Err(_) => Err(Error::with_msg(format!(
"can not parse the http error body: {:?}",
String::from_utf8_lossy(&buf)
))),
Err(e) => Ready(Some(Err(e.into()))),
},
Ready(None) => Ready(None),
Pending => Pending,
}
}
}