TCP: Fixes - allow for live monitoring of ACK progress
This commit is contained in:
@@ -268,6 +268,313 @@ TEST_CASE("TCPImageCommTest_DataFatalAck_PropagatesToPusher", "[TCP]") {
|
||||
sender.join();
|
||||
for (auto &t : receivers) t.join();
|
||||
|
||||
for (auto &p : puller)
|
||||
p->Disconnect();
|
||||
}
|
||||
|
||||
TEST_CASE("TCPImageCommTest_GetAckProgress_Correct", "[TCP]") {
|
||||
const size_t nframes = 64;
|
||||
const int64_t npullers = 2;
|
||||
const int64_t images_per_file = 8;
|
||||
|
||||
DiffractionExperiment x(DetJF(1));
|
||||
x.Raw();
|
||||
x.PedestalG0Frames(0).NumTriggers(1).UseInternalPacketGenerator(false).IncidentEnergy_keV(12.4)
|
||||
.ImagesPerTrigger(nframes).Compression(CompressionAlgorithm::NO_COMPRESSION);
|
||||
|
||||
std::mt19937 g1(123);
|
||||
std::uniform_int_distribution<uint16_t> dist;
|
||||
std::vector<uint16_t> image1(x.GetPixelsNum() * nframes);
|
||||
for (auto &i : image1) i = dist(g1);
|
||||
|
||||
std::vector<std::string> addr{
|
||||
"tcp://127.0.0.1:19021",
|
||||
"tcp://127.0.0.1:19022"
|
||||
};
|
||||
|
||||
std::vector<std::unique_ptr<TCPImagePuller>> puller;
|
||||
for (int i = 0; i < npullers; i++) {
|
||||
puller.push_back(std::make_unique<TCPImagePuller>(addr[i], 64 * 1024 * 1024));
|
||||
}
|
||||
|
||||
TCPStreamPusher pusher(
|
||||
addr,
|
||||
64 * 1024 * 1024,
|
||||
128 * 1024,
|
||||
8192
|
||||
);
|
||||
|
||||
std::thread sender([&] {
|
||||
std::vector<uint8_t> serialization_buffer(16 * 1024 * 1024);
|
||||
CBORStream2Serializer serializer(serialization_buffer.data(), serialization_buffer.size());
|
||||
|
||||
StartMessage start{
|
||||
.images_per_file = images_per_file,
|
||||
.write_master_file = true
|
||||
};
|
||||
EndMessage end{};
|
||||
|
||||
pusher.StartDataCollection(start);
|
||||
|
||||
for (int64_t i = 0; i < static_cast<int64_t>(nframes); i++) {
|
||||
DataMessage data_message;
|
||||
data_message.number = i;
|
||||
data_message.image = CompressedImage(image1.data() + i * x.GetPixelsNum(),
|
||||
x.GetPixelsNum() * sizeof(uint16_t),
|
||||
x.GetXPixelsNum(),
|
||||
x.GetYPixelsNum(),
|
||||
x.GetImageMode(),
|
||||
x.GetCompressionAlgorithm());
|
||||
serializer.SerializeImage(data_message);
|
||||
REQUIRE(pusher.SendImage(serialization_buffer.data(), serializer.GetBufferSize(), i));
|
||||
}
|
||||
|
||||
REQUIRE(pusher.EndDataCollection(end));
|
||||
|
||||
const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5);
|
||||
while (std::chrono::steady_clock::now() < deadline) {
|
||||
auto progress = pusher.GetAckProgress();
|
||||
REQUIRE(progress.has_value());
|
||||
if (progress->data_acked_total == nframes)
|
||||
break;
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(10));
|
||||
}
|
||||
|
||||
auto progress = pusher.GetAckProgress();
|
||||
REQUIRE(progress.has_value());
|
||||
REQUIRE(progress->data_sent == nframes);
|
||||
REQUIRE(progress->data_acked_ok == nframes / 2);
|
||||
REQUIRE(progress->data_acked_bad == nframes / 2);
|
||||
REQUIRE(progress->data_acked_total == nframes);
|
||||
REQUIRE(progress->data_ack_pending == 0);
|
||||
});
|
||||
|
||||
std::vector<std::thread> receivers;
|
||||
receivers.reserve(npullers);
|
||||
|
||||
for (int w = 0; w < npullers; w++) {
|
||||
receivers.emplace_back([&, w] {
|
||||
bool seen_end = false;
|
||||
uint64_t processed = 0;
|
||||
|
||||
while (!seen_end) {
|
||||
auto out = puller[w]->PollImage(std::chrono::seconds(10));
|
||||
REQUIRE(out.has_value());
|
||||
REQUIRE(out->cbor != nullptr);
|
||||
REQUIRE(out->tcp_msg != nullptr);
|
||||
|
||||
const auto &h = out->tcp_msg->header;
|
||||
|
||||
if (out->cbor->start_message) {
|
||||
PullerAckMessage ack;
|
||||
ack.ack_for = TCPFrameType::START;
|
||||
ack.ok = true;
|
||||
ack.fatal = false;
|
||||
ack.run_number = h.run_number;
|
||||
ack.socket_number = h.socket_number;
|
||||
ack.image_number = 0;
|
||||
ack.processed_images = 0;
|
||||
ack.error_code = TCPAckCode::None;
|
||||
REQUIRE(puller[w]->SendAck(ack));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (out->cbor->data_message) {
|
||||
auto number = out->cbor->data_message->number;
|
||||
|
||||
processed++;
|
||||
PullerAckMessage ack;
|
||||
ack.ack_for = TCPFrameType::DATA;
|
||||
ack.ok = (number % 2 == 0) ? true : false;
|
||||
ack.fatal = false;
|
||||
ack.run_number = h.run_number;
|
||||
ack.socket_number = h.socket_number;
|
||||
ack.image_number = number;
|
||||
ack.processed_images = processed;
|
||||
ack.error_code = TCPAckCode::None;
|
||||
REQUIRE(puller[w]->SendAck(ack));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (out->cbor->end_message) {
|
||||
PullerAckMessage ack;
|
||||
ack.ack_for = TCPFrameType::END;
|
||||
ack.ok = true;
|
||||
ack.fatal = false;
|
||||
ack.run_number = h.run_number;
|
||||
ack.socket_number = h.socket_number;
|
||||
ack.image_number = 0;
|
||||
ack.processed_images = processed;
|
||||
ack.error_code = TCPAckCode::None;
|
||||
REQUIRE(puller[w]->SendAck(ack));
|
||||
seen_end = true;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
sender.join();
|
||||
for (auto &t : receivers) t.join();
|
||||
|
||||
for (auto &p : puller)
|
||||
p->Disconnect();
|
||||
}
|
||||
|
||||
TEST_CASE("TCPImageCommTest_GetAckProgress_InFlightPending", "[TCP]") {
|
||||
const size_t nframes = 128;
|
||||
const int64_t npullers = 2;
|
||||
const int64_t images_per_file = 8;
|
||||
|
||||
DiffractionExperiment x(DetJF(1));
|
||||
x.Raw();
|
||||
x.PedestalG0Frames(0).NumTriggers(1).UseInternalPacketGenerator(false).IncidentEnergy_keV(12.4)
|
||||
.ImagesPerTrigger(nframes).Compression(CompressionAlgorithm::NO_COMPRESSION);
|
||||
|
||||
std::mt19937 g1(321);
|
||||
std::uniform_int_distribution<uint16_t> dist;
|
||||
std::vector<uint16_t> image1(x.GetPixelsNum() * nframes);
|
||||
for (auto &i : image1) i = dist(g1);
|
||||
|
||||
std::vector<std::string> addr{
|
||||
"tcp://127.0.0.1:19031",
|
||||
"tcp://127.0.0.1:19032"
|
||||
};
|
||||
|
||||
std::vector<std::unique_ptr<TCPImagePuller>> puller;
|
||||
for (int i = 0; i < npullers; i++) {
|
||||
puller.push_back(std::make_unique<TCPImagePuller>(addr[i], 64 * 1024 * 1024));
|
||||
}
|
||||
|
||||
TCPStreamPusher pusher(
|
||||
addr,
|
||||
64 * 1024 * 1024,
|
||||
128 * 1024,
|
||||
8192
|
||||
);
|
||||
|
||||
std::atomic<bool> observed_pending{false};
|
||||
|
||||
std::thread sender([&] {
|
||||
std::vector<uint8_t> serialization_buffer(16 * 1024 * 1024);
|
||||
CBORStream2Serializer serializer(serialization_buffer.data(), serialization_buffer.size());
|
||||
|
||||
StartMessage start{
|
||||
.images_per_file = images_per_file,
|
||||
.write_master_file = true
|
||||
};
|
||||
EndMessage end{};
|
||||
|
||||
pusher.StartDataCollection(start);
|
||||
|
||||
for (int64_t i = 0; i < static_cast<int64_t>(nframes); i++) {
|
||||
DataMessage data_message;
|
||||
data_message.number = i;
|
||||
data_message.image = CompressedImage(image1.data() + i * x.GetPixelsNum(),
|
||||
x.GetPixelsNum() * sizeof(uint16_t),
|
||||
x.GetXPixelsNum(),
|
||||
x.GetYPixelsNum(),
|
||||
x.GetImageMode(),
|
||||
x.GetCompressionAlgorithm());
|
||||
serializer.SerializeImage(data_message);
|
||||
REQUIRE(pusher.SendImage(serialization_buffer.data(), serializer.GetBufferSize(), i));
|
||||
|
||||
if (i >= 16 && !observed_pending.load()) {
|
||||
auto progress = pusher.GetAckProgress();
|
||||
REQUIRE(progress.has_value());
|
||||
if (progress->data_sent > progress->data_acked_total && progress->data_ack_pending > 0) {
|
||||
observed_pending = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
REQUIRE(pusher.EndDataCollection(end));
|
||||
REQUIRE(observed_pending.load());
|
||||
|
||||
const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5);
|
||||
while (std::chrono::steady_clock::now() < deadline) {
|
||||
auto progress = pusher.GetAckProgress();
|
||||
REQUIRE(progress.has_value());
|
||||
if (progress->data_acked_total == nframes)
|
||||
break;
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(10));
|
||||
}
|
||||
|
||||
auto progress = pusher.GetAckProgress();
|
||||
REQUIRE(progress.has_value());
|
||||
REQUIRE(progress->data_sent == nframes);
|
||||
REQUIRE(progress->data_acked_ok == nframes);
|
||||
REQUIRE(progress->data_acked_bad == 0);
|
||||
REQUIRE(progress->data_acked_total == nframes);
|
||||
REQUIRE(progress->data_ack_pending == 0);
|
||||
});
|
||||
|
||||
std::vector<std::thread> receivers;
|
||||
receivers.reserve(npullers);
|
||||
|
||||
for (int w = 0; w < npullers; w++) {
|
||||
receivers.emplace_back([&, w] {
|
||||
bool seen_end = false;
|
||||
uint64_t processed = 0;
|
||||
|
||||
while (!seen_end) {
|
||||
auto out = puller[w]->PollImage(std::chrono::seconds(10));
|
||||
REQUIRE(out.has_value());
|
||||
REQUIRE(out->cbor != nullptr);
|
||||
REQUIRE(out->tcp_msg != nullptr);
|
||||
|
||||
const auto &h = out->tcp_msg->header;
|
||||
|
||||
if (out->cbor->start_message) {
|
||||
PullerAckMessage ack;
|
||||
ack.ack_for = TCPFrameType::START;
|
||||
ack.ok = true;
|
||||
ack.fatal = false;
|
||||
ack.run_number = h.run_number;
|
||||
ack.socket_number = h.socket_number;
|
||||
ack.image_number = 0;
|
||||
ack.processed_images = 0;
|
||||
ack.error_code = TCPAckCode::None;
|
||||
REQUIRE(puller[w]->SendAck(ack));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (out->cbor->data_message) {
|
||||
processed++;
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(3));
|
||||
|
||||
PullerAckMessage ack;
|
||||
ack.ack_for = TCPFrameType::DATA;
|
||||
ack.ok = true;
|
||||
ack.fatal = false;
|
||||
ack.run_number = h.run_number;
|
||||
ack.socket_number = h.socket_number;
|
||||
ack.image_number = static_cast<uint64_t>(out->cbor->data_message->number);
|
||||
ack.processed_images = processed;
|
||||
ack.error_code = TCPAckCode::None;
|
||||
REQUIRE(puller[w]->SendAck(ack));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (out->cbor->end_message) {
|
||||
PullerAckMessage ack;
|
||||
ack.ack_for = TCPFrameType::END;
|
||||
ack.ok = true;
|
||||
ack.fatal = false;
|
||||
ack.run_number = h.run_number;
|
||||
ack.socket_number = h.socket_number;
|
||||
ack.image_number = 0;
|
||||
ack.processed_images = processed;
|
||||
ack.error_code = TCPAckCode::None;
|
||||
REQUIRE(puller[w]->SendAck(ack));
|
||||
seen_end = true;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
sender.join();
|
||||
for (auto &t : receivers) t.join();
|
||||
|
||||
for (auto &p : puller)
|
||||
p->Disconnect();
|
||||
}
|
||||
Reference in New Issue
Block a user