sf-daq/sf_daq_buffer
0
0
Fork 0
mirror of https://github.com/paulscherrerinstitute/sf_daq_buffer.git synced 2026-05-07 14:34:12 +02:00
Code Issues Packages Projects Releases Wiki Activity

Major project refactoring WIP

Browse Source
This commit is contained in:
babic_a
2020-05-20 11:45:34 +02:00
parent 4e7eefcdc6
commit b45b7d17fa
45 changed files with 48 additions and 3868 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core-buffer/test/CMakeLists.txt
+10 -10
View File
@@ -1,7 +1,7 @@
add_executable(core-buffer_tests main.cpp)
add_executable(core-buffer-tests main.cpp)
target_link_libraries(core-buffer_tests
core-buffer
target_link_libraries(core-buffer-tests
core-buffer-lib
external
hdf5
hdf5_hl
@@ -9,10 +9,10 @@ target_link_libraries(core-buffer_tests
zmq
gtest)
add_executable(perf-sf_writer perf/perf_WriterH5Writer.cpp)
target_link_libraries(perf-sf_writer
core-buffer
hdf5
hdf5_hl
hdf5_cpp
gtest)
#add_executable(perf-sf_writer perf/perf_WriterH5Writer.cpp)
#target_link_libraries(perf-sf_writer
# core-buffer
# hdf5
# hdf5_hl
# hdf5_cpp
# gtest)
core-buffer/test/main.cpp
-9
View File
@@ -1,15 +1,6 @@
#include "gtest/gtest.h"
#include "test_UdpReceiver.cpp"
#include "test_BufferBinaryWriter.cpp"
#include "test_buffer_utils.cpp"
#include "test_BufferH5Writer.cpp"
#include "test_ReplayH5Reader.cpp"
#include "test_WriterH5Writer.cpp"
#include "test_FastQueue.cpp"
#include "test_LiveRecvModule.cpp"
#include "test_BufferUdpReceiver.cpp"
#include "test_bitshuffle.cpp"
#include "test_WriterZmqReceiver.cpp"
using namespace std;
core-buffer/test/mock/udp.hpp
-16
View File
@@ -1,16 +0,0 @@
#ifndef MOCK_UDP_H
#define MOCK_UDP_H
const int MOCK_UDP_PORT(13000);
sockaddr_in get_server_address(uint16_t udp_port)
{
sockaddr_in server_address = {0};
server_address.sin_family = AF_INET;
server_address.sin_addr.s_addr = INADDR_ANY;
server_address.sin_port = htons(udp_port);
return server_address;
}
#endif
core-buffer/test/perf/perf_WriterH5Writer.cpp
-90
View File
@@ -1,90 +0,0 @@
#include <iostream>
#include "buffer_config.hpp"
#include "zmq.h"
#include <string>
#include <RingBuffer.hpp>
#include <thread>
#include <chrono>
#include "WriterH5Writer.hpp"
using namespace std;
using namespace core_buffer;
int main (int argc, char *argv[])
{
if (argc != 4) {
cout << endl;
cout << "Usage: sf_writer ";
cout << " [output_file] [start_pulse_id] [stop_pulse_id]";
cout << endl;
cout << "\toutput_file: Complete path to the output file." << endl;
cout << "\tstart_pulse_id: Start pulse_id of retrieval." << endl;
cout << "\tstop_pulse_id: Stop pulse_id of retrieval." << endl;
cout << endl;
exit(-1);
}
string output_file = string(argv[1]);
uint64_t start_pulse_id = (uint64_t) atoll(argv[2]);
uint64_t stop_pulse_id = (uint64_t) atoll(argv[3]);
size_t n_modules = 32;
size_t n_frames = stop_pulse_id - start_pulse_id;
WriterH5Writer writer(output_file, n_frames, n_modules);
// TODO: Remove stats trash.
int i_write = 0;
size_t total_ms = 0;
size_t max_ms = 0;
size_t min_ms = 10000; // 10 seconds should be a safe first value.
auto start_time = chrono::steady_clock::now();
auto metadata = make_shared<ImageMetadata>();
auto data = make_unique<char[]>(MODULE_N_BYTES*n_modules);
auto current_pulse_id = start_pulse_id;
while (current_pulse_id <= stop_pulse_id) {
writer.write(metadata.get(), data.get());
current_pulse_id++;
i_write++;
auto end_time = chrono::steady_clock::now();
// TODO: Some poor statistics.
auto ms_duration = chrono::duration_cast<chrono::milliseconds>(
end_time-start_time).count();
total_ms += ms_duration;
if (ms_duration > max_ms) {
max_ms = ms_duration;
}
if (ms_duration < min_ms) {
min_ms = ms_duration;
}
if (i_write==100) {
cout << "avg_write_ms " << total_ms / 100;
cout << " min_write_ms " << min_ms;
cout << " max_write_ms " << max_ms << endl;
i_write = 0;
total_ms = 0;
max_ms = 0;
min_ms = 0;
}
start_time = chrono::steady_clock::now();
}
writer.close_file();
return 0;
}
core-buffer/test/test_BufferBinaryWriter.cpp
-86
View File
@@ -1,86 +0,0 @@
#include <BufferBinaryWriter.hpp>
#include "BufferUtils.hpp"
#include <fcntl.h>
#include "gtest/gtest.h"
TEST(BinaryWriter, basic_interaction)
{
auto root_folder = ".";
auto device_name = "test_device";
uint64_t pulse_id = 5;
BufferBinaryWriter writer(device_name, root_folder);
BufferBinaryFormat frame_data;
frame_data.pulse_id = 1;
frame_data.frame_id = 2;
frame_data.daq_rec = 3;
frame_data.n_recv_packets = 4;
ASSERT_EQ(frame_data.FORMAT_MARKER, JF_FORMAT_START_BYTE);
writer.write(5, &frame_data);
auto output_filename =
BufferUtils::get_filename(root_folder, device_name, pulse_id);
auto read_fd = open(output_filename.c_str(), O_RDONLY);
ASSERT_NE(read_fd, -1);
auto file_frame_index = BufferUtils::get_file_frame_index(pulse_id);
BufferBinaryFormat read_data;
::lseek(read_fd, file_frame_index * sizeof(BufferBinaryFormat), SEEK_SET);
::read(read_fd, &read_data, sizeof(BufferBinaryFormat));
ASSERT_EQ(frame_data.FORMAT_MARKER, JF_FORMAT_START_BYTE);
ASSERT_EQ(frame_data.FORMAT_MARKER, read_data.FORMAT_MARKER);
ASSERT_EQ(frame_data.pulse_id, read_data.pulse_id);
ASSERT_EQ(frame_data.frame_id, read_data.frame_id);
ASSERT_EQ(frame_data.daq_rec, read_data.daq_rec);
ASSERT_EQ(frame_data.n_recv_packets, read_data.n_recv_packets);
}
TEST(BinaryWriter, test_format_marker)
{
auto root_folder = ".";
auto device_name = "test_device";
uint64_t pulse_id = 5;
BufferBinaryWriter writer(device_name, root_folder);
BufferBinaryFormat frame_data;
frame_data.pulse_id = 1;
frame_data.frame_id = 2;
frame_data.daq_rec = 3;
frame_data.n_recv_packets = 4;
writer.write(5, &frame_data);
auto output_filename =
BufferUtils::get_filename(root_folder, device_name, pulse_id);
auto read_fd = open(output_filename.c_str(), O_RDONLY);
ASSERT_NE(read_fd, -1);
auto file_frame_index = BufferUtils::get_file_frame_index(pulse_id);
BufferBinaryFormat read_data;
// One frame before should be empty.
::lseek(read_fd, (file_frame_index-1) * sizeof(BufferBinaryFormat), SEEK_SET);
::read(read_fd, &read_data, sizeof(BufferBinaryFormat));
ASSERT_NE(read_data.FORMAT_MARKER, JF_FORMAT_START_BYTE);
// One frame after should be empty as well.
::lseek(read_fd, (file_frame_index+1) * sizeof(BufferBinaryFormat), SEEK_SET);
::read(read_fd, &read_data, sizeof(BufferBinaryFormat));
ASSERT_NE(read_data.FORMAT_MARKER, JF_FORMAT_START_BYTE);
// But this frame should be here.
::lseek(read_fd, (file_frame_index) * sizeof(BufferBinaryFormat), SEEK_SET);
::read(read_fd, &read_data, sizeof(BufferBinaryFormat));
ASSERT_EQ(read_data.FORMAT_MARKER, JF_FORMAT_START_BYTE);
}
core-buffer/test/test_BufferH5Writer.cpp
-130
View File
@@ -1,130 +0,0 @@
#include "BufferH5Writer.hpp"
#include "gtest/gtest.h"
using namespace core_buffer;
TEST(BufferH5Writer, basic_interaction)
{
auto root_folder = ".";
auto device_name = "fast_device";
size_t pulse_id = 1;
auto buffer = make_unique<char[]>(JUNGFRAU_DATA_BYTES_PER_FRAME);
ModuleFrame metadata;
metadata.pulse_id = 1;
metadata.frame_index = 2;
metadata.daq_rec = 3;
metadata.n_received_packets = 128;
BufferH5Writer writer(root_folder, device_name);
writer.set_pulse_id(pulse_id);
writer.write(&metadata, buffer.get());
writer.close_file();
auto filename = BufferUtils::get_filename(
root_folder, device_name, pulse_id);
auto file_frame_index = BufferUtils::get_file_frame_index(pulse_id);
H5::H5File input_file(filename, H5F_ACC_RDONLY);
auto image_dataset = input_file.openDataSet("image");
size_t image_buffer_n_bytes = JUNGFRAU_DATA_BYTES_PER_FRAME * FILE_MOD;
auto image_buffer = make_unique<uint16_t[]>(image_buffer_n_bytes);
image_dataset.read(image_buffer.get(), H5::PredType::NATIVE_UINT16);
auto metadata_dataset = input_file.openDataSet("metadata");
auto metadata_buffer = make_unique<ModuleFrame[]>(FILE_MOD);
metadata_dataset.read(metadata_buffer.get(), H5::PredType::NATIVE_UINT64);
EXPECT_EQ(metadata_buffer[file_frame_index].pulse_id, 1);
EXPECT_EQ(metadata_buffer[file_frame_index].frame_index, 2);
EXPECT_EQ(metadata_buffer[file_frame_index].daq_rec, 3);
EXPECT_EQ(metadata_buffer[file_frame_index].n_received_packets, 128);
}
//
//TEST(BufferH5Writer, SWMR)
//{
// auto root_folder = ".";
// auto device_name = "fast_device";
// size_t pulse_id = 0;
//
// auto i_write_buffer = make_unique<char[]>(JUNGFRAU_DATA_BYTES_PER_FRAME);
// size_t image_buffer_n_bytes = JUNGFRAU_DATA_BYTES_PER_FRAME * FILE_MOD;
// auto i_read_buffer = make_unique<uint16_t[]>(image_buffer_n_bytes);
//
// ModuleFrame m_write_buffer = {1, 2, 3, 4, 5};
// auto m_read_buffer = make_unique<ModuleFrame[]>(FILE_MOD);
//
// for (size_t i=0; i<MODULE_N_PIXELS; i++) {
// uint16_t* image_ptr = (uint16_t*)(i_write_buffer.get());
// image_ptr[i] = 99;
// }
//
// BufferH5Writer writer(device_name, root_folder);
// // This creates the file.
// writer.set_pulse_id(0);
//
// auto filename = BufferUtils::get_filename(
// root_folder, device_name, pulse_id);
//
// H5::H5File input_file(filename, H5F_ACC_RDONLY | H5F_ACC_SWMR_READ);
// auto image_dataset = input_file.openDataSet("image");
// auto metadata_dataset = input_file.openDataSet("metadata");
//
// // The data was not yet written to file, so 0 is expected.
// image_dataset.read(i_read_buffer.get(), H5::PredType::NATIVE_UINT16);
// EXPECT_EQ((i_read_buffer.get())[0], 0);
// EXPECT_EQ((i_read_buffer.get())[512 * 1024], 0);
//
// // The data was not yet written to file, so 0 is expected.
// metadata_dataset.read(m_read_buffer.get(), H5::PredType::NATIVE_UINT64);
// EXPECT_EQ((m_read_buffer.get())[0].pulse_id, 0);
// EXPECT_EQ((m_read_buffer.get())[1].pulse_id, 0);
//
// // Flushing after every frame should ensure that the reader can see this.
// writer.set_pulse_id(0);
// writer.write(&m_write_buffer, i_write_buffer.get());
//
// image_dataset.read(i_read_buffer.get(), H5::PredType::NATIVE_UINT16);
// // Frame 0 was written, so we are expecting data in just the first frame.
// EXPECT_EQ((i_read_buffer.get())[0], 99);
// EXPECT_EQ((i_read_buffer.get())[512 * 1024], 0);
//
// // Frame 0 written, metadata for frame 0 expected.
// metadata_dataset.read(m_read_buffer.get(), H5::PredType::NATIVE_UINT64);
// EXPECT_EQ((m_read_buffer.get())[0].pulse_id, 1);
// EXPECT_EQ((m_read_buffer.get())[0].frame_index, 2);
// EXPECT_EQ((m_read_buffer.get())[0].daq_rec, 3);
// EXPECT_EQ((m_read_buffer.get())[0].n_received_packets, 4);
// EXPECT_EQ((m_read_buffer.get())[0].module_id, 5);
//
// EXPECT_EQ((m_read_buffer.get())[1].pulse_id, 0);
// EXPECT_EQ((m_read_buffer.get())[1].frame_index, 0);
// EXPECT_EQ((m_read_buffer.get())[1].daq_rec, 0);
// EXPECT_EQ((m_read_buffer.get())[1].n_received_packets, 0);
// EXPECT_EQ((m_read_buffer.get())[1].module_id, 0);
//
// writer.set_pulse_id(1);
// writer.write(&m_write_buffer, i_write_buffer.get());
//
// image_dataset.read(i_read_buffer.get(), H5::PredType::NATIVE_UINT16);
// // Both frame written, and we should access both.
// EXPECT_EQ((i_read_buffer.get())[0], 99);
// EXPECT_EQ((i_read_buffer.get())[512 * 1024], 99);
//
// // Both frame written, and we should access both.
// metadata_dataset.read(m_read_buffer.get(), H5::PredType::NATIVE_UINT64);
// EXPECT_EQ((m_read_buffer.get())[0].pulse_id, 1);
// EXPECT_EQ((m_read_buffer.get())[0].frame_index, 2);
// EXPECT_EQ((m_read_buffer.get())[0].daq_rec, 3);
// EXPECT_EQ((m_read_buffer.get())[0].n_received_packets, 4);
// EXPECT_EQ((m_read_buffer.get())[0].module_id, 5);
//
// EXPECT_EQ((m_read_buffer.get())[1].pulse_id, 1);
// EXPECT_EQ((m_read_buffer.get())[1].frame_index, 2);
// EXPECT_EQ((m_read_buffer.get())[1].daq_rec, 3);
// EXPECT_EQ((m_read_buffer.get())[1].n_received_packets, 4);
// EXPECT_EQ((m_read_buffer.get())[1].module_id, 5);
//}
core-buffer/test/test_BufferUdpReceiver.cpp
-239
View File
@@ -1,239 +0,0 @@
#include <netinet/in.h>
#include <jungfrau.hpp>
#include "gtest/gtest.h"
#include "BufferUdpReceiver.hpp"
#include "mock/udp.hpp"
#include <thread>
#include <chrono>
#include <future>
using namespace std;
TEST(BufferUdpReceiver, simple_recv)
{
auto n_packets = JUNGFRAU_N_PACKETS_PER_FRAME;
int source_id = 1234;
int n_frames = 5;
uint16_t udp_port = MOCK_UDP_PORT;
auto server_address = get_server_address(udp_port);
auto send_socket_fd = socket(AF_INET, SOCK_DGRAM, 0);
ASSERT_TRUE(send_socket_fd >= 0);
BufferUdpReceiver udp_receiver(udp_port, source_id);
auto handle = async(launch::async, [&](){
for (int i_frame=0; i_frame < n_frames; i_frame++){
for (size_t i_packet=0; i_packet<n_packets; i_packet++) {
jungfrau_packet send_udp_buffer;
send_udp_buffer.packetnum = i_packet;
send_udp_buffer.bunchid = i_frame + 1;
send_udp_buffer.framenum = i_frame + 1000;
send_udp_buffer.debug = i_frame + 10000;
::sendto(
send_socket_fd,
&send_udp_buffer,
JUNGFRAU_BYTES_PER_PACKET,
0,
(sockaddr*) &server_address,
sizeof(server_address));
}
}
});
handle.wait();
ModuleFrame metadata;
auto frame_buffer = make_unique<char[]>(JUNGFRAU_DATA_BYTES_PER_FRAME);
for (int i_frame=0; i_frame < n_frames; i_frame++) {
auto pulse_id = udp_receiver.get_frame_from_udp(
metadata, frame_buffer.get());
ASSERT_EQ(i_frame + 1, pulse_id);
ASSERT_EQ(metadata.frame_index, i_frame + 1000);
ASSERT_EQ(metadata.daq_rec, i_frame + 10000);
// -1 because we skipped a packet.
ASSERT_EQ(metadata.n_received_packets, n_packets);
ASSERT_EQ(metadata.module_id, source_id);
}
::close(send_socket_fd);
}
TEST(BufferUdpReceiver, missing_middle_packet)
{
auto n_packets = JUNGFRAU_N_PACKETS_PER_FRAME;
int source_id = 1234;
int n_frames = 3;
uint16_t udp_port = MOCK_UDP_PORT;
auto server_address = get_server_address(udp_port);
auto send_socket_fd = socket(AF_INET, SOCK_DGRAM, 0);
ASSERT_TRUE(send_socket_fd >= 0);
BufferUdpReceiver udp_receiver(udp_port, source_id);
auto handle = async(launch::async, [&](){
for (int i_frame=0; i_frame < n_frames; i_frame++){
for (size_t i_packet=0; i_packet<n_packets; i_packet++) {
// Skip some random middle packet.
if (i_packet == 10) {
continue;
}
jungfrau_packet send_udp_buffer;
send_udp_buffer.packetnum = i_packet;
send_udp_buffer.bunchid = i_frame + 1;
send_udp_buffer.framenum = i_frame + 1000;
send_udp_buffer.debug = i_frame + 10000;
::sendto(
send_socket_fd,
&send_udp_buffer,
JUNGFRAU_BYTES_PER_PACKET,
0,
(sockaddr*) &server_address,
sizeof(server_address));
}
}
});
handle.wait();
ModuleFrame metadata;
auto frame_buffer = make_unique<char[]>(JUNGFRAU_DATA_BYTES_PER_FRAME);
for (int i_frame=0; i_frame < n_frames; i_frame++) {
auto pulse_id = udp_receiver.get_frame_from_udp(
metadata, frame_buffer.get());
ASSERT_EQ(i_frame + 1, pulse_id);
ASSERT_EQ(metadata.frame_index, i_frame + 1000);
ASSERT_EQ(metadata.daq_rec, i_frame + 10000);
// -1 because we skipped a packet.
ASSERT_EQ(metadata.n_received_packets, n_packets-1);
ASSERT_EQ(metadata.module_id, source_id);
}
::close(send_socket_fd);
}
TEST(BufferUdpReceiver, missing_first_packet)
{
auto n_packets = JUNGFRAU_N_PACKETS_PER_FRAME;
int source_id = 1234;
int n_frames = 3;
uint16_t udp_port = MOCK_UDP_PORT;
auto server_address = get_server_address(udp_port);
auto send_socket_fd = socket(AF_INET, SOCK_DGRAM, 0);
ASSERT_TRUE(send_socket_fd >= 0);
BufferUdpReceiver udp_receiver(udp_port, source_id);
auto handle = async(launch::async, [&](){
for (int i_frame=0; i_frame < n_frames; i_frame++){
for (size_t i_packet=0; i_packet<n_packets; i_packet++) {
// Skip first packet.
if (i_packet == 0) {
continue;
}
jungfrau_packet send_udp_buffer;
send_udp_buffer.packetnum = i_packet;
send_udp_buffer.bunchid = i_frame + 1;
send_udp_buffer.framenum = i_frame + 1000;
send_udp_buffer.debug = i_frame + 10000;
::sendto(
send_socket_fd,
&send_udp_buffer,
JUNGFRAU_BYTES_PER_PACKET,
0,
(sockaddr*) &server_address,
sizeof(server_address));
}
}
});
handle.wait();
ModuleFrame metadata;
auto frame_buffer = make_unique<char[]>(JUNGFRAU_DATA_BYTES_PER_FRAME);
for (int i_frame=0; i_frame < n_frames; i_frame++) {
auto pulse_id = udp_receiver.get_frame_from_udp(
metadata, frame_buffer.get());
ASSERT_EQ(i_frame + 1, pulse_id);
ASSERT_EQ(metadata.frame_index, i_frame + 1000);
ASSERT_EQ(metadata.daq_rec, i_frame + 10000);
// -1 because we skipped a packet.
ASSERT_EQ(metadata.n_received_packets, n_packets-1);
ASSERT_EQ(metadata.module_id, source_id);
}
::close(send_socket_fd);
}
TEST(BufferUdpReceiver, missing_last_packet)
{
auto n_packets = JUNGFRAU_N_PACKETS_PER_FRAME;
int source_id = 1234;
int n_frames = 3;
uint16_t udp_port = MOCK_UDP_PORT;
auto server_address = get_server_address(udp_port);
auto send_socket_fd = socket(AF_INET, SOCK_DGRAM, 0);
ASSERT_TRUE(send_socket_fd >= 0);
BufferUdpReceiver udp_receiver(udp_port, source_id);
auto handle = async(launch::async, [&](){
for (int i_frame=0; i_frame < n_frames; i_frame++){
for (size_t i_packet=0; i_packet<n_packets; i_packet++) {
// Skip the last packet.
if (i_packet == n_packets-1) {
continue;
}
jungfrau_packet send_udp_buffer;
send_udp_buffer.packetnum = i_packet;
send_udp_buffer.bunchid = i_frame + 1;
send_udp_buffer.framenum = i_frame + 1000;
send_udp_buffer.debug = i_frame + 10000;
::sendto(
send_socket_fd,
&send_udp_buffer,
JUNGFRAU_BYTES_PER_PACKET,
0,
(sockaddr*) &server_address,
sizeof(server_address));
}
}
});
handle.wait();
ModuleFrame metadata;
auto frame_buffer = make_unique<char[]>(JUNGFRAU_DATA_BYTES_PER_FRAME);
// n_frames -1 because the last frame is not complete.
for (int i_frame=0; i_frame < n_frames - 1; i_frame++) {
auto pulse_id = udp_receiver.get_frame_from_udp(
metadata, frame_buffer.get());
ASSERT_EQ(i_frame + 1, pulse_id);
ASSERT_EQ(metadata.frame_index, i_frame + 1000);
ASSERT_EQ(metadata.daq_rec, i_frame + 10000);
// -1 because we skipped a packet.
ASSERT_EQ(metadata.n_received_packets, n_packets-1);
ASSERT_EQ(metadata.module_id, source_id);
}
::close(send_socket_fd);
}
core-buffer/test/test_FastQueue.cpp
-147
View File
@@ -1,147 +0,0 @@
#include "FastQueue.hpp"
#include "gtest/gtest.h"
using namespace core_buffer;
TEST(FastQueue, basic_interaction)
{
size_t n_slots = 5;
size_t slot_data_n_bytes = MODULE_N_BYTES * 2;
FastQueue<ImageMetadata> queue(slot_data_n_bytes, n_slots);
int slot_id;
// The queue at the beginning should be empty.
ASSERT_EQ(queue.read(), -1);
// Cannot commit a slot until you reserve it.
ASSERT_THROW(queue.commit(), runtime_error);
// Cannot release a slot until its ready.
ASSERT_THROW(queue.release(), runtime_error);
// Reserve a slot.
slot_id = queue.reserve();
ASSERT_NE(slot_id, -1);
// But you cannot reserve 2 slots at once.
ASSERT_EQ(queue.reserve(), -1);
// And cannot read this slot until its committed.
ASSERT_EQ(queue.read(), -1);
auto detector_frame = queue.get_metadata_buffer(slot_id);
char* meta_ptr = (char*) detector_frame;
char* data_ptr = (char*) queue.get_data_buffer(slot_id);
queue.commit();
slot_id = queue.read();
// Once the slot is committed we should be able to read it.
ASSERT_NE(slot_id, -1);
// You can read the same slot multiple times.
ASSERT_NE(queue.read(), -1);
// The 2 buffers should match the committed slot.
ASSERT_EQ(meta_ptr, (char*)(queue.get_metadata_buffer(slot_id)));
ASSERT_EQ(data_ptr, (char*)(queue.get_data_buffer(slot_id)));
queue.release();
}
TEST(FastQueue, queue_full)
{
size_t n_slots = 5;
size_t slot_data_n_bytes = MODULE_N_BYTES * 2;
FastQueue<ImageMetadata> queue(slot_data_n_bytes, n_slots);
// There is nothing to be read in the queue.
ASSERT_EQ(queue.read(), -1);
for (size_t i=0; i<n_slots; i++) {
// Business as usual here, we still have slots left.
ASSERT_NE(queue.reserve(), -1);
queue.commit();
}
// There are no more slots available.
ASSERT_EQ(queue.reserve(), -1);
// We now read the first slot.
ASSERT_EQ(queue.read(), 0);
// But until we release it we cannot re-use it.
ASSERT_EQ(queue.reserve(), -1);
queue.release();
// After the release, the first slot is again ready for writing.
ASSERT_EQ(queue.reserve(), 0);
}
TEST(FastQueue, data_transfer)
{
size_t n_slots = 5;
size_t slot_data_n_bytes = MODULE_N_BYTES * 2;
FastQueue<ImageMetadata> queue(slot_data_n_bytes, n_slots);
int write_slot_id = queue.reserve();
auto w_metadata = queue.get_metadata_buffer(write_slot_id);
w_metadata->pulse_id = 1;
w_metadata->frame_index = 2;
w_metadata->daq_rec = 3;
w_metadata->is_good_frame = 4;
auto w_data = (uint16_t*)(queue.get_data_buffer(write_slot_id));
for (size_t i=0; i<MODULE_N_PIXELS; i++) {
w_data[i] = (uint16_t) i;
}
queue.commit();
auto read_slot_id = queue.read();
auto r_metadata = queue.get_metadata_buffer(read_slot_id);
EXPECT_EQ(w_metadata->pulse_id,
r_metadata->pulse_id);
EXPECT_EQ(w_metadata->frame_index,
r_metadata->frame_index);
EXPECT_EQ(w_metadata->daq_rec,
r_metadata->daq_rec);
EXPECT_EQ(w_metadata->is_good_frame,
r_metadata->is_good_frame);
auto r_data = (uint16_t*)(queue.get_data_buffer(read_slot_id));
for (size_t i=0; i<MODULE_N_PIXELS; i++) {
ASSERT_EQ(r_data[i], (uint16_t) i);
}
}
TEST(FaseQueue, array_parameter)
{
size_t n_modules = 32;
FastQueue<ModuleFrameBuffer> queue(
n_modules * MODULE_N_BYTES,
WRITER_FASTQUEUE_N_SLOTS);
ModuleFrame frame;
auto slot_id = queue.reserve();
auto metadata = queue.get_metadata_buffer(slot_id);
for (int i_module=0; i_module<n_modules; i_module++) {
auto& module_metadata = metadata->module[i_module];
frame.pulse_id = i_module;
frame.frame_index = i_module;
frame.daq_rec = i_module;
frame.n_received_packets = i_module;
frame.module_id = i_module;
ModuleFrame* p_metadata = &module_metadata;
memcpy(p_metadata, &frame, sizeof(ModuleFrame));
}
for (int i_module=0; i_module<n_modules; i_module++) {
auto& module_metadata = metadata->module[i_module];
ASSERT_EQ(module_metadata.pulse_id, i_module);
ASSERT_EQ(module_metadata.frame_index, i_module);
ASSERT_EQ(module_metadata.daq_rec, i_module);
ASSERT_EQ(module_metadata.n_received_packets, i_module);
ASSERT_EQ(module_metadata.module_id, i_module);
}
}
core-buffer/test/test_LiveRecvModule.cpp
-87
View File
@@ -1,87 +0,0 @@
#include <zmq.h>
#include "LiveRecvModule.hpp"
#include "gtest/gtest.h"
#include "buffer_config.hpp"
#include <future>
using namespace std;
using namespace core_buffer;
TEST(LiveRecvModule, transfer_test) {
// TODO: Make this test work again.
// auto ctx = zmq_ctx_new();
//
// size_t n_modules = 32;
// size_t n_slots = 5;
// FastQueue<ModuleFrameBuffer> queue(MODULE_N_BYTES * n_modules, n_slots);
//
// void *sockets[n_modules];
// for (size_t i = 0; i < n_modules; i++) {
// sockets[i] = zmq_socket(ctx, ZMQ_PUB);
//
// int linger = 0;
// if (zmq_setsockopt(sockets[i], ZMQ_LINGER, &linger,
// sizeof(linger)) != 0) {
// throw runtime_error(zmq_strerror(errno));
// }
//
// stringstream ipc_addr;
// ipc_addr << BUFFER_LIVE_IPC_URL << i;
// const auto ipc = ipc_addr.str();
//
// if (zmq_bind(sockets[i], ipc.c_str()) != 0) {
// throw runtime_error(zmq_strerror(errno));
// }
// }
//
// LiveRecvModule recv_module(queue, n_modules, ctx, BUFFER_LIVE_IPC_URL);
//
// // Nothing should be committed, queue, should be empty.
// ASSERT_EQ(queue.read(), -1);
//
// ModuleFrame metadata;
// auto data = make_unique<char[]>(MODULE_N_BYTES);
//
// for (size_t i = 0; i < n_modules; i++) {
// metadata.pulse_id = 1;
// metadata.frame_index = 2;
// metadata.daq_rec = 3;
// metadata.n_received_packets = 4;
// metadata.module_id = i;
//
// zmq_send(sockets[i], &metadata, sizeof(ModuleFrame), ZMQ_SNDMORE);
// zmq_send(sockets[i], data.get(), MODULE_N_BYTES, 0);
// }
//
// this_thread::sleep_for(chrono::milliseconds(100));
//
// auto slot_id = queue.read();
// // We should have the first Detector frame in the buffer.
// //ASSERT_NE(slot_id, -1);
//
// auto recv_stopped = async(launch::async, [&](){
// recv_module.stop();
// });
//
// this_thread::sleep_for(chrono::milliseconds(100));
//
// for (size_t i = 0; i < n_modules; i++) {
// metadata.pulse_id = 1;
// metadata.frame_index = 2;
// metadata.daq_rec = 3;
// metadata.n_received_packets = 4;
// metadata.module_id = i;
//
// zmq_send(sockets[i], &metadata, sizeof(ModuleFrame), ZMQ_SNDMORE);
// zmq_send(sockets[i], data.get(), MODULE_N_BYTES, 0);
// }
//
// recv_stopped.wait();
//
// for (size_t i = 0; i < n_modules; i++) {
// zmq_close(sockets[i]);
// }
//
// zmq_ctx_destroy(ctx);
// cout << "We are finished" << endl;
}
core-buffer/test/test_ReplayH5Reader.cpp
-110
View File
@@ -1,110 +0,0 @@
#include "ReplayH5Reader.hpp"
#include "BufferH5Writer.hpp"
#include "gtest/gtest.h"
using namespace core_buffer;
TEST(ReplayH5Reader, basic_interaction)
{
auto root_folder = ".";
auto device_name = "fast_device";
// This 2 must be compatible by design.
BufferH5Writer writer(root_folder, device_name);
ReplayH5Reader reader(root_folder, device_name);
size_t pulse_id = 65;
ModuleFrame w_metadata;
ModuleFrame r_metadata;
auto w_frame_buffer = make_unique<uint16_t[]>(MODULE_N_PIXELS);
auto r_frame_buffer = make_unique<uint16_t[]>(MODULE_N_PIXELS);
// Setup test values.
w_metadata.pulse_id = pulse_id;
w_metadata.frame_index = 2;
w_metadata.daq_rec = 3;
w_metadata.n_received_packets = 128;
w_metadata.module_id = 4;
for (size_t i=0; i<MODULE_N_PIXELS; i++) {
w_frame_buffer[i] = i % 100;
}
// Write to file.
writer.set_pulse_id(pulse_id);
writer.write(&w_metadata, (char*)&(w_frame_buffer[0]));
writer.close_file();
auto frame_present = reader.get_frame(
pulse_id, &r_metadata, (char*)&(r_frame_buffer[0]));
ASSERT_EQ(frame_present, true);
// Metadata has to match.
EXPECT_EQ(r_metadata.pulse_id, w_metadata.pulse_id);
EXPECT_EQ(r_metadata.frame_index, w_metadata.frame_index);
EXPECT_EQ(r_metadata.daq_rec, w_metadata.daq_rec);
EXPECT_EQ(r_metadata.n_received_packets, w_metadata.n_received_packets);
EXPECT_EQ(r_metadata.module_id, w_metadata.module_id);
// Data as well.
for (size_t i=0; i<MODULE_N_PIXELS; i++) {
w_frame_buffer[i] = r_frame_buffer[i];
}
reader.close_file();
}
TEST(ReplayH5Reader, missing_frame)
{
auto root_folder = ".";
auto device_name = "fast_device";
// This 2 must be compatible by design.
BufferH5Writer writer(root_folder, device_name);
ReplayH5Reader reader(root_folder, device_name);
size_t pulse_id = 65;
ModuleFrame w_metadata;
ModuleFrame r_metadata;
auto w_frame_buffer = make_unique<uint16_t[]>(MODULE_N_PIXELS);
auto r_frame_buffer = make_unique<uint16_t[]>(MODULE_N_PIXELS);
// Setup test values.
w_metadata.pulse_id = pulse_id;
w_metadata.frame_index = 2;
w_metadata.daq_rec = 3;
w_metadata.n_received_packets = 128;
w_metadata.module_id = 4;
for (size_t i=0; i<MODULE_N_PIXELS; i++) {
w_frame_buffer[i] = i % 100;
}
// Write to file.
writer.set_pulse_id(pulse_id);
writer.write(&w_metadata, (char*)&(w_frame_buffer[0]));
writer.close_file();
// But read another pulse_id, that should be empty.
auto frame_present = reader.get_frame(
pulse_id-1, &r_metadata, (char*)&(r_frame_buffer[0]));
ASSERT_EQ(frame_present, false);
// All metadata has to be 0, expect pulse_id.
EXPECT_EQ(r_metadata.pulse_id, pulse_id-1);
EXPECT_EQ(r_metadata.frame_index, 0);
EXPECT_EQ(r_metadata.daq_rec, 0);
EXPECT_EQ(r_metadata.n_received_packets, 0);
EXPECT_EQ(r_metadata.module_id, 0);
// Data as well.
for (size_t i=0; i<MODULE_N_PIXELS; i++) {
r_frame_buffer[i] = 0;
}
reader.close_file();
}
core-buffer/test/test_UdpReceiver.cpp
-170
View File
@@ -1,170 +0,0 @@
#include <netinet/in.h>
#include <jungfrau.hpp>
#include "gtest/gtest.h"
#include "UdpReceiver.hpp"
#include "mock/udp.hpp"
#include <thread>
#include <chrono>
using namespace std;
TEST(UdpReceiver, simple_recv)
{
uint16_t udp_port = MOCK_UDP_PORT;
auto send_socket_fd = socket(AF_INET,SOCK_DGRAM,0);
ASSERT_TRUE(send_socket_fd >= 0);
UdpReceiver udp_receiver;
udp_receiver.bind(udp_port);
jungfrau_packet send_udp_buffer;
send_udp_buffer.packetnum = 91;
send_udp_buffer.framenum = 92;
send_udp_buffer.bunchid = 93;
send_udp_buffer.debug = 94;
auto server_address = get_server_address(udp_port);
::sendto(
send_socket_fd,
&send_udp_buffer,
JUNGFRAU_BYTES_PER_PACKET,
0,
(sockaddr*) &server_address,
sizeof(server_address));
this_thread::sleep_for(chrono::milliseconds(100));
jungfrau_packet recv_udp_buffer;
ASSERT_TRUE(udp_receiver.receive(
&recv_udp_buffer, JUNGFRAU_BYTES_PER_PACKET));
EXPECT_EQ(send_udp_buffer.packetnum, recv_udp_buffer.packetnum);
EXPECT_EQ(send_udp_buffer.framenum, recv_udp_buffer.framenum);
EXPECT_EQ(send_udp_buffer.bunchid, recv_udp_buffer.bunchid);
EXPECT_EQ(send_udp_buffer.debug, recv_udp_buffer.debug);
ASSERT_FALSE(udp_receiver.receive(
&recv_udp_buffer, JUNGFRAU_BYTES_PER_PACKET));
udp_receiver.disconnect();
::close(send_socket_fd);
}
TEST(UdpReceiver, false_recv)
{
uint16_t udp_port = MOCK_UDP_PORT;
auto send_socket_fd = socket(AF_INET,SOCK_DGRAM,0);
ASSERT_TRUE(send_socket_fd >= 0);
UdpReceiver udp_receiver;
udp_receiver.bind(udp_port);
jungfrau_packet send_udp_buffer;
jungfrau_packet recv_udp_buffer;
auto server_address = get_server_address(udp_port);
::sendto(
send_socket_fd,
&send_udp_buffer,
JUNGFRAU_BYTES_PER_PACKET-1,
0,
(sockaddr*) &server_address,
sizeof(server_address));
ASSERT_FALSE(udp_receiver.receive(
&recv_udp_buffer, JUNGFRAU_BYTES_PER_PACKET));
::sendto(
send_socket_fd,
&send_udp_buffer,
JUNGFRAU_BYTES_PER_PACKET,
0,
(sockaddr*) &server_address,
sizeof(server_address));
ASSERT_TRUE(udp_receiver.receive(
&recv_udp_buffer, JUNGFRAU_BYTES_PER_PACKET));
::sendto(
send_socket_fd,
&send_udp_buffer,
JUNGFRAU_BYTES_PER_PACKET-1,
0,
(sockaddr*) &server_address,
sizeof(server_address));
ASSERT_TRUE(udp_receiver.receive(
&recv_udp_buffer, JUNGFRAU_BYTES_PER_PACKET-1));
udp_receiver.disconnect();
::close(send_socket_fd);
}
TEST(UdpReceiver, receive_many)
{
auto n_msg_buffer = JUNGFRAU_N_PACKETS_PER_FRAME;
jungfrau_packet recv_buffer[n_msg_buffer];
iovec recv_buff_ptr[n_msg_buffer];
struct mmsghdr msgs[n_msg_buffer];
struct sockaddr_in sockFrom[n_msg_buffer];
for (int i = 0; i < n_msg_buffer; i++) {
recv_buff_ptr[i].iov_base = (void*) &(recv_buffer[i]);
recv_buff_ptr[i].iov_len = sizeof(jungfrau_packet);
msgs[i].msg_hdr.msg_iov = &recv_buff_ptr[i];
msgs[i].msg_hdr.msg_iovlen = 1;
msgs[i].msg_hdr.msg_name = &sockFrom[i];
msgs[i].msg_hdr.msg_namelen = sizeof(sockaddr_in);
}
uint16_t udp_port = MOCK_UDP_PORT;
auto send_socket_fd = socket(AF_INET,SOCK_DGRAM,0);
ASSERT_TRUE(send_socket_fd >= 0);
UdpReceiver udp_receiver;
udp_receiver.bind(udp_port);
jungfrau_packet send_udp_buffer;
auto server_address = get_server_address(udp_port);
send_udp_buffer.bunchid = 0;
::sendto(
send_socket_fd,
&send_udp_buffer,
JUNGFRAU_BYTES_PER_PACKET,
0,
(sockaddr*) &server_address,
sizeof(server_address));
send_udp_buffer.bunchid = 1;
::sendto(
send_socket_fd,
&send_udp_buffer,
JUNGFRAU_BYTES_PER_PACKET,
0,
(sockaddr*) &server_address,
sizeof(server_address));
this_thread::sleep_for(chrono::milliseconds(10));
auto n_msgs = udp_receiver.receive_many(msgs, JUNGFRAU_N_PACKETS_PER_FRAME);
ASSERT_EQ(n_msgs, 2);
for (size_t i=0;i<n_msgs;i++) {
ASSERT_EQ(msgs[i].msg_len, JUNGFRAU_BYTES_PER_PACKET);
ASSERT_EQ(recv_buffer[i].bunchid, i);
}
n_msgs = udp_receiver.receive_many(msgs, JUNGFRAU_N_PACKETS_PER_FRAME);
ASSERT_EQ(n_msgs, -1);
udp_receiver.disconnect();
::close(send_socket_fd);
}
core-buffer/test/test_WriterH5Writer.cpp
-92
View File
@@ -1,92 +0,0 @@
#include "WriterH5Writer.hpp"
#include "gtest/gtest.h"
#include "bitshuffle/bitshuffle.h"
using namespace core_buffer;
TEST(WriterH5Writer, basic_interaction)
{
size_t n_modules = 2;
size_t n_frames = 5;
auto data = make_unique<char[]>(n_modules*MODULE_N_BYTES);
auto metadata = make_shared<ImageMetadataBuffer>();
// Needed by writer.
metadata->data_n_bytes[0] = 500;
metadata->n_pulses_in_buffer = 1;
WriterH5Writer writer("ignore.h5", n_frames, n_modules);
writer.write(metadata.get(), data.get());
writer.close_file();
}
TEST(WriterH5Writer, test_compression)
{
// size_t n_modules = 2;
// size_t n_frames = 2;
//
// auto comp_buffer_size = bshuf_compress_lz4_bound(
// MODULE_N_PIXELS, PIXEL_N_BYTES, MODULE_N_PIXELS);
//
// auto f_raw_buffer = make_unique<uint16_t[]>(MODULE_N_PIXELS);
// auto f_comp_buffer = make_unique<char[]>(comp_buffer_size);
//
// auto i_comp_buffer = make_unique<char[]>(
// (comp_buffer_size * n_modules) + BSHUF_LZ4_HEADER_BYTES);
// auto i_raw_buffer = make_unique<uint16_t[]>(
// MODULE_N_PIXELS * n_modules * n_frames);
//
// bshuf_write_uint64_BE(&i_comp_buffer[0],
// MODULE_N_BYTES * n_modules);
// bshuf_write_uint32_BE(&i_comp_buffer[8],
// MODULE_N_PIXELS * PIXEL_N_BYTES);
//
// size_t total_compressed_size = BSHUF_LZ4_HEADER_BYTES;
// for (int i_module=0; i_module<n_modules; i_module++) {
//
// for (size_t i=0; i<MODULE_N_PIXELS; i++) {
// f_raw_buffer[i] = (uint16_t)((i % 100) + (i_module*100));
// }
//
// auto compressed_size = bshuf_compress_lz4(
// f_raw_buffer.get(), f_comp_buffer.get(),
// MODULE_N_PIXELS, PIXEL_N_BYTES, MODULE_N_PIXELS);
//
// memcpy((i_comp_buffer.get() + total_compressed_size),
// f_comp_buffer.get(),
// compressed_size);
//
// total_compressed_size += compressed_size;
// }
//
// auto metadata = make_shared<ImageMetadata>();
// metadata->data_n_bytes = total_compressed_size;
//
// metadata->is_good_frame = 1;
// metadata->frame_index = 3;
// metadata->pulse_id = 3;
// metadata->daq_rec = 3;
//
// auto result = bshuf_decompress_lz4(
// &i_comp_buffer[12], &i_raw_buffer[0],
// MODULE_N_PIXELS*n_modules, PIXEL_N_BYTES, MODULE_N_PIXELS);
//
// WriterH5Writer writer("ignore.h5", n_frames, n_modules);
// writer.write(metadata.get(), &i_comp_buffer[0]);
// writer.close_file();
//
// H5::H5File reader("ignore.h5", H5F_ACC_RDONLY);
// auto image_dataset = reader.openDataSet("image");
// image_dataset.read(&i_raw_buffer[0], H5::PredType::NATIVE_UINT16);
//
// for (int i_module=0; i_module<n_modules; i_module++) {
// for (int i_pixel=0; i_pixel<MODULE_N_PIXELS; i_pixel++) {
// size_t offset = (i_module * MODULE_N_PIXELS) + i_pixel;
// ASSERT_EQ(i_raw_buffer[offset],
// (uint16_t)((i_pixel % 100) + (i_module*100)));
// }
// }
}
core-buffer/test/test_WriterZmqReceiver.cpp
-74
View File
@@ -1,74 +0,0 @@
#include <gtest/gtest.h>
#include "WriterZmqReceiver.hpp"
#include "bitshuffle/bitshuffle.h"
#include "zmq.h"
TEST(WriterZmqReceiver, basic_test)
{
size_t n_modules = 4;
uint64_t pulse_id = 12345;
auto ctx = zmq_ctx_new();
zmq_ctx_set (ctx, ZMQ_IO_THREADS, 1);
void* sockets[n_modules];
for (size_t i = 0; i < n_modules; i++) {
sockets[i] = zmq_socket(ctx, ZMQ_PUSH);
int linger = 0;
if (zmq_setsockopt(sockets[i], ZMQ_LINGER, &linger,
sizeof(linger)) != 0) {
throw runtime_error(zmq_strerror(errno));
}
stringstream ipc_addr;
ipc_addr << REPLAY_STREAM_IPC_URL << i;
const auto ipc = ipc_addr.str();
if (zmq_bind(sockets[i], ipc.c_str()) != 0) {
throw runtime_error(zmq_strerror(errno));
}
}
this_thread::sleep_for(chrono::milliseconds(100));
WriterZmqReceiver receiver(ctx, REPLAY_STREAM_IPC_URL, n_modules);
this_thread::sleep_for(chrono::milliseconds(100));
size_t compressed_frame_size = 5000;
auto frame_buffer = make_unique<char[]>(compressed_frame_size);
ImageMetadata image_metadata;
auto compress_size = bshuf_compress_lz4_bound(
MODULE_N_PIXELS, PIXEL_N_BYTES, MODULE_N_PIXELS);
auto image_buffer = make_unique<char[]>(compress_size * n_modules);
for (size_t i = 0; i < n_modules; i++) {
StreamModuleFrame frame_metadata;
frame_metadata.metadata.pulse_id = pulse_id;
frame_metadata.metadata.frame_index = pulse_id + 100;
frame_metadata.metadata.n_received_packets = 128;
frame_metadata.metadata.daq_rec = 4;
frame_metadata.is_frame_present = 1;
frame_metadata.data_n_bytes = compressed_frame_size;
zmq_send(sockets[i],
&frame_metadata,
sizeof(StreamModuleFrame),
ZMQ_SNDMORE);
zmq_send(sockets[i],
(char*)(frame_buffer.get()),
compressed_frame_size,
0);
}
receiver.get_next_image(pulse_id, &image_metadata, image_buffer.get());
EXPECT_EQ(pulse_id, image_metadata.pulse_id);
EXPECT_EQ(image_metadata.is_good_frame, 1);
EXPECT_EQ(image_metadata.daq_rec, 4);
EXPECT_EQ(image_metadata.data_n_bytes,
5000*n_modules);
// 5000*n_modules+BSHUF_LZ4_HEADER_BYTES);
}
core-buffer/test/test_bitshuffle.cpp
+1
View File
@@ -6,6 +6,7 @@ extern "C" {
}
using namespace std;
using namespace core_buffer;
TEST(bitshuffle, simple_compression)
{