sf-daq/sf_daq_buffer
0
0
Fork 0
mirror of https://github.com/paulscherrerinstitute/sf_daq_buffer.git synced 2026-04-22 05:04:37 +02:00
Code Issues Packages Projects Releases Wiki Activity

Test partial buffering

Browse Source
This commit is contained in:
babic_a
2020-04-22 15:12:19 +02:00
parent 70b8172a36
commit d5fb4f4dc0
2 changed files with 108 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core-writer/include/RingBuffer.hpp
+5 -5
View File
@@ -16,17 +16,17 @@
#pragma pack(1)
struct FileBufferMetadata {
// Needed by RingBuffer
const uint64_t frame_bytes_size = 2*512*1024*1000;
const uint64_t frame_bytes_size = 2*512*1024*50;
uint64_t buffer_slot_index;
uint64_t start_pulse_id;
uint64_t stop_pulse_id;
uint16_t module_id;
uint64_t pulse_id[1000];
uint64_t frame_index[1000];
uint32_t daq_rec[1000];
uint16_t n_received_packets[1000];
uint64_t pulse_id[50];
uint64_t frame_index[50];
uint32_t daq_rec[50];
uint16_t n_received_packets[50];
};
#pragma pack(pop)
sf-buffer/src/sf_replay.cpp
+103 -43
View File
@@ -7,6 +7,7 @@
#include "BufferUtils.hpp"
#include "zmq.h"
const int STREAM_BLOCK_SIZE = 50;
using namespace std;
@@ -40,7 +41,7 @@ int main (int argc, char *argv[]) {
metadata_buffer->module_id = module_id;
auto image_buffer = make_unique<uint16_t[]>(
BufferUtils::FILE_MOD * 512 * 1024);
STREAM_BLOCK_SIZE * 512 * 1024);
auto path_suffixes = BufferUtils::get_path_suffixes(
start_pulse_id, stop_pulse_id);
@@ -82,58 +83,117 @@ int main (int argc, char *argv[]) {
H5::H5File input_file(filename, H5F_ACC_RDONLY);
auto image_dataset = input_file.openDataSet("image");
image_dataset.read(
image_buffer.get(), H5::PredType::NATIVE_UINT16);
auto pulse_id_dataset = input_file.openDataSet("pulse_id");
pulse_id_dataset.read(
metadata_buffer->pulse_id, H5::PredType::NATIVE_UINT64);
auto frame_id_dataset = input_file.openDataSet("frame_id");
frame_id_dataset.read(
metadata_buffer->frame_index, H5::PredType::NATIVE_UINT64);
auto daq_rec_dataset = input_file.openDataSet("daq_rec");
daq_rec_dataset.read(
metadata_buffer->daq_rec, H5::PredType::NATIVE_UINT32);
auto received_packets_dataset =
input_file.openDataSet("received_packets");
// Load first
hsize_t file_dim[3] = {BufferUtils::FILE_MOD, 512, 1024};
H5::DataSpace file_space (3, file_dim);
hsize_t b_count[] = {STREAM_BLOCK_SIZE, 512, 1024};
hsize_t b_start[] = {0, 0, 0};
file_space.selectHyperslab(H5S_SELECT_SET, b_count, b_start);
image_dataset.read(
image_buffer.get(), H5::PredType::NATIVE_UINT16,
H5::DataSpace::ALL, file_space);
hsize_t meta_dim[2] = {BufferUtils::FILE_MOD, 1};
H5::DataSpace meta_space (2, meta_dim);
hsize_t m_count[] = {STREAM_BLOCK_SIZE, 1};
hsize_t m_start[] = {0, 0, 0};
meta_space.selectHyperslab(H5S_SELECT_SET, m_count, m_start);
pulse_id_dataset.read(
metadata_buffer->pulse_id, H5::PredType::NATIVE_UINT64,
H5::DataSpace::ALL, meta_space);
frame_id_dataset.read(
metadata_buffer->frame_index, H5::PredType::NATIVE_UINT64,
H5::DataSpace::ALL, meta_space);
daq_rec_dataset.read(
metadata_buffer->daq_rec, H5::PredType::NATIVE_UINT32,
H5::DataSpace::ALL, meta_space);
received_packets_dataset.read(
metadata_buffer->n_received_packets,
H5::PredType::NATIVE_UINT16);
H5::PredType::NATIVE_UINT16,
H5::DataSpace::ALL, meta_space);
for (
size_t start_pulse=0;
start_pulse<BufferUtils::FILE_MOD;
start_pulse+STREAM_BLOCK_SIZE) {
for (size_t i_frame=0; i_frame < STREAM_BLOCK_SIZE; i_frame++) {
ModuleFrame module_frame = {
metadata_buffer->pulse_id[i_frame],
metadata_buffer->frame_index[i_frame],
metadata_buffer->daq_rec[i_frame],
metadata_buffer->n_received_packets[i_frame],
module_id
};
zmq_send(socket,
&module_frame,
sizeof(ModuleFrame),
ZMQ_SNDMORE);
zmq_send(socket,
(char*) (image_buffer.get() + (i_frame * 512 * 1024)),
512 * 1024 * 2,
0);
}
// Load next
if (start_pulse + STREAM_BLOCK_SIZE < BufferUtils::FILE_MOD) {
hsize_t file_dim[3] = {BufferUtils::FILE_MOD, 512, 1024};
H5::DataSpace file_space (3, file_dim);
hsize_t b_count[] = {STREAM_BLOCK_SIZE, 512, 1024};
hsize_t b_start[] = {start_pulse, 0, 0};
file_space.selectHyperslab(H5S_SELECT_SET, b_count, b_start);
image_dataset.read(
image_buffer.get(), H5::PredType::NATIVE_UINT16,
H5::DataSpace::ALL, file_space);
hsize_t meta_dim[2] = {BufferUtils::FILE_MOD, 1};
H5::DataSpace meta_space (2, meta_dim);
hsize_t m_count[] = {STREAM_BLOCK_SIZE, 1};
hsize_t m_start[] = {start_pulse, 0, 0};
meta_space.selectHyperslab(H5S_SELECT_SET, m_count, m_start);
pulse_id_dataset.read(
metadata_buffer->pulse_id, H5::PredType::NATIVE_UINT64,
H5::DataSpace::ALL, meta_space);
frame_id_dataset.read(
metadata_buffer->frame_index, H5::PredType::NATIVE_UINT64,
H5::DataSpace::ALL, meta_space);
daq_rec_dataset.read(
metadata_buffer->daq_rec, H5::PredType::NATIVE_UINT32,
H5::DataSpace::ALL, meta_space);
received_packets_dataset.read(
metadata_buffer->n_received_packets,
H5::PredType::NATIVE_UINT16,
H5::DataSpace::ALL, meta_space);
}
// Wait for sync.
zmq_recv(meta_socket, nullptr, 0, 0);
}
input_file.close();
size_t send_counter = 0;
for (size_t i_frame=0; i_frame<BufferUtils::FILE_MOD; i_frame++) {
ModuleFrame module_frame = {
metadata_buffer->pulse_id[i_frame],
metadata_buffer->frame_index[i_frame],
metadata_buffer->daq_rec[i_frame],
metadata_buffer->n_received_packets[i_frame],
module_id
};
zmq_send(socket,
&module_frame,
sizeof(ModuleFrame),
ZMQ_SNDMORE);
zmq_send(socket,
(char*) (image_buffer.get() + (i_frame * 512 * 1024)),
512 * 1024 * 2,
0);
send_counter++;
if (send_counter == 50) {
// Wait for the sync message.
zmq_recv(meta_socket, nullptr, 0, 0);
send_counter = 0;
}
}
}
zmq_close(socket);