From 5b9b650803bad67a0f24b0b4b19aad43ef95fcbf Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 2 Sep 2020 13:39:26 +0200 Subject: [PATCH 001/126] Add stub implementation of RamBufferWriter --- core-buffer/include/RamBufferWriter.hpp | 22 +++++++++++++ core-buffer/src/RamBufferWriter.cpp | 43 +++++++++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 core-buffer/include/RamBufferWriter.hpp create mode 100644 core-buffer/src/RamBufferWriter.cpp diff --git a/core-buffer/include/RamBufferWriter.hpp b/core-buffer/include/RamBufferWriter.hpp new file mode 100644 index 0000000..4f689f3 --- /dev/null +++ b/core-buffer/include/RamBufferWriter.hpp @@ -0,0 +1,22 @@ +#ifndef SF_DAQ_BUFFER_RAMBUFFERWRITER_HPP +#define SF_DAQ_BUFFER_RAMBUFFERWRITER_HPP + +#include + +class RamBufferWriter { + const int module_n_; + const std::string detector_name_; + + int shm_fd_; + +public: + RamBufferWriter(const std::string& detector_name, + const int module_n, + const size_t n_modules, + const size_t n_slots); + ~RamBufferWriter(); + +}; + + +#endif //SF_DAQ_BUFFER_RAMBUFFERWRITER_HPP diff --git a/core-buffer/src/RamBufferWriter.cpp b/core-buffer/src/RamBufferWriter.cpp new file mode 100644 index 0000000..49d11cf --- /dev/null +++ b/core-buffer/src/RamBufferWriter.cpp @@ -0,0 +1,43 @@ +#include +#include +#include +#include +#include +#include +#include "RamBufferWriter.hpp" +#include "buffer_config.hpp" + +using namespace std; +using namespace buffer_config; + +RamBufferWriter::RamBufferWriter( + const string &detector_name, + const int module_n, + const size_t n_modules, + const size_t n_slots) : + detector_name_(detector_name), + module_n_(module_n) + +{ + shm_fd_ = shm_open(detector_name_.c_str(), O_RDWR | O_CREAT, 0777); + if (shm_fd_ < 0) { + throw runtime_error(strerror(errno)); + } + + size_t slot_size = MODULE_N_BYTES + sizeof(ModuleFrame); + size_t buffer_size = slot_size * n_modules * n_slots; + + if ((ftruncate(shm_fd_, buffer_size)) == -1) { + throw runtime_error(strerror(errno)); + } + + void* buffer = mmap(NULL, buffer_size, PROT_WRITE, MAP_SHARED, shm_fd_, 0); + if (buffer == (void*)-1) { + throw runtime_error(strerror(errno)); + } +} + +RamBufferWriter::~RamBufferWriter() +{ + shm_unlink(detector_name_.c_str()); +} From 20f5557cc1455ddf4324b9b79dbb9b0848f8e33a Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 2 Sep 2020 14:00:30 +0200 Subject: [PATCH 002/126] Add proper cleanup --- core-buffer/include/RamBufferWriter.hpp | 2 ++ core-buffer/src/RamBufferWriter.cpp | 15 ++++++++++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/core-buffer/include/RamBufferWriter.hpp b/core-buffer/include/RamBufferWriter.hpp index 4f689f3..64b6df9 100644 --- a/core-buffer/include/RamBufferWriter.hpp +++ b/core-buffer/include/RamBufferWriter.hpp @@ -8,6 +8,8 @@ class RamBufferWriter { const std::string detector_name_; int shm_fd_; + size_t buffer_size_; + void* buffer_; public: RamBufferWriter(const std::string& detector_name, diff --git a/core-buffer/src/RamBufferWriter.cpp b/core-buffer/src/RamBufferWriter.cpp index 49d11cf..0aaa818 100644 --- a/core-buffer/src/RamBufferWriter.cpp +++ b/core-buffer/src/RamBufferWriter.cpp @@ -25,19 +25,24 @@ RamBufferWriter::RamBufferWriter( } size_t slot_size = MODULE_N_BYTES + sizeof(ModuleFrame); - size_t buffer_size = slot_size * n_modules * n_slots; + buffer_size_ = slot_size * n_modules * n_slots; - if ((ftruncate(shm_fd_, buffer_size)) == -1) { + if ((ftruncate(shm_fd_, buffer_size_)) == -1) { throw runtime_error(strerror(errno)); } - void* buffer = mmap(NULL, buffer_size, PROT_WRITE, MAP_SHARED, shm_fd_, 0); - if (buffer == (void*)-1) { + // TODO: Test with MAP_HUGETLB + void* ptr = mmap(NULL, buffer_size_, PROT_WRITE, MAP_SHARED, shm_fd_, 0); + if (ptr == MAP_FAILED) { throw runtime_error(strerror(errno)); } + + buffer_ = ptr; } RamBufferWriter::~RamBufferWriter() { - shm_unlink(detector_name_.c_str()); + munmap(buffer_, buffer_size_); + shm_unlink(detector_name_.c_str()); + close(shm_fd_); } From 28d407fb0ec8aabe29b060a67049064d7e554327 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 2 Sep 2020 15:16:02 +0200 Subject: [PATCH 003/126] Implement common class for RamBuffer --- core-buffer/include/RamBuffer.hpp | 32 ++++++++++++ core-buffer/include/RamBufferWriter.hpp | 24 --------- core-buffer/src/RamBuffer.cpp | 66 +++++++++++++++++++++++++ core-buffer/src/RamBufferWriter.cpp | 48 ------------------ 4 files changed, 98 insertions(+), 72 deletions(-) create mode 100644 core-buffer/include/RamBuffer.hpp delete mode 100644 core-buffer/include/RamBufferWriter.hpp create mode 100644 core-buffer/src/RamBuffer.cpp delete mode 100644 core-buffer/src/RamBufferWriter.cpp diff --git a/core-buffer/include/RamBuffer.hpp b/core-buffer/include/RamBuffer.hpp new file mode 100644 index 0000000..486268f --- /dev/null +++ b/core-buffer/include/RamBuffer.hpp @@ -0,0 +1,32 @@ +#ifndef SF_DAQ_BUFFER_RAMBUFFER_HPP +#define SF_DAQ_BUFFER_RAMBUFFER_HPP + +#include +#include "formats.hpp" + +class RamBuffer { + const std::string detector_name_; + const int module_n_; + const size_t n_slots_; + const size_t image_size_; + const size_t buffer_size_; + + int shm_fd_; + void* buffer_; + + ModuleFrame* meta_buffer_; + char* image_buffer_; + + +public: + RamBuffer(const std::string& detector_name, + const size_t n_modules, + const int module_n, + const size_t n_slots); + ~RamBuffer(); + + void write_frame(const ModuleFrame& metadata, const char* data); +}; + + +#endif //SF_DAQ_BUFFER_RAMBUFFER_HPP diff --git a/core-buffer/include/RamBufferWriter.hpp b/core-buffer/include/RamBufferWriter.hpp deleted file mode 100644 index 64b6df9..0000000 --- a/core-buffer/include/RamBufferWriter.hpp +++ /dev/null @@ -1,24 +0,0 @@ -#ifndef SF_DAQ_BUFFER_RAMBUFFERWRITER_HPP -#define SF_DAQ_BUFFER_RAMBUFFERWRITER_HPP - -#include - -class RamBufferWriter { - const int module_n_; - const std::string detector_name_; - - int shm_fd_; - size_t buffer_size_; - void* buffer_; - -public: - RamBufferWriter(const std::string& detector_name, - const int module_n, - const size_t n_modules, - const size_t n_slots); - ~RamBufferWriter(); - -}; - - -#endif //SF_DAQ_BUFFER_RAMBUFFERWRITER_HPP diff --git a/core-buffer/src/RamBuffer.cpp b/core-buffer/src/RamBuffer.cpp new file mode 100644 index 0000000..1268eba --- /dev/null +++ b/core-buffer/src/RamBuffer.cpp @@ -0,0 +1,66 @@ +#include +#include +#include +#include +#include +#include "RamBuffer.hpp" +#include "buffer_config.hpp" + +using namespace std; +using namespace buffer_config; + +RamBuffer::RamBuffer( + const string &detector_name, + const size_t n_modules, + const int module_n, + const size_t n_slots) : + detector_name_(detector_name), + module_n_(module_n), + n_slots_(n_slots), + image_size_(MODULE_N_BYTES * n_modules), + buffer_size_((sizeof(ModuleFrame) + image_size_) * n_slots) +{ + shm_fd_ = shm_open(detector_name_.c_str(), O_RDWR | O_CREAT, 0777); + if (shm_fd_ < 0) { + throw runtime_error(strerror(errno)); + } + + if ((ftruncate(shm_fd_, buffer_size_)) == -1) { + throw runtime_error(strerror(errno)); + } + + // TODO: Test with MAP_HUGETLB + buffer_ = mmap(NULL, buffer_size_, PROT_WRITE, MAP_SHARED, shm_fd_, 0); + if (buffer_ == MAP_FAILED) { + throw runtime_error(strerror(errno)); + } + + meta_buffer_ = (ModuleFrame *) buffer_; + // Image buffer start right after metadata buffer. + image_buffer_ = (char *) (meta_buffer_ + n_slots); +} + +RamBuffer::~RamBuffer() +{ + munmap(buffer_, buffer_size_); + close(shm_fd_); + shm_unlink(detector_name_.c_str()); + +} + +void RamBuffer::write_frame( + const ModuleFrame &metadata, + const char *data) +{ + size_t slot_n = metadata.pulse_id % n_slots_; + + ModuleFrame *meta = meta_buffer_ + slot_n; + *meta = metadata; + + char *frame = image_buffer_ + + (image_size_ * slot_n) + + (MODULE_N_BYTES * module_n_); + + memcpy(frame, data, MODULE_N_BYTES); +} + diff --git a/core-buffer/src/RamBufferWriter.cpp b/core-buffer/src/RamBufferWriter.cpp deleted file mode 100644 index 0aaa818..0000000 --- a/core-buffer/src/RamBufferWriter.cpp +++ /dev/null @@ -1,48 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include "RamBufferWriter.hpp" -#include "buffer_config.hpp" - -using namespace std; -using namespace buffer_config; - -RamBufferWriter::RamBufferWriter( - const string &detector_name, - const int module_n, - const size_t n_modules, - const size_t n_slots) : - detector_name_(detector_name), - module_n_(module_n) - -{ - shm_fd_ = shm_open(detector_name_.c_str(), O_RDWR | O_CREAT, 0777); - if (shm_fd_ < 0) { - throw runtime_error(strerror(errno)); - } - - size_t slot_size = MODULE_N_BYTES + sizeof(ModuleFrame); - buffer_size_ = slot_size * n_modules * n_slots; - - if ((ftruncate(shm_fd_, buffer_size_)) == -1) { - throw runtime_error(strerror(errno)); - } - - // TODO: Test with MAP_HUGETLB - void* ptr = mmap(NULL, buffer_size_, PROT_WRITE, MAP_SHARED, shm_fd_, 0); - if (ptr == MAP_FAILED) { - throw runtime_error(strerror(errno)); - } - - buffer_ = ptr; -} - -RamBufferWriter::~RamBufferWriter() -{ - munmap(buffer_, buffer_size_); - shm_unlink(detector_name_.c_str()); - close(shm_fd_); -} From b3ddb922e980623bf6d4a1b77d57ffd760968c49 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Fri, 4 Sep 2020 10:35:50 +0200 Subject: [PATCH 004/126] Improve on shared buffer class --- core-buffer/include/RamBuffer.hpp | 11 +++++--- core-buffer/src/RamBuffer.cpp | 45 ++++++++++++++++++++----------- 2 files changed, 37 insertions(+), 19 deletions(-) diff --git a/core-buffer/include/RamBuffer.hpp b/core-buffer/include/RamBuffer.hpp index 486268f..fb22d2f 100644 --- a/core-buffer/include/RamBuffer.hpp +++ b/core-buffer/include/RamBuffer.hpp @@ -8,6 +8,7 @@ class RamBuffer { const std::string detector_name_; const int module_n_; const size_t n_slots_; + const size_t meta_size_; const size_t image_size_; const size_t buffer_size_; @@ -17,15 +18,17 @@ class RamBuffer { ModuleFrame* meta_buffer_; char* image_buffer_; - public: RamBuffer(const std::string& detector_name, const size_t n_modules, - const int module_n, - const size_t n_slots); + const size_t n_slots, + const int module_n=0); ~RamBuffer(); - void write_frame(const ModuleFrame& metadata, const char* data); + void write_frame(const ModuleFrame *src_meta, const char *src_data) const; + void read_image(const uint64_t pulse_id, + ModuleFrame *&dst_meta, + char *&dst_data) const; }; diff --git a/core-buffer/src/RamBuffer.cpp b/core-buffer/src/RamBuffer.cpp index 1268eba..8008fd2 100644 --- a/core-buffer/src/RamBuffer.cpp +++ b/core-buffer/src/RamBuffer.cpp @@ -12,13 +12,14 @@ using namespace buffer_config; RamBuffer::RamBuffer( const string &detector_name, const size_t n_modules, - const int module_n, - const size_t n_slots) : + const size_t n_slots, + const int module_n) : detector_name_(detector_name), - module_n_(module_n), n_slots_(n_slots), + module_n_(module_n), + meta_size_(sizeof(ModuleFrame) * n_modules), image_size_(MODULE_N_BYTES * n_modules), - buffer_size_((sizeof(ModuleFrame) + image_size_) * n_slots) + buffer_size_((meta_size_ + image_size_) * n_slots) { shm_fd_ = shm_open(detector_name_.c_str(), O_RDWR | O_CREAT, 0777); if (shm_fd_ < 0) { @@ -37,7 +38,7 @@ RamBuffer::RamBuffer( meta_buffer_ = (ModuleFrame *) buffer_; // Image buffer start right after metadata buffer. - image_buffer_ = (char *) (meta_buffer_ + n_slots); + image_buffer_ = (char *) (meta_buffer_ + (n_modules * n_slots)); } RamBuffer::~RamBuffer() @@ -45,22 +46,36 @@ RamBuffer::~RamBuffer() munmap(buffer_, buffer_size_); close(shm_fd_); shm_unlink(detector_name_.c_str()); - } void RamBuffer::write_frame( - const ModuleFrame &metadata, - const char *data) + const ModuleFrame *src_meta, + const char *src_data) const { - size_t slot_n = metadata.pulse_id % n_slots_; + const size_t slot_n = src_meta->pulse_id % n_slots_; - ModuleFrame *meta = meta_buffer_ + slot_n; - *meta = metadata; + ModuleFrame *dst_meta = meta_buffer_ + + (meta_size_ * slot_n) + + module_n_; - char *frame = image_buffer_ + - (image_size_ * slot_n) + - (MODULE_N_BYTES * module_n_); + char *dst_data = image_buffer_ + + (image_size_ * slot_n) + + (MODULE_N_BYTES * module_n_); - memcpy(frame, data, MODULE_N_BYTES); + memcpy(dst_meta, src_meta, sizeof(ModuleFrame)); + memcpy(dst_data, src_data, MODULE_N_BYTES); +} + +void RamBuffer::read_image(const uint64_t pulse_id, + ModuleFrame *&dst_meta, + char *&dst_data) const +{ + const size_t slot_n = pulse_id % n_slots_; + + ModuleFrame *src_meta = meta_buffer_ + + (meta_size_ * slot_n); + + char *src_data = image_buffer_ + + (image_size_ * slot_n); } From e12b8bdd519dc53c766cd83c8f5001295c88fdcc Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Fri, 4 Sep 2020 11:40:48 +0200 Subject: [PATCH 005/126] Add new ImageMetadata format --- core-buffer/include/formats.hpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/core-buffer/include/formats.hpp b/core-buffer/include/formats.hpp index a48cb8b..ad462b1 100644 --- a/core-buffer/include/formats.hpp +++ b/core-buffer/include/formats.hpp @@ -15,6 +15,17 @@ struct ModuleFrame { }; #pragma pack(pop) + +#pragma pack(push) +#pragma pack(1) +struct ImageMetadata { + uint64_t pulse_id; + uint64_t frame_index; + uint32_t daq_rec; + uint32_t is_good_frame; +}; +#pragma pack(pop) + struct ModuleFrameBuffer { ModuleFrame module[JUNGFRAU_N_MODULES]; }; From 15302c85974dc966c615b048cccb5c6f461b4560 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Fri, 4 Sep 2020 12:14:43 +0200 Subject: [PATCH 006/126] Rename field to reflect reality --- core-buffer/include/formats.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core-buffer/include/formats.hpp b/core-buffer/include/formats.hpp index ad462b1..731dd61 100644 --- a/core-buffer/include/formats.hpp +++ b/core-buffer/include/formats.hpp @@ -22,7 +22,7 @@ struct ImageMetadata { uint64_t pulse_id; uint64_t frame_index; uint32_t daq_rec; - uint32_t is_good_frame; + uint32_t is_good_image; }; #pragma pack(pop) From 4a031d6e620f6ae87cdd1b594ad47e60a7da0882 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Fri, 4 Sep 2020 12:31:57 +0200 Subject: [PATCH 007/126] Refactor RamBuffer --- core-buffer/include/RamBuffer.hpp | 10 ++-- core-buffer/include/buffer_config.hpp | 2 + core-buffer/src/RamBuffer.cpp | 68 ++++++++++++++++++++++----- 3 files changed, 62 insertions(+), 18 deletions(-) diff --git a/core-buffer/include/RamBuffer.hpp b/core-buffer/include/RamBuffer.hpp index fb22d2f..fa1501d 100644 --- a/core-buffer/include/RamBuffer.hpp +++ b/core-buffer/include/RamBuffer.hpp @@ -6,8 +6,9 @@ class RamBuffer { const std::string detector_name_; + const int n_modules_; const int module_n_; - const size_t n_slots_; + const size_t meta_size_; const size_t image_size_; const size_t buffer_size_; @@ -20,15 +21,12 @@ class RamBuffer { public: RamBuffer(const std::string& detector_name, - const size_t n_modules, - const size_t n_slots, + const int n_modules, const int module_n=0); ~RamBuffer(); void write_frame(const ModuleFrame *src_meta, const char *src_data) const; - void read_image(const uint64_t pulse_id, - ModuleFrame *&dst_meta, - char *&dst_data) const; + char* read_image(const uint64_t pulse_id, ImageMetadata &image_meta) const; }; diff --git a/core-buffer/include/buffer_config.hpp b/core-buffer/include/buffer_config.hpp index 6f58c22..2f1f662 100644 --- a/core-buffer/include/buffer_config.hpp +++ b/core-buffer/include/buffer_config.hpp @@ -40,6 +40,8 @@ namespace buffer_config { const int BUFFER_ZMQ_SNDHWM = 100; // IPC address of the live stream. const std::string BUFFER_LIVE_IPC_URL = "ipc:///tmp/sf-live-"; + // Number of image slots in ram buffer - 10 seconds should be enough + const int RAM_BUFFER_N_SLOTS = 100 * 10; } #endif //BUFFERCONFIG_HPP diff --git a/core-buffer/src/RamBuffer.cpp b/core-buffer/src/RamBuffer.cpp index 8008fd2..f5dfa6a 100644 --- a/core-buffer/src/RamBuffer.cpp +++ b/core-buffer/src/RamBuffer.cpp @@ -11,15 +11,14 @@ using namespace buffer_config; RamBuffer::RamBuffer( const string &detector_name, - const size_t n_modules, - const size_t n_slots, + const int n_modules, const int module_n) : detector_name_(detector_name), - n_slots_(n_slots), + n_modules_(n_modules), module_n_(module_n), - meta_size_(sizeof(ModuleFrame) * n_modules), - image_size_(MODULE_N_BYTES * n_modules), - buffer_size_((meta_size_ + image_size_) * n_slots) + meta_size_(sizeof(ModuleFrame) * n_modules_), + image_size_(MODULE_N_BYTES * n_modules_), + buffer_size_((meta_size_ + image_size_) * RAM_BUFFER_N_SLOTS) { shm_fd_ = shm_open(detector_name_.c_str(), O_RDWR | O_CREAT, 0777); if (shm_fd_ < 0) { @@ -38,7 +37,7 @@ RamBuffer::RamBuffer( meta_buffer_ = (ModuleFrame *) buffer_; // Image buffer start right after metadata buffer. - image_buffer_ = (char *) (meta_buffer_ + (n_modules * n_slots)); + image_buffer_ = (char*)buffer_ + (meta_size_ * RAM_BUFFER_N_SLOTS); } RamBuffer::~RamBuffer() @@ -52,7 +51,7 @@ void RamBuffer::write_frame( const ModuleFrame *src_meta, const char *src_data) const { - const size_t slot_n = src_meta->pulse_id % n_slots_; + const size_t slot_n = src_meta->pulse_id % RAM_BUFFER_N_SLOTS; ModuleFrame *dst_meta = meta_buffer_ + (meta_size_ * slot_n) + @@ -66,16 +65,61 @@ void RamBuffer::write_frame( memcpy(dst_data, src_data, MODULE_N_BYTES); } -void RamBuffer::read_image(const uint64_t pulse_id, - ModuleFrame *&dst_meta, - char *&dst_data) const +char* RamBuffer::read_image(const uint64_t pulse_id, + ImageMetadata &image_meta) const { - const size_t slot_n = pulse_id % n_slots_; + const size_t slot_n = pulse_id % RAM_BUFFER_N_SLOTS; ModuleFrame *src_meta = meta_buffer_ + (meta_size_ * slot_n); char *src_data = image_buffer_ + (image_size_ * slot_n); + + auto is_pulse_init = false; + auto is_good_image = true; + + for (int i_module=0; i_module < n_modules_; i_module++) { + ModuleFrame *frame_meta = src_meta + i_module; + + auto is_good_frame = + frame_meta->n_recv_packets == JF_N_PACKETS_PER_FRAME; + + if (!is_good_frame) { + is_good_image = false; + continue; + } + + if (!is_pulse_init) { + + if (frame_meta->pulse_id != pulse_id) { + throw runtime_error("Wrong pulse_id in ram buffer slot."); + } + + image_meta.pulse_id = frame_meta->pulse_id; + image_meta.frame_index = frame_meta->frame_index; + image_meta.daq_rec = frame_meta->daq_rec; + + is_pulse_init = 1; + } + + if (is_good_image) { + if (frame_meta->pulse_id != image_meta.pulse_id) { + is_good_image = false; + } + + if (frame_meta->frame_index != image_meta.frame_index) { + is_good_image = false; + } + + if (frame_meta->daq_rec != image_meta.daq_rec) { + is_good_image = false; + } + } + } + + image_meta.is_good_image = is_good_image; + + return src_data; } From 352aefef26f1290ac073d2bc54aeb5ae7096a620 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Fri, 4 Sep 2020 12:45:59 +0200 Subject: [PATCH 008/126] Add back n_slots to RamBuffer In tests you don't want it to consume all your ram. --- core-buffer/include/RamBuffer.hpp | 4 +++- core-buffer/src/RamBuffer.cpp | 12 +++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/core-buffer/include/RamBuffer.hpp b/core-buffer/include/RamBuffer.hpp index fa1501d..d871ac6 100644 --- a/core-buffer/include/RamBuffer.hpp +++ b/core-buffer/include/RamBuffer.hpp @@ -8,6 +8,7 @@ class RamBuffer { const std::string detector_name_; const int n_modules_; const int module_n_; + const int n_slots_; const size_t meta_size_; const size_t image_size_; @@ -22,7 +23,8 @@ class RamBuffer { public: RamBuffer(const std::string& detector_name, const int n_modules, - const int module_n=0); + const int module_n=0, + const int n_slots=RAM_BUFFER_N_SLOTS); ~RamBuffer(); void write_frame(const ModuleFrame *src_meta, const char *src_data) const; diff --git a/core-buffer/src/RamBuffer.cpp b/core-buffer/src/RamBuffer.cpp index f5dfa6a..376dfaa 100644 --- a/core-buffer/src/RamBuffer.cpp +++ b/core-buffer/src/RamBuffer.cpp @@ -12,13 +12,15 @@ using namespace buffer_config; RamBuffer::RamBuffer( const string &detector_name, const int n_modules, - const int module_n) : + const int module_n, + const int n_slots) : detector_name_(detector_name), n_modules_(n_modules), module_n_(module_n), + n_slots_(n_slots), meta_size_(sizeof(ModuleFrame) * n_modules_), image_size_(MODULE_N_BYTES * n_modules_), - buffer_size_((meta_size_ + image_size_) * RAM_BUFFER_N_SLOTS) + buffer_size_((meta_size_ + image_size_) * n_slots_) { shm_fd_ = shm_open(detector_name_.c_str(), O_RDWR | O_CREAT, 0777); if (shm_fd_ < 0) { @@ -37,7 +39,7 @@ RamBuffer::RamBuffer( meta_buffer_ = (ModuleFrame *) buffer_; // Image buffer start right after metadata buffer. - image_buffer_ = (char*)buffer_ + (meta_size_ * RAM_BUFFER_N_SLOTS); + image_buffer_ = (char*)buffer_ + (meta_size_ * n_slots_); } RamBuffer::~RamBuffer() @@ -51,7 +53,7 @@ void RamBuffer::write_frame( const ModuleFrame *src_meta, const char *src_data) const { - const size_t slot_n = src_meta->pulse_id % RAM_BUFFER_N_SLOTS; + const size_t slot_n = src_meta->pulse_id % n_slots_; ModuleFrame *dst_meta = meta_buffer_ + (meta_size_ * slot_n) + @@ -68,7 +70,7 @@ void RamBuffer::write_frame( char* RamBuffer::read_image(const uint64_t pulse_id, ImageMetadata &image_meta) const { - const size_t slot_n = pulse_id % RAM_BUFFER_N_SLOTS; + const size_t slot_n = pulse_id % n_slots_; ModuleFrame *src_meta = meta_buffer_ + (meta_size_ * slot_n); From bf1a5a928bc9fc6513b6bbd1074add6d4b5a0121 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Fri, 4 Sep 2020 12:54:41 +0200 Subject: [PATCH 009/126] Remove module id from RamBuffer Module id is already present in the buffer package. --- core-buffer/include/RamBuffer.hpp | 2 -- core-buffer/src/RamBuffer.cpp | 8 +++----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/core-buffer/include/RamBuffer.hpp b/core-buffer/include/RamBuffer.hpp index d871ac6..d6c841a 100644 --- a/core-buffer/include/RamBuffer.hpp +++ b/core-buffer/include/RamBuffer.hpp @@ -7,7 +7,6 @@ class RamBuffer { const std::string detector_name_; const int n_modules_; - const int module_n_; const int n_slots_; const size_t meta_size_; @@ -23,7 +22,6 @@ class RamBuffer { public: RamBuffer(const std::string& detector_name, const int n_modules, - const int module_n=0, const int n_slots=RAM_BUFFER_N_SLOTS); ~RamBuffer(); diff --git a/core-buffer/src/RamBuffer.cpp b/core-buffer/src/RamBuffer.cpp index 376dfaa..106c61a 100644 --- a/core-buffer/src/RamBuffer.cpp +++ b/core-buffer/src/RamBuffer.cpp @@ -12,11 +12,9 @@ using namespace buffer_config; RamBuffer::RamBuffer( const string &detector_name, const int n_modules, - const int module_n, const int n_slots) : detector_name_(detector_name), n_modules_(n_modules), - module_n_(module_n), n_slots_(n_slots), meta_size_(sizeof(ModuleFrame) * n_modules_), image_size_(MODULE_N_BYTES * n_modules_), @@ -53,15 +51,15 @@ void RamBuffer::write_frame( const ModuleFrame *src_meta, const char *src_data) const { - const size_t slot_n = src_meta->pulse_id % n_slots_; + const int slot_n = src_meta->pulse_id % n_slots_; ModuleFrame *dst_meta = meta_buffer_ + (meta_size_ * slot_n) + - module_n_; + src_meta->module_id; char *dst_data = image_buffer_ + (image_size_ * slot_n) + - (MODULE_N_BYTES * module_n_); + (MODULE_N_BYTES * src_meta->module_id); memcpy(dst_meta, src_meta, sizeof(ModuleFrame)); memcpy(dst_data, src_data, MODULE_N_BYTES); From 13f46908c5742735004bbf8e93cdc06bc6fed595 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Fri, 4 Sep 2020 13:00:31 +0200 Subject: [PATCH 010/126] Small fixes to RamBuffer before tests --- core-buffer/include/RamBuffer.hpp | 2 +- core-buffer/src/RamBuffer.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core-buffer/include/RamBuffer.hpp b/core-buffer/include/RamBuffer.hpp index d6c841a..8fb28e7 100644 --- a/core-buffer/include/RamBuffer.hpp +++ b/core-buffer/include/RamBuffer.hpp @@ -22,7 +22,7 @@ class RamBuffer { public: RamBuffer(const std::string& detector_name, const int n_modules, - const int n_slots=RAM_BUFFER_N_SLOTS); + const int n_slots=buffer_config::RAM_BUFFER_N_SLOTS); ~RamBuffer(); void write_frame(const ModuleFrame *src_meta, const char *src_data) const; diff --git a/core-buffer/src/RamBuffer.cpp b/core-buffer/src/RamBuffer.cpp index 106c61a..0f083ca 100644 --- a/core-buffer/src/RamBuffer.cpp +++ b/core-buffer/src/RamBuffer.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include #include From 395b986cc6f0666a329df075238174e7f1809de0 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Fri, 4 Sep 2020 13:00:49 +0200 Subject: [PATCH 011/126] Add test stub for RamBuffer --- core-buffer/test/CMakeLists.txt | 1 + core-buffer/test/main.cpp | 1 + core-buffer/test/test_RamBuffer.cpp | 29 +++++++++++++++++++++++++++++ 3 files changed, 31 insertions(+) create mode 100644 core-buffer/test/test_RamBuffer.cpp diff --git a/core-buffer/test/CMakeLists.txt b/core-buffer/test/CMakeLists.txt index f127886..bf61f07 100644 --- a/core-buffer/test/CMakeLists.txt +++ b/core-buffer/test/CMakeLists.txt @@ -3,6 +3,7 @@ add_executable(core-buffer-tests main.cpp) target_link_libraries(core-buffer-tests core-buffer-lib external + rt hdf5 hdf5_cpp zmq diff --git a/core-buffer/test/main.cpp b/core-buffer/test/main.cpp index e379d99..732038f 100644 --- a/core-buffer/test/main.cpp +++ b/core-buffer/test/main.cpp @@ -1,6 +1,7 @@ #include "gtest/gtest.h" #include "test_buffer_utils.cpp" #include "test_bitshuffle.cpp" +#include "test_RamBuffer.cpp" using namespace std; diff --git a/core-buffer/test/test_RamBuffer.cpp b/core-buffer/test/test_RamBuffer.cpp new file mode 100644 index 0000000..d01a328 --- /dev/null +++ b/core-buffer/test/test_RamBuffer.cpp @@ -0,0 +1,29 @@ +#include "gtest/gtest.h" +#include "RamBuffer.hpp" + +using namespace std; +using namespace buffer_config; + +TEST(RamBuffer, simple_store) +{ + const int n_modules = 3; + RamBuffer buffer("test_detector", n_modules, 10); + + ModuleFrame frame_meta; + frame_meta.pulse_id = 123523; + frame_meta.daq_rec = 1234; + frame_meta.frame_index = 12342300; + frame_meta.n_recv_packets = JF_N_PACKETS_PER_FRAME; + + auto frame_buffer = make_unique(MODULE_N_PIXELS); + + + for (size_t i = 0; i < MODULE_N_PIXELS; i++) { + frame_buffer[i] = i % 100; + } + for (int i_module=0; i_module Date: Fri, 4 Sep 2020 13:11:49 +0200 Subject: [PATCH 012/126] Initialize the struct in case no frame is good in a slot --- core-buffer/src/RamBuffer.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core-buffer/src/RamBuffer.cpp b/core-buffer/src/RamBuffer.cpp index 0f083ca..3ea0105 100644 --- a/core-buffer/src/RamBuffer.cpp +++ b/core-buffer/src/RamBuffer.cpp @@ -120,6 +120,12 @@ char* RamBuffer::read_image(const uint64_t pulse_id, image_meta.is_good_image = is_good_image; + if (!is_pulse_init) { + image_meta.pulse_id = pulse_id; + image_meta.frame_index = 0; + image_meta.daq_rec = 0; + } + return src_data; } From 25d65f99377b8dd0ff980bbd6d12f1acf785840d Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Fri, 4 Sep 2020 13:14:08 +0200 Subject: [PATCH 013/126] Improve on basic test --- core-buffer/test/test_RamBuffer.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/core-buffer/test/test_RamBuffer.cpp b/core-buffer/test/test_RamBuffer.cpp index d01a328..b4b8ddb 100644 --- a/core-buffer/test/test_RamBuffer.cpp +++ b/core-buffer/test/test_RamBuffer.cpp @@ -21,9 +21,17 @@ TEST(RamBuffer, simple_store) for (size_t i = 0; i < MODULE_N_PIXELS; i++) { frame_buffer[i] = i % 100; } + for (int i_module=0; i_module Date: Fri, 4 Sep 2020 13:49:22 +0200 Subject: [PATCH 014/126] Change start scripts for new sf_buffer --- scripts/JF01-buffer-worker.sh | 3 ++- scripts/JF02-buffer-worker.sh | 3 ++- scripts/JF06-buffer-worker.sh | 3 ++- scripts/JF06_4M-buffer-worker.sh | 3 ++- scripts/JF07-buffer-worker.sh | 3 ++- scripts/JF13-buffer-worker.sh | 3 ++- 6 files changed, 12 insertions(+), 6 deletions(-) diff --git a/scripts/JF01-buffer-worker.sh b/scripts/JF01-buffer-worker.sh index b907af5..311471a 100644 --- a/scripts/JF01-buffer-worker.sh +++ b/scripts/JF01-buffer-worker.sh @@ -16,5 +16,6 @@ coreAssociatedBuffer=(12 12 12) initialUDPport=50010 port=$((${initialUDPport}+10#${M})) DETECTOR=JF01T03V01 +N_MODULES=3 -taskset -c ${coreAssociatedBuffer[10#${M}]} /usr/bin/sf_buffer ${DETECTOR} M${M} ${port} /gpfs/photonics/swissfel/buffer/${DETECTOR} ${M} +taskset -c ${coreAssociatedBuffer[10#${M}]} /usr/bin/sf_buffer ${DETECTOR} ${N_MODULES} M${M} ${port} /gpfs/photonics/swissfel/buffer/${DETECTOR} ${M} diff --git a/scripts/JF02-buffer-worker.sh b/scripts/JF02-buffer-worker.sh index 226db66..a9fc4c0 100644 --- a/scripts/JF02-buffer-worker.sh +++ b/scripts/JF02-buffer-worker.sh @@ -16,5 +16,6 @@ coreAssociatedBuffer=(39 39 39 40 40 40 41 41 41) initialUDPport=50020 port=$((${initialUDPport}+10#${M})) DETECTOR=JF02T09V02 +N_MODULES=9 -taskset -c ${coreAssociatedBuffer[10#${M}]} /usr/bin/sf_buffer ${DETECTOR} M${M} ${port} /gpfs/photonics/swissfel/buffer/${DETECTOR} ${M} +taskset -c ${coreAssociatedBuffer[10#${M}]} /usr/bin/sf_buffer ${DETECTOR} ${N_MODULES} M${M} ${port} /gpfs/photonics/swissfel/buffer/${DETECTOR} ${M} diff --git a/scripts/JF06-buffer-worker.sh b/scripts/JF06-buffer-worker.sh index dd9c721..731ac88 100644 --- a/scripts/JF06-buffer-worker.sh +++ b/scripts/JF06-buffer-worker.sh @@ -16,5 +16,6 @@ coreAssociatedBuffer=(22 22 23 23 24 24 25 25 26 26 27 27 28 28 29 29 30 30 31 3 initialUDPport=50060 port=$((${initialUDPport}+10#${M})) DETECTOR=JF06T32V02 +N_MODULES=32 -taskset -c ${coreAssociatedBuffer[10#${M}]} /usr/bin/sf_buffer ${DETECTOR} M${M} ${port} /gpfs/photonics/swissfel/buffer/${DETECTOR} ${M} +taskset -c ${coreAssociatedBuffer[10#${M}]} /usr/bin/sf_buffer ${DETECTOR} ${N_MODULES} M${M} ${port} /gpfs/photonics/swissfel/buffer/${DETECTOR} ${M} diff --git a/scripts/JF06_4M-buffer-worker.sh b/scripts/JF06_4M-buffer-worker.sh index c67c0e5..6a6e924 100644 --- a/scripts/JF06_4M-buffer-worker.sh +++ b/scripts/JF06_4M-buffer-worker.sh @@ -16,5 +16,6 @@ coreAssociatedBuffer=(22 23 24 25 26 27 28 29) initialUDPport=50060 port=$((${initialUDPport}+10#${M})) DETECTOR=JF06T08V01 +N_MODULES=8 -taskset -c ${coreAssociatedBuffer[10#${M}]} /usr/bin/sf_buffer ${DETECTOR} M${M} ${port} /gpfs/photonics/swissfel/buffer/${DETECTOR} ${M} +taskset -c ${coreAssociatedBuffer[10#${M}]} /usr/bin/sf_buffer ${DETECTOR} ${N_MODULES} M${M} ${port} /gpfs/photonics/swissfel/buffer/${DETECTOR} ${M} diff --git a/scripts/JF07-buffer-worker.sh b/scripts/JF07-buffer-worker.sh index 5d744e1..2698a36 100644 --- a/scripts/JF07-buffer-worker.sh +++ b/scripts/JF07-buffer-worker.sh @@ -23,5 +23,6 @@ coreAssociatedBuffer=(4 4 4 4 5 5 5 5 6 6 6 6 7 7 7 7 8 8 8 8 9 9 9 9 10 10 10 1 initialUDPport=50100 port=$((${initialUDPport}+10#${M})) DETECTOR=JF07T32V01 +N_MODULES=32 -taskset -c ${coreAssociatedBuffer[10#${M}]} /usr/bin/sf_buffer ${DETECTOR} M${M} ${port} /gpfs/photonics/swissfel/buffer/${DETECTOR} ${M} +taskset -c ${coreAssociatedBuffer[10#${M}]} /usr/bin/sf_buffer ${DETECTOR} ${N_MODULES} M${M} ${port} /gpfs/photonics/swissfel/buffer/${DETECTOR} ${M} diff --git a/scripts/JF13-buffer-worker.sh b/scripts/JF13-buffer-worker.sh index c269179..21fff2a 100644 --- a/scripts/JF13-buffer-worker.sh +++ b/scripts/JF13-buffer-worker.sh @@ -16,5 +16,6 @@ coreAssociatedBuffer=(13) initialUDPport=50190 port=$((${initialUDPport}+10#${M})) DETECTOR=JF13T01V01 +N_MODULES=1 -taskset -c ${coreAssociatedBuffer[10#${M}]} /usr/bin/sf_buffer ${DETECTOR} M${M} ${port} /gpfs/photonics/swissfel/buffer/${DETECTOR} ${M} +taskset -c ${coreAssociatedBuffer[10#${M}]} /usr/bin/sf_buffer ${DETECTOR} ${N_MODULES} M${M} ${port} /gpfs/photonics/swissfel/buffer/${DETECTOR} ${M} From befeb93d784c928b43be193a5ed87c30a76cea52 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Fri, 4 Sep 2020 13:49:51 +0200 Subject: [PATCH 015/126] Refactor buffer to use RamBUffer --- sf-buffer/CMakeLists.txt | 3 ++- sf-buffer/src/main.cpp | 57 ++++++++++------------------------------ 2 files changed, 16 insertions(+), 44 deletions(-) diff --git a/sf-buffer/CMakeLists.txt b/sf-buffer/CMakeLists.txt index 5063955..bbd2cfe 100644 --- a/sf-buffer/CMakeLists.txt +++ b/sf-buffer/CMakeLists.txt @@ -11,7 +11,8 @@ add_executable(sf-buffer src/main.cpp) set_target_properties(sf-buffer PROPERTIES OUTPUT_NAME sf_buffer) target_link_libraries(sf-buffer sf-buffer-lib - zmq) + zmq + rt) enable_testing() add_subdirectory(test/) diff --git a/sf-buffer/src/main.cpp b/sf-buffer/src/main.cpp index a82ff25..0e39832 100644 --- a/sf-buffer/src/main.cpp +++ b/sf-buffer/src/main.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include "formats.hpp" #include "buffer_config.hpp" @@ -46,11 +47,12 @@ int main (int argc, char *argv[]) { if (argc != 6) { cout << endl; - cout << "Usage: sf_buffer [detector_name] [device_name]"; + cout << "Usage: sf_buffer [detector_name] [n_modules] [device_name]"; cout << " [udp_port] [root_folder] [source_id]"; cout << endl; cout << "\tdetector_name: Detector name, example JF07T32V01" << endl; - cout << "\tdevice_name: Name to write to disk."; + cout << "\tn_modules: Number of modules in the detector." << endl; + cout << "\tdevice_name: Name to write to disk." << endl; cout << "\tudp_port: UDP port to connect to." << endl; cout << "\troot_folder: FS root folder." << endl; cout << "\tsource_id: ID of the source for live stream." << endl; @@ -60,10 +62,11 @@ int main (int argc, char *argv[]) { } string detector_name = string(argv[1]); - string device_name = string(argv[2]); - int udp_port = atoi(argv[3]); - string root_folder = string(argv[4]); - int source_id = atoi(argv[5]); + int n_modules = atoi(argv[2]); + string device_name = string(argv[3]); + int udp_port = atoi(argv[4]); + string root_folder = string(argv[5]); + int source_id = atoi(argv[6]); uint64_t stats_counter(0); uint64_t n_missed_packets = 0; @@ -71,45 +74,22 @@ int main (int argc, char *argv[]) { BufferBinaryWriter writer(root_folder, device_name); FrameUdpReceiver receiver(udp_port, source_id); + RamBuffer buffer(detector_name, n_modules); auto binary_buffer = new BufferBinaryFormat(); auto socket = get_live_stream_socket(detector_name, source_id); - size_t write_total_us = 0; - size_t write_max_us = 0; - size_t send_total_us = 0; - size_t send_max_us = 0; - while (true) { auto pulse_id = receiver.get_frame_from_udp( binary_buffer->metadata, binary_buffer->data); - auto start_time = steady_clock::now(); - writer.write(pulse_id, binary_buffer); - auto write_end_time = steady_clock::now(); - size_t write_us_duration = duration_cast( - write_end_time-start_time).count(); + buffer.write_frame(&(binary_buffer->metadata), + &(binary_buffer->data[0])); - start_time = steady_clock::now(); - - zmq_send(socket, &(binary_buffer->metadata), sizeof(ModuleFrame), - ZMQ_SNDMORE); - zmq_send(socket, binary_buffer->data, MODULE_N_BYTES, 0); - - auto send_end_time = steady_clock::now(); - size_t send_us_duration = duration_cast( - send_end_time-start_time).count(); - - // TODO: Make real statistics, please. - stats_counter++; - write_total_us += write_us_duration; - send_total_us += send_us_duration; - - write_max_us = max(write_max_us, write_us_duration); - send_max_us = max(send_max_us, send_us_duration); + zmq_send(socket, &pulse_id, sizeof(pulse_id), 0); if (binary_buffer->metadata.n_recv_packets < JF_N_PACKETS_PER_FRAME) { n_missed_packets += JF_N_PACKETS_PER_FRAME - @@ -117,25 +97,16 @@ int main (int argc, char *argv[]) { n_corrupted_frames++; } + stats_counter++; if (stats_counter == STATS_MODULO) { cout << "sf_buffer:device_name " << device_name; cout << " sf_buffer:n_missed_packets " << n_missed_packets; cout << " sf_buffer:n_corrupted_frames " << n_corrupted_frames; - - cout << " sf_buffer:write_total_us " << write_total_us/STATS_MODULO; - cout << " sf_buffer:write_max_us " << write_max_us; - cout << " sf_buffer:send_total_us " << send_total_us/STATS_MODULO; - cout << " sf_buffer:send_max_us " << send_max_us; cout << endl; stats_counter = 0; n_missed_packets = 0; n_corrupted_frames = 0; - - write_total_us = 0; - write_max_us = 0; - send_total_us = 0; - send_max_us = 0; } } } From ac58965758a8c8bf6ae93f03c9ea765f08128389 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Fri, 4 Sep 2020 17:00:01 +0200 Subject: [PATCH 016/126] Adapt Live sender to RamBuffer --- sf-stream/include/ZmqLiveSender.hpp | 2 +- sf-stream/src/ZmqLiveSender.cpp | 40 +++++------------------------ 2 files changed, 8 insertions(+), 34 deletions(-) diff --git a/sf-stream/include/ZmqLiveSender.hpp b/sf-stream/include/ZmqLiveSender.hpp index ee197d5..5e38aba 100644 --- a/sf-stream/include/ZmqLiveSender.hpp +++ b/sf-stream/include/ZmqLiveSender.hpp @@ -38,7 +38,7 @@ public: const LiveStreamConfig& config); ~ZmqLiveSender(); - void send(const ModuleFrameBuffer* meta, const char* data); + void send(const ImageMetadata* meta, const char* data); }; diff --git a/sf-stream/src/ZmqLiveSender.cpp b/sf-stream/src/ZmqLiveSender.cpp index 897c6c2..6e7b52c 100644 --- a/sf-stream/src/ZmqLiveSender.cpp +++ b/sf-stream/src/ZmqLiveSender.cpp @@ -88,7 +88,7 @@ ZmqLiveSender::~ZmqLiveSender() zmq_close(socket_live_); } -void ZmqLiveSender::send(const ModuleFrameBuffer *meta, const char *data) +void ZmqLiveSender::send(const ImageMetadata *meta, const char *data) { uint16_t data_empty [] = { 0, 0, 0, 0}; @@ -96,41 +96,15 @@ void ZmqLiveSender::send(const ModuleFrameBuffer *meta, const char *data) auto& header_alloc = header.GetAllocator(); string text_header; - uint64_t pulse_id = 0; - uint64_t frame_index = 0; - uint64_t daq_rec = 0; - bool is_good_frame = true; - - for (size_t i_module = 0; i_module < config_.n_modules; i_module++) { - // TODO: Place this tests in the appropriate spot. - auto& module_metadata = meta->module[i_module]; - if (i_module == 0) { - pulse_id = module_metadata.pulse_id; - frame_index = module_metadata.frame_index; - daq_rec = module_metadata.daq_rec; - - if (module_metadata.n_recv_packets != 128 ) is_good_frame = false; - } else { - if (module_metadata.pulse_id != pulse_id) is_good_frame = false; - - if (module_metadata.frame_index != frame_index) is_good_frame = false; - - if (module_metadata.daq_rec != daq_rec) is_good_frame = false; - - if (module_metadata.n_recv_packets != 128 ) is_good_frame = false; - } - } - - if(zmq_send(socket_pulse_, &pulse_id, sizeof(pulse_id), 0) == -1) { + if(zmq_send(socket_pulse_, &meta->pulse_id, sizeof(uint64_t), 0) == -1) { throw runtime_error(zmq_strerror(errno)); } // TODO: Here we need to send to streamvis and live analysis metadata(probably need to operate still on them) and data(not every frame) - - header.AddMember("frame", frame_index, header_alloc); - header.AddMember("is_good_frame", is_good_frame, header_alloc); - header.AddMember("daq_rec", daq_rec, header_alloc); - header.AddMember("pulse_id", pulse_id, header_alloc); + header.AddMember("frame", meta->frame_index, header_alloc); + header.AddMember("is_good_frame", meta->is_good_image, header_alloc); + header.AddMember("daq_rec", meta->daq_rec, header_alloc); + header.AddMember("pulse_id", meta->pulse_id, header_alloc); rapidjson::Value pedestal_file; pedestal_file.SetString(config_.PEDE_FILENAME.c_str(), header_alloc); @@ -144,7 +118,7 @@ void ZmqLiveSender::send(const ModuleFrameBuffer *meta, const char *data) rapidjson::Value run_name; run_name.SetString( - to_string(uint64_t(pulse_id/10000)*10000).c_str(), + to_string(uint64_t(meta->pulse_id/10000)*10000).c_str(), header_alloc); header.AddMember("run_name", run_name, header_alloc); From 1ddee279dbccb121c08ca833a31c3ff19d27228a Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Fri, 4 Sep 2020 17:15:13 +0200 Subject: [PATCH 017/126] Throw away useless metrics --- sf-stream/src/main.cpp | 29 +---------------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/sf-stream/src/main.cpp b/sf-stream/src/main.cpp index 07e0ac3..687495e 100644 --- a/sf-stream/src/main.cpp +++ b/sf-stream/src/main.cpp @@ -43,41 +43,18 @@ int main (int argc, char *argv[]) // TODO: Remove stats trash. int stats_counter = 0; - size_t read_total_us = 0; - size_t read_max_us = 0; - size_t send_total_us = 0; - size_t send_max_us = 0; while (true) { - auto start_time = steady_clock::now(); - - auto n_lost_pulses = receiver.get_next_image(meta, data); + auto n_lost_pulses = receiver.get_next_pulse_id(); if (n_lost_pulses > 0) { cout << "sf_stream:sync_lost_pulses " << n_lost_pulses << endl; } - auto end_time = steady_clock::now(); - size_t read_us_duration = duration_cast( - end_time - start_time).count(); - - start_time = steady_clock::now(); - sender.send(meta, data); - end_time = steady_clock::now(); - size_t send_us_duration = duration_cast( - end_time - start_time).count(); - - // TODO: Some poor statistics. stats_counter++; - read_total_us += read_us_duration; - send_total_us += send_us_duration; - - read_max_us = max(read_max_us, read_us_duration); - send_max_us = max(send_max_us, send_us_duration); - if (stats_counter == STATS_MODULO) { cout << "sf_stream:read_us " << read_total_us / STATS_MODULO; cout << " sf_stream:read_max_us " << read_max_us; @@ -86,10 +63,6 @@ int main (int argc, char *argv[]) cout << endl; stats_counter = 0; - read_total_us = 0; - read_max_us = 0; - send_total_us = 0; - send_max_us = 0; } } } From 639fe3790a782c45da96175dc722a5024274e8e3 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Fri, 18 Sep 2020 15:25:31 +0200 Subject: [PATCH 018/126] Refactor pulse_id receiver --- sf-stream/include/ZmqLiveReceiver.hpp | 33 ----- sf-stream/include/ZmqPulseReceiver.hpp | 29 +++++ sf-stream/include/stream_config.hpp | 4 +- sf-stream/src/ZmqLiveReceiver.cpp | 171 ------------------------- sf-stream/src/ZmqPulseReceiver.cpp | 130 +++++++++++++++++++ sf-stream/src/main.cpp | 4 +- 6 files changed, 164 insertions(+), 207 deletions(-) delete mode 100644 sf-stream/include/ZmqLiveReceiver.hpp create mode 100644 sf-stream/include/ZmqPulseReceiver.hpp delete mode 100644 sf-stream/src/ZmqLiveReceiver.cpp create mode 100644 sf-stream/src/ZmqPulseReceiver.cpp diff --git a/sf-stream/include/ZmqLiveReceiver.hpp b/sf-stream/include/ZmqLiveReceiver.hpp deleted file mode 100644 index c12e34e..0000000 --- a/sf-stream/include/ZmqLiveReceiver.hpp +++ /dev/null @@ -1,33 +0,0 @@ -#ifndef SF_DAQ_BUFFER_ZMQLIVERECEIVER_HPP -#define SF_DAQ_BUFFER_ZMQLIVERECEIVER_HPP - - -#include -#include -#include - -#include "formats.hpp" - -class ZmqLiveReceiver { - - const size_t n_modules_; - void* ctx_; - const std::string ipc_prefix_; - std::vector sockets_; - - void* connect_socket(size_t module_id); - void recv_single_module(void* socket, ModuleFrame* meta, char* data); - uint64_t align_modules(ModuleFrameBuffer *meta, char *data); - -public: - ZmqLiveReceiver(const size_t n_modules, - void* ctx, - const std::string& ipc_prefix); - - ~ZmqLiveReceiver(); - - uint64_t get_next_image(ModuleFrameBuffer* meta, char* data); -}; - - -#endif //SF_DAQ_BUFFER_ZMQLIVERECEIVER_HPP diff --git a/sf-stream/include/ZmqPulseReceiver.hpp b/sf-stream/include/ZmqPulseReceiver.hpp new file mode 100644 index 0000000..f0c177b --- /dev/null +++ b/sf-stream/include/ZmqPulseReceiver.hpp @@ -0,0 +1,29 @@ +#ifndef SF_DAQ_BUFFER_ZMQPULSERECEIVER_HPP +#define SF_DAQ_BUFFER_ZMQPULSERECEIVER_HPP + + +#include +#include +#include + +#include "formats.hpp" + +class ZmqPulseReceiver { + + const std::vector ipc_urls_; + const int n_modules_; + void* ctx_; + + std::vector sockets_; + + void* connect_socket(const std::string url); + +public: + ZmqPulseReceiver(const std::vector& ipc_urls, void* ctx); + ~ZmqPulseReceiver(); + + uint64_t get_next_pulse_id(); +}; + + +#endif //SF_DAQ_BUFFER_ZMQPULSERECEIVER_HPP diff --git a/sf-stream/include/stream_config.hpp b/sf-stream/include/stream_config.hpp index ac28d86..614141c 100644 --- a/sf-stream/include/stream_config.hpp +++ b/sf-stream/include/stream_config.hpp @@ -7,9 +7,11 @@ namespace stream_config // Size of buffer between the receiving and sending part. const int STREAM_FASTQUEUE_SLOTS = 5; // If the modules are offset more than 1000 pulses, crush. - const uint64_t PULSE_OFFSET_LIMIT = 1000; + const uint64_t PULSE_OFFSET_LIMIT = 100; // SNDHWM for live processing socket. const int PROCESSING_ZMQ_SNDHWM = 10; // Keep the last second of pulses in the buffer. const int PULSE_ZMQ_SNDHWM = 100; + // Number of times we try to re-sync in case of failure. + const int SYNC_RETRY_LIMIT = 3; } diff --git a/sf-stream/src/ZmqLiveReceiver.cpp b/sf-stream/src/ZmqLiveReceiver.cpp deleted file mode 100644 index 5ac915e..0000000 --- a/sf-stream/src/ZmqLiveReceiver.cpp +++ /dev/null @@ -1,171 +0,0 @@ -#include "ZmqLiveReceiver.hpp" - -#include -#include -#include -#include - -#include "buffer_config.hpp" -#include "stream_config.hpp" - -using namespace std; -using namespace chrono; -using namespace buffer_config; -using namespace stream_config; - - -ZmqLiveReceiver::ZmqLiveReceiver( - const size_t n_modules, - void* ctx, - const std::string &ipc_prefix) : - n_modules_(n_modules), - ctx_(ctx), - ipc_prefix_(ipc_prefix), - sockets_(n_modules) -{ - for (size_t i = 0; i < n_modules_; i++) { - sockets_[i] = connect_socket(i); - } -} - -ZmqLiveReceiver::~ZmqLiveReceiver() -{ - for (auto& socket:sockets_) { - zmq_close(socket); - } -} - -void* ZmqLiveReceiver::connect_socket(size_t module_id) -{ - void* socket = zmq_socket(ctx_, ZMQ_SUB); - if (socket == nullptr) { - throw runtime_error(zmq_strerror(errno)); - } - - int rcvhwm = STREAM_RCVHWM; - if (zmq_setsockopt(socket, ZMQ_RCVHWM, &rcvhwm, sizeof(rcvhwm)) != 0) { - throw runtime_error(zmq_strerror(errno)); - } - - int linger = 0; - if (zmq_setsockopt(socket, ZMQ_LINGER, &linger, sizeof(linger)) != 0) { - throw runtime_error(zmq_strerror(errno)); - } - - stringstream ipc_addr; - ipc_addr << ipc_prefix_ << module_id; - const auto ipc = ipc_addr.str(); - - if (zmq_connect(socket, ipc.c_str()) != 0) { - throw runtime_error(zmq_strerror(errno)); - } - - if (zmq_setsockopt(socket, ZMQ_SUBSCRIBE, "", 0) != 0) { - throw runtime_error(zmq_strerror(errno)); - } - - return socket; -} - -void ZmqLiveReceiver::recv_single_module( - void* socket, ModuleFrame* meta, char* data) -{ - auto n_bytes_meta = zmq_recv(socket, meta, sizeof(ModuleFrame), 0); - - if (n_bytes_meta == -1) { - throw runtime_error(zmq_strerror(errno)); - } - if (n_bytes_meta != sizeof(ModuleFrame)) { - throw runtime_error("Stream header of wrong size."); - } - if (meta->pulse_id == 0) { - throw runtime_error("Received invalid pulse_id=0."); - } - - auto n_bytes_frame = zmq_recv(socket, data, MODULE_N_BYTES, 0); - - if (n_bytes_frame == -1) { - throw runtime_error(zmq_strerror(errno)); - } - if (n_bytes_frame != MODULE_N_BYTES) { - throw runtime_error("Stream data of wrong size."); - } -} - -uint64_t ZmqLiveReceiver::align_modules(ModuleFrameBuffer *meta, char *data) -{ - uint64_t max_pulse_id = 0; - uint64_t min_pulse_id = numeric_limits::max(); - - // First pass - determine current min and max pulse_id. - for (size_t i_module = 0; i_module < n_modules_; i_module++) { - auto& module_meta = meta->module[i_module]; - - min_pulse_id = min(min_pulse_id, module_meta.pulse_id); - max_pulse_id = max(max_pulse_id, module_meta.pulse_id); - } - - auto max_diff = max_pulse_id - min_pulse_id; - if (max_diff > PULSE_OFFSET_LIMIT) { - stringstream err_msg; - - err_msg << "[ZmqLiveReceiver::align_modules]"; - err_msg << " PULSE_OFFSET_LIMIT exceeded."; - err_msg << " Modules out of sync for " << max_diff << " pulses."; - - for (size_t i_module = 0; i_module < n_modules_; i_module++) { - auto& module_meta = meta->module[i_module]; - - err_msg << " (" << module_meta.module_id << ", "; - err_msg << module_meta.pulse_id << "),"; - } - - err_msg << endl; - - throw runtime_error(err_msg.str()); - } - - // Second pass - align all receivers to max_pulse_id. - for (size_t i_module = 0; i_module < n_modules_; i_module++) { - auto& module_meta = meta->module[i_module]; - - while (module_meta.pulse_id < max_pulse_id) { - recv_single_module( - sockets_[i_module], - &module_meta, - data + (MODULE_N_BYTES * i_module)); - } - - if (module_meta.pulse_id != max_pulse_id) { - throw runtime_error("Cannot align pulse_ids."); - } - } - - return max_pulse_id - min_pulse_id; -} - -uint64_t ZmqLiveReceiver::get_next_image(ModuleFrameBuffer* meta, char* data) -{ - uint64_t frame_pulse_id; - bool sync_needed = false; - - for (size_t i_module = 0; i_module < n_modules_; i_module++) { - auto& module_meta = meta->module[i_module]; - - char* buffer = data + (MODULE_N_BYTES * i_module); - recv_single_module(sockets_[i_module], &module_meta, buffer); - - if (i_module == 0) { - frame_pulse_id = module_meta.pulse_id; - } else if (frame_pulse_id != module_meta.pulse_id) { - sync_needed = true; - } - } - - if (sync_needed) { - auto lost_pulses = align_modules(meta, data); - return lost_pulses; - } - - return 0; -} diff --git a/sf-stream/src/ZmqPulseReceiver.cpp b/sf-stream/src/ZmqPulseReceiver.cpp new file mode 100644 index 0000000..85abeab --- /dev/null +++ b/sf-stream/src/ZmqPulseReceiver.cpp @@ -0,0 +1,130 @@ +#include "ZmqPulseReceiver.hpp" + +#include +#include +#include +#include +#include + +#include "stream_config.hpp" + +using namespace std; +using namespace chrono; +using namespace buffer_config; +using namespace stream_config; + + +ZmqPulseReceiver::ZmqPulseReceiver( + const vector& ipc_urls, + void* ctx) : + ipc_urls_(ipc_urls), + n_modules_(ipc_urls_.size()), + ctx_(ctx) +{ + sockets_.reserve(ipc_urls_.size()); + + for (const auto& url : ipc_urls_) { + sockets_.push_back(connect_socket(url)); + } +} + +ZmqPulseReceiver::~ZmqPulseReceiver() +{ + for (auto& socket:sockets_) { + zmq_close(socket); + } +} + +void* ZmqPulseReceiver::connect_socket(const string url) +{ + void* socket = zmq_socket(ctx_, ZMQ_SUB); + if (socket == nullptr) { + throw runtime_error(zmq_strerror(errno)); + } + + int rcvhwm = STREAM_RCVHWM; + if (zmq_setsockopt(socket, ZMQ_RCVHWM, &rcvhwm, sizeof(rcvhwm)) != 0) { + throw runtime_error(zmq_strerror(errno)); + } + + int linger = 0; + if (zmq_setsockopt(socket, ZMQ_LINGER, &linger, sizeof(linger)) != 0) { + throw runtime_error(zmq_strerror(errno)); + } + + if (zmq_connect(socket, url.c_str()) != 0) { + throw runtime_error(zmq_strerror(errno)); + } + + if (zmq_setsockopt(socket, ZMQ_SUBSCRIBE, "", 0) != 0) { + throw runtime_error(zmq_strerror(errno)); + } + + return socket; +} + +uint64_t ZmqPulseReceiver::get_next_pulse_id() +{ + uint64_t pulses[n_modules_]; + + bool modules_in_sync = true; + for (int i = 0; i < n_modules_; i++) { + zmq_recv(sockets_[i], &pulses[i], sizeof(uint64_t), 0); + + if (pulses[0] != pulses[i]) { + modules_in_sync = false; + } + } + + if (modules_in_sync) { + return pulses[0]; + } + + for (int i_sync; i_sync < SYNC_RETRY_LIMIT; i_sync++) { + uint64_t min_pulse_id = 0; + uint64_t max_pulse_id = numeric_limits::max(); + + for (int i = 0; i < n_modules_; i++) { + min_pulse_id = min(min_pulse_id, pulses[i]); + max_pulse_id = max(max_pulse_id, pulses[i]); + } + + auto max_diff = max_pulse_id - min_pulse_id; + if (max_diff > PULSE_OFFSET_LIMIT) { + stringstream err_msg; + err_msg << "[ZmqLiveReceiver::get_next_pulse_id]"; + err_msg << " PULSE_OFFSET_LIMIT exceeded."; + err_msg << " max_diff=" << max_diff << " pulses."; + + for (int i = 0; i < n_modules_; i++) { + err_msg << " (" << ipc_urls_[i] << ", "; + err_msg << pulses[i] << "),"; + } + err_msg << endl; + + throw runtime_error(err_msg.str()); + } + + modules_in_sync = true; + for (int i = 0; i < n_modules_; i++) { + while (pulses[i] < max_pulse_id) { + zmq_recv(sockets_[i], &pulses[i], sizeof(uint64_t), 0); + } + + if (pulses[i] != max_pulse_id) { + modules_in_sync = false; + } + } + + if (modules_in_sync) { + return pulses[0]; + } + } + + stringstream err_msg; + err_msg << "[ZmqLiveReceiver::get_next_pulse_id]"; + err_msg << " SYNC_RETRY_LIMIT exceeded."; + err_msg << endl; + + throw runtime_error(err_msg.str()); +} diff --git a/sf-stream/src/main.cpp b/sf-stream/src/main.cpp index 687495e..796c2a5 100644 --- a/sf-stream/src/main.cpp +++ b/sf-stream/src/main.cpp @@ -7,7 +7,7 @@ #include "buffer_config.hpp" #include "stream_config.hpp" #include "ZmqLiveSender.hpp" -#include "ZmqLiveReceiver.hpp" +#include "ZmqPulseReceiver.hpp" using namespace std; using namespace chrono; @@ -38,7 +38,7 @@ int main (int argc, char *argv[]) auto ctx = zmq_ctx_new(); zmq_ctx_set (ctx, ZMQ_IO_THREADS, STREAM_ZMQ_IO_THREADS); - ZmqLiveReceiver receiver(config.n_modules, ctx, RECV_IPC_URL); + ZmqPulseReceiver receiver(config.n_modules, ctx, RECV_IPC_URL); ZmqLiveSender sender(ctx, config); // TODO: Remove stats trash. From f84f4fbcdbcd8cb0dd21d39d0a3ba89a0b0759f2 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Fri, 18 Sep 2020 15:29:34 +0200 Subject: [PATCH 019/126] Make function const --- sf-stream/include/ZmqPulseReceiver.hpp | 2 +- sf-stream/src/ZmqPulseReceiver.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sf-stream/include/ZmqPulseReceiver.hpp b/sf-stream/include/ZmqPulseReceiver.hpp index f0c177b..1590283 100644 --- a/sf-stream/include/ZmqPulseReceiver.hpp +++ b/sf-stream/include/ZmqPulseReceiver.hpp @@ -22,7 +22,7 @@ public: ZmqPulseReceiver(const std::vector& ipc_urls, void* ctx); ~ZmqPulseReceiver(); - uint64_t get_next_pulse_id(); + uint64_t get_next_pulse_id() const; }; diff --git a/sf-stream/src/ZmqPulseReceiver.cpp b/sf-stream/src/ZmqPulseReceiver.cpp index 85abeab..ed94dd8 100644 --- a/sf-stream/src/ZmqPulseReceiver.cpp +++ b/sf-stream/src/ZmqPulseReceiver.cpp @@ -63,7 +63,7 @@ void* ZmqPulseReceiver::connect_socket(const string url) return socket; } -uint64_t ZmqPulseReceiver::get_next_pulse_id() +uint64_t ZmqPulseReceiver::get_next_pulse_id() const { uint64_t pulses[n_modules_]; From 5a22344e841c4efd129f7a02af78a11f7b221061 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Mon, 21 Sep 2020 09:45:13 +0200 Subject: [PATCH 020/126] Externalize the socket creation --- sf-stream/src/main.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/sf-stream/src/main.cpp b/sf-stream/src/main.cpp index 796c2a5..367f875 100644 --- a/sf-stream/src/main.cpp +++ b/sf-stream/src/main.cpp @@ -38,7 +38,13 @@ int main (int argc, char *argv[]) auto ctx = zmq_ctx_new(); zmq_ctx_set (ctx, ZMQ_IO_THREADS, STREAM_ZMQ_IO_THREADS); - ZmqPulseReceiver receiver(config.n_modules, ctx, RECV_IPC_URL); + // TODO: This should be passed to the service and not calculated here. + vector ipc_urls; + for (int i=0; i Date: Mon, 21 Sep 2020 09:48:38 +0200 Subject: [PATCH 021/126] Add reminder for improvement --- core-buffer/src/RamBuffer.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core-buffer/src/RamBuffer.cpp b/core-buffer/src/RamBuffer.cpp index 3ea0105..939c6a4 100644 --- a/core-buffer/src/RamBuffer.cpp +++ b/core-buffer/src/RamBuffer.cpp @@ -106,6 +106,7 @@ char* RamBuffer::read_image(const uint64_t pulse_id, if (is_good_image) { if (frame_meta->pulse_id != image_meta.pulse_id) { is_good_image = false; + // TODO: Add some diagnostics in case this happens. } if (frame_meta->frame_index != image_meta.frame_index) { @@ -121,7 +122,7 @@ char* RamBuffer::read_image(const uint64_t pulse_id, image_meta.is_good_image = is_good_image; if (!is_pulse_init) { - image_meta.pulse_id = pulse_id; + image_meta.pulse_id = 0; image_meta.frame_index = 0; image_meta.daq_rec = 0; } From 3e097a5cd867ce9340c72a39f35f6205cd3a55bf Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Mon, 21 Sep 2020 09:56:06 +0200 Subject: [PATCH 022/126] Adjust stream sender to use reference --- sf-stream/include/ZmqLiveSender.hpp | 2 +- sf-stream/src/ZmqLiveSender.cpp | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/sf-stream/include/ZmqLiveSender.hpp b/sf-stream/include/ZmqLiveSender.hpp index 5e38aba..96e5424 100644 --- a/sf-stream/include/ZmqLiveSender.hpp +++ b/sf-stream/include/ZmqLiveSender.hpp @@ -38,7 +38,7 @@ public: const LiveStreamConfig& config); ~ZmqLiveSender(); - void send(const ImageMetadata* meta, const char* data); + void send(const ImageMetadata& meta, const char* data); }; diff --git a/sf-stream/src/ZmqLiveSender.cpp b/sf-stream/src/ZmqLiveSender.cpp index 6e7b52c..d71485e 100644 --- a/sf-stream/src/ZmqLiveSender.cpp +++ b/sf-stream/src/ZmqLiveSender.cpp @@ -88,7 +88,7 @@ ZmqLiveSender::~ZmqLiveSender() zmq_close(socket_live_); } -void ZmqLiveSender::send(const ImageMetadata *meta, const char *data) +void ZmqLiveSender::send(const ImageMetadata& meta, const char *data) { uint16_t data_empty [] = { 0, 0, 0, 0}; @@ -96,15 +96,15 @@ void ZmqLiveSender::send(const ImageMetadata *meta, const char *data) auto& header_alloc = header.GetAllocator(); string text_header; - if(zmq_send(socket_pulse_, &meta->pulse_id, sizeof(uint64_t), 0) == -1) { + if(zmq_send(socket_pulse_, &meta.pulse_id, sizeof(uint64_t), 0) == -1) { throw runtime_error(zmq_strerror(errno)); } // TODO: Here we need to send to streamvis and live analysis metadata(probably need to operate still on them) and data(not every frame) - header.AddMember("frame", meta->frame_index, header_alloc); - header.AddMember("is_good_frame", meta->is_good_image, header_alloc); - header.AddMember("daq_rec", meta->daq_rec, header_alloc); - header.AddMember("pulse_id", meta->pulse_id, header_alloc); + header.AddMember("frame", meta.frame_index, header_alloc); + header.AddMember("is_good_frame", meta.is_good_image, header_alloc); + header.AddMember("daq_rec", meta.daq_rec, header_alloc); + header.AddMember("pulse_id", meta.pulse_id, header_alloc); rapidjson::Value pedestal_file; pedestal_file.SetString(config_.PEDE_FILENAME.c_str(), header_alloc); @@ -118,7 +118,7 @@ void ZmqLiveSender::send(const ImageMetadata *meta, const char *data) rapidjson::Value run_name; run_name.SetString( - to_string(uint64_t(meta->pulse_id/10000)*10000).c_str(), + to_string(uint64_t(meta.pulse_id/10000)*10000).c_str(), header_alloc); header.AddMember("run_name", run_name, header_alloc); From 5dbaf3319f1ef7335bc1395e19a096b8dd11ada9 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Mon, 21 Sep 2020 10:16:05 +0200 Subject: [PATCH 023/126] Add sync attempt output counter --- sf-stream/src/ZmqPulseReceiver.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sf-stream/src/ZmqPulseReceiver.cpp b/sf-stream/src/ZmqPulseReceiver.cpp index ed94dd8..e21a94b 100644 --- a/sf-stream/src/ZmqPulseReceiver.cpp +++ b/sf-stream/src/ZmqPulseReceiver.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include "stream_config.hpp" @@ -81,6 +82,8 @@ uint64_t ZmqPulseReceiver::get_next_pulse_id() const } for (int i_sync; i_sync < SYNC_RETRY_LIMIT; i_sync++) { + cout << "Sync attempt " << i_sync << endl; + uint64_t min_pulse_id = 0; uint64_t max_pulse_id = numeric_limits::max(); From 53a2690567b5811318a78cdf3e093955579d8121 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Mon, 21 Sep 2020 10:16:27 +0200 Subject: [PATCH 024/126] Improve statistics and logging for sf_stream --- sf-stream/src/main.cpp | 45 +++++++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/sf-stream/src/main.cpp b/sf-stream/src/main.cpp index 367f875..0c3d500 100644 --- a/sf-stream/src/main.cpp +++ b/sf-stream/src/main.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include "buffer_config.hpp" #include "stream_config.hpp" @@ -32,9 +33,6 @@ int main (int argc, char *argv[]) auto config = read_json_config(string(argv[1])); string RECV_IPC_URL = BUFFER_LIVE_IPC_URL + config.DETECTOR_NAME + "-"; - ModuleFrameBuffer* meta = new ModuleFrameBuffer(); - char* data = new char[config.n_modules * MODULE_N_BYTES]; - auto ctx = zmq_ctx_new(); zmq_ctx_set (ctx, ZMQ_IO_THREADS, STREAM_ZMQ_IO_THREADS); @@ -45,30 +43,41 @@ int main (int argc, char *argv[]) } ZmqPulseReceiver receiver(ipc_urls, ctx); + RamBuffer ram_buffer(config.DETECTOR_NAME, config.n_modules); ZmqLiveSender sender(ctx, config); // TODO: Remove stats trash. - int stats_counter = 0; + uint64_t last_pulse_id = 0; + uint64_t last_pulse_id_range = 0; + uint16_t n_good_images = 0; + ImageMetadata meta; while (true) { - - auto n_lost_pulses = receiver.get_next_pulse_id(); - - if (n_lost_pulses > 0) { - cout << "sf_stream:sync_lost_pulses " << n_lost_pulses << endl; - } + auto pulse_id = receiver.get_next_pulse_id(); + char* data = ram_buffer.read_image(pulse_id, meta); sender.send(meta, data); - stats_counter++; - if (stats_counter == STATS_MODULO) { - cout << "sf_stream:read_us " << read_total_us / STATS_MODULO; - cout << " sf_stream:read_max_us " << read_max_us; - cout << " sf_stream:send_us " << send_total_us / STATS_MODULO; - cout << " sf_stream:send_max_us " << send_max_us; - cout << endl; + // TODO: This logic works only at 100Hz. Fix it systematically. + uint64_t sync_lost_pulses = pulse_id - last_pulse_id; + if (last_pulse_id > 0 && sync_lost_pulses > 1) { + cout << "sf_stream:sync_lost_pulses " << sync_lost_pulses << endl; + } + last_pulse_id = pulse_id; - stats_counter = 0; + uint64_t curr_pulse_id_range = pulse_id / 10000; + if (last_pulse_id_range != curr_pulse_id_range) { + if (last_pulse_id_range > 0) { + cout << "sf_stream:n_good_images " << n_good_images; + cout << endl; + } + + last_pulse_id_range = curr_pulse_id_range; + n_good_images = 0; + } + + if (meta.is_good_image) { + n_good_images++; } } } From d230eeec004effddf34c7d34ea03cd457a8dcec8 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Mon, 21 Sep 2020 10:36:49 +0200 Subject: [PATCH 025/126] Add link library --- sf-stream/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sf-stream/CMakeLists.txt b/sf-stream/CMakeLists.txt index fafe832..ada511a 100644 --- a/sf-stream/CMakeLists.txt +++ b/sf-stream/CMakeLists.txt @@ -12,7 +12,8 @@ set_target_properties(sf-stream PROPERTIES OUTPUT_NAME sf_stream) target_link_libraries(sf-stream sf-stream-lib zmq - pthread) + pthread + rt) enable_testing() add_subdirectory(test/) From 6f3b186215b49903ae1b572651b4dd311de3d9b7 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Mon, 21 Sep 2020 11:00:08 +0200 Subject: [PATCH 026/126] Start refactoring buffer writer --- jf-live-daq/src/main.cpp | 49 -------- .../CMakeLists.txt | 0 sf-buffer-writer/src/main.cpp | 112 ++++++++++++++++++ .../test/CMakeLists.txt | 0 .../test/main.cpp | 0 5 files changed, 112 insertions(+), 49 deletions(-) delete mode 100644 jf-live-daq/src/main.cpp rename {jf-live-daq => sf-buffer-writer}/CMakeLists.txt (100%) create mode 100644 sf-buffer-writer/src/main.cpp rename {jf-live-daq => sf-buffer-writer}/test/CMakeLists.txt (100%) rename {jf-live-daq => sf-buffer-writer}/test/main.cpp (100%) diff --git a/jf-live-daq/src/main.cpp b/jf-live-daq/src/main.cpp deleted file mode 100644 index 6069a0b..0000000 --- a/jf-live-daq/src/main.cpp +++ /dev/null @@ -1,49 +0,0 @@ -#include -#include - -void receive() -{ - -} - -void assemble() -{ - -} - -void write() -{ - -} - -int main(int argc, char** argv) -{ - // Initialize the MPI environment - MPI_Init(NULL, NULL); - - // Get the number of processes - int world_size; - MPI_Comm_size(MPI_COMM_WORLD, &world_size); - - // Get the rank of the process - int world_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); - - // Get the name of the processor - char processor_name[MPI_MAX_PROCESSOR_NAME]; - int name_len; - MPI_Get_processor_name(processor_name, &name_len); - - const int n_modules = 16; - - if (world_rank == 0) { - assemble(); - } else if (world_rank <= n_modules) { - receive(); - } else { - write(); - } - - // Finalize the MPI environment. - MPI_Finalize(); -} diff --git a/jf-live-daq/CMakeLists.txt b/sf-buffer-writer/CMakeLists.txt similarity index 100% rename from jf-live-daq/CMakeLists.txt rename to sf-buffer-writer/CMakeLists.txt diff --git a/sf-buffer-writer/src/main.cpp b/sf-buffer-writer/src/main.cpp new file mode 100644 index 0000000..0e39832 --- /dev/null +++ b/sf-buffer-writer/src/main.cpp @@ -0,0 +1,112 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "formats.hpp" +#include "buffer_config.hpp" +#include "jungfrau.hpp" +#include "FrameUdpReceiver.hpp" +#include "BufferBinaryWriter.hpp" + +using namespace std; +using namespace chrono; +using namespace buffer_config; + +void* get_live_stream_socket(const string& detector_name, const int source_id) +{ + stringstream ipc_stream; + string LIVE_IPC_URL = BUFFER_LIVE_IPC_URL + detector_name + "-"; + ipc_stream << LIVE_IPC_URL << source_id; + const auto ipc_address = ipc_stream.str(); + + void* ctx = zmq_ctx_new(); + void* socket = zmq_socket(ctx, ZMQ_PUB); + + const int sndhwm = BUFFER_ZMQ_SNDHWM; + if (zmq_setsockopt(socket, ZMQ_SNDHWM, &sndhwm, sizeof(sndhwm)) != 0) { + throw runtime_error(zmq_strerror(errno)); + } + + const int linger = 0; + if (zmq_setsockopt(socket, ZMQ_LINGER, &linger, sizeof(linger)) != 0) { + throw runtime_error(zmq_strerror(errno)); + } + + if (zmq_bind(socket, ipc_address.c_str()) != 0) { + throw runtime_error(zmq_strerror(errno)); + } + + return socket; +} + +int main (int argc, char *argv[]) { + + if (argc != 6) { + cout << endl; + cout << "Usage: sf_buffer [detector_name] [n_modules] [device_name]"; + cout << " [udp_port] [root_folder] [source_id]"; + cout << endl; + cout << "\tdetector_name: Detector name, example JF07T32V01" << endl; + cout << "\tn_modules: Number of modules in the detector." << endl; + cout << "\tdevice_name: Name to write to disk." << endl; + cout << "\tudp_port: UDP port to connect to." << endl; + cout << "\troot_folder: FS root folder." << endl; + cout << "\tsource_id: ID of the source for live stream." << endl; + cout << endl; + + exit(-1); + } + + string detector_name = string(argv[1]); + int n_modules = atoi(argv[2]); + string device_name = string(argv[3]); + int udp_port = atoi(argv[4]); + string root_folder = string(argv[5]); + int source_id = atoi(argv[6]); + + uint64_t stats_counter(0); + uint64_t n_missed_packets = 0; + uint64_t n_corrupted_frames = 0; + + BufferBinaryWriter writer(root_folder, device_name); + FrameUdpReceiver receiver(udp_port, source_id); + RamBuffer buffer(detector_name, n_modules); + + auto binary_buffer = new BufferBinaryFormat(); + auto socket = get_live_stream_socket(detector_name, source_id); + + while (true) { + + auto pulse_id = receiver.get_frame_from_udp( + binary_buffer->metadata, binary_buffer->data); + + writer.write(pulse_id, binary_buffer); + + buffer.write_frame(&(binary_buffer->metadata), + &(binary_buffer->data[0])); + + zmq_send(socket, &pulse_id, sizeof(pulse_id), 0); + + if (binary_buffer->metadata.n_recv_packets < JF_N_PACKETS_PER_FRAME) { + n_missed_packets += JF_N_PACKETS_PER_FRAME - + binary_buffer->metadata.n_recv_packets; + n_corrupted_frames++; + } + + stats_counter++; + if (stats_counter == STATS_MODULO) { + cout << "sf_buffer:device_name " << device_name; + cout << " sf_buffer:n_missed_packets " << n_missed_packets; + cout << " sf_buffer:n_corrupted_frames " << n_corrupted_frames; + cout << endl; + + stats_counter = 0; + n_missed_packets = 0; + n_corrupted_frames = 0; + } + } +} diff --git a/jf-live-daq/test/CMakeLists.txt b/sf-buffer-writer/test/CMakeLists.txt similarity index 100% rename from jf-live-daq/test/CMakeLists.txt rename to sf-buffer-writer/test/CMakeLists.txt diff --git a/jf-live-daq/test/main.cpp b/sf-buffer-writer/test/main.cpp similarity index 100% rename from jf-live-daq/test/main.cpp rename to sf-buffer-writer/test/main.cpp From f883d9c298077be4b39041ad7fb864e0f5e32eaa Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Mon, 21 Sep 2020 11:01:23 +0200 Subject: [PATCH 027/126] Cleanup project from files moved to sf_broker --- CMakeLists.txt | 1 + core-broker/core-broker/__init__.py | 0 core-broker/core-broker/broker_client.py | 71 ------------ core-broker/core-broker/broker_debugger.py | 70 ------------ epics-writer/Readme.md | 34 ------ epics-writer/conda-recipe/build.sh | 3 - epics-writer/conda-recipe/meta.yaml | 20 ---- epics-writer/epics_writer/__init__.py | 0 epics-writer/epics_writer/start.py | 99 ----------------- epics-writer/epics_writer/writer.py | 120 --------------------- epics-writer/setup.py | 15 --- epics-writer/tests/__init__.py | 0 epics-writer/tests/channels.txt | 6 -- epics-writer/tests/test_download_data.py | 34 ------ 14 files changed, 1 insertion(+), 472 deletions(-) delete mode 100644 core-broker/core-broker/__init__.py delete mode 100644 core-broker/core-broker/broker_client.py delete mode 100644 core-broker/core-broker/broker_debugger.py delete mode 100644 epics-writer/Readme.md delete mode 100644 epics-writer/conda-recipe/build.sh delete mode 100644 epics-writer/conda-recipe/meta.yaml delete mode 100644 epics-writer/epics_writer/__init__.py delete mode 100644 epics-writer/epics_writer/start.py delete mode 100644 epics-writer/epics_writer/writer.py delete mode 100644 epics-writer/setup.py delete mode 100644 epics-writer/tests/__init__.py delete mode 100644 epics-writer/tests/channels.txt delete mode 100644 epics-writer/tests/test_download_data.py diff --git a/CMakeLists.txt b/CMakeLists.txt index d480f91..f67c4f0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -30,6 +30,7 @@ add_subdirectory( add_subdirectory("core-buffer") add_subdirectory("sf-buffer") +add_subdirectory("sf-buffer-writer") add_subdirectory("sf-stream") add_subdirectory("sf-writer") #add_subdirectory("jf-live-writer") diff --git a/core-broker/core-broker/__init__.py b/core-broker/core-broker/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/core-broker/core-broker/broker_client.py b/core-broker/core-broker/broker_client.py deleted file mode 100644 index 2b676fe..0000000 --- a/core-broker/core-broker/broker_client.py +++ /dev/null @@ -1,71 +0,0 @@ -import json - -from pika import BlockingConnection, ConnectionParameters, BasicProperties - - -class BrokerClient(object): - - REQUEST_EXCHANGE = "request" - STATUS_EXCHANGE = "status" - DEFAULT_BROKER_URL = "127.0.0.1" - - def __init__(self, broker_url=DEFAULT_BROKER_URL): - self.connection = BlockingConnection(ConnectionParameters(broker_url)) - self.channel = self.connection.channel() - - self.channel.exchange_declare(exchange=self.REQUEST_EXCHANGE, - exchange_type="topic") - - self.channel.exchange_declare(exchange=self.STATUS_EXCHANGE, - exchange_type="fanout") - - def close(self): - self.connection.close() - - def request_write(self, - output_prefix, - metadata=None, - detectors=None, - bsread_channels=None, - epics_pvs=None): - - routing_key = "." - - if detectors: - for detector in detectors: - routing_key += detector + "." - - if bsread_channels: - routing_key += "bsread" + "." - - if epics_pvs: - routing_key += "epics" + "." - - body_bytes = json.dumps({ - "output_prefix": output_prefix, - "metadata": metadata, - "detectors": detectors, - "bsread_channels": bsread_channels, - "epics_pvs": epics_pvs - }).encode() - - self.channel.basic_publish(exchange=self.REQUEST_EXCHANGE, - routing_key=routing_key, - body=body_bytes) - - status_header = { - "action": "write_request", - "source": "BrokerClient", - "routing_key": routing_key - } - - self.channel.basic_publish(exchange=self.STATUS_EXCHANGE, - properties=BasicProperties( - headers=status_header), - routing_key="", - body=body_bytes) - - -broker = BrokerClient() -broker.request_write("/tmp/test", epics_pvs=["test"]) -broker.close() diff --git a/core-broker/core-broker/broker_debugger.py b/core-broker/core-broker/broker_debugger.py deleted file mode 100644 index 52b504b..0000000 --- a/core-broker/core-broker/broker_debugger.py +++ /dev/null @@ -1,70 +0,0 @@ -from datetime import datetime -import json - -from pika import BlockingConnection, ConnectionParameters - -DEFAULT_BROKER_URL = "127.0.0.1" -STATUS_EXCHANGE = "status" - -COLOR_END_MARKER = '\x1b[0m' - - -def get_color_for_action(action): - - color_mapping = { - "write_request": "\x1b[34;1m", - "write_start": "\x1b[1;33;1m", - "write_finished": "\x1b[1;32;1m" - } - - return color_mapping.get(action, "") - - -def on_status(channel, method_frame, header_frame, body): - header = header_frame.headers - request = json.loads(body.decode()) - - action = header["action"] - source = header["source"] - - action_output = get_color_for_action(action) + action + COLOR_END_MARKER - time_output = datetime.utcnow().strftime("%Y%m%d-%H:%M:%S.%f") - - print("[%s] %s %s" % (time_output, action_output, source)) - print(request) - - -def connect_to_broker(broker_url): - connection = BlockingConnection(ConnectionParameters(broker_url)) - channel = connection.channel() - - channel.exchange_declare(exchange=STATUS_EXCHANGE, - exchange_type="fanout") - queue = channel.queue_declare(queue="", exclusive=True).method.queue - channel.queue_bind(queue=queue, - exchange=STATUS_EXCHANGE) - - channel.basic_consume(queue, on_status) - - try: - channel.start_consuming() - except KeyboardInterrupt: - channel.stop_consuming() - - -def main(): - import argparse - parser = argparse.ArgumentParser( - description="Connect and listen to broker events.") - - parser.add_argument('--broker_url', dest='broker_url', - default=DEFAULT_BROKER_URL, - help='RabbitMQ broker URL') - - args = parser.parse_args() - - connect_to_broker(broker_url=args.broker_url) - - -if __name__ == '__main__': - main() diff --git a/epics-writer/Readme.md b/epics-writer/Readme.md deleted file mode 100644 index 5f6ab51..0000000 --- a/epics-writer/Readme.md +++ /dev/null @@ -1,34 +0,0 @@ -# Overview - -Simple server to dump Epics Channel Access data to an HDF5 file. -The server gets an http callback from the Broker whenever there was an acquisition. - - -__Note: THIS IS/WAS A FRIDAY AFTERNOON HACK TO MAKE THE SWISSFEL DAQ WORK__ - - -The format of the request is as follows: -``` -{ - 'range': { - 'startPulseId': 100, - 'endPulseId': 120 - }, - - 'parameters': { - 'general/created': 'test', - 'general/user': 'tester', - 'general/process': 'test_process', - 'general/instrument': 'mac', - 'output_file': '/bla/test.h5'} -} - -``` - -Right now this server needs to run on the same server than the - -# Testing - -```bash -curl -XPUT -d '{"range":{"startPulseId": 7281433214, "endPulseId": 7281489688}, "parameters":{"output_file":"test.h5"}}' http://localhost:10200/notify -``` diff --git a/epics-writer/conda-recipe/build.sh b/epics-writer/conda-recipe/build.sh deleted file mode 100644 index d7a34f9..0000000 --- a/epics-writer/conda-recipe/build.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -$PYTHON setup.py install # Python command to install the script diff --git a/epics-writer/conda-recipe/meta.yaml b/epics-writer/conda-recipe/meta.yaml deleted file mode 100644 index 8b464c1..0000000 --- a/epics-writer/conda-recipe/meta.yaml +++ /dev/null @@ -1,20 +0,0 @@ -package: - name: epics-writer - version: 0.0.1 - -source: - path: .. - -build: - noarch: python - entry_points: - - epics-writer = epics_writer.start:main - -requirements: - build: - - python - run: - - python - - data_api >=0.7.6 - - requests - - pika diff --git a/epics-writer/epics_writer/__init__.py b/epics-writer/epics_writer/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/epics-writer/epics_writer/start.py b/epics-writer/epics_writer/start.py deleted file mode 100644 index 3647d6d..0000000 --- a/epics-writer/epics_writer/start.py +++ /dev/null @@ -1,99 +0,0 @@ -import json -from pika import BlockingConnection, ConnectionParameters, BasicProperties - -from epics_writer.writer import write_epics_pvs - - -DEFAULT_BROKER_URL = "127.0.0.1" -REQUEST_EXCHANGE = "request" -STATUS_EXCHANGE = "status" -QUEUE_NAME = "epics" -OUTPUT_FILE_SUFFIX = ".PVCHANNELS.h5" - - -def update_status(channel, body, action, file, message=None): - status_header = { - "action": action, - "source": "epics_writer", - "routing_key": QUEUE_NAME, - "file": file, - "message": message - } - - channel.basic_publish(exchange=STATUS_EXCHANGE, - properties=BasicProperties( - headers=status_header), - routing_key="", - body=body) - - -def on_message(channel, method_frame, header_frame, body): - - output_file = None - - try: - request = json.loads(body.decode()) - output_prefix = request["output_prefix"] - start_pulse_id = 1 - stop_pulse_id = 10 - metadata = request["metadata"] - epics_pvs = request["epics_pvs"] - - output_file = output_prefix + OUTPUT_FILE_SUFFIX - - update_status(channel, body, "write_start", output_file) - - write_epics_pvs(output_file=output_file, - start_pulse_id=start_pulse_id, - stop_pulse_id=stop_pulse_id, - metadata=metadata, - epics_pvs=epics_pvs) - - except Exception as e: - channel.basic_reject(delivery_tag=method_frame.delivery_tag, - requeue=False) - - update_status(channel, body, "write_rejected", output_file, str(e)) - - else: - channel.basic_ack(delivery_tag=method_frame.delivery_tag) - - update_status(channel, body, "write_finished", output_file) - - -def connect_to_broker(broker_url): - connection = BlockingConnection(ConnectionParameters(broker_url)) - channel = connection.channel() - - channel.exchange_declare(exchange=STATUS_EXCHANGE, - exchange_type="fanout") - channel.exchange_declare(exchange=REQUEST_EXCHANGE, - exchange_type="topic") - - channel.queue_declare(queue=QUEUE_NAME, auto_delete=True) - channel.queue_bind(queue=QUEUE_NAME, - exchange=REQUEST_EXCHANGE, - routing_key="*.%s.*" % QUEUE_NAME) - channel.basic_qos(prefetch_count=1) - channel.basic_consume(QUEUE_NAME, on_message) - - try: - channel.start_consuming() - except KeyboardInterrupt: - channel.stop_consuming() - - -def main(): - import argparse - parser = argparse.ArgumentParser(description='Epics HDF5 writer') - parser.add_argument('--broker_url', dest='broker_url', - default=DEFAULT_BROKER_URL, - help='RabbitMQ broker URL') - - args = parser.parse_args() - - connect_to_broker(broker_url=args.broker_url) - - -if __name__ == '__main__': - main() diff --git a/epics-writer/epics_writer/writer.py b/epics-writer/epics_writer/writer.py deleted file mode 100644 index fe9bc06..0000000 --- a/epics-writer/epics_writer/writer.py +++ /dev/null @@ -1,120 +0,0 @@ -import datetime -import time -import logging -import requests -import data_api - -logger = logging.getLogger(__name__) - -DATA_API_QUERY_URL = "https://data-api.psi.ch/sf/query" - - -def write_epics_pvs(output_file, start_pulse_id, stop_pulse_id, metadata, epics_pvs): - - start_date = get_pulse_id_date_mapping(start_pulse_id) - stop_date = get_pulse_id_date_mapping(stop_pulse_id) - - data = get_data(epics_pvs, start=start_date, stop=stop_date) - # TODO: Merge metadata to data. - - if data: - logger.info("Persist data to hdf5 file") - data_api.to_hdf5(data, output_file, overwrite=True, compression=None, shuffle=False) - else: - logger.error("No data retrieved") - open(output_file + "_NO_DATA", 'a').close() - - -def get_data(channel_list, start=None, stop=None, base_url=None): - logger.info("Requesting range %s to %s for channels: " % (start, stop, channel_list)) - - query = {"range": {"startDate": datetime.datetime.isoformat(start), - "endDate": datetime.datetime.isoformat(stop), - "startExpansion": True}, - "channels": channel_list, - "fields": ["pulseId", "globalSeconds", "globalDate", "value", - "eventCount"]} - logger.debug(query) - - response = requests.post(DATA_API_QUERY_URL, json=query) - - # Check for successful return of data - if response.status_code != 200: - logger.info("Data retrievali failed, sleep for another time and try") - - itry = 0 - while itry < 5: - itry += 1 - time.sleep(60) - response = requests.post(DATA_API_QUERY_URL, json=query) - if response.status_code == 200: - break - - logger.info("Data retrieval failed, post attempt %d" % itry) - - if response.status_code != 200: - raise RuntimeError("Unable to retrieve data from server: ", response) - - logger.info("Data retieval is successful") - - data = response.json() - - return data_api.client._build_pandas_data_frame(data, index_field="globalDate") - - -def get_pulse_id_date_mapping(pulse_id): - # See https://jira.psi.ch/browse/ATEST-897 for more details ... - logger.info("Retrieve pulse-id/date mapping for pulse_id %s" % pulse_id) - - try: - - query = {"range": {"startPulseId": 0, - "endPulseId": pulse_id}, - "limit": 1, - "ordering": "desc", - "channels": ["SIN-CVME-TIFGUN-EVR0:BUNCH-1-OK"], - "fields": ["pulseId", "globalDate"]} - - for c in range(10): - - response = requests.post("https://data-api.psi.ch/sf/query", json=query) - - # Check for successful return of data - if response.status_code != 200: - raise RuntimeError("Unable to retrieve data from server: ", response) - - data = response.json() - - if len(data[0]["data"]) == 0 or not "pulseId" in data[0]["data"][0]: - raise RuntimeError( - "Didn't get good responce from data_api : %s " % data) - - if not pulse_id == data[0]["data"][0]["pulseId"]: - logger.info("retrieval failed") - if c == 0: - ref_date = data[0]["data"][0]["globalDate"] - ref_date = dateutil.parser.parse(ref_date) - - now_date = datetime.datetime.now() - now_date = pytz.timezone('Europe/Zurich').localize( - now_date) - - check_date = ref_date + datetime.timedelta( - seconds=24) # 20 seconds should be enough - delta_date = check_date - now_date - - s = delta_date.seconds - logger.info("retry in " + str(s) + " seconds ") - if not s <= 0: - time.sleep(s) - continue - - raise RuntimeError('Unable to retrieve mapping') - - date = data[0]["data"][0]["globalDate"] - date = dateutil.parser.parse(date) - dates.append(date) - break - - except Exception as e: - raise RuntimeError('Unable to retrieve pulse_id date mapping') from e diff --git a/epics-writer/setup.py b/epics-writer/setup.py deleted file mode 100644 index d4358df..0000000 --- a/epics-writer/setup.py +++ /dev/null @@ -1,15 +0,0 @@ -from setuptools import setup - -setup( - name="cadump", - version="0.0.12", - author="Paul Scherrer Institute", - author_email="daq@psi.ch", - description="Interface to dump data from archiver/databuffer", - packages=["cadump"], - entry_points={ - 'console_scripts': [ - 'cadump_server = cadump.cadump:main', - ], - } -) diff --git a/epics-writer/tests/__init__.py b/epics-writer/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/epics-writer/tests/channels.txt b/epics-writer/tests/channels.txt deleted file mode 100644 index 63868d2..0000000 --- a/epics-writer/tests/channels.txt +++ /dev/null @@ -1,6 +0,0 @@ -S10-CPCL-VM1MGC:LOAD - -ONE - TWO -# Comment that need to be removed -THREE # one more comment at the end that need to be removed diff --git a/epics-writer/tests/test_download_data.py b/epics-writer/tests/test_download_data.py deleted file mode 100644 index 87a7194..0000000 --- a/epics-writer/tests/test_download_data.py +++ /dev/null @@ -1,34 +0,0 @@ -import unittest -from unittest import TestCase -from cadump import cadump -import logging - -class TestDownloadData(TestCase): - - def test_download_data(self): - config = { - 'range': { - 'startPulseId': 9618913001, - 'endPulseId': 9618923000 - }, - - 'parameters': { - 'general/created': 'test', - 'general/user': 'tester', - 'general/process': 'test_process', - 'general/instrument': 'mac', - 'output_file': 'test.h5'} # this is usually the full path - } - - cadump.base_url = "https://data-api.psi.ch/sf" - cadump.download_data(config) - # self.fail() - - def test_read_channels(self): - channels = cadump.read_channels("channels.txt") - logging.info(channels) - self.assertEqual(len(channels), 4) - - -if __name__ == '__main__': - unittest.main() From 8627caed11c4961728431be74d60510b50986b1b Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Mon, 21 Sep 2020 11:04:14 +0200 Subject: [PATCH 028/126] Fix buffer writer build files --- sf-buffer-writer/CMakeLists.txt | 30 ++++++++++------------------ sf-buffer-writer/test/CMakeLists.txt | 9 +++------ 2 files changed, 14 insertions(+), 25 deletions(-) diff --git a/sf-buffer-writer/CMakeLists.txt b/sf-buffer-writer/CMakeLists.txt index f2763dc..636556d 100644 --- a/sf-buffer-writer/CMakeLists.txt +++ b/sf-buffer-writer/CMakeLists.txt @@ -1,26 +1,18 @@ - -find_package(MPI REQUIRED) -# Because of openmpi. -add_definitions(-DOMPI_SKIP_MPICXX) - file(GLOB SOURCES src/*.cpp) -add_library(jf-live-daq-lib STATIC ${SOURCES}) -target_include_directories(jf-live-daq-lib - PUBLIC include/ - SYSTEM ${MPI_INCLUDE_PATH}) - -target_link_libraries(jf-live-daq-lib +add_library(sf-buffer-writer-lib STATIC ${SOURCES}) +target_include_directories(sf-buffer-writer-lib PUBLIC include/) +target_link_libraries(sf-buffer-writer-lib external - core-buffer-lib - ${MPI_LIBRARIES}) + core-buffer-lib) -add_executable(jf-live-daq src/main.cpp) -set_target_properties(jf-live-daq PROPERTIES OUTPUT_NAME jf_live_daq) -target_link_libraries(jf-live-daq - jf-live-daq-lib - ) +add_executable(sf-buffer-writer src/main.cpp) +set_target_properties(sf-buffer-writer PROPERTIES OUTPUT_NAME sf_buffer) +target_link_libraries(sf-buffer-writer + sf-buffer-writer-lib + zmq + rt) enable_testing() -add_subdirectory(test/) \ No newline at end of file +add_subdirectory(test/) diff --git a/sf-buffer-writer/test/CMakeLists.txt b/sf-buffer-writer/test/CMakeLists.txt index 7dc93bb..7d1417c 100644 --- a/sf-buffer-writer/test/CMakeLists.txt +++ b/sf-buffer-writer/test/CMakeLists.txt @@ -1,10 +1,7 @@ -add_executable(jf-live-daq-tests main.cpp) +add_executable(sf-buffer-writer-tests main.cpp) -target_link_libraries(jf-live-daq-tests - jf-live-daq-lib - hdf5 - hdf5_hl - hdf5_cpp +target_link_libraries(sf-buffer-writer-tests + sf-buffer-writer-lib zmq gtest ) From 29fce99f672e8b61c891cf64fffa0fbb339f9683 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Mon, 21 Sep 2020 11:05:58 +0200 Subject: [PATCH 029/126] Move buffer writing to sf-buffer-writer --- .../include/BufferBinaryWriter.hpp | 0 {sf-buffer => sf-buffer-writer}/src/BufferBinaryWriter.cpp | 0 sf-buffer-writer/src/main.cpp | 1 - sf-buffer/src/main.cpp | 4 ---- 4 files changed, 5 deletions(-) rename {sf-buffer => sf-buffer-writer}/include/BufferBinaryWriter.hpp (100%) rename {sf-buffer => sf-buffer-writer}/src/BufferBinaryWriter.cpp (100%) diff --git a/sf-buffer/include/BufferBinaryWriter.hpp b/sf-buffer-writer/include/BufferBinaryWriter.hpp similarity index 100% rename from sf-buffer/include/BufferBinaryWriter.hpp rename to sf-buffer-writer/include/BufferBinaryWriter.hpp diff --git a/sf-buffer/src/BufferBinaryWriter.cpp b/sf-buffer-writer/src/BufferBinaryWriter.cpp similarity index 100% rename from sf-buffer/src/BufferBinaryWriter.cpp rename to sf-buffer-writer/src/BufferBinaryWriter.cpp diff --git a/sf-buffer-writer/src/main.cpp b/sf-buffer-writer/src/main.cpp index 0e39832..8b14409 100644 --- a/sf-buffer-writer/src/main.cpp +++ b/sf-buffer-writer/src/main.cpp @@ -9,7 +9,6 @@ #include "formats.hpp" #include "buffer_config.hpp" #include "jungfrau.hpp" -#include "FrameUdpReceiver.hpp" #include "BufferBinaryWriter.hpp" using namespace std; diff --git a/sf-buffer/src/main.cpp b/sf-buffer/src/main.cpp index 0e39832..ed49728 100644 --- a/sf-buffer/src/main.cpp +++ b/sf-buffer/src/main.cpp @@ -10,7 +10,6 @@ #include "buffer_config.hpp" #include "jungfrau.hpp" #include "FrameUdpReceiver.hpp" -#include "BufferBinaryWriter.hpp" using namespace std; using namespace chrono; @@ -72,7 +71,6 @@ int main (int argc, char *argv[]) { uint64_t n_missed_packets = 0; uint64_t n_corrupted_frames = 0; - BufferBinaryWriter writer(root_folder, device_name); FrameUdpReceiver receiver(udp_port, source_id); RamBuffer buffer(detector_name, n_modules); @@ -84,8 +82,6 @@ int main (int argc, char *argv[]) { auto pulse_id = receiver.get_frame_from_udp( binary_buffer->metadata, binary_buffer->data); - writer.write(pulse_id, binary_buffer); - buffer.write_frame(&(binary_buffer->metadata), &(binary_buffer->data[0])); From f6006f7a251cbc498eeded1153ddd7023dcc5aec Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Mon, 21 Sep 2020 11:06:43 +0200 Subject: [PATCH 030/126] Moved also tests --- sf-buffer-writer/test/main.cpp | 1 + {sf-buffer => sf-buffer-writer}/test/test_BufferBinaryWriter.cpp | 0 sf-buffer/test/main.cpp | 1 - 3 files changed, 1 insertion(+), 1 deletion(-) rename {sf-buffer => sf-buffer-writer}/test/test_BufferBinaryWriter.cpp (100%) diff --git a/sf-buffer-writer/test/main.cpp b/sf-buffer-writer/test/main.cpp index 1ea4d8a..1c656b8 100644 --- a/sf-buffer-writer/test/main.cpp +++ b/sf-buffer-writer/test/main.cpp @@ -1,4 +1,5 @@ #include "gtest/gtest.h" +#include "test_BufferBinaryWriter.cpp" using namespace std; diff --git a/sf-buffer/test/test_BufferBinaryWriter.cpp b/sf-buffer-writer/test/test_BufferBinaryWriter.cpp similarity index 100% rename from sf-buffer/test/test_BufferBinaryWriter.cpp rename to sf-buffer-writer/test/test_BufferBinaryWriter.cpp diff --git a/sf-buffer/test/main.cpp b/sf-buffer/test/main.cpp index a8035f5..8f2cd01 100644 --- a/sf-buffer/test/main.cpp +++ b/sf-buffer/test/main.cpp @@ -1,6 +1,5 @@ #include "gtest/gtest.h" #include "test_PacketUdpReceiver.cpp" -#include "test_BufferBinaryWriter.cpp" #include "test_FrameUdpReceiver.cpp" using namespace std; From cb7f37f08bc5f37dc6a28208af48d3a20eee97c0 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Mon, 21 Sep 2020 11:19:15 +0200 Subject: [PATCH 031/126] Rename buffer project --- CMakeLists.txt | 3 +- sf-buffer-recv/CMakeLists.txt | 18 ++++++++++ {sf-buffer => sf-buffer-recv}/README.md | 0 .../include/FrameUdpReceiver.hpp | 0 .../include/PacketUdpReceiver.hpp | 0 .../src/FrameUdpReceiver.cpp | 0 .../src/PacketUdpReceiver.cpp | 0 {sf-buffer => sf-buffer-recv}/src/main.cpp | 0 sf-buffer-recv/test/CMakeLists.txt | 8 +++++ {sf-buffer => sf-buffer-recv}/test/main.cpp | 0 .../test/mock/udp.hpp | 0 .../test/test_FrameUdpReceiver.cpp | 0 .../test/test_PacketUdpReceiver.cpp | 0 sf-buffer-writer/src/main.cpp | 34 +++---------------- sf-buffer/CMakeLists.txt | 18 ---------- sf-buffer/test/CMakeLists.txt | 8 ----- 16 files changed, 32 insertions(+), 57 deletions(-) create mode 100644 sf-buffer-recv/CMakeLists.txt rename {sf-buffer => sf-buffer-recv}/README.md (100%) rename {sf-buffer => sf-buffer-recv}/include/FrameUdpReceiver.hpp (100%) rename {sf-buffer => sf-buffer-recv}/include/PacketUdpReceiver.hpp (100%) rename {sf-buffer => sf-buffer-recv}/src/FrameUdpReceiver.cpp (100%) rename {sf-buffer => sf-buffer-recv}/src/PacketUdpReceiver.cpp (100%) rename {sf-buffer => sf-buffer-recv}/src/main.cpp (100%) create mode 100644 sf-buffer-recv/test/CMakeLists.txt rename {sf-buffer => sf-buffer-recv}/test/main.cpp (100%) rename {sf-buffer => sf-buffer-recv}/test/mock/udp.hpp (100%) rename {sf-buffer => sf-buffer-recv}/test/test_FrameUdpReceiver.cpp (100%) rename {sf-buffer => sf-buffer-recv}/test/test_PacketUdpReceiver.cpp (100%) delete mode 100644 sf-buffer/CMakeLists.txt delete mode 100644 sf-buffer/test/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index f67c4f0..c490a02 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,9 +29,8 @@ add_subdirectory( EXCLUDE_FROM_ALL) add_subdirectory("core-buffer") -add_subdirectory("sf-buffer") +add_subdirectory("sf-buffer-recv") add_subdirectory("sf-buffer-writer") add_subdirectory("sf-stream") add_subdirectory("sf-writer") #add_subdirectory("jf-live-writer") -#add_subdirectory("jf-live-daq") \ No newline at end of file diff --git a/sf-buffer-recv/CMakeLists.txt b/sf-buffer-recv/CMakeLists.txt new file mode 100644 index 0000000..f523a4f --- /dev/null +++ b/sf-buffer-recv/CMakeLists.txt @@ -0,0 +1,18 @@ +file(GLOB SOURCES + src/*.cpp) + +add_library(sf-buffer-recv-lib STATIC ${SOURCES}) +target_include_directories(sf-buffer-recv-lib PUBLIC include/) +target_link_libraries(sf-buffer-recv-lib + external + core-buffer-lib) + +add_executable(sf-buffer-recv src/main.cpp) +set_target_properties(sf-buffer-recv PROPERTIES OUTPUT_NAME sf_buffer) +target_link_libraries(sf-buffer-recv + sf-buffer-recv-lib + zmq + rt) + +enable_testing() +add_subdirectory(test/) diff --git a/sf-buffer/README.md b/sf-buffer-recv/README.md similarity index 100% rename from sf-buffer/README.md rename to sf-buffer-recv/README.md diff --git a/sf-buffer/include/FrameUdpReceiver.hpp b/sf-buffer-recv/include/FrameUdpReceiver.hpp similarity index 100% rename from sf-buffer/include/FrameUdpReceiver.hpp rename to sf-buffer-recv/include/FrameUdpReceiver.hpp diff --git a/sf-buffer/include/PacketUdpReceiver.hpp b/sf-buffer-recv/include/PacketUdpReceiver.hpp similarity index 100% rename from sf-buffer/include/PacketUdpReceiver.hpp rename to sf-buffer-recv/include/PacketUdpReceiver.hpp diff --git a/sf-buffer/src/FrameUdpReceiver.cpp b/sf-buffer-recv/src/FrameUdpReceiver.cpp similarity index 100% rename from sf-buffer/src/FrameUdpReceiver.cpp rename to sf-buffer-recv/src/FrameUdpReceiver.cpp diff --git a/sf-buffer/src/PacketUdpReceiver.cpp b/sf-buffer-recv/src/PacketUdpReceiver.cpp similarity index 100% rename from sf-buffer/src/PacketUdpReceiver.cpp rename to sf-buffer-recv/src/PacketUdpReceiver.cpp diff --git a/sf-buffer/src/main.cpp b/sf-buffer-recv/src/main.cpp similarity index 100% rename from sf-buffer/src/main.cpp rename to sf-buffer-recv/src/main.cpp diff --git a/sf-buffer-recv/test/CMakeLists.txt b/sf-buffer-recv/test/CMakeLists.txt new file mode 100644 index 0000000..a03fdca --- /dev/null +++ b/sf-buffer-recv/test/CMakeLists.txt @@ -0,0 +1,8 @@ +add_executable(sf-buffer-recv-tests main.cpp) + +target_link_libraries(sf-buffer-recv-tests + core-buffer-lib + sf-buffer-recv-lib + gtest + ) + diff --git a/sf-buffer/test/main.cpp b/sf-buffer-recv/test/main.cpp similarity index 100% rename from sf-buffer/test/main.cpp rename to sf-buffer-recv/test/main.cpp diff --git a/sf-buffer/test/mock/udp.hpp b/sf-buffer-recv/test/mock/udp.hpp similarity index 100% rename from sf-buffer/test/mock/udp.hpp rename to sf-buffer-recv/test/mock/udp.hpp diff --git a/sf-buffer/test/test_FrameUdpReceiver.cpp b/sf-buffer-recv/test/test_FrameUdpReceiver.cpp similarity index 100% rename from sf-buffer/test/test_FrameUdpReceiver.cpp rename to sf-buffer-recv/test/test_FrameUdpReceiver.cpp diff --git a/sf-buffer/test/test_PacketUdpReceiver.cpp b/sf-buffer-recv/test/test_PacketUdpReceiver.cpp similarity index 100% rename from sf-buffer/test/test_PacketUdpReceiver.cpp rename to sf-buffer-recv/test/test_PacketUdpReceiver.cpp diff --git a/sf-buffer-writer/src/main.cpp b/sf-buffer-writer/src/main.cpp index 8b14409..bcc33ba 100644 --- a/sf-buffer-writer/src/main.cpp +++ b/sf-buffer-writer/src/main.cpp @@ -15,33 +15,6 @@ using namespace std; using namespace chrono; using namespace buffer_config; -void* get_live_stream_socket(const string& detector_name, const int source_id) -{ - stringstream ipc_stream; - string LIVE_IPC_URL = BUFFER_LIVE_IPC_URL + detector_name + "-"; - ipc_stream << LIVE_IPC_URL << source_id; - const auto ipc_address = ipc_stream.str(); - - void* ctx = zmq_ctx_new(); - void* socket = zmq_socket(ctx, ZMQ_PUB); - - const int sndhwm = BUFFER_ZMQ_SNDHWM; - if (zmq_setsockopt(socket, ZMQ_SNDHWM, &sndhwm, sizeof(sndhwm)) != 0) { - throw runtime_error(zmq_strerror(errno)); - } - - const int linger = 0; - if (zmq_setsockopt(socket, ZMQ_LINGER, &linger, sizeof(linger)) != 0) { - throw runtime_error(zmq_strerror(errno)); - } - - if (zmq_bind(socket, ipc_address.c_str()) != 0) { - throw runtime_error(zmq_strerror(errno)); - } - - return socket; -} - int main (int argc, char *argv[]) { if (argc != 6) { @@ -67,16 +40,19 @@ int main (int argc, char *argv[]) { string root_folder = string(argv[5]); int source_id = atoi(argv[6]); + + string LIVE_IPC_URL = BUFFER_LIVE_IPC_URL + detector_name + "-"; + ipc_stream << LIVE_IPC_URL << source_id; + const auto ipc_address = ipc_stream.str(); + uint64_t stats_counter(0); uint64_t n_missed_packets = 0; uint64_t n_corrupted_frames = 0; BufferBinaryWriter writer(root_folder, device_name); - FrameUdpReceiver receiver(udp_port, source_id); RamBuffer buffer(detector_name, n_modules); auto binary_buffer = new BufferBinaryFormat(); - auto socket = get_live_stream_socket(detector_name, source_id); while (true) { diff --git a/sf-buffer/CMakeLists.txt b/sf-buffer/CMakeLists.txt deleted file mode 100644 index bbd2cfe..0000000 --- a/sf-buffer/CMakeLists.txt +++ /dev/null @@ -1,18 +0,0 @@ -file(GLOB SOURCES - src/*.cpp) - -add_library(sf-buffer-lib STATIC ${SOURCES}) -target_include_directories(sf-buffer-lib PUBLIC include/) -target_link_libraries(sf-buffer-lib - external - core-buffer-lib) - -add_executable(sf-buffer src/main.cpp) -set_target_properties(sf-buffer PROPERTIES OUTPUT_NAME sf_buffer) -target_link_libraries(sf-buffer - sf-buffer-lib - zmq - rt) - -enable_testing() -add_subdirectory(test/) diff --git a/sf-buffer/test/CMakeLists.txt b/sf-buffer/test/CMakeLists.txt deleted file mode 100644 index 4ae80a8..0000000 --- a/sf-buffer/test/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -add_executable(sf-buffer-tests main.cpp) - -target_link_libraries(sf-buffer-tests - core-buffer-lib - sf-buffer-lib - gtest - ) - From d858b4dc56a7b5a3f86de194ff18227e3746fc9c Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 22 Sep 2020 10:15:20 +0200 Subject: [PATCH 032/126] Fix memory offset bug in RamBuffer --- core-buffer/include/RamBuffer.hpp | 10 +++++--- core-buffer/src/RamBuffer.cpp | 40 +++++++++++++++++++++---------- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/core-buffer/include/RamBuffer.hpp b/core-buffer/include/RamBuffer.hpp index 8fb28e7..f0b0fd6 100644 --- a/core-buffer/include/RamBuffer.hpp +++ b/core-buffer/include/RamBuffer.hpp @@ -9,9 +9,9 @@ class RamBuffer { const int n_modules_; const int n_slots_; - const size_t meta_size_; - const size_t image_size_; - const size_t buffer_size_; + const size_t meta_bytes_; + const size_t image_bytes_; + const size_t buffer_bytes_; int shm_fd_; void* buffer_; @@ -26,6 +26,10 @@ public: ~RamBuffer(); void write_frame(const ModuleFrame *src_meta, const char *src_data) const; + void read_frame(const uint64_t pulse_id, + const uint64_t module_id, + ModuleFrame *&meta, + char *&data) const; char* read_image(const uint64_t pulse_id, ImageMetadata &image_meta) const; }; diff --git a/core-buffer/src/RamBuffer.cpp b/core-buffer/src/RamBuffer.cpp index 939c6a4..77613b1 100644 --- a/core-buffer/src/RamBuffer.cpp +++ b/core-buffer/src/RamBuffer.cpp @@ -16,33 +16,34 @@ RamBuffer::RamBuffer( detector_name_(detector_name), n_modules_(n_modules), n_slots_(n_slots), - meta_size_(sizeof(ModuleFrame) * n_modules_), - image_size_(MODULE_N_BYTES * n_modules_), - buffer_size_((meta_size_ + image_size_) * n_slots_) + meta_bytes_(sizeof(ModuleFrame) * n_modules_), + image_bytes_(MODULE_N_BYTES * n_modules_), + buffer_bytes_((meta_bytes_ + image_bytes_) * n_slots_) { shm_fd_ = shm_open(detector_name_.c_str(), O_RDWR | O_CREAT, 0777); if (shm_fd_ < 0) { throw runtime_error(strerror(errno)); } - if ((ftruncate(shm_fd_, buffer_size_)) == -1) { + if ((ftruncate(shm_fd_, buffer_bytes_)) == -1) { throw runtime_error(strerror(errno)); } // TODO: Test with MAP_HUGETLB - buffer_ = mmap(NULL, buffer_size_, PROT_WRITE, MAP_SHARED, shm_fd_, 0); + buffer_ = mmap(NULL, buffer_bytes_, PROT_WRITE, MAP_SHARED, shm_fd_, 0); if (buffer_ == MAP_FAILED) { throw runtime_error(strerror(errno)); } + // Metadata buffer is located at the start of the memory region. meta_buffer_ = (ModuleFrame *) buffer_; // Image buffer start right after metadata buffer. - image_buffer_ = (char*)buffer_ + (meta_size_ * n_slots_); + image_buffer_ = (char*)buffer_ + (meta_bytes_ * n_slots_); } RamBuffer::~RamBuffer() { - munmap(buffer_, buffer_size_); + munmap(buffer_, buffer_bytes_); close(shm_fd_); shm_unlink(detector_name_.c_str()); } @@ -54,27 +55,40 @@ void RamBuffer::write_frame( const int slot_n = src_meta->pulse_id % n_slots_; ModuleFrame *dst_meta = meta_buffer_ + - (meta_size_ * slot_n) + + (n_modules_ * slot_n) + src_meta->module_id; char *dst_data = image_buffer_ + - (image_size_ * slot_n) + + (image_bytes_ * slot_n) + (MODULE_N_BYTES * src_meta->module_id); memcpy(dst_meta, src_meta, sizeof(ModuleFrame)); memcpy(dst_data, src_data, MODULE_N_BYTES); } +void RamBuffer::read_frame( + const uint64_t pulse_id, + const uint64_t module_id, + ModuleFrame*& meta, + char*& data) const +{ + const size_t slot_n = pulse_id % n_slots_; + + meta = meta_buffer_ + (n_modules_ * slot_n) + module_id; + + data = image_buffer_ + + (image_bytes_ * slot_n) + + (MODULE_N_BYTES * module_id); +} + char* RamBuffer::read_image(const uint64_t pulse_id, ImageMetadata &image_meta) const { const size_t slot_n = pulse_id % n_slots_; - ModuleFrame *src_meta = meta_buffer_ + - (meta_size_ * slot_n); + ModuleFrame *src_meta = meta_buffer_ + (n_modules_ * slot_n); - char *src_data = image_buffer_ + - (image_size_ * slot_n); + char *src_data = image_buffer_ + (image_bytes_ * slot_n); auto is_pulse_init = false; auto is_good_image = true; From 259e3dd822999ad5cde6bb16f036a65f7cc1099a Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 22 Sep 2020 10:29:21 +0200 Subject: [PATCH 033/126] Start refactoring of buffer writer --- sf-buffer-writer/src/main.cpp | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/sf-buffer-writer/src/main.cpp b/sf-buffer-writer/src/main.cpp index bcc33ba..8ef837f 100644 --- a/sf-buffer-writer/src/main.cpp +++ b/sf-buffer-writer/src/main.cpp @@ -19,13 +19,12 @@ int main (int argc, char *argv[]) { if (argc != 6) { cout << endl; - cout << "Usage: sf_buffer [detector_name] [n_modules] [device_name]"; - cout << " [udp_port] [root_folder] [source_id]"; + cout << "Usage: sf_buffer_writer [detector_name] [n_modules]"; + cout << " [device_name] [root_folder] [source_id]"; cout << endl; cout << "\tdetector_name: Detector name, example JF07T32V01" << endl; cout << "\tn_modules: Number of modules in the detector." << endl; cout << "\tdevice_name: Name to write to disk." << endl; - cout << "\tudp_port: UDP port to connect to." << endl; cout << "\troot_folder: FS root folder." << endl; cout << "\tsource_id: ID of the source for live stream." << endl; cout << endl; @@ -36,24 +35,21 @@ int main (int argc, char *argv[]) { string detector_name = string(argv[1]); int n_modules = atoi(argv[2]); string device_name = string(argv[3]); - int udp_port = atoi(argv[4]); - string root_folder = string(argv[5]); - int source_id = atoi(argv[6]); + string root_folder = string(argv[4]); + int source_id = atoi(argv[5]); - - string LIVE_IPC_URL = BUFFER_LIVE_IPC_URL + detector_name + "-"; - ipc_stream << LIVE_IPC_URL << source_id; - const auto ipc_address = ipc_stream.str(); - - uint64_t stats_counter(0); - uint64_t n_missed_packets = 0; - uint64_t n_corrupted_frames = 0; + string ipc_address = BUFFER_LIVE_IPC_URL + + detector_name + "-" + + to_string(source_id); BufferBinaryWriter writer(root_folder, device_name); RamBuffer buffer(detector_name, n_modules); auto binary_buffer = new BufferBinaryFormat(); + ImageMetadata* meta = nullptr; + char* data = nullptr; + while (true) { auto pulse_id = receiver.get_frame_from_udp( From a5629616d9b7665d3e5cf04c7799d3da5e21fa29 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Thu, 24 Sep 2020 08:33:45 +0200 Subject: [PATCH 034/126] Create common connect and bind zmq methods --- core-buffer/include/BufferUtils.hpp | 5 +++ core-buffer/include/buffer_config.hpp | 2 + core-buffer/src/BufferUtils.cpp | 63 ++++++++++++++++++++++++++- 3 files changed, 69 insertions(+), 1 deletion(-) diff --git a/core-buffer/include/BufferUtils.hpp b/core-buffer/include/BufferUtils.hpp index 37f9f62..5216a2d 100644 --- a/core-buffer/include/BufferUtils.hpp +++ b/core-buffer/include/BufferUtils.hpp @@ -18,6 +18,11 @@ namespace BufferUtils const std::string& filename_to_write); void create_destination_folder(const std::string& output_file); + + void* bind_socket( + void* ctx, const std::string& detector_name, const int source_id); + void* connect_socket( + void* ctx, const std::string& detector_name, const int source_id); } #endif //BUFFER_UTILS_HPP diff --git a/core-buffer/include/buffer_config.hpp b/core-buffer/include/buffer_config.hpp index 2f1f662..73c97b5 100644 --- a/core-buffer/include/buffer_config.hpp +++ b/core-buffer/include/buffer_config.hpp @@ -38,6 +38,8 @@ namespace buffer_config { const int BUFFER_UDP_US_TIMEOUT = 2 * 1000; // HWM for live stream from buffer. const int BUFFER_ZMQ_SNDHWM = 100; + // HWM for live stream from buffer. + const int BUFFER_ZMQ_RCVHWM = 100; // IPC address of the live stream. const std::string BUFFER_LIVE_IPC_URL = "ipc:///tmp/sf-live-"; // Number of image slots in ram buffer - 10 seconds should be enough diff --git a/core-buffer/src/BufferUtils.cpp b/core-buffer/src/BufferUtils.cpp index c2a60a4..a7346aa 100644 --- a/core-buffer/src/BufferUtils.cpp +++ b/core-buffer/src/BufferUtils.cpp @@ -2,8 +2,10 @@ #include #include +#include using namespace std; +using namespace buffer_config; string BufferUtils::get_filename( std::string detector_folder, @@ -60,4 +62,63 @@ void BufferUtils::create_destination_folder(const string& output_file) string create_folder_command("mkdir -p " + output_folder); system(create_folder_command.c_str()); } -} \ No newline at end of file +} + +void* BufferUtils::connect_socket( + void* ctx, const string& detector_name, const int source_id) +{ + string ipc_address = BUFFER_LIVE_IPC_URL + + detector_name + "-" + + to_string(source_id); + + void* socket = zmq_socket(ctx, ZMQ_SUB); + if (socket == nullptr) { + throw runtime_error(zmq_strerror(errno)); + } + + int rcvhwm = BUFFER_ZMQ_RCVHWM; + if (zmq_setsockopt(socket, ZMQ_RCVHWM, &rcvhwm, sizeof(rcvhwm)) != 0) { + throw runtime_error(zmq_strerror(errno)); + } + + int linger = 0; + if (zmq_setsockopt(socket, ZMQ_LINGER, &linger, sizeof(linger)) != 0) { + throw runtime_error(zmq_strerror(errno)); + } + + if (zmq_connect(socket, ipc_address.c_str()) != 0) { + throw runtime_error(zmq_strerror(errno)); + } + + if (zmq_setsockopt(socket, ZMQ_SUBSCRIBE, "", 0) != 0) { + throw runtime_error(zmq_strerror(errno)); + } + + return socket; +} + +void* BufferUtils::bind_socket( + void* ctx, const string& detector_name, const int source_id) +{ + string ipc_address = BUFFER_LIVE_IPC_URL + + detector_name + "-" + + to_string(source_id); + + void* socket = zmq_socket(ctx, ZMQ_PUB); + + const int sndhwm = BUFFER_ZMQ_SNDHWM; + if (zmq_setsockopt(socket, ZMQ_SNDHWM, &sndhwm, sizeof(sndhwm)) != 0) { + throw runtime_error(zmq_strerror(errno)); + } + + const int linger = 0; + if (zmq_setsockopt(socket, ZMQ_LINGER, &linger, sizeof(linger)) != 0) { + throw runtime_error(zmq_strerror(errno)); + } + + if (zmq_bind(socket, ipc_address.c_str()) != 0) { + throw runtime_error(zmq_strerror(errno)); + } + + return socket; +} From b54f351089036e9e8f144fb46a0ff4c54aa7fa8d Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Thu, 24 Sep 2020 08:37:02 +0200 Subject: [PATCH 035/126] Fix const correctness in buffer --- core-buffer/include/BufferUtils.hpp | 8 ++++---- core-buffer/src/BufferUtils.cpp | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/core-buffer/include/BufferUtils.hpp b/core-buffer/include/BufferUtils.hpp index 5216a2d..fa02a3f 100644 --- a/core-buffer/include/BufferUtils.hpp +++ b/core-buffer/include/BufferUtils.hpp @@ -7,11 +7,11 @@ namespace BufferUtils { std::string get_filename( - std::string detector_folder, - std::string module_name, - uint64_t pulse_id); + const std::string& detector_folder, + const std::string& module_name, + const uint64_t pulse_id); - std::size_t get_file_frame_index(uint64_t pulse_id); + std::size_t get_file_frame_index(const uint64_t pulse_id); void update_latest_file( const std::string& latest_filename, diff --git a/core-buffer/src/BufferUtils.cpp b/core-buffer/src/BufferUtils.cpp index a7346aa..388c945 100644 --- a/core-buffer/src/BufferUtils.cpp +++ b/core-buffer/src/BufferUtils.cpp @@ -8,9 +8,9 @@ using namespace std; using namespace buffer_config; string BufferUtils::get_filename( - std::string detector_folder, - std::string module_name, - uint64_t pulse_id) + const std::string& detector_folder, + const std::string& module_name, + const uint64_t pulse_id) { uint64_t data_folder = pulse_id / buffer_config::FOLDER_MOD; data_folder *= buffer_config::FOLDER_MOD; @@ -27,7 +27,7 @@ string BufferUtils::get_filename( return folder.str(); } -size_t BufferUtils::get_file_frame_index(uint64_t pulse_id) +size_t BufferUtils::get_file_frame_index(const uint64_t pulse_id) { uint64_t file_base = pulse_id / buffer_config::FILE_MOD; file_base *= buffer_config::FILE_MOD; From d3d98c8c7075caf9b43c46e2851e10a7f099bd77 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Thu, 24 Sep 2020 09:04:12 +0200 Subject: [PATCH 036/126] Add repetition rate to sf-buffer-recv --- sf-buffer-recv/src/main.cpp | 45 ++++++++++++------------------------- 1 file changed, 14 insertions(+), 31 deletions(-) diff --git a/sf-buffer-recv/src/main.cpp b/sf-buffer-recv/src/main.cpp index ed49728..f0f25dd 100644 --- a/sf-buffer-recv/src/main.cpp +++ b/sf-buffer-recv/src/main.cpp @@ -10,44 +10,18 @@ #include "buffer_config.hpp" #include "jungfrau.hpp" #include "FrameUdpReceiver.hpp" +#include "BufferUtils.hpp" using namespace std; using namespace chrono; using namespace buffer_config; -void* get_live_stream_socket(const string& detector_name, const int source_id) -{ - stringstream ipc_stream; - string LIVE_IPC_URL = BUFFER_LIVE_IPC_URL + detector_name + "-"; - ipc_stream << LIVE_IPC_URL << source_id; - const auto ipc_address = ipc_stream.str(); - - void* ctx = zmq_ctx_new(); - void* socket = zmq_socket(ctx, ZMQ_PUB); - - const int sndhwm = BUFFER_ZMQ_SNDHWM; - if (zmq_setsockopt(socket, ZMQ_SNDHWM, &sndhwm, sizeof(sndhwm)) != 0) { - throw runtime_error(zmq_strerror(errno)); - } - - const int linger = 0; - if (zmq_setsockopt(socket, ZMQ_LINGER, &linger, sizeof(linger)) != 0) { - throw runtime_error(zmq_strerror(errno)); - } - - if (zmq_bind(socket, ipc_address.c_str()) != 0) { - throw runtime_error(zmq_strerror(errno)); - } - - return socket; -} - int main (int argc, char *argv[]) { if (argc != 6) { cout << endl; - cout << "Usage: sf_buffer [detector_name] [n_modules] [device_name]"; - cout << " [udp_port] [root_folder] [source_id]"; + cout << "Usage: sf_buffer_recv [detector_name] [n_modules]"; + cout << " [device_name] [udp_port] [root_folder] [source_id]"; cout << endl; cout << "\tdetector_name: Detector name, example JF07T32V01" << endl; cout << "\tn_modules: Number of modules in the detector." << endl; @@ -70,12 +44,13 @@ int main (int argc, char *argv[]) { uint64_t stats_counter(0); uint64_t n_missed_packets = 0; uint64_t n_corrupted_frames = 0; + auto stats_interval_start = steady_clock::now(); FrameUdpReceiver receiver(udp_port, source_id); RamBuffer buffer(detector_name, n_modules); - auto binary_buffer = new BufferBinaryFormat(); - auto socket = get_live_stream_socket(detector_name, source_id); + auto ctx = zmq_ctx_new(); + auto socket = BufferUtils::bind_socket(ctx, detector_name, source_id); while (true) { @@ -87,6 +62,7 @@ int main (int argc, char *argv[]) { zmq_send(socket, &pulse_id, sizeof(pulse_id), 0); + // TODO: Isolate in a class. if (binary_buffer->metadata.n_recv_packets < JF_N_PACKETS_PER_FRAME) { n_missed_packets += JF_N_PACKETS_PER_FRAME - binary_buffer->metadata.n_recv_packets; @@ -95,14 +71,21 @@ int main (int argc, char *argv[]) { stats_counter++; if (stats_counter == STATS_MODULO) { + auto interval_ms_duration = duration_cast( + stats_interval_start-steady_clock::now()).count(); + // * 1000 because milliseconds, 0.5 for truncation. + int rep_rate = ((stats_counter/interval_ms_duration) * 1000) + 0.5; + cout << "sf_buffer:device_name " << device_name; cout << " sf_buffer:n_missed_packets " << n_missed_packets; cout << " sf_buffer:n_corrupted_frames " << n_corrupted_frames; + cout << " sf_buffer:repetition_rate " << rep_rate; cout << endl; stats_counter = 0; n_missed_packets = 0; n_corrupted_frames = 0; + stats_interval_start = steady_clock::now(); } } } From f16a81ed52618dcc958fbd3392adf0ef3032594f Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Thu, 24 Sep 2020 09:10:28 +0200 Subject: [PATCH 037/126] Change interface to use reference This makes the receiver nicer. --- core-buffer/include/RamBuffer.hpp | 2 +- core-buffer/src/RamBuffer.cpp | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/core-buffer/include/RamBuffer.hpp b/core-buffer/include/RamBuffer.hpp index f0b0fd6..a202033 100644 --- a/core-buffer/include/RamBuffer.hpp +++ b/core-buffer/include/RamBuffer.hpp @@ -25,7 +25,7 @@ public: const int n_slots=buffer_config::RAM_BUFFER_N_SLOTS); ~RamBuffer(); - void write_frame(const ModuleFrame *src_meta, const char *src_data) const; + void write_frame(const ModuleFrame &src_meta, const char *src_data) const; void read_frame(const uint64_t pulse_id, const uint64_t module_id, ModuleFrame *&meta, diff --git a/core-buffer/src/RamBuffer.cpp b/core-buffer/src/RamBuffer.cpp index 77613b1..0179528 100644 --- a/core-buffer/src/RamBuffer.cpp +++ b/core-buffer/src/RamBuffer.cpp @@ -49,20 +49,20 @@ RamBuffer::~RamBuffer() } void RamBuffer::write_frame( - const ModuleFrame *src_meta, + const ModuleFrame& src_meta, const char *src_data) const { - const int slot_n = src_meta->pulse_id % n_slots_; + const int slot_n = src_meta.pulse_id % n_slots_; ModuleFrame *dst_meta = meta_buffer_ + (n_modules_ * slot_n) + - src_meta->module_id; + src_meta.module_id; char *dst_data = image_buffer_ + (image_bytes_ * slot_n) + - (MODULE_N_BYTES * src_meta->module_id); + (MODULE_N_BYTES * src_meta.module_id); - memcpy(dst_meta, src_meta, sizeof(ModuleFrame)); + memcpy(dst_meta, &src_meta, sizeof(ModuleFrame)); memcpy(dst_data, src_data, MODULE_N_BYTES); } From caaa6befbd6863211ff31ef190b7ead57787593a Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Thu, 24 Sep 2020 09:11:44 +0200 Subject: [PATCH 038/126] Buffer recv refactoring completed --- sf-buffer-recv/src/main.cpp | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/sf-buffer-recv/src/main.cpp b/sf-buffer-recv/src/main.cpp index f0f25dd..e8d177c 100644 --- a/sf-buffer-recv/src/main.cpp +++ b/sf-buffer-recv/src/main.cpp @@ -2,7 +2,6 @@ #include #include #include -#include #include #include @@ -52,20 +51,20 @@ int main (int argc, char *argv[]) { auto ctx = zmq_ctx_new(); auto socket = BufferUtils::bind_socket(ctx, detector_name, source_id); + ModuleFrame meta; + char* data = new char[MODULE_N_BYTES]; + while (true) { - auto pulse_id = receiver.get_frame_from_udp( - binary_buffer->metadata, binary_buffer->data); + auto pulse_id = receiver.get_frame_from_udp(meta, data); - buffer.write_frame(&(binary_buffer->metadata), - &(binary_buffer->data[0])); + buffer.write_frame(meta, data); zmq_send(socket, &pulse_id, sizeof(pulse_id), 0); // TODO: Isolate in a class. - if (binary_buffer->metadata.n_recv_packets < JF_N_PACKETS_PER_FRAME) { - n_missed_packets += JF_N_PACKETS_PER_FRAME - - binary_buffer->metadata.n_recv_packets; + if (meta.n_recv_packets < JF_N_PACKETS_PER_FRAME) { + n_missed_packets += JF_N_PACKETS_PER_FRAME - meta.n_recv_packets; n_corrupted_frames++; } @@ -88,4 +87,6 @@ int main (int argc, char *argv[]) { stats_interval_start = steady_clock::now(); } } + + delete[] data; } From c3593636622c6d90b412216f6653369b50e9f00e Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Thu, 24 Sep 2020 09:39:26 +0200 Subject: [PATCH 039/126] FrameStatistics class --- sf-buffer-recv/include/FrameStats.hpp | 27 +++++++++++++++ sf-buffer-recv/src/FrameStats.cpp | 49 +++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 sf-buffer-recv/include/FrameStats.hpp create mode 100644 sf-buffer-recv/src/FrameStats.cpp diff --git a/sf-buffer-recv/include/FrameStats.hpp b/sf-buffer-recv/include/FrameStats.hpp new file mode 100644 index 0000000..9d6fa71 --- /dev/null +++ b/sf-buffer-recv/include/FrameStats.hpp @@ -0,0 +1,27 @@ +#include +#include +#include + +#ifndef SF_DAQ_BUFFER_FRAMESTATS_HPP +#define SF_DAQ_BUFFER_FRAMESTATS_HPP + + +class FrameStats { + const std::string source_name_; + size_t stats_modulo_; + + int frames_counter_; + int n_missed_packets_; + int n_corrupted_frames_; + std::chrono::time_point stats_interval_start_; + + void reset_counters(); + void print_stats(); + +public: + FrameStats(const std::string &source_name, const size_t stats_modulo); + void record_stats(const ModuleFrame &meta); +}; + + +#endif //SF_DAQ_BUFFER_FRAMESTATS_HPP diff --git a/sf-buffer-recv/src/FrameStats.cpp b/sf-buffer-recv/src/FrameStats.cpp new file mode 100644 index 0000000..ded7649 --- /dev/null +++ b/sf-buffer-recv/src/FrameStats.cpp @@ -0,0 +1,49 @@ +#include +#include "FrameStats.hpp" + +using namespace std; +using namespace chrono; + +FrameStats::FrameStats(const string& source_name, const size_t stats_modulo) : + source_name_(source_name), + stats_modulo_(stats_modulo) +{ + reset_counters(); +} + +void FrameStats::reset_counters() +{ + frames_counter_ = 0; + n_missed_packets_ = 0; + n_corrupted_frames_ = 0; + stats_interval_start_ = steady_clock::now(); +} + +void FrameStats::record_stats(const ModuleFrame &meta) +{ + if (meta.n_recv_packets < JF_N_PACKETS_PER_FRAME) { + n_missed_packets_ += JF_N_PACKETS_PER_FRAME - meta.n_recv_packets; + n_corrupted_frames_++; + } + + frames_counter_++; + + if (frames_counter_ == stats_modulo_) { + print_stats(); + reset_counters(); + } +} + +void FrameStats::print_stats() +{ + auto interval_ms_duration = duration_cast( + stats_interval_start_-steady_clock::now()).count(); + // * 1000 because milliseconds, 0.5 for truncation. + int rep_rate = ((frames_counter_/interval_ms_duration) * 1000) + 0.5; + + cout << "sf_buffer:device_name " << source_name_; + cout << " sf_buffer:n_missed_packets " << n_missed_packets_; + cout << " sf_buffer:n_corrupted_frames " << n_corrupted_frames_; + cout << " sf_buffer:repetition_rate " << rep_rate; + cout << endl; +} From da7377d5019f157bd6a5c3a06897396ebe85f8a7 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Thu, 24 Sep 2020 09:39:46 +0200 Subject: [PATCH 040/126] Move statistics to FrameStats class --- sf-buffer-recv/src/main.cpp | 34 ++++------------------------------ 1 file changed, 4 insertions(+), 30 deletions(-) diff --git a/sf-buffer-recv/src/main.cpp b/sf-buffer-recv/src/main.cpp index e8d177c..2590272 100644 --- a/sf-buffer-recv/src/main.cpp +++ b/sf-buffer-recv/src/main.cpp @@ -10,6 +10,7 @@ #include "jungfrau.hpp" #include "FrameUdpReceiver.hpp" #include "BufferUtils.hpp" +#include "FrameStats.hpp" using namespace std; using namespace chrono; @@ -40,52 +41,25 @@ int main (int argc, char *argv[]) { string root_folder = string(argv[5]); int source_id = atoi(argv[6]); - uint64_t stats_counter(0); - uint64_t n_missed_packets = 0; - uint64_t n_corrupted_frames = 0; - auto stats_interval_start = steady_clock::now(); - FrameUdpReceiver receiver(udp_port, source_id); RamBuffer buffer(detector_name, n_modules); auto ctx = zmq_ctx_new(); auto socket = BufferUtils::bind_socket(ctx, detector_name, source_id); + FrameStats stats(device_name, STATS_MODULO); + ModuleFrame meta; char* data = new char[MODULE_N_BYTES]; while (true) { - auto pulse_id = receiver.get_frame_from_udp(meta, data); buffer.write_frame(meta, data); zmq_send(socket, &pulse_id, sizeof(pulse_id), 0); - // TODO: Isolate in a class. - if (meta.n_recv_packets < JF_N_PACKETS_PER_FRAME) { - n_missed_packets += JF_N_PACKETS_PER_FRAME - meta.n_recv_packets; - n_corrupted_frames++; - } - - stats_counter++; - if (stats_counter == STATS_MODULO) { - auto interval_ms_duration = duration_cast( - stats_interval_start-steady_clock::now()).count(); - // * 1000 because milliseconds, 0.5 for truncation. - int rep_rate = ((stats_counter/interval_ms_duration) * 1000) + 0.5; - - cout << "sf_buffer:device_name " << device_name; - cout << " sf_buffer:n_missed_packets " << n_missed_packets; - cout << " sf_buffer:n_corrupted_frames " << n_corrupted_frames; - cout << " sf_buffer:repetition_rate " << rep_rate; - cout << endl; - - stats_counter = 0; - n_missed_packets = 0; - n_corrupted_frames = 0; - stats_interval_start = steady_clock::now(); - } + stats.record_stats(meta); } delete[] data; From 19a60e42c829dce421943906596df1f2aa63f671 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Thu, 24 Sep 2020 09:40:42 +0200 Subject: [PATCH 041/126] Remove unused import --- sf-buffer-recv/src/main.cpp | 2 -- sf-buffer-writer/src/main.cpp | 8 ++++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/sf-buffer-recv/src/main.cpp b/sf-buffer-recv/src/main.cpp index 2590272..905321b 100644 --- a/sf-buffer-recv/src/main.cpp +++ b/sf-buffer-recv/src/main.cpp @@ -1,13 +1,11 @@ #include #include #include -#include #include #include #include "formats.hpp" #include "buffer_config.hpp" -#include "jungfrau.hpp" #include "FrameUdpReceiver.hpp" #include "BufferUtils.hpp" #include "FrameStats.hpp" diff --git a/sf-buffer-writer/src/main.cpp b/sf-buffer-writer/src/main.cpp index 8ef837f..fa4ef59 100644 --- a/sf-buffer-writer/src/main.cpp +++ b/sf-buffer-writer/src/main.cpp @@ -7,6 +7,7 @@ #include #include "formats.hpp" +#include "BufferUtils.hpp" #include "buffer_config.hpp" #include "jungfrau.hpp" #include "BufferBinaryWriter.hpp" @@ -38,15 +39,14 @@ int main (int argc, char *argv[]) { string root_folder = string(argv[4]); int source_id = atoi(argv[5]); - string ipc_address = BUFFER_LIVE_IPC_URL + - detector_name + "-" + - to_string(source_id); - BufferBinaryWriter writer(root_folder, device_name); RamBuffer buffer(detector_name, n_modules); auto binary_buffer = new BufferBinaryFormat(); + auto ctx = zmq_ctx_new(); + auto socket = BufferUtils::connect_socket(ctx, detector_name, source_id); + ImageMetadata* meta = nullptr; char* data = nullptr; From 55882bd0d13ad07d61e01ba82fed9d3f18841213 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Thu, 24 Sep 2020 09:51:02 +0200 Subject: [PATCH 042/126] Shorten metadata name in buffer format --- sf-buffer-recv/README.md | 6 ++--- .../test/test_BufferBinaryWriter.cpp | 26 +++++++++---------- sf-stream/README.md | 22 ++++++++-------- sf-writer/src/ImageAssembler.cpp | 2 +- sf-writer/test/test_ImageAssembler.cpp | 2 +- 5 files changed, 29 insertions(+), 29 deletions(-) diff --git a/sf-buffer-recv/README.md b/sf-buffer-recv/README.md index a738087..504281f 100644 --- a/sf-buffer-recv/README.md +++ b/sf-buffer-recv/README.md @@ -63,7 +63,7 @@ struct ModuleFrame { #pragma pack(1) struct BufferBinaryFormat { const char FORMAT_MARKER = 0xBE; - ModuleFrame metadata; + ModuleFrame meta; char data[buffer_config::MODULE_N_BYTES]; }; #pragma pack(pop) @@ -74,7 +74,7 @@ struct BufferBinaryFormat { Each frame is composed by: - **FORMAT\_MARKER** (0xBE) - a control byte to determine the validity of the frame. -- **ModuleFrame** - frame metadata used in image assembly phase. +- **ModuleFrame** - frame meta used in image assembly phase. - **Data** - assembled frame from a single module. Frames are written one after another to a specific offset in the file. The @@ -139,7 +139,7 @@ and the received data. - VerifyH5DataConsistency.py checks the consistency between the H5 file and buffer. -- BinaryBufferReader.py reads the buffer and prints metadata. The class inside +- BinaryBufferReader.py reads the buffer and prints meta. The class inside can also be used in external scripts. ### ZMQ sending diff --git a/sf-buffer-writer/test/test_BufferBinaryWriter.cpp b/sf-buffer-writer/test/test_BufferBinaryWriter.cpp index 5076c34..92e5d92 100644 --- a/sf-buffer-writer/test/test_BufferBinaryWriter.cpp +++ b/sf-buffer-writer/test/test_BufferBinaryWriter.cpp @@ -12,10 +12,10 @@ TEST(BinaryWriter, basic_interaction) BufferBinaryWriter writer(module_name, detector_folder); BufferBinaryFormat frame_data; - frame_data.metadata.pulse_id = 1; - frame_data.metadata.frame_index = 2; - frame_data.metadata.daq_rec = 3; - frame_data.metadata.n_recv_packets = 4; + frame_data.meta.pulse_id = 1; + frame_data.meta.frame_index = 2; + frame_data.meta.daq_rec = 3; + frame_data.meta.n_recv_packets = 4; writer.write(5, &frame_data); @@ -33,11 +33,11 @@ TEST(BinaryWriter, basic_interaction) ::read(read_fd, &read_data, sizeof(BufferBinaryFormat)); ASSERT_EQ(frame_data.FORMAT_MARKER, read_data.FORMAT_MARKER); - ASSERT_EQ(frame_data.metadata.pulse_id, read_data.metadata.pulse_id); - ASSERT_EQ(frame_data.metadata.frame_index, read_data.metadata.frame_index); - ASSERT_EQ(frame_data.metadata.daq_rec, read_data.metadata.daq_rec); - ASSERT_EQ(frame_data.metadata.n_recv_packets, - read_data.metadata.n_recv_packets); + ASSERT_EQ(frame_data.meta.pulse_id, read_data.meta.pulse_id); + ASSERT_EQ(frame_data.meta.frame_index, read_data.meta.frame_index); + ASSERT_EQ(frame_data.meta.daq_rec, read_data.meta.daq_rec); + ASSERT_EQ(frame_data.meta.n_recv_packets, + read_data.meta.n_recv_packets); } TEST(BinaryWriter, test_format_marker) @@ -49,10 +49,10 @@ TEST(BinaryWriter, test_format_marker) BufferBinaryWriter writer(module_name, detector_folder); BufferBinaryFormat frame_data; - frame_data.metadata.pulse_id = 1; - frame_data.metadata.frame_index = 2; - frame_data.metadata.daq_rec = 3; - frame_data.metadata.n_recv_packets = 4; + frame_data.meta.pulse_id = 1; + frame_data.meta.frame_index = 2; + frame_data.meta.daq_rec = 3; + frame_data.meta.n_recv_packets = 4; writer.write(5, &frame_data); diff --git a/sf-stream/README.md b/sf-stream/README.md index 1ccd5fd..ce21a4d 100644 --- a/sf-stream/README.md +++ b/sf-stream/README.md @@ -6,13 +6,13 @@ per detector. It currently has 3 output streams: -- **Full data full metadata** rate stream (send all images and metadata) -- **Reduced data full metadata** rate stream (send less images, but -all metadata) +- **Full data full meta** rate stream (send all images and meta) +- **Reduced data full meta** rate stream (send less images, but +all meta) - **Pulse_id** stream (send only the current pulse_id) In addition to receiving and assembling images, sf-stream also calculates -additional metadata and constructs the structures needed to send data in +additional meta and constructs the structures needed to send data in Array 1.0 protocol. This component does not guarantee that the streams will always contain all @@ -103,12 +103,12 @@ arrived. We devide the ZMQ sending to 3 types of stream: - Data processing stream. This is basically the complete stream from -the detector with all metadata and data. It can be described as full data full -metadata stream. Only 1 client at the time can be connected to this stream +the detector with all meta and data. It can be described as full data full +meta stream. Only 1 client at the time can be connected to this stream (PUSH/PULL for load balancing). -- Live viewing stream. This is a reduced data full metadata stream. We send -metadata for all frames, but data only for subset of them (10Hz, for example). +- Live viewing stream. This is a reduced data full meta stream. We send +meta for all frames, but data only for subset of them (10Hz, for example). Any number of clients can connect to the 10Hz stream, because we use PUB/SUB for this socket. @@ -137,7 +137,7 @@ We use following fields in the JSON header: |type|string|Value: "uint16"| |shape|Array[uint64]|Shape of the image in stream| -### Full data full metadata stream +### Full data full meta stream This stream runs at detector frequency and uses PUSH/PULL to distribute data to max 1 client (this client can have many processes, but it needs to be a @@ -151,9 +151,9 @@ for purposes of online analysis. Given the large amount of data on this stream only "pre-approved" applications that can handle the load should be attached here. -### Reduced data full metadata stream +### Reduced data full meta stream -This streams also runs at detector frequency for JSON headers (metadata), but +This streams also runs at detector frequency for JSON headers (meta), but it sends only part of the images in the stream. The rest of the images are sent as empty buffers (the receiver needs to be aware of this behaviour, as Array 1.0 alone does not define it). diff --git a/sf-writer/src/ImageAssembler.cpp b/sf-writer/src/ImageAssembler.cpp index ac1b851..1a9a5e6 100644 --- a/sf-writer/src/ImageAssembler.cpp +++ b/sf-writer/src/ImageAssembler.cpp @@ -86,7 +86,7 @@ void ImageAssembler::process( memcpy( &(frame_meta_buffer_[meta_offset]), - &(frame.metadata), + &(frame.meta), sizeof(ModuleFrame)); meta_offset += meta_offset_step; diff --git a/sf-writer/test/test_ImageAssembler.cpp b/sf-writer/test/test_ImageAssembler.cpp index 1482442..97bdbb3 100644 --- a/sf-writer/test/test_ImageAssembler.cpp +++ b/sf-writer/test/test_ImageAssembler.cpp @@ -50,7 +50,7 @@ TEST(ImageAssembler, reconstruction) for (size_t i_module=0; i_module < n_modules; i_module++) { for (size_t i_pulse=0; i_pulse < BUFFER_BLOCK_SIZE; i_pulse++) { - auto& frame_meta = buffer_block->frame[i_pulse].metadata; + auto& frame_meta = buffer_block->frame[i_pulse].meta; frame_meta.pulse_id = 100 + i_pulse; frame_meta.daq_rec = 1000 + i_pulse; From 1a381ee1bb26a583df7fc90d0e7381da0231e22a Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Thu, 24 Sep 2020 09:55:30 +0200 Subject: [PATCH 043/126] Fix const correctness --- sf-buffer-writer/src/BufferBinaryWriter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sf-buffer-writer/src/BufferBinaryWriter.cpp b/sf-buffer-writer/src/BufferBinaryWriter.cpp index f892704..78eef20 100644 --- a/sf-buffer-writer/src/BufferBinaryWriter.cpp +++ b/sf-buffer-writer/src/BufferBinaryWriter.cpp @@ -29,7 +29,7 @@ BufferBinaryWriter::~BufferBinaryWriter() } void BufferBinaryWriter::write( - uint64_t pulse_id, + const uint64_t pulse_id, const BufferBinaryFormat* buffer) { auto current_frame_file = From ef9fa031ebab29590d49f71f10a08d72787a76bb Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Thu, 24 Sep 2020 09:55:46 +0200 Subject: [PATCH 044/126] Shorten metadata name --- core-buffer/include/formats.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core-buffer/include/formats.hpp b/core-buffer/include/formats.hpp index 731dd61..dfa3589 100644 --- a/core-buffer/include/formats.hpp +++ b/core-buffer/include/formats.hpp @@ -34,7 +34,7 @@ struct ModuleFrameBuffer { #pragma pack(1) struct BufferBinaryFormat { const char FORMAT_MARKER = 0xBE; - ModuleFrame metadata; + ModuleFrame meta; char data[buffer_config::MODULE_N_BYTES]; }; #pragma pack(pop) From 3553541c968b8b96de2d1d91e6fb08c052488ae0 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Thu, 24 Sep 2020 11:32:44 +0200 Subject: [PATCH 045/126] Fix interface for reading frames --- core-buffer/include/RamBuffer.hpp | 4 ++-- core-buffer/src/RamBuffer.cpp | 15 +++++++++------ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/core-buffer/include/RamBuffer.hpp b/core-buffer/include/RamBuffer.hpp index a202033..45b67a1 100644 --- a/core-buffer/include/RamBuffer.hpp +++ b/core-buffer/include/RamBuffer.hpp @@ -28,8 +28,8 @@ public: void write_frame(const ModuleFrame &src_meta, const char *src_data) const; void read_frame(const uint64_t pulse_id, const uint64_t module_id, - ModuleFrame *&meta, - char *&data) const; + ModuleFrame &meta, + char *data) const; char* read_image(const uint64_t pulse_id, ImageMetadata &image_meta) const; }; diff --git a/core-buffer/src/RamBuffer.cpp b/core-buffer/src/RamBuffer.cpp index 0179528..bcf614b 100644 --- a/core-buffer/src/RamBuffer.cpp +++ b/core-buffer/src/RamBuffer.cpp @@ -69,16 +69,19 @@ void RamBuffer::write_frame( void RamBuffer::read_frame( const uint64_t pulse_id, const uint64_t module_id, - ModuleFrame*& meta, - char*& data) const + ModuleFrame& dst_meta, + char* dst_data) const { const size_t slot_n = pulse_id % n_slots_; - meta = meta_buffer_ + (n_modules_ * slot_n) + module_id; + ModuleFrame *src_meta = meta_buffer_ + (n_modules_ * slot_n) + module_id; - data = image_buffer_ + - (image_bytes_ * slot_n) + - (MODULE_N_BYTES * module_id); + char *src_data = image_buffer_ + + (image_bytes_ * slot_n) + + (MODULE_N_BYTES * module_id); + + memcpy(&dst_meta, src_meta, sizeof(ModuleFrame)); + memcpy(dst_data, src_data, MODULE_N_BYTES); } char* RamBuffer::read_image(const uint64_t pulse_id, From 4c654f4f7990ed42fd2cd013272d2a8f08df1233 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Thu, 24 Sep 2020 11:59:54 +0200 Subject: [PATCH 046/126] Fix sf-buffer-writer main --- sf-buffer-writer/src/main.cpp | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/sf-buffer-writer/src/main.cpp b/sf-buffer-writer/src/main.cpp index fa4ef59..e9e2c71 100644 --- a/sf-buffer-writer/src/main.cpp +++ b/sf-buffer-writer/src/main.cpp @@ -40,33 +40,22 @@ int main (int argc, char *argv[]) { int source_id = atoi(argv[5]); BufferBinaryWriter writer(root_folder, device_name); - RamBuffer buffer(detector_name, n_modules); - - auto binary_buffer = new BufferBinaryFormat(); + RamBuffer ram_buff(detector_name, n_modules); + auto file_buff = new BufferBinaryFormat(); auto ctx = zmq_ctx_new(); auto socket = BufferUtils::connect_socket(ctx, detector_name, source_id); - ImageMetadata* meta = nullptr; - char* data = nullptr; - + uint64_t pulse_id; while (true) { - auto pulse_id = receiver.get_frame_from_udp( - binary_buffer->metadata, binary_buffer->data); + zmq_recv(socket, &pulse_id, sizeof(pulse_id), 0); - writer.write(pulse_id, binary_buffer); + // TODO: Memory copy here. Optimize this one out. + ram_buff.read_frame( + pulse_id, source_id, file_buff->meta, file_buff->data); - buffer.write_frame(&(binary_buffer->metadata), - &(binary_buffer->data[0])); - - zmq_send(socket, &pulse_id, sizeof(pulse_id), 0); - - if (binary_buffer->metadata.n_recv_packets < JF_N_PACKETS_PER_FRAME) { - n_missed_packets += JF_N_PACKETS_PER_FRAME - - binary_buffer->metadata.n_recv_packets; - n_corrupted_frames++; - } + writer.write(pulse_id, file_buff); stats_counter++; if (stats_counter == STATS_MODULO) { From 29652e0a25434554efa02d792c380c107957b1c8 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Thu, 24 Sep 2020 12:37:46 +0200 Subject: [PATCH 047/126] Transfer statistics into separate class --- sf-buffer-writer/src/main.cpp | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/sf-buffer-writer/src/main.cpp b/sf-buffer-writer/src/main.cpp index e9e2c71..5788a96 100644 --- a/sf-buffer-writer/src/main.cpp +++ b/sf-buffer-writer/src/main.cpp @@ -1,10 +1,8 @@ #include #include #include -#include -#include -#include #include +#include #include "formats.hpp" #include "BufferUtils.hpp" @@ -13,7 +11,6 @@ #include "BufferBinaryWriter.hpp" using namespace std; -using namespace chrono; using namespace buffer_config; int main (int argc, char *argv[]) { @@ -43,6 +40,8 @@ int main (int argc, char *argv[]) { RamBuffer ram_buff(detector_name, n_modules); auto file_buff = new BufferBinaryFormat(); + BufferStats stats(device_name, STATS_MODULO); + auto ctx = zmq_ctx_new(); auto socket = BufferUtils::connect_socket(ctx, detector_name, source_id); @@ -51,22 +50,14 @@ int main (int argc, char *argv[]) { zmq_recv(socket, &pulse_id, sizeof(pulse_id), 0); + stats.start_frame_write(); + // TODO: Memory copy here. Optimize this one out. ram_buff.read_frame( pulse_id, source_id, file_buff->meta, file_buff->data); writer.write(pulse_id, file_buff); - stats_counter++; - if (stats_counter == STATS_MODULO) { - cout << "sf_buffer:device_name " << device_name; - cout << " sf_buffer:n_missed_packets " << n_missed_packets; - cout << " sf_buffer:n_corrupted_frames " << n_corrupted_frames; - cout << endl; - - stats_counter = 0; - n_missed_packets = 0; - n_corrupted_frames = 0; - } + stats.end_frame_write(); } } From 92ebbde3307d7bee5f8fd82794e4c8b115a1fa00 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Thu, 24 Sep 2020 12:42:35 +0200 Subject: [PATCH 048/126] Create BufferStats class --- sf-buffer-writer/include/BufferStats.hpp | 28 +++++++++++++ sf-buffer-writer/src/BufferStats.cpp | 50 ++++++++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 sf-buffer-writer/include/BufferStats.hpp create mode 100644 sf-buffer-writer/src/BufferStats.cpp diff --git a/sf-buffer-writer/include/BufferStats.hpp b/sf-buffer-writer/include/BufferStats.hpp new file mode 100644 index 0000000..62f238e --- /dev/null +++ b/sf-buffer-writer/include/BufferStats.hpp @@ -0,0 +1,28 @@ +#include +#include +#include + +#ifndef SF_DAQ_BUFFER_FRAMESTATS_HPP +#define SF_DAQ_BUFFER_FRAMESTATS_HPP + + +class BufferStats { + const std::string source_name_; + size_t stats_modulo_; + + int frames_counter_; + uint32_t total_buffer_write_us_; + uint32_t max_buffer_write_us_; + std::chrono::time_point stats_interval_start_; + + void reset_counters(); + void print_stats(); + +public: + BufferStats(const std::string &source_name, const size_t stats_modulo); + void start_frame_write(); + void end_frame_write(); +}; + + +#endif //SF_DAQ_BUFFER_FRAMESTATS_HPP diff --git a/sf-buffer-writer/src/BufferStats.cpp b/sf-buffer-writer/src/BufferStats.cpp new file mode 100644 index 0000000..5e297e2 --- /dev/null +++ b/sf-buffer-writer/src/BufferStats.cpp @@ -0,0 +1,50 @@ +#include +#include "BufferStats.hpp" + +using namespace std; +using namespace chrono; + +BufferStats::BufferStats(const string& source_name, const size_t stats_modulo) : + source_name_(source_name), + stats_modulo_(stats_modulo) +{ + reset_counters(); +} + +void BufferStats::reset_counters() +{ + frames_counter_ = 0; + total_buffer_write_us_ = 0; + max_buffer_write_us_ = 0; +} + +void BufferStats::start_frame_write() +{ + stats_interval_start_ = steady_clock::now(); +} + +void BufferStats::end_frame_write() +{ + frames_counter_++; + + uint32_t write_us_duration = duration_cast( + steady_clock::now()-stats_interval_start_).count(); + + total_buffer_write_us_ += write_us_duration; + max_buffer_write_us_ = max(max_buffer_write_us_, write_us_duration); + + if (frames_counter_ == stats_modulo_) { + print_stats(); + reset_counters(); + } +} + +void BufferStats::print_stats() +{ + float avg_buffer_write_us = total_buffer_write_us_ / frames_counter_; + + cout << "sf_buffer:device_name " << source_name_; + cout << " sf_buffer:avg_buffer_write_us " << avg_buffer_write_us; + cout << " sf_buffer:max_buffer_write_us " << max_buffer_write_us_; + cout << endl; +} From 677f9c44c49aded9177554f3ac8e27d360dc93c8 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Thu, 24 Sep 2020 13:03:39 +0200 Subject: [PATCH 049/126] Since we are sending only pulse_ids adjust n threads --- sf-stream/include/stream_config.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sf-stream/include/stream_config.hpp b/sf-stream/include/stream_config.hpp index 614141c..9b7e693 100644 --- a/sf-stream/include/stream_config.hpp +++ b/sf-stream/include/stream_config.hpp @@ -1,7 +1,7 @@ namespace stream_config { // N of IO threads to receive data from modules. - const int STREAM_ZMQ_IO_THREADS = 2; + const int STREAM_ZMQ_IO_THREADS = 1; // How long should the RECV queue be. const size_t STREAM_RCVHWM = 100; // Size of buffer between the receiving and sending part. From 7e1d1dae2b76af10ceb0a61e1e6172332f2e2517 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Thu, 24 Sep 2020 13:04:28 +0200 Subject: [PATCH 050/126] Make use of common connect function in Pulse Receiver --- sf-stream/include/ZmqPulseReceiver.hpp | 10 +++--- sf-stream/src/ZmqPulseReceiver.cpp | 48 ++++++-------------------- 2 files changed, 16 insertions(+), 42 deletions(-) diff --git a/sf-stream/include/ZmqPulseReceiver.hpp b/sf-stream/include/ZmqPulseReceiver.hpp index 1590283..0b79781 100644 --- a/sf-stream/include/ZmqPulseReceiver.hpp +++ b/sf-stream/include/ZmqPulseReceiver.hpp @@ -10,16 +10,16 @@ class ZmqPulseReceiver { - const std::vector ipc_urls_; - const int n_modules_; void* ctx_; + const int n_modules_; std::vector sockets_; - void* connect_socket(const std::string url); - public: - ZmqPulseReceiver(const std::vector& ipc_urls, void* ctx); + ZmqPulseReceiver( + void* ctx, + const std::string& detector_name, + const int n_modules); ~ZmqPulseReceiver(); uint64_t get_next_pulse_id() const; diff --git a/sf-stream/src/ZmqPulseReceiver.cpp b/sf-stream/src/ZmqPulseReceiver.cpp index e21a94b..5d64813 100644 --- a/sf-stream/src/ZmqPulseReceiver.cpp +++ b/sf-stream/src/ZmqPulseReceiver.cpp @@ -1,4 +1,5 @@ #include "ZmqPulseReceiver.hpp" +#include "BufferUtils.hpp" #include #include @@ -16,16 +17,17 @@ using namespace stream_config; ZmqPulseReceiver::ZmqPulseReceiver( - const vector& ipc_urls, - void* ctx) : - ipc_urls_(ipc_urls), - n_modules_(ipc_urls_.size()), - ctx_(ctx) + void * ctx, + const string& detector_name, + const int n_modules) : + ctx_(ctx), + n_modules_(n_modules) { - sockets_.reserve(ipc_urls_.size()); + sockets_.reserve(n_modules_); - for (const auto& url : ipc_urls_) { - sockets_.push_back(connect_socket(url)); + for (int i=0; i Date: Thu, 24 Sep 2020 13:06:12 +0200 Subject: [PATCH 051/126] Rename zmq pulse receiver --- ...PulseReceiver.hpp => ZmqPulseSyncReceiver.hpp} | 12 ++++++------ ...PulseReceiver.cpp => ZmqPulseSyncReceiver.cpp} | 8 ++++---- sf-stream/src/main.cpp | 15 +++------------ 3 files changed, 13 insertions(+), 22 deletions(-) rename sf-stream/include/{ZmqPulseReceiver.hpp => ZmqPulseSyncReceiver.hpp} (58%) rename sf-stream/src/{ZmqPulseReceiver.cpp => ZmqPulseSyncReceiver.cpp} (93%) diff --git a/sf-stream/include/ZmqPulseReceiver.hpp b/sf-stream/include/ZmqPulseSyncReceiver.hpp similarity index 58% rename from sf-stream/include/ZmqPulseReceiver.hpp rename to sf-stream/include/ZmqPulseSyncReceiver.hpp index 0b79781..aa00035 100644 --- a/sf-stream/include/ZmqPulseReceiver.hpp +++ b/sf-stream/include/ZmqPulseSyncReceiver.hpp @@ -1,5 +1,5 @@ -#ifndef SF_DAQ_BUFFER_ZMQPULSERECEIVER_HPP -#define SF_DAQ_BUFFER_ZMQPULSERECEIVER_HPP +#ifndef SF_DAQ_BUFFER_ZMQPULSESYNCRECEIVER_HPP +#define SF_DAQ_BUFFER_ZMQPULSESYNCRECEIVER_HPP #include @@ -8,7 +8,7 @@ #include "formats.hpp" -class ZmqPulseReceiver { +class ZmqPulseSyncReceiver { void* ctx_; const int n_modules_; @@ -16,14 +16,14 @@ class ZmqPulseReceiver { std::vector sockets_; public: - ZmqPulseReceiver( + ZmqPulseSyncReceiver( void* ctx, const std::string& detector_name, const int n_modules); - ~ZmqPulseReceiver(); + ~ZmqPulseSyncReceiver(); uint64_t get_next_pulse_id() const; }; -#endif //SF_DAQ_BUFFER_ZMQPULSERECEIVER_HPP +#endif //SF_DAQ_BUFFER_ZMQPULSESYNCRECEIVER_HPP diff --git a/sf-stream/src/ZmqPulseReceiver.cpp b/sf-stream/src/ZmqPulseSyncReceiver.cpp similarity index 93% rename from sf-stream/src/ZmqPulseReceiver.cpp rename to sf-stream/src/ZmqPulseSyncReceiver.cpp index 5d64813..79de2e6 100644 --- a/sf-stream/src/ZmqPulseReceiver.cpp +++ b/sf-stream/src/ZmqPulseSyncReceiver.cpp @@ -1,4 +1,4 @@ -#include "ZmqPulseReceiver.hpp" +#include "ZmqPulseSyncReceiver.hpp" #include "BufferUtils.hpp" #include @@ -16,7 +16,7 @@ using namespace buffer_config; using namespace stream_config; -ZmqPulseReceiver::ZmqPulseReceiver( +ZmqPulseSyncReceiver::ZmqPulseSyncReceiver( void * ctx, const string& detector_name, const int n_modules) : @@ -31,14 +31,14 @@ ZmqPulseReceiver::ZmqPulseReceiver( } } -ZmqPulseReceiver::~ZmqPulseReceiver() +ZmqPulseSyncReceiver::~ZmqPulseSyncReceiver() { for (auto& socket:sockets_) { zmq_close(socket); } } -uint64_t ZmqPulseReceiver::get_next_pulse_id() const +uint64_t ZmqPulseSyncReceiver::get_next_pulse_id() const { uint64_t pulses[n_modules_]; diff --git a/sf-stream/src/main.cpp b/sf-stream/src/main.cpp index 0c3d500..9a5bcdd 100644 --- a/sf-stream/src/main.cpp +++ b/sf-stream/src/main.cpp @@ -1,17 +1,14 @@ #include #include -#include -#include #include #include #include "buffer_config.hpp" #include "stream_config.hpp" #include "ZmqLiveSender.hpp" -#include "ZmqPulseReceiver.hpp" +#include "ZmqPulseSyncReceiver.hpp" using namespace std; -using namespace chrono; using namespace buffer_config; using namespace stream_config; @@ -34,15 +31,9 @@ int main (int argc, char *argv[]) string RECV_IPC_URL = BUFFER_LIVE_IPC_URL + config.DETECTOR_NAME + "-"; auto ctx = zmq_ctx_new(); - zmq_ctx_set (ctx, ZMQ_IO_THREADS, STREAM_ZMQ_IO_THREADS); + zmq_ctx_set(ctx, ZMQ_IO_THREADS, STREAM_ZMQ_IO_THREADS); - // TODO: This should be passed to the service and not calculated here. - vector ipc_urls; - for (int i=0; i Date: Wed, 30 Sep 2020 10:45:40 +0200 Subject: [PATCH 052/126] Add configuration loading code to core-buffer --- core-buffer/include/BufferUtils.hpp | 16 ++++++++++++++++ core-buffer/src/BufferUtils.cpp | 24 ++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/core-buffer/include/BufferUtils.hpp b/core-buffer/include/BufferUtils.hpp index fa02a3f..f258d99 100644 --- a/core-buffer/include/BufferUtils.hpp +++ b/core-buffer/include/BufferUtils.hpp @@ -6,6 +6,20 @@ namespace BufferUtils { + + struct DetectorConfig { + const std::string streamvis_address; + const int reduction_factor_streamvis; + const std::string live_analysis_address; + const int reduction_factor_live_analysis; + const std::string PEDE_FILENAME; + const std::string GAIN_FILENAME; + const std::string DETECTOR_NAME; + const int n_modules; + const std::string pulse_address; + }; + + std::string get_filename( const std::string& detector_folder, const std::string& module_name, @@ -23,6 +37,8 @@ namespace BufferUtils void* ctx, const std::string& detector_name, const int source_id); void* connect_socket( void* ctx, const std::string& detector_name, const int source_id); + + DetectorConfig read_json_config(const std::string& filename); } #endif //BUFFER_UTILS_HPP diff --git a/core-buffer/src/BufferUtils.cpp b/core-buffer/src/BufferUtils.cpp index 388c945..69fd24d 100644 --- a/core-buffer/src/BufferUtils.cpp +++ b/core-buffer/src/BufferUtils.cpp @@ -3,6 +3,10 @@ #include #include #include +#include +#include +#include +#include using namespace std; using namespace buffer_config; @@ -122,3 +126,23 @@ void* BufferUtils::bind_socket( return socket; } + +BufferUtils::DetectorConfig read_json_config(const std::string& filename) +{ + std::ifstream ifs(filename); + rapidjson::IStreamWrapper isw(ifs); + rapidjson::Document config_parameters; + config_parameters.ParseStream(isw); + + return { + config_parameters["streamvis_stream"].GetString(), + config_parameters["streamvis_rate"].GetInt(), + config_parameters["live_stream"].GetString(), + config_parameters["live_rate"].GetInt(), + config_parameters["pedestal_file"].GetString(), + config_parameters["gain_file"].GetString(), + config_parameters["detector_name"].GetString(), + config_parameters["n_modules"].GetInt(), + "tcp://127.0.0.1:51234" + }; +} From 2c7ceb292da47b39156dcde7a52a93ac2ae903b7 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 30 Sep 2020 10:47:06 +0200 Subject: [PATCH 053/126] Cleanup core-buffer CMake file --- core-buffer/CMakeLists.txt | 39 ++------------------------------------ 1 file changed, 2 insertions(+), 37 deletions(-) diff --git a/core-buffer/CMakeLists.txt b/core-buffer/CMakeLists.txt index f8e3b5c..e87a3b2 100644 --- a/core-buffer/CMakeLists.txt +++ b/core-buffer/CMakeLists.txt @@ -5,43 +5,8 @@ file(GLOB SOURCES add_library(core-buffer-lib STATIC ${SOURCES}) target_include_directories(core-buffer-lib PUBLIC include/) - -#if(CMAKE_BUILD_TYPE STREQUAL "Debug") -# target_compile_definitions(core-buffer-lib PRIVATE DEBUG_OUTPUT) -#endif() - -#add_executable(sf-replay src/replay/sf_replay.cpp) -#set_target_properties(sf-replay PROPERTIES OUTPUT_NAME sf_replay) -#target_link_libraries(sf-replay -# core-buffer -# external -# zmq -# hdf5 -# hdf5_hl -# hdf5_cpp -# boost_system -# pthread) -# -#add_executable(sf-writer src/writer/sf_writer.cpp) -#set_target_properties(sf-writer PROPERTIES OUTPUT_NAME sf_writer) -#target_link_libraries(sf-writer -# core-buffer -# external -# zmq -# hdf5 -# hdf5_hl -# hdf5_cpp -# boost_system -# pthread) -# -#add_executable(sf-stream src/stream/sf_stream.cpp) -#set_target_properties(sf-stream PROPERTIES OUTPUT_NAME sf_stream) -#target_link_libraries(sf-stream -# core-buffer -# zmq -# jsoncpp -# boost_system -# pthread) +target_link_libraries(core-buffer-lib + external) enable_testing() add_subdirectory(test/) \ No newline at end of file From a7efe6492ebea751e2f625441814fae110d072bf Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 30 Sep 2020 10:48:13 +0200 Subject: [PATCH 054/126] Use core-buffer load config --- sf-stream/include/ZmqLiveSender.hpp | 25 +++---------------------- sf-stream/src/ZmqLiveSender.cpp | 25 ++++--------------------- sf-stream/src/main.cpp | 10 +++++----- 3 files changed, 12 insertions(+), 48 deletions(-) diff --git a/sf-stream/include/ZmqLiveSender.hpp b/sf-stream/include/ZmqLiveSender.hpp index 96e5424..4a0e02d 100644 --- a/sf-stream/include/ZmqLiveSender.hpp +++ b/sf-stream/include/ZmqLiveSender.hpp @@ -2,32 +2,13 @@ #define SF_DAQ_BUFFER_ZMQLIVESENDER_HPP #include -#include -#include -#include -#include -#include - #include "formats.hpp" +#include "BufferUtils.hpp" -struct LiveStreamConfig { - const std::string streamvis_address; - const int reduction_factor_streamvis; - const std::string live_analysis_address; - const int reduction_factor_live_analysis; - const std::string PEDE_FILENAME; - const std::string GAIN_FILENAME; - const std::string DETECTOR_NAME; - const int n_modules; - const std::string pulse_address; -}; - -LiveStreamConfig read_json_config(const std::string filename); - class ZmqLiveSender { const void* ctx_; - const LiveStreamConfig config_; + const BufferUtils::DetectorConfig config_; void* socket_streamvis_; void* socket_live_; @@ -35,7 +16,7 @@ class ZmqLiveSender { public: ZmqLiveSender(void* ctx, - const LiveStreamConfig& config); + const BufferUtils::DetectorConfig& config); ~ZmqLiveSender(); void send(const ImageMetadata& meta, const char* data); diff --git a/sf-stream/src/ZmqLiveSender.cpp b/sf-stream/src/ZmqLiveSender.cpp index d71485e..cf0ee64 100644 --- a/sf-stream/src/ZmqLiveSender.cpp +++ b/sf-stream/src/ZmqLiveSender.cpp @@ -3,33 +3,16 @@ #include "zmq.h" #include +#include +#include +#include using namespace std; using namespace stream_config; -LiveStreamConfig read_json_config(const std::string filename) -{ - std::ifstream ifs(filename); - rapidjson::IStreamWrapper isw(ifs); - rapidjson::Document config_parameters; - config_parameters.ParseStream(isw); - - return { - config_parameters["streamvis_stream"].GetString(), - config_parameters["streamvis_rate"].GetInt(), - config_parameters["live_stream"].GetString(), - config_parameters["live_rate"].GetInt(), - config_parameters["pedestal_file"].GetString(), - config_parameters["gain_file"].GetString(), - config_parameters["detector_name"].GetString(), - config_parameters["n_modules"].GetInt(), - "tcp://127.0.0.1:51234" - }; -} - ZmqLiveSender::ZmqLiveSender( void* ctx, - const LiveStreamConfig& config) : + const BufferUtils::DetectorConfig& config) : ctx_(ctx), config_(config) { diff --git a/sf-stream/src/main.cpp b/sf-stream/src/main.cpp index 9a5bcdd..8adb1f3 100644 --- a/sf-stream/src/main.cpp +++ b/sf-stream/src/main.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include "buffer_config.hpp" #include "stream_config.hpp" @@ -17,17 +18,16 @@ int main (int argc, char *argv[]) if (argc != 2) { cout << endl; cout << "Usage: sf_stream "; - cout << " [config_json_file]"; + cout << " [detector_json_filename]"; cout << endl; - cout << "\tconfig_json_file: json file with the configuration " - "parameters(detector name, number of modules, pedestal and " - "gain files" << endl; + cout << "\tdetector_json_filename: json file with the configuration " + "of the detector." << endl; cout << endl; exit(-1); } - auto config = read_json_config(string(argv[1])); + auto config = BufferUtils::read_json_config(string(argv[1])); string RECV_IPC_URL = BUFFER_LIVE_IPC_URL + config.DETECTOR_NAME + "-"; auto ctx = zmq_ctx_new(); From ae61ccae17a7bfd9e744d3be5d7b6421f9862cef Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 30 Sep 2020 11:34:54 +0200 Subject: [PATCH 055/126] Fix namespace of function --- core-buffer/src/BufferUtils.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core-buffer/src/BufferUtils.cpp b/core-buffer/src/BufferUtils.cpp index 69fd24d..b811d21 100644 --- a/core-buffer/src/BufferUtils.cpp +++ b/core-buffer/src/BufferUtils.cpp @@ -127,7 +127,8 @@ void* BufferUtils::bind_socket( return socket; } -BufferUtils::DetectorConfig read_json_config(const std::string& filename) +BufferUtils::DetectorConfig BufferUtils::read_json_config( + const std::string& filename) { std::ifstream ifs(filename); rapidjson::IStreamWrapper isw(ifs); From 5b3613295246efab4c371c14e81968df2760a482 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 30 Sep 2020 11:35:16 +0200 Subject: [PATCH 056/126] Fix build for sf_stream --- sf-stream/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sf-stream/CMakeLists.txt b/sf-stream/CMakeLists.txt index ada511a..9957767 100644 --- a/sf-stream/CMakeLists.txt +++ b/sf-stream/CMakeLists.txt @@ -10,6 +10,8 @@ target_link_libraries(sf-stream-lib add_executable(sf-stream src/main.cpp) set_target_properties(sf-stream PROPERTIES OUTPUT_NAME sf_stream) target_link_libraries(sf-stream + external + core-buffer-lib sf-stream-lib zmq pthread From f42298e264a46bb939112779251c2be91138a439 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 30 Sep 2020 11:38:39 +0200 Subject: [PATCH 057/126] Improve attribute reading from sf_stream --- sf-stream/src/main.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sf-stream/src/main.cpp b/sf-stream/src/main.cpp index 8adb1f3..987ee42 100644 --- a/sf-stream/src/main.cpp +++ b/sf-stream/src/main.cpp @@ -15,11 +15,9 @@ using namespace stream_config; int main (int argc, char *argv[]) { - if (argc != 2) { - cout << endl; - cout << "Usage: sf_stream "; - cout << " [detector_json_filename]"; + if (argc != 1) { cout << endl; + cout << "Usage: sf_stream [detector_json_filename]" << endl; cout << "\tdetector_json_filename: json file with the configuration " "of the detector." << endl; cout << endl; From 96100c7a7a43b855adcc1dded84f9a229dd1bd00 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 30 Sep 2020 11:39:41 +0200 Subject: [PATCH 058/126] Rename project --- {sf-buffer-recv => jf-udp-recv}/CMakeLists.txt | 0 {sf-buffer-recv => jf-udp-recv}/README.md | 0 {sf-buffer-recv => jf-udp-recv}/include/FrameStats.hpp | 0 {sf-buffer-recv => jf-udp-recv}/include/FrameUdpReceiver.hpp | 0 {sf-buffer-recv => jf-udp-recv}/include/PacketUdpReceiver.hpp | 0 {sf-buffer-recv => jf-udp-recv}/src/FrameStats.cpp | 0 {sf-buffer-recv => jf-udp-recv}/src/FrameUdpReceiver.cpp | 0 {sf-buffer-recv => jf-udp-recv}/src/PacketUdpReceiver.cpp | 0 {sf-buffer-recv => jf-udp-recv}/src/main.cpp | 0 {sf-buffer-recv => jf-udp-recv}/test/CMakeLists.txt | 0 {sf-buffer-recv => jf-udp-recv}/test/main.cpp | 0 {sf-buffer-recv => jf-udp-recv}/test/mock/udp.hpp | 0 {sf-buffer-recv => jf-udp-recv}/test/test_FrameUdpReceiver.cpp | 0 {sf-buffer-recv => jf-udp-recv}/test/test_PacketUdpReceiver.cpp | 0 14 files changed, 0 insertions(+), 0 deletions(-) rename {sf-buffer-recv => jf-udp-recv}/CMakeLists.txt (100%) rename {sf-buffer-recv => jf-udp-recv}/README.md (100%) rename {sf-buffer-recv => jf-udp-recv}/include/FrameStats.hpp (100%) rename {sf-buffer-recv => jf-udp-recv}/include/FrameUdpReceiver.hpp (100%) rename {sf-buffer-recv => jf-udp-recv}/include/PacketUdpReceiver.hpp (100%) rename {sf-buffer-recv => jf-udp-recv}/src/FrameStats.cpp (100%) rename {sf-buffer-recv => jf-udp-recv}/src/FrameUdpReceiver.cpp (100%) rename {sf-buffer-recv => jf-udp-recv}/src/PacketUdpReceiver.cpp (100%) rename {sf-buffer-recv => jf-udp-recv}/src/main.cpp (100%) rename {sf-buffer-recv => jf-udp-recv}/test/CMakeLists.txt (100%) rename {sf-buffer-recv => jf-udp-recv}/test/main.cpp (100%) rename {sf-buffer-recv => jf-udp-recv}/test/mock/udp.hpp (100%) rename {sf-buffer-recv => jf-udp-recv}/test/test_FrameUdpReceiver.cpp (100%) rename {sf-buffer-recv => jf-udp-recv}/test/test_PacketUdpReceiver.cpp (100%) diff --git a/sf-buffer-recv/CMakeLists.txt b/jf-udp-recv/CMakeLists.txt similarity index 100% rename from sf-buffer-recv/CMakeLists.txt rename to jf-udp-recv/CMakeLists.txt diff --git a/sf-buffer-recv/README.md b/jf-udp-recv/README.md similarity index 100% rename from sf-buffer-recv/README.md rename to jf-udp-recv/README.md diff --git a/sf-buffer-recv/include/FrameStats.hpp b/jf-udp-recv/include/FrameStats.hpp similarity index 100% rename from sf-buffer-recv/include/FrameStats.hpp rename to jf-udp-recv/include/FrameStats.hpp diff --git a/sf-buffer-recv/include/FrameUdpReceiver.hpp b/jf-udp-recv/include/FrameUdpReceiver.hpp similarity index 100% rename from sf-buffer-recv/include/FrameUdpReceiver.hpp rename to jf-udp-recv/include/FrameUdpReceiver.hpp diff --git a/sf-buffer-recv/include/PacketUdpReceiver.hpp b/jf-udp-recv/include/PacketUdpReceiver.hpp similarity index 100% rename from sf-buffer-recv/include/PacketUdpReceiver.hpp rename to jf-udp-recv/include/PacketUdpReceiver.hpp diff --git a/sf-buffer-recv/src/FrameStats.cpp b/jf-udp-recv/src/FrameStats.cpp similarity index 100% rename from sf-buffer-recv/src/FrameStats.cpp rename to jf-udp-recv/src/FrameStats.cpp diff --git a/sf-buffer-recv/src/FrameUdpReceiver.cpp b/jf-udp-recv/src/FrameUdpReceiver.cpp similarity index 100% rename from sf-buffer-recv/src/FrameUdpReceiver.cpp rename to jf-udp-recv/src/FrameUdpReceiver.cpp diff --git a/sf-buffer-recv/src/PacketUdpReceiver.cpp b/jf-udp-recv/src/PacketUdpReceiver.cpp similarity index 100% rename from sf-buffer-recv/src/PacketUdpReceiver.cpp rename to jf-udp-recv/src/PacketUdpReceiver.cpp diff --git a/sf-buffer-recv/src/main.cpp b/jf-udp-recv/src/main.cpp similarity index 100% rename from sf-buffer-recv/src/main.cpp rename to jf-udp-recv/src/main.cpp diff --git a/sf-buffer-recv/test/CMakeLists.txt b/jf-udp-recv/test/CMakeLists.txt similarity index 100% rename from sf-buffer-recv/test/CMakeLists.txt rename to jf-udp-recv/test/CMakeLists.txt diff --git a/sf-buffer-recv/test/main.cpp b/jf-udp-recv/test/main.cpp similarity index 100% rename from sf-buffer-recv/test/main.cpp rename to jf-udp-recv/test/main.cpp diff --git a/sf-buffer-recv/test/mock/udp.hpp b/jf-udp-recv/test/mock/udp.hpp similarity index 100% rename from sf-buffer-recv/test/mock/udp.hpp rename to jf-udp-recv/test/mock/udp.hpp diff --git a/sf-buffer-recv/test/test_FrameUdpReceiver.cpp b/jf-udp-recv/test/test_FrameUdpReceiver.cpp similarity index 100% rename from sf-buffer-recv/test/test_FrameUdpReceiver.cpp rename to jf-udp-recv/test/test_FrameUdpReceiver.cpp diff --git a/sf-buffer-recv/test/test_PacketUdpReceiver.cpp b/jf-udp-recv/test/test_PacketUdpReceiver.cpp similarity index 100% rename from sf-buffer-recv/test/test_PacketUdpReceiver.cpp rename to jf-udp-recv/test/test_PacketUdpReceiver.cpp From 5df36a81a0071e08ed814478fe4b1e3429a9c7b6 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 30 Sep 2020 11:44:02 +0200 Subject: [PATCH 059/126] Rename udp receiver --- CMakeLists.txt | 2 +- jf-udp-recv/CMakeLists.txt | 14 +++++++------- jf-udp-recv/test/CMakeLists.txt | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c490a02..d436e88 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,7 +29,7 @@ add_subdirectory( EXCLUDE_FROM_ALL) add_subdirectory("core-buffer") -add_subdirectory("sf-buffer-recv") +add_subdirectory("jf-udp-recv") add_subdirectory("sf-buffer-writer") add_subdirectory("sf-stream") add_subdirectory("sf-writer") diff --git a/jf-udp-recv/CMakeLists.txt b/jf-udp-recv/CMakeLists.txt index f523a4f..3c83127 100644 --- a/jf-udp-recv/CMakeLists.txt +++ b/jf-udp-recv/CMakeLists.txt @@ -1,16 +1,16 @@ file(GLOB SOURCES src/*.cpp) -add_library(sf-buffer-recv-lib STATIC ${SOURCES}) -target_include_directories(sf-buffer-recv-lib PUBLIC include/) -target_link_libraries(sf-buffer-recv-lib +add_library(jf-udp-recv-lib STATIC ${SOURCES}) +target_include_directories(jf-udp-recv-lib PUBLIC include/) +target_link_libraries(jf-udp-recv-lib external core-buffer-lib) -add_executable(sf-buffer-recv src/main.cpp) -set_target_properties(sf-buffer-recv PROPERTIES OUTPUT_NAME sf_buffer) -target_link_libraries(sf-buffer-recv - sf-buffer-recv-lib +add_executable(jf-udp-recv src/main.cpp) +set_target_properties(jf-udp-recv PROPERTIES OUTPUT_NAME jf_udp_recv) +target_link_libraries(jf-udp-recv + jf-udp-recv-lib zmq rt) diff --git a/jf-udp-recv/test/CMakeLists.txt b/jf-udp-recv/test/CMakeLists.txt index a03fdca..25c729a 100644 --- a/jf-udp-recv/test/CMakeLists.txt +++ b/jf-udp-recv/test/CMakeLists.txt @@ -1,8 +1,8 @@ -add_executable(sf-buffer-recv-tests main.cpp) +add_executable(jf-udp-recv-tests main.cpp) -target_link_libraries(sf-buffer-recv-tests +target_link_libraries(jf-udp-recv-tests core-buffer-lib - sf-buffer-recv-lib + jf-udp-recv-lib gtest ) From 1c5a498355fd3a71e382fafb9b7a10b4322c5574 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 30 Sep 2020 11:46:41 +0200 Subject: [PATCH 060/126] Rename project to jf-buffer-writer --- CMakeLists.txt | 2 +- jf-buffer-writer/CMakeLists.txt | 18 ++++++++++++++++++ .../include/BufferBinaryWriter.hpp | 0 .../include/BufferStats.hpp | 0 .../src/BufferBinaryWriter.cpp | 0 .../src/BufferStats.cpp | 0 .../src/main.cpp | 0 jf-buffer-writer/test/CMakeLists.txt | 7 +++++++ .../test/main.cpp | 0 .../test/test_BufferBinaryWriter.cpp | 0 sf-buffer-writer/CMakeLists.txt | 18 ------------------ sf-buffer-writer/test/CMakeLists.txt | 7 ------- 12 files changed, 26 insertions(+), 26 deletions(-) create mode 100644 jf-buffer-writer/CMakeLists.txt rename {sf-buffer-writer => jf-buffer-writer}/include/BufferBinaryWriter.hpp (100%) rename {sf-buffer-writer => jf-buffer-writer}/include/BufferStats.hpp (100%) rename {sf-buffer-writer => jf-buffer-writer}/src/BufferBinaryWriter.cpp (100%) rename {sf-buffer-writer => jf-buffer-writer}/src/BufferStats.cpp (100%) rename {sf-buffer-writer => jf-buffer-writer}/src/main.cpp (100%) create mode 100644 jf-buffer-writer/test/CMakeLists.txt rename {sf-buffer-writer => jf-buffer-writer}/test/main.cpp (100%) rename {sf-buffer-writer => jf-buffer-writer}/test/test_BufferBinaryWriter.cpp (100%) delete mode 100644 sf-buffer-writer/CMakeLists.txt delete mode 100644 sf-buffer-writer/test/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index d436e88..1fd9a7e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -30,7 +30,7 @@ add_subdirectory( add_subdirectory("core-buffer") add_subdirectory("jf-udp-recv") -add_subdirectory("sf-buffer-writer") +add_subdirectory("jf-buffer-writer") add_subdirectory("sf-stream") add_subdirectory("sf-writer") #add_subdirectory("jf-live-writer") diff --git a/jf-buffer-writer/CMakeLists.txt b/jf-buffer-writer/CMakeLists.txt new file mode 100644 index 0000000..a389502 --- /dev/null +++ b/jf-buffer-writer/CMakeLists.txt @@ -0,0 +1,18 @@ +file(GLOB SOURCES + src/*.cpp) + +add_library(jf-buffer-writer-lib STATIC ${SOURCES}) +target_include_directories(jf-buffer-writer-lib PUBLIC include/) +target_link_libraries(jf-buffer-writer-lib + external + core-buffer-lib) + +add_executable(jf-buffer-writer src/main.cpp) +set_target_properties(jf-buffer-writer PROPERTIES OUTPUT_NAME jf_buffer_writer) +target_link_libraries(jf-buffer-writer + jf-buffer-writer-lib + zmq + rt) + +enable_testing() +add_subdirectory(test/) diff --git a/sf-buffer-writer/include/BufferBinaryWriter.hpp b/jf-buffer-writer/include/BufferBinaryWriter.hpp similarity index 100% rename from sf-buffer-writer/include/BufferBinaryWriter.hpp rename to jf-buffer-writer/include/BufferBinaryWriter.hpp diff --git a/sf-buffer-writer/include/BufferStats.hpp b/jf-buffer-writer/include/BufferStats.hpp similarity index 100% rename from sf-buffer-writer/include/BufferStats.hpp rename to jf-buffer-writer/include/BufferStats.hpp diff --git a/sf-buffer-writer/src/BufferBinaryWriter.cpp b/jf-buffer-writer/src/BufferBinaryWriter.cpp similarity index 100% rename from sf-buffer-writer/src/BufferBinaryWriter.cpp rename to jf-buffer-writer/src/BufferBinaryWriter.cpp diff --git a/sf-buffer-writer/src/BufferStats.cpp b/jf-buffer-writer/src/BufferStats.cpp similarity index 100% rename from sf-buffer-writer/src/BufferStats.cpp rename to jf-buffer-writer/src/BufferStats.cpp diff --git a/sf-buffer-writer/src/main.cpp b/jf-buffer-writer/src/main.cpp similarity index 100% rename from sf-buffer-writer/src/main.cpp rename to jf-buffer-writer/src/main.cpp diff --git a/jf-buffer-writer/test/CMakeLists.txt b/jf-buffer-writer/test/CMakeLists.txt new file mode 100644 index 0000000..4e32fd8 --- /dev/null +++ b/jf-buffer-writer/test/CMakeLists.txt @@ -0,0 +1,7 @@ +add_executable(jf-buffer-writer-tests main.cpp) + +target_link_libraries(jf-buffer-writer-tests + jf-buffer-writer-lib + zmq + gtest + ) diff --git a/sf-buffer-writer/test/main.cpp b/jf-buffer-writer/test/main.cpp similarity index 100% rename from sf-buffer-writer/test/main.cpp rename to jf-buffer-writer/test/main.cpp diff --git a/sf-buffer-writer/test/test_BufferBinaryWriter.cpp b/jf-buffer-writer/test/test_BufferBinaryWriter.cpp similarity index 100% rename from sf-buffer-writer/test/test_BufferBinaryWriter.cpp rename to jf-buffer-writer/test/test_BufferBinaryWriter.cpp diff --git a/sf-buffer-writer/CMakeLists.txt b/sf-buffer-writer/CMakeLists.txt deleted file mode 100644 index 636556d..0000000 --- a/sf-buffer-writer/CMakeLists.txt +++ /dev/null @@ -1,18 +0,0 @@ -file(GLOB SOURCES - src/*.cpp) - -add_library(sf-buffer-writer-lib STATIC ${SOURCES}) -target_include_directories(sf-buffer-writer-lib PUBLIC include/) -target_link_libraries(sf-buffer-writer-lib - external - core-buffer-lib) - -add_executable(sf-buffer-writer src/main.cpp) -set_target_properties(sf-buffer-writer PROPERTIES OUTPUT_NAME sf_buffer) -target_link_libraries(sf-buffer-writer - sf-buffer-writer-lib - zmq - rt) - -enable_testing() -add_subdirectory(test/) diff --git a/sf-buffer-writer/test/CMakeLists.txt b/sf-buffer-writer/test/CMakeLists.txt deleted file mode 100644 index 7d1417c..0000000 --- a/sf-buffer-writer/test/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -add_executable(sf-buffer-writer-tests main.cpp) - -target_link_libraries(sf-buffer-writer-tests - sf-buffer-writer-lib - zmq - gtest - ) From 0afe9480dcf7e0a05c4914a42a17011195bd737b Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 30 Sep 2020 11:52:54 +0200 Subject: [PATCH 061/126] Rename source_id to module_id --- jf-udp-recv/include/FrameUdpReceiver.hpp | 4 ++-- jf-udp-recv/src/FrameUdpReceiver.cpp | 6 +++--- jf-udp-recv/src/main.cpp | 5 +++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/jf-udp-recv/include/FrameUdpReceiver.hpp b/jf-udp-recv/include/FrameUdpReceiver.hpp index 6dece49..eea4d46 100644 --- a/jf-udp-recv/include/FrameUdpReceiver.hpp +++ b/jf-udp-recv/include/FrameUdpReceiver.hpp @@ -7,7 +7,7 @@ #include "buffer_config.hpp" class FrameUdpReceiver { - const int source_id_; + const int module_id_; PacketUdpReceiver udp_receiver_; @@ -27,7 +27,7 @@ class FrameUdpReceiver { const int n_packets, ModuleFrame& metadata, char* frame_buffer); public: - FrameUdpReceiver(const uint16_t port, const int source_id); + FrameUdpReceiver(const uint16_t port, const int module_id); virtual ~FrameUdpReceiver(); uint64_t get_frame_from_udp(ModuleFrame& metadata, char* frame_buffer); }; diff --git a/jf-udp-recv/src/FrameUdpReceiver.cpp b/jf-udp-recv/src/FrameUdpReceiver.cpp index bcad86e..8bff530 100644 --- a/jf-udp-recv/src/FrameUdpReceiver.cpp +++ b/jf-udp-recv/src/FrameUdpReceiver.cpp @@ -7,8 +7,8 @@ using namespace buffer_config; FrameUdpReceiver::FrameUdpReceiver( const uint16_t port, - const int source_id) : - source_id_(source_id) + const int module_id) : + module_id_(module_id) { udp_receiver_.bind(port); @@ -33,7 +33,7 @@ inline void FrameUdpReceiver::init_frame( frame_metadata.pulse_id = packet_buffer_[i_packet].bunchid; frame_metadata.frame_index = packet_buffer_[i_packet].framenum; frame_metadata.daq_rec = (uint64_t) packet_buffer_[i_packet].debug; - frame_metadata.module_id = (int64_t) source_id_; + frame_metadata.module_id = (int64_t) module_id_; } inline void FrameUdpReceiver::copy_packet_to_buffers( diff --git a/jf-udp-recv/src/main.cpp b/jf-udp-recv/src/main.cpp index 905321b..298e816 100644 --- a/jf-udp-recv/src/main.cpp +++ b/jf-udp-recv/src/main.cpp @@ -31,15 +31,16 @@ int main (int argc, char *argv[]) { exit(-1); } + /usr/bin/sf_buffer ${DETECTOR} ${N_MODULES} M${M} ${port} /gpfs/photonics/swissfel/buffer/${DETECTOR} ${M} string detector_name = string(argv[1]); int n_modules = atoi(argv[2]); string device_name = string(argv[3]); int udp_port = atoi(argv[4]); string root_folder = string(argv[5]); - int source_id = atoi(argv[6]); + int module_id = atoi(argv[6]); - FrameUdpReceiver receiver(udp_port, source_id); + FrameUdpReceiver receiver(udp_port, module_id); RamBuffer buffer(detector_name, n_modules); auto ctx = zmq_ctx_new(); From 3d2121609401dfc5b7e2e0930c1910e569f61fed Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 30 Sep 2020 13:34:09 +0200 Subject: [PATCH 062/126] Add udp port to config file --- core-buffer/include/BufferUtils.hpp | 2 +- core-buffer/src/BufferUtils.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core-buffer/include/BufferUtils.hpp b/core-buffer/include/BufferUtils.hpp index f258d99..2295345 100644 --- a/core-buffer/include/BufferUtils.hpp +++ b/core-buffer/include/BufferUtils.hpp @@ -16,7 +16,7 @@ namespace BufferUtils const std::string GAIN_FILENAME; const std::string DETECTOR_NAME; const int n_modules; - const std::string pulse_address; + const int start_udp_port; }; diff --git a/core-buffer/src/BufferUtils.cpp b/core-buffer/src/BufferUtils.cpp index b811d21..62396b1 100644 --- a/core-buffer/src/BufferUtils.cpp +++ b/core-buffer/src/BufferUtils.cpp @@ -144,6 +144,6 @@ BufferUtils::DetectorConfig BufferUtils::read_json_config( config_parameters["gain_file"].GetString(), config_parameters["detector_name"].GetString(), config_parameters["n_modules"].GetInt(), - "tcp://127.0.0.1:51234" + config_parameters["start_udp_port"].GetInt() }; } From 79f6eaebc9db0a0d96ff2e3debeb22791e1d839f Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 30 Sep 2020 13:38:35 +0200 Subject: [PATCH 063/126] Shorten help string in console --- sf-stream/src/main.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sf-stream/src/main.cpp b/sf-stream/src/main.cpp index 987ee42..1e4d103 100644 --- a/sf-stream/src/main.cpp +++ b/sf-stream/src/main.cpp @@ -18,8 +18,7 @@ int main (int argc, char *argv[]) if (argc != 1) { cout << endl; cout << "Usage: sf_stream [detector_json_filename]" << endl; - cout << "\tdetector_json_filename: json file with the configuration " - "of the detector." << endl; + cout << "\tdetector_json_filename: detector config file path." << endl; cout << endl; exit(-1); From fd878691e2206b2b167b27cb26c09392446f3272 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 30 Sep 2020 13:40:43 +0200 Subject: [PATCH 064/126] Transition UDP receiver to detector config file --- jf-udp-recv/src/main.cpp | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/jf-udp-recv/src/main.cpp b/jf-udp-recv/src/main.cpp index 298e816..aa47330 100644 --- a/jf-udp-recv/src/main.cpp +++ b/jf-udp-recv/src/main.cpp @@ -16,37 +16,29 @@ using namespace buffer_config; int main (int argc, char *argv[]) { - if (argc != 6) { + if (argc != 2) { cout << endl; - cout << "Usage: sf_buffer_recv [detector_name] [n_modules]"; - cout << " [device_name] [udp_port] [root_folder] [source_id]"; + cout << "Usage: jf_udp_recv [detector_json_filename] [module_id]"; cout << endl; - cout << "\tdetector_name: Detector name, example JF07T32V01" << endl; - cout << "\tn_modules: Number of modules in the detector." << endl; - cout << "\tdevice_name: Name to write to disk." << endl; - cout << "\tudp_port: UDP port to connect to." << endl; - cout << "\troot_folder: FS root folder." << endl; - cout << "\tsource_id: ID of the source for live stream." << endl; + cout << "\tdetector_json_filename: detector config file path." << endl; + cout << "\tmodule_id: id of the module for this process." << endl; cout << endl; exit(-1); } - /usr/bin/sf_buffer ${DETECTOR} ${N_MODULES} M${M} ${port} /gpfs/photonics/swissfel/buffer/${DETECTOR} ${M} - string detector_name = string(argv[1]); - int n_modules = atoi(argv[2]); - string device_name = string(argv[3]); - int udp_port = atoi(argv[4]); - string root_folder = string(argv[5]); - int module_id = atoi(argv[6]); + auto config = BufferUtils::read_json_config(string(argv[1])); + int module_id = atoi(argv[2]); + auto udp_port = config.start_udp_port + module_id; FrameUdpReceiver receiver(udp_port, module_id); - RamBuffer buffer(detector_name, n_modules); + RamBuffer buffer(config.DETECTOR_NAME, config.n_modules); auto ctx = zmq_ctx_new(); - auto socket = BufferUtils::bind_socket(ctx, detector_name, source_id); + auto socket = BufferUtils::bind_socket( + ctx, config.DETECTOR_NAME, module_id); - FrameStats stats(device_name, STATS_MODULO); + FrameStats stats("M" + to_string(module_id), STATS_MODULO); ModuleFrame meta; char* data = new char[MODULE_N_BYTES]; From b8b152e5ed20dcdeb38553514cf58cc203a2d12f Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 30 Sep 2020 13:49:57 +0200 Subject: [PATCH 065/126] Rename Detector Name to lower case --- core-buffer/include/BufferUtils.hpp | 3 ++- jf-udp-recv/src/main.cpp | 15 +++++++-------- sf-stream/src/ZmqLiveSender.cpp | 2 +- sf-stream/src/main.cpp | 6 +++--- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/core-buffer/include/BufferUtils.hpp b/core-buffer/include/BufferUtils.hpp index 2295345..ab33f69 100644 --- a/core-buffer/include/BufferUtils.hpp +++ b/core-buffer/include/BufferUtils.hpp @@ -14,7 +14,8 @@ namespace BufferUtils const int reduction_factor_live_analysis; const std::string PEDE_FILENAME; const std::string GAIN_FILENAME; - const std::string DETECTOR_NAME; + + const std::string detector_name; const int n_modules; const int start_udp_port; }; diff --git a/jf-udp-recv/src/main.cpp b/jf-udp-recv/src/main.cpp index aa47330..28a8350 100644 --- a/jf-udp-recv/src/main.cpp +++ b/jf-udp-recv/src/main.cpp @@ -13,6 +13,7 @@ using namespace std; using namespace chrono; using namespace buffer_config; +using namespace BufferUtils; int main (int argc, char *argv[]) { @@ -27,18 +28,16 @@ int main (int argc, char *argv[]) { exit(-1); } - auto config = BufferUtils::read_json_config(string(argv[1])); - int module_id = atoi(argv[2]); + const auto config = read_json_config(string(argv[1])); + const int module_id = atoi(argv[2]); - auto udp_port = config.start_udp_port + module_id; + const auto udp_port = config.start_udp_port + module_id; FrameUdpReceiver receiver(udp_port, module_id); - RamBuffer buffer(config.DETECTOR_NAME, config.n_modules); + RamBuffer buffer(config.detector_name, config.n_modules); + FrameStats stats("M" + to_string(module_id), STATS_MODULO); auto ctx = zmq_ctx_new(); - auto socket = BufferUtils::bind_socket( - ctx, config.DETECTOR_NAME, module_id); - - FrameStats stats("M" + to_string(module_id), STATS_MODULO); + auto socket = bind_socket(ctx, config.detector_name, module_id); ModuleFrame meta; char* data = new char[MODULE_N_BYTES]; diff --git a/sf-stream/src/ZmqLiveSender.cpp b/sf-stream/src/ZmqLiveSender.cpp index cf0ee64..04f3d05 100644 --- a/sf-stream/src/ZmqLiveSender.cpp +++ b/sf-stream/src/ZmqLiveSender.cpp @@ -106,7 +106,7 @@ void ZmqLiveSender::send(const ImageMetadata& meta, const char *data) header.AddMember("run_name", run_name, header_alloc); rapidjson::Value detector_name; - detector_name.SetString(config_.DETECTOR_NAME.c_str(), header_alloc); + detector_name.SetString(config_.detector_name.c_str(), header_alloc); header.AddMember("detector_name", detector_name, header_alloc); header.AddMember("htype", "array-1.0", header_alloc); diff --git a/sf-stream/src/main.cpp b/sf-stream/src/main.cpp index 1e4d103..45cc85d 100644 --- a/sf-stream/src/main.cpp +++ b/sf-stream/src/main.cpp @@ -25,13 +25,13 @@ int main (int argc, char *argv[]) } auto config = BufferUtils::read_json_config(string(argv[1])); - string RECV_IPC_URL = BUFFER_LIVE_IPC_URL + config.DETECTOR_NAME + "-"; + string RECV_IPC_URL = BUFFER_LIVE_IPC_URL + config.detector_name + "-"; auto ctx = zmq_ctx_new(); zmq_ctx_set(ctx, ZMQ_IO_THREADS, STREAM_ZMQ_IO_THREADS); - ZmqPulseSyncReceiver receiver(ctx, config.DETECTOR_NAME, config.n_modules); - RamBuffer ram_buffer(config.DETECTOR_NAME, config.n_modules); + ZmqPulseSyncReceiver receiver(ctx, config.detector_name, config.n_modules); + RamBuffer ram_buffer(config.detector_name, config.n_modules); ZmqLiveSender sender(ctx, config); // TODO: Remove stats trash. From 430a1507ff1f5bbc5c70abc3e95a3f3016869203 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 30 Sep 2020 13:56:41 +0200 Subject: [PATCH 066/126] Add buffer folder to detector config --- core-buffer/include/BufferUtils.hpp | 1 + core-buffer/src/BufferUtils.cpp | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/core-buffer/include/BufferUtils.hpp b/core-buffer/include/BufferUtils.hpp index ab33f69..bb09dea 100644 --- a/core-buffer/include/BufferUtils.hpp +++ b/core-buffer/include/BufferUtils.hpp @@ -18,6 +18,7 @@ namespace BufferUtils const std::string detector_name; const int n_modules; const int start_udp_port; + const std::string buffer_folder; }; diff --git a/core-buffer/src/BufferUtils.cpp b/core-buffer/src/BufferUtils.cpp index 62396b1..4e861df 100644 --- a/core-buffer/src/BufferUtils.cpp +++ b/core-buffer/src/BufferUtils.cpp @@ -144,6 +144,7 @@ BufferUtils::DetectorConfig BufferUtils::read_json_config( config_parameters["gain_file"].GetString(), config_parameters["detector_name"].GetString(), config_parameters["n_modules"].GetInt(), - config_parameters["start_udp_port"].GetInt() + config_parameters["start_udp_port"].GetInt(), + config_parameters["buffer_folder"].GetString() }; } From bf6bba801db265e4c8fd3d10627967d66e7b013f Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 30 Sep 2020 14:02:48 +0200 Subject: [PATCH 067/126] Make it more explicit --- jf-udp-recv/src/main.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jf-udp-recv/src/main.cpp b/jf-udp-recv/src/main.cpp index 28a8350..af31050 100644 --- a/jf-udp-recv/src/main.cpp +++ b/jf-udp-recv/src/main.cpp @@ -34,7 +34,8 @@ int main (int argc, char *argv[]) { const auto udp_port = config.start_udp_port + module_id; FrameUdpReceiver receiver(udp_port, module_id); RamBuffer buffer(config.detector_name, config.n_modules); - FrameStats stats("M" + to_string(module_id), STATS_MODULO); + const string module_name = "M" + to_string(module_id); + FrameStats stats(module_name, STATS_MODULO); auto ctx = zmq_ctx_new(); auto socket = bind_socket(ctx, config.detector_name, module_id); From 8a3d0d1baa97f8bbd8b21af7a9d9922078d824c6 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 30 Sep 2020 14:17:33 +0200 Subject: [PATCH 068/126] Simplify buffer writer --- jf-buffer-writer/src/main.cpp | 35 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/jf-buffer-writer/src/main.cpp b/jf-buffer-writer/src/main.cpp index 5788a96..1046802 100644 --- a/jf-buffer-writer/src/main.cpp +++ b/jf-buffer-writer/src/main.cpp @@ -12,40 +12,35 @@ using namespace std; using namespace buffer_config; +using namespace BufferUtils; int main (int argc, char *argv[]) { - if (argc != 6) { + if (argc != 2) { cout << endl; - cout << "Usage: sf_buffer_writer [detector_name] [n_modules]"; - cout << " [device_name] [root_folder] [source_id]"; + cout << "Usage: jf_buffer_writer [detector_json_filename] [module_id]"; cout << endl; - cout << "\tdetector_name: Detector name, example JF07T32V01" << endl; - cout << "\tn_modules: Number of modules in the detector." << endl; - cout << "\tdevice_name: Name to write to disk." << endl; - cout << "\troot_folder: FS root folder." << endl; - cout << "\tsource_id: ID of the source for live stream." << endl; + cout << "\tdetector_json_filename: detector config file path." << endl; + cout << "\tmodule_id: id of the module for this process." << endl; cout << endl; exit(-1); } - string detector_name = string(argv[1]); - int n_modules = atoi(argv[2]); - string device_name = string(argv[3]); - string root_folder = string(argv[4]); - int source_id = atoi(argv[5]); + const auto config = read_json_config(string(argv[1])); + const int module_id = atoi(argv[2]); - BufferBinaryWriter writer(root_folder, device_name); - RamBuffer ram_buff(detector_name, n_modules); - auto file_buff = new BufferBinaryFormat(); - - BufferStats stats(device_name, STATS_MODULO); + const auto module_name = "M" + to_string(module_id); + BufferBinaryWriter writer(config.buffer_folder, module_name); + RamBuffer ram_buff(config.detector_name, config.n_modules); + BufferStats stats(module_name, STATS_MODULO); auto ctx = zmq_ctx_new(); - auto socket = BufferUtils::connect_socket(ctx, detector_name, source_id); + auto socket = connect_socket(ctx, config.detector_name, module_id); + auto file_buff = new BufferBinaryFormat(); uint64_t pulse_id; + while (true) { zmq_recv(socket, &pulse_id, sizeof(pulse_id), 0); @@ -54,7 +49,7 @@ int main (int argc, char *argv[]) { // TODO: Memory copy here. Optimize this one out. ram_buff.read_frame( - pulse_id, source_id, file_buff->meta, file_buff->data); + pulse_id, module_id, file_buff->meta, file_buff->data); writer.write(pulse_id, file_buff); From db4aeacad9e0b69219088d75d888aebe6c5261bd Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 30 Sep 2020 17:03:40 +0200 Subject: [PATCH 069/126] Whitespace fix --- jf-buffer-writer/src/main.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/jf-buffer-writer/src/main.cpp b/jf-buffer-writer/src/main.cpp index 1046802..83eba59 100644 --- a/jf-buffer-writer/src/main.cpp +++ b/jf-buffer-writer/src/main.cpp @@ -42,7 +42,6 @@ int main (int argc, char *argv[]) { uint64_t pulse_id; while (true) { - zmq_recv(socket, &pulse_id, sizeof(pulse_id), 0); stats.start_frame_write(); From 9c4478343b275705a0d4f699233e13e566e29048 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Thu, 1 Oct 2020 09:39:08 +0200 Subject: [PATCH 070/126] Change logs output format to InfluxDB --- jf-udp-recv/include/FrameStats.hpp | 7 +++++-- jf-udp-recv/src/FrameStats.cpp | 28 +++++++++++++++++++++------- jf-udp-recv/src/main.cpp | 3 +-- 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/jf-udp-recv/include/FrameStats.hpp b/jf-udp-recv/include/FrameStats.hpp index 9d6fa71..dd4ef95 100644 --- a/jf-udp-recv/include/FrameStats.hpp +++ b/jf-udp-recv/include/FrameStats.hpp @@ -7,7 +7,8 @@ class FrameStats { - const std::string source_name_; + const std::string detector_name_; + const int module_id_; size_t stats_modulo_; int frames_counter_; @@ -19,7 +20,9 @@ class FrameStats { void print_stats(); public: - FrameStats(const std::string &source_name, const size_t stats_modulo); + FrameStats(const std::string &detector_name, + const int module_id, + const size_t stats_modulo); void record_stats(const ModuleFrame &meta); }; diff --git a/jf-udp-recv/src/FrameStats.cpp b/jf-udp-recv/src/FrameStats.cpp index ded7649..d25493a 100644 --- a/jf-udp-recv/src/FrameStats.cpp +++ b/jf-udp-recv/src/FrameStats.cpp @@ -4,9 +4,13 @@ using namespace std; using namespace chrono; -FrameStats::FrameStats(const string& source_name, const size_t stats_modulo) : - source_name_(source_name), - stats_modulo_(stats_modulo) +FrameStats::FrameStats( + const std::string &detector_name, + const int module_id, + const size_t stats_modulo) : + detector_name_(detector_name), + module_id_(module_id), + stats_modulo_(stats_modulo) { reset_counters(); } @@ -41,9 +45,19 @@ void FrameStats::print_stats() // * 1000 because milliseconds, 0.5 for truncation. int rep_rate = ((frames_counter_/interval_ms_duration) * 1000) + 0.5; - cout << "sf_buffer:device_name " << source_name_; - cout << " sf_buffer:n_missed_packets " << n_missed_packets_; - cout << " sf_buffer:n_corrupted_frames " << n_corrupted_frames_; - cout << " sf_buffer:repetition_rate " << rep_rate; + uint64_t timestamp = time_point_cast( + system_clock::now()).time_since_epoch().count(); + + // Output in InfluxDB line protocol + cout << "jf-udp-recv"; + cout << ",detector_name=" << detector_name_; + cout << ",module_name=M" << module_id_; + cout << " "; + cout << ",n_missed_packets=" << n_missed_packets_ << "i"; + cout << ",n_corrupted_frames=" << n_corrupted_frames_ << "i"; + cout << ",repetition_rate=" << rep_rate << "i"; + cout << " "; + cout << timestamp; + cout << endl; } diff --git a/jf-udp-recv/src/main.cpp b/jf-udp-recv/src/main.cpp index af31050..6d9961e 100644 --- a/jf-udp-recv/src/main.cpp +++ b/jf-udp-recv/src/main.cpp @@ -34,8 +34,7 @@ int main (int argc, char *argv[]) { const auto udp_port = config.start_udp_port + module_id; FrameUdpReceiver receiver(udp_port, module_id); RamBuffer buffer(config.detector_name, config.n_modules); - const string module_name = "M" + to_string(module_id); - FrameStats stats(module_name, STATS_MODULO); + FrameStats stats(config.detector_name, module_id, STATS_MODULO); auto ctx = zmq_ctx_new(); auto socket = bind_socket(ctx, config.detector_name, module_id); From d776e78ab0366d6749077e83c1b2b2a94ff62680 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Thu, 1 Oct 2020 09:42:22 +0200 Subject: [PATCH 071/126] Fix extra character in logs output --- jf-udp-recv/src/FrameStats.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jf-udp-recv/src/FrameStats.cpp b/jf-udp-recv/src/FrameStats.cpp index d25493a..92815cb 100644 --- a/jf-udp-recv/src/FrameStats.cpp +++ b/jf-udp-recv/src/FrameStats.cpp @@ -53,11 +53,10 @@ void FrameStats::print_stats() cout << ",detector_name=" << detector_name_; cout << ",module_name=M" << module_id_; cout << " "; - cout << ",n_missed_packets=" << n_missed_packets_ << "i"; + cout << "n_missed_packets=" << n_missed_packets_ << "i"; cout << ",n_corrupted_frames=" << n_corrupted_frames_ << "i"; cout << ",repetition_rate=" << rep_rate << "i"; cout << " "; cout << timestamp; - cout << endl; } From 2399b8d59e624df57d41533a5e4713c8ba65072d Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Thu, 1 Oct 2020 10:02:58 +0200 Subject: [PATCH 072/126] Fixed InfluxDB statistics on jf-buffer-writer --- jf-buffer-writer/include/BufferStats.hpp | 8 ++++++-- jf-buffer-writer/src/BufferStats.cpp | 25 ++++++++++++++++++------ jf-buffer-writer/src/main.cpp | 2 +- 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/jf-buffer-writer/include/BufferStats.hpp b/jf-buffer-writer/include/BufferStats.hpp index 62f238e..3aff6cb 100644 --- a/jf-buffer-writer/include/BufferStats.hpp +++ b/jf-buffer-writer/include/BufferStats.hpp @@ -7,7 +7,8 @@ class BufferStats { - const std::string source_name_; + const std::string detector_name_; + const int module_id_; size_t stats_modulo_; int frames_counter_; @@ -19,7 +20,10 @@ class BufferStats { void print_stats(); public: - BufferStats(const std::string &source_name, const size_t stats_modulo); + BufferStats( + const std::string &detector_name, + const int module_id, + const size_t stats_modulo); void start_frame_write(); void end_frame_write(); }; diff --git a/jf-buffer-writer/src/BufferStats.cpp b/jf-buffer-writer/src/BufferStats.cpp index 5e297e2..325e930 100644 --- a/jf-buffer-writer/src/BufferStats.cpp +++ b/jf-buffer-writer/src/BufferStats.cpp @@ -4,9 +4,13 @@ using namespace std; using namespace chrono; -BufferStats::BufferStats(const string& source_name, const size_t stats_modulo) : - source_name_(source_name), - stats_modulo_(stats_modulo) +BufferStats::BufferStats( + const string& detector_name, + const int module_id, + const size_t stats_modulo) : + detector_name_(detector_name), + module_id_(module_id), + stats_modulo_(stats_modulo) { reset_counters(); } @@ -43,8 +47,17 @@ void BufferStats::print_stats() { float avg_buffer_write_us = total_buffer_write_us_ / frames_counter_; - cout << "sf_buffer:device_name " << source_name_; - cout << " sf_buffer:avg_buffer_write_us " << avg_buffer_write_us; - cout << " sf_buffer:max_buffer_write_us " << max_buffer_write_us_; + uint64_t timestamp = time_point_cast( + system_clock::now()).time_since_epoch().count(); + + // Output in InfluxDB line protocol + cout << "jf-buffer-writer"; + cout << ",detector_name=" << detector_name_; + cout << ",module_name=M" << module_id_; + cout << " "; + cout << "avg_buffer_write_us=" << avg_buffer_write_us; + cout << ",max_buffer_write_us=" << max_buffer_write_us_ << "i"; + cout << " "; + cout << timestamp; cout << endl; } diff --git a/jf-buffer-writer/src/main.cpp b/jf-buffer-writer/src/main.cpp index 83eba59..81c7705 100644 --- a/jf-buffer-writer/src/main.cpp +++ b/jf-buffer-writer/src/main.cpp @@ -33,7 +33,7 @@ int main (int argc, char *argv[]) { const auto module_name = "M" + to_string(module_id); BufferBinaryWriter writer(config.buffer_folder, module_name); RamBuffer ram_buff(config.detector_name, config.n_modules); - BufferStats stats(module_name, STATS_MODULO); + BufferStats stats(config.detector_name, module_id, STATS_MODULO); auto ctx = zmq_ctx_new(); auto socket = connect_socket(ctx, config.detector_name, module_id); From b00ce59fa0aaaa95f74367252946c0fbcbf1d7e2 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 12 Jan 2021 13:44:05 +0100 Subject: [PATCH 073/126] Fix the number of input parameters to the executable --- jf-udp-recv/src/main.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jf-udp-recv/src/main.cpp b/jf-udp-recv/src/main.cpp index 6d9961e..49668a1 100644 --- a/jf-udp-recv/src/main.cpp +++ b/jf-udp-recv/src/main.cpp @@ -1,7 +1,6 @@ #include #include #include -#include #include #include "formats.hpp" @@ -17,7 +16,7 @@ using namespace BufferUtils; int main (int argc, char *argv[]) { - if (argc != 2) { + if (argc != 3) { cout << endl; cout << "Usage: jf_udp_recv [detector_json_filename] [module_id]"; cout << endl; From d3c36413111255cdd5fba31121bcf022e5ce8f71 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 12 Jan 2021 13:44:40 +0100 Subject: [PATCH 074/126] Fix indent in for loop --- jf-udp-recv/src/FrameUdpReceiver.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/jf-udp-recv/src/FrameUdpReceiver.cpp b/jf-udp-recv/src/FrameUdpReceiver.cpp index 8bff530..1c0833b 100644 --- a/jf-udp-recv/src/FrameUdpReceiver.cpp +++ b/jf-udp-recv/src/FrameUdpReceiver.cpp @@ -54,10 +54,9 @@ inline uint64_t FrameUdpReceiver::process_packets( ModuleFrame& metadata, char* frame_buffer) { - for ( - int i_packet=start_offset; - i_packet < packet_buffer_n_packets_; - i_packet++) { + for (int i_packet=start_offset; + i_packet < packet_buffer_n_packets_; + i_packet++) { // First packet for this frame. if (metadata.pulse_id == 0) { From 12a426f63af8dea623e4cc9b39b5cf122f3db0a4 Mon Sep 17 00:00:00 2001 From: Dmitry Ozerov Date: Fri, 20 Nov 2020 11:50:34 +0100 Subject: [PATCH 075/126] changes in jungfrau header package for 5.0.0 slsDetector version --- core-buffer/include/jungfrau.hpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/core-buffer/include/jungfrau.hpp b/core-buffer/include/jungfrau.hpp index 8e9e333..99156e6 100644 --- a/core-buffer/include/jungfrau.hpp +++ b/core-buffer/include/jungfrau.hpp @@ -4,16 +4,15 @@ #include #define JUNGFRAU_N_MODULES 32 -#define JUNGFRAU_BYTES_PER_PACKET 8246 +#define JUNGFRAU_BYTES_PER_PACKET 8240 #define JUNGFRAU_DATA_BYTES_PER_PACKET 8192 #define JF_N_PACKETS_PER_FRAME 128 #define JUNGFRAU_DATA_BYTES_PER_FRAME 1048576 -// 6 bytes + 48 bytes + 8192 bytes = 8246 bytes +// 48 bytes + 8192 bytes = 8240 bytes #pragma pack(push) #pragma pack(2) struct jungfrau_packet { - char emptyheader[6]; uint64_t framenum; uint32_t exptime; uint32_t packetnum; @@ -35,4 +34,4 @@ struct jungfrau_packet { #pragma pack(pop) -#endif \ No newline at end of file +#endif From 19e08c3718e9ca3505d3fc6d911acaa263750f99 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 12 Jan 2021 16:08:08 +0100 Subject: [PATCH 076/126] Invert the clocks to get a positive duration --- jf-udp-recv/src/FrameStats.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jf-udp-recv/src/FrameStats.cpp b/jf-udp-recv/src/FrameStats.cpp index 92815cb..c3c91f4 100644 --- a/jf-udp-recv/src/FrameStats.cpp +++ b/jf-udp-recv/src/FrameStats.cpp @@ -41,7 +41,7 @@ void FrameStats::record_stats(const ModuleFrame &meta) void FrameStats::print_stats() { auto interval_ms_duration = duration_cast( - stats_interval_start_-steady_clock::now()).count(); + steady_clock::now()-stats_interval_start_).count(); // * 1000 because milliseconds, 0.5 for truncation. int rep_rate = ((frames_counter_/interval_ms_duration) * 1000) + 0.5; From 87e47173d121d9dd61aa8e8af13a35a26c5ed214 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 12 Jan 2021 16:19:58 +0100 Subject: [PATCH 077/126] Fix repetition rate in statistics to nice numbers --- jf-udp-recv/src/FrameStats.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/jf-udp-recv/src/FrameStats.cpp b/jf-udp-recv/src/FrameStats.cpp index c3c91f4..8beb291 100644 --- a/jf-udp-recv/src/FrameStats.cpp +++ b/jf-udp-recv/src/FrameStats.cpp @@ -42,9 +42,8 @@ void FrameStats::print_stats() { auto interval_ms_duration = duration_cast( steady_clock::now()-stats_interval_start_).count(); - // * 1000 because milliseconds, 0.5 for truncation. - int rep_rate = ((frames_counter_/interval_ms_duration) * 1000) + 0.5; - + // * 1000 because milliseconds, + 250 because of truncation. + int rep_rate = ((frames_counter_ * 1000) + 250) / interval_ms_duration; uint64_t timestamp = time_point_cast( system_clock::now()).time_since_epoch().count(); From 4c914ad66eb2e57d0adebd973a64293c1b9a1847 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 12 Jan 2021 18:17:15 +0100 Subject: [PATCH 078/126] Fix number of parameters check in main --- jf-buffer-writer/src/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jf-buffer-writer/src/main.cpp b/jf-buffer-writer/src/main.cpp index 81c7705..c8eb032 100644 --- a/jf-buffer-writer/src/main.cpp +++ b/jf-buffer-writer/src/main.cpp @@ -16,7 +16,7 @@ using namespace BufferUtils; int main (int argc, char *argv[]) { - if (argc != 2) { + if (argc != 3) { cout << endl; cout << "Usage: jf_buffer_writer [detector_json_filename] [module_id]"; cout << endl; From c90114bec99e497b6810ca0fdcd24d9140265097 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 13 Jan 2021 12:45:57 +0100 Subject: [PATCH 079/126] Add 0 in module_id < 10 --- jf-buffer-writer/src/main.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/jf-buffer-writer/src/main.cpp b/jf-buffer-writer/src/main.cpp index c8eb032..e1eb961 100644 --- a/jf-buffer-writer/src/main.cpp +++ b/jf-buffer-writer/src/main.cpp @@ -30,7 +30,10 @@ int main (int argc, char *argv[]) { const auto config = read_json_config(string(argv[1])); const int module_id = atoi(argv[2]); - const auto module_name = "M" + to_string(module_id); + // TODO: Remove this also in reader. + const string module_prefix = (module_id < 10) ? "0" : ""; + const auto module_name = "M" + module_prefix + to_string(module_id); + BufferBinaryWriter writer(config.buffer_folder, module_name); RamBuffer ram_buff(config.detector_name, config.n_modules); BufferStats stats(config.detector_name, module_id, STATS_MODULO); From 6ee8f7d89d494a63d2abf409c433a4d56d3daf85 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 13 Jan 2021 15:54:28 +0100 Subject: [PATCH 080/126] Remove pulse_id stream from streamer This is not needed anymore since we have a different architecture. --- sf-stream/include/ZmqLiveSender.hpp | 1 - sf-stream/src/ZmqLiveSender.cpp | 24 ------------------------ 2 files changed, 25 deletions(-) diff --git a/sf-stream/include/ZmqLiveSender.hpp b/sf-stream/include/ZmqLiveSender.hpp index 4a0e02d..5a0a1b4 100644 --- a/sf-stream/include/ZmqLiveSender.hpp +++ b/sf-stream/include/ZmqLiveSender.hpp @@ -12,7 +12,6 @@ class ZmqLiveSender { void* socket_streamvis_; void* socket_live_; - void* socket_pulse_; public: ZmqLiveSender(void* ctx, diff --git a/sf-stream/src/ZmqLiveSender.cpp b/sf-stream/src/ZmqLiveSender.cpp index 04f3d05..724bf04 100644 --- a/sf-stream/src/ZmqLiveSender.cpp +++ b/sf-stream/src/ZmqLiveSender.cpp @@ -43,26 +43,6 @@ ZmqLiveSender::ZmqLiveSender( } } - { - socket_pulse_ = zmq_socket(ctx, ZMQ_PUB); - - if (zmq_bind(socket_pulse_, config.pulse_address.c_str()) != 0) { - throw runtime_error(zmq_strerror(errno)); - } - - const int sndhwm = PULSE_ZMQ_SNDHWM; - if (zmq_setsockopt( - socket_pulse_, ZMQ_SNDHWM, &sndhwm, sizeof(sndhwm)) != 0) { - throw runtime_error(zmq_strerror(errno)); - } - - const int linger = 0; - if (zmq_setsockopt( - socket_pulse_, ZMQ_LINGER, &linger, sizeof(linger)) != 0) { - throw runtime_error(zmq_strerror(errno)); - } - } - } ZmqLiveSender::~ZmqLiveSender() @@ -79,10 +59,6 @@ void ZmqLiveSender::send(const ImageMetadata& meta, const char *data) auto& header_alloc = header.GetAllocator(); string text_header; - if(zmq_send(socket_pulse_, &meta.pulse_id, sizeof(uint64_t), 0) == -1) { - throw runtime_error(zmq_strerror(errno)); - } - // TODO: Here we need to send to streamvis and live analysis metadata(probably need to operate still on them) and data(not every frame) header.AddMember("frame", meta.frame_index, header_alloc); header.AddMember("is_good_frame", meta.is_good_image, header_alloc); From cb9ddb3693cd9f12eb24eaab831dc13e2ec22a3d Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 13 Jan 2021 16:14:56 +0100 Subject: [PATCH 081/126] Fix the count of input parameters in streamer --- sf-stream/src/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sf-stream/src/main.cpp b/sf-stream/src/main.cpp index 45cc85d..dbd588f 100644 --- a/sf-stream/src/main.cpp +++ b/sf-stream/src/main.cpp @@ -15,7 +15,7 @@ using namespace stream_config; int main (int argc, char *argv[]) { - if (argc != 1) { + if (argc != 2) { cout << endl; cout << "Usage: sf_stream [detector_json_filename]" << endl; cout << "\tdetector_json_filename: detector config file path." << endl; From e9bb30eac82034e688ab9841b86c9e24098cf938 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 13 Jan 2021 16:18:47 +0100 Subject: [PATCH 082/126] Fix streamer sync attempts for loop --- sf-stream/src/ZmqPulseSyncReceiver.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sf-stream/src/ZmqPulseSyncReceiver.cpp b/sf-stream/src/ZmqPulseSyncReceiver.cpp index 79de2e6..eb57cb9 100644 --- a/sf-stream/src/ZmqPulseSyncReceiver.cpp +++ b/sf-stream/src/ZmqPulseSyncReceiver.cpp @@ -55,7 +55,7 @@ uint64_t ZmqPulseSyncReceiver::get_next_pulse_id() const return pulses[0]; } - for (int i_sync; i_sync < SYNC_RETRY_LIMIT; i_sync++) { + for (int i_sync=0; i_sync < SYNC_RETRY_LIMIT; i_sync++) { cout << "Sync attempt " << i_sync << endl; uint64_t min_pulse_id = 0; @@ -69,7 +69,7 @@ uint64_t ZmqPulseSyncReceiver::get_next_pulse_id() const auto max_diff = max_pulse_id - min_pulse_id; if (max_diff > PULSE_OFFSET_LIMIT) { stringstream err_msg; - err_msg << "[ZmqLiveReceiver::get_next_pulse_id]"; + err_msg << "[ZmqPulseSyncReceiver::get_next_pulse_id]"; err_msg << " PULSE_OFFSET_LIMIT exceeded."; err_msg << " max_diff=" << max_diff << " pulses."; From dd52eea55b0870b9e739ecf2d472b336ef18aacd Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 13 Jan 2021 16:20:51 +0100 Subject: [PATCH 083/126] Reverse sync difference criteria in streamer --- sf-stream/src/ZmqPulseSyncReceiver.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sf-stream/src/ZmqPulseSyncReceiver.cpp b/sf-stream/src/ZmqPulseSyncReceiver.cpp index eb57cb9..f309389 100644 --- a/sf-stream/src/ZmqPulseSyncReceiver.cpp +++ b/sf-stream/src/ZmqPulseSyncReceiver.cpp @@ -58,8 +58,8 @@ uint64_t ZmqPulseSyncReceiver::get_next_pulse_id() const for (int i_sync=0; i_sync < SYNC_RETRY_LIMIT; i_sync++) { cout << "Sync attempt " << i_sync << endl; - uint64_t min_pulse_id = 0; - uint64_t max_pulse_id = numeric_limits::max(); + uint64_t min_pulse_id = numeric_limits::max();; + uint64_t max_pulse_id = 0; for (int i = 0; i < n_modules_; i++) { min_pulse_id = min(min_pulse_id, pulses[i]); From 0cf973802da0017d3775b680087b755f4b81786a Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Thu, 14 Jan 2021 15:45:41 +0100 Subject: [PATCH 084/126] Add stream stats modulo to config file --- sf-stream/include/stream_config.hpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sf-stream/include/stream_config.hpp b/sf-stream/include/stream_config.hpp index 9b7e693..4e4eec4 100644 --- a/sf-stream/include/stream_config.hpp +++ b/sf-stream/include/stream_config.hpp @@ -14,4 +14,7 @@ namespace stream_config const int PULSE_ZMQ_SNDHWM = 100; // Number of times we try to re-sync in case of failure. const int SYNC_RETRY_LIMIT = 3; + + // Number of pulses between each statistics print out. + const size_t STREAM_STATS_MODULO = 10000; } From 2937326dbf97ccf8b2fb69c034e0a4aa878144a2 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Fri, 15 Jan 2021 10:59:43 +0100 Subject: [PATCH 085/126] Return n_lost_pulses in sync from receiver In order to have proper statistics we need to return the number of lost pulses from the receiver. We cannot just count the gap in pulse_id as the beam might be operated at different frequencies. --- sf-stream/include/ZmqPulseSyncReceiver.hpp | 7 ++++++- sf-stream/src/ZmqPulseSyncReceiver.cpp | 18 +++++++++++++----- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/sf-stream/include/ZmqPulseSyncReceiver.hpp b/sf-stream/include/ZmqPulseSyncReceiver.hpp index aa00035..624de34 100644 --- a/sf-stream/include/ZmqPulseSyncReceiver.hpp +++ b/sf-stream/include/ZmqPulseSyncReceiver.hpp @@ -8,6 +8,11 @@ #include "formats.hpp" +struct PulseAndSync { + const uint64_t pulse_id; + const uint32_t n_lost_pulses; +}; + class ZmqPulseSyncReceiver { void* ctx_; @@ -22,7 +27,7 @@ public: const int n_modules); ~ZmqPulseSyncReceiver(); - uint64_t get_next_pulse_id() const; + PulseAndSync get_next_pulse_id() const; }; diff --git a/sf-stream/src/ZmqPulseSyncReceiver.cpp b/sf-stream/src/ZmqPulseSyncReceiver.cpp index f309389..ead65ee 100644 --- a/sf-stream/src/ZmqPulseSyncReceiver.cpp +++ b/sf-stream/src/ZmqPulseSyncReceiver.cpp @@ -38,7 +38,7 @@ ZmqPulseSyncReceiver::~ZmqPulseSyncReceiver() } } -uint64_t ZmqPulseSyncReceiver::get_next_pulse_id() const +PulseAndSync ZmqPulseSyncReceiver::get_next_pulse_id() const { uint64_t pulses[n_modules_]; @@ -52,12 +52,12 @@ uint64_t ZmqPulseSyncReceiver::get_next_pulse_id() const } if (modules_in_sync) { - return pulses[0]; + return {pulses[0], 0}; } + // How many pulses we lost in total to get the next pulse_id. + uint32_t n_lost_pulses = 0; for (int i_sync=0; i_sync < SYNC_RETRY_LIMIT; i_sync++) { - cout << "Sync attempt " << i_sync << endl; - uint64_t min_pulse_id = numeric_limits::max();; uint64_t max_pulse_id = 0; @@ -83,18 +83,26 @@ uint64_t ZmqPulseSyncReceiver::get_next_pulse_id() const } modules_in_sync = true; + // Max pulses we lost in this sync attempt. + uint32_t i_sync_lost_pulses = 0; for (int i = 0; i < n_modules_; i++) { + // How many pulses we lost for this specific module. + uint32_t i_module_lost_pulses = 0; while (pulses[i] < max_pulse_id) { zmq_recv(sockets_[i], &pulses[i], sizeof(uint64_t), 0); + i_module_lost_pulses++; } + i_sync_lost_pulses = max(i_sync_lost_pulses, i_module_lost_pulses); + if (pulses[i] != max_pulse_id) { modules_in_sync = false; } } + n_lost_pulses += i_sync_lost_pulses; if (modules_in_sync) { - return pulses[0]; + return {pulses[0], n_lost_pulses}; } } From f58ec4270c9be471f43bcaae67401648d4405589 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Fri, 15 Jan 2021 11:06:47 +0100 Subject: [PATCH 086/126] Implementation fo StreamStats for outputting Grafana metrics --- sf-stream/include/StreamStats.hpp | 30 ++++++++++++++ sf-stream/src/StreamStats.cpp | 65 +++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100644 sf-stream/include/StreamStats.hpp create mode 100644 sf-stream/src/StreamStats.cpp diff --git a/sf-stream/include/StreamStats.hpp b/sf-stream/include/StreamStats.hpp new file mode 100644 index 0000000..a91db57 --- /dev/null +++ b/sf-stream/include/StreamStats.hpp @@ -0,0 +1,30 @@ +#ifndef SF_DAQ_BUFFER_STREAMSTATS_HPP +#define SF_DAQ_BUFFER_STREAMSTATS_HPP + +#include +#include +#include + +class StreamStats { + const std::string detector_name_; + const std::string stream_name_; + const size_t stats_modulo_; + + int image_counter_; + int n_corrupted_images_; + int n_sync_lost_images_; + std::chrono::time_point stats_interval_start_; + + void reset_counters(); + void print_stats(); + +public: + StreamStats(const std::string &detector_name, + const std::string &stream_name, + const size_t stats_modulo); + + void record_stats(const ImageMetadata &meta, const uint32_t n_lost_pulses); +}; + + +#endif //SF_DAQ_BUFFER_STREAMSTATS_HPP diff --git a/sf-stream/src/StreamStats.cpp b/sf-stream/src/StreamStats.cpp new file mode 100644 index 0000000..190f1bc --- /dev/null +++ b/sf-stream/src/StreamStats.cpp @@ -0,0 +1,65 @@ +#include "StreamStats.hpp" + +#include + +using namespace std; +using namespace chrono; + +StreamStats::StreamStats( + const std::string &detector_name, + const std::string &stream_name, + const size_t stats_modulo) : + detector_name_(detector_name), + stream_name_(stream_name), + stats_modulo_(stats_modulo) +{ + reset_counters(); +} + +void StreamStats::reset_counters() +{ + image_counter_ = 0; + n_sync_lost_images_ = 0; + n_corrupted_images_ = 0; + stats_interval_start_ = steady_clock::now(); +} + +void StreamStats::record_stats( + const ImageMetadata &meta, const uint32_t n_lost_pulses) +{ + image_counter_++; + n_sync_lost_images_ += n_lost_pulses; + + if (!meta.is_good_image) { + n_corrupted_images_++; + } + + if (image_counter_ == stats_modulo_) { + print_stats(); + reset_counters(); + } +} + +void StreamStats::print_stats() +{ + auto interval_ms_duration = duration_cast( + steady_clock::now()-stats_interval_start_).count(); + // * 1000 because milliseconds, + 250 because of truncation. + int rep_rate = ((image_counter_ * 1000) + 250) / interval_ms_duration; + uint64_t timestamp = time_point_cast( + system_clock::now()).time_since_epoch().count(); + + // Output in InfluxDB line protocol + cout << "sf-stream"; + cout << ",detector_name=" << detector_name_; + cout << ",stream_name=" << stream_name_; + cout << " "; + cout << "n_processed_images=" << image_counter_ << "i"; + cout << ",n_corrupted_images=" << n_corrupted_images_ << "i"; + cout << ",n_sync_lost_images=" << n_sync_lost_images_ << "i"; + cout << ",repetition_rate=" << rep_rate << "i"; + cout << " "; + cout << timestamp; + cout << endl; +} + From 3ed5bcb33021573f0c8e820de0376dab3f8cdbc8 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Fri, 15 Jan 2021 11:07:31 +0100 Subject: [PATCH 087/126] Statistics for sf-stream --- sf-stream/src/main.cpp | 42 +++++++++++------------------------------- 1 file changed, 11 insertions(+), 31 deletions(-) diff --git a/sf-stream/src/main.cpp b/sf-stream/src/main.cpp index dbd588f..9ebb111 100644 --- a/sf-stream/src/main.cpp +++ b/sf-stream/src/main.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include "buffer_config.hpp" #include "stream_config.hpp" @@ -15,57 +16,36 @@ using namespace stream_config; int main (int argc, char *argv[]) { - if (argc != 2) { + if (argc != 3) { cout << endl; - cout << "Usage: sf_stream [detector_json_filename]" << endl; + cout << "Usage: sf_stream [detector_json_filename]" + " [stream_name]" << endl; cout << "\tdetector_json_filename: detector config file path." << endl; cout << endl; exit(-1); } + const auto stream_name = string(argv[2]); + // TODO: Add stream_name to config reading - multiple stream definitions. auto config = BufferUtils::read_json_config(string(argv[1])); - string RECV_IPC_URL = BUFFER_LIVE_IPC_URL + config.detector_name + "-"; + string RECV_IPC_URL = BUFFER_LIVE_IPC_URL + config.detector_name + "-"; auto ctx = zmq_ctx_new(); zmq_ctx_set(ctx, ZMQ_IO_THREADS, STREAM_ZMQ_IO_THREADS); ZmqPulseSyncReceiver receiver(ctx, config.detector_name, config.n_modules); RamBuffer ram_buffer(config.detector_name, config.n_modules); + StreamStats stats(config.detector_name, stream_name, STREAM_STATS_MODULO); ZmqLiveSender sender(ctx, config); - // TODO: Remove stats trash. - uint64_t last_pulse_id = 0; - uint64_t last_pulse_id_range = 0; - uint16_t n_good_images = 0; - ImageMetadata meta; while (true) { - auto pulse_id = receiver.get_next_pulse_id(); - char* data = ram_buffer.read_image(pulse_id, meta); + auto pulse_and_sync = receiver.get_next_pulse_id(); + char* data = ram_buffer.read_image(pulse_and_sync.pulse_id, meta); sender.send(meta, data); - // TODO: This logic works only at 100Hz. Fix it systematically. - uint64_t sync_lost_pulses = pulse_id - last_pulse_id; - if (last_pulse_id > 0 && sync_lost_pulses > 1) { - cout << "sf_stream:sync_lost_pulses " << sync_lost_pulses << endl; - } - last_pulse_id = pulse_id; - - uint64_t curr_pulse_id_range = pulse_id / 10000; - if (last_pulse_id_range != curr_pulse_id_range) { - if (last_pulse_id_range > 0) { - cout << "sf_stream:n_good_images " << n_good_images; - cout << endl; - } - - last_pulse_id_range = curr_pulse_id_range; - n_good_images = 0; - } - - if (meta.is_good_image) { - n_good_images++; - } + stats.record_stats(meta, pulse_and_sync.n_lost_pulses); } } From ba9532591caf128d222040d277b0c394df609f04 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Fri, 15 Jan 2021 11:14:54 +0100 Subject: [PATCH 088/126] Add details about unexpected pulse_id in ram buffer error --- core-buffer/src/RamBuffer.cpp | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/core-buffer/src/RamBuffer.cpp b/core-buffer/src/RamBuffer.cpp index bcf614b..581ab73 100644 --- a/core-buffer/src/RamBuffer.cpp +++ b/core-buffer/src/RamBuffer.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include "RamBuffer.hpp" #include "buffer_config.hpp" @@ -110,7 +111,21 @@ char* RamBuffer::read_image(const uint64_t pulse_id, if (!is_pulse_init) { if (frame_meta->pulse_id != pulse_id) { - throw runtime_error("Wrong pulse_id in ram buffer slot."); + stringstream err_msg; + err_msg << "[RamBuffer::read_image]"; + err_msg << " Unexpected pulse_id in ram buffer."; + err_msg << " expected=" << pulse_id; + err_msg << " got=" << frame_meta->pulse_id; + + for (int i = 0; i < n_modules_; i++) { + ModuleFrame *meta = src_meta + i_module; + + err_msg << " (module " << i << ", "; + err_msg << meta->pulse_id << "),"; + } + err_msg << endl; + + throw runtime_error(err_msg.str()); } image_meta.pulse_id = frame_meta->pulse_id; From fec900ff63b77d565d5444136fa163303cc42ac1 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Fri, 15 Jan 2021 11:15:28 +0100 Subject: [PATCH 089/126] Increase the sf_stream output frequency to 10 seconds --- sf-stream/include/stream_config.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sf-stream/include/stream_config.hpp b/sf-stream/include/stream_config.hpp index 4e4eec4..8f8b977 100644 --- a/sf-stream/include/stream_config.hpp +++ b/sf-stream/include/stream_config.hpp @@ -16,5 +16,5 @@ namespace stream_config const int SYNC_RETRY_LIMIT = 3; // Number of pulses between each statistics print out. - const size_t STREAM_STATS_MODULO = 10000; + const size_t STREAM_STATS_MODULO = 1000; } From 0a3db103c5f4bbe44bc358a6989d8101bab28499 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Fri, 15 Jan 2021 11:18:07 +0100 Subject: [PATCH 090/126] Change stats names to use _ instead of - Using - is causing problems in Influx -> we move to use the executable name instead of the project name as the main identifier. --- jf-buffer-writer/src/BufferStats.cpp | 2 +- jf-udp-recv/src/FrameStats.cpp | 2 +- sf-stream/src/StreamStats.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/jf-buffer-writer/src/BufferStats.cpp b/jf-buffer-writer/src/BufferStats.cpp index 325e930..173a35c 100644 --- a/jf-buffer-writer/src/BufferStats.cpp +++ b/jf-buffer-writer/src/BufferStats.cpp @@ -51,7 +51,7 @@ void BufferStats::print_stats() system_clock::now()).time_since_epoch().count(); // Output in InfluxDB line protocol - cout << "jf-buffer-writer"; + cout << "jf_buffer_writer"; cout << ",detector_name=" << detector_name_; cout << ",module_name=M" << module_id_; cout << " "; diff --git a/jf-udp-recv/src/FrameStats.cpp b/jf-udp-recv/src/FrameStats.cpp index 8beb291..103e4fe 100644 --- a/jf-udp-recv/src/FrameStats.cpp +++ b/jf-udp-recv/src/FrameStats.cpp @@ -48,7 +48,7 @@ void FrameStats::print_stats() system_clock::now()).time_since_epoch().count(); // Output in InfluxDB line protocol - cout << "jf-udp-recv"; + cout << "jf_udp_recv"; cout << ",detector_name=" << detector_name_; cout << ",module_name=M" << module_id_; cout << " "; diff --git a/sf-stream/src/StreamStats.cpp b/sf-stream/src/StreamStats.cpp index 190f1bc..bed448d 100644 --- a/sf-stream/src/StreamStats.cpp +++ b/sf-stream/src/StreamStats.cpp @@ -50,7 +50,7 @@ void StreamStats::print_stats() system_clock::now()).time_since_epoch().count(); // Output in InfluxDB line protocol - cout << "sf-stream"; + cout << "sf_stream"; cout << ",detector_name=" << detector_name_; cout << ",stream_name=" << stream_name_; cout << " "; From e013c4a6f8fca49e2fa740ece69fdbef284f11e9 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Fri, 15 Jan 2021 18:18:54 +0100 Subject: [PATCH 091/126] Increase number of packets read at once to 128 --- core-buffer/include/buffer_config.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core-buffer/include/buffer_config.hpp b/core-buffer/include/buffer_config.hpp index 73c97b5..293308b 100644 --- a/core-buffer/include/buffer_config.hpp +++ b/core-buffer/include/buffer_config.hpp @@ -28,7 +28,7 @@ namespace buffer_config { const size_t BUFFER_BLOCK_SIZE = 100; - const size_t BUFFER_UDP_N_RECV_MSG = 64; + const size_t BUFFER_UDP_N_RECV_MSG = 128; // Size of UDP recv buffer const int BUFFER_UDP_RCVBUF_N_SLOTS = 100; // 8246 bytes for each UDP packet. From 20bf00cbbec4c164f2ca50f5da2b29dcfbc7b572 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 19 Jan 2021 10:52:18 +0100 Subject: [PATCH 092/126] Adjust RamBuffer test --- core-buffer/test/test_RamBuffer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core-buffer/test/test_RamBuffer.cpp b/core-buffer/test/test_RamBuffer.cpp index b4b8ddb..55c3a7c 100644 --- a/core-buffer/test/test_RamBuffer.cpp +++ b/core-buffer/test/test_RamBuffer.cpp @@ -25,7 +25,7 @@ TEST(RamBuffer, simple_store) for (int i_module=0; i_module Date: Tue, 19 Jan 2021 10:56:14 +0100 Subject: [PATCH 093/126] Add missing link library to sf_writer --- sf-writer/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/sf-writer/CMakeLists.txt b/sf-writer/CMakeLists.txt index dc87ad2..5b520f5 100644 --- a/sf-writer/CMakeLists.txt +++ b/sf-writer/CMakeLists.txt @@ -11,6 +11,7 @@ add_executable(sf-writer src/main.cpp) set_target_properties(sf-writer PROPERTIES OUTPUT_NAME sf_writer) target_link_libraries(sf-writer sf-writer-lib + zmq hdf5 hdf5_hl hdf5_cpp From ccca4225d40ed68b7e3ec34265f678744f4d881e Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 19 Jan 2021 12:59:50 +0100 Subject: [PATCH 094/126] Change bind and connect socket helper functions to take string --- core-buffer/include/BufferUtils.hpp | 9 +++++++-- core-buffer/src/BufferUtils.cpp | 8 ++++---- jf-buffer-writer/src/main.cpp | 3 ++- jf-udp-recv/src/main.cpp | 2 +- sf-stream/src/ZmqPulseSyncReceiver.cpp | 2 +- 5 files changed, 15 insertions(+), 9 deletions(-) diff --git a/core-buffer/include/BufferUtils.hpp b/core-buffer/include/BufferUtils.hpp index bb09dea..18a3fe5 100644 --- a/core-buffer/include/BufferUtils.hpp +++ b/core-buffer/include/BufferUtils.hpp @@ -36,9 +36,14 @@ namespace BufferUtils void create_destination_folder(const std::string& output_file); void* bind_socket( - void* ctx, const std::string& detector_name, const int source_id); + void* ctx, + const std::string& detector_name, + const std::string& stream_name); + void* connect_socket( - void* ctx, const std::string& detector_name, const int source_id); + void* ctx, + const std::string& detector_name, + const std::string& stream_name); DetectorConfig read_json_config(const std::string& filename); } diff --git a/core-buffer/src/BufferUtils.cpp b/core-buffer/src/BufferUtils.cpp index 4e861df..147d4e8 100644 --- a/core-buffer/src/BufferUtils.cpp +++ b/core-buffer/src/BufferUtils.cpp @@ -69,11 +69,11 @@ void BufferUtils::create_destination_folder(const string& output_file) } void* BufferUtils::connect_socket( - void* ctx, const string& detector_name, const int source_id) + void* ctx, const string& detector_name, const string& stream_name) { string ipc_address = BUFFER_LIVE_IPC_URL + detector_name + "-" + - to_string(source_id); + stream_name; void* socket = zmq_socket(ctx, ZMQ_SUB); if (socket == nullptr) { @@ -102,11 +102,11 @@ void* BufferUtils::connect_socket( } void* BufferUtils::bind_socket( - void* ctx, const string& detector_name, const int source_id) + void* ctx, const string& detector_name, const string& stream_name) { string ipc_address = BUFFER_LIVE_IPC_URL + detector_name + "-" + - to_string(source_id); + stream_name; void* socket = zmq_socket(ctx, ZMQ_PUB); diff --git a/jf-buffer-writer/src/main.cpp b/jf-buffer-writer/src/main.cpp index e1eb961..cd65986 100644 --- a/jf-buffer-writer/src/main.cpp +++ b/jf-buffer-writer/src/main.cpp @@ -39,7 +39,8 @@ int main (int argc, char *argv[]) { BufferStats stats(config.detector_name, module_id, STATS_MODULO); auto ctx = zmq_ctx_new(); - auto socket = connect_socket(ctx, config.detector_name, module_id); + auto socket = connect_socket( + ctx, config.detector_name, to_string(module_id)); auto file_buff = new BufferBinaryFormat(); uint64_t pulse_id; diff --git a/jf-udp-recv/src/main.cpp b/jf-udp-recv/src/main.cpp index 49668a1..2194b80 100644 --- a/jf-udp-recv/src/main.cpp +++ b/jf-udp-recv/src/main.cpp @@ -36,7 +36,7 @@ int main (int argc, char *argv[]) { FrameStats stats(config.detector_name, module_id, STATS_MODULO); auto ctx = zmq_ctx_new(); - auto socket = bind_socket(ctx, config.detector_name, module_id); + auto socket = bind_socket(ctx, config.detector_name, to_string(module_id)); ModuleFrame meta; char* data = new char[MODULE_N_BYTES]; diff --git a/sf-stream/src/ZmqPulseSyncReceiver.cpp b/sf-stream/src/ZmqPulseSyncReceiver.cpp index ead65ee..96221f3 100644 --- a/sf-stream/src/ZmqPulseSyncReceiver.cpp +++ b/sf-stream/src/ZmqPulseSyncReceiver.cpp @@ -27,7 +27,7 @@ ZmqPulseSyncReceiver::ZmqPulseSyncReceiver( for (int i=0; i Date: Tue, 19 Jan 2021 13:22:20 +0100 Subject: [PATCH 095/126] Separate image assembly and image retrieval in RamBuffer --- core-buffer/include/RamBuffer.hpp | 4 +++- core-buffer/src/RamBuffer.cpp | 14 ++++++++------ core-buffer/test/test_RamBuffer.cpp | 2 +- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/core-buffer/include/RamBuffer.hpp b/core-buffer/include/RamBuffer.hpp index 45b67a1..91872cb 100644 --- a/core-buffer/include/RamBuffer.hpp +++ b/core-buffer/include/RamBuffer.hpp @@ -30,7 +30,9 @@ public: const uint64_t module_id, ModuleFrame &meta, char *data) const; - char* read_image(const uint64_t pulse_id, ImageMetadata &image_meta) const; + char* read_image(const uint64_t pulse_id) const; + void assemble_image( + const uint64_t pulse_id, ImageMetadata &image_meta) const; }; diff --git a/core-buffer/src/RamBuffer.cpp b/core-buffer/src/RamBuffer.cpp index 581ab73..ff14011 100644 --- a/core-buffer/src/RamBuffer.cpp +++ b/core-buffer/src/RamBuffer.cpp @@ -85,15 +85,12 @@ void RamBuffer::read_frame( memcpy(dst_data, src_data, MODULE_N_BYTES); } -char* RamBuffer::read_image(const uint64_t pulse_id, - ImageMetadata &image_meta) const +void RamBuffer::assemble_image( + const uint64_t pulse_id, ImageMetadata &image_meta) const { const size_t slot_n = pulse_id % n_slots_; - ModuleFrame *src_meta = meta_buffer_ + (n_modules_ * slot_n); - char *src_data = image_buffer_ + (image_bytes_ * slot_n); - auto is_pulse_init = false; auto is_good_image = true; @@ -158,7 +155,12 @@ char* RamBuffer::read_image(const uint64_t pulse_id, image_meta.frame_index = 0; image_meta.daq_rec = 0; } +} + +char* RamBuffer::read_image(const uint64_t pulse_id) const +{ + const size_t slot_n = pulse_id % n_slots_; + char *src_data = image_buffer_ + (image_bytes_ * slot_n); return src_data; } - diff --git a/core-buffer/test/test_RamBuffer.cpp b/core-buffer/test/test_RamBuffer.cpp index 55c3a7c..d28dd81 100644 --- a/core-buffer/test/test_RamBuffer.cpp +++ b/core-buffer/test/test_RamBuffer.cpp @@ -29,7 +29,7 @@ TEST(RamBuffer, simple_store) } ImageMetadata image_meta; - buffer.read_image(frame_meta.pulse_id, image_meta); + buffer.assemble_image(frame_meta.pulse_id, image_meta); ASSERT_EQ(image_meta.pulse_id, frame_meta.pulse_id); ASSERT_EQ(image_meta.daq_rec, frame_meta.daq_rec); ASSERT_EQ(image_meta.frame_index, frame_meta.frame_index); From 65fb6c929f1d87693287c823b4158daaee39691d Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 19 Jan 2021 13:40:16 +0100 Subject: [PATCH 096/126] Remove hdf5 linking to core-buffer --- core-buffer/test/CMakeLists.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/core-buffer/test/CMakeLists.txt b/core-buffer/test/CMakeLists.txt index bf61f07..aa1173a 100644 --- a/core-buffer/test/CMakeLists.txt +++ b/core-buffer/test/CMakeLists.txt @@ -4,7 +4,5 @@ target_link_libraries(core-buffer-tests core-buffer-lib external rt - hdf5 - hdf5_cpp zmq gtest) From 39d714f53829841452a93ac8c9429f9e4c8a7cba Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 19 Jan 2021 14:02:57 +0100 Subject: [PATCH 097/126] Add jf-assembler executable This service reconstructs the various modules and sends out a image metadata stream for further consumers. --- CMakeLists.txt | 1 + jf-assembler/CMakeLists.txt | 21 ++ jf-assembler/README.md | 179 ++++++++++++++++++ jf-assembler/include/AssemblerStats.hpp | 28 +++ jf-assembler/include/ZmqPulseSyncReceiver.hpp | 34 ++++ jf-assembler/include/assembler_config.hpp | 14 ++ jf-assembler/src/AssemblerStats.cpp | 62 ++++++ jf-assembler/src/ZmqPulseSyncReceiver.cpp | 115 +++++++++++ jf-assembler/src/main.cpp | 50 +++++ jf-assembler/test/CMakeLists.txt | 7 + jf-assembler/test/main.cpp | 8 + 11 files changed, 519 insertions(+) create mode 100644 jf-assembler/CMakeLists.txt create mode 100644 jf-assembler/README.md create mode 100644 jf-assembler/include/AssemblerStats.hpp create mode 100644 jf-assembler/include/ZmqPulseSyncReceiver.hpp create mode 100644 jf-assembler/include/assembler_config.hpp create mode 100644 jf-assembler/src/AssemblerStats.cpp create mode 100644 jf-assembler/src/ZmqPulseSyncReceiver.cpp create mode 100644 jf-assembler/src/main.cpp create mode 100644 jf-assembler/test/CMakeLists.txt create mode 100644 jf-assembler/test/main.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 1fd9a7e..c739d65 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -31,6 +31,7 @@ add_subdirectory( add_subdirectory("core-buffer") add_subdirectory("jf-udp-recv") add_subdirectory("jf-buffer-writer") +add_subdirectory("jf-assembler") add_subdirectory("sf-stream") add_subdirectory("sf-writer") #add_subdirectory("jf-live-writer") diff --git a/jf-assembler/CMakeLists.txt b/jf-assembler/CMakeLists.txt new file mode 100644 index 0000000..95e755c --- /dev/null +++ b/jf-assembler/CMakeLists.txt @@ -0,0 +1,21 @@ +file(GLOB SOURCES + src/*.cpp) + +add_library(jf-assembler-lib STATIC ${SOURCES}) +target_include_directories(jf-assembler-lib PUBLIC include/) +target_link_libraries(jf-assembler-lib + external + core-buffer-lib) + +add_executable(jf-assembler src/main.cpp) +set_target_properties(jf-assembler PROPERTIES OUTPUT_NAME jf_assembler) +target_link_libraries(jf-assembler + external + core-buffer-lib + jf-assembler-lib + zmq + pthread + rt) + +enable_testing() +add_subdirectory(test/) diff --git a/jf-assembler/README.md b/jf-assembler/README.md new file mode 100644 index 0000000..ce21a4d --- /dev/null +++ b/jf-assembler/README.md @@ -0,0 +1,179 @@ +# sf-stream +sf-stream is the component that receives a live stream of frame data from +sf-buffers over ZMQ and assembles them into images. This images are then +sent again over ZMQ to external components. There is always only 1 sf-stream +per detector. + +It currently has 3 output streams: + +- **Full data full meta** rate stream (send all images and meta) +- **Reduced data full meta** rate stream (send less images, but +all meta) +- **Pulse_id** stream (send only the current pulse_id) + +In addition to receiving and assembling images, sf-stream also calculates +additional meta and constructs the structures needed to send data in +Array 1.0 protocol. + +This component does not guarantee that the streams will always contain all +the data - it can happen that frame resynchronization is needed, and in this +case 1 or more frames could potentially be lost. This happens so rarely that in +practice is not a problem. + +## Overview + +![image_stream_overview](../docs/sf_daq_buffer-overview-stream.jpg) + +sf-stream is a single threaded application (without counting the ZMQ IO threads) +that is used for providing live assembled images to anyone willing to listen. + +In addition, it also provides a pulse_id stream, which is the most immediate +pulse_id feedback we currently have in case we need to synchronize external +components to the current machine pulse_id. + +## ZMQ receiving +Each ZMQ stream is coming from a separate sf-buffer. This means that we have as +many connections as we have modules in a detector. + +Messages are multipart (2 parts) and are received in PUB/SUB mode. + +There is no need for special synchronization between modules as we expect that +frames will always be in the correct order and all modules will provide the +same frame more or less at the same time. If any of this 2 conditions is not +met, the detector is not working properly and we cannot guaranty that sf-stream +will work correctly. + +Nonetheless we provide the capability to synchronize the streams in image +assembly phase - this is needed rarely, but occasionally happens. In this sort +of hiccups we usually loose only a couple of consecutive images. + +### Messages format +Each message is composed by 2 parts: + +- Serialization of ModuleFrame in the first part. +- Frame data in the second part. + +Module frame is defined as: +```c++ +#pragma pack(push) +#pragma pack(1) +struct ModuleFrame { + uint64_t pulse_id; + uint64_t frame_index; + uint64_t daq_rec; + uint64_t n_recv_packets; + uint64_t module_id; +}; +#pragma pack(pop) +``` + +The frame data is a 1MB (1024*512 pixels * 2 bytes/pixel) blob of data in +**uint16** representing the detector image. + +## Image assembly +We first synchronize the modules. We do this by reading all sockets and +deciding the largest frame pulse_id among them (max_pulse_id). We then calculate +the diff between a specific socket pulse_id and the max_pulse_id. +This difference tells us how many messages we need to discard from a specific socket. + +This discarding is the source of possible missing images in the output stream. +It can happen in 3 cases: + +- At least one of the detector modules did not sent any packets for the specific +pulse_id. +- All the packets from a specific module for a pulse_id were lost before UDP +receiving them. +- ZMQ HWM was reached (either on the sf-buffer or sf-stream) and the message was +dropped. + +All this 3 cases are highly unlikely, so synchronization is mostly needed when +first starting sf-stream. Different sockets connect to sf-buffers at different +times. Apart from the initial synchronization there should be no need to +re-synchronize modules in a healthy running environment. + +If an image is missing any ZMQ messages from sf-buffers (not all modules data +arrived), the image will be dropped. We do not do partial reconstruction in +sf-stream. However, it is important to note, that this does not cover the case +where frames are incomplete (missing UDP packets on sf-buffer) - we still +assemble this images as long as at least 1 packet/frame for a specific pulse_id +arrived. + +## ZMQ sending + +We devide the ZMQ sending to 3 types of stream: + +- Data processing stream. This is basically the complete stream from +the detector with all meta and data. It can be described as full data full +meta stream. Only 1 client at the time can be connected to this stream +(PUSH/PULL for load balancing). + +- Live viewing stream. This is a reduced data full meta stream. We send +meta for all frames, but data only for subset of them (10Hz, for example). +Any number of clients can connect to the 10Hz stream, because we use PUB/SUB +for this socket. + +- Pulse_id stream. This is a stream that sends out only the current pulse_id. +It can be used to synchronize any external system with the current pulse_id +being recorded. Multiple clients can connect to this stream. + +In the data processing and live viewing stream we use +[Array 1.0](https://github.com/paulscherrerinstitute/htypes/blob/master/array-1.0.md) +as our protocol to be compatible with currently available external components. + +We use following fields in the JSON header: + +| Name | Type | Comment | +| --- | --- | --- | +| pulse_id | uint64 |bunchid from detector header| +|frame|uint64|frame_index from detector header| +|is_good_frame|bool|true if all packets for this frame are present| +|daq_rec|uint32|daqrec from detector header| +|pedestal_file|string|Path to pedestal file| +|gain_file|string|Path to gain file| +|number_frames_expected|int|Number of expected frames| +|run_name|string|Name of the run| +|detector_name|string|Name of the detector| +|htype|string|Value: "array-1.0"| +|type|string|Value: "uint16"| +|shape|Array[uint64]|Shape of the image in stream| + +### Full data full meta stream + +This stream runs at detector frequency and uses PUSH/PULL to distribute data +to max 1 client (this client can have many processes, but it needs to be a +single logical entity, since the images are evenly distributed to all +connected sockets). + +![image_full_stream](../docs/sf_daq_buffer-FullStream.jpg) + +The goal here is to provide a complete copy of the detector image stream +for purposes of online analysis. Given the large amount of data on this +stream only "pre-approved" applications that can handle the load should be +attached here. + +### Reduced data full meta stream + +This streams also runs at detector frequency for JSON headers (meta), but +it sends only part of the images in the stream. The rest of the images are +sent as empty buffers (the receiver needs to be aware of this behaviour, as +Array 1.0 alone does not define it). + +![image_reduced_stream](../docs/sf_daq_buffer-ReducedStream.jpg) + +This is the lightweight version of the image stream. Any number of clients +can connect to this stream (PUB/SUB) but no client can do load +balancing automatically (it would require PUSH/PULL). + +This is a "public interface" for anyone who wants to get detector data live, +and can do with only a subset of images. + +### Pulse_id stream + +This stream runs ar detector frequency in PUB/SUB mode. The only thing it +does is sends out the pulse_id (of the just received image) in uint64_t +format. + +![image_pulse_stream](../docs/sf_daq_buffer-PulseStream.jpg) + +This is also a "public interface" for anyone who wants to get the current +system pulse_id. \ No newline at end of file diff --git a/jf-assembler/include/AssemblerStats.hpp b/jf-assembler/include/AssemblerStats.hpp new file mode 100644 index 0000000..a8267a6 --- /dev/null +++ b/jf-assembler/include/AssemblerStats.hpp @@ -0,0 +1,28 @@ +#ifndef SF_DAQ_BUFFER_ASSEMBLERSTATS_HPP +#define SF_DAQ_BUFFER_ASSEMBLERSTATS_HPP + +#include +#include +#include + +class AssemblerStats { + const std::string detector_name_; + const size_t stats_modulo_; + + int image_counter_; + int n_corrupted_images_; + int n_sync_lost_images_; + std::chrono::time_point stats_interval_start_; + + void reset_counters(); + void print_stats(); + +public: + AssemblerStats(const std::string &detector_name, + const size_t stats_modulo); + + void record_stats(const ImageMetadata &meta, const uint32_t n_lost_pulses); +}; + + +#endif //SF_DAQ_BUFFER_ASSEMBLERSTATS_HPP diff --git a/jf-assembler/include/ZmqPulseSyncReceiver.hpp b/jf-assembler/include/ZmqPulseSyncReceiver.hpp new file mode 100644 index 0000000..624de34 --- /dev/null +++ b/jf-assembler/include/ZmqPulseSyncReceiver.hpp @@ -0,0 +1,34 @@ +#ifndef SF_DAQ_BUFFER_ZMQPULSESYNCRECEIVER_HPP +#define SF_DAQ_BUFFER_ZMQPULSESYNCRECEIVER_HPP + + +#include +#include +#include + +#include "formats.hpp" + +struct PulseAndSync { + const uint64_t pulse_id; + const uint32_t n_lost_pulses; +}; + +class ZmqPulseSyncReceiver { + + void* ctx_; + const int n_modules_; + + std::vector sockets_; + +public: + ZmqPulseSyncReceiver( + void* ctx, + const std::string& detector_name, + const int n_modules); + ~ZmqPulseSyncReceiver(); + + PulseAndSync get_next_pulse_id() const; +}; + + +#endif //SF_DAQ_BUFFER_ZMQPULSESYNCRECEIVER_HPP diff --git a/jf-assembler/include/assembler_config.hpp b/jf-assembler/include/assembler_config.hpp new file mode 100644 index 0000000..b0e277d --- /dev/null +++ b/jf-assembler/include/assembler_config.hpp @@ -0,0 +1,14 @@ +namespace assembler_config +{ + // N of IO threads to send image metadata. + const int ASSEMBLER_ZMQ_IO_THREADS = 1; + + // If the modules are offset more than 1000 pulses, crush. + const uint64_t PULSE_OFFSET_LIMIT = 100; + + // Number of times we try to re-sync in case of failure. + const int SYNC_RETRY_LIMIT = 3; + + // Number of pulses between each statistics print out. + const size_t ASSEMBLER_STATS_MODULO = 1000; +} diff --git a/jf-assembler/src/AssemblerStats.cpp b/jf-assembler/src/AssemblerStats.cpp new file mode 100644 index 0000000..e295da6 --- /dev/null +++ b/jf-assembler/src/AssemblerStats.cpp @@ -0,0 +1,62 @@ +#include "AssemblerStats.hpp" + +#include + +using namespace std; +using namespace chrono; + +AssemblerStats::AssemblerStats( + const std::string &detector_name, + const size_t stats_modulo) : + detector_name_(detector_name), + stats_modulo_(stats_modulo) +{ + reset_counters(); +} + +void AssemblerStats::reset_counters() +{ + image_counter_ = 0; + n_sync_lost_images_ = 0; + n_corrupted_images_ = 0; + stats_interval_start_ = steady_clock::now(); +} + +void AssemblerStats::record_stats( + const ImageMetadata &meta, const uint32_t n_lost_pulses) +{ + image_counter_++; + n_sync_lost_images_ += n_lost_pulses; + + if (!meta.is_good_image) { + n_corrupted_images_++; + } + + if (image_counter_ == stats_modulo_) { + print_stats(); + reset_counters(); + } +} + +void AssemblerStats::print_stats() +{ + auto interval_ms_duration = duration_cast( + steady_clock::now()-stats_interval_start_).count(); + // * 1000 because milliseconds, + 250 because of truncation. + int rep_rate = ((image_counter_ * 1000) + 250) / interval_ms_duration; + uint64_t timestamp = time_point_cast( + system_clock::now()).time_since_epoch().count(); + + // Output in InfluxDB line protocol + cout << "jf_assembler"; + cout << ",detector_name=" << detector_name_; + cout << " "; + cout << "n_processed_images=" << image_counter_ << "i"; + cout << ",n_corrupted_images=" << n_corrupted_images_ << "i"; + cout << ",n_sync_lost_images=" << n_sync_lost_images_ << "i"; + cout << ",repetition_rate=" << rep_rate << "i"; + cout << " "; + cout << timestamp; + cout << endl; +} + diff --git a/jf-assembler/src/ZmqPulseSyncReceiver.cpp b/jf-assembler/src/ZmqPulseSyncReceiver.cpp new file mode 100644 index 0000000..6dbe2fc --- /dev/null +++ b/jf-assembler/src/ZmqPulseSyncReceiver.cpp @@ -0,0 +1,115 @@ +#include "ZmqPulseSyncReceiver.hpp" +#include "BufferUtils.hpp" + +#include +#include +#include +#include +#include +#include + +#include "assembler_config.hpp" + +using namespace std; +using namespace chrono; +using namespace buffer_config; +using namespace assembler_config; + + +ZmqPulseSyncReceiver::ZmqPulseSyncReceiver( + void * ctx, + const string& detector_name, + const int n_modules) : + ctx_(ctx), + n_modules_(n_modules) +{ + sockets_.reserve(n_modules_); + + for (int i=0; i::max();; + uint64_t max_pulse_id = 0; + + for (int i = 0; i < n_modules_; i++) { + min_pulse_id = min(min_pulse_id, pulses[i]); + max_pulse_id = max(max_pulse_id, pulses[i]); + } + + auto max_diff = max_pulse_id - min_pulse_id; + if (max_diff > PULSE_OFFSET_LIMIT) { + stringstream err_msg; + err_msg << "[ZmqPulseSyncReceiver::get_next_pulse_id]"; + err_msg << " PULSE_OFFSET_LIMIT exceeded."; + err_msg << " max_diff=" << max_diff << " pulses."; + + for (int i = 0; i < n_modules_; i++) { + err_msg << " (module " << i << ", "; + err_msg << pulses[i] << "),"; + } + err_msg << endl; + + throw runtime_error(err_msg.str()); + } + + modules_in_sync = true; + // Max pulses we lost in this sync attempt. + uint32_t i_sync_lost_pulses = 0; + for (int i = 0; i < n_modules_; i++) { + // How many pulses we lost for this specific module. + uint32_t i_module_lost_pulses = 0; + while (pulses[i] < max_pulse_id) { + zmq_recv(sockets_[i], &pulses[i], sizeof(uint64_t), 0); + i_module_lost_pulses++; + } + + i_sync_lost_pulses = max(i_sync_lost_pulses, i_module_lost_pulses); + + if (pulses[i] != max_pulse_id) { + modules_in_sync = false; + } + } + n_lost_pulses += i_sync_lost_pulses; + + if (modules_in_sync) { + return {pulses[0], n_lost_pulses}; + } + } + + stringstream err_msg; + err_msg << "[ZmqLiveReceiver::get_next_pulse_id]"; + err_msg << " SYNC_RETRY_LIMIT exceeded."; + err_msg << endl; + + throw runtime_error(err_msg.str()); +} diff --git a/jf-assembler/src/main.cpp b/jf-assembler/src/main.cpp new file mode 100644 index 0000000..22ede02 --- /dev/null +++ b/jf-assembler/src/main.cpp @@ -0,0 +1,50 @@ +#include +#include +#include +#include +#include +#include + +#include "buffer_config.hpp" +#include "assembler_config.hpp" +#include "ZmqPulseSyncReceiver.hpp" + +using namespace std; +using namespace buffer_config; +using namespace assembler_config; + +int main (int argc, char *argv[]) +{ + if (argc != 2) { + cout << endl; + cout << "Usage: jf_assembler [detector_json_filename]" << endl; + cout << "\tdetector_json_filename: detector config file path." << endl; + cout << endl; + + exit(-1); + } + + auto config = BufferUtils::read_json_config(string(argv[1])); + auto const stream_name = "assembler"; + + string RECV_IPC_URL = BUFFER_LIVE_IPC_URL + config.detector_name + "-"; + auto ctx = zmq_ctx_new(); + zmq_ctx_set(ctx, ZMQ_IO_THREADS, ASSEMBLER_ZMQ_IO_THREADS); + + ZmqPulseSyncReceiver receiver(ctx, config.detector_name, config.n_modules); + RamBuffer ram_buffer(config.detector_name, config.n_modules); + AssemblerStats stats(config.detector_name, ASSEMBLER_STATS_MODULO); + + auto sender = BufferUtils::bind_socket( + ctx, config.detector_name, stream_name); + + ImageMetadata meta; + while (true) { + auto pulse_and_sync = receiver.get_next_pulse_id(); + ram_buffer.assemble_image(pulse_and_sync.pulse_id, meta); + + zmq_send(sender, &meta, sizeof(meta), 0); + + stats.record_stats(meta, pulse_and_sync.n_lost_pulses); + } +} diff --git a/jf-assembler/test/CMakeLists.txt b/jf-assembler/test/CMakeLists.txt new file mode 100644 index 0000000..bb240e7 --- /dev/null +++ b/jf-assembler/test/CMakeLists.txt @@ -0,0 +1,7 @@ +add_executable(jf-assembler-tests main.cpp) + +target_link_libraries(jf-assembler-tests + jf-assembler-lib + gtest + ) + diff --git a/jf-assembler/test/main.cpp b/jf-assembler/test/main.cpp new file mode 100644 index 0000000..e819294 --- /dev/null +++ b/jf-assembler/test/main.cpp @@ -0,0 +1,8 @@ +#include "gtest/gtest.h" + +using namespace std; + +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} From 53e24c9f1a8316eb4705733aeaac4a83f42361df Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 19 Jan 2021 14:06:57 +0100 Subject: [PATCH 098/126] Reorganize assembler main code --- jf-assembler/src/main.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/jf-assembler/src/main.cpp b/jf-assembler/src/main.cpp index 22ede02..e1b76ab 100644 --- a/jf-assembler/src/main.cpp +++ b/jf-assembler/src/main.cpp @@ -5,7 +5,6 @@ #include #include -#include "buffer_config.hpp" #include "assembler_config.hpp" #include "ZmqPulseSyncReceiver.hpp" @@ -27,17 +26,15 @@ int main (int argc, char *argv[]) auto config = BufferUtils::read_json_config(string(argv[1])); auto const stream_name = "assembler"; - string RECV_IPC_URL = BUFFER_LIVE_IPC_URL + config.detector_name + "-"; auto ctx = zmq_ctx_new(); zmq_ctx_set(ctx, ZMQ_IO_THREADS, ASSEMBLER_ZMQ_IO_THREADS); + auto sender = BufferUtils::bind_socket( + ctx, config.detector_name, stream_name); ZmqPulseSyncReceiver receiver(ctx, config.detector_name, config.n_modules); RamBuffer ram_buffer(config.detector_name, config.n_modules); AssemblerStats stats(config.detector_name, ASSEMBLER_STATS_MODULO); - auto sender = BufferUtils::bind_socket( - ctx, config.detector_name, stream_name); - ImageMetadata meta; while (true) { auto pulse_and_sync = receiver.get_next_pulse_id(); From d3d51d2957298ea76b90b4c864e380ebf289ba37 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 19 Jan 2021 16:57:41 +0100 Subject: [PATCH 099/126] Adjust stats to new stream workings --- sf-stream/include/StreamStats.hpp | 3 +-- sf-stream/src/StreamStats.cpp | 5 +---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/sf-stream/include/StreamStats.hpp b/sf-stream/include/StreamStats.hpp index a91db57..bca5ce0 100644 --- a/sf-stream/include/StreamStats.hpp +++ b/sf-stream/include/StreamStats.hpp @@ -12,7 +12,6 @@ class StreamStats { int image_counter_; int n_corrupted_images_; - int n_sync_lost_images_; std::chrono::time_point stats_interval_start_; void reset_counters(); @@ -23,7 +22,7 @@ public: const std::string &stream_name, const size_t stats_modulo); - void record_stats(const ImageMetadata &meta, const uint32_t n_lost_pulses); + void record_stats(const ImageMetadata &meta); }; diff --git a/sf-stream/src/StreamStats.cpp b/sf-stream/src/StreamStats.cpp index bed448d..7408629 100644 --- a/sf-stream/src/StreamStats.cpp +++ b/sf-stream/src/StreamStats.cpp @@ -19,16 +19,14 @@ StreamStats::StreamStats( void StreamStats::reset_counters() { image_counter_ = 0; - n_sync_lost_images_ = 0; n_corrupted_images_ = 0; stats_interval_start_ = steady_clock::now(); } void StreamStats::record_stats( - const ImageMetadata &meta, const uint32_t n_lost_pulses) + const ImageMetadata &meta) { image_counter_++; - n_sync_lost_images_ += n_lost_pulses; if (!meta.is_good_image) { n_corrupted_images_++; @@ -56,7 +54,6 @@ void StreamStats::print_stats() cout << " "; cout << "n_processed_images=" << image_counter_ << "i"; cout << ",n_corrupted_images=" << n_corrupted_images_ << "i"; - cout << ",n_sync_lost_images=" << n_sync_lost_images_ << "i"; cout << ",repetition_rate=" << rep_rate << "i"; cout << " "; cout << timestamp; From 402a2b978d002bbfa41d7e435e23f7ce77f9f90e Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 19 Jan 2021 16:58:07 +0100 Subject: [PATCH 100/126] Rework sf-stream to attach to assembler --- sf-stream/src/main.cpp | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/sf-stream/src/main.cpp b/sf-stream/src/main.cpp index 9ebb111..dbe313a 100644 --- a/sf-stream/src/main.cpp +++ b/sf-stream/src/main.cpp @@ -5,10 +5,8 @@ #include #include -#include "buffer_config.hpp" #include "stream_config.hpp" #include "ZmqLiveSender.hpp" -#include "ZmqPulseSyncReceiver.hpp" using namespace std; using namespace buffer_config; @@ -30,22 +28,22 @@ int main (int argc, char *argv[]) // TODO: Add stream_name to config reading - multiple stream definitions. auto config = BufferUtils::read_json_config(string(argv[1])); - string RECV_IPC_URL = BUFFER_LIVE_IPC_URL + config.detector_name + "-"; auto ctx = zmq_ctx_new(); zmq_ctx_set(ctx, ZMQ_IO_THREADS, STREAM_ZMQ_IO_THREADS); + auto receiver = BufferUtils::connect_socket( + ctx, config.detector_name, "assembler"); - ZmqPulseSyncReceiver receiver(ctx, config.detector_name, config.n_modules); RamBuffer ram_buffer(config.detector_name, config.n_modules); StreamStats stats(config.detector_name, stream_name, STREAM_STATS_MODULO); ZmqLiveSender sender(ctx, config); ImageMetadata meta; while (true) { - auto pulse_and_sync = receiver.get_next_pulse_id(); - char* data = ram_buffer.read_image(pulse_and_sync.pulse_id, meta); + zmq_recv(receiver, &meta, sizeof(meta), 0); + char* data = ram_buffer.read_image(meta.pulse_id); sender.send(meta, data); - stats.record_stats(meta, pulse_and_sync.n_lost_pulses); + stats.record_stats(meta); } } From 6bbbd734c7f0ec9901a4e4dc3929d477bf635ceb Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 19 Jan 2021 17:20:06 +0100 Subject: [PATCH 101/126] Add jf-live-writer First implementation of the image buffer writer for the Jungfrau --- CMakeLists.txt | 2 +- jf-live-writer/include/BinaryReader.hpp | 28 --- jf-live-writer/include/BufferStats.hpp | 32 +++ jf-live-writer/include/JFH5LiveWriter.hpp | 49 ----- jf-live-writer/include/LiveImageAssembler.hpp | 51 ----- jf-live-writer/include/live_writer_config.hpp | 6 +- jf-live-writer/src/BinaryReader.cpp | 102 --------- jf-live-writer/src/BufferStats.cpp | 63 ++++++ jf-live-writer/src/JFH5LiveWriter.cpp | 133 ------------ jf-live-writer/src/LiveImageAssembler.cpp | 159 -------------- jf-live-writer/src/main.cpp | 195 +++--------------- 11 files changed, 121 insertions(+), 699 deletions(-) delete mode 100644 jf-live-writer/include/BinaryReader.hpp create mode 100644 jf-live-writer/include/BufferStats.hpp delete mode 100644 jf-live-writer/include/JFH5LiveWriter.hpp delete mode 100644 jf-live-writer/include/LiveImageAssembler.hpp delete mode 100644 jf-live-writer/src/BinaryReader.cpp create mode 100644 jf-live-writer/src/BufferStats.cpp delete mode 100644 jf-live-writer/src/JFH5LiveWriter.cpp delete mode 100644 jf-live-writer/src/LiveImageAssembler.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index c739d65..63c25dc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,4 +34,4 @@ add_subdirectory("jf-buffer-writer") add_subdirectory("jf-assembler") add_subdirectory("sf-stream") add_subdirectory("sf-writer") -#add_subdirectory("jf-live-writer") +add_subdirectory("jf-live-writer") diff --git a/jf-live-writer/include/BinaryReader.hpp b/jf-live-writer/include/BinaryReader.hpp deleted file mode 100644 index 85d2a0c..0000000 --- a/jf-live-writer/include/BinaryReader.hpp +++ /dev/null @@ -1,28 +0,0 @@ -#ifndef SF_DAQ_BUFFER_BINARYREADER_HPP -#define SF_DAQ_BUFFER_BINARYREADER_HPP - - -#include - -class BinaryReader { - - const std::string detector_folder_; - const std::string module_name_; - - std::string current_input_file_; - int input_file_fd_; - - void open_file(const std::string& filename); - void close_current_file(); - -public: - BinaryReader(const std::string &detector_folder, - const std::string &module_name); - - ~BinaryReader(); - - void get_frame(const uint64_t pulse_id, BufferBinaryFormat *buffer); -}; - - -#endif //SF_DAQ_BUFFER_BINARYREADER_HPP diff --git a/jf-live-writer/include/BufferStats.hpp b/jf-live-writer/include/BufferStats.hpp new file mode 100644 index 0000000..3aff6cb --- /dev/null +++ b/jf-live-writer/include/BufferStats.hpp @@ -0,0 +1,32 @@ +#include +#include +#include + +#ifndef SF_DAQ_BUFFER_FRAMESTATS_HPP +#define SF_DAQ_BUFFER_FRAMESTATS_HPP + + +class BufferStats { + const std::string detector_name_; + const int module_id_; + size_t stats_modulo_; + + int frames_counter_; + uint32_t total_buffer_write_us_; + uint32_t max_buffer_write_us_; + std::chrono::time_point stats_interval_start_; + + void reset_counters(); + void print_stats(); + +public: + BufferStats( + const std::string &detector_name, + const int module_id, + const size_t stats_modulo); + void start_frame_write(); + void end_frame_write(); +}; + + +#endif //SF_DAQ_BUFFER_FRAMESTATS_HPP diff --git a/jf-live-writer/include/JFH5LiveWriter.hpp b/jf-live-writer/include/JFH5LiveWriter.hpp deleted file mode 100644 index a417631..0000000 --- a/jf-live-writer/include/JFH5LiveWriter.hpp +++ /dev/null @@ -1,49 +0,0 @@ -#ifndef SFWRITER_HPP -#define SFWRITER_HPP - -#include -#include -#include - -#include "LiveImageAssembler.hpp" - -const auto& H5_UINT64 = H5::PredType::NATIVE_UINT64; -const auto& H5_UINT32 = H5::PredType::NATIVE_UINT32; -const auto& H5_UINT16 = H5::PredType::NATIVE_UINT16; -const auto& H5_UINT8 = H5::PredType::NATIVE_UINT8; - -class JFH5LiveWriter { - - const std::string detector_name_; - const size_t n_modules_; - const size_t n_pulses_; - - size_t write_index_; - - H5::H5File file_; - H5::DataSet image_dataset_; - - uint64_t* b_pulse_id_; - uint64_t* b_frame_index_; - uint32_t* b_daq_rec_; - uint8_t* b_is_good_frame_ ; - - void init_file(const std::string &output_file); - void write_dataset(const std::string name, - const void *buffer, - const H5::PredType &type); - void write_metadata(); - std::string get_detector_name(const std::string& detector_folder); - - void close_file(); - -public: - JFH5LiveWriter(const std::string& output_file, - const std::string& detector_folder, - const size_t n_modules, - const size_t n_pulses); - ~JFH5LiveWriter(); - void write(const ImageMetadata* metadata, const char* data); -}; - -#endif //SFWRITER_HPP diff --git a/jf-live-writer/include/LiveImageAssembler.hpp b/jf-live-writer/include/LiveImageAssembler.hpp deleted file mode 100644 index 5bcb749..0000000 --- a/jf-live-writer/include/LiveImageAssembler.hpp +++ /dev/null @@ -1,51 +0,0 @@ -#ifndef SF_DAQ_BUFFER_LIVEIMAGEASSEMBLER_HPP -#define SF_DAQ_BUFFER_LIVEIMAGEASSEMBLER_HPP - -#include - -#include "buffer_config.hpp" -#include "formats.hpp" - -const uint64_t IA_EMPTY_SLOT_VALUE = 0; - -struct ImageMetadata -{ - uint64_t pulse_id; - uint64_t frame_index; - uint32_t daq_rec; - uint8_t is_good_image; -}; - -class LiveImageAssembler { - const size_t n_modules_; - const size_t image_buffer_slot_n_bytes_; - - char* image_buffer_; - ImageMetadata* image_meta_buffer_; - ModuleFrame* frame_meta_buffer_; - std::atomic_int* buffer_status_; - std::atomic_uint64_t* buffer_pulse_id_; - - size_t get_data_offset(const uint64_t slot_id, const int i_module); - size_t get_frame_metadata_offset(const uint64_t slot_id, const int i_module); - -public: - LiveImageAssembler(const size_t n_modules); - - virtual ~LiveImageAssembler(); - - bool is_slot_free(const uint64_t pulse_id); - bool is_slot_full(const uint64_t pulse_id); - - void process(const uint64_t pulse_id, - const int i_module, - const BufferBinaryFormat* block_buffer); - - void free_slot(const uint64_t pulse_id); - - ImageMetadata* get_metadata_buffer(const uint64_t pulse_id); - char* get_data_buffer(const uint64_t pulse_id); -}; - - -#endif //SF_DAQ_BUFFER_LIVEIMAGEASSEMBLER_HPP diff --git a/jf-live-writer/include/live_writer_config.hpp b/jf-live-writer/include/live_writer_config.hpp index 0a62457..76d9b05 100644 --- a/jf-live-writer/include/live_writer_config.hpp +++ b/jf-live-writer/include/live_writer_config.hpp @@ -2,8 +2,6 @@ namespace live_writer_config { - // MS to retry reading from the image assembler. - const size_t ASSEMBLER_RETRY_MS = 5; - // Number of slots in the reconstruction buffer. - const size_t WRITER_IA_N_SLOTS = 200; + // N of IO threads to receive data from modules. + const int LIVE_ZMQ_IO_THREADS = 1; } \ No newline at end of file diff --git a/jf-live-writer/src/BinaryReader.cpp b/jf-live-writer/src/BinaryReader.cpp deleted file mode 100644 index 0512ac7..0000000 --- a/jf-live-writer/src/BinaryReader.cpp +++ /dev/null @@ -1,102 +0,0 @@ -#include "BinaryReader.hpp" - -#include -#include -#include -#include - -#include "BufferUtils.hpp" -#include "buffer_config.hpp" - -using namespace std; -using namespace buffer_config; - -BinaryReader::BinaryReader( - const std::string &detector_folder, - const std::string &module_name) : - detector_folder_(detector_folder), - module_name_(module_name), - current_input_file_(""), - input_file_fd_(-1) -{} - -BinaryReader::~BinaryReader() -{ - close_current_file(); -} - -void BinaryReader::get_frame( - const uint64_t pulse_id, BufferBinaryFormat* buffer) -{ - - auto current_frame_file = BufferUtils::get_filename( - detector_folder_, module_name_, pulse_id); - - if (current_frame_file != current_input_file_) { - open_file(current_frame_file); - } - - size_t file_index = BufferUtils::get_file_frame_index(pulse_id); - size_t n_bytes_offset = file_index * sizeof(BufferBinaryFormat); - - auto lseek_result = lseek(input_file_fd_, n_bytes_offset, SEEK_SET); - if (lseek_result < 0) { - stringstream err_msg; - - err_msg << "[BinaryReader::get_frame]"; - err_msg << " Error while lseek on file "; - err_msg << current_input_file_ << " for n_bytes_offset "; - err_msg << n_bytes_offset << ": " << strerror(errno) << endl; - - throw runtime_error(err_msg.str()); - } - - auto n_bytes = ::read(input_file_fd_, buffer, sizeof(BufferBinaryFormat)); - - if (n_bytes < sizeof(BufferBinaryFormat)) { - stringstream err_msg; - - err_msg << "[BinaryReader::get_block]"; - err_msg << " Error while reading from file "; - err_msg << current_input_file_ << ": " << strerror(errno) << endl; - - throw runtime_error(err_msg.str()); - } -} - -void BinaryReader::open_file(const std::string& filename) -{ - close_current_file(); - - input_file_fd_ = open(filename.c_str(), O_RDONLY); - - if (input_file_fd_ < 0) { - stringstream err_msg; - - err_msg << "[BinaryReader::open_file]"; - err_msg << " Cannot open file " << filename << ": "; - err_msg << strerror(errno) << endl; - - throw runtime_error(err_msg.str()); - } - - current_input_file_ = filename; -} - -void BinaryReader::close_current_file() -{ - if (input_file_fd_ != -1) { - if (close(input_file_fd_) < 0) { - stringstream err_msg; - - err_msg << "[BinaryWriter::close_current_file]"; - err_msg << " Error while closing file " << current_input_file_; - err_msg << ": " << strerror(errno) << endl; - - throw runtime_error(err_msg.str()); - } - - input_file_fd_ = -1; - current_input_file_ = ""; - } -} diff --git a/jf-live-writer/src/BufferStats.cpp b/jf-live-writer/src/BufferStats.cpp new file mode 100644 index 0000000..173a35c --- /dev/null +++ b/jf-live-writer/src/BufferStats.cpp @@ -0,0 +1,63 @@ +#include +#include "BufferStats.hpp" + +using namespace std; +using namespace chrono; + +BufferStats::BufferStats( + const string& detector_name, + const int module_id, + const size_t stats_modulo) : + detector_name_(detector_name), + module_id_(module_id), + stats_modulo_(stats_modulo) +{ + reset_counters(); +} + +void BufferStats::reset_counters() +{ + frames_counter_ = 0; + total_buffer_write_us_ = 0; + max_buffer_write_us_ = 0; +} + +void BufferStats::start_frame_write() +{ + stats_interval_start_ = steady_clock::now(); +} + +void BufferStats::end_frame_write() +{ + frames_counter_++; + + uint32_t write_us_duration = duration_cast( + steady_clock::now()-stats_interval_start_).count(); + + total_buffer_write_us_ += write_us_duration; + max_buffer_write_us_ = max(max_buffer_write_us_, write_us_duration); + + if (frames_counter_ == stats_modulo_) { + print_stats(); + reset_counters(); + } +} + +void BufferStats::print_stats() +{ + float avg_buffer_write_us = total_buffer_write_us_ / frames_counter_; + + uint64_t timestamp = time_point_cast( + system_clock::now()).time_since_epoch().count(); + + // Output in InfluxDB line protocol + cout << "jf_buffer_writer"; + cout << ",detector_name=" << detector_name_; + cout << ",module_name=M" << module_id_; + cout << " "; + cout << "avg_buffer_write_us=" << avg_buffer_write_us; + cout << ",max_buffer_write_us=" << max_buffer_write_us_ << "i"; + cout << " "; + cout << timestamp; + cout << endl; +} diff --git a/jf-live-writer/src/JFH5LiveWriter.cpp b/jf-live-writer/src/JFH5LiveWriter.cpp deleted file mode 100644 index 5928a6e..0000000 --- a/jf-live-writer/src/JFH5LiveWriter.cpp +++ /dev/null @@ -1,133 +0,0 @@ -#include "JFH5LiveWriter.hpp" - -#include -#include - - -#include "buffer_config.hpp" - -using namespace std; -using namespace buffer_config; - -JFH5LiveWriter::JFH5LiveWriter(const string& output_file, - const string& detector_folder, - const size_t n_modules, - const size_t n_pulses) : - detector_name_(get_detector_name(detector_folder)), - n_modules_(n_modules), - n_pulses_(n_pulses), - write_index_(0) -{ - b_pulse_id_ = new uint64_t[n_pulses_]; - b_frame_index_= new uint64_t[n_pulses_]; - b_daq_rec_ = new uint32_t[n_pulses_]; - b_is_good_frame_ = new uint8_t[n_pulses_]; - - init_file(output_file); -} - -void JFH5LiveWriter::init_file(const string& output_file) -{ - file_ = H5::H5File(output_file, H5F_ACC_TRUNC); - file_.createGroup("/data"); - file_.createGroup("/data/" + detector_name_); - - H5::DataSpace att_space(H5S_SCALAR); - H5::DataType data_type = H5::StrType(0, H5T_VARIABLE); - - file_.createGroup("/general"); - auto detector_dataset = file_.createDataSet( - "/general/detector_name", data_type ,att_space); - - detector_dataset.write(detector_name_, data_type); - - hsize_t image_dataset_dims[3] = - {n_pulses_, n_modules_ * MODULE_Y_SIZE, MODULE_X_SIZE}; - - H5::DataSpace image_dataspace(3, image_dataset_dims); - - hsize_t image_dataset_chunking[3] = - {1, n_modules_ * MODULE_Y_SIZE, MODULE_X_SIZE}; - H5::DSetCreatPropList image_dataset_properties; - image_dataset_properties.setChunk(3, image_dataset_chunking); - - image_dataset_ = file_.createDataSet( - "/data/" + detector_name_ + "/data", - H5_UINT16, - image_dataspace, - image_dataset_properties); -} - - -std::string JFH5LiveWriter::get_detector_name(const string& detector_folder) -{ - size_t last_separator; - if ((last_separator = detector_folder.rfind("/")) == string::npos) { - return detector_folder; - } - - return detector_folder.substr(last_separator + 1); -} - -JFH5LiveWriter::~JFH5LiveWriter() -{ - close_file(); - - delete[] b_pulse_id_; - delete[] b_frame_index_; - delete[] b_daq_rec_; - delete[] b_is_good_frame_; -} - -void JFH5LiveWriter::write_dataset( - const string name, const void* buffer, const H5::PredType& type) -{ - hsize_t b_m_dims[] = {n_pulses_}; - H5::DataSpace b_m_space (1, b_m_dims); - - hsize_t f_m_dims[] = {n_pulses_, 1}; - H5::DataSpace f_m_space(2, f_m_dims); - - auto complete_name = "/data/" + detector_name_ + "/" + name; - auto dataset = file_.createDataSet(complete_name, type, f_m_space); - - dataset.write(buffer, type, b_m_space, f_m_space); - - dataset.close(); -} - -void JFH5LiveWriter::write_metadata() -{ - write_dataset("pulse_id", &b_pulse_id_, H5_UINT64); - write_dataset("frame_index", &b_frame_index_, H5_UINT64); - write_dataset("daq_rec", &b_daq_rec_, H5_UINT32); - write_dataset("is_good_frame", &b_is_good_frame_, H5_UINT8); -} - -void JFH5LiveWriter::close_file() -{ - if (file_.getId() == -1) { - return; - } - - image_dataset_.close(); - - write_metadata(); - - file_.close(); -} - -void JFH5LiveWriter::write(const ImageMetadata* metadata, const char* data) -{ - hsize_t offset[] = {write_index_, 0, 0}; - - H5DOwrite_chunk(image_dataset_.getId(), H5P_DEFAULT, 0, - offset, MODULE_N_BYTES * n_modules_, data); - - b_pulse_id_[write_index_] = metadata->pulse_id; - b_frame_index_[write_index_] = metadata->frame_index; - b_daq_rec_[write_index_] = metadata->daq_rec; - b_is_good_frame_[write_index_] = metadata->is_good_image; - - write_index_++; -} diff --git a/jf-live-writer/src/LiveImageAssembler.cpp b/jf-live-writer/src/LiveImageAssembler.cpp deleted file mode 100644 index 57cf48b..0000000 --- a/jf-live-writer/src/LiveImageAssembler.cpp +++ /dev/null @@ -1,159 +0,0 @@ -#include - -#include "LiveImageAssembler.hpp" -#include "buffer_config.hpp" -#include "live_writer_config.hpp" - -using namespace std; -using namespace buffer_config; -using namespace live_writer_config; - -LiveImageAssembler::LiveImageAssembler(const size_t n_modules) : - n_modules_(n_modules), - image_buffer_slot_n_bytes_(MODULE_N_BYTES * n_modules_) -{ - image_buffer_ = new char[WRITER_IA_N_SLOTS * image_buffer_slot_n_bytes_]; - image_meta_buffer_ = new ImageMetadata[WRITER_IA_N_SLOTS]; - frame_meta_buffer_ = new ModuleFrame[WRITER_IA_N_SLOTS * n_modules]; - buffer_status_ = new atomic_int[WRITER_IA_N_SLOTS]; - buffer_pulse_id_ = new atomic_uint64_t[WRITER_IA_N_SLOTS]; - - for (size_t i=0; i < WRITER_IA_N_SLOTS; i++) { - free_slot(i); - } -} - -LiveImageAssembler::~LiveImageAssembler() -{ - delete[] image_buffer_; - delete[] image_meta_buffer_; -} - -bool LiveImageAssembler::is_slot_free(const uint64_t pulse_id) -{ - auto slot_id = pulse_id % WRITER_IA_N_SLOTS; - - uint64_t slot_pulse_id = IA_EMPTY_SLOT_VALUE; - if (buffer_pulse_id_[slot_id].compare_exchange_strong( - slot_pulse_id, pulse_id)) { - return true; - } - - auto is_free = buffer_status_[slot_id].load(memory_order_relaxed) > 0; - return is_free && (slot_pulse_id == pulse_id); -} - -bool LiveImageAssembler::is_slot_full(const uint64_t pulse_id) -{ - auto slot_id = pulse_id % WRITER_IA_N_SLOTS; - return buffer_status_[slot_id].load(memory_order_relaxed) == 0; -} - -size_t LiveImageAssembler::get_data_offset( - const uint64_t slot_id, const int i_module) -{ - size_t slot_i_offset = slot_id * image_buffer_slot_n_bytes_; - size_t module_i_offset = i_module * MODULE_N_BYTES; - - return slot_i_offset + module_i_offset; -} - -size_t LiveImageAssembler::get_frame_metadata_offset( - const uint64_t slot_id, const int i_module) -{ - size_t slot_m_offset = slot_id * n_modules_; - size_t module_m_offset = i_module; - - return slot_m_offset + module_m_offset; -} - -void LiveImageAssembler::process( - const uint64_t pulse_id, - const int i_module, - const BufferBinaryFormat* file_buffer) -{ - const auto slot_id = pulse_id % WRITER_IA_N_SLOTS; - - auto frame_meta_offset = get_frame_metadata_offset(slot_id, i_module); - auto image_offset = get_data_offset(slot_id, i_module); - - memcpy( - &(frame_meta_buffer_[frame_meta_offset]), - &(file_buffer->metadata), - sizeof(file_buffer->metadata)); - - memcpy( - image_buffer_ + image_offset, - &(file_buffer->data[0]), - MODULE_N_BYTES); - - buffer_status_[slot_id].fetch_sub(1, memory_order_relaxed); -} - -void LiveImageAssembler::free_slot(const uint64_t pulse_id) -{ - auto slot_id = pulse_id % WRITER_IA_N_SLOTS; - buffer_status_[slot_id].store(n_modules_, memory_order_relaxed); - buffer_pulse_id_[slot_id].store(IA_EMPTY_SLOT_VALUE, memory_order_relaxed); -} - -ImageMetadata* LiveImageAssembler::get_metadata_buffer(const uint64_t pulse_id) -{ - const auto slot_id = pulse_id % WRITER_IA_N_SLOTS; - - ImageMetadata& image_meta = image_meta_buffer_[slot_id]; - - auto frame_meta_offset = get_frame_metadata_offset(slot_id, 0); - - auto is_pulse_init = false; - image_meta.is_good_image = 1; - image_meta.pulse_id = 0; - - for (size_t i_module=0; i_module < n_modules_; i_module++) { - - auto& frame_meta = frame_meta_buffer_[frame_meta_offset]; - frame_meta_offset += 1; - - auto is_good_frame = - frame_meta.n_recv_packets == JF_N_PACKETS_PER_FRAME; - - if (!is_good_frame) { - image_meta.pulse_id = 0; - continue; - } - - if (!is_pulse_init) { - image_meta.pulse_id = frame_meta.pulse_id; - image_meta.frame_index = frame_meta.frame_index; - image_meta.daq_rec = frame_meta.daq_rec; - - is_pulse_init = true; - } - - if (image_meta.is_good_image == 1) { - if (frame_meta.pulse_id != image_meta.pulse_id) { - image_meta.is_good_image = 0; - } - - if (frame_meta.frame_index != image_meta.frame_index) { - image_meta.is_good_image = 0; - } - - if (frame_meta.daq_rec != image_meta.daq_rec) { - image_meta.is_good_image = 0; - } - - if (frame_meta.n_recv_packets != JF_N_PACKETS_PER_FRAME) { - image_meta.is_good_image = 0; - } - } - } - - return &image_meta; -} - -char* LiveImageAssembler::get_data_buffer(const uint64_t pulse_id) -{ - auto slot_id = pulse_id % WRITER_IA_N_SLOTS; - return image_buffer_ + (slot_id * image_buffer_slot_n_bytes_); -} diff --git a/jf-live-writer/src/main.cpp b/jf-live-writer/src/main.cpp index 139a34f..1b912f8 100644 --- a/jf-live-writer/src/main.cpp +++ b/jf-live-writer/src/main.cpp @@ -1,195 +1,46 @@ #include #include -#include -#include -#include - -#include "zmq.h" +#include +#include +#include #include "live_writer_config.hpp" -#include "buffer_config.hpp" -#include "bitshuffle/bitshuffle.h" -#include "JFH5LiveWriter.hpp" -#include "LiveImageAssembler.hpp" -#include "BinaryReader.hpp" +#include "../../jf-buffer-writer/include/BufferStats.hpp" + using namespace std; -using namespace chrono; using namespace buffer_config; using namespace live_writer_config; -void read_buffer( - const string detector_folder, - const string module_name, - const int i_module, - const vector& pulse_ids_to_write, - LiveImageAssembler& image_assembler, - void* ctx) -{ - BinaryReader reader(detector_folder, module_name); - auto frame_buffer = new BufferBinaryFormat(); - - void* socket = zmq_socket(ctx, ZMQ_SUB); - if (socket == nullptr) { - throw runtime_error(zmq_strerror(errno)); - } - - int rcvhwm = 100; - if (zmq_setsockopt(socket, ZMQ_RCVHWM, &rcvhwm, sizeof(rcvhwm)) != 0) { - throw runtime_error(zmq_strerror(errno)); - } - - int linger = 0; - if (zmq_setsockopt(socket, ZMQ_LINGER, &linger, sizeof(linger)) != 0) { - throw runtime_error(zmq_strerror(errno)); - } - - // In milliseconds. - int rcvto = 2000; - if (zmq_setsockopt(socket, ZMQ_RCVTIMEO, &rcvto, sizeof(rcvto)) != 0 ){ - throw runtime_error(zmq_strerror(errno)); - } - - if (zmq_connect(socket, "tcp://127.0.0.1:51234") != 0) { - throw runtime_error(zmq_strerror(errno)); - } - - if (zmq_setsockopt(socket, ZMQ_SUBSCRIBE, "", 0) != 0) { - throw runtime_error(zmq_strerror(errno)); - } - - const uint64_t PULSE_ID_DELAY = 100; - - uint64_t live_pulse_id = pulse_ids_to_write.front(); - for (uint64_t pulse_id:pulse_ids_to_write) { - - while(!image_assembler.is_slot_free(pulse_id)) { - this_thread::sleep_for(chrono::milliseconds(ASSEMBLER_RETRY_MS)); - } - - auto start_time = steady_clock::now(); - - // Enforce a delay of 1 second for writing. - while (live_pulse_id - pulse_id < PULSE_ID_DELAY) { - if (zmq_recv(socket, &live_pulse_id, - sizeof(live_pulse_id), 0) == -1) { - if (errno == EAGAIN) { - throw runtime_error("Did not receive pulse_id in time."); - } else { - throw runtime_error(zmq_strerror(errno)); - } - } - } - - reader.get_frame(pulse_id, frame_buffer); - - auto end_time = steady_clock::now(); - uint64_t read_us_duration = duration_cast( - end_time-start_time).count(); - - start_time = steady_clock::now(); - - image_assembler.process(pulse_id, i_module, frame_buffer); - - end_time = steady_clock::now(); - uint64_t compose_us_duration = duration_cast( - end_time-start_time).count(); - - cout << "sf_writer:avg_read_us "; - cout << read_us_duration / BUFFER_BLOCK_SIZE << endl; - cout << "sf_writer:avg_assemble_us "; - cout << compose_us_duration / BUFFER_BLOCK_SIZE << endl; - } - - delete frame_buffer; -} - int main (int argc, char *argv[]) { - if (argc != 7) { + if (argc != 3) { cout << endl; - cout << "Usage: sf_writer [output_file] [detector_folder] [n_modules]"; - cout << " [start_pulse_id] [n_pulses] [pulse_id_step]"; - cout << endl; - cout << "\toutput_file: Complete path to the output file." << endl; - cout << "\tdetector_folder: Absolute path to detector buffer." << endl; - cout << "\tn_modules: number of modules" << endl; - cout << "\tstart_pulse_id: Start pulse_id of retrieval." << endl; - cout << "\tn_pulses: Number of pulses to write." << endl; - cout << "\tpulse_id_step: 1==100Hz, 2==50hz, 4==25Hz.." << endl; + cout << "Usage: jf_live_writer [detector_json_filename]" + " [stream_name]" << endl; + cout << "\tdetector_json_filename: detector config file path." << endl; cout << endl; exit(-1); } - string output_file = string(argv[1]); - const string detector_folder = string(argv[2]); - size_t n_modules = atoi(argv[3]); - uint64_t start_pulse_id = (uint64_t) atoll(argv[4]); - size_t n_pulses = (size_t) atoll(argv[5]); - int pulse_id_step = atoi(argv[6]); - - std::vector pulse_ids_to_write; - uint64_t i_pulse_id = start_pulse_id; - for (size_t i=0; i reading_threads(n_modules); - for (size_t i_module=0; i_module( - end_time-start_time).count(); - - image_assembler.free_slot(pulse_id); - - cout << "sf_writer:avg_write_us "; - cout << write_us_duration / BUFFER_BLOCK_SIZE << endl; - } - - for (auto& reading_thread : reading_threads) { - if (reading_thread.joinable()) { - reading_thread.join(); - } - } - - return 0; } From 2a9e4f2b7c8b450f8a888dbe45864d4f09c60eb3 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 19 Jan 2021 18:35:28 +0100 Subject: [PATCH 102/126] Implementation of WriterStats for the live writer --- .../{BufferStats.hpp => WriterStats.hpp} | 12 ++++----- .../src/{BufferStats.cpp => WriterStats.cpp} | 26 +++++++++---------- 2 files changed, 17 insertions(+), 21 deletions(-) rename jf-live-writer/include/{BufferStats.hpp => WriterStats.hpp} (75%) rename jf-live-writer/src/{BufferStats.cpp => WriterStats.cpp} (66%) diff --git a/jf-live-writer/include/BufferStats.hpp b/jf-live-writer/include/WriterStats.hpp similarity index 75% rename from jf-live-writer/include/BufferStats.hpp rename to jf-live-writer/include/WriterStats.hpp index 3aff6cb..cb023a7 100644 --- a/jf-live-writer/include/BufferStats.hpp +++ b/jf-live-writer/include/WriterStats.hpp @@ -6,12 +6,11 @@ #define SF_DAQ_BUFFER_FRAMESTATS_HPP -class BufferStats { +class WriterStats { const std::string detector_name_; - const int module_id_; size_t stats_modulo_; - int frames_counter_; + int image_counter_; uint32_t total_buffer_write_us_; uint32_t max_buffer_write_us_; std::chrono::time_point stats_interval_start_; @@ -20,12 +19,11 @@ class BufferStats { void print_stats(); public: - BufferStats( + WriterStats( const std::string &detector_name, - const int module_id, const size_t stats_modulo); - void start_frame_write(); - void end_frame_write(); + void start_image_write(); + void end_image_write(); }; diff --git a/jf-live-writer/src/BufferStats.cpp b/jf-live-writer/src/WriterStats.cpp similarity index 66% rename from jf-live-writer/src/BufferStats.cpp rename to jf-live-writer/src/WriterStats.cpp index 173a35c..1d67947 100644 --- a/jf-live-writer/src/BufferStats.cpp +++ b/jf-live-writer/src/WriterStats.cpp @@ -1,35 +1,33 @@ #include -#include "BufferStats.hpp" +#include "WriterStats.hpp" using namespace std; using namespace chrono; -BufferStats::BufferStats( +WriterStats::WriterStats( const string& detector_name, - const int module_id, const size_t stats_modulo) : detector_name_(detector_name), - module_id_(module_id), stats_modulo_(stats_modulo) { reset_counters(); } -void BufferStats::reset_counters() +void WriterStats::reset_counters() { - frames_counter_ = 0; + image_counter_ = 0; total_buffer_write_us_ = 0; max_buffer_write_us_ = 0; } -void BufferStats::start_frame_write() +void WriterStats::start_image_write() { stats_interval_start_ = steady_clock::now(); } -void BufferStats::end_frame_write() +void WriterStats::end_image_write() { - frames_counter_++; + image_counter_++; uint32_t write_us_duration = duration_cast( steady_clock::now()-stats_interval_start_).count(); @@ -37,15 +35,15 @@ void BufferStats::end_frame_write() total_buffer_write_us_ += write_us_duration; max_buffer_write_us_ = max(max_buffer_write_us_, write_us_duration); - if (frames_counter_ == stats_modulo_) { + if (image_counter_ == stats_modulo_) { print_stats(); reset_counters(); } } -void BufferStats::print_stats() +void WriterStats::print_stats() { - float avg_buffer_write_us = total_buffer_write_us_ / frames_counter_; + float avg_buffer_write_us = total_buffer_write_us_ / image_counter_; uint64_t timestamp = time_point_cast( system_clock::now()).time_since_epoch().count(); @@ -53,9 +51,9 @@ void BufferStats::print_stats() // Output in InfluxDB line protocol cout << "jf_buffer_writer"; cout << ",detector_name=" << detector_name_; - cout << ",module_name=M" << module_id_; cout << " "; - cout << "avg_buffer_write_us=" << avg_buffer_write_us; + cout << "n_written_images=" << image_counter_ << "i"; + cout << " ,avg_buffer_write_us=" << avg_buffer_write_us; cout << ",max_buffer_write_us=" << max_buffer_write_us_ << "i"; cout << " "; cout << timestamp; From 94749585d892ad343b42b0bd6920686e884def92 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 19 Jan 2021 19:09:19 +0100 Subject: [PATCH 103/126] Add ImageBinaryWriter implementation --- jf-live-writer/include/ImageBinaryWriter.hpp | 33 ++++ jf-live-writer/src/ImageBinaryWriter.cpp | 165 +++++++++++++++++++ 2 files changed, 198 insertions(+) create mode 100644 jf-live-writer/include/ImageBinaryWriter.hpp create mode 100644 jf-live-writer/src/ImageBinaryWriter.cpp diff --git a/jf-live-writer/include/ImageBinaryWriter.hpp b/jf-live-writer/include/ImageBinaryWriter.hpp new file mode 100644 index 0000000..8e6ebfb --- /dev/null +++ b/jf-live-writer/include/ImageBinaryWriter.hpp @@ -0,0 +1,33 @@ +#ifndef BINARYWRITER_HPP +#define BINARYWRITER_HPP + +#include + +#include "formats.hpp" + +class ImageBinaryWriter { + + const size_t MAX_FILE_BYTES = + buffer_config::FILE_MOD * sizeof(BufferBinaryFormat); + + const std::string detector_folder_; + std::string latest_filename_; + + std::string current_output_filename_; + int output_file_fd_; + + void open_file(const std::string& filename); + void close_current_file(); + + +public: + ImageBinaryWriter(const std::string& detector_folder); + + virtual ~ImageBinaryWriter(); + + void write(const uint64_t pulse_id, const BufferBinaryFormat* buffer); + +}; + + +#endif //BINARYWRITER_HPP diff --git a/jf-live-writer/src/ImageBinaryWriter.cpp b/jf-live-writer/src/ImageBinaryWriter.cpp new file mode 100644 index 0000000..c3f70f5 --- /dev/null +++ b/jf-live-writer/src/ImageBinaryWriter.cpp @@ -0,0 +1,165 @@ +#include "ImageBinaryWriter.hpp" + +#include +#include +#include "date.h" +#include +#include +#include +#include + +#include "BufferUtils.hpp" + +using namespace std; + +ImageBinaryWriter::ImageBinaryWriter( + const string& detector_folder): + detector_folder_(detector_folder), + latest_filename_(detector_folder + "/LATEST"), + current_output_filename_(""), + output_file_fd_(-1) +{ +} + +ImageBinaryWriter::~ImageBinaryWriter() +{ + close_current_file(); +} + +void ImageBinaryWriter::write( + const uint64_t pulse_id, + const BufferBinaryFormat* buffer) +{ + auto current_frame_file = + BufferUtils::get_filename(detector_folder_, module_name_, pulse_id); + + if (current_frame_file != current_output_filename_) { + open_file(current_frame_file); + } + + size_t n_bytes_offset = + BufferUtils::get_file_frame_index(pulse_id) * + sizeof(BufferBinaryFormat); + + auto lseek_result = lseek(output_file_fd_, n_bytes_offset, SEEK_SET); + if (lseek_result < 0) { + stringstream err_msg; + + using namespace date; + using namespace chrono; + err_msg << "[" << system_clock::now() << "]"; + err_msg << "[BufferBinaryWriter::write]"; + err_msg << " Error while lseek on file "; + err_msg << current_output_filename_; + err_msg << " for n_bytes_offset "; + err_msg << n_bytes_offset << ": "; + err_msg << strerror(errno) << endl; + + throw runtime_error(err_msg.str()); + } + + auto n_bytes = ::write(output_file_fd_, buffer, sizeof(BufferBinaryFormat)); + if (n_bytes < sizeof(BufferBinaryFormat)) { + stringstream err_msg; + + using namespace date; + using namespace chrono; + err_msg << "[" << system_clock::now() << "]"; + err_msg << "[BufferBinaryWriter::write]"; + err_msg << " Error while writing to file "; + err_msg << current_output_filename_ << ": "; + err_msg << strerror(errno) << endl; + + throw runtime_error(err_msg.str()); + } +} + +void ImageBinaryWriter::open_file(const std::string& filename) +{ + close_current_file(); + + BufferUtils::create_destination_folder(filename); + + output_file_fd_ = ::open(filename.c_str(), O_WRONLY | O_CREAT, + S_IRWXU | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH); + if (output_file_fd_ < 0) { + stringstream err_msg; + + using namespace date; + using namespace chrono; + err_msg << "[" << system_clock::now() << "]"; + err_msg << "[BinaryWriter::open_file]"; + err_msg << " Cannot create file "; + err_msg << filename << ": "; + err_msg << strerror(errno) << endl; + + throw runtime_error(err_msg.str()); + } + + // TODO: Remove context if test successful. + + /** Setting the buffer file size in advance to try to lower the number of + metadata updates on GPFS. */ + { + // TODO: Try instead to use fallocate. + if (lseek(output_file_fd_, MAX_FILE_BYTES, SEEK_SET) < 0) { + stringstream err_msg; + + using namespace date; + using namespace chrono; + err_msg << "[" << system_clock::now() << "]"; + err_msg << "[BufferBinaryWriter::open_file]"; + err_msg << " Error while lseek on end of file "; + err_msg << current_output_filename_; + err_msg << " for MAX_FILE_BYTES "; + err_msg << MAX_FILE_BYTES << ": "; + err_msg << strerror(errno) << endl; + + throw runtime_error(err_msg.str()); + } + + const uint8_t mark = 255; + if(::write(output_file_fd_, &mark, sizeof(mark)) != sizeof(mark)) { + stringstream err_msg; + + using namespace date; + using namespace chrono; + err_msg << "[" << system_clock::now() << "]"; + err_msg << "[BufferBinaryWriter::open_file]"; + err_msg << " Error while writing to file "; + err_msg << current_output_filename_ << ": "; + err_msg << strerror(errno) << endl; + + throw runtime_error(err_msg.str()); + } + } + + + current_output_filename_ = filename; +} + +void ImageBinaryWriter::close_current_file() +{ + if (output_file_fd_ != -1) { + if (close(output_file_fd_) < 0) { + stringstream err_msg; + + using namespace date; + using namespace chrono; + err_msg << "[" << system_clock::now() << "]"; + err_msg << "[BufferBinaryWriter::close_current_file]"; + err_msg << " Error while closing file "; + err_msg << current_output_filename_ << ": "; + err_msg << strerror(errno) << endl; + + throw runtime_error(err_msg.str()); + } + + output_file_fd_ = -1; + + BufferUtils::update_latest_file( + latest_filename_, current_output_filename_); + + current_output_filename_ = ""; + } +} \ No newline at end of file From b39fd93431ea518f690b4ab8a7107c97b9c4f2a6 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 20 Jan 2021 11:10:39 +0100 Subject: [PATCH 104/126] Add method for creating image buffer files --- core-buffer/include/BufferUtils.hpp | 4 ++++ core-buffer/src/BufferUtils.cpp | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/core-buffer/include/BufferUtils.hpp b/core-buffer/include/BufferUtils.hpp index 18a3fe5..1403bd0 100644 --- a/core-buffer/include/BufferUtils.hpp +++ b/core-buffer/include/BufferUtils.hpp @@ -27,6 +27,10 @@ namespace BufferUtils const std::string& module_name, const uint64_t pulse_id); + std::string get_image_filename( + const std::string& detector_folder, + const uint64_t pulse_id); + std::size_t get_file_frame_index(const uint64_t pulse_id); void update_latest_file( diff --git a/core-buffer/src/BufferUtils.cpp b/core-buffer/src/BufferUtils.cpp index 147d4e8..c763966 100644 --- a/core-buffer/src/BufferUtils.cpp +++ b/core-buffer/src/BufferUtils.cpp @@ -11,6 +11,24 @@ using namespace std; using namespace buffer_config; +string BufferUtils::get_image_filename( + const std::string& detector_folder, + const uint64_t pulse_id) +{ + uint64_t data_folder = pulse_id / buffer_config::FOLDER_MOD; + data_folder *= buffer_config::FOLDER_MOD; + + uint64_t data_file = pulse_id / buffer_config::FILE_MOD; + data_file *= buffer_config::FILE_MOD; + + stringstream folder; + folder << detector_folder << "/"; + folder << data_folder << "/"; + folder << data_file << buffer_config::FILE_EXTENSION; + + return folder.str(); +} + string BufferUtils::get_filename( const std::string& detector_folder, const std::string& module_name, From 407b00890d9e28a3cda6eee60cbf11adba4447bb Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Thu, 21 Jan 2021 10:16:38 +0100 Subject: [PATCH 105/126] Finish BinaryWriter for Images in buffer --- jf-live-writer/include/ImageBinaryWriter.hpp | 21 ++--- jf-live-writer/src/ImageBinaryWriter.cpp | 80 +++++++------------- 2 files changed, 41 insertions(+), 60 deletions(-) diff --git a/jf-live-writer/include/ImageBinaryWriter.hpp b/jf-live-writer/include/ImageBinaryWriter.hpp index 8e6ebfb..ce80655 100644 --- a/jf-live-writer/include/ImageBinaryWriter.hpp +++ b/jf-live-writer/include/ImageBinaryWriter.hpp @@ -1,15 +1,16 @@ -#ifndef BINARYWRITER_HPP -#define BINARYWRITER_HPP +#ifndef IMAGEBINARYWRITER_HPP +#define IMAGEBINARYWRITER_HPP #include #include "formats.hpp" + + class ImageBinaryWriter { - - const size_t MAX_FILE_BYTES = - buffer_config::FILE_MOD * sizeof(BufferBinaryFormat); - + const size_t IMAGE_BYTES; + const size_t IMAGE_SLOT_BYTES; + const size_t MAX_FILE_BYTES; const std::string detector_folder_; std::string latest_filename_; @@ -21,13 +22,15 @@ class ImageBinaryWriter { public: - ImageBinaryWriter(const std::string& detector_folder); + ImageBinaryWriter( + const std::string& detector_folder, + const uint64_t image_n_bytes); virtual ~ImageBinaryWriter(); - void write(const uint64_t pulse_id, const BufferBinaryFormat* buffer); + void write(const ImageMetadata meta, const char* data); }; -#endif //BINARYWRITER_HPP +#endif //IMAGEBINARYWRITER_HPP diff --git a/jf-live-writer/src/ImageBinaryWriter.cpp b/jf-live-writer/src/ImageBinaryWriter.cpp index c3f70f5..d5f9900 100644 --- a/jf-live-writer/src/ImageBinaryWriter.cpp +++ b/jf-live-writer/src/ImageBinaryWriter.cpp @@ -11,9 +11,14 @@ #include "BufferUtils.hpp" using namespace std; +using namespace buffer_config; ImageBinaryWriter::ImageBinaryWriter( - const string& detector_folder): + const string& detector_folder, + const size_t image_n_bytes): + IMAGE_BYTES(image_n_bytes), + IMAGE_SLOT_BYTES(IMAGE_BYTES + sizeof(ImageMetadata)), + MAX_FILE_BYTES(IMAGE_SLOT_BYTES * FILE_MOD), detector_folder_(detector_folder), latest_filename_(detector_folder + "/LATEST"), current_output_filename_(""), @@ -26,20 +31,17 @@ ImageBinaryWriter::~ImageBinaryWriter() close_current_file(); } -void ImageBinaryWriter::write( - const uint64_t pulse_id, - const BufferBinaryFormat* buffer) +void ImageBinaryWriter::write(const ImageMetadata meta, const char* data) { auto current_frame_file = - BufferUtils::get_filename(detector_folder_, module_name_, pulse_id); + BufferUtils::get_image_filename(detector_folder_, meta.pulse_id); if (current_frame_file != current_output_filename_) { open_file(current_frame_file); } size_t n_bytes_offset = - BufferUtils::get_file_frame_index(pulse_id) * - sizeof(BufferBinaryFormat); + BufferUtils::get_file_frame_index(meta.pulse_id) * IMAGE_SLOT_BYTES; auto lseek_result = lseek(output_file_fd_, n_bytes_offset, SEEK_SET); if (lseek_result < 0) { @@ -48,7 +50,7 @@ void ImageBinaryWriter::write( using namespace date; using namespace chrono; err_msg << "[" << system_clock::now() << "]"; - err_msg << "[BufferBinaryWriter::write]"; + err_msg << "[ImageBinaryWriter::write]"; err_msg << " Error while lseek on file "; err_msg << current_output_filename_; err_msg << " for n_bytes_offset "; @@ -58,8 +60,23 @@ void ImageBinaryWriter::write( throw runtime_error(err_msg.str()); } - auto n_bytes = ::write(output_file_fd_, buffer, sizeof(BufferBinaryFormat)); - if (n_bytes < sizeof(BufferBinaryFormat)) { + auto n_bytes_meta = ::write(output_file_fd_, &meta, sizeof(ImageMetadata)); + if (n_bytes_meta < sizeof(ImageMetadata)) { + stringstream err_msg; + + using namespace date; + using namespace chrono; + err_msg << "[" << system_clock::now() << "]"; + err_msg << "[BufferBinaryWriter::write]"; + err_msg << " Error while writing to file "; + err_msg << current_output_filename_ << ": "; + err_msg << strerror(errno) << endl; + + throw runtime_error(err_msg.str()); + } + + auto n_bytes_data = ::write(output_file_fd_, data, IMAGE_BYTES); + if (n_bytes_data < sizeof(IMAGE_BYTES)) { stringstream err_msg; using namespace date; @@ -88,7 +105,7 @@ void ImageBinaryWriter::open_file(const std::string& filename) using namespace date; using namespace chrono; err_msg << "[" << system_clock::now() << "]"; - err_msg << "[BinaryWriter::open_file]"; + err_msg << "[ImageBinaryWriter::open_file]"; err_msg << " Cannot create file "; err_msg << filename << ": "; err_msg << strerror(errno) << endl; @@ -96,45 +113,6 @@ void ImageBinaryWriter::open_file(const std::string& filename) throw runtime_error(err_msg.str()); } - // TODO: Remove context if test successful. - - /** Setting the buffer file size in advance to try to lower the number of - metadata updates on GPFS. */ - { - // TODO: Try instead to use fallocate. - if (lseek(output_file_fd_, MAX_FILE_BYTES, SEEK_SET) < 0) { - stringstream err_msg; - - using namespace date; - using namespace chrono; - err_msg << "[" << system_clock::now() << "]"; - err_msg << "[BufferBinaryWriter::open_file]"; - err_msg << " Error while lseek on end of file "; - err_msg << current_output_filename_; - err_msg << " for MAX_FILE_BYTES "; - err_msg << MAX_FILE_BYTES << ": "; - err_msg << strerror(errno) << endl; - - throw runtime_error(err_msg.str()); - } - - const uint8_t mark = 255; - if(::write(output_file_fd_, &mark, sizeof(mark)) != sizeof(mark)) { - stringstream err_msg; - - using namespace date; - using namespace chrono; - err_msg << "[" << system_clock::now() << "]"; - err_msg << "[BufferBinaryWriter::open_file]"; - err_msg << " Error while writing to file "; - err_msg << current_output_filename_ << ": "; - err_msg << strerror(errno) << endl; - - throw runtime_error(err_msg.str()); - } - } - - current_output_filename_ = filename; } @@ -147,7 +125,7 @@ void ImageBinaryWriter::close_current_file() using namespace date; using namespace chrono; err_msg << "[" << system_clock::now() << "]"; - err_msg << "[BufferBinaryWriter::close_current_file]"; + err_msg << "[ImageBinaryWriter::close_current_file]"; err_msg << " Error while closing file "; err_msg << current_output_filename_ << ": "; err_msg << strerror(errno) << endl; From 2ed8b0fd1b3f3939060f5f1062b1f347501fddac Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Thu, 21 Jan 2021 10:29:19 +0100 Subject: [PATCH 106/126] Add avg throughput to writer stats --- jf-live-writer/include/WriterStats.hpp | 6 ++++-- jf-live-writer/src/WriterStats.cpp | 20 ++++++++++++++------ 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/jf-live-writer/include/WriterStats.hpp b/jf-live-writer/include/WriterStats.hpp index cb023a7..775a46f 100644 --- a/jf-live-writer/include/WriterStats.hpp +++ b/jf-live-writer/include/WriterStats.hpp @@ -8,7 +8,8 @@ class WriterStats { const std::string detector_name_; - size_t stats_modulo_; + const size_t stats_modulo_; + const size_t image_n_bytes_; int image_counter_; uint32_t total_buffer_write_us_; @@ -21,7 +22,8 @@ class WriterStats { public: WriterStats( const std::string &detector_name, - const size_t stats_modulo); + const size_t stats_modulo, + const size_t image_n_bytes); void start_image_write(); void end_image_write(); }; diff --git a/jf-live-writer/src/WriterStats.cpp b/jf-live-writer/src/WriterStats.cpp index 1d67947..51a0922 100644 --- a/jf-live-writer/src/WriterStats.cpp +++ b/jf-live-writer/src/WriterStats.cpp @@ -6,9 +6,11 @@ using namespace chrono; WriterStats::WriterStats( const string& detector_name, - const size_t stats_modulo) : + const size_t stats_modulo, + const size_t image_n_bytes) : detector_name_(detector_name), - stats_modulo_(stats_modulo) + stats_modulo_(stats_modulo), + image_n_bytes_(image_n_bytes) { reset_counters(); } @@ -43,19 +45,25 @@ void WriterStats::end_image_write() void WriterStats::print_stats() { - float avg_buffer_write_us = total_buffer_write_us_ / image_counter_; + const float avg_buffer_write_us = total_buffer_write_us_ / image_counter_; - uint64_t timestamp = time_point_cast( + const uint64_t timestamp = time_point_cast( system_clock::now()).time_since_epoch().count(); + const uint64_t avg_throughput = + // bytes -> megabytes + (image_n_bytes_ / 1024 / 1024) / + // micro seconds -> seconds + (avg_buffer_write_us * 1000 * 1000); + // Output in InfluxDB line protocol cout << "jf_buffer_writer"; cout << ",detector_name=" << detector_name_; cout << " "; cout << "n_written_images=" << image_counter_ << "i"; cout << " ,avg_buffer_write_us=" << avg_buffer_write_us; - cout << ",max_buffer_write_us=" << max_buffer_write_us_ << "i"; - cout << " "; + cout << " ,max_buffer_write_us=" << max_buffer_write_us_ << "i"; + cout << " ,avg_throughput=" << avg_throughput; cout << timestamp; cout << endl; } From 6a1fb79f9f6ca5823dc01d37d9018ae390481133 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Thu, 21 Jan 2021 10:32:15 +0100 Subject: [PATCH 107/126] Fix live_writer tests --- jf-live-writer/test/CMakeLists.txt | 4 +--- jf-live-writer/test/main.cpp | 2 -- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/jf-live-writer/test/CMakeLists.txt b/jf-live-writer/test/CMakeLists.txt index 1079fc2..8f806b0 100644 --- a/jf-live-writer/test/CMakeLists.txt +++ b/jf-live-writer/test/CMakeLists.txt @@ -2,9 +2,7 @@ add_executable(jf-live-writer-tests main.cpp) target_link_libraries(jf-live-writer-tests jf-live-writer-lib - hdf5 - hdf5_hl - hdf5_cpp zmq + rt gtest ) diff --git a/jf-live-writer/test/main.cpp b/jf-live-writer/test/main.cpp index 69b7f53..e819294 100644 --- a/jf-live-writer/test/main.cpp +++ b/jf-live-writer/test/main.cpp @@ -1,7 +1,5 @@ #include "gtest/gtest.h" -#include "test_BinaryReader.cpp" - using namespace std; int main(int argc, char **argv) { From d0d5776675d037b9e708c1fad657fc2a1c3ccc24 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Thu, 21 Jan 2021 10:32:50 +0100 Subject: [PATCH 108/126] Adjust the main for the jf-live-writer to ImageWriter --- jf-live-writer/CMakeLists.txt | 7 ++----- jf-live-writer/src/main.cpp | 22 +++++++++++++++------- jf-live-writer/test/test_BinaryReader.cpp | 10 ---------- 3 files changed, 17 insertions(+), 22 deletions(-) delete mode 100644 jf-live-writer/test/test_BinaryReader.cpp diff --git a/jf-live-writer/CMakeLists.txt b/jf-live-writer/CMakeLists.txt index 6028bfe..f96af5a 100644 --- a/jf-live-writer/CMakeLists.txt +++ b/jf-live-writer/CMakeLists.txt @@ -11,11 +11,8 @@ add_executable(jf-live-writer src/main.cpp) set_target_properties(jf-live-writer PROPERTIES OUTPUT_NAME jf_live_writer) target_link_libraries(jf-live-writer jf-live-writer-lib - sf-writer-lib - hdf5 - hdf5_hl - hdf5_cpp - pthread + zmq + rt ) enable_testing() diff --git a/jf-live-writer/src/main.cpp b/jf-live-writer/src/main.cpp index 1b912f8..aa47b8c 100644 --- a/jf-live-writer/src/main.cpp +++ b/jf-live-writer/src/main.cpp @@ -3,8 +3,9 @@ #include #include #include +#include #include "live_writer_config.hpp" -#include "../../jf-buffer-writer/include/BufferStats.hpp" +#include "WriterStats.hpp" using namespace std; @@ -16,15 +17,16 @@ int main (int argc, char *argv[]) if (argc != 3) { cout << endl; cout << "Usage: jf_live_writer [detector_json_filename]" - " [stream_name]" << endl; + " [writer_id]" << endl; cout << "\tdetector_json_filename: detector config file path." << endl; + cout << "\twriter_id: Index of this writer instance." << endl; cout << endl; exit(-1); } - const auto stream_name = string(argv[2]); auto config = BufferUtils::read_json_config(string(argv[1])); + const int writer_id = atoi(argv[2]); auto ctx = zmq_ctx_new(); zmq_ctx_set(ctx, ZMQ_IO_THREADS, LIVE_ZMQ_IO_THREADS); @@ -32,15 +34,21 @@ int main (int argc, char *argv[]) ctx, config.detector_name, "assembler"); RamBuffer ram_buffer(config.detector_name, config.n_modules); - BufferStats stats(config.detector_name, stream_name, STATS_MODULO); - ImageMetadata meta; + const uint64_t image_n_bytes = config.n_modules * MODULE_N_BYTES; + ImageBinaryWriter writer(config.detector_name, image_n_bytes); + + WriterStats stats(config.detector_name, STATS_MODULO, image_n_bytes); + + ImageMetadata meta = {}; while (true) { zmq_recv(receiver, &meta, sizeof(meta), 0); char* data = ram_buffer.read_image(meta.pulse_id); - sender.send(meta, data); + stats.start_image_write(); - stats.record_stats(meta); + writer.write(meta, data); + + stats.end_image_write(); } } diff --git a/jf-live-writer/test/test_BinaryReader.cpp b/jf-live-writer/test/test_BinaryReader.cpp deleted file mode 100644 index cc30157..0000000 --- a/jf-live-writer/test/test_BinaryReader.cpp +++ /dev/null @@ -1,10 +0,0 @@ -#include -#include "gtest/gtest.h" - -TEST(BinaryReader, basic_interaction) { - // TODO: Write some real tests. - auto detector_folder = "test_device"; - auto module_name = "M1"; - BinaryReader reader(detector_folder, module_name); -} - From aba739ce877f23e70dc33938bdbb3e2b5e387783 Mon Sep 17 00:00:00 2001 From: Dmitry Ozerov Date: Tue, 16 Feb 2021 16:25:28 +0100 Subject: [PATCH 109/126] print statistics from udp writer every N seconds (not N frames) check that frame packets sending is finished (new frame) by new trigger number, not pulse_id check if pulse_id of the frame is more or less correct (in case not - do not store that frame in the ram buffer) --- core-buffer/include/buffer_config.hpp | 6 ++++-- jf-udp-recv/include/FrameStats.hpp | 7 ++++--- jf-udp-recv/src/FrameStats.cpp | 18 +++++++++++++---- jf-udp-recv/src/FrameUdpReceiver.cpp | 6 +++--- jf-udp-recv/src/main.cpp | 28 +++++++++++++++++++++++---- 5 files changed, 49 insertions(+), 16 deletions(-) diff --git a/core-buffer/include/buffer_config.hpp b/core-buffer/include/buffer_config.hpp index 293308b..b2e68ca 100644 --- a/core-buffer/include/buffer_config.hpp +++ b/core-buffer/include/buffer_config.hpp @@ -20,8 +20,10 @@ namespace buffer_config { const size_t FOLDER_MOD = 100000; // Extension of our file format. const std::string FILE_EXTENSION = ".bin"; - // Number of pulses between each statistics print out. - const size_t STATS_MODULO = 100; + // Number of pulses between each statistics print out (buffer_writer, stream2vis...) + const size_t STATS_MODULO = 1000; + // Number of seconds after which statistics is print out (udp_recv) + const size_t STATS_TIME = 10; // If the RB is empty, how much time to wait before trying to read it again. const size_t RB_READ_RETRY_INTERVAL_MS = 5; // How many frames to read at once from file. diff --git a/jf-udp-recv/include/FrameStats.hpp b/jf-udp-recv/include/FrameStats.hpp index dd4ef95..7839a38 100644 --- a/jf-udp-recv/include/FrameStats.hpp +++ b/jf-udp-recv/include/FrameStats.hpp @@ -9,11 +9,12 @@ class FrameStats { const std::string detector_name_; const int module_id_; - size_t stats_modulo_; + size_t stats_time_; int frames_counter_; int n_missed_packets_; int n_corrupted_frames_; + int n_corrupted_pulse_id_; std::chrono::time_point stats_interval_start_; void reset_counters(); @@ -22,8 +23,8 @@ class FrameStats { public: FrameStats(const std::string &detector_name, const int module_id, - const size_t stats_modulo); - void record_stats(const ModuleFrame &meta); + const size_t stats_time); + void record_stats(const ModuleFrame &meta, const bool bad_pulse_id); }; diff --git a/jf-udp-recv/src/FrameStats.cpp b/jf-udp-recv/src/FrameStats.cpp index 103e4fe..28161c7 100644 --- a/jf-udp-recv/src/FrameStats.cpp +++ b/jf-udp-recv/src/FrameStats.cpp @@ -7,10 +7,10 @@ using namespace chrono; FrameStats::FrameStats( const std::string &detector_name, const int module_id, - const size_t stats_modulo) : + const size_t stats_time) : detector_name_(detector_name), module_id_(module_id), - stats_modulo_(stats_modulo) + stats_time_(stats_time) { reset_counters(); } @@ -20,11 +20,17 @@ void FrameStats::reset_counters() frames_counter_ = 0; n_missed_packets_ = 0; n_corrupted_frames_ = 0; + n_corrupted_pulse_id_ = 0; stats_interval_start_ = steady_clock::now(); } -void FrameStats::record_stats(const ModuleFrame &meta) +void FrameStats::record_stats(const ModuleFrame &meta, const bool bad_pulse_id) { + + if (bad_pulse_id) { + n_corrupted_pulse_id_++; + } + if (meta.n_recv_packets < JF_N_PACKETS_PER_FRAME) { n_missed_packets_ += JF_N_PACKETS_PER_FRAME - meta.n_recv_packets; n_corrupted_frames_++; @@ -32,7 +38,10 @@ void FrameStats::record_stats(const ModuleFrame &meta) frames_counter_++; - if (frames_counter_ == stats_modulo_) { + auto time_passed = duration_cast( + steady_clock::now()-stats_interval_start_).count(); + + if (time_passed >= stats_time_*1000) { print_stats(); reset_counters(); } @@ -55,6 +64,7 @@ void FrameStats::print_stats() cout << "n_missed_packets=" << n_missed_packets_ << "i"; cout << ",n_corrupted_frames=" << n_corrupted_frames_ << "i"; cout << ",repetition_rate=" << rep_rate << "i"; + cout << ",n_corrupted_pulse_ids=" << n_corrupted_pulse_id_ << "i"; cout << " "; cout << timestamp; cout << endl; diff --git a/jf-udp-recv/src/FrameUdpReceiver.cpp b/jf-udp-recv/src/FrameUdpReceiver.cpp index 1c0833b..cb78a7a 100644 --- a/jf-udp-recv/src/FrameUdpReceiver.cpp +++ b/jf-udp-recv/src/FrameUdpReceiver.cpp @@ -63,8 +63,8 @@ inline uint64_t FrameUdpReceiver::process_packets( init_frame(metadata, i_packet); // Happens if the last packet from the previous frame gets lost. - // In the jungfrau_packet, pulse_id is called bunchid. - } else if (metadata.pulse_id != packet_buffer_[i_packet].bunchid) { + // In the jungfrau_packet, framenum is the trigger number (how many triggers from detector power-on) happened + } else if (metadata.frame_index != packet_buffer_[i_packet].framenum) { packet_buffer_loaded_ = true; // Continue on this packet. packet_buffer_offset_ = i_packet; @@ -134,4 +134,4 @@ uint64_t FrameUdpReceiver::get_frame_from_udp( return pulse_id; } } -} \ No newline at end of file +} diff --git a/jf-udp-recv/src/main.cpp b/jf-udp-recv/src/main.cpp index 2194b80..bca57e2 100644 --- a/jf-udp-recv/src/main.cpp +++ b/jf-udp-recv/src/main.cpp @@ -33,7 +33,7 @@ int main (int argc, char *argv[]) { const auto udp_port = config.start_udp_port + module_id; FrameUdpReceiver receiver(udp_port, module_id); RamBuffer buffer(config.detector_name, config.n_modules); - FrameStats stats(config.detector_name, module_id, STATS_MODULO); + FrameStats stats(config.detector_name, module_id, STATS_TIME); auto ctx = zmq_ctx_new(); auto socket = bind_socket(ctx, config.detector_name, to_string(module_id)); @@ -41,14 +41,34 @@ int main (int argc, char *argv[]) { ModuleFrame meta; char* data = new char[MODULE_N_BYTES]; + uint64_t pulse_id_previous = 0; + uint64_t frame_index_previous = 0; + while (true) { + auto pulse_id = receiver.get_frame_from_udp(meta, data); - buffer.write_frame(meta, data); + bool bad_pulse_id = false; - zmq_send(socket, &pulse_id, sizeof(pulse_id), 0); + if ( ( meta.frame_index != (frame_index_previous+1) ) || + ( (pulse_id-pulse_id_previous) < 0 ) || + ( (pulse_id-pulse_id_previous) > 1000 ) ) { + + bad_pulse_id = true; + + } else { + + buffer.write_frame(meta, data); + + zmq_send(socket, &pulse_id, sizeof(pulse_id), 0); + + } + + stats.record_stats(meta, bad_pulse_id); + + pulse_id_previous = pulse_id; + frame_index_previous = meta.frame_index; - stats.record_stats(meta); } delete[] data; From d0df2677defea40fdbd37cfbc5875db82e7852b3 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 17 Feb 2021 12:46:58 +0100 Subject: [PATCH 110/126] Remove old writer --- CMakeLists.txt | 1 - jf-live-writer/include/ImageBinaryWriter.hpp | 36 --- jf-live-writer/src/ImageBinaryWriter.cpp | 143 ----------- sf-writer/CMakeLists.txt | 22 -- sf-writer/README.md | 42 --- sf-writer/include/BufferBinaryReader.hpp | 28 -- sf-writer/include/ImageAssembler.hpp | 53 ---- sf-writer/include/writer_config.hpp | 9 - sf-writer/src/BufferBinaryReader.cpp | 107 -------- sf-writer/src/ImageAssembler.cpp | 186 -------------- sf-writer/src/main.cpp | 158 ------------ sf-writer/test/CMakeLists.txt | 10 - sf-writer/test/main.cpp | 10 - sf-writer/test/mock/data.hpp | 72 ------ sf-writer/test/perf/perf_WriterH5Writer.cpp | 90 ------- sf-writer/test/test_ImageAssembler.cpp | 90 ------- sf-writer/test/test_JFH5Writer.cpp | 254 ------------------- 17 files changed, 1311 deletions(-) delete mode 100644 jf-live-writer/include/ImageBinaryWriter.hpp delete mode 100644 jf-live-writer/src/ImageBinaryWriter.cpp delete mode 100644 sf-writer/CMakeLists.txt delete mode 100644 sf-writer/README.md delete mode 100644 sf-writer/include/BufferBinaryReader.hpp delete mode 100644 sf-writer/include/ImageAssembler.hpp delete mode 100644 sf-writer/include/writer_config.hpp delete mode 100644 sf-writer/src/BufferBinaryReader.cpp delete mode 100644 sf-writer/src/ImageAssembler.cpp delete mode 100644 sf-writer/src/main.cpp delete mode 100644 sf-writer/test/CMakeLists.txt delete mode 100644 sf-writer/test/main.cpp delete mode 100644 sf-writer/test/mock/data.hpp delete mode 100644 sf-writer/test/perf/perf_WriterH5Writer.cpp delete mode 100644 sf-writer/test/test_ImageAssembler.cpp delete mode 100644 sf-writer/test/test_JFH5Writer.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 63c25dc..ff244cf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,5 +33,4 @@ add_subdirectory("jf-udp-recv") add_subdirectory("jf-buffer-writer") add_subdirectory("jf-assembler") add_subdirectory("sf-stream") -add_subdirectory("sf-writer") add_subdirectory("jf-live-writer") diff --git a/jf-live-writer/include/ImageBinaryWriter.hpp b/jf-live-writer/include/ImageBinaryWriter.hpp deleted file mode 100644 index ce80655..0000000 --- a/jf-live-writer/include/ImageBinaryWriter.hpp +++ /dev/null @@ -1,36 +0,0 @@ -#ifndef IMAGEBINARYWRITER_HPP -#define IMAGEBINARYWRITER_HPP - -#include - -#include "formats.hpp" - - - -class ImageBinaryWriter { - const size_t IMAGE_BYTES; - const size_t IMAGE_SLOT_BYTES; - const size_t MAX_FILE_BYTES; - const std::string detector_folder_; - std::string latest_filename_; - - std::string current_output_filename_; - int output_file_fd_; - - void open_file(const std::string& filename); - void close_current_file(); - - -public: - ImageBinaryWriter( - const std::string& detector_folder, - const uint64_t image_n_bytes); - - virtual ~ImageBinaryWriter(); - - void write(const ImageMetadata meta, const char* data); - -}; - - -#endif //IMAGEBINARYWRITER_HPP diff --git a/jf-live-writer/src/ImageBinaryWriter.cpp b/jf-live-writer/src/ImageBinaryWriter.cpp deleted file mode 100644 index d5f9900..0000000 --- a/jf-live-writer/src/ImageBinaryWriter.cpp +++ /dev/null @@ -1,143 +0,0 @@ -#include "ImageBinaryWriter.hpp" - -#include -#include -#include "date.h" -#include -#include -#include -#include - -#include "BufferUtils.hpp" - -using namespace std; -using namespace buffer_config; - -ImageBinaryWriter::ImageBinaryWriter( - const string& detector_folder, - const size_t image_n_bytes): - IMAGE_BYTES(image_n_bytes), - IMAGE_SLOT_BYTES(IMAGE_BYTES + sizeof(ImageMetadata)), - MAX_FILE_BYTES(IMAGE_SLOT_BYTES * FILE_MOD), - detector_folder_(detector_folder), - latest_filename_(detector_folder + "/LATEST"), - current_output_filename_(""), - output_file_fd_(-1) -{ -} - -ImageBinaryWriter::~ImageBinaryWriter() -{ - close_current_file(); -} - -void ImageBinaryWriter::write(const ImageMetadata meta, const char* data) -{ - auto current_frame_file = - BufferUtils::get_image_filename(detector_folder_, meta.pulse_id); - - if (current_frame_file != current_output_filename_) { - open_file(current_frame_file); - } - - size_t n_bytes_offset = - BufferUtils::get_file_frame_index(meta.pulse_id) * IMAGE_SLOT_BYTES; - - auto lseek_result = lseek(output_file_fd_, n_bytes_offset, SEEK_SET); - if (lseek_result < 0) { - stringstream err_msg; - - using namespace date; - using namespace chrono; - err_msg << "[" << system_clock::now() << "]"; - err_msg << "[ImageBinaryWriter::write]"; - err_msg << " Error while lseek on file "; - err_msg << current_output_filename_; - err_msg << " for n_bytes_offset "; - err_msg << n_bytes_offset << ": "; - err_msg << strerror(errno) << endl; - - throw runtime_error(err_msg.str()); - } - - auto n_bytes_meta = ::write(output_file_fd_, &meta, sizeof(ImageMetadata)); - if (n_bytes_meta < sizeof(ImageMetadata)) { - stringstream err_msg; - - using namespace date; - using namespace chrono; - err_msg << "[" << system_clock::now() << "]"; - err_msg << "[BufferBinaryWriter::write]"; - err_msg << " Error while writing to file "; - err_msg << current_output_filename_ << ": "; - err_msg << strerror(errno) << endl; - - throw runtime_error(err_msg.str()); - } - - auto n_bytes_data = ::write(output_file_fd_, data, IMAGE_BYTES); - if (n_bytes_data < sizeof(IMAGE_BYTES)) { - stringstream err_msg; - - using namespace date; - using namespace chrono; - err_msg << "[" << system_clock::now() << "]"; - err_msg << "[BufferBinaryWriter::write]"; - err_msg << " Error while writing to file "; - err_msg << current_output_filename_ << ": "; - err_msg << strerror(errno) << endl; - - throw runtime_error(err_msg.str()); - } -} - -void ImageBinaryWriter::open_file(const std::string& filename) -{ - close_current_file(); - - BufferUtils::create_destination_folder(filename); - - output_file_fd_ = ::open(filename.c_str(), O_WRONLY | O_CREAT, - S_IRWXU | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH); - if (output_file_fd_ < 0) { - stringstream err_msg; - - using namespace date; - using namespace chrono; - err_msg << "[" << system_clock::now() << "]"; - err_msg << "[ImageBinaryWriter::open_file]"; - err_msg << " Cannot create file "; - err_msg << filename << ": "; - err_msg << strerror(errno) << endl; - - throw runtime_error(err_msg.str()); - } - - current_output_filename_ = filename; -} - -void ImageBinaryWriter::close_current_file() -{ - if (output_file_fd_ != -1) { - if (close(output_file_fd_) < 0) { - stringstream err_msg; - - using namespace date; - using namespace chrono; - err_msg << "[" << system_clock::now() << "]"; - err_msg << "[ImageBinaryWriter::close_current_file]"; - err_msg << " Error while closing file "; - err_msg << current_output_filename_ << ": "; - err_msg << strerror(errno) << endl; - - throw runtime_error(err_msg.str()); - } - - output_file_fd_ = -1; - - BufferUtils::update_latest_file( - latest_filename_, current_output_filename_); - - current_output_filename_ = ""; - } -} \ No newline at end of file diff --git a/sf-writer/CMakeLists.txt b/sf-writer/CMakeLists.txt deleted file mode 100644 index 5b520f5..0000000 --- a/sf-writer/CMakeLists.txt +++ /dev/null @@ -1,22 +0,0 @@ -file(GLOB SOURCES - src/*.cpp) - -add_library(sf-writer-lib STATIC ${SOURCES}) -target_include_directories(sf-writer-lib PUBLIC include/) -target_link_libraries(sf-writer-lib - external - core-buffer-lib) - -add_executable(sf-writer src/main.cpp) -set_target_properties(sf-writer PROPERTIES OUTPUT_NAME sf_writer) -target_link_libraries(sf-writer - sf-writer-lib - zmq - hdf5 - hdf5_hl - hdf5_cpp - pthread - ) - -enable_testing() -add_subdirectory(test/) \ No newline at end of file diff --git a/sf-writer/README.md b/sf-writer/README.md deleted file mode 100644 index 606129e..0000000 --- a/sf-writer/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# sf-writer - -sf-writer reads the binary buffer from disk, assembled the images and writes -them down in HDF5 format. - -## Data request ranges - -Data request ranges are composed of: - -- start_pulse_id (first pulse_id to be included in the file) -- stop_pulse_id (last pulse_id to be included in the file) -- pulse_id_step (how many pulses to skip between images.) - -pulse_id_step can be used to write data at different frequencies: - -- pulse_id_step == 1 (100Hz, write very pulse_id) -- pulse_id_step == 2 (50hz, write every second pulse) -- pulse_id_step == 10 (10Hz, write every 10th pulse) - -The next pulse_id to be written is calculated internally as: - -```c++ -auto next_pulse_id = currnet_pulse_id + pulse_id_step; -``` - -The loop criteria for writing is: - -```c++ -for ( - auto curr_pulse_id = start_pulse_id; - curr_pulse_id <= stop_pulse_id; - curr_pulse_id += pulse_id_step -) { - // Write curr_pulse_id to output file. -} -``` - -**Warning** - -If your stop_pulse_id cannot be reached by adding step_pulse_id to -start_pulse_id (start_pulse_id + (n * pulse_id_step) != stop_pulse_id for any n) -it will not be included in the final file. \ No newline at end of file diff --git a/sf-writer/include/BufferBinaryReader.hpp b/sf-writer/include/BufferBinaryReader.hpp deleted file mode 100644 index 0350cba..0000000 --- a/sf-writer/include/BufferBinaryReader.hpp +++ /dev/null @@ -1,28 +0,0 @@ -#ifndef SF_DAQ_BUFFER_BUFFERBINARYREADER_HPP -#define SF_DAQ_BUFFER_BUFFERBINARYREADER_HPP - - -#include - -class BufferBinaryReader { - - const std::string detector_folder_; - const std::string module_name_; - - std::string current_input_file_; - int input_file_fd_; - - void open_file(const std::string& filename); - void close_current_file(); - -public: - BufferBinaryReader(const std::string &detector_folder, - const std::string &module_name); - - ~BufferBinaryReader(); - - void get_block(const uint64_t block_id, BufferBinaryBlock *buffer); -}; - - -#endif //SF_DAQ_BUFFER_BUFFERBINARYREADER_HPP diff --git a/sf-writer/include/ImageAssembler.hpp b/sf-writer/include/ImageAssembler.hpp deleted file mode 100644 index ce98962..0000000 --- a/sf-writer/include/ImageAssembler.hpp +++ /dev/null @@ -1,53 +0,0 @@ -#ifndef SF_DAQ_BUFFER_IMAGEASSEMBLER_HPP -#define SF_DAQ_BUFFER_IMAGEASSEMBLER_HPP - -#include - -#include "buffer_config.hpp" -#include "formats.hpp" - -const uint64_t IA_EMPTY_SLOT_VALUE = 0; - -struct ImageMetadataBlock -{ - uint64_t pulse_id[buffer_config::BUFFER_BLOCK_SIZE]; - uint64_t frame_index[buffer_config::BUFFER_BLOCK_SIZE]; - uint32_t daq_rec[buffer_config::BUFFER_BLOCK_SIZE]; - uint8_t is_good_image[buffer_config::BUFFER_BLOCK_SIZE]; - uint64_t block_start_pulse_id; - uint64_t block_stop_pulse_id; -}; - -class ImageAssembler { - const size_t n_modules_; - const size_t image_buffer_slot_n_bytes_; - - char* image_buffer_; - ImageMetadataBlock* meta_buffer_; - ModuleFrame* frame_meta_buffer_; - std::atomic_int* buffer_status_; - std::atomic_uint64_t* buffer_bunch_id_; - - size_t get_data_offset(const uint64_t slot_id, const int i_module); - size_t get_metadata_offset(const uint64_t slot_id, const int i_module); - -public: - ImageAssembler(const size_t n_modules); - - virtual ~ImageAssembler(); - - bool is_slot_free(const uint64_t bunch_id); - bool is_slot_full(const uint64_t bunch_id); - - void process(const uint64_t bunch_id, - const int i_module, - const BufferBinaryBlock* block_buffer); - - void free_slot(const uint64_t bunch_id); - - ImageMetadataBlock* get_metadata_buffer(const uint64_t bunch_id); - char* get_data_buffer(const uint64_t bunch_id); -}; - - -#endif //SF_DAQ_BUFFER_IMAGEASSEMBLER_HPP diff --git a/sf-writer/include/writer_config.hpp b/sf-writer/include/writer_config.hpp deleted file mode 100644 index 6a60d77..0000000 --- a/sf-writer/include/writer_config.hpp +++ /dev/null @@ -1,9 +0,0 @@ -#include - -namespace writer_config -{ - // MS to retry reading from the image assembler. - const size_t ASSEMBLER_RETRY_MS = 5; - // Number of slots in the reconstruction buffer. - const size_t WRITER_IA_N_SLOTS = 2; -} \ No newline at end of file diff --git a/sf-writer/src/BufferBinaryReader.cpp b/sf-writer/src/BufferBinaryReader.cpp deleted file mode 100644 index e76c5c5..0000000 --- a/sf-writer/src/BufferBinaryReader.cpp +++ /dev/null @@ -1,107 +0,0 @@ -#include "BufferBinaryReader.hpp" - -#include -#include -#include -#include -#include - -#include "BufferUtils.hpp" -#include "writer_config.hpp" -#include "buffer_config.hpp" - -using namespace std; -using namespace writer_config; -using namespace buffer_config; - -BufferBinaryReader::BufferBinaryReader( - const std::string &detector_folder, - const std::string &module_name) : - detector_folder_(detector_folder), - module_name_(module_name), - current_input_file_(""), - input_file_fd_(-1) -{} - -BufferBinaryReader::~BufferBinaryReader() -{ - close_current_file(); -} - -void BufferBinaryReader::get_block( - const uint64_t block_id, BufferBinaryBlock* buffer) -{ - uint64_t block_start_pulse_id = block_id * BUFFER_BLOCK_SIZE; - auto current_block_file = BufferUtils::get_filename( - detector_folder_, module_name_, block_start_pulse_id); - - if (current_block_file != current_input_file_) { - open_file(current_block_file); - } - - size_t file_start_index = - BufferUtils::get_file_frame_index(block_start_pulse_id); - size_t n_bytes_offset = file_start_index * sizeof(BufferBinaryFormat); - - auto lseek_result = lseek(input_file_fd_, n_bytes_offset, SEEK_SET); - if (lseek_result < 0) { - stringstream err_msg; - - err_msg << "[BufferBinaryReader::get_block]"; - err_msg << " Error while lseek on file "; - err_msg << current_input_file_ << " for n_bytes_offset "; - err_msg << n_bytes_offset << ": " << strerror(errno) << endl; - - throw runtime_error(err_msg.str()); - } - - auto n_bytes = ::read(input_file_fd_, buffer, - sizeof(BufferBinaryFormat) * BUFFER_BLOCK_SIZE); - - if (n_bytes < sizeof(BufferBinaryFormat)) { - stringstream err_msg; - - err_msg << "[BufferBinaryReader::get_block]"; - err_msg << " Error while reading from file "; - err_msg << current_input_file_ << ": " << strerror(errno) << endl; - - throw runtime_error(err_msg.str()); - } -} - -void BufferBinaryReader::open_file(const std::string& filename) -{ - close_current_file(); - - input_file_fd_ = open(filename.c_str(), O_RDONLY); - - if (input_file_fd_ < 0) { - stringstream err_msg; - - err_msg << "[BufferBinaryReader::open_file]"; - err_msg << " Cannot open file " << filename << ": "; - err_msg << strerror(errno) << endl; - - throw runtime_error(err_msg.str()); - } - - current_input_file_ = filename; -} - -void BufferBinaryReader::close_current_file() -{ - if (input_file_fd_ != -1) { - if (close(input_file_fd_) < 0) { - stringstream err_msg; - - err_msg << "[BinaryWriter::close_current_file]"; - err_msg << " Error while closing file " << current_input_file_; - err_msg << ": " << strerror(errno) << endl; - - throw runtime_error(err_msg.str()); - } - - input_file_fd_ = -1; - current_input_file_ = ""; - } -} diff --git a/sf-writer/src/ImageAssembler.cpp b/sf-writer/src/ImageAssembler.cpp deleted file mode 100644 index 1a9a5e6..0000000 --- a/sf-writer/src/ImageAssembler.cpp +++ /dev/null @@ -1,186 +0,0 @@ -#include - -#include "ImageAssembler.hpp" -#include "writer_config.hpp" -#include "buffer_config.hpp" - -using namespace std; -using namespace writer_config; -using namespace buffer_config; - -ImageAssembler::ImageAssembler(const size_t n_modules) : - n_modules_(n_modules), - image_buffer_slot_n_bytes_(BUFFER_BLOCK_SIZE * MODULE_N_BYTES * n_modules_) -{ - image_buffer_ = new char[WRITER_IA_N_SLOTS * image_buffer_slot_n_bytes_]; - meta_buffer_ = new ImageMetadataBlock[WRITER_IA_N_SLOTS]; - frame_meta_buffer_ = - new ModuleFrame[WRITER_IA_N_SLOTS * n_modules * BUFFER_BLOCK_SIZE]; - buffer_status_ = new atomic_int[WRITER_IA_N_SLOTS]; - buffer_bunch_id_ = new atomic_uint64_t[WRITER_IA_N_SLOTS]; - - for (size_t i=0; i < WRITER_IA_N_SLOTS; i++) { - free_slot(i); - } -} - -ImageAssembler::~ImageAssembler() -{ - delete[] image_buffer_; - delete[] meta_buffer_; -} - -bool ImageAssembler::is_slot_free(const uint64_t bunch_id) -{ - auto slot_id = bunch_id % WRITER_IA_N_SLOTS; - - uint64_t slot_bunch_id = IA_EMPTY_SLOT_VALUE; - if (buffer_bunch_id_[slot_id].compare_exchange_strong( - slot_bunch_id, bunch_id)) { - return true; - } - - auto is_free = buffer_status_[slot_id].load(memory_order_relaxed) > 0; - return is_free && (slot_bunch_id == bunch_id); -} - -bool ImageAssembler::is_slot_full(const uint64_t bunch_id) -{ - auto slot_id = bunch_id % WRITER_IA_N_SLOTS; - return buffer_status_[slot_id].load(memory_order_relaxed) == 0; -} - -size_t ImageAssembler::get_data_offset( - const uint64_t slot_id, const int i_module) -{ - size_t slot_i_offset = slot_id * image_buffer_slot_n_bytes_; - size_t module_i_offset = i_module * MODULE_N_BYTES; - - return slot_i_offset + module_i_offset; -} - -size_t ImageAssembler::get_metadata_offset( - const uint64_t slot_id, const int i_module) -{ - size_t n_metadata_in_slot = n_modules_ * BUFFER_BLOCK_SIZE; - size_t slot_m_offset = slot_id * n_metadata_in_slot; - size_t module_m_offset = i_module; - - return slot_m_offset + module_m_offset; -} - -void ImageAssembler::process( - const uint64_t bunch_id, - const int i_module, - const BufferBinaryBlock* block_buffer) -{ - const auto slot_id = bunch_id % WRITER_IA_N_SLOTS; - - auto meta_offset = get_metadata_offset(slot_id, i_module); - const auto meta_offset_step = n_modules_; - - auto image_offset = get_data_offset(slot_id, i_module); - const auto image_offset_step = MODULE_N_BYTES * n_modules_; - - for (const auto& frame : block_buffer->frame) { - - memcpy( - &(frame_meta_buffer_[meta_offset]), - &(frame.meta), - sizeof(ModuleFrame)); - - meta_offset += meta_offset_step; - - memcpy( - image_buffer_ + image_offset, - &(frame.data[0]), - MODULE_N_BYTES); - - image_offset += image_offset_step; - } - - buffer_status_[slot_id].fetch_sub(1, memory_order_relaxed); -} - -void ImageAssembler::free_slot(const uint64_t bunch_id) -{ - auto slot_id = bunch_id % WRITER_IA_N_SLOTS; - buffer_status_[slot_id].store(n_modules_, memory_order_relaxed); - buffer_bunch_id_[slot_id].store(IA_EMPTY_SLOT_VALUE, memory_order_relaxed); -} - -ImageMetadataBlock* ImageAssembler::get_metadata_buffer(const uint64_t bunch_id) -{ - const auto slot_id = bunch_id % WRITER_IA_N_SLOTS; - - auto& image_pulse_id = meta_buffer_[slot_id].pulse_id; - auto& image_frame_index = meta_buffer_[slot_id].frame_index; - auto& image_daq_rec = meta_buffer_[slot_id].daq_rec; - auto& image_is_good_frame = meta_buffer_[slot_id].is_good_image; - - auto meta_offset = get_metadata_offset(slot_id, 0); - const auto meta_offset_step = 1; - - uint64_t start_pulse_id = bunch_id * BUFFER_BLOCK_SIZE; - meta_buffer_[slot_id].block_start_pulse_id = start_pulse_id; - - uint64_t stop_pulse_id = start_pulse_id + BUFFER_BLOCK_SIZE - 1; - meta_buffer_[slot_id].block_stop_pulse_id = stop_pulse_id; - - for (size_t i_pulse=0; i_pulse < BUFFER_BLOCK_SIZE; i_pulse++) { - - auto is_pulse_init = false; - image_is_good_frame[i_pulse] = 1; - image_pulse_id[i_pulse] = 0; - - for (size_t i_module=0; i_module < n_modules_; i_module++) { - - auto& frame_meta = frame_meta_buffer_[meta_offset]; - auto is_good_frame = - frame_meta.n_recv_packets == JF_N_PACKETS_PER_FRAME; - - if (!is_good_frame) { - image_is_good_frame[i_pulse] = 0; - // TODO: Update meta_offset only once in the loop. - meta_offset += meta_offset_step; - continue; - } - - if (!is_pulse_init) { - image_pulse_id[i_pulse] = frame_meta.pulse_id; - image_frame_index[i_pulse] = frame_meta.frame_index; - image_daq_rec[i_pulse] = frame_meta.daq_rec; - - is_pulse_init = true; - } - - if (image_is_good_frame[i_pulse] == 1) { - if (frame_meta.pulse_id != image_pulse_id[i_pulse]) { - image_is_good_frame[i_pulse] = 0; - } - - if (frame_meta.frame_index != image_frame_index[i_pulse]) { - image_is_good_frame[i_pulse] = 0; - } - - if (frame_meta.daq_rec != image_daq_rec[i_pulse]) { - image_is_good_frame[i_pulse] = 0; - } - - if (frame_meta.n_recv_packets != JF_N_PACKETS_PER_FRAME) { - image_is_good_frame[i_pulse] = 0; - } - } - - meta_offset += meta_offset_step; - } - } - - return &(meta_buffer_[slot_id]); -} - -char* ImageAssembler::get_data_buffer(const uint64_t bunch_id) -{ - auto slot_id = bunch_id % WRITER_IA_N_SLOTS; - return image_buffer_ + (slot_id * image_buffer_slot_n_bytes_); -} diff --git a/sf-writer/src/main.cpp b/sf-writer/src/main.cpp deleted file mode 100644 index 643d47f..0000000 --- a/sf-writer/src/main.cpp +++ /dev/null @@ -1,158 +0,0 @@ -#include -#include -#include -#include -#include - -#include "date.h" -#include "zmq.h" -#include "writer_config.hpp" -#include "buffer_config.hpp" -#include "bitshuffle/bitshuffle.h" -#include "JFH5Writer.hpp" -#include "ImageAssembler.hpp" -#include "BufferBinaryReader.hpp" - -using namespace std; -using namespace chrono; -using namespace writer_config; -using namespace buffer_config; - -void read_buffer( - const string detector_folder, - const string module_name, - const int i_module, - const vector& buffer_blocks, - ImageAssembler& image_assembler) -{ - BufferBinaryReader block_reader(detector_folder, module_name); - auto block_buffer = new BufferBinaryBlock(); - - for (uint64_t block_id:buffer_blocks) { - - while(!image_assembler.is_slot_free(block_id)) { - this_thread::sleep_for(chrono::milliseconds(ASSEMBLER_RETRY_MS)); - } - - auto start_time = steady_clock::now(); - - block_reader.get_block(block_id, block_buffer); - - auto end_time = steady_clock::now(); - uint64_t read_us_duration = duration_cast( - end_time-start_time).count(); - - start_time = steady_clock::now(); - - image_assembler.process(block_id, i_module, block_buffer); - - end_time = steady_clock::now(); - uint64_t compose_us_duration = duration_cast( - end_time-start_time).count(); - - cout << "sf_writer:avg_read_us "; - cout << read_us_duration / BUFFER_BLOCK_SIZE << endl; - cout << "sf_writer:avg_assemble_us "; - cout << compose_us_duration / BUFFER_BLOCK_SIZE << endl; - } - - delete block_buffer; -} - -int main (int argc, char *argv[]) -{ - if (argc != 7) { - cout << endl; - cout << "Usage: sf_writer [output_file] [detector_folder] [n_modules]"; - cout << " [start_pulse_id] [stop_pulse_id] [pulse_id_step]"; - cout << endl; - cout << "\toutput_file: Complete path to the output file." << endl; - cout << "\tdetector_folder: Absolute path to detector buffer." << endl; - cout << "\tn_modules: number of modules" << endl; - cout << "\tstart_pulse_id: Start pulse_id of retrieval." << endl; - cout << "\tstop_pulse_id: Stop pulse_id of retrieval." << endl; - cout << "\tpulse_id_step: 1==100Hz, 2==50hz, 4==25Hz.." << endl; - cout << endl; - - exit(-1); - } - - string output_file = string(argv[1]); - const string detector_folder = string(argv[2]); - size_t n_modules = atoi(argv[3]); - uint64_t start_pulse_id = (uint64_t) atoll(argv[4]); - uint64_t stop_pulse_id = (uint64_t) atoll(argv[5]); - int pulse_id_step = atoi(argv[6]); - - // Align start (up) and stop(down) pulse_id with pulse_id_step. - if (start_pulse_id % pulse_id_step != 0) { - start_pulse_id += pulse_id_step - (start_pulse_id % pulse_id_step); - } - if (stop_pulse_id % pulse_id_step != 0) { - stop_pulse_id -= (start_pulse_id % pulse_id_step); - } - - uint64_t start_block = start_pulse_id / BUFFER_BLOCK_SIZE; - uint64_t stop_block = stop_pulse_id / BUFFER_BLOCK_SIZE; - - // Generate list of buffer blocks that need to be loaded. - std::vector buffer_blocks; - for (uint64_t i_block=start_block; i_block <= stop_block; i_block++) { - buffer_blocks.push_back(i_block); - } - - ImageAssembler image_assembler(n_modules); - - std::vector reading_threads(n_modules); - for (size_t i_module=0; i_module( - end_time-start_time).count(); - - image_assembler.free_slot(block_id); - - cout << "sf_writer:avg_write_us "; - cout << write_us_duration / BUFFER_BLOCK_SIZE << endl; - } - - for (auto& reading_thread : reading_threads) { - if (reading_thread.joinable()) { - reading_thread.join(); - } - } - - return 0; -} diff --git a/sf-writer/test/CMakeLists.txt b/sf-writer/test/CMakeLists.txt deleted file mode 100644 index 2b9f4f0..0000000 --- a/sf-writer/test/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -add_executable(sf-writer-tests main.cpp) - -target_link_libraries(sf-writer-tests - sf-writer-lib - hdf5 - hdf5_hl - hdf5_cpp - zmq - gtest - ) diff --git a/sf-writer/test/main.cpp b/sf-writer/test/main.cpp deleted file mode 100644 index 65c6feb..0000000 --- a/sf-writer/test/main.cpp +++ /dev/null @@ -1,10 +0,0 @@ -#include "gtest/gtest.h" -#include "test_JFH5Writer.cpp" -#include "test_ImageAssembler.cpp" - -using namespace std; - -int main(int argc, char **argv) { - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/sf-writer/test/mock/data.hpp b/sf-writer/test/mock/data.hpp deleted file mode 100644 index 0588de5..0000000 --- a/sf-writer/test/mock/data.hpp +++ /dev/null @@ -1,72 +0,0 @@ -#ifndef SF_DAQ_BUFFER_DATA_HPP -#define SF_DAQ_BUFFER_DATA_HPP - -#include -#include - -#include "buffer_config.hpp" - -auto get_test_block_metadata( - const uint64_t start_pulse_id, - const uint64_t stop_pulse_id, - const int pulse_id_step) -{ - using namespace std; - using namespace buffer_config; - - auto metadata = make_shared(); - - uint64_t block_id = start_pulse_id / BUFFER_BLOCK_SIZE; - - metadata->block_start_pulse_id = block_id * BUFFER_BLOCK_SIZE; - metadata->block_stop_pulse_id = - metadata->block_start_pulse_id + BUFFER_BLOCK_SIZE - 1; - - if (metadata->block_stop_pulse_id < stop_pulse_id) { - throw runtime_error("stop_pulse_id in next block"); - } - - auto offset = start_pulse_id - metadata->block_start_pulse_id; - - for (uint64_t pulse_id = start_pulse_id; - pulse_id <= stop_pulse_id; - pulse_id++, offset++) { - - if (pulse_id % pulse_id_step != 0) { - metadata->is_good_image[offset] = 0; - continue; - } - - metadata->pulse_id[offset] = pulse_id; - metadata->frame_index[offset] = pulse_id + 10; - metadata->daq_rec[offset] = pulse_id + 100; - metadata->is_good_image[offset] = 1; - } - - return metadata; -} - -auto get_test_block_data(const size_t n_modules) -{ - using namespace std; - using namespace buffer_config; - - auto image_buffer = make_unique( - MODULE_N_PIXELS * n_modules * BUFFER_BLOCK_SIZE); - - for (int i_block=0; i_block<=BUFFER_BLOCK_SIZE; i_block++) { - for (int i_module=0; i_module -#include "buffer_config.hpp" -#include "zmq.h" -#include -#include -#include -#include -#include "WriterH5Writer.hpp" - -using namespace std; -using namespace core_buffer; - - -int main (int argc, char *argv[]) -{ - if (argc != 4) { - cout << endl; - cout << "Usage: sf_writer "; - cout << " [output_file] [start_pulse_id] [stop_pulse_id]"; - cout << endl; - cout << "\toutput_file: Complete path to the output file." << endl; - cout << "\tstart_pulse_id: Start pulse_id of retrieval." << endl; - cout << "\tstop_pulse_id: Stop pulse_id of retrieval." << endl; - cout << endl; - - exit(-1); - } - - string output_file = string(argv[1]); - uint64_t start_pulse_id = (uint64_t) atoll(argv[2]); - uint64_t stop_pulse_id = (uint64_t) atoll(argv[3]); - - size_t n_modules = 32; - - size_t n_frames = stop_pulse_id - start_pulse_id; - WriterH5Writer writer(output_file, n_frames, n_modules); - - // TODO: Remove stats trash. - int i_write = 0; - size_t total_ms = 0; - size_t max_ms = 0; - size_t min_ms = 10000; // 10 seconds should be a safe first value. - - auto start_time = chrono::steady_clock::now(); - - auto metadata = make_shared(); - auto data = make_unique(MODULE_N_BYTES*n_modules); - - auto current_pulse_id = start_pulse_id; - while (current_pulse_id <= stop_pulse_id) { - - writer.write(metadata.get(), data.get()); - current_pulse_id++; - - i_write++; - - auto end_time = chrono::steady_clock::now(); - - // TODO: Some poor statistics. - - auto ms_duration = chrono::duration_cast( - end_time-start_time).count(); - total_ms += ms_duration; - - if (ms_duration > max_ms) { - max_ms = ms_duration; - } - - if (ms_duration < min_ms) { - min_ms = ms_duration; - } - - if (i_write==100) { - cout << "avg_write_ms " << total_ms / 100; - cout << " min_write_ms " << min_ms; - cout << " max_write_ms " << max_ms << endl; - - i_write = 0; - total_ms = 0; - max_ms = 0; - min_ms = 0; - } - - start_time = chrono::steady_clock::now(); - } - - writer.close_file(); - - return 0; -} diff --git a/sf-writer/test/test_ImageAssembler.cpp b/sf-writer/test/test_ImageAssembler.cpp deleted file mode 100644 index 97bdbb3..0000000 --- a/sf-writer/test/test_ImageAssembler.cpp +++ /dev/null @@ -1,90 +0,0 @@ -#include - -#include "ImageAssembler.hpp" -#include "gtest/gtest.h" - -using namespace std; -using namespace buffer_config; - -TEST(ImageAssembler, basic_interaction) -{ - size_t n_modules = 3; - uint64_t bunch_id = 0; - - ImageAssembler assembler(n_modules); - - ASSERT_EQ(assembler.is_slot_free(bunch_id), true); - - auto buffer_block = make_unique(); - auto buffer_ptr = buffer_block.get(); - - for (size_t i_module=0; i_module < n_modules; i_module++) { - assembler.process(bunch_id, i_module, buffer_ptr); - } - - ASSERT_EQ(assembler.is_slot_full(bunch_id), true); - - auto metadata = assembler.get_metadata_buffer(bunch_id); - auto data = assembler.get_data_buffer(bunch_id); - - assembler.free_slot(bunch_id); - ASSERT_EQ(assembler.is_slot_free(bunch_id), true); - - for (size_t i_pulse = 0; i_pulse < BUFFER_BLOCK_SIZE; i_pulse++) { - ASSERT_EQ(metadata->is_good_image[i_pulse], 0); - } -} - -TEST(ImageAssembler, reconstruction) -{ - size_t n_modules = 2; - uint64_t bunch_id = 0; - - ImageAssembler assembler(n_modules); - - ASSERT_EQ(assembler.is_slot_free(bunch_id), true); - - auto buffer_block = make_unique(); - auto buffer_ptr = buffer_block.get(); - - for (size_t i_module=0; i_module < n_modules; i_module++) { - - for (size_t i_pulse=0; i_pulse < BUFFER_BLOCK_SIZE; i_pulse++) { - auto& frame_meta = buffer_block->frame[i_pulse].meta; - - frame_meta.pulse_id = 100 + i_pulse; - frame_meta.daq_rec = 1000 + i_pulse; - frame_meta.frame_index = 10000 + i_pulse; - frame_meta.n_recv_packets = JF_N_PACKETS_PER_FRAME; - - for (size_t i_pixel=0; i_pixel < MODULE_N_PIXELS; i_pixel++) { - buffer_block->frame[i_pulse].data[i_pixel] = - (i_module * 10) + (i_pixel % 100); - } - } - - assembler.process(bunch_id, i_module, buffer_ptr); - } - - ASSERT_EQ(assembler.is_slot_full(bunch_id), true); - - auto metadata = assembler.get_metadata_buffer(bunch_id); - auto data = assembler.get_data_buffer(bunch_id); - - assembler.free_slot(bunch_id); - ASSERT_EQ(assembler.is_slot_free(bunch_id), true); - - ASSERT_EQ(metadata->block_start_pulse_id, 0); - ASSERT_EQ(metadata->block_stop_pulse_id, BUFFER_BLOCK_SIZE-1); - - for (size_t i_pulse = 0; i_pulse < BUFFER_BLOCK_SIZE; i_pulse++) { - ASSERT_EQ(metadata->pulse_id[i_pulse], 100 + i_pulse); - ASSERT_EQ(metadata->daq_rec[i_pulse], 1000 + i_pulse); - ASSERT_EQ(metadata->frame_index[i_pulse], 10000 + i_pulse); - ASSERT_EQ(metadata->is_good_image[i_pulse], 1); - - for (size_t i_module=0; i_module < n_modules; i_module++) { - // TODO: Check assembled image. - } - } -} diff --git a/sf-writer/test/test_JFH5Writer.cpp b/sf-writer/test/test_JFH5Writer.cpp deleted file mode 100644 index 2a2b5a9..0000000 --- a/sf-writer/test/test_JFH5Writer.cpp +++ /dev/null @@ -1,254 +0,0 @@ -#include - -#include "JFH5Writer.hpp" -#include "gtest/gtest.h" -#include "bitshuffle/bitshuffle.h" -#include "mock/data.hpp" - -using namespace std; -using namespace buffer_config; - -TEST(JFH5Writer, basic_interaction) -{ - size_t n_modules = 2; - uint64_t start_pulse_id = 1; - uint64_t stop_pulse_id = 5; - - auto data = make_unique(n_modules*MODULE_N_BYTES*BUFFER_BLOCK_SIZE); - auto metadata = make_shared(); - - // Needed by writer. - metadata->block_start_pulse_id = 0; - metadata->block_stop_pulse_id = BUFFER_BLOCK_SIZE - 1; - - JFH5Writer writer("ignore.h5", "detector", - n_modules, start_pulse_id, stop_pulse_id, 1); - writer.write(metadata.get(), data.get()); -} - -TEST(JFH5Writer, test_writing) -{ - size_t n_modules = 2; - uint64_t start_pulse_id = 5; - uint64_t stop_pulse_id = 10; - auto n_images = stop_pulse_id - start_pulse_id + 1; - - auto meta = get_test_block_metadata(start_pulse_id, stop_pulse_id, 1); - auto data = get_test_block_data(n_modules); - - string detector_name = "detector"; - - // The writer closes the file on destruction. - { - JFH5Writer writer( - "ignore.h5", detector_name, - n_modules, start_pulse_id, stop_pulse_id, 1); - writer.write(meta.get(), (char*)(&data[0])); - } - - H5::H5File reader("ignore.h5", H5F_ACC_RDONLY); - auto image_dataset = reader.openDataSet("/data/detector/data"); - image_dataset.read(&data[0], H5::PredType::NATIVE_UINT16); - - for (int i_image=0; i_image < n_images; i_image++) { - for (int i_module=0; i_module(n_images); - auto pulse_id_dataset = reader.openDataSet("/data/detector/pulse_id"); - pulse_id_dataset.read(&pulse_id_data[0], H5::PredType::NATIVE_UINT64); - - auto frame_index_data = make_unique(n_images); - auto frame_index_dataset = reader.openDataSet("/data/detector/frame_index"); - frame_index_dataset.read(&frame_index_data[0], H5::PredType::NATIVE_UINT64); - - auto daq_rec_data = make_unique(n_images); - auto daq_rec_dataset = reader.openDataSet("/data/detector/daq_rec"); - daq_rec_dataset.read(&daq_rec_data[0], H5::PredType::NATIVE_UINT32); - - auto is_good_frame_data = make_unique(n_images); - auto is_good_frame_dataset = - reader.openDataSet("/data/detector/is_good_frame"); - is_good_frame_dataset.read( - &is_good_frame_data[0], H5::PredType::NATIVE_UINT8); - - auto name_dataset = reader.openDataSet("/general/detector_name"); - string read_detector_name; - name_dataset.read(read_detector_name, name_dataset.getDataType()); - - ASSERT_EQ(detector_name, read_detector_name); - - for (uint64_t pulse_id=start_pulse_id; - pulse_id<=stop_pulse_id; - pulse_id++) { - - ASSERT_EQ(pulse_id_data[pulse_id - start_pulse_id], pulse_id); - ASSERT_EQ(frame_index_data[pulse_id - start_pulse_id], pulse_id + 10); - ASSERT_EQ(daq_rec_data[pulse_id - start_pulse_id], pulse_id + 100); - ASSERT_EQ(is_good_frame_data[pulse_id - start_pulse_id], 1); - } -} - -TEST(JFH5Writer, test_step_pulse_id) -{ - // Start pulse id (5) larger than stop pulse id (4). - ASSERT_THROW(JFH5Writer writer("ignore.h5", "d", 1 , 5, 4, 1), - runtime_error); - - // Start pulse id (5) is equal to stop pulse id (5). - ASSERT_NO_THROW(JFH5Writer writer("ignore.h5", "d", 1, 5, 5, 1)); - - // The step is exactly on start nad stop pulse id. - ASSERT_NO_THROW(JFH5Writer writer("ignore.h5", "d", 1, 5, 5, 5)); - - // No pulses in given range with step = 10 - ASSERT_THROW(JFH5Writer writer("ignore.h5", "d", 1, 1, 9, 10), - runtime_error); - - // Stop pulse id is divisible by step, but start is not. - ASSERT_THROW(JFH5Writer writer("ignore.h5", "d", 1, 5, 10, 10), - runtime_error); - - // Start pulse id is divisible by step, but stop is not. - ASSERT_THROW(JFH5Writer writer("ignore.h5", "d", 1, 10, 19, 10), - runtime_error); - - // Should be ok. - ASSERT_NO_THROW(JFH5Writer("ignore.h5", "d", 1, 1234, 1234, 1)); - // Should be ok. - ASSERT_NO_THROW(JFH5Writer("ignore.h5", "d", 1, 1234, 4567, 1)); - // Should be ok. - ASSERT_NO_THROW(JFH5Writer("ignore.h5", "d", 1, 4, 4, 4)); - - // stop smaller than start. - ASSERT_THROW(JFH5Writer("ignore.h5", "d", 1, 1234, 1233, 1), - runtime_error); - // step is not valid for 100Hz. - ASSERT_THROW(JFH5Writer("ignore.h5", "d", 1, 1234, 1234, 3), - runtime_error); - // start not divisible by step. - ASSERT_THROW(JFH5Writer("ignore.h5", "d", 1, 10, 10, 4), - runtime_error); - // stop not divisible by step - ASSERT_THROW(JFH5Writer("ignore.h5", "d", 1, 8, 10, 4), - runtime_error); -} - -void test_writing_with_step( - uint64_t start_pulse_id, uint64_t stop_pulse_id, size_t step) -{ - size_t n_modules = 3; - - size_t n_images = 1; - n_images += (stop_pulse_id / step); - n_images -= start_pulse_id / step; - - auto meta = get_test_block_metadata(start_pulse_id, stop_pulse_id, step); - auto data = get_test_block_data(n_modules); - - // Verify the metadata has the layout we want to test (50Hz). - for (size_t i_pulse=0; i_pulsepulse_id[i_pulse], 500 + i_pulse); - } else { - ASSERT_EQ(meta->pulse_id[i_pulse], 0); - } - } - - string path_root = "/path/to/"; - string expected_detector_name = "detector"; - - // The writer closes the file on destruction. - { - JFH5Writer writer( - "ignore.h5", path_root + expected_detector_name, - n_modules, start_pulse_id, stop_pulse_id, step); - writer.write(meta.get(), (char*)(&data[0])); - } - - H5::H5File reader("ignore.h5", H5F_ACC_RDONLY); - auto image_dataset = reader.openDataSet("/data/detector/data"); - image_dataset.read(&data[0], H5::PredType::NATIVE_UINT16); - - hsize_t dims[3]; - image_dataset.getSpace().getSimpleExtentDims(dims); - ASSERT_EQ(dims[0], n_images); - ASSERT_EQ(dims[1], n_modules * MODULE_Y_SIZE); - ASSERT_EQ(dims[2], MODULE_X_SIZE); - - auto pulse_id_data = make_unique(n_images); - auto pulse_id_dataset = reader.openDataSet("/data/detector/pulse_id"); - pulse_id_dataset.read(&pulse_id_data[0], H5::PredType::NATIVE_UINT64); - pulse_id_dataset.getSpace().getSimpleExtentDims(dims); - ASSERT_EQ(dims[0], n_images); - ASSERT_EQ(dims[1], 1); - - auto frame_index_data = make_unique(n_images); - auto frame_index_dataset = reader.openDataSet("/data/detector/frame_index"); - frame_index_dataset.read(&frame_index_data[0], H5::PredType::NATIVE_UINT64); - frame_index_dataset.getSpace().getSimpleExtentDims(dims); - ASSERT_EQ(dims[0], n_images); - ASSERT_EQ(dims[1], 1); - - auto daq_rec_data = make_unique(n_images); - auto daq_rec_dataset = reader.openDataSet("/data/detector/daq_rec"); - daq_rec_dataset.read(&daq_rec_data[0], H5::PredType::NATIVE_UINT32); - daq_rec_dataset.getSpace().getSimpleExtentDims(dims); - ASSERT_EQ(dims[0], n_images); - ASSERT_EQ(dims[1], 1); - - auto is_good_frame_data = make_unique(n_images); - auto is_good_frame_dataset = - reader.openDataSet("/data/detector/is_good_frame"); - is_good_frame_dataset.read( - &is_good_frame_data[0], H5::PredType::NATIVE_UINT8); - is_good_frame_dataset.getSpace().getSimpleExtentDims(dims); - ASSERT_EQ(dims[0], n_images); - ASSERT_EQ(dims[1], 1); - - auto name_dataset = reader.openDataSet("/general/detector_name"); - string read_detector_name; - name_dataset.read(read_detector_name, name_dataset.getDataType()); - - ASSERT_EQ(expected_detector_name, read_detector_name); - - uint64_t i_pulse = 0; - for (uint64_t pulse_id=start_pulse_id; - pulse_id<=stop_pulse_id; - pulse_id++) { - - if (pulse_id % step != 0) { - continue; - } - - ASSERT_EQ(pulse_id_data[i_pulse], pulse_id); - ASSERT_EQ(frame_index_data[i_pulse], pulse_id + 10); - ASSERT_EQ(daq_rec_data[i_pulse], pulse_id + 100); - ASSERT_EQ(is_good_frame_data[i_pulse], 1); - - i_pulse++; - } -} - -TEST(JFH5Writer, test_writing_with_step) -{ - // TODO: Write with any number of steps. - // 100Hz - test_writing_with_step(500, 599, 1); - // 50Hz - test_writing_with_step(500, 598, 2); - // 25Hz - test_writing_with_step(500, 596, 4); - // 10Hz - test_writing_with_step(500, 590, 10); - // 1Hz - test_writing_with_step(500, 500, 100); -} From 11ba1c399c58eddcd9b490ab447c4487a6f050e1 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 17 Feb 2021 12:47:23 +0100 Subject: [PATCH 111/126] Moved H5 writer to new project --- {sf-writer => jf-live-writer}/include/JFH5Writer.hpp | 0 {sf-writer => jf-live-writer}/src/JFH5Writer.cpp | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {sf-writer => jf-live-writer}/include/JFH5Writer.hpp (100%) rename {sf-writer => jf-live-writer}/src/JFH5Writer.cpp (100%) diff --git a/sf-writer/include/JFH5Writer.hpp b/jf-live-writer/include/JFH5Writer.hpp similarity index 100% rename from sf-writer/include/JFH5Writer.hpp rename to jf-live-writer/include/JFH5Writer.hpp diff --git a/sf-writer/src/JFH5Writer.cpp b/jf-live-writer/src/JFH5Writer.cpp similarity index 100% rename from sf-writer/src/JFH5Writer.cpp rename to jf-live-writer/src/JFH5Writer.cpp From cdfff368248ffb1e64cd1dce6faeacb3673f00f7 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Mon, 22 Feb 2021 10:45:23 +0100 Subject: [PATCH 112/126] Broker agent stream format --- jf-live-writer/include/broker_format.hpp | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 jf-live-writer/include/broker_format.hpp diff --git a/jf-live-writer/include/broker_format.hpp b/jf-live-writer/include/broker_format.hpp new file mode 100644 index 0000000..9943d9c --- /dev/null +++ b/jf-live-writer/include/broker_format.hpp @@ -0,0 +1,21 @@ +#ifndef SF_DAQ_BUFFER_BROKER_FORMAT_HPP +#define SF_DAQ_BUFFER_BROKER_FORMAT_HPP + +#include "formats.hpp" + +const static uint8_t OP_CONTINUE = 0; +const static uint8_t OP_START = 1; +const static uint8_t OP_END = 2; + +#pragma pack(push) +#pragma pack(1) +struct StoreStream { + uint8_t op_code; + uint32_t i_image; + uint32_t n_images; + int64_t run_id; + + ImageMetadata image_metadata; +}; +#pragma pack(pop) +#endif //SF_DAQ_BUFFER_BROKER_FORMAT_HPP From 29d3e4c1cb09244c8b6a4b0a41808839eba0bada Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 24 Feb 2021 08:56:29 +0100 Subject: [PATCH 113/126] Rewrite the writer main function --- jf-live-writer/src/main.cpp | 50 +++++++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/jf-live-writer/src/main.cpp b/jf-live-writer/src/main.cpp index aa47b8c..417c919 100644 --- a/jf-live-writer/src/main.cpp +++ b/jf-live-writer/src/main.cpp @@ -3,9 +3,11 @@ #include #include #include -#include #include "live_writer_config.hpp" #include "WriterStats.hpp" +#include "broker_format.hpp" +#include +#include using namespace std; @@ -17,38 +19,64 @@ int main (int argc, char *argv[]) if (argc != 3) { cout << endl; cout << "Usage: jf_live_writer [detector_json_filename]" - " [writer_id]" << endl; + " [bits_per_pixel]" << endl; cout << "\tdetector_json_filename: detector config file path." << endl; - cout << "\twriter_id: Index of this writer instance." << endl; + cout << "\tbits_per_pixel: Number of bits in each pixel." << endl; cout << endl; exit(-1); } - auto config = BufferUtils::read_json_config(string(argv[1])); - const int writer_id = atoi(argv[2]); + auto const config = BufferUtils::read_json_config(string(argv[1])); + auto const bits_per_pixel = atoi(argv[2]); + + MPI_Init(NULL, NULL); + + int n_writers; + MPI_Comm_size(MPI_COMM_WORLD, &n_writers); + + int i_writer; + MPI_Comm_size(MPI_COMM_WORLD, &i_writer); auto ctx = zmq_ctx_new(); zmq_ctx_set(ctx, ZMQ_IO_THREADS, LIVE_ZMQ_IO_THREADS); auto receiver = BufferUtils::connect_socket( - ctx, config.detector_name, "assembler"); + ctx, config.detector_name, "broker-agent"); RamBuffer ram_buffer(config.detector_name, config.n_modules); - const uint64_t image_n_bytes = config.n_modules * MODULE_N_BYTES; - ImageBinaryWriter writer(config.detector_name, image_n_bytes); + const uint64_t image_n_bytes = + config.image_y_size * config.image_x_size * bits_per_pixel; + JFH5Writer writer(config); WriterStats stats(config.detector_name, STATS_MODULO, image_n_bytes); - ImageMetadata meta = {}; + StoreStream meta = {}; while (true) { zmq_recv(receiver, &meta, sizeof(meta), 0); - char* data = ram_buffer.read_image(meta.pulse_id); + + if (meta.op_code == OP_START) { + writer.open_run(meta.run_id, meta.n_images); + continue; + } + + if (meta.op_code == OP_END) { + writer.close_run(meta.run_id); + continue; + } + + if (meta.i_image % n_writers != i_writer) { + continue; + } + + char* data = ram_buffer.read_image(meta.image_metadata.pulse_id); stats.start_image_write(); - writer.write(meta, data); + writer.write(meta.run_id, meta.image_metadata, data); stats.end_image_write(); } + + MPI_Finalize(); } From 3c9a9d8bce70bf906a73809a84f3ffbcf7095036 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 24 Feb 2021 08:56:54 +0100 Subject: [PATCH 114/126] Add output symlink name to writer config --- jf-live-writer/include/live_writer_config.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jf-live-writer/include/live_writer_config.hpp b/jf-live-writer/include/live_writer_config.hpp index 76d9b05..d47c531 100644 --- a/jf-live-writer/include/live_writer_config.hpp +++ b/jf-live-writer/include/live_writer_config.hpp @@ -4,4 +4,6 @@ namespace live_writer_config { // N of IO threads to receive data from modules. const int LIVE_ZMQ_IO_THREADS = 1; + + const std::string OUTPUT_FOLDER_SYMLINK = "OUTPUT/" } \ No newline at end of file From 1ac041f51ce5787f033a6a6292447ec8a84b7be4 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 24 Feb 2021 08:57:15 +0100 Subject: [PATCH 115/126] Make the live writer an MPI build --- jf-live-writer/CMakeLists.txt | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/jf-live-writer/CMakeLists.txt b/jf-live-writer/CMakeLists.txt index f96af5a..b6f1045 100644 --- a/jf-live-writer/CMakeLists.txt +++ b/jf-live-writer/CMakeLists.txt @@ -1,17 +1,29 @@ +find_package(MPI REQUIRED) +# Because of openmpi. +add_definitions(-DOMPI_SKIP_MPICXX) + file(GLOB SOURCES src/*.cpp) add_library(jf-live-writer-lib STATIC ${SOURCES}) -target_include_directories(jf-live-writer-lib PUBLIC include/) +target_include_directories(jf-live-writer-lib + PUBLIC include/ + SYSTEM ${MPI_INCLUDE_PATH}) + target_link_libraries(jf-live-writer-lib external - core-buffer-lib) + core-buffer-lib + ${MPI_LIBRARIES} + ) add_executable(jf-live-writer src/main.cpp) set_target_properties(jf-live-writer PROPERTIES OUTPUT_NAME jf_live_writer) target_link_libraries(jf-live-writer jf-live-writer-lib zmq + hdf5 + hdf5_hl + hdf5_cpp rt ) From b2694fb8495ab424581a9f8bb299894b0e483792 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 24 Feb 2021 08:57:34 +0100 Subject: [PATCH 116/126] Add image size to detector config --- core-buffer/include/BufferUtils.hpp | 2 ++ core-buffer/src/BufferUtils.cpp | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/core-buffer/include/BufferUtils.hpp b/core-buffer/include/BufferUtils.hpp index 1403bd0..a1219ff 100644 --- a/core-buffer/include/BufferUtils.hpp +++ b/core-buffer/include/BufferUtils.hpp @@ -19,6 +19,8 @@ namespace BufferUtils const int n_modules; const int start_udp_port; const std::string buffer_folder; + const int image_y_size; + const int image_x_size; }; diff --git a/core-buffer/src/BufferUtils.cpp b/core-buffer/src/BufferUtils.cpp index c763966..be85b4b 100644 --- a/core-buffer/src/BufferUtils.cpp +++ b/core-buffer/src/BufferUtils.cpp @@ -163,6 +163,8 @@ BufferUtils::DetectorConfig BufferUtils::read_json_config( config_parameters["detector_name"].GetString(), config_parameters["n_modules"].GetInt(), config_parameters["start_udp_port"].GetInt(), - config_parameters["buffer_folder"].GetString() + config_parameters["buffer_folder"].GetString(), + config_parameters["image_y_size"].GetInt(), + config_parameters["image_x_size"].GetInt() }; } From fb9ec376f68923b1b89cad460669b57d778bf3ef Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 24 Feb 2021 09:01:11 +0100 Subject: [PATCH 117/126] Add readme stub to H5 Writer --- jf-live-writer/README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 jf-live-writer/README.md diff --git a/jf-live-writer/README.md b/jf-live-writer/README.md new file mode 100644 index 0000000..c1463f1 --- /dev/null +++ b/jf-live-writer/README.md @@ -0,0 +1,12 @@ +# jf-live-writer + +## Install PHDF5 manually +``` +wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.10/hdf5-1.10.7/src/hdf5-1.10.7.tar.gz +tar -xzf hdf5-1.10.7.tar.gz +cd hdf5-1.10.7 +./configure --enable-parallel +make +make install +``` + From d8706f65daeb9184c14056b6e2b8802e68a2f21f Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 24 Feb 2021 11:04:36 +0100 Subject: [PATCH 118/126] Updated HDF5 version to latest --- jf-live-writer/README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/jf-live-writer/README.md b/jf-live-writer/README.md index c1463f1..ce4349c 100644 --- a/jf-live-writer/README.md +++ b/jf-live-writer/README.md @@ -2,11 +2,12 @@ ## Install PHDF5 manually ``` -wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.10/hdf5-1.10.7/src/hdf5-1.10.7.tar.gz -tar -xzf hdf5-1.10.7.tar.gz +wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.12/hdf5-1.12.0/src/hdf5-1.12.0.tar.gz +tar -xzf hdf5-1.12.0.tar.gz cd hdf5-1.10.7 -./configure --enable-parallel -make +./configure --enable-parallel make install +sudo ln -v -s `pwd`/hdf5/lib/* /usr/lib64/ +sudo ln -v -s `pwd`/hdf5/include/* /usr/include/ ``` From a485736af4a690d57e8d162d0ac740ba993ba71e Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Thu, 25 Feb 2021 10:21:49 +0100 Subject: [PATCH 119/126] Reimplemented writer --- jf-live-writer/include/JFH5Writer.hpp | 58 ++-- jf-live-writer/src/JFH5Writer.cpp | 376 +++++++++++--------------- 2 files changed, 180 insertions(+), 254 deletions(-) diff --git a/jf-live-writer/include/JFH5Writer.hpp b/jf-live-writer/include/JFH5Writer.hpp index afd4a76..fb4a96a 100644 --- a/jf-live-writer/include/JFH5Writer.hpp +++ b/jf-live-writer/include/JFH5Writer.hpp @@ -3,48 +3,46 @@ #include #include -#include +#include +#include -#include "ImageAssembler.hpp" +extern "C" { + #include +} class JFH5Writer { + const std::string root_folder_; const std::string detector_name_; - const size_t n_modules_; - const uint64_t start_pulse_id_; - const uint64_t stop_pulse_id_; - const size_t pulse_id_step_; - const size_t n_images_; - const size_t n_total_pulses_; - size_t meta_write_index_; - size_t data_write_index_; + const uint32_t image_y_size_; + const uint32_t image_x_size_; - H5::H5File file_; - H5::DataSet image_dataset_; + static const int64_t NO_RUN_ID; + int64_t current_run_id_ = NO_RUN_ID; - uint64_t* b_pulse_id_; - uint64_t* b_frame_index_; - uint32_t* b_daq_rec_; - uint8_t* b_is_good_frame_ ; - - size_t get_n_pulses_in_range(const uint64_t start_pulse_id, - const uint64_t stop_pulse_id, - const int pulse_id_step); - - void write_metadata(); - std::string get_device_name(const std::string& device); + hid_t file_id_ = -1; + hid_t image_dataset_id_ = -1; + hid_t pulse_dataset_id_= -1; + hid_t frame_dataset_id_ = -1; + hid_t daq_rec_dataset_id_ = -1; + hid_t is_good_dataset_id_ = -1; + void open_file(const std::string& output_file, const uint32_t n_images); void close_file(); public: - JFH5Writer(const std::string& output_file, - const std::string& device, - const size_t n_modules, - const uint64_t start_pulse_id, - const uint64_t stop_pulse_id, - const size_t pulse_id_step); + JFH5Writer(const BufferUtils::DetectorConfig config); ~JFH5Writer(); - void write(const ImageMetadataBlock* metadata, const char* data); + void open_run(const int64_t run_id, const uint32_t n_images); + void close_run(); + + void write_data(const int64_t run_id, + const uint32_t index, + const char* data); + + void write_meta(const int64_t run_id, + const uint32_t index, + const ImageMetadata& meta); }; #endif //SFWRITER_HPP diff --git a/jf-live-writer/src/JFH5Writer.cpp b/jf-live-writer/src/JFH5Writer.cpp index 784597d..535d233 100644 --- a/jf-live-writer/src/JFH5Writer.cpp +++ b/jf-live-writer/src/JFH5Writer.cpp @@ -2,213 +2,181 @@ #include #include -#include +#include -#include "writer_config.hpp" + +#include "live_writer_config.hpp" #include "buffer_config.hpp" +#include "formats.hpp" -//extern "C" -//{ -// #include "H5DOpublic.h" -// #include -//} - -using namespace std; -using namespace writer_config; -using namespace buffer_config; - -JFH5Writer::JFH5Writer(const string& output_file, - const string& device, - const size_t n_modules, - const uint64_t start_pulse_id, - const uint64_t stop_pulse_id, - const size_t pulse_id_step) : - detector_name_(get_device_name(device)), - n_modules_(n_modules), - start_pulse_id_(start_pulse_id), - stop_pulse_id_(stop_pulse_id), - pulse_id_step_(pulse_id_step), - n_images_(get_n_pulses_in_range(start_pulse_id, - stop_pulse_id, - pulse_id_step)), - n_total_pulses_(stop_pulse_id_ - start_pulse_id_ + 1), - meta_write_index_(0), - data_write_index_(0) +extern "C" { - -// bshuf_register_h5filter(); - - file_ = H5::H5File(output_file, H5F_ACC_TRUNC); - file_.createGroup("/data"); - file_.createGroup("/data/" + detector_name_); - - H5::DataSpace att_space(H5S_SCALAR); - H5::DataType data_type = H5::StrType(0, H5T_VARIABLE); - - file_.createGroup("/general"); - auto detector_dataset = file_.createDataSet( - "/general/detector_name", data_type ,att_space); - - detector_dataset.write(detector_name_, data_type); - - hsize_t image_dataset_dims[3] = - {n_images_, n_modules * MODULE_Y_SIZE, MODULE_X_SIZE}; - - H5::DataSpace image_dataspace(3, image_dataset_dims); - - hsize_t image_dataset_chunking[3] = - {1, n_modules * MODULE_Y_SIZE, MODULE_X_SIZE}; - H5::DSetCreatPropList image_dataset_properties; - image_dataset_properties.setChunk(3, image_dataset_chunking); - -// // block_size, compression type -// uint compression_prop[] = -// {MODULE_N_PIXELS, //block size -// BSHUF_H5_COMPRESS_LZ4}; // Compression type -// -// H5Pset_filter(image_dataset_properties.getId(), -// BSHUF_H5FILTER, -// H5Z_FLAG_MANDATORY, -// 2, -// &(compression_prop[0])); - - image_dataset_ = file_.createDataSet( - "/data/" + detector_name_ + "/data", - H5::PredType::NATIVE_UINT16, - image_dataspace, - image_dataset_properties); - - b_pulse_id_ = new uint64_t[n_total_pulses_]; - b_frame_index_= new uint64_t[n_total_pulses_]; - b_daq_rec_ = new uint32_t[n_total_pulses_]; - b_is_good_frame_ = new uint8_t[n_total_pulses_]; + #include "H5DOpublic.h" + #include } -std::string JFH5Writer::get_device_name(const std::string& device) -{ - size_t last_separator; - if ((last_separator = device.rfind("/")) == string::npos) { - return device; - } +using namespace std; +using namespace buffer_config; +using namespace live_writer_config; - return device.substr(last_separator+1); +JFH5Writer::JFH5Writer(const BufferUtils::DetectorConfig config): + root_folder_(config.buffer_folder), + detector_name_(config.detector_name), + image_x_size_(config.image_x_size), + image_y_size_(config.image_y_size) +{ } JFH5Writer::~JFH5Writer() { close_file(); - - delete[] b_pulse_id_; - delete[] b_frame_index_; - delete[] b_daq_rec_; - delete[] b_is_good_frame_; } -size_t JFH5Writer::get_n_pulses_in_range( - const uint64_t start_pulse_id, - const uint64_t stop_pulse_id, - const int pulse_id_step) +void JFH5Writer::open_run(const int64_t run_id, const uint32_t n_images) { - if (stop_pulse_id < start_pulse_id) { - throw runtime_error("stop_pulse_id smaller than start_pulse_id."); - } + close_file(); - if (100 % pulse_id_step != 0) { - throw runtime_error("100 is not divisible by the pulse_id_step."); - } + const string output_folder = root_folder_ + "/" + OUTPUT_FOLDER_SYMLINK; + // TODO: Maybe add leading zeros to filename? + const string output_file = output_folder + to_string(run_id) + ".h5"; - if (start_pulse_id % pulse_id_step != 0) { - throw runtime_error("start_pulse_id not divisible by pulse_id_step."); - } + open_file(output_file, n_images); - if (stop_pulse_id % pulse_id_step != 0) { - throw runtime_error("stop_pulse_id not divisible by pulse_id_step."); - } - - size_t n_pulses = 1; - n_pulses += (stop_pulse_id / pulse_id_step); - n_pulses -= start_pulse_id / pulse_id_step; - - if (n_pulses == 0) { - throw runtime_error("Zero pulses to write in given range."); - } - - return n_pulses; + current_run_id_ = run_id; } -void JFH5Writer::write_metadata() +void JFH5Writer::close_run() { - hsize_t b_m_dims[] = {n_total_pulses_}; - hsize_t b_m_count[] = {n_images_}; - hsize_t b_m_start[] = {0}; - hsize_t b_m_stride[] = {pulse_id_step_}; - H5::DataSpace b_m_space (1, b_m_dims); - b_m_space.selectHyperslab(H5S_SELECT_SET, b_m_count, b_m_start, b_m_stride); + close_file(); + current_run_id_ = NO_RUN_ID; +} - hsize_t f_m_dims[] = {n_images_, 1}; - H5::DataSpace f_m_space(2, f_m_dims); +void JFH5Writer::open_file(const string& output_file, const uint32_t n_images) +{ + // Create file + auto fcpl_id = H5Pcreate(H5P_FILE_ACCESS); + if (fcpl_id == -1) { + throw runtime_error("Error in file access property list."); + } - auto pulse_id_dataset = file_.createDataSet( - "/data/" + detector_name_ + "/pulse_id", - H5::PredType::NATIVE_UINT64, f_m_space); - pulse_id_dataset.write( - b_pulse_id_, H5::PredType::NATIVE_UINT64, b_m_space, f_m_space); - pulse_id_dataset.close(); + if (H5Pset_fapl_mpio(fcpl_id, MPI_COMM_WORLD, MPI_INFO_NULL) < 0) { + throw runtime_error("Cannot set mpio to property list."); + } - auto frame_index_dataset = file_.createDataSet( - "/data/" + detector_name_ + "/frame_index", - H5::PredType::NATIVE_UINT64, f_m_space); - frame_index_dataset.write( - b_frame_index_, H5::PredType::NATIVE_UINT64, b_m_space, f_m_space); - frame_index_dataset.close(); + file_id_ = H5Fcreate( + output_file.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, fcpl_id); + if (file_id_ < 0) { + throw runtime_error("Cannot create output file."); + } - auto daq_rec_dataset = file_.createDataSet( - "/data/" + detector_name_ + "/daq_rec", - H5::PredType::NATIVE_UINT32, f_m_space); - daq_rec_dataset.write( - b_daq_rec_, H5::PredType::NATIVE_UINT32, b_m_space, f_m_space); - daq_rec_dataset.close(); + H5Pclose(fcpl_id); - auto is_good_frame_dataset = file_.createDataSet( - "/data/" + detector_name_ + "/is_good_frame", - H5::PredType::NATIVE_UINT8, f_m_space); - is_good_frame_dataset.write( - b_is_good_frame_, H5::PredType::NATIVE_UINT8, b_m_space, f_m_space); - is_good_frame_dataset.close(); + // Create group + auto data_group_id = H5Gcreate(file_id_, detector_name_.c_str(), + H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + if (data_group_id < 0) { + throw runtime_error("Cannot create data group."); + } + + // Create image dataset. + auto dcpl_id = H5Pcreate(H5P_DATASET_CREATE); + if (dcpl_id < 0) { + throw runtime_error("Error in creating dataset create property list."); + } + + hsize_t image_dataset_chunking[] = {1, image_y_size_, image_x_size_}; + if (H5Pset_chunk(dcpl_id, 3, image_dataset_chunking) < 0) { + throw runtime_error("Cannot set image dataset chunking."); + } + + if (H5Pset_fill_time(dcpl_id, H5D_FILL_TIME_NEVER) < 0) { + throw runtime_error("Cannot set image dataset fill time."); + } + + if (H5Pset_alloc_time(dcpl_id, H5D_ALLOC_TIME_EARLY) < 0) { + throw runtime_error("Cannot set image dataset allocation time."); + } + + hsize_t image_dataset_dims[] = {n_images, image_y_size_, image_x_size_}; + auto image_space_id = H5Screate_simple(3, image_dataset_dims, NULL); + if (image_space_id < 0) { + throw runtime_error("Cannot create image dataset space."); + } + + // TODO: Enable compression. +// bshuf_register_h5filter(); +// uint filter_prop[] = {PIXEL_N_BYTES, BSHUF_H5_COMPRESS_LZ4}; +// if (H5Pset_filter(dcpl_id, BSHUF_H5FILTER, H5Z_FLAG_MANDATORY, +// 2, filter_prop) < 0) { +// throw runtime_error("Cannot set compression filter on dataset."); +// } + + image_dataset_id_ = H5Dcreate( + data_group_id, "data", H5T_NATIVE_INT, image_space_id, + H5P_DEFAULT, dcpl_id, H5P_DEFAULT); + if (image_dataset_id_ < 0) { + throw runtime_error("Cannot create image dataset."); + } + + // Create metadata datasets. + hsize_t meta_dataset_dims[] = {n_images}; + auto meta_space_id = H5Screate_simple(1, meta_dataset_dims, NULL); + if (meta_space_id < 0) { + throw runtime_error("Cannot create meta dataset space."); + } + + auto create_meta_dataset = [&](string name, hid_t data_type) { + auto dataset_id = H5Dcreate( + data_group_id, name.c_str(), data_type, meta_space_id, + H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + if (dataset_id < 0) { + throw runtime_error("Cannot create " + name + " dataset."); + } + + return dataset_id; + }; + + pulse_dataset_id_ = create_meta_dataset("pulse_id", H5T_NATIVE_UINT64); + frame_dataset_id_ = create_meta_dataset("frame_index", H5T_NATIVE_UINT64); + daq_rec_dataset_id_ = create_meta_dataset("daq_rec", H5T_NATIVE_UINT32); + is_good_dataset_id_ = create_meta_dataset("is_good_frame", H5T_NATIVE_UINT8); + + H5Sclose(meta_space_id); + H5Sclose(image_space_id); + H5Pclose(dcpl_id); + H5Gclose(data_group_id); } void JFH5Writer::close_file() { - if (file_.getId() == -1) { + if (file_id_ < 0) { return; } - image_dataset_.close(); + H5Dclose(image_dataset_id_); + image_dataset_id_ = -1; - write_metadata(); + H5Dclose(pulse_dataset_id_); + pulse_dataset_id_ = -1; - file_.close(); + H5Dclose(frame_dataset_id_); + frame_dataset_id_ = -1; + + H5Dclose(daq_rec_dataset_id_); + daq_rec_dataset_id_ = -1; + + H5Dclose(is_good_dataset_id_); + is_good_dataset_id_ = -1; + + H5Fclose(file_id_); + file_id_ = -1; } -void JFH5Writer::write( - const ImageMetadataBlock* metadata, const char* data) +void JFH5Writer::write_data( + const int64_t run_id, const uint32_t index, const char* data) { - size_t n_images_offset = 0; - if (start_pulse_id_ > metadata->block_start_pulse_id) { - n_images_offset = start_pulse_id_ - metadata->block_start_pulse_id; - } - - if (n_images_offset > BUFFER_BLOCK_SIZE) { - throw runtime_error("Received unexpected block for start_pulse_id."); - } - - size_t n_images_to_copy = BUFFER_BLOCK_SIZE - n_images_offset; - if (stop_pulse_id_ < metadata->block_stop_pulse_id) { - n_images_to_copy -= metadata->block_stop_pulse_id - stop_pulse_id_; - } - - if (n_images_to_copy < 1) { - throw runtime_error("Received unexpected block for stop_pulse_id."); + if (run_id != current_run_id_) { + throw runtime_error("Invalid run_id."); } // hsize_t b_i_dims[3] = {BUFFER_BLOCK_SIZE, @@ -234,60 +202,20 @@ void JFH5Writer::write( // image_dataset_.write( // data, H5::PredType::NATIVE_UINT16, b_i_space, f_i_space); - // TODO: Can the i_image++ be made more efficient? - for (size_t i_image=n_images_offset; - i_image < n_images_offset + n_images_to_copy; - i_image++) { + hsize_t offset[] = {data_write_index_, 0, 0}; + size_t data_offset = i_image * MODULE_N_BYTES * n_modules_; + + H5DOwrite_chunk( + image_dataset_.getId(), + H5P_DEFAULT, + 0, + offset, + MODULE_N_BYTES * n_modules_, + data + data_offset); +} + +void JFH5Writer::write_meta( + const int64_t run_id, const uint32_t index, const ImageMetadata& meta) +{ - if (i_image % pulse_id_step_ != 0) { - continue; - } - - hsize_t offset[] = {data_write_index_, 0, 0}; - size_t data_offset = i_image * MODULE_N_BYTES * n_modules_; - - H5DOwrite_chunk( - image_dataset_.getId(), - H5P_DEFAULT, - 0, - offset, - MODULE_N_BYTES * n_modules_, - data + data_offset); - - data_write_index_++; - } - - // pulse_id - { - auto b_current_ptr = b_pulse_id_ + meta_write_index_; - memcpy(b_current_ptr, - &(metadata->pulse_id[n_images_offset]), - sizeof(uint64_t) * n_images_to_copy); - } - - // frame_index - { - auto b_current_ptr = b_frame_index_ + meta_write_index_; - memcpy(b_current_ptr, - &(metadata->frame_index[n_images_offset]), - sizeof(uint64_t) * n_images_to_copy); - } - - // daq_rec - { - auto b_current_ptr = b_daq_rec_ + meta_write_index_; - memcpy(b_current_ptr, - &(metadata->daq_rec[n_images_offset]), - sizeof(uint32_t) * n_images_to_copy); - } - - // is_good_frame - { - auto b_current_ptr = b_is_good_frame_ + meta_write_index_; - memcpy(b_current_ptr, - &(metadata->is_good_image[n_images_offset]), - sizeof(uint8_t) * n_images_to_copy); - } - - meta_write_index_ += n_images_to_copy; } From 6ad5b9538dd9f544bdbe9d06f69113cffa3bb0ed Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Thu, 25 Feb 2021 10:22:21 +0100 Subject: [PATCH 120/126] Remove unused OP code --- jf-live-writer/include/broker_format.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/jf-live-writer/include/broker_format.hpp b/jf-live-writer/include/broker_format.hpp index 9943d9c..3e8d3cd 100644 --- a/jf-live-writer/include/broker_format.hpp +++ b/jf-live-writer/include/broker_format.hpp @@ -3,7 +3,6 @@ #include "formats.hpp" -const static uint8_t OP_CONTINUE = 0; const static uint8_t OP_START = 1; const static uint8_t OP_END = 2; From 9a3df073311e02aef49cb66adf50e194a38fbc35 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Thu, 25 Feb 2021 10:26:59 +0100 Subject: [PATCH 121/126] Adjust main in live writer for metadata writes --- jf-live-writer/src/main.cpp | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/jf-live-writer/src/main.cpp b/jf-live-writer/src/main.cpp index 417c919..25cdfa9 100644 --- a/jf-live-writer/src/main.cpp +++ b/jf-live-writer/src/main.cpp @@ -61,21 +61,23 @@ int main (int argc, char *argv[]) } if (meta.op_code == OP_END) { - writer.close_run(meta.run_id); + writer.close_run(); continue; } - if (meta.i_image % n_writers != i_writer) { - continue; + // Fair distribution of images among writers. + if (meta.i_image % n_writers == i_writer) { + char* data = ram_buffer.read_image(meta.image_metadata.pulse_id); + + stats.start_image_write(); + writer.write_data(meta.run_id, meta.i_image, data); + stats.end_image_write(); } - char* data = ram_buffer.read_image(meta.image_metadata.pulse_id); - - stats.start_image_write(); - - writer.write(meta.run_id, meta.image_metadata, data); - - stats.end_image_write(); + // Only the first instance writes metadata. + if (i_writer == 0) { + writer.write_meta(meta.run_id, meta.i_image, meta.image_metadata); + } } MPI_Finalize(); From 592741048a6dd50b69f827707589a9ff38c8b941 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Thu, 25 Feb 2021 10:47:46 +0100 Subject: [PATCH 122/126] Add image details to store stream format --- jf-live-writer/include/broker_format.hpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/jf-live-writer/include/broker_format.hpp b/jf-live-writer/include/broker_format.hpp index 3e8d3cd..ecdca44 100644 --- a/jf-live-writer/include/broker_format.hpp +++ b/jf-live-writer/include/broker_format.hpp @@ -9,10 +9,13 @@ const static uint8_t OP_END = 2; #pragma pack(push) #pragma pack(1) struct StoreStream { - uint8_t op_code; + int64_t run_id; uint32_t i_image; uint32_t n_images; - int64_t run_id; + uint32_t image_y_size; + uint32_t image_x_size; + uint32_t op_code; + uint32_t bits_per_pixel; ImageMetadata image_metadata; }; From 676e8215d9ec4717130cf4e2f675df1f518a4a0f Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Thu, 25 Feb 2021 10:49:59 +0100 Subject: [PATCH 123/126] Moving image size initialization into each run --- jf-live-writer/include/JFH5Writer.hpp | 17 +++++++++++++---- jf-live-writer/src/JFH5Writer.cpp | 9 +++++++-- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/jf-live-writer/include/JFH5Writer.hpp b/jf-live-writer/include/JFH5Writer.hpp index fb4a96a..18becef 100644 --- a/jf-live-writer/include/JFH5Writer.hpp +++ b/jf-live-writer/include/JFH5Writer.hpp @@ -14,12 +14,15 @@ class JFH5Writer { const std::string root_folder_; const std::string detector_name_; - const uint32_t image_y_size_; - const uint32_t image_x_size_; static const int64_t NO_RUN_ID; - int64_t current_run_id_ = NO_RUN_ID; + // Run specific variables. + int64_t current_run_id_ = NO_RUN_ID; + uint32_t image_y_size_ = 0; + uint32_t image_x_size_ = 0; + + // Open file specific variables. hid_t file_id_ = -1; hid_t image_dataset_id_ = -1; hid_t pulse_dataset_id_= -1; @@ -27,13 +30,19 @@ class JFH5Writer { hid_t daq_rec_dataset_id_ = -1; hid_t is_good_dataset_id_ = -1; + hid_t get_datatype(const int bits_per_pixel); void open_file(const std::string& output_file, const uint32_t n_images); void close_file(); public: JFH5Writer(const BufferUtils::DetectorConfig config); ~JFH5Writer(); - void open_run(const int64_t run_id, const uint32_t n_images); + + void open_run(const int64_t run_id, + const uint32_t n_images, + const uint32_t image_y_size, + const uint32_t image_x_size, + const uint32_t bits_per_pixel); void close_run(); void write_data(const int64_t run_id, diff --git a/jf-live-writer/src/JFH5Writer.cpp b/jf-live-writer/src/JFH5Writer.cpp index 535d233..6e6c03e 100644 --- a/jf-live-writer/src/JFH5Writer.cpp +++ b/jf-live-writer/src/JFH5Writer.cpp @@ -22,8 +22,6 @@ using namespace live_writer_config; JFH5Writer::JFH5Writer(const BufferUtils::DetectorConfig config): root_folder_(config.buffer_folder), detector_name_(config.detector_name), - image_x_size_(config.image_x_size), - image_y_size_(config.image_y_size) { } @@ -32,6 +30,8 @@ JFH5Writer::~JFH5Writer() close_file(); } + + void JFH5Writer::open_run(const int64_t run_id, const uint32_t n_images) { close_file(); @@ -53,6 +53,7 @@ void JFH5Writer::close_run() void JFH5Writer::open_file(const string& output_file, const uint32_t n_images) { + // Create file auto fcpl_id = H5Pcreate(H5P_FILE_ACCESS); if (fcpl_id == -1) { @@ -217,5 +218,9 @@ void JFH5Writer::write_data( void JFH5Writer::write_meta( const int64_t run_id, const uint32_t index, const ImageMetadata& meta) { + if (run_id != current_run_id_) { + throw runtime_error("Invalid run_id."); + } + } From 2324ebde4f8c2bc5ea37491e50796bd0a854eadd Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Thu, 25 Feb 2021 10:51:04 +0100 Subject: [PATCH 124/126] Adjust main to new signature --- jf-live-writer/src/main.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/jf-live-writer/src/main.cpp b/jf-live-writer/src/main.cpp index 25cdfa9..4d77d29 100644 --- a/jf-live-writer/src/main.cpp +++ b/jf-live-writer/src/main.cpp @@ -56,7 +56,11 @@ int main (int argc, char *argv[]) zmq_recv(receiver, &meta, sizeof(meta), 0); if (meta.op_code == OP_START) { - writer.open_run(meta.run_id, meta.n_images); + writer.open_run(meta.run_id, + meta.n_images, + meta.image_y_size, + meta.image_x_size, + meta.bits_per_pixel); continue; } From 8583597a51471d17eb6469d60e208f8efb77b065 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Fri, 26 Feb 2021 12:03:20 +0100 Subject: [PATCH 125/126] Bits per pixel added to live writer --- jf-live-writer/include/JFH5Writer.hpp | 1 + jf-live-writer/src/JFH5Writer.cpp | 26 +++++++++++++++++--------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/jf-live-writer/include/JFH5Writer.hpp b/jf-live-writer/include/JFH5Writer.hpp index 18becef..3a681f2 100644 --- a/jf-live-writer/include/JFH5Writer.hpp +++ b/jf-live-writer/include/JFH5Writer.hpp @@ -21,6 +21,7 @@ class JFH5Writer { int64_t current_run_id_ = NO_RUN_ID; uint32_t image_y_size_ = 0; uint32_t image_x_size_ = 0; + uint32_t bits_per_pixel_ = 0; // Open file specific variables. hid_t file_id_ = -1; diff --git a/jf-live-writer/src/JFH5Writer.cpp b/jf-live-writer/src/JFH5Writer.cpp index 6e6c03e..a9d2a52 100644 --- a/jf-live-writer/src/JFH5Writer.cpp +++ b/jf-live-writer/src/JFH5Writer.cpp @@ -30,30 +30,38 @@ JFH5Writer::~JFH5Writer() close_file(); } - - -void JFH5Writer::open_run(const int64_t run_id, const uint32_t n_images) +void JFH5Writer::open_run(const int64_t run_id, + const uint32_t n_images, + const uint32_t image_y_size, + const uint32_t image_x_size, + const uint32_t bits_per_pixel) { - close_file(); + close_run(); const string output_folder = root_folder_ + "/" + OUTPUT_FOLDER_SYMLINK; // TODO: Maybe add leading zeros to filename? const string output_file = output_folder + to_string(run_id) + ".h5"; - open_file(output_file, n_images); - current_run_id_ = run_id; + image_y_size_ = image_y_size; + image_x_size_ = image_x_size; + bits_per_pixel_ = bits_per_pixel; + + open_file(output_file, n_images); } void JFH5Writer::close_run() { close_file(); + current_run_id_ = NO_RUN_ID; + image_y_size_ = 0; + image_x_size_ = 0; + bits_per_pixel_ = 0; } void JFH5Writer::open_file(const string& output_file, const uint32_t n_images) { - // Create file auto fcpl_id = H5Pcreate(H5P_FILE_ACCESS); if (fcpl_id == -1) { @@ -113,8 +121,8 @@ void JFH5Writer::open_file(const string& output_file, const uint32_t n_images) // } image_dataset_id_ = H5Dcreate( - data_group_id, "data", H5T_NATIVE_INT, image_space_id, - H5P_DEFAULT, dcpl_id, H5P_DEFAULT); + data_group_id, "data", get_datatype(bits_per_pixel_), + image_space_id, H5P_DEFAULT, dcpl_id, H5P_DEFAULT); if (image_dataset_id_ < 0) { throw runtime_error("Cannot create image dataset."); } From 00696c572b403b6abf27f93c63841d30f78d61f0 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Fri, 26 Feb 2021 12:03:48 +0100 Subject: [PATCH 126/126] Remove live writer from build until finished --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ff244cf..f392ac3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,4 +33,4 @@ add_subdirectory("jf-udp-recv") add_subdirectory("jf-buffer-writer") add_subdirectory("jf-assembler") add_subdirectory("sf-stream") -add_subdirectory("jf-live-writer") +#add_subdirectory("jf-live-writer")