From 9497b00c454e9196dc3159caa020a8407427879f Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 26 May 2020 13:34:27 +0200 Subject: [PATCH 001/111] Modify BufferBinaryFormat --- sf-buffer/include/BufferBinaryFormat.hpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/sf-buffer/include/BufferBinaryFormat.hpp b/sf-buffer/include/BufferBinaryFormat.hpp index d9f72cd..768c27e 100644 --- a/sf-buffer/include/BufferBinaryFormat.hpp +++ b/sf-buffer/include/BufferBinaryFormat.hpp @@ -12,10 +12,7 @@ struct BufferBinaryFormat { BufferBinaryFormat() : FORMAT_MARKER(JF_FORMAT_START_BYTE) {}; const char FORMAT_MARKER; - uint64_t pulse_id; - uint64_t frame_id; - uint32_t daq_rec; - uint16_t n_recv_packets; + ModuleFrame metadata; char data[JUNGFRAU_DATA_BYTES_PER_FRAME]; }; #pragma pack(pop) From 6abdd0fb04f25bfb41f07b1db0719cd182e1f27f Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 26 May 2020 13:34:36 +0200 Subject: [PATCH 002/111] New version of sf_buffer with binary --- sf-buffer/src/main.cpp | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/sf-buffer/src/main.cpp b/sf-buffer/src/main.cpp index 22965ed..de614a9 100644 --- a/sf-buffer/src/main.cpp +++ b/sf-buffer/src/main.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #include "zmq.h" #include "buffer_config.hpp" #include "jungfrau.hpp" @@ -11,6 +11,7 @@ #include #include #include +#include using namespace std; using namespace core_buffer; @@ -59,7 +60,7 @@ int main (int argc, char *argv[]) { uint64_t n_corrupted_frames = 0; uint64_t last_pulse_id = 0; - BufferH5Writer writer(root_folder, device_name); + BufferBinaryWriter writer(root_folder, device_name); BufferUdpReceiver receiver(udp_port, source_id); pid_t tid; @@ -67,8 +68,7 @@ int main (int argc, char *argv[]) { int ret = setpriority(PRIO_PROCESS, tid, 0); if (ret == -1) throw runtime_error("cannot set nice"); - ModuleFrame metadata; - auto frame_buffer = new char[MODULE_N_BYTES * JUNGFRAU_N_MODULES]; + BufferBinaryFormat* binary_buffer = new BufferBinaryFormat(); size_t write_total_us = 0; size_t write_max_us = 0; @@ -77,12 +77,12 @@ int main (int argc, char *argv[]) { while (true) { - auto pulse_id = receiver.get_frame_from_udp(metadata, frame_buffer); + auto pulse_id = receiver.get_frame_from_udp( + binary_buffer->metadata, binary_buffer->data); auto start_time = chrono::steady_clock::now(); - writer.set_pulse_id(pulse_id); - writer.write(&metadata, frame_buffer); + writer.write(pulse_id, binary_buffer); auto write_end_time = chrono::steady_clock::now(); auto write_us_duration = chrono::duration_cast( @@ -90,8 +90,9 @@ int main (int argc, char *argv[]) { start_time = chrono::steady_clock::now(); - zmq_send(socket, &metadata, sizeof(ModuleFrame), ZMQ_SNDMORE); - zmq_send(socket, frame_buffer, MODULE_N_BYTES, 0); + zmq_send(socket, &(binary_buffer->metadata), sizeof(ModuleFrame), + ZMQ_SNDMORE); + zmq_send(socket, binary_buffer->data, MODULE_N_BYTES, 0); auto send_end_time = chrono::steady_clock::now(); auto send_us_duration = chrono::duration_cast( @@ -110,9 +111,9 @@ int main (int argc, char *argv[]) { send_max_us = send_us_duration; } - if (metadata.n_received_packets < JF_N_PACKETS_PER_FRAME) { - n_missed_packets += - JF_N_PACKETS_PER_FRAME - metadata.n_received_packets; + if (binary_buffer->metadata.n_received_packets < JF_N_PACKETS_PER_FRAME) { + n_missed_packets += JF_N_PACKETS_PER_FRAME - + binary_buffer->metadata.n_received_packets; n_corrupted_frames++; } @@ -146,5 +147,5 @@ int main (int argc, char *argv[]) { } } - delete[] frame_buffer; + delete binary_buffer; } From c0668a97ba4af1722fef53d5138d8b00e7e45821 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 26 May 2020 14:56:17 +0200 Subject: [PATCH 003/111] Turn around attribute order to match HDF5 writer --- sf-buffer/include/BufferBinaryWriter.hpp | 6 +++--- sf-buffer/src/BufferBinaryWriter.cpp | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sf-buffer/include/BufferBinaryWriter.hpp b/sf-buffer/include/BufferBinaryWriter.hpp index ee90afb..8be8855 100644 --- a/sf-buffer/include/BufferBinaryWriter.hpp +++ b/sf-buffer/include/BufferBinaryWriter.hpp @@ -6,8 +6,8 @@ class BufferBinaryWriter { - const std::string device_name_; const std::string root_folder_; + const std::string device_name_; std::string latest_filename_; std::string current_output_filename_; @@ -19,8 +19,8 @@ class BufferBinaryWriter { public: BufferBinaryWriter( - const std::string& device_name, - const std::string& root_folder); + const std::string& root_folder, + const std::string& device_name); virtual ~BufferBinaryWriter(); diff --git a/sf-buffer/src/BufferBinaryWriter.cpp b/sf-buffer/src/BufferBinaryWriter.cpp index 750ef76..51fa275 100644 --- a/sf-buffer/src/BufferBinaryWriter.cpp +++ b/sf-buffer/src/BufferBinaryWriter.cpp @@ -12,10 +12,10 @@ using namespace std; BufferBinaryWriter::BufferBinaryWriter( - const string& device_name, - const string& root_folder) : - device_name_(device_name), + const string& root_folder, + const string& device_name): root_folder_(root_folder), + device_name_(device_name), latest_filename_(root_folder + "/" + device_name + "/LATEST"), current_output_filename_(""), output_file_fd_(-1) From 69c8501af74b403c25b6d3f9dccaf6f7413857cb Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 26 May 2020 14:56:40 +0200 Subject: [PATCH 004/111] We are writing binary data now --- core-buffer/include/buffer_config.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core-buffer/include/buffer_config.hpp b/core-buffer/include/buffer_config.hpp index e1b87a6..97a0df8 100644 --- a/core-buffer/include/buffer_config.hpp +++ b/core-buffer/include/buffer_config.hpp @@ -19,7 +19,7 @@ namespace core_buffer { // Must be power of 10 and >= than FILE_MOD. const size_t FOLDER_MOD = 100000; // Extension of our file format. - const std::string FILE_EXTENSION = ".h5"; + const std::string FILE_EXTENSION = ".bin"; // Number of pulses between each statistics print out. const size_t STATS_MODULO = 100; // If the RB is empty, how much time to wait before trying to read it again. From 0bdbe0238176f50b3d1ef9e011d6773363965f3d Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 26 May 2020 14:57:23 +0200 Subject: [PATCH 005/111] Adjust RECV buffer to bytes instead of msgs --- core-buffer/include/buffer_config.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core-buffer/include/buffer_config.hpp b/core-buffer/include/buffer_config.hpp index 97a0df8..de9361d 100644 --- a/core-buffer/include/buffer_config.hpp +++ b/core-buffer/include/buffer_config.hpp @@ -31,7 +31,7 @@ namespace core_buffer { const int BUFFER_UDP_RCVBUF_N_SLOTS = 100; // +1 for packet headers. const int BUFFER_UDP_RCVBUF_BYTES = - (128 * BUFFER_UDP_RCVBUF_N_SLOTS); + (128 * BUFFER_UDP_RCVBUF_N_SLOTS * 8246); // Microseconds timeout for UDP recv. const int BUFFER_UDP_US_TIMEOUT = 2 * 1000; // HWM for live stream from buffer. From 5047d00ae3e3697dfadf9ca660c7a77b5ea171cc Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 26 May 2020 19:16:25 +0200 Subject: [PATCH 006/111] Small optimization --- sf-buffer/src/BufferBinaryWriter.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sf-buffer/src/BufferBinaryWriter.cpp b/sf-buffer/src/BufferBinaryWriter.cpp index 51fa275..7430038 100644 --- a/sf-buffer/src/BufferBinaryWriter.cpp +++ b/sf-buffer/src/BufferBinaryWriter.cpp @@ -118,7 +118,7 @@ void BufferBinaryWriter::close_current_file() BufferUtils::update_latest_file( latest_filename_, current_output_filename_); - } - current_output_filename_ = ""; + current_output_filename_ = ""; + } } \ No newline at end of file From 299e61f569e97165a8041831743115b63612bb56 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 26 May 2020 19:34:54 +0200 Subject: [PATCH 007/111] Rename class to reflect better its role --- .../include/{WriterH5Writer.hpp => JFH5Writer.hpp} | 10 +++++----- sf-writer/include/WriterZmqReceiver.hpp | 2 +- sf-writer/src/{WriterH5Writer.cpp => JFH5Writer.cpp} | 10 +++++----- sf-writer/src/main.cpp | 4 ++-- sf-writer/test/manual/test_sf_writer_recv.cpp | 2 +- sf-writer/test/test_WriterH5Writer.cpp | 4 ++-- 6 files changed, 16 insertions(+), 16 deletions(-) rename sf-writer/include/{WriterH5Writer.hpp => JFH5Writer.hpp} (75%) rename sf-writer/src/{WriterH5Writer.cpp => JFH5Writer.cpp} (96%) diff --git a/sf-writer/include/WriterH5Writer.hpp b/sf-writer/include/JFH5Writer.hpp similarity index 75% rename from sf-writer/include/WriterH5Writer.hpp rename to sf-writer/include/JFH5Writer.hpp index d1983d0..d4cdb59 100644 --- a/sf-writer/include/WriterH5Writer.hpp +++ b/sf-writer/include/JFH5Writer.hpp @@ -7,7 +7,7 @@ #include "buffer_config.hpp" #include "formats.hpp" -class WriterH5Writer { +class JFH5Writer { const size_t n_frames_; const size_t n_modules_; @@ -23,10 +23,10 @@ class WriterH5Writer { public: - WriterH5Writer(const std::string& output_file, - const size_t n_frames, - const size_t n_modules); - ~WriterH5Writer(); + JFH5Writer(const std::string& output_file, + const size_t n_frames, + const size_t n_modules); + ~JFH5Writer(); void write(const ImageMetadataBuffer* metadata, const char* data); void close_file(); }; diff --git a/sf-writer/include/WriterZmqReceiver.hpp b/sf-writer/include/WriterZmqReceiver.hpp index b02f9e0..c66ba91 100644 --- a/sf-writer/include/WriterZmqReceiver.hpp +++ b/sf-writer/include/WriterZmqReceiver.hpp @@ -2,7 +2,7 @@ #define SF_DAQ_BUFFER_WRITERZMQRECEIVER_HPP #include -#include "WriterH5Writer.hpp" +#include "JFH5Writer.hpp" #include #include diff --git a/sf-writer/src/WriterH5Writer.cpp b/sf-writer/src/JFH5Writer.cpp similarity index 96% rename from sf-writer/src/WriterH5Writer.cpp rename to sf-writer/src/JFH5Writer.cpp index 0d921c0..94d89fe 100644 --- a/sf-writer/src/WriterH5Writer.cpp +++ b/sf-writer/src/JFH5Writer.cpp @@ -1,4 +1,4 @@ -#include "WriterH5Writer.hpp" +#include "JFH5Writer.hpp" #include @@ -11,7 +11,7 @@ using namespace std; using namespace core_buffer; -WriterH5Writer::WriterH5Writer( +JFH5Writer::JFH5Writer( const string& output_file, const size_t n_frames, const size_t n_modules) : @@ -90,12 +90,12 @@ WriterH5Writer::WriterH5Writer( } -WriterH5Writer::~WriterH5Writer() +JFH5Writer::~JFH5Writer() { close_file(); } -void WriterH5Writer::close_file() +void JFH5Writer::close_file() { image_dataset_.close(); pulse_id_dataset_.close(); @@ -106,7 +106,7 @@ void WriterH5Writer::close_file() file_.close(); } -void WriterH5Writer::write( +void JFH5Writer::write( const ImageMetadataBuffer* metadata, const char* data) { auto n_images_in_buffer = metadata->n_images; diff --git a/sf-writer/src/main.cpp b/sf-writer/src/main.cpp index 2a3774f..fda91ff 100644 --- a/sf-writer/src/main.cpp +++ b/sf-writer/src/main.cpp @@ -6,7 +6,7 @@ #include #include #include -#include "WriterH5Writer.hpp" +#include "JFH5Writer.hpp" #include #include #include "date.h" @@ -97,7 +97,7 @@ int main (int argc, char *argv[]) ref(queue), start_pulse_id, stop_pulse_id); size_t n_frames = stop_pulse_id - start_pulse_id + 1; - WriterH5Writer writer(output_file, n_frames, n_modules); + JFH5Writer writer(output_file, n_frames, n_modules); auto current_pulse_id = start_pulse_id; // "<= stop_pulse_id" because we include the last pulse_id. diff --git a/sf-writer/test/manual/test_sf_writer_recv.cpp b/sf-writer/test/manual/test_sf_writer_recv.cpp index 4f2e463..9a505d8 100644 --- a/sf-writer/test/manual/test_sf_writer_recv.cpp +++ b/sf-writer/test/manual/test_sf_writer_recv.cpp @@ -4,7 +4,7 @@ #include #include #include -#include "WriterH5Writer.hpp" +#include "JFH5Writer.hpp" #include #include #include "date.h" diff --git a/sf-writer/test/test_WriterH5Writer.cpp b/sf-writer/test/test_WriterH5Writer.cpp index fd3f91c..e5236f9 100644 --- a/sf-writer/test/test_WriterH5Writer.cpp +++ b/sf-writer/test/test_WriterH5Writer.cpp @@ -1,5 +1,5 @@ -#include "WriterH5Writer.hpp" +#include "JFH5Writer.hpp" #include "gtest/gtest.h" #include "bitshuffle/bitshuffle.h" @@ -18,7 +18,7 @@ TEST(WriterH5Writer, basic_interaction) metadata->data_n_bytes[0] = 500; metadata->n_pulses_in_buffer = 1; - WriterH5Writer writer("ignore.h5", n_frames, n_modules); + JFH5Writer writer("ignore.h5", n_frames, n_modules); writer.write(metadata.get(), data.get()); writer.close_file(); } From 3a13302e0f4fcbe4131a4afffb91505e08e07e45 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 26 May 2020 19:44:26 +0200 Subject: [PATCH 008/111] Replace slow call with C++17 stdlib --- sf-buffer/src/WriterUtils.cpp | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/sf-buffer/src/WriterUtils.cpp b/sf-buffer/src/WriterUtils.cpp index 7653bee..4adfeba 100644 --- a/sf-buffer/src/WriterUtils.cpp +++ b/sf-buffer/src/WriterUtils.cpp @@ -1,5 +1,6 @@ #include #include +#include #include "WriterUtils.hpp" #include "date.h" @@ -41,9 +42,20 @@ void WriterUtils::create_destination_folder(const string& output_file) auto file_separator_index = output_file.rfind('/'); if (file_separator_index != string::npos) { + string output_folder(output_file.substr(0, file_separator_index)); - string create_folder_command("mkdir -p " + output_folder); - system(create_folder_command.c_str()); + if(!filesystem::create_directories(output_folder)) { + stringstream err_msg; + + using namespace date; + using namespace chrono; + err_msg << "[" << system_clock::now() << "]"; + err_msg << "[WriterUtils::create_destination_folder]"; + err_msg << " Cannot create directory "; + err_msg << output_folder << endl; + + throw runtime_error(err_msg.str()); + } } } From 13a35a7668cd2a04b2b9110802c65c1c1fb6ee39 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 26 May 2020 20:10:18 +0200 Subject: [PATCH 009/111] Refactorig JFH5 Writer --- sf-writer/include/JFH5Writer.hpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sf-writer/include/JFH5Writer.hpp b/sf-writer/include/JFH5Writer.hpp index d4cdb59..402e648 100644 --- a/sf-writer/include/JFH5Writer.hpp +++ b/sf-writer/include/JFH5Writer.hpp @@ -29,6 +29,11 @@ public: ~JFH5Writer(); void write(const ImageMetadataBuffer* metadata, const char* data); void close_file(); + + uint64_t* b_pulse_id_; + uint64_t* b_frame_index_; + uint32_t* b_daq_rec_; + uint8_t* b_is_good_frame_ ; }; From 04a467c2b46b97b45e63abd675f280426eb311bc Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 26 May 2020 20:51:34 +0200 Subject: [PATCH 010/111] Make JF writer metadata buffering --- sf-writer/src/JFH5Writer.cpp | 97 +++++++++++++++++++++--------------- 1 file changed, 57 insertions(+), 40 deletions(-) diff --git a/sf-writer/src/JFH5Writer.cpp b/sf-writer/src/JFH5Writer.cpp index 94d89fe..1b0f4c9 100644 --- a/sf-writer/src/JFH5Writer.cpp +++ b/sf-writer/src/JFH5Writer.cpp @@ -1,5 +1,6 @@ #include "JFH5Writer.hpp" #include +#include //extern "C" @@ -54,49 +55,55 @@ JFH5Writer::JFH5Writer( hsize_t metadata_dataset_dims[] = {n_frames_, 1}; H5::DataSpace metadata_dataspace(2, metadata_dataset_dims); - // Chunk cannot be larger than n_frames. - auto metadata_chunk_size = WRITER_METADATA_CHUNK_N_IMAGES; - if (n_frames < metadata_chunk_size) { - metadata_chunk_size = n_frames; - } - - hsize_t metadata_dataset_chunking[] = {metadata_chunk_size, 1}; - H5::DSetCreatPropList metadata_dataset_properties; - metadata_dataset_properties.setChunk(2, metadata_dataset_chunking); - pulse_id_dataset_ = file_.createDataSet( "pulse_id", H5::PredType::NATIVE_UINT64, - metadata_dataspace, - metadata_dataset_properties); + metadata_dataspace); frame_index_dataset_ = file_.createDataSet( "frame_index", H5::PredType::NATIVE_UINT64, - metadata_dataspace, - metadata_dataset_properties); + metadata_dataspace); daq_rec_dataset_ = file_.createDataSet( "daq_rec", H5::PredType::NATIVE_UINT32, - metadata_dataspace, - metadata_dataset_properties); + metadata_dataspace); is_good_frame_dataset_ = file_.createDataSet( "is_good_frame", H5::PredType::NATIVE_UINT8, - metadata_dataspace, - metadata_dataset_properties); + metadata_dataspace); + b_pulse_id_ = new uint64_t[n_frames_]; + b_frame_index_= new uint64_t[n_frames_]; + b_daq_rec_ = new uint32_t[n_frames_]; + b_is_good_frame_ = new uint8_t[n_frames_]; } JFH5Writer::~JFH5Writer() { close_file(); + + delete[] b_pulse_id_; + delete[] b_frame_index_; + delete[] b_daq_rec_; + delete[] b_is_good_frame_; } void JFH5Writer::close_file() { + + pulse_id_dataset_.write(b_pulse_id_, H5::PredType::NATIVE_UINT64); + + frame_index_dataset_.write(b_frame_index_, + H5::PredType::NATIVE_UINT64); + + daq_rec_dataset_.write(b_daq_rec_, H5::PredType::NATIVE_UINT32); + + is_good_frame_dataset_.write(b_is_good_frame_, + H5::PredType::NATIVE_UINT8); + image_dataset_.close(); pulse_id_dataset_.close(); frame_index_dataset_.close(); @@ -132,31 +139,41 @@ void JFH5Writer::write( data, H5::PredType::NATIVE_UINT16, b_i_space, f_i_space); - hsize_t b_m_dims[2] = {n_images_in_buffer, 1}; - H5::DataSpace b_m_space (2, b_m_dims); + { + auto b_current_ptr = b_pulse_id_ + current_write_index_; + const uint64_t* metadata_pulse_id_ptr = + &(metadata->pulse_id[0]) + current_write_index_; + memcpy(b_current_ptr, + metadata_pulse_id_ptr, + sizeof(uint64_t) * n_images_in_buffer); + } - hsize_t f_m_dims[] = {n_frames_, 1}; - H5::DataSpace f_m_space(2, f_m_dims); + { + auto b_current_ptr = b_frame_index_ + current_write_index_; + const uint64_t* metadata_pulse_id_ptr = + &(metadata->frame_index[0]) + current_write_index_; + memcpy(b_current_ptr, + metadata_pulse_id_ptr, + sizeof(uint64_t) * n_images_in_buffer); + } - hsize_t meta_count[] = {n_images_in_buffer, 1}; - hsize_t meta_start[] = {current_write_index_, 0}; - f_m_space.selectHyperslab(H5S_SELECT_SET, meta_count, meta_start); + { + auto b_current_ptr = b_daq_rec_ + current_write_index_; + const uint32_t* metadata_pulse_id_ptr = + &(metadata->daq_rec[0]) + current_write_index_; + memcpy(b_current_ptr, + metadata_pulse_id_ptr, + sizeof(uint32_t) * n_images_in_buffer); + } - pulse_id_dataset_.write( - &(metadata->pulse_id), H5::PredType::NATIVE_UINT64, - b_m_space, f_m_space); - - frame_index_dataset_.write( - &(metadata->frame_index), H5::PredType::NATIVE_UINT64, - b_m_space, f_m_space); - - daq_rec_dataset_.write( - &(metadata->daq_rec), H5::PredType::NATIVE_UINT32, - b_m_space, f_m_space); - - is_good_frame_dataset_.write( - &(metadata->is_good_image), H5::PredType::NATIVE_UINT8, - b_m_space, f_m_space); + { + auto b_current_ptr = b_is_good_frame_ + current_write_index_; + const uint8_t* metadata_pulse_id_ptr = + &(metadata->is_good_image[0]) + current_write_index_; + memcpy(b_current_ptr, + metadata_pulse_id_ptr, + sizeof(uint8_t) * n_images_in_buffer); + } current_write_index_ += n_images_in_buffer; } From 9940f54e1acf5e745e047aa08fd8900a13a4efe5 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 26 May 2020 21:21:15 +0200 Subject: [PATCH 011/111] Implement BufferBlock --- core-buffer/include/formats.hpp | 19 ++++++++++++++++--- sf-buffer/include/BufferBinaryFormat.hpp | 20 -------------------- sf-buffer/src/main.cpp | 2 +- 3 files changed, 17 insertions(+), 24 deletions(-) delete mode 100644 sf-buffer/include/BufferBinaryFormat.hpp diff --git a/core-buffer/include/formats.hpp b/core-buffer/include/formats.hpp index e1083ff..2e9acec 100644 --- a/core-buffer/include/formats.hpp +++ b/core-buffer/include/formats.hpp @@ -13,13 +13,26 @@ struct ImageMetadataBuffer uint16_t n_images; }; +const char BUFFER_FORMAT_START_BYTE = 0xBE; + #pragma pack(push) #pragma pack(1) -struct ReplayBuffer +struct BufferBinaryFormat { + + BufferBinaryFormat() : FORMAT_MARKER(BUFFER_FORMAT_START_BYTE) {}; + + const char FORMAT_MARKER; + ModuleFrame metadata; + char data[core_buffer::MODULE_N_BYTES]; +}; +#pragma pack(pop) + +#pragma pack(push) +#pragma pack(1) +struct BufferBlock { - ModuleFrame metadata[core_buffer::REPLAY_READ_BUFFER_SIZE]; + BufferBinaryFormat frame[core_buffer::REPLAY_READ_BUFFER_SIZE]; uint64_t start_pulse_id; - uint16_t n_frames; }; #pragma pack(pop) diff --git a/sf-buffer/include/BufferBinaryFormat.hpp b/sf-buffer/include/BufferBinaryFormat.hpp deleted file mode 100644 index 768c27e..0000000 --- a/sf-buffer/include/BufferBinaryFormat.hpp +++ /dev/null @@ -1,20 +0,0 @@ -#ifndef JFFILEFORMAT_HPP -#define JFFILEFORMAT_HPP - -#include "jungfrau.hpp" - -const char JF_FORMAT_START_BYTE = 0xBE; - -#pragma pack(push) -#pragma pack(1) -struct BufferBinaryFormat { - - BufferBinaryFormat() : FORMAT_MARKER(JF_FORMAT_START_BYTE) {}; - - const char FORMAT_MARKER; - ModuleFrame metadata; - char data[JUNGFRAU_DATA_BYTES_PER_FRAME]; -}; -#pragma pack(pop) - -#endif // JFFILEFORMAT_HPP \ No newline at end of file diff --git a/sf-buffer/src/main.cpp b/sf-buffer/src/main.cpp index de614a9..b6aadc6 100644 --- a/sf-buffer/src/main.cpp +++ b/sf-buffer/src/main.cpp @@ -11,7 +11,7 @@ #include #include #include -#include +#include "formats.hpp" using namespace std; using namespace core_buffer; From 6653421696637e18e0cd0429083bd494a6d1684c Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 09:24:24 +0200 Subject: [PATCH 012/111] Rename buffer block size --- core-buffer/include/buffer_config.hpp | 2 +- core-buffer/include/formats.hpp | 2 +- sf-replay/src/ReplayH5Reader.cpp | 18 +++++++++--------- sf-replay/src/ReplayZmqSender.cpp | 2 +- sf-replay/src/main.cpp | 4 ++-- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/core-buffer/include/buffer_config.hpp b/core-buffer/include/buffer_config.hpp index de9361d..b605a79 100644 --- a/core-buffer/include/buffer_config.hpp +++ b/core-buffer/include/buffer_config.hpp @@ -53,7 +53,7 @@ namespace core_buffer { // Address where Replay streams and writer receives. const std::string REPLAY_STREAM_IPC_URL = "ipc:///tmp/sf-replay-"; // How many frames to read at once from file. - const size_t REPLAY_READ_BUFFER_SIZE = 100; + const size_t BUFFER_BLOCK_SIZE = 100; // How many slots to have in the internal queue between read and send. const int REPLAY_FASTQUEUE_N_SLOTS = 2; // How many frames do we buffer in send. diff --git a/core-buffer/include/formats.hpp b/core-buffer/include/formats.hpp index 2e9acec..d35c056 100644 --- a/core-buffer/include/formats.hpp +++ b/core-buffer/include/formats.hpp @@ -31,7 +31,7 @@ struct BufferBinaryFormat { #pragma pack(1) struct BufferBlock { - BufferBinaryFormat frame[core_buffer::REPLAY_READ_BUFFER_SIZE]; + BufferBinaryFormat frame[core_buffer::BUFFER_BLOCK_SIZE]; uint64_t start_pulse_id; }; #pragma pack(pop) diff --git a/sf-replay/src/ReplayH5Reader.cpp b/sf-replay/src/ReplayH5Reader.cpp index 9a126ab..56f8851 100644 --- a/sf-replay/src/ReplayH5Reader.cpp +++ b/sf-replay/src/ReplayH5Reader.cpp @@ -46,22 +46,22 @@ void ReplayH5Reader::get_buffer( } auto file_index = BufferUtils::get_file_frame_index(pulse_id); - auto cache_start_index = file_index / REPLAY_READ_BUFFER_SIZE; - cache_start_index *= REPLAY_READ_BUFFER_SIZE; + auto cache_start_index = file_index / BUFFER_BLOCK_SIZE; + cache_start_index *= BUFFER_BLOCK_SIZE; uint64_t b_start_pulse_id = pulse_id - (file_index - cache_start_index); metadata->start_pulse_id = b_start_pulse_id; - metadata->n_frames = REPLAY_READ_BUFFER_SIZE; + metadata->n_frames = BUFFER_BLOCK_SIZE; - hsize_t b_m_dims[2] = {REPLAY_READ_BUFFER_SIZE, ModuleFrame_N_FIELDS}; + hsize_t b_m_dims[2] = {BUFFER_BLOCK_SIZE, ModuleFrame_N_FIELDS}; H5::DataSpace b_m_space (2, b_m_dims); - hsize_t b_m_count[] = {REPLAY_READ_BUFFER_SIZE, ModuleFrame_N_FIELDS}; + hsize_t b_m_count[] = {BUFFER_BLOCK_SIZE, ModuleFrame_N_FIELDS}; hsize_t b_m_start[] = {cache_start_index, 0}; b_m_space.selectHyperslab(H5S_SELECT_SET, b_m_count, b_m_start); hsize_t f_m_dims[2] = {FILE_MOD, ModuleFrame_N_FIELDS}; H5::DataSpace f_m_space (2, f_m_dims); - hsize_t f_m_count[] = {REPLAY_READ_BUFFER_SIZE, ModuleFrame_N_FIELDS}; + hsize_t f_m_count[] = {BUFFER_BLOCK_SIZE, ModuleFrame_N_FIELDS}; hsize_t pulse_id_start[] = {cache_start_index, 0}; f_m_space.selectHyperslab(H5S_SELECT_SET, f_m_count, pulse_id_start); @@ -70,17 +70,17 @@ void ReplayH5Reader::get_buffer( H5::PredType::NATIVE_UINT64, b_m_space, f_m_space); hsize_t b_f_dims[3] = - {REPLAY_READ_BUFFER_SIZE, MODULE_Y_SIZE, MODULE_X_SIZE}; + {BUFFER_BLOCK_SIZE, MODULE_Y_SIZE, MODULE_X_SIZE}; H5::DataSpace b_f_space (3, b_f_dims); hsize_t b_f_count[] = - {REPLAY_READ_BUFFER_SIZE, MODULE_Y_SIZE, MODULE_X_SIZE}; + {BUFFER_BLOCK_SIZE, MODULE_Y_SIZE, MODULE_X_SIZE}; hsize_t b_f_start[] = {0, 0, 0}; b_f_space.selectHyperslab(H5S_SELECT_SET, b_f_count, b_f_start); hsize_t f_frame_dims[3] = {FILE_MOD, MODULE_Y_SIZE, MODULE_X_SIZE}; H5::DataSpace f_f_space (3, f_frame_dims); hsize_t f_f_count[] = - {REPLAY_READ_BUFFER_SIZE, MODULE_Y_SIZE, MODULE_X_SIZE}; + {BUFFER_BLOCK_SIZE, MODULE_Y_SIZE, MODULE_X_SIZE}; hsize_t f_f_start[] = {cache_start_index, 0, 0}; f_f_space.selectHyperslab(H5S_SELECT_SET, f_f_count, f_f_start); diff --git a/sf-replay/src/ReplayZmqSender.cpp b/sf-replay/src/ReplayZmqSender.cpp index 3d982ae..71b357d 100644 --- a/sf-replay/src/ReplayZmqSender.cpp +++ b/sf-replay/src/ReplayZmqSender.cpp @@ -44,5 +44,5 @@ void ReplayZmqSender::close() { void ReplayZmqSender::send(const ReplayBuffer* metadata, const char* data) { zmq_send(socket_, metadata, sizeof(ReplayBuffer), ZMQ_SNDMORE); - zmq_send(socket_, data, MODULE_N_BYTES * REPLAY_READ_BUFFER_SIZE, 0); + zmq_send(socket_, data, MODULE_N_BYTES * BUFFER_BLOCK_SIZE, 0); } \ No newline at end of file diff --git a/sf-replay/src/main.cpp b/sf-replay/src/main.cpp index e2cddc8..e97e04a 100644 --- a/sf-replay/src/main.cpp +++ b/sf-replay/src/main.cpp @@ -47,7 +47,7 @@ void sf_replay ( // TODO: Proper statistics cout << "sf_replay:avg_read_us "; - cout << read_us_duration / REPLAY_READ_BUFFER_SIZE << endl; + cout << read_us_duration / BUFFER_BLOCK_SIZE << endl; } } @@ -76,7 +76,7 @@ int main (int argc, char *argv[]) { const auto stop_pulse_id = (uint64_t) atoll(argv[6]); FastQueue queue( - MODULE_N_BYTES * REPLAY_READ_BUFFER_SIZE, + MODULE_N_BYTES * BUFFER_BLOCK_SIZE, REPLAY_FASTQUEUE_N_SLOTS); thread file_read_thread(sf_replay, From 981f76cac1d87dc22b5c8dd121dfc139f30f80dd Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 10:15:57 +0200 Subject: [PATCH 013/111] Redefined formats for image storage --- core-buffer/include/formats.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/core-buffer/include/formats.hpp b/core-buffer/include/formats.hpp index d35c056..ef3a9fb 100644 --- a/core-buffer/include/formats.hpp +++ b/core-buffer/include/formats.hpp @@ -6,11 +6,11 @@ struct ImageMetadataBuffer { - uint64_t pulse_id[core_buffer::WRITER_DATA_CACHE_N_IMAGES]; - uint64_t frame_index[core_buffer::WRITER_DATA_CACHE_N_IMAGES]; - uint32_t daq_rec[core_buffer::WRITER_DATA_CACHE_N_IMAGES]; - uint8_t is_good_image[core_buffer::WRITER_DATA_CACHE_N_IMAGES]; - uint16_t n_images; + uint64_t pulse_id[core_buffer::BUFFER_BLOCK_SIZE]; + uint64_t frame_index[core_buffer::BUFFER_BLOCK_SIZE]; + uint32_t daq_rec[core_buffer::BUFFER_BLOCK_SIZE]; + uint8_t is_good_image[core_buffer::BUFFER_BLOCK_SIZE]; + uint64_t start_pulse_id; }; const char BUFFER_FORMAT_START_BYTE = 0xBE; From 96314db1790f2012c5bde03d084e859de3b1b86d Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 10:16:17 +0200 Subject: [PATCH 014/111] Calculate exact number of images to copy from buffer --- sf-writer/src/JFH5Writer.cpp | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/sf-writer/src/JFH5Writer.cpp b/sf-writer/src/JFH5Writer.cpp index 1b0f4c9..3dcb0ce 100644 --- a/sf-writer/src/JFH5Writer.cpp +++ b/sf-writer/src/JFH5Writer.cpp @@ -116,10 +116,15 @@ void JFH5Writer::close_file() void JFH5Writer::write( const ImageMetadataBuffer* metadata, const char* data) { - auto n_images_in_buffer = metadata->n_images; + size_t n_images_to_copy = min(n_frames_ - current_write_index_, + BUFFER_BLOCK_SIZE); + + if (n_images_to_copy < 1) { + return; + } hsize_t b_i_dims[3] = { - n_images_in_buffer, + n_images_to_copy, MODULE_Y_SIZE*n_modules_, MODULE_X_SIZE}; H5::DataSpace b_i_space(3, b_i_dims); @@ -129,7 +134,7 @@ void JFH5Writer::write( MODULE_X_SIZE}; H5::DataSpace f_i_space(3, f_i_dims); - hsize_t i_count[] = {n_images_in_buffer, + hsize_t i_count[] = {n_images_to_copy, MODULE_Y_SIZE*n_modules_, MODULE_X_SIZE}; hsize_t i_start[] = {current_write_index_, 0, 0}; @@ -145,7 +150,7 @@ void JFH5Writer::write( &(metadata->pulse_id[0]) + current_write_index_; memcpy(b_current_ptr, metadata_pulse_id_ptr, - sizeof(uint64_t) * n_images_in_buffer); + sizeof(uint64_t) * n_images_to_copy); } { @@ -154,7 +159,7 @@ void JFH5Writer::write( &(metadata->frame_index[0]) + current_write_index_; memcpy(b_current_ptr, metadata_pulse_id_ptr, - sizeof(uint64_t) * n_images_in_buffer); + sizeof(uint64_t) * n_images_to_copy); } { @@ -163,7 +168,7 @@ void JFH5Writer::write( &(metadata->daq_rec[0]) + current_write_index_; memcpy(b_current_ptr, metadata_pulse_id_ptr, - sizeof(uint32_t) * n_images_in_buffer); + sizeof(uint32_t) * n_images_to_copy); } { @@ -172,8 +177,8 @@ void JFH5Writer::write( &(metadata->is_good_image[0]) + current_write_index_; memcpy(b_current_ptr, metadata_pulse_id_ptr, - sizeof(uint8_t) * n_images_in_buffer); + sizeof(uint8_t) * n_images_to_copy); } - current_write_index_ += n_images_in_buffer; + current_write_index_ += n_images_to_copy; } From 65d587c80b7877ecfc5a9c33ea503615d9b96f6e Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 10:18:44 +0200 Subject: [PATCH 015/111] Refactoring of JF writer --- sf-writer/include/JFH5Writer.hpp | 4 ++-- sf-writer/src/JFH5Writer.cpp | 36 +++++++++++++++++--------------- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/sf-writer/include/JFH5Writer.hpp b/sf-writer/include/JFH5Writer.hpp index 402e648..2724f3d 100644 --- a/sf-writer/include/JFH5Writer.hpp +++ b/sf-writer/include/JFH5Writer.hpp @@ -9,7 +9,7 @@ class JFH5Writer { - const size_t n_frames_; + const size_t n_images_; const size_t n_modules_; size_t current_write_index_; @@ -24,7 +24,7 @@ class JFH5Writer { public: JFH5Writer(const std::string& output_file, - const size_t n_frames, + const size_t n_images, const size_t n_modules); ~JFH5Writer(); void write(const ImageMetadataBuffer* metadata, const char* data); diff --git a/sf-writer/src/JFH5Writer.cpp b/sf-writer/src/JFH5Writer.cpp index 3dcb0ce..affd557 100644 --- a/sf-writer/src/JFH5Writer.cpp +++ b/sf-writer/src/JFH5Writer.cpp @@ -14,9 +14,9 @@ using namespace core_buffer; JFH5Writer::JFH5Writer( const string& output_file, - const size_t n_frames, + const size_t n_images, const size_t n_modules) : - n_frames_(n_frames), + n_images_(n_images), n_modules_(n_modules), current_write_index_(0) { @@ -26,7 +26,7 @@ JFH5Writer::JFH5Writer( file_ = H5::H5File(output_file, H5F_ACC_TRUNC); hsize_t image_dataset_dims[3] = - {n_frames_, n_modules * MODULE_Y_SIZE, MODULE_X_SIZE}; + {n_images_, n_modules * MODULE_Y_SIZE, MODULE_X_SIZE}; H5::DataSpace image_dataspace(3, image_dataset_dims); @@ -52,7 +52,7 @@ JFH5Writer::JFH5Writer( image_dataspace, image_dataset_properties); - hsize_t metadata_dataset_dims[] = {n_frames_, 1}; + hsize_t metadata_dataset_dims[] = {n_images_, 1}; H5::DataSpace metadata_dataspace(2, metadata_dataset_dims); pulse_id_dataset_ = file_.createDataSet( @@ -75,10 +75,10 @@ JFH5Writer::JFH5Writer( H5::PredType::NATIVE_UINT8, metadata_dataspace); - b_pulse_id_ = new uint64_t[n_frames_]; - b_frame_index_= new uint64_t[n_frames_]; - b_daq_rec_ = new uint32_t[n_frames_]; - b_is_good_frame_ = new uint8_t[n_frames_]; + b_pulse_id_ = new uint64_t[n_images_]; + b_frame_index_= new uint64_t[n_images_]; + b_daq_rec_ = new uint32_t[n_images_]; + b_is_good_frame_ = new uint8_t[n_images_]; } JFH5Writer::~JFH5Writer() @@ -116,34 +116,33 @@ void JFH5Writer::close_file() void JFH5Writer::write( const ImageMetadataBuffer* metadata, const char* data) { - size_t n_images_to_copy = min(n_frames_ - current_write_index_, + size_t n_images_to_copy = min(n_images_ - current_write_index_, BUFFER_BLOCK_SIZE); if (n_images_to_copy < 1) { return; } - hsize_t b_i_dims[3] = { - n_images_to_copy, - MODULE_Y_SIZE*n_modules_, - MODULE_X_SIZE}; + hsize_t b_i_dims[3] = {n_images_to_copy, + MODULE_Y_SIZE * n_modules_, + MODULE_X_SIZE}; H5::DataSpace b_i_space(3, b_i_dims); - hsize_t f_i_dims[3] = {n_frames_, + hsize_t f_i_dims[3] = {n_images_, MODULE_Y_SIZE * n_modules_, MODULE_X_SIZE}; H5::DataSpace f_i_space(3, f_i_dims); hsize_t i_count[] = {n_images_to_copy, - MODULE_Y_SIZE*n_modules_, + MODULE_Y_SIZE * n_modules_, MODULE_X_SIZE}; hsize_t i_start[] = {current_write_index_, 0, 0}; f_i_space.selectHyperslab(H5S_SELECT_SET, i_count, i_start); image_dataset_.write( - data, H5::PredType::NATIVE_UINT16, - b_i_space, f_i_space); + data, H5::PredType::NATIVE_UINT16, b_i_space, f_i_space); + // pulse_id { auto b_current_ptr = b_pulse_id_ + current_write_index_; const uint64_t* metadata_pulse_id_ptr = @@ -153,6 +152,7 @@ void JFH5Writer::write( sizeof(uint64_t) * n_images_to_copy); } + // frame_index { auto b_current_ptr = b_frame_index_ + current_write_index_; const uint64_t* metadata_pulse_id_ptr = @@ -162,6 +162,7 @@ void JFH5Writer::write( sizeof(uint64_t) * n_images_to_copy); } + // daq_rec { auto b_current_ptr = b_daq_rec_ + current_write_index_; const uint32_t* metadata_pulse_id_ptr = @@ -171,6 +172,7 @@ void JFH5Writer::write( sizeof(uint32_t) * n_images_to_copy); } + // is_good_frame { auto b_current_ptr = b_is_good_frame_ + current_write_index_; const uint8_t* metadata_pulse_id_ptr = From 01776e12adc19bc1f5d45390028307d2307416e5 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 10:21:02 +0200 Subject: [PATCH 016/111] Rename ImageMetadataBlock --- core-buffer/include/formats.hpp | 4 ++-- core-buffer/src/FastQueue.cpp | 2 +- sf-writer/include/JFH5Writer.hpp | 2 +- sf-writer/include/WriterZmqReceiver.hpp | 2 +- sf-writer/src/JFH5Writer.cpp | 2 +- sf-writer/src/WriterZmqReceiver.cpp | 2 +- sf-writer/src/main.cpp | 4 ++-- sf-writer/test/manual/test_sf_writer_recv.cpp | 4 ++-- sf-writer/test/test_WriterH5Writer.cpp | 2 +- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/core-buffer/include/formats.hpp b/core-buffer/include/formats.hpp index ef3a9fb..80bea40 100644 --- a/core-buffer/include/formats.hpp +++ b/core-buffer/include/formats.hpp @@ -4,7 +4,7 @@ #include "buffer_config.hpp" #include "jungfrau.hpp" -struct ImageMetadataBuffer +struct ImageMetadataBlock { uint64_t pulse_id[core_buffer::BUFFER_BLOCK_SIZE]; uint64_t frame_index[core_buffer::BUFFER_BLOCK_SIZE]; @@ -29,7 +29,7 @@ struct BufferBinaryFormat { #pragma pack(push) #pragma pack(1) -struct BufferBlock +struct BufferBinaryBlock { BufferBinaryFormat frame[core_buffer::BUFFER_BLOCK_SIZE]; uint64_t start_pulse_id; diff --git a/core-buffer/src/FastQueue.cpp b/core-buffer/src/FastQueue.cpp index 1595278..c3f9daf 100644 --- a/core-buffer/src/FastQueue.cpp +++ b/core-buffer/src/FastQueue.cpp @@ -103,7 +103,7 @@ void FastQueue::release() read_slot_id_ %= n_slots_; } -template class FastQueue; +template class FastQueue; template class FastQueue; template class FastQueue; template class FastQueue; diff --git a/sf-writer/include/JFH5Writer.hpp b/sf-writer/include/JFH5Writer.hpp index 2724f3d..4e96e9a 100644 --- a/sf-writer/include/JFH5Writer.hpp +++ b/sf-writer/include/JFH5Writer.hpp @@ -27,7 +27,7 @@ public: const size_t n_images, const size_t n_modules); ~JFH5Writer(); - void write(const ImageMetadataBuffer* metadata, const char* data); + void write(const ImageMetadataBlock* metadata, const char* data); void close_file(); uint64_t* b_pulse_id_; diff --git a/sf-writer/include/WriterZmqReceiver.hpp b/sf-writer/include/WriterZmqReceiver.hpp index c66ba91..96bc16c 100644 --- a/sf-writer/include/WriterZmqReceiver.hpp +++ b/sf-writer/include/WriterZmqReceiver.hpp @@ -26,7 +26,7 @@ public: void get_next_buffer( const uint64_t start_pulse_id, - ImageMetadataBuffer* i_meta, + ImageMetadataBlock* i_meta, char* image_buffer); }; diff --git a/sf-writer/src/JFH5Writer.cpp b/sf-writer/src/JFH5Writer.cpp index affd557..1abd2fb 100644 --- a/sf-writer/src/JFH5Writer.cpp +++ b/sf-writer/src/JFH5Writer.cpp @@ -114,7 +114,7 @@ void JFH5Writer::close_file() } void JFH5Writer::write( - const ImageMetadataBuffer* metadata, const char* data) + const ImageMetadataBlock* metadata, const char* data) { size_t n_images_to_copy = min(n_images_ - current_write_index_, BUFFER_BLOCK_SIZE); diff --git a/sf-writer/src/WriterZmqReceiver.cpp b/sf-writer/src/WriterZmqReceiver.cpp index 18d38dc..b01b6f5 100644 --- a/sf-writer/src/WriterZmqReceiver.cpp +++ b/sf-writer/src/WriterZmqReceiver.cpp @@ -50,7 +50,7 @@ WriterZmqReceiver::~WriterZmqReceiver() void WriterZmqReceiver::get_next_buffer( const uint64_t start_pulse_id, - ImageMetadataBuffer* i_meta, + ImageMetadataBlock* i_meta, char* image_buffer) { auto n_images_in_buffer = WRITER_DATA_CACHE_N_IMAGES; diff --git a/sf-writer/src/main.cpp b/sf-writer/src/main.cpp index fda91ff..d4da930 100644 --- a/sf-writer/src/main.cpp +++ b/sf-writer/src/main.cpp @@ -20,7 +20,7 @@ void receive_replay( void* ctx, const string ipc_prefix, const size_t n_modules, - FastQueue& queue, + FastQueue& queue, const uint64_t start_pulse_id, const uint64_t stop_pulse_id) { @@ -84,7 +84,7 @@ int main (int argc, char *argv[]) size_t n_modules = 32; - FastQueue queue( + FastQueue queue( MODULE_N_BYTES * n_modules * WRITER_DATA_CACHE_N_IMAGES, WRITER_FASTQUEUE_N_SLOTS); diff --git a/sf-writer/test/manual/test_sf_writer_recv.cpp b/sf-writer/test/manual/test_sf_writer_recv.cpp index 9a505d8..b78c48f 100644 --- a/sf-writer/test/manual/test_sf_writer_recv.cpp +++ b/sf-writer/test/manual/test_sf_writer_recv.cpp @@ -18,7 +18,7 @@ void receive_replay( void* ctx, const string ipc_prefix, const size_t n_modules, - FastQueue& queue, + FastQueue& queue, const uint64_t start_pulse_id, const uint64_t stop_pulse_id) { @@ -89,7 +89,7 @@ int main (int argc, char *argv[]) size_t n_modules = 32; - FastQueue queue( + FastQueue queue( MODULE_N_BYTES * n_modules * WRITER_DATA_CACHE_N_IMAGES, WRITER_FASTQUEUE_N_SLOTS); diff --git a/sf-writer/test/test_WriterH5Writer.cpp b/sf-writer/test/test_WriterH5Writer.cpp index e5236f9..abcc4be 100644 --- a/sf-writer/test/test_WriterH5Writer.cpp +++ b/sf-writer/test/test_WriterH5Writer.cpp @@ -12,7 +12,7 @@ TEST(WriterH5Writer, basic_interaction) size_t n_frames = 5; auto data = make_unique(n_modules*MODULE_N_BYTES); - auto metadata = make_shared(); + auto metadata = make_shared(); // Needed by writer. metadata->data_n_bytes[0] = 500; From 0f6d63aded37ccde4539ac480034f35c1a4eaebb Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 10:22:44 +0200 Subject: [PATCH 017/111] Send BufferBinaryBlock from sf_replay --- sf-replay/include/ReplayZmqSender.hpp | 2 +- sf-replay/src/ReplayZmqSender.cpp | 6 ++++-- sf-replay/src/main.cpp | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/sf-replay/include/ReplayZmqSender.hpp b/sf-replay/include/ReplayZmqSender.hpp index 845c9f0..7396b1e 100644 --- a/sf-replay/include/ReplayZmqSender.hpp +++ b/sf-replay/include/ReplayZmqSender.hpp @@ -16,7 +16,7 @@ public: void close(); - void send(const ReplayBuffer* metadata, const char* data); + void send(const BufferBinaryBlock* metadata, const char* data); }; diff --git a/sf-replay/src/ReplayZmqSender.cpp b/sf-replay/src/ReplayZmqSender.cpp index 71b357d..27be144 100644 --- a/sf-replay/src/ReplayZmqSender.cpp +++ b/sf-replay/src/ReplayZmqSender.cpp @@ -41,8 +41,10 @@ void ReplayZmqSender::close() { zmq_ctx_destroy(ctx_); } -void ReplayZmqSender::send(const ReplayBuffer* metadata, const char* data) +void ReplayZmqSender::send( + const BufferBinaryBlock* metadata, + const char* data) { - zmq_send(socket_, metadata, sizeof(ReplayBuffer), ZMQ_SNDMORE); + zmq_send(socket_, metadata, sizeof(BufferBinaryBlock), ZMQ_SNDMORE); zmq_send(socket_, data, MODULE_N_BYTES * BUFFER_BLOCK_SIZE, 0); } \ No newline at end of file diff --git a/sf-replay/src/main.cpp b/sf-replay/src/main.cpp index e97e04a..81d9937 100644 --- a/sf-replay/src/main.cpp +++ b/sf-replay/src/main.cpp @@ -14,7 +14,7 @@ using namespace chrono; void sf_replay ( const string device, const string channel_name, - FastQueue& queue, + FastQueue& queue, const uint64_t start_pulse_id, const uint64_t stop_pulse_id ) @@ -75,7 +75,7 @@ int main (int argc, char *argv[]) { const auto start_pulse_id = (uint64_t) atoll(argv[5]); const auto stop_pulse_id = (uint64_t) atoll(argv[6]); - FastQueue queue( + FastQueue queue( MODULE_N_BYTES * BUFFER_BLOCK_SIZE, REPLAY_FASTQUEUE_N_SLOTS); From 72a9b3bcc991533b02118196528f9eea05d6f24d Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 10:27:14 +0200 Subject: [PATCH 018/111] Small refactoring --- sf-buffer/include/BufferBinaryWriter.hpp | 3 ++- sf-buffer/include/BufferH5Writer.hpp | 5 +++-- sf-buffer/src/BufferBinaryWriter.cpp | 13 +++++++++---- sf-buffer/src/BufferH5Writer.cpp | 3 ++- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/sf-buffer/include/BufferBinaryWriter.hpp b/sf-buffer/include/BufferBinaryWriter.hpp index 8be8855..e401236 100644 --- a/sf-buffer/include/BufferBinaryWriter.hpp +++ b/sf-buffer/include/BufferBinaryWriter.hpp @@ -2,7 +2,8 @@ #define BINARYWRITER_HPP #include -#include "BufferBinaryFormat.hpp" + +#include "formats.hpp" class BufferBinaryWriter { diff --git a/sf-buffer/include/BufferH5Writer.hpp b/sf-buffer/include/BufferH5Writer.hpp index a4f61b2..922d4df 100644 --- a/sf-buffer/include/BufferH5Writer.hpp +++ b/sf-buffer/include/BufferH5Writer.hpp @@ -6,9 +6,10 @@ #include #include #include -#include "jungfrau.hpp" #include -#include + +#include "jungfrau.hpp" +#include "buffer_config.hpp" class BufferH5Writer { diff --git a/sf-buffer/src/BufferBinaryWriter.cpp b/sf-buffer/src/BufferBinaryWriter.cpp index 7430038..6e189a6 100644 --- a/sf-buffer/src/BufferBinaryWriter.cpp +++ b/sf-buffer/src/BufferBinaryWriter.cpp @@ -1,13 +1,15 @@ #include "BufferBinaryWriter.hpp" + #include #include #include "date.h" #include #include #include -#include #include -#include + +#include "BufferUtils.hpp" +#include "WriterUtils.hpp" using namespace std; @@ -27,7 +29,9 @@ BufferBinaryWriter::~BufferBinaryWriter() close_current_file(); } -void BufferBinaryWriter::write(uint64_t pulse_id, const BufferBinaryFormat* buffer) +void BufferBinaryWriter::write( + uint64_t pulse_id, + const BufferBinaryFormat* buffer) { auto current_frame_file = BufferUtils::get_filename(root_folder_, device_name_, pulse_id); @@ -37,7 +41,8 @@ void BufferBinaryWriter::write(uint64_t pulse_id, const BufferBinaryFormat* buff } size_t n_bytes_offset = - BufferUtils::get_file_frame_index(pulse_id) * sizeof(BufferBinaryFormat); + BufferUtils::get_file_frame_index(pulse_id) * + sizeof(BufferBinaryFormat); auto lseek_result = lseek(output_file_fd_, n_bytes_offset, SEEK_SET); if (lseek_result < 0) { diff --git a/sf-buffer/src/BufferH5Writer.cpp b/sf-buffer/src/BufferH5Writer.cpp index ed7ce39..c2706ec 100644 --- a/sf-buffer/src/BufferH5Writer.cpp +++ b/sf-buffer/src/BufferH5Writer.cpp @@ -84,7 +84,8 @@ void BufferH5Writer::set_pulse_id(const uint64_t pulse_id) if (h5_file_.getId() != -1) { auto latest_filename = output_filename_; close_file(); - BufferUtils::update_latest_file(LATEST_filename_, latest_filename); + BufferUtils::update_latest_file( + LATEST_filename_, latest_filename); } WriterUtils::create_destination_folder(new_output_filename); From 32f4ef6dba258bfc21b9f9875245c03bfef46393 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 10:43:22 +0200 Subject: [PATCH 019/111] Change logic from pulse_id to block sending --- sf-replay/src/main.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/sf-replay/src/main.cpp b/sf-replay/src/main.cpp index 81d9937..e7b65b7 100644 --- a/sf-replay/src/main.cpp +++ b/sf-replay/src/main.cpp @@ -89,10 +89,13 @@ int main (int argc, char *argv[]) { ReplayZmqSender sender(ipc_id, source_id); - // "<= stop_pulse_id" because we include the stop_pulse_id in the file. - for (uint64_t curr_pulse_id=start_pulse_id; - curr_pulse_id <= stop_pulse_id; - curr_pulse_id++) { + uint64_t start_block = start_pulse_id / BUFFER_BLOCK_SIZE; + uint64_t stop_block = stop_pulse_id / BUFFER_BLOCK_SIZE; + + // "<= stop_block" because we include the stop_block in the transfer. + for (uint64_t curr_block=start_block; + curr_block <= stop_block; + curr_block++) { int slot_id; while((slot_id = queue.read()) == -1) { From d42eca0e7a8ab29433274b6368657fd3aa4f860c Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 10:54:34 +0200 Subject: [PATCH 020/111] Adjust to sending only 1 binary block --- sf-replay/include/ReplayZmqSender.hpp | 2 +- sf-replay/src/ReplayZmqSender.cpp | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/sf-replay/include/ReplayZmqSender.hpp b/sf-replay/include/ReplayZmqSender.hpp index 7396b1e..5851ad4 100644 --- a/sf-replay/include/ReplayZmqSender.hpp +++ b/sf-replay/include/ReplayZmqSender.hpp @@ -16,7 +16,7 @@ public: void close(); - void send(const BufferBinaryBlock* metadata, const char* data); + void send(const BufferBinaryBlock* block_data); }; diff --git a/sf-replay/src/ReplayZmqSender.cpp b/sf-replay/src/ReplayZmqSender.cpp index 27be144..f16e028 100644 --- a/sf-replay/src/ReplayZmqSender.cpp +++ b/sf-replay/src/ReplayZmqSender.cpp @@ -41,10 +41,7 @@ void ReplayZmqSender::close() { zmq_ctx_destroy(ctx_); } -void ReplayZmqSender::send( - const BufferBinaryBlock* metadata, - const char* data) +void ReplayZmqSender::send(const BufferBinaryBlock* block_data) { - zmq_send(socket_, metadata, sizeof(BufferBinaryBlock), ZMQ_SNDMORE); - zmq_send(socket_, data, MODULE_N_BYTES * BUFFER_BLOCK_SIZE, 0); + zmq_send(socket_, block_data, sizeof(BufferBinaryBlock), ZMQ_SNDMORE); } \ No newline at end of file From 44ee05953a521c2540b79aca297cc0ee96097a40 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 10:54:55 +0200 Subject: [PATCH 021/111] Conversion to block logic --- sf-replay/src/main.cpp | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/sf-replay/src/main.cpp b/sf-replay/src/main.cpp index e7b65b7..fb04ecf 100644 --- a/sf-replay/src/main.cpp +++ b/sf-replay/src/main.cpp @@ -4,7 +4,7 @@ #include #include "buffer_config.hpp" -#include "ReplayH5Reader.hpp" +#include "BufferBinaryReader.hpp" #include "ReplayZmqSender.hpp" using namespace std; @@ -19,12 +19,15 @@ void sf_replay ( const uint64_t stop_pulse_id ) { - ReplayH5Reader file_reader(device, channel_name); + BufferBinaryReader block_reader(device, channel_name); - // "<= stop_pulse_id" because we include the stop_pulse_id in the file. - for (uint64_t curr_pulse_id=start_pulse_id; - curr_pulse_id <= stop_pulse_id; - curr_pulse_id++) { + uint64_t start_block = start_pulse_id / BUFFER_BLOCK_SIZE; + uint64_t stop_block = stop_pulse_id / BUFFER_BLOCK_SIZE; + + // "<= stop_block" because we include the stop_block in the transfer. + for (uint64_t curr_block=start_block; + curr_block <= stop_block; + curr_block++) { int slot_id; while((slot_id = queue.reserve()) == -1) { @@ -35,13 +38,12 @@ void sf_replay ( auto start_time = steady_clock::now(); auto metadata = queue.get_metadata_buffer(slot_id); - auto buffer = queue.get_data_buffer(slot_id); - file_reader.get_buffer(curr_pulse_id, metadata, buffer); + block_reader.get_block(curr_block, metadata); auto end_time = steady_clock::now(); - uint64_t read_us_duration = - duration_cast(end_time-start_time).count(); + uint64_t read_us_duration = duration_cast( + end_time-start_time).count(); queue.commit(); @@ -75,9 +77,8 @@ int main (int argc, char *argv[]) { const auto start_pulse_id = (uint64_t) atoll(argv[5]); const auto stop_pulse_id = (uint64_t) atoll(argv[6]); - FastQueue queue( - MODULE_N_BYTES * BUFFER_BLOCK_SIZE, - REPLAY_FASTQUEUE_N_SLOTS); + // 0 bytes for data since everything is in the header. + FastQueue queue(0, REPLAY_FASTQUEUE_N_SLOTS); thread file_read_thread(sf_replay, device, channel_name, ref(queue), @@ -103,12 +104,11 @@ int main (int argc, char *argv[]) { RB_READ_RETRY_INTERVAL_MS)); } - auto m_buffer = queue.get_metadata_buffer(slot_id); - auto f_buffer = queue.get_data_buffer(slot_id); + auto block_buffer = queue.get_metadata_buffer(slot_id); auto start_time = steady_clock::now(); - sender.send(m_buffer, f_buffer); + sender.send(block_buffer); auto end_time = steady_clock::now(); uint64_t send_us_duration = From 65227ecaa40ceba729a978bad981075f978d8d48 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 10:56:40 +0200 Subject: [PATCH 022/111] Add reminder to write test --- core-buffer/test/test_FastQueue.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core-buffer/test/test_FastQueue.cpp b/core-buffer/test/test_FastQueue.cpp index 0d31142..e720c21 100644 --- a/core-buffer/test/test_FastQueue.cpp +++ b/core-buffer/test/test_FastQueue.cpp @@ -145,4 +145,6 @@ TEST(FaseQueue, array_parameter) ASSERT_EQ(module_metadata.n_received_packets, i_module); ASSERT_EQ(module_metadata.module_id, i_module); } -} \ No newline at end of file +} + +// TODO: Test with payload of zero (metadata only). \ No newline at end of file From 2003d9c982e721b3884b926b5fc2107cf0202ed3 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 11:13:55 +0200 Subject: [PATCH 023/111] Definition of BufferBinaryReader --- sf-replay/src/BufferBinaryReader.hpp | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 sf-replay/src/BufferBinaryReader.hpp diff --git a/sf-replay/src/BufferBinaryReader.hpp b/sf-replay/src/BufferBinaryReader.hpp new file mode 100644 index 0000000..0dbfa03 --- /dev/null +++ b/sf-replay/src/BufferBinaryReader.hpp @@ -0,0 +1,28 @@ +#ifndef SF_DAQ_BUFFER_BUFFERBINARYREADER_HPP +#define SF_DAQ_BUFFER_BUFFERBINARYREADER_HPP + + +#include + +class BufferBinaryReader { + + const std::string device_; + const std::string channel_name_; + + std::string current_input_file_; + int input_file_fd_; + + void open_file(const std::string& filename); + void close_current_file(); + +public: + BufferBinaryReader(const std::string &device, + const std::string &channel_name); + + virtual ~BufferBinaryReader(); + + void get_block(const uint64_t block_number, BufferBinaryBlock *buffer); +}; + + +#endif //SF_DAQ_BUFFER_BUFFERBINARYREADER_HPP From 1fc4bd6e44b590abb78a6d681386648ed18508cc Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 11:14:05 +0200 Subject: [PATCH 024/111] Adjust names for block read --- sf-replay/src/main.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/sf-replay/src/main.cpp b/sf-replay/src/main.cpp index fb04ecf..f6242d7 100644 --- a/sf-replay/src/main.cpp +++ b/sf-replay/src/main.cpp @@ -1,8 +1,7 @@ #include #include -#include -#include +#include "FastQueue.hpp" #include "buffer_config.hpp" #include "BufferBinaryReader.hpp" #include "ReplayZmqSender.hpp" @@ -37,9 +36,9 @@ void sf_replay ( auto start_time = steady_clock::now(); - auto metadata = queue.get_metadata_buffer(slot_id); + auto block_buffer = queue.get_metadata_buffer(slot_id); - block_reader.get_block(curr_block, metadata); + block_reader.get_block(curr_block, block_buffer); auto end_time = steady_clock::now(); uint64_t read_us_duration = duration_cast( From 666042c5b7fe1381ec3f5c56db6a8a24ad8b2790 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 11:18:08 +0200 Subject: [PATCH 025/111] Implement boilerplate for BufferBinaryReader.cpp --- sf-replay/src/BufferBinaryReader.cpp | 76 ++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 sf-replay/src/BufferBinaryReader.cpp diff --git a/sf-replay/src/BufferBinaryReader.cpp b/sf-replay/src/BufferBinaryReader.cpp new file mode 100644 index 0000000..15df0d6 --- /dev/null +++ b/sf-replay/src/BufferBinaryReader.cpp @@ -0,0 +1,76 @@ +#include "BufferBinaryReader.hpp" + +#include +#include +#include +#include +#include + +using namespace std; + +BufferBinaryReader::BufferBinaryReader( + const std::string &device, + const std::string &channel_name) : + device_(device), + channel_name_(channel_name), + current_input_file_(""), + input_file_fd_(-1) +{ + +} + +BufferBinaryReader::~BufferBinaryReader() +{ + close_current_file(); +} + +void BufferBinaryReader::get_block( + const uint64_t block_number, BufferBinaryBlock *buffer) +{ + +} + +void BufferBinaryReader::open_file(const std::string& filename) +{ + close_current_file(); + + input_file_fd_ = open(filename.c_str(), O_RDONLY); + + if (input_file_fd_ < 0) { + stringstream err_msg; + + using namespace date; + using namespace chrono; + err_msg << "[" << system_clock::now() << "]"; + err_msg << "[BufferBinaryReader::open_file]"; + err_msg << " Cannot open file "; + err_msg << filename << ": "; + err_msg << strerror(errno) << endl; + + throw runtime_error(err_msg.str()); + } + + current_input_file_ = filename; +} + +void BufferBinaryReader::close_current_file() +{ + if (input_file_fd_ != -1) { + if (close(input_file_fd_) < 0) { + stringstream err_msg; + + using namespace date; + using namespace chrono; + err_msg << "[" << system_clock::now() << "]"; + err_msg << "[BinaryWriter::close_current_file]"; + err_msg << " Error while closing file "; + err_msg << current_input_file_ << ": "; + err_msg << strerror(errno) << endl; + + throw runtime_error(err_msg.str()); + } + + input_file_fd_ = -1; + current_input_file_ = ""; + } +} From ed84e7d0b655c06d28b7bd9c637acf852d949457 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 11:31:37 +0200 Subject: [PATCH 026/111] Implement block read --- sf-replay/src/BufferBinaryReader.cpp | 49 +++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/sf-replay/src/BufferBinaryReader.cpp b/sf-replay/src/BufferBinaryReader.cpp index 15df0d6..a20806c 100644 --- a/sf-replay/src/BufferBinaryReader.cpp +++ b/sf-replay/src/BufferBinaryReader.cpp @@ -6,7 +6,10 @@ #include #include +#include "BufferUtils.hpp" + using namespace std; +using namespace core_buffer; BufferBinaryReader::BufferBinaryReader( const std::string &device, @@ -25,9 +28,53 @@ BufferBinaryReader::~BufferBinaryReader() } void BufferBinaryReader::get_block( - const uint64_t block_number, BufferBinaryBlock *buffer) + const uint64_t block_number, BufferBinaryBlock* buffer) { + uint64_t block_start_pulse_id = block_number * BUFFER_BLOCK_SIZE; + auto current_block_file = BufferUtils::get_filename( + device_, channel_name_, block_start_pulse_id); + if (current_block_file != current_input_file_) { + open_file(current_block_file); + } + + size_t file_start_index = + BufferUtils::get_file_frame_index(block_start_pulse_id); + size_t n_bytes_offset = file_start_index * sizeof(BufferBinaryFormat); + + auto lseek_result = lseek(input_file_fd_, n_bytes_offset, SEEK_SET); + if (lseek_result < 0) { + stringstream err_msg; + + using namespace date; + using namespace chrono; + err_msg << "[" << system_clock::now() << "]"; + err_msg << "[BufferBinaryReader::get_block]"; + err_msg << " Error while lseek on file "; + err_msg << current_input_file_; + err_msg << " for n_bytes_offset "; + err_msg << n_bytes_offset << ": "; + err_msg << strerror(errno) << endl; + + throw runtime_error(err_msg.str()); + } + + auto n_bytes = ::read(input_file_fd_, buffer, + sizeof(BufferBinaryFormat) * BUFFER_BLOCK_SIZE); + + if (n_bytes < sizeof(BufferBinaryFormat)) { + stringstream err_msg; + + using namespace date; + using namespace chrono; + err_msg << "[" << system_clock::now() << "]"; + err_msg << "[BufferBinaryReader::get_block]"; + err_msg << " Error while reading from file "; + err_msg << current_input_file_ << ": "; + err_msg << strerror(errno) << endl; + + throw runtime_error(err_msg.str()); + } } void BufferBinaryReader::open_file(const std::string& filename) From 1f7c292d863b9a37cbd600c6e7c26a8cb4dedea8 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 11:35:08 +0200 Subject: [PATCH 027/111] Rename replay zmq sender to BlockZmqSender --- sf-replay/include/BlockZmqSender.hpp | 23 +++++++++++++++++++ sf-replay/include/ReplayZmqSender.hpp | 23 ------------------- ...ReplayZmqSender.cpp => BlockZmqSender.cpp} | 12 +++++----- sf-replay/src/main.cpp | 4 ++-- 4 files changed, 31 insertions(+), 31 deletions(-) create mode 100644 sf-replay/include/BlockZmqSender.hpp delete mode 100644 sf-replay/include/ReplayZmqSender.hpp rename sf-replay/src/{ReplayZmqSender.cpp => BlockZmqSender.cpp} (79%) diff --git a/sf-replay/include/BlockZmqSender.hpp b/sf-replay/include/BlockZmqSender.hpp new file mode 100644 index 0000000..e4eb813 --- /dev/null +++ b/sf-replay/include/BlockZmqSender.hpp @@ -0,0 +1,23 @@ +#ifndef SF_DAQ_BUFFER_BLOCKZMQSENDER_HPP +#define SF_DAQ_BUFFER_BLOCKZMQSENDER_HPP + +#include +#include +#include + +class BlockZmqSender { + + void* ctx_; + void* socket_; + +public: + BlockZmqSender(const std::string& ipc_id, const int source_id); + virtual ~BlockZmqSender(); + + void close(); + + void send(const BufferBinaryBlock* block_data); +}; + + +#endif //SF_DAQ_BUFFER_BLOCKZMQSENDER_HPP diff --git a/sf-replay/include/ReplayZmqSender.hpp b/sf-replay/include/ReplayZmqSender.hpp deleted file mode 100644 index 5851ad4..0000000 --- a/sf-replay/include/ReplayZmqSender.hpp +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef SF_DAQ_BUFFER_REPLAYZMQSENDER_HPP -#define SF_DAQ_BUFFER_REPLAYZMQSENDER_HPP - -#include -#include -#include - -class ReplayZmqSender { - - void* ctx_; - void* socket_; - -public: - ReplayZmqSender(const std::string& ipc_id, const int source_id); - virtual ~ReplayZmqSender(); - - void close(); - - void send(const BufferBinaryBlock* block_data); -}; - - -#endif //SF_DAQ_BUFFER_REPLAYZMQSENDER_HPP diff --git a/sf-replay/src/ReplayZmqSender.cpp b/sf-replay/src/BlockZmqSender.cpp similarity index 79% rename from sf-replay/src/ReplayZmqSender.cpp rename to sf-replay/src/BlockZmqSender.cpp index f16e028..fc6667f 100644 --- a/sf-replay/src/ReplayZmqSender.cpp +++ b/sf-replay/src/BlockZmqSender.cpp @@ -1,4 +1,4 @@ -#include "ReplayZmqSender.hpp" +#include "BlockZmqSender.hpp" #include #include @@ -9,7 +9,7 @@ using namespace std; using namespace core_buffer; -ReplayZmqSender::ReplayZmqSender(const string& ipc_id, const int source_id) +BlockZmqSender::BlockZmqSender(const string& ipc_id, const int source_id) { auto ipc_base = REPLAY_STREAM_IPC_URL + ipc_id + "-"; stringstream ipc_stream; @@ -31,17 +31,17 @@ ReplayZmqSender::ReplayZmqSender(const string& ipc_id, const int source_id) throw runtime_error(zmq_strerror (errno)); } -ReplayZmqSender::~ReplayZmqSender() +BlockZmqSender::~BlockZmqSender() { close(); } -void ReplayZmqSender::close() { +void BlockZmqSender::close() { zmq_close(socket_); zmq_ctx_destroy(ctx_); } -void ReplayZmqSender::send(const BufferBinaryBlock* block_data) +void BlockZmqSender::send(const BufferBinaryBlock* block_data) { zmq_send(socket_, block_data, sizeof(BufferBinaryBlock), ZMQ_SNDMORE); -} \ No newline at end of file +} diff --git a/sf-replay/src/main.cpp b/sf-replay/src/main.cpp index f6242d7..5b96e46 100644 --- a/sf-replay/src/main.cpp +++ b/sf-replay/src/main.cpp @@ -4,7 +4,7 @@ #include "FastQueue.hpp" #include "buffer_config.hpp" #include "BufferBinaryReader.hpp" -#include "ReplayZmqSender.hpp" +#include "BlockZmqSender.hpp" using namespace std; using namespace core_buffer; @@ -87,7 +87,7 @@ int main (int argc, char *argv[]) { uint64_t max_send_us = 0; uint64_t n_stats = 0; - ReplayZmqSender sender(ipc_id, source_id); + BlockZmqSender sender(ipc_id, source_id); uint64_t start_block = start_pulse_id / BUFFER_BLOCK_SIZE; uint64_t stop_block = stop_pulse_id / BUFFER_BLOCK_SIZE; From 048010255c4bacc078df6b24866f5dba89ccf340 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 12:07:01 +0200 Subject: [PATCH 028/111] Add dependency to replay lib --- sf-writer/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/sf-writer/CMakeLists.txt b/sf-writer/CMakeLists.txt index a66a5b1..d75df61 100644 --- a/sf-writer/CMakeLists.txt +++ b/sf-writer/CMakeLists.txt @@ -11,6 +11,7 @@ add_executable(sf-writer src/main.cpp) set_target_properties(sf-writer PROPERTIES OUTPUT_NAME sf_writer) target_link_libraries(sf-writer sf-writer-lib + sf-replay-lib zmq hdf5 hdf5_cpp From 79ffc144bda6d23093cebe791e2f52ad188ee19c Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 12:11:45 +0200 Subject: [PATCH 029/111] Moved header to correct folder --- sf-replay/{src => include}/BufferBinaryReader.hpp | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename sf-replay/{src => include}/BufferBinaryReader.hpp (100%) diff --git a/sf-replay/src/BufferBinaryReader.hpp b/sf-replay/include/BufferBinaryReader.hpp similarity index 100% rename from sf-replay/src/BufferBinaryReader.hpp rename to sf-replay/include/BufferBinaryReader.hpp From 6995e71543ccbc28991fbb4e1ee854d0734f3b42 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 12:13:08 +0200 Subject: [PATCH 030/111] Exchange implementation --- core-buffer/src/FastQueue.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core-buffer/src/FastQueue.cpp b/core-buffer/src/FastQueue.cpp index c3f9daf..da679e4 100644 --- a/core-buffer/src/FastQueue.cpp +++ b/core-buffer/src/FastQueue.cpp @@ -104,6 +104,6 @@ void FastQueue::release() } template class FastQueue; -template class FastQueue; +template class FastQueue; template class FastQueue; template class FastQueue; From 759803389ecbfbdbce78455cbc288b14254471d8 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 12:21:20 +0200 Subject: [PATCH 031/111] Major writer re-write --- sf-writer/src/main.cpp | 113 +++++++++++++++++++++-------------------- 1 file changed, 58 insertions(+), 55 deletions(-) diff --git a/sf-writer/src/main.cpp b/sf-writer/src/main.cpp index d4da930..92f6e8a 100644 --- a/sf-writer/src/main.cpp +++ b/sf-writer/src/main.cpp @@ -1,63 +1,58 @@ #include #include -#include "buffer_config.hpp" -#include "zmq.h" #include -#include #include #include -#include "JFH5Writer.hpp" -#include #include + #include "date.h" +#include "zmq.h" +#include "jungfrau.hpp" +#include "buffer_config.hpp" #include "bitshuffle/bitshuffle.h" #include "WriterZmqReceiver.hpp" +#include "JFH5Writer.hpp" +#include "BufferBinaryReader.hpp" using namespace std; using namespace core_buffer; +using namespace chrono; -void receive_replay( - void* ctx, - const string ipc_prefix, - const size_t n_modules, - FastQueue& queue, - const uint64_t start_pulse_id, - const uint64_t stop_pulse_id) +void read_buffer( + const string device, + const string channel_name, + const vector& blocks) { - try { - WriterZmqReceiver receiver(ctx, ipc_prefix, n_modules, stop_pulse_id); + BufferBinaryReader block_reader(device, channel_name); - uint64_t current_pulse_id=start_pulse_id; - // "<= stop_pulse_id" because we include the last pulse_id. - while(current_pulse_id<=stop_pulse_id) { - int slot_id; - while((slot_id = queue.reserve()) == -1) { - this_thread::sleep_for(chrono::milliseconds( - RB_READ_RETRY_INTERVAL_MS)); - } + // "<= stop_block" because we include the stop_block in the transfer. + for (uint64_t curr_block=start_block; + curr_block <= stop_block; + curr_block++) { - auto metadata = queue.get_metadata_buffer(slot_id); - auto buffer = queue.get_data_buffer(slot_id); - - receiver.get_next_buffer( - current_pulse_id, metadata, buffer); - - queue.commit(); - current_pulse_id += metadata->n_images; + int slot_id; + while((slot_id = queue.reserve()) == -1) { + this_thread::sleep_for(chrono::milliseconds( + RB_READ_RETRY_INTERVAL_MS)); } - } catch (const std::exception& e) { - using namespace date; - using namespace chrono; + auto start_time = steady_clock::now(); - cout << "[" << system_clock::now() << "]"; - cout << "[sf_writer::receive_replay]"; - cout << " Stopped because of exception: " << endl; - cout << e.what() << endl; + auto block_buffer = queue.get_metadata_buffer(slot_id); - throw; + block_reader.get_block(curr_block, block_buffer); + + auto end_time = steady_clock::now(); + uint64_t read_us_duration = duration_cast( + end_time-start_time).count(); + + queue.commit(); + + // TODO: Proper statistics + cout << "sf_replay:avg_read_us "; + cout << read_us_duration / BUFFER_BLOCK_SIZE << endl; } } @@ -65,11 +60,11 @@ int main (int argc, char *argv[]) { if (argc != 5) { cout << endl; - cout << "Usage: sf_writer "; - cout << " [ipc_id] [output_file] [start_pulse_id] [stop_pulse_id]"; + cout << "Usage: sf_writer [output_file] [device]"; + cout << " [start_pulse_id] [stop_pulse_id]"; cout << endl; - cout << "\tipc_id: Unique identifier for ipc." << endl; cout << "\toutput_file: Complete path to the output file." << endl; + cout << "\tdevice: Name of detector." << endl; cout << "\tstart_pulse_id: Start pulse_id of retrieval." << endl; cout << "\tstop_pulse_id: Stop pulse_id of retrieval." << endl; cout << endl; @@ -77,24 +72,32 @@ int main (int argc, char *argv[]) exit(-1); } - const string ipc_id = string(argv[1]); - string output_file = string(argv[2]); - uint64_t start_pulse_id = (uint64_t) atoll(argv[3]); - uint64_t stop_pulse_id = (uint64_t) atoll(argv[4]); - + string output_file = string(argv[1]); + const string device = string(argv[2]); + uint64_t start_pulse_id = (uint64_t) atoll(argv[4]); + uint64_t stop_pulse_id = (uint64_t) atoll(argv[5]); size_t n_modules = 32; - FastQueue queue( - MODULE_N_BYTES * n_modules * WRITER_DATA_CACHE_N_IMAGES, - WRITER_FASTQUEUE_N_SLOTS); + uint64_t start_block = start_pulse_id / BUFFER_BLOCK_SIZE; + uint64_t stop_block = stop_pulse_id / BUFFER_BLOCK_SIZE; + auto n_blocks = stop_block - start_block + 1; - auto ctx = zmq_ctx_new(); - zmq_ctx_set (ctx, ZMQ_IO_THREADS, WRITER_ZMQ_IO_THREADS); + // Generate list of buffer blocks that need to be loaded. + std::vector buffer_blocks(n_blocks); + for (uint64_t curr_block=start_block; + curr_block<=stop_block; + curr_block++) { + buffer_blocks.push_back(curr_block); + } - auto ipc_base = REPLAY_STREAM_IPC_URL + ipc_id + "-"; - thread replay_receive_thread(receive_replay, - ctx, ipc_base, n_modules, - ref(queue), start_pulse_id, stop_pulse_id); + std::vector reading_threads(n_modules); + for (size_t i_module=0; i_module Date: Wed, 27 May 2020 12:24:14 +0200 Subject: [PATCH 032/111] Cleanup threads after yourself --- sf-writer/src/main.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/sf-writer/src/main.cpp b/sf-writer/src/main.cpp index 92f6e8a..8a1c6f8 100644 --- a/sf-writer/src/main.cpp +++ b/sf-writer/src/main.cpp @@ -150,7 +150,11 @@ int main (int argc, char *argv[]) writer.close_file(); - //wait till receive thread is finished - replay_receive_thread.join(); + for (auto& reading_thread :reading_threads) { + if (reading_thread.joinable()) { + reading_thread.join(); + } + } + return 0; } From b868059dbcb812ad2b1c6c6dd6cfce1f81270c45 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 12:39:00 +0200 Subject: [PATCH 033/111] Implement reading stub --- sf-writer/src/main.cpp | 62 +++++++++++++----------------------------- 1 file changed, 19 insertions(+), 43 deletions(-) diff --git a/sf-writer/src/main.cpp b/sf-writer/src/main.cpp index 8a1c6f8..3c8db4c 100644 --- a/sf-writer/src/main.cpp +++ b/sf-writer/src/main.cpp @@ -21,39 +21,36 @@ using namespace chrono; void read_buffer( const string device, const string channel_name, - const vector& blocks) + const vector& buffer_blocks) { BufferBinaryReader block_reader(device, channel_name); + auto block_buffer = new BufferBinaryBlock(); - - - // "<= stop_block" because we include the stop_block in the transfer. - for (uint64_t curr_block=start_block; - curr_block <= stop_block; - curr_block++) { - - int slot_id; - while((slot_id = queue.reserve()) == -1) { - this_thread::sleep_for(chrono::milliseconds( - RB_READ_RETRY_INTERVAL_MS)); - } - + for (uint64_t block_number:buffer_blocks) { auto start_time = steady_clock::now(); - auto block_buffer = queue.get_metadata_buffer(slot_id); - - block_reader.get_block(curr_block, block_buffer); + block_reader.get_block(block_number, block_buffer); auto end_time = steady_clock::now(); uint64_t read_us_duration = duration_cast( end_time-start_time).count(); - queue.commit(); + start_time = steady_clock::now(); + + // TODO: Send to composition. + + end_time = steady_clock::now(); + uint64_t compose_us_duration = duration_cast( + end_time-start_time).count(); // TODO: Proper statistics cout << "sf_replay:avg_read_us "; cout << read_us_duration / BUFFER_BLOCK_SIZE << endl; + cout << "sf_replay:avg_compose_us "; + cout << compose_us_duration / BUFFER_BLOCK_SIZE << endl; } + + delete block_buffer; } int main (int argc, char *argv[]) @@ -92,9 +89,7 @@ int main (int argc, char *argv[]) std::vector reading_threads(n_modules); for (size_t i_module=0; i_modulen_images; i++) { - if (metadata->pulse_id[i] != current_pulse_id) { - throw runtime_error("Wrong pulse id from receiver thread."); - } - - current_pulse_id++; - } - - auto end_time = chrono::steady_clock::now(); - auto read_us_duration = chrono::duration_cast( - end_time-start_time).count(); - - start_time = chrono::steady_clock::now(); + auto start_time = steady_clock::now(); writer.write(metadata, data); - end_time = chrono::steady_clock::now(); + auto end_time = steady_clock::now(); auto write_us_duration = chrono::duration_cast( end_time-start_time).count(); queue.release(); - auto avg_read_us = read_us_duration / metadata->n_images;; - auto avg_write_us = write_us_duration / metadata->n_images;; - - cout << "sf_writer:avg_read_us " << avg_read_us; - cout << " sf_writer:avg_write_us " << avg_write_us; - cout << endl; + cout << "sf_writer:avg_write_us "; + cout << write_us_duration / BUFFER_BLOCK_SIZE << endl; } writer.close_file(); From 37d4d0d772c227157173bb7140b5ac4915e76c35 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 12:44:26 +0200 Subject: [PATCH 034/111] Add stub class for image assembly --- sf-writer/src/ImageAssembler.cpp | 1 + sf-writer/src/ImageAssembler.hpp | 10 ++++++++++ 2 files changed, 11 insertions(+) create mode 100644 sf-writer/src/ImageAssembler.cpp create mode 100644 sf-writer/src/ImageAssembler.hpp diff --git a/sf-writer/src/ImageAssembler.cpp b/sf-writer/src/ImageAssembler.cpp new file mode 100644 index 0000000..58049fc --- /dev/null +++ b/sf-writer/src/ImageAssembler.cpp @@ -0,0 +1 @@ +#include "ImageAssembler.hpp" diff --git a/sf-writer/src/ImageAssembler.hpp b/sf-writer/src/ImageAssembler.hpp new file mode 100644 index 0000000..5abb76a --- /dev/null +++ b/sf-writer/src/ImageAssembler.hpp @@ -0,0 +1,10 @@ +#ifndef SF_DAQ_BUFFER_IMAGEASSEMBLER_HPP +#define SF_DAQ_BUFFER_IMAGEASSEMBLER_HPP + + +class ImageAssembler { + +}; + + +#endif //SF_DAQ_BUFFER_IMAGEASSEMBLER_HPP From 3d98a9e9087c6b7d319e1f434c3a4b4ed9a26bdc Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 13:07:04 +0200 Subject: [PATCH 035/111] Add start and stop pulse id to JF writer --- sf-writer/include/JFH5Writer.hpp | 7 +++++-- sf-writer/src/JFH5Writer.cpp | 12 +++++++----- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/sf-writer/include/JFH5Writer.hpp b/sf-writer/include/JFH5Writer.hpp index 4e96e9a..b8c559c 100644 --- a/sf-writer/include/JFH5Writer.hpp +++ b/sf-writer/include/JFH5Writer.hpp @@ -9,8 +9,10 @@ class JFH5Writer { - const size_t n_images_; + const size_t start_pulse_id_; + const uint64_t stop_pulse_id_; const size_t n_modules_; + const size_t n_images_; size_t current_write_index_; H5::H5File file_; @@ -24,7 +26,8 @@ class JFH5Writer { public: JFH5Writer(const std::string& output_file, - const size_t n_images, + const uint64_t start_pulse_id, + const uint64_t stop_pulse_id, const size_t n_modules); ~JFH5Writer(); void write(const ImageMetadataBlock* metadata, const char* data); diff --git a/sf-writer/src/JFH5Writer.cpp b/sf-writer/src/JFH5Writer.cpp index 1abd2fb..5b1b5a7 100644 --- a/sf-writer/src/JFH5Writer.cpp +++ b/sf-writer/src/JFH5Writer.cpp @@ -12,12 +12,14 @@ using namespace std; using namespace core_buffer; -JFH5Writer::JFH5Writer( - const string& output_file, - const size_t n_images, - const size_t n_modules) : - n_images_(n_images), +JFH5Writer::JFH5Writer(const std::string& output_file, + const uint64_t start_pulse_id, + const uint64_t stop_pulse_id, + const size_t n_modules) : + start_pulse_id_(start_pulse_id), + stop_pulse_id_(stop_pulse_id), n_modules_(n_modules), + n_images_(stop_pulse_id - start_pulse_id + 1), current_write_index_(0) { From ba383f11f6d574c98505c66294e53b1378402cfa Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 13:07:16 +0200 Subject: [PATCH 036/111] Improve ImageAssembler interface --- sf-writer/src/ImageAssembler.hpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sf-writer/src/ImageAssembler.hpp b/sf-writer/src/ImageAssembler.hpp index 5abb76a..60f0088 100644 --- a/sf-writer/src/ImageAssembler.hpp +++ b/sf-writer/src/ImageAssembler.hpp @@ -3,6 +3,16 @@ class ImageAssembler { +public: + void process(const int slot_id, + const int i_module, + const BufferBinaryBlock* block_buffer); + int get_free_slot(); + int get_full_slot(); + void free_slot(int slot_id); + + const ImageMetadataBlock* get_metadata_buffer(int slot_id); + const char* get_data_buffer(int slot_id); }; From 7f6985f538b9a2f6a42cc5a76603d04fdaaec4c2 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 13:08:21 +0200 Subject: [PATCH 037/111] Improve ImageAssembler workflow --- sf-writer/src/main.cpp | 44 ++++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/sf-writer/src/main.cpp b/sf-writer/src/main.cpp index 3c8db4c..d5b0031 100644 --- a/sf-writer/src/main.cpp +++ b/sf-writer/src/main.cpp @@ -1,5 +1,4 @@ #include -#include #include #include #include @@ -7,12 +6,11 @@ #include "date.h" #include "zmq.h" -#include "jungfrau.hpp" #include "buffer_config.hpp" #include "bitshuffle/bitshuffle.h" -#include "WriterZmqReceiver.hpp" #include "JFH5Writer.hpp" #include "BufferBinaryReader.hpp" +#include "ImageAssembler.hpp" using namespace std; using namespace core_buffer; @@ -21,12 +19,21 @@ using namespace chrono; void read_buffer( const string device, const string channel_name, - const vector& buffer_blocks) + const int i_module, + const vector& buffer_blocks, + ImageAssembler& image_assembler) { BufferBinaryReader block_reader(device, channel_name); auto block_buffer = new BufferBinaryBlock(); for (uint64_t block_number:buffer_blocks) { + + int slot_id; + while((slot_id = image_assembler.get_free_slot()) == -1) { + this_thread::sleep_for(chrono::milliseconds( + RB_READ_RETRY_INTERVAL_MS)); + } + auto start_time = steady_clock::now(); block_reader.get_block(block_number, block_buffer); @@ -37,16 +44,15 @@ void read_buffer( start_time = steady_clock::now(); - // TODO: Send to composition. + image_assembler.process(slot_id, i_module, block_buffer); end_time = steady_clock::now(); uint64_t compose_us_duration = duration_cast( end_time-start_time).count(); - // TODO: Proper statistics cout << "sf_replay:avg_read_us "; cout << read_us_duration / BUFFER_BLOCK_SIZE << endl; - cout << "sf_replay:avg_compose_us "; + cout << "sf_replay:avg_assemble_us "; cout << compose_us_duration / BUFFER_BLOCK_SIZE << endl; } @@ -79,6 +85,8 @@ int main (int argc, char *argv[]) uint64_t stop_block = stop_pulse_id / BUFFER_BLOCK_SIZE; auto n_blocks = stop_block - start_block + 1; + ImageAssembler image_assembler; + // Generate list of buffer blocks that need to be loaded. std::vector buffer_blocks(n_blocks); for (uint64_t curr_block=start_block; @@ -91,24 +99,26 @@ int main (int argc, char *argv[]) for (size_t i_module=0; i_module( end_time-start_time).count(); - queue.release(); + image_assembler.free_slot(slot_id); cout << "sf_writer:avg_write_us "; cout << write_us_duration / BUFFER_BLOCK_SIZE << endl; From ea5473846c764cfb50cc6d7f5678931f92442048 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 13:11:16 +0200 Subject: [PATCH 038/111] Add TODO for later implementation --- sf-writer/src/JFH5Writer.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sf-writer/src/JFH5Writer.cpp b/sf-writer/src/JFH5Writer.cpp index 5b1b5a7..3d0b7d5 100644 --- a/sf-writer/src/JFH5Writer.cpp +++ b/sf-writer/src/JFH5Writer.cpp @@ -118,6 +118,8 @@ void JFH5Writer::close_file() void JFH5Writer::write( const ImageMetadataBlock* metadata, const char* data) { + // TODO: Implement proper block offsetting. + size_t n_images_offset = 0; size_t n_images_to_copy = min(n_images_ - current_write_index_, BUFFER_BLOCK_SIZE); From 468176adba13394bc11397be6b01b603294f2366 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 13:43:19 +0200 Subject: [PATCH 039/111] Add block verification to writer --- core-buffer/include/formats.hpp | 3 ++- sf-writer/include/JFH5Writer.hpp | 2 +- sf-writer/src/JFH5Writer.cpp | 17 +++++++++++++---- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/core-buffer/include/formats.hpp b/core-buffer/include/formats.hpp index 80bea40..72222b0 100644 --- a/core-buffer/include/formats.hpp +++ b/core-buffer/include/formats.hpp @@ -10,7 +10,8 @@ struct ImageMetadataBlock uint64_t frame_index[core_buffer::BUFFER_BLOCK_SIZE]; uint32_t daq_rec[core_buffer::BUFFER_BLOCK_SIZE]; uint8_t is_good_image[core_buffer::BUFFER_BLOCK_SIZE]; - uint64_t start_pulse_id; + uint64_t block_first_pulse_id; + uint64_t block_last_pulse_id; }; const char BUFFER_FORMAT_START_BYTE = 0xBE; diff --git a/sf-writer/include/JFH5Writer.hpp b/sf-writer/include/JFH5Writer.hpp index b8c559c..6d51341 100644 --- a/sf-writer/include/JFH5Writer.hpp +++ b/sf-writer/include/JFH5Writer.hpp @@ -9,7 +9,7 @@ class JFH5Writer { - const size_t start_pulse_id_; + const uint64_t start_pulse_id_; const uint64_t stop_pulse_id_; const size_t n_modules_; const size_t n_images_; diff --git a/sf-writer/src/JFH5Writer.cpp b/sf-writer/src/JFH5Writer.cpp index 3d0b7d5..dcff400 100644 --- a/sf-writer/src/JFH5Writer.cpp +++ b/sf-writer/src/JFH5Writer.cpp @@ -118,13 +118,22 @@ void JFH5Writer::close_file() void JFH5Writer::write( const ImageMetadataBlock* metadata, const char* data) { - // TODO: Implement proper block offsetting. size_t n_images_offset = 0; - size_t n_images_to_copy = min(n_images_ - current_write_index_, - BUFFER_BLOCK_SIZE); + if (start_pulse_id_ > metadata->block_first_pulse_id) { + n_images_offset = start_pulse_id_ - metadata->block_first_pulse_id; + } + + if (n_images_offset > BUFFER_BLOCK_SIZE) { + throw runtime_error("Received unexpected block for start_pulse_id."); + } + + size_t n_images_to_copy = BUFFER_BLOCK_SIZE - n_images_offset; + if (stop_pulse_id_ < metadata->block_last_pulse_id) { + n_images_to_copy -= metadata->block_last_pulse_id - stop_pulse_id_; + } if (n_images_to_copy < 1) { - return; + throw runtime_error("Received unexpected block for stop_pulse_id."); } hsize_t b_i_dims[3] = {n_images_to_copy, From f850fd2ab6bfc72bd25b59fcb4dd3258a8470800 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 13:52:40 +0200 Subject: [PATCH 040/111] Implement offset for images --- sf-writer/src/JFH5Writer.cpp | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/sf-writer/src/JFH5Writer.cpp b/sf-writer/src/JFH5Writer.cpp index dcff400..c2407fa 100644 --- a/sf-writer/src/JFH5Writer.cpp +++ b/sf-writer/src/JFH5Writer.cpp @@ -136,21 +136,25 @@ void JFH5Writer::write( throw runtime_error("Received unexpected block for stop_pulse_id."); } - hsize_t b_i_dims[3] = {n_images_to_copy, + hsize_t b_i_dims[3] = {BUFFER_BLOCK_SIZE, MODULE_Y_SIZE * n_modules_, MODULE_X_SIZE}; H5::DataSpace b_i_space(3, b_i_dims); + hsize_t b_i_count[] = {n_images_to_copy, + MODULE_Y_SIZE * n_modules_, + MODULE_X_SIZE}; + hsize_t b_i_start[] = {n_images_offset, 0, 0}; + b_i_space.selectHyperslab(H5S_SELECT_SET, b_i_count, b_i_start); hsize_t f_i_dims[3] = {n_images_, MODULE_Y_SIZE * n_modules_, MODULE_X_SIZE}; H5::DataSpace f_i_space(3, f_i_dims); - - hsize_t i_count[] = {n_images_to_copy, - MODULE_Y_SIZE * n_modules_, - MODULE_X_SIZE}; - hsize_t i_start[] = {current_write_index_, 0, 0}; - f_i_space.selectHyperslab(H5S_SELECT_SET, i_count, i_start); + hsize_t f_i_count[] = {n_images_to_copy, + MODULE_Y_SIZE * n_modules_, + MODULE_X_SIZE}; + hsize_t f_i_start[] = {current_write_index_, 0, 0}; + f_i_space.selectHyperslab(H5S_SELECT_SET, f_i_count, f_i_start); image_dataset_.write( data, H5::PredType::NATIVE_UINT16, b_i_space, f_i_space); From a1c93430e33ef1059937b66a211770b49e6393f5 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 13:59:14 +0200 Subject: [PATCH 041/111] Add offset also to metadata --- sf-writer/src/JFH5Writer.cpp | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/sf-writer/src/JFH5Writer.cpp b/sf-writer/src/JFH5Writer.cpp index c2407fa..d2a5231 100644 --- a/sf-writer/src/JFH5Writer.cpp +++ b/sf-writer/src/JFH5Writer.cpp @@ -162,40 +162,32 @@ void JFH5Writer::write( // pulse_id { auto b_current_ptr = b_pulse_id_ + current_write_index_; - const uint64_t* metadata_pulse_id_ptr = - &(metadata->pulse_id[0]) + current_write_index_; memcpy(b_current_ptr, - metadata_pulse_id_ptr, + &(metadata->pulse_id[n_images_offset]), sizeof(uint64_t) * n_images_to_copy); } // frame_index { auto b_current_ptr = b_frame_index_ + current_write_index_; - const uint64_t* metadata_pulse_id_ptr = - &(metadata->frame_index[0]) + current_write_index_; memcpy(b_current_ptr, - metadata_pulse_id_ptr, + &(metadata->frame_index[n_images_offset]), sizeof(uint64_t) * n_images_to_copy); } // daq_rec { auto b_current_ptr = b_daq_rec_ + current_write_index_; - const uint32_t* metadata_pulse_id_ptr = - &(metadata->daq_rec[0]) + current_write_index_; memcpy(b_current_ptr, - metadata_pulse_id_ptr, + &(metadata->daq_rec[n_images_offset]), sizeof(uint32_t) * n_images_to_copy); } // is_good_frame { auto b_current_ptr = b_is_good_frame_ + current_write_index_; - const uint8_t* metadata_pulse_id_ptr = - &(metadata->is_good_image[0]) + current_write_index_; memcpy(b_current_ptr, - metadata_pulse_id_ptr, + &(metadata->is_good_image[n_images_offset]), sizeof(uint8_t) * n_images_to_copy); } From b7381bce68fc18561e6dd128b53aaaace796bb63 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 14:01:59 +0200 Subject: [PATCH 042/111] Use proper attribute for ImageAssembler retry --- core-buffer/include/buffer_config.hpp | 4 ++-- sf-writer/src/main.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core-buffer/include/buffer_config.hpp b/core-buffer/include/buffer_config.hpp index b605a79..1862b05 100644 --- a/core-buffer/include/buffer_config.hpp +++ b/core-buffer/include/buffer_config.hpp @@ -63,8 +63,8 @@ namespace core_buffer { const int WRITER_RCVHWM = 100; // ZMQ threads for receiving data from sf_replay. const int WRITER_ZMQ_IO_THREADS = 2; - // Size of buffer between the receiving and writing part. - const int WRITER_FASTQUEUE_N_SLOTS = 5; + // MS to retry reading from the image assembler. + const size_t WRITER_IMAGE_ASSEMBLER_RETRY_MS = 5; // How large are metadata chunks in the HDF5. const size_t WRITER_METADATA_CHUNK_N_IMAGES = 100; // How large should the data cache be in N images. diff --git a/sf-writer/src/main.cpp b/sf-writer/src/main.cpp index d5b0031..4784e71 100644 --- a/sf-writer/src/main.cpp +++ b/sf-writer/src/main.cpp @@ -31,7 +31,7 @@ void read_buffer( int slot_id; while((slot_id = image_assembler.get_free_slot()) == -1) { this_thread::sleep_for(chrono::milliseconds( - RB_READ_RETRY_INTERVAL_MS)); + WRITER_IMAGE_ASSEMBLER_RETRY_MS)); } auto start_time = steady_clock::now(); @@ -114,7 +114,7 @@ int main (int argc, char *argv[]) int slot_id; while((slot_id = image_assembler.get_full_slot()) == -1) { this_thread::sleep_for(chrono::milliseconds( - RB_READ_RETRY_INTERVAL_MS)); + WRITER_IMAGE_ASSEMBLER_RETRY_MS)); } auto metadata = image_assembler.get_metadata_buffer(slot_id); From 719b3ed0f73272bf0d7328fa1d3e0e5c73105171 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 14:03:43 +0200 Subject: [PATCH 043/111] Rename block_number to block_id --- sf-replay/include/BufferBinaryReader.hpp | 2 +- sf-replay/src/BufferBinaryReader.cpp | 4 ++-- sf-writer/src/main.cpp | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sf-replay/include/BufferBinaryReader.hpp b/sf-replay/include/BufferBinaryReader.hpp index 0dbfa03..cca17fc 100644 --- a/sf-replay/include/BufferBinaryReader.hpp +++ b/sf-replay/include/BufferBinaryReader.hpp @@ -21,7 +21,7 @@ public: virtual ~BufferBinaryReader(); - void get_block(const uint64_t block_number, BufferBinaryBlock *buffer); + void get_block(const uint64_t block_id, BufferBinaryBlock *buffer); }; diff --git a/sf-replay/src/BufferBinaryReader.cpp b/sf-replay/src/BufferBinaryReader.cpp index a20806c..0c1eeee 100644 --- a/sf-replay/src/BufferBinaryReader.cpp +++ b/sf-replay/src/BufferBinaryReader.cpp @@ -28,9 +28,9 @@ BufferBinaryReader::~BufferBinaryReader() } void BufferBinaryReader::get_block( - const uint64_t block_number, BufferBinaryBlock* buffer) + const uint64_t block_id, BufferBinaryBlock* buffer) { - uint64_t block_start_pulse_id = block_number * BUFFER_BLOCK_SIZE; + uint64_t block_start_pulse_id = block_id * BUFFER_BLOCK_SIZE; auto current_block_file = BufferUtils::get_filename( device_, channel_name_, block_start_pulse_id); diff --git a/sf-writer/src/main.cpp b/sf-writer/src/main.cpp index 4784e71..4b64d77 100644 --- a/sf-writer/src/main.cpp +++ b/sf-writer/src/main.cpp @@ -26,7 +26,7 @@ void read_buffer( BufferBinaryReader block_reader(device, channel_name); auto block_buffer = new BufferBinaryBlock(); - for (uint64_t block_number:buffer_blocks) { + for (uint64_t block_id:buffer_blocks) { int slot_id; while((slot_id = image_assembler.get_free_slot()) == -1) { @@ -36,7 +36,7 @@ void read_buffer( auto start_time = steady_clock::now(); - block_reader.get_block(block_number, block_buffer); + block_reader.get_block(block_id, block_buffer); auto end_time = steady_clock::now(); uint64_t read_us_duration = duration_cast( @@ -109,7 +109,7 @@ int main (int argc, char *argv[]) JFH5Writer writer(output_file, start_pulse_id, stop_pulse_id, n_modules); - for (uint64_t block_number:buffer_blocks) { + for (uint64_t block_id:buffer_blocks) { int slot_id; while((slot_id = image_assembler.get_full_slot()) == -1) { From 5a7efaf1d0857d4bf51ff83eff026e33fe18e2e2 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 14:11:08 +0200 Subject: [PATCH 044/111] Improve interface of ImageAssembler --- sf-writer/src/ImageAssembler.hpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sf-writer/src/ImageAssembler.hpp b/sf-writer/src/ImageAssembler.hpp index 60f0088..9834ba7 100644 --- a/sf-writer/src/ImageAssembler.hpp +++ b/sf-writer/src/ImageAssembler.hpp @@ -4,16 +4,19 @@ class ImageAssembler { public: + ImageAssembler(); + ImageMetadataBlock* get_metadata_buffer(const int slot_id); + virtual ~ImageAssembler(); + void process(const int slot_id, const int i_module, const BufferBinaryBlock* block_buffer); int get_free_slot(); int get_full_slot(); - void free_slot(int slot_id); + void free_slot(const int slot_id); - const ImageMetadataBlock* get_metadata_buffer(int slot_id); - const char* get_data_buffer(int slot_id); + const char* get_data_buffer(const int slot_id); }; From 75c17247407288d39b92abb5709fef9ed97b3698 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 14:11:21 +0200 Subject: [PATCH 045/111] Add empty stub implementation of ImageAssembler --- sf-writer/src/ImageAssembler.cpp | 43 ++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/sf-writer/src/ImageAssembler.cpp b/sf-writer/src/ImageAssembler.cpp index 58049fc..f6cc69a 100644 --- a/sf-writer/src/ImageAssembler.cpp +++ b/sf-writer/src/ImageAssembler.cpp @@ -1 +1,44 @@ #include "ImageAssembler.hpp" + +ImageAssembler::ImageAssembler() +{ + +} + +ImageAssembler::~ImageAssembler() +{ + +} + +void ImageAssembler::process( + const int slot_id, + const int i_module, + const BufferBinaryBlock* block_buffer) +{ + +} + +void ImageAssembler::get_free_slot() +{ + +} + +void ImageAssembler::get_full_slot() +{ + +} + +void ImageAssembler::free_slot(const int slot_id) +{ + +} + +char* ImageAssembler::get_data_buffer(const int slot_id) +{ + +} + +ImageMetadataBlock* ImageAssembler::get_metadata_buffer(const int slot_id) +{ + +} \ No newline at end of file From d3ebb5b9d541a51d1a647be58cb6ff8d90cd6125 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 14:19:03 +0200 Subject: [PATCH 046/111] Construction initialization --- sf-writer/src/ImageAssembler.cpp | 12 +++++++++--- sf-writer/src/ImageAssembler.hpp | 10 +++++++++- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/sf-writer/src/ImageAssembler.cpp b/sf-writer/src/ImageAssembler.cpp index f6cc69a..d5d1e9d 100644 --- a/sf-writer/src/ImageAssembler.cpp +++ b/sf-writer/src/ImageAssembler.cpp @@ -1,13 +1,19 @@ #include "ImageAssembler.hpp" -ImageAssembler::ImageAssembler() -{ +using namespace std; +using namespace core_buffer; +ImageAssembler::ImageAssembler(const size_t n_modules) : + n_modules_(n_modules) +{ + image_buffer_ = new char[IA_N_SLOTS * MODULE_N_BYTES * n_modules_]; + metadata_buffer_ = new ImageMetadataBlock[IA_N_SLOTS]; } ImageAssembler::~ImageAssembler() { - + delete[] image_buffer_; + delete[] metadata_buffer_; } void ImageAssembler::process( diff --git a/sf-writer/src/ImageAssembler.hpp b/sf-writer/src/ImageAssembler.hpp index 9834ba7..845bcfb 100644 --- a/sf-writer/src/ImageAssembler.hpp +++ b/sf-writer/src/ImageAssembler.hpp @@ -1,10 +1,18 @@ #ifndef SF_DAQ_BUFFER_IMAGEASSEMBLER_HPP #define SF_DAQ_BUFFER_IMAGEASSEMBLER_HPP +#include "formats.hpp" + +const size_t IA_N_SLOTS = 2; class ImageAssembler { + const size_t n_modules_; + + char* image_buffer_; + ImageMetadataBlock* metadata_buffer_; + public: - ImageAssembler(); + ImageAssembler(const size_t n_modules); ImageMetadataBlock* get_metadata_buffer(const int slot_id); virtual ~ImageAssembler(); From 29fc304f02ee08b2133742791433dc3adedbbce3 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 14:20:20 +0200 Subject: [PATCH 047/111] Clean ImageAssembler --- sf-writer/src/ImageAssembler.cpp | 10 ---------- sf-writer/src/ImageAssembler.hpp | 9 +++++---- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/sf-writer/src/ImageAssembler.cpp b/sf-writer/src/ImageAssembler.cpp index d5d1e9d..f654fd4 100644 --- a/sf-writer/src/ImageAssembler.cpp +++ b/sf-writer/src/ImageAssembler.cpp @@ -24,16 +24,6 @@ void ImageAssembler::process( } -void ImageAssembler::get_free_slot() -{ - -} - -void ImageAssembler::get_full_slot() -{ - -} - void ImageAssembler::free_slot(const int slot_id) { diff --git a/sf-writer/src/ImageAssembler.hpp b/sf-writer/src/ImageAssembler.hpp index 845bcfb..d2c4540 100644 --- a/sf-writer/src/ImageAssembler.hpp +++ b/sf-writer/src/ImageAssembler.hpp @@ -13,18 +13,19 @@ class ImageAssembler { public: ImageAssembler(const size_t n_modules); - ImageMetadataBlock* get_metadata_buffer(const int slot_id); + virtual ~ImageAssembler(); + int get_free_slot(); void process(const int slot_id, const int i_module, const BufferBinaryBlock* block_buffer); - int get_free_slot(); + int get_full_slot(); void free_slot(const int slot_id); - - const char* get_data_buffer(const int slot_id); + ImageMetadataBlock* get_metadata_buffer(const int slot_id); + char* get_data_buffer(const int slot_id); }; From 7d619aa90e3ccd48ff0da16f740fed35bd09cb5f Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 14:24:21 +0200 Subject: [PATCH 048/111] Add buffer retrieval methods --- sf-writer/src/ImageAssembler.cpp | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/sf-writer/src/ImageAssembler.cpp b/sf-writer/src/ImageAssembler.cpp index f654fd4..3d9b6c2 100644 --- a/sf-writer/src/ImageAssembler.cpp +++ b/sf-writer/src/ImageAssembler.cpp @@ -16,6 +16,11 @@ ImageAssembler::~ImageAssembler() delete[] metadata_buffer_; } +int ImageAssembler::get_free_slot() +{ + +} + void ImageAssembler::process( const int slot_id, const int i_module, @@ -24,17 +29,22 @@ void ImageAssembler::process( } +int ImageAssembler::get_full_slot() +{ + +} + void ImageAssembler::free_slot(const int slot_id) { } -char* ImageAssembler::get_data_buffer(const int slot_id) -{ - -} - ImageMetadataBlock* ImageAssembler::get_metadata_buffer(const int slot_id) { + return &(metadata_buffer_[slot_id]); +} -} \ No newline at end of file +char* ImageAssembler::get_data_buffer(const int slot_id) +{ + return image_buffer_ + (slot_id * MODULE_N_BYTES * n_modules_); +} From a0be1a501015481b781324059cbb0040877bd639 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 14:28:50 +0200 Subject: [PATCH 049/111] Correct calculation of buffer size --- sf-writer/src/ImageAssembler.cpp | 7 ++++--- sf-writer/src/ImageAssembler.hpp | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/sf-writer/src/ImageAssembler.cpp b/sf-writer/src/ImageAssembler.cpp index 3d9b6c2..329017f 100644 --- a/sf-writer/src/ImageAssembler.cpp +++ b/sf-writer/src/ImageAssembler.cpp @@ -4,9 +4,10 @@ using namespace std; using namespace core_buffer; ImageAssembler::ImageAssembler(const size_t n_modules) : - n_modules_(n_modules) + n_modules_(n_modules), + image_buffer_slot_n_bytes_(BUFFER_BLOCK_SIZE * MODULE_N_BYTES * n_modules_) { - image_buffer_ = new char[IA_N_SLOTS * MODULE_N_BYTES * n_modules_]; + image_buffer_ = new char[IA_N_SLOTS * image_buffer_slot_n_bytes_]; metadata_buffer_ = new ImageMetadataBlock[IA_N_SLOTS]; } @@ -46,5 +47,5 @@ ImageMetadataBlock* ImageAssembler::get_metadata_buffer(const int slot_id) char* ImageAssembler::get_data_buffer(const int slot_id) { - return image_buffer_ + (slot_id * MODULE_N_BYTES * n_modules_); + return image_buffer_ + (slot_id * image_buffer_slot_n_bytes_); } diff --git a/sf-writer/src/ImageAssembler.hpp b/sf-writer/src/ImageAssembler.hpp index d2c4540..a34cb59 100644 --- a/sf-writer/src/ImageAssembler.hpp +++ b/sf-writer/src/ImageAssembler.hpp @@ -7,6 +7,7 @@ const size_t IA_N_SLOTS = 2; class ImageAssembler { const size_t n_modules_; + const size_t image_buffer_slot_n_bytes_; char* image_buffer_; ImageMetadataBlock* metadata_buffer_; From 774aeaa7091bcefb95225f084bf5d3ae1f00f38d Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 14:40:20 +0200 Subject: [PATCH 050/111] Add image assembly memcpy --- sf-writer/src/ImageAssembler.cpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/sf-writer/src/ImageAssembler.cpp b/sf-writer/src/ImageAssembler.cpp index 329017f..32515ac 100644 --- a/sf-writer/src/ImageAssembler.cpp +++ b/sf-writer/src/ImageAssembler.cpp @@ -1,3 +1,4 @@ +#include #include "ImageAssembler.hpp" using namespace std; @@ -27,7 +28,19 @@ void ImageAssembler::process( const int i_module, const BufferBinaryBlock* block_buffer) { + size_t slot_offset = slot_id * image_buffer_slot_n_bytes_; + size_t module_image_offset = i_module * MODULE_N_BYTES; + for (size_t i_pulse=0; i_pulse < BUFFER_BLOCK_SIZE; i_pulse++) { + size_t image_offset = i_pulse * MODULE_N_BYTES * n_modules_; + + memcpy( + image_buffer_ + slot_offset + image_offset + module_image_offset, + &(block_buffer->frame[i_pulse].data[0]), + MODULE_N_BYTES); + + + } } int ImageAssembler::get_full_slot() From 7005746292687b3f4724f2f13fcbb8fe911ea694 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 15:26:30 +0200 Subject: [PATCH 051/111] Tentative buffer implementation --- sf-writer/src/ImageAssembler.cpp | 27 +++++++++++++++++++++++++-- sf-writer/src/ImageAssembler.hpp | 3 +++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/sf-writer/src/ImageAssembler.cpp b/sf-writer/src/ImageAssembler.cpp index 32515ac..3329c01 100644 --- a/sf-writer/src/ImageAssembler.cpp +++ b/sf-writer/src/ImageAssembler.cpp @@ -10,6 +10,11 @@ ImageAssembler::ImageAssembler(const size_t n_modules) : { image_buffer_ = new char[IA_N_SLOTS * image_buffer_slot_n_bytes_]; metadata_buffer_ = new ImageMetadataBlock[IA_N_SLOTS]; + buffer_status_ = new atomic_int[IA_N_SLOTS]; + + for (size_t i=0; iframe[0].metadata.pulse_id; + metadata_buffer_[slot_id].block_last_pulse_id = + block_buffer->frame[BUFFER_BLOCK_SIZE-1].metadata.pulse_id; + } + size_t slot_offset = slot_id * image_buffer_slot_n_bytes_; size_t module_image_offset = i_module * MODULE_N_BYTES; @@ -39,7 +52,17 @@ void ImageAssembler::process( &(block_buffer->frame[i_pulse].data[0]), MODULE_N_BYTES); - + // TODO: Temp workaround. We need to synchronize this access. + if (i_module == 0) { + metadata_buffer_[slot_id].pulse_id[i_pulse] = + block_buffer->frame[i_pulse].metadata.pulse_id; + metadata_buffer_[slot_id].frame_index[i_pulse] = + block_buffer->frame[i_pulse].metadata.frame_index; + metadata_buffer_[slot_id].daq_rec[i_pulse] = + block_buffer->frame[i_pulse].metadata.daq_rec; +// metadata_buffer_[slot_id].is_good_image[i_pulse] = +// block_buffer->frame[i_pulse].is_good_image.pulse_id; + } } } @@ -50,7 +73,7 @@ int ImageAssembler::get_full_slot() void ImageAssembler::free_slot(const int slot_id) { - + buffer_status_[slot_id] = n_modules_; } ImageMetadataBlock* ImageAssembler::get_metadata_buffer(const int slot_id) diff --git a/sf-writer/src/ImageAssembler.hpp b/sf-writer/src/ImageAssembler.hpp index a34cb59..10f7fdf 100644 --- a/sf-writer/src/ImageAssembler.hpp +++ b/sf-writer/src/ImageAssembler.hpp @@ -1,6 +1,8 @@ #ifndef SF_DAQ_BUFFER_IMAGEASSEMBLER_HPP #define SF_DAQ_BUFFER_IMAGEASSEMBLER_HPP +#include + #include "formats.hpp" const size_t IA_N_SLOTS = 2; @@ -11,6 +13,7 @@ class ImageAssembler { char* image_buffer_; ImageMetadataBlock* metadata_buffer_; + std::atomic_int* buffer_status_; public: ImageAssembler(const size_t n_modules); From e06ae47d6ca9ee664851ae1466044d119163cc95 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 15:35:17 +0200 Subject: [PATCH 052/111] Add buffer status initialization --- sf-writer/src/ImageAssembler.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sf-writer/src/ImageAssembler.cpp b/sf-writer/src/ImageAssembler.cpp index 3329c01..f87270f 100644 --- a/sf-writer/src/ImageAssembler.cpp +++ b/sf-writer/src/ImageAssembler.cpp @@ -13,7 +13,7 @@ ImageAssembler::ImageAssembler(const size_t n_modules) : buffer_status_ = new atomic_int[IA_N_SLOTS]; for (size_t i=0; iframe[i_pulse].is_good_image.pulse_id; } } + + buffer_status_[slot_id]--; } int ImageAssembler::get_full_slot() From 67d27c4618e0dea8c6c6fba20931964f5b0b0636 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 15:47:45 +0200 Subject: [PATCH 053/111] Implemented read_slot_id --- sf-writer/src/ImageAssembler.cpp | 16 +++++++++++++++- sf-writer/src/ImageAssembler.hpp | 2 ++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/sf-writer/src/ImageAssembler.cpp b/sf-writer/src/ImageAssembler.cpp index f87270f..1bc4f24 100644 --- a/sf-writer/src/ImageAssembler.cpp +++ b/sf-writer/src/ImageAssembler.cpp @@ -15,6 +15,8 @@ ImageAssembler::ImageAssembler(const size_t n_modules) : for (size_t i=0; i Date: Wed, 27 May 2020 15:51:59 +0200 Subject: [PATCH 054/111] Implement write index as well --- sf-writer/src/ImageAssembler.cpp | 9 +++++++-- sf-writer/src/ImageAssembler.hpp | 1 + 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/sf-writer/src/ImageAssembler.cpp b/sf-writer/src/ImageAssembler.cpp index 1bc4f24..b090489 100644 --- a/sf-writer/src/ImageAssembler.cpp +++ b/sf-writer/src/ImageAssembler.cpp @@ -17,6 +17,7 @@ ImageAssembler::ImageAssembler(const size_t n_modules) : } read_slot_id_ = 0; + write_slot_id_ = 0; } ImageAssembler::~ImageAssembler() @@ -27,7 +28,11 @@ ImageAssembler::~ImageAssembler() int ImageAssembler::get_free_slot() { + if (buffer_status_[write_slot_id_] > 0) { + return write_slot_id_; + } + return -1; } void ImageAssembler::process( @@ -71,8 +76,8 @@ void ImageAssembler::process( // 1 because fetch is done before subtraction. if (previous_status == 1) { - write_slot_id_++; - read_slot_id_ %= IA_N_SLOTS; + auto next_write_slot_id = (int)((write_slot_id_+1) % IA_N_SLOTS); + write_slot_id_ = next_write_slot_id; } } diff --git a/sf-writer/src/ImageAssembler.hpp b/sf-writer/src/ImageAssembler.hpp index 1b4d2be..525b4c4 100644 --- a/sf-writer/src/ImageAssembler.hpp +++ b/sf-writer/src/ImageAssembler.hpp @@ -16,6 +16,7 @@ class ImageAssembler { std::atomic_int* buffer_status_; int read_slot_id_; + std::atomic_int write_slot_id_; public: ImageAssembler(const size_t n_modules); From ed86780629449add4964e90266eb7e665b25bff7 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 16:01:12 +0200 Subject: [PATCH 055/111] Refactoring files --- sf-replay/include/ReplayH5Reader.hpp | 34 ------- sf-replay/src/ReplayH5Reader.cpp | 89 ------------------- sf-writer/{src => include}/ImageAssembler.hpp | 0 sf-writer/src/WriterZmqReceiver.cpp | 2 +- sf-writer/src/main.cpp | 4 +- 5 files changed, 3 insertions(+), 126 deletions(-) delete mode 100644 sf-replay/include/ReplayH5Reader.hpp delete mode 100644 sf-replay/src/ReplayH5Reader.cpp rename sf-writer/{src => include}/ImageAssembler.hpp (100%) diff --git a/sf-replay/include/ReplayH5Reader.hpp b/sf-replay/include/ReplayH5Reader.hpp deleted file mode 100644 index 009a028..0000000 --- a/sf-replay/include/ReplayH5Reader.hpp +++ /dev/null @@ -1,34 +0,0 @@ -#ifndef SF_DAQ_BUFFER_REPLAYH5READER_HPP -#define SF_DAQ_BUFFER_REPLAYH5READER_HPP - -#include -#include -#include - -#include "formats.hpp" - -class ReplayH5Reader { - - const std::string device_; - const std::string channel_name_; - - H5::H5File current_file_; - std::string current_filename_; - H5::DataSet dset_metadata_; - H5::DataSet dset_frame_; - -public: - ReplayH5Reader( - const std::string device, - const std::string channel_name); - virtual ~ReplayH5Reader(); - - void close_file(); - void get_buffer( - const uint64_t pulse_id, - ReplayBuffer* metadata, - char* frame_buffer); -}; - - -#endif //SF_DAQ_BUFFER_REPLAYH5READER_HPP diff --git a/sf-replay/src/ReplayH5Reader.cpp b/sf-replay/src/ReplayH5Reader.cpp deleted file mode 100644 index 56f8851..0000000 --- a/sf-replay/src/ReplayH5Reader.cpp +++ /dev/null @@ -1,89 +0,0 @@ -#include "ReplayH5Reader.hpp" - -#include "BufferUtils.hpp" - -using namespace std; -using namespace core_buffer; - -ReplayH5Reader::ReplayH5Reader( - const string device, - const string channel_name) : - device_(device), - channel_name_(channel_name) -{ -} - -ReplayH5Reader::~ReplayH5Reader() -{ - close_file(); -} - -void ReplayH5Reader::close_file() -{ - if (current_file_.getId() != -1) { - dset_metadata_.close(); - dset_frame_.close(); - current_file_.close(); - } -} - -void ReplayH5Reader::get_buffer( - const uint64_t pulse_id, - ReplayBuffer* metadata, - char* data) -{ - auto pulse_filename = BufferUtils::get_filename( - device_, channel_name_, pulse_id); - - if (pulse_filename != current_filename_) { - close_file(); - - current_filename_ = pulse_filename; - current_file_ = H5::H5File(current_filename_, H5F_ACC_RDONLY); - - dset_metadata_ = current_file_.openDataSet(BUFFER_H5_METADATA_DATASET); - dset_frame_ = current_file_.openDataSet(BUFFER_H5_FRAME_DATASET); - } - - auto file_index = BufferUtils::get_file_frame_index(pulse_id); - auto cache_start_index = file_index / BUFFER_BLOCK_SIZE; - cache_start_index *= BUFFER_BLOCK_SIZE; - - uint64_t b_start_pulse_id = pulse_id - (file_index - cache_start_index); - metadata->start_pulse_id = b_start_pulse_id; - metadata->n_frames = BUFFER_BLOCK_SIZE; - - hsize_t b_m_dims[2] = {BUFFER_BLOCK_SIZE, ModuleFrame_N_FIELDS}; - H5::DataSpace b_m_space (2, b_m_dims); - hsize_t b_m_count[] = {BUFFER_BLOCK_SIZE, ModuleFrame_N_FIELDS}; - hsize_t b_m_start[] = {cache_start_index, 0}; - b_m_space.selectHyperslab(H5S_SELECT_SET, b_m_count, b_m_start); - - hsize_t f_m_dims[2] = {FILE_MOD, ModuleFrame_N_FIELDS}; - H5::DataSpace f_m_space (2, f_m_dims); - hsize_t f_m_count[] = {BUFFER_BLOCK_SIZE, ModuleFrame_N_FIELDS}; - hsize_t pulse_id_start[] = {cache_start_index, 0}; - f_m_space.selectHyperslab(H5S_SELECT_SET, f_m_count, pulse_id_start); - - dset_metadata_.read( - &metadata->metadata[0], - H5::PredType::NATIVE_UINT64, b_m_space, f_m_space); - - hsize_t b_f_dims[3] = - {BUFFER_BLOCK_SIZE, MODULE_Y_SIZE, MODULE_X_SIZE}; - H5::DataSpace b_f_space (3, b_f_dims); - hsize_t b_f_count[] = - {BUFFER_BLOCK_SIZE, MODULE_Y_SIZE, MODULE_X_SIZE}; - hsize_t b_f_start[] = {0, 0, 0}; - b_f_space.selectHyperslab(H5S_SELECT_SET, b_f_count, b_f_start); - - hsize_t f_frame_dims[3] = {FILE_MOD, MODULE_Y_SIZE, MODULE_X_SIZE}; - H5::DataSpace f_f_space (3, f_frame_dims); - hsize_t f_f_count[] = - {BUFFER_BLOCK_SIZE, MODULE_Y_SIZE, MODULE_X_SIZE}; - hsize_t f_f_start[] = {cache_start_index, 0, 0}; - f_f_space.selectHyperslab(H5S_SELECT_SET, f_f_count, f_f_start); - - dset_frame_.read( - data, H5::PredType::NATIVE_UINT16, b_f_space, f_f_space); -} diff --git a/sf-writer/src/ImageAssembler.hpp b/sf-writer/include/ImageAssembler.hpp similarity index 100% rename from sf-writer/src/ImageAssembler.hpp rename to sf-writer/include/ImageAssembler.hpp diff --git a/sf-writer/src/WriterZmqReceiver.cpp b/sf-writer/src/WriterZmqReceiver.cpp index b01b6f5..0d3d8f8 100644 --- a/sf-writer/src/WriterZmqReceiver.cpp +++ b/sf-writer/src/WriterZmqReceiver.cpp @@ -59,7 +59,7 @@ void WriterZmqReceiver::get_next_buffer( n_images_in_buffer = images_left; } - i_meta->n_images = (uint16_t)n_images_in_buffer; +// i_meta->n_images = (uint16_t)n_images_in_buffer; for (uint64_t i_pulse=0; i_pulse buffer_blocks(n_blocks); From 5b6a6379e3a4325b0c15a900df821ec6fb697a24 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 16:10:59 +0200 Subject: [PATCH 056/111] Fix 2 small bugs --- sf-writer/src/main.cpp | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/sf-writer/src/main.cpp b/sf-writer/src/main.cpp index 73b20e8..13218d8 100644 --- a/sf-writer/src/main.cpp +++ b/sf-writer/src/main.cpp @@ -77,8 +77,8 @@ int main (int argc, char *argv[]) string output_file = string(argv[1]); const string device = string(argv[2]); - uint64_t start_pulse_id = (uint64_t) atoll(argv[4]); - uint64_t stop_pulse_id = (uint64_t) atoll(argv[5]); + uint64_t start_pulse_id = (uint64_t) atoll(argv[3]); + uint64_t stop_pulse_id = (uint64_t) atoll(argv[4]); size_t n_modules = 32; uint64_t start_block = start_pulse_id / BUFFER_BLOCK_SIZE; @@ -97,7 +97,14 @@ int main (int argc, char *argv[]) std::vector reading_threads(n_modules); for (size_t i_module=0; i_module Date: Wed, 27 May 2020 16:19:42 +0200 Subject: [PATCH 057/111] Bug in vector init --- sf-writer/src/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sf-writer/src/main.cpp b/sf-writer/src/main.cpp index 13218d8..7978eeb 100644 --- a/sf-writer/src/main.cpp +++ b/sf-writer/src/main.cpp @@ -88,7 +88,7 @@ int main (int argc, char *argv[]) ImageAssembler image_assembler(n_modules); // Generate list of buffer blocks that need to be loaded. - std::vector buffer_blocks(n_blocks); + std::vector buffer_blocks; for (uint64_t curr_block=start_block; curr_block<=stop_block; curr_block++) { From cf0245d3f3c470a7ae16121300985d09072c1d8d Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 16:51:32 +0200 Subject: [PATCH 058/111] Implement external slot_id book keeping --- sf-writer/include/ImageAssembler.hpp | 10 +++---- sf-writer/src/ImageAssembler.cpp | 44 +++++++++------------------- sf-writer/src/main.cpp | 10 ++++--- 3 files changed, 24 insertions(+), 40 deletions(-) diff --git a/sf-writer/include/ImageAssembler.hpp b/sf-writer/include/ImageAssembler.hpp index 525b4c4..87f1070 100644 --- a/sf-writer/include/ImageAssembler.hpp +++ b/sf-writer/include/ImageAssembler.hpp @@ -15,20 +15,18 @@ class ImageAssembler { ImageMetadataBlock* metadata_buffer_; std::atomic_int* buffer_status_; - int read_slot_id_; - std::atomic_int write_slot_id_; - public: ImageAssembler(const size_t n_modules); virtual ~ImageAssembler(); - int get_free_slot(); - void process(const int slot_id, + bool is_slot_free(const int slot_id); + bool is_slot_full(const int slot_id); + + void process(int slot_id, const int i_module, const BufferBinaryBlock* block_buffer); - int get_full_slot(); void free_slot(const int slot_id); ImageMetadataBlock* get_metadata_buffer(const int slot_id); diff --git a/sf-writer/src/ImageAssembler.cpp b/sf-writer/src/ImageAssembler.cpp index b090489..fe271c4 100644 --- a/sf-writer/src/ImageAssembler.cpp +++ b/sf-writer/src/ImageAssembler.cpp @@ -15,9 +15,6 @@ ImageAssembler::ImageAssembler(const size_t n_modules) : for (size_t i=0; i 0) { - return write_slot_id_; - } + return buffer_status_[slot_id % IA_N_SLOTS] > 0; +} - return -1; +bool ImageAssembler::is_slot_full(const int slot_id) +{ + return buffer_status_[slot_id % IA_N_SLOTS] == 0; } void ImageAssembler::process( - const int slot_id, + int slot_id, const int i_module, const BufferBinaryBlock* block_buffer) { + slot_id %= IA_N_SLOTS; + // TODO: Temp workaround. Make proper initialization. if (i_module == 0) { metadata_buffer_[slot_id].block_first_pulse_id = @@ -72,37 +72,21 @@ void ImageAssembler::process( } } - auto previous_status = buffer_status_[slot_id].fetch_sub(1); - - // 1 because fetch is done before subtraction. - if (previous_status == 1) { - auto next_write_slot_id = (int)((write_slot_id_+1) % IA_N_SLOTS); - write_slot_id_ = next_write_slot_id; - } -} - -int ImageAssembler::get_full_slot() -{ - if (buffer_status_[read_slot_id_] == 0) { - return read_slot_id_; - } - - return -1; + buffer_status_[slot_id]--; } void ImageAssembler::free_slot(const int slot_id) { - buffer_status_[slot_id] = n_modules_; - - read_slot_id_ = (int)((read_slot_id_+1) % IA_N_SLOTS); + buffer_status_[slot_id % IA_N_SLOTS] = n_modules_; } ImageMetadataBlock* ImageAssembler::get_metadata_buffer(const int slot_id) { - return &(metadata_buffer_[slot_id]); + return &(metadata_buffer_[slot_id % IA_N_SLOTS]); } char* ImageAssembler::get_data_buffer(const int slot_id) { - return image_buffer_ + (slot_id * image_buffer_slot_n_bytes_); + return image_buffer_ + + ((slot_id % IA_N_SLOTS) * image_buffer_slot_n_bytes_); } diff --git a/sf-writer/src/main.cpp b/sf-writer/src/main.cpp index 7978eeb..56bb458 100644 --- a/sf-writer/src/main.cpp +++ b/sf-writer/src/main.cpp @@ -26,10 +26,10 @@ void read_buffer( BufferBinaryReader block_reader(device, channel_name); auto block_buffer = new BufferBinaryBlock(); + int slot_id = 0; for (uint64_t block_id:buffer_blocks) { - int slot_id; - while((slot_id = image_assembler.get_free_slot()) == -1) { + while(!image_assembler.is_slot_free(slot_id)) { this_thread::sleep_for(chrono::milliseconds( WRITER_IMAGE_ASSEMBLER_RETRY_MS)); } @@ -45,6 +45,7 @@ void read_buffer( start_time = steady_clock::now(); image_assembler.process(slot_id, i_module, block_buffer); + slot_id++; end_time = steady_clock::now(); uint64_t compose_us_duration = duration_cast( @@ -116,10 +117,10 @@ int main (int argc, char *argv[]) JFH5Writer writer(output_file, start_pulse_id, stop_pulse_id, n_modules); + int slot_id = 0; for (uint64_t block_id:buffer_blocks) { - int slot_id; - while((slot_id = image_assembler.get_full_slot()) == -1) { + while(!image_assembler.is_slot_full(slot_id)) { this_thread::sleep_for(chrono::milliseconds( WRITER_IMAGE_ASSEMBLER_RETRY_MS)); } @@ -136,6 +137,7 @@ int main (int argc, char *argv[]) end_time-start_time).count(); image_assembler.free_slot(slot_id); + slot_id++; cout << "sf_writer:avg_write_us "; cout << write_us_duration / BUFFER_BLOCK_SIZE << endl; From 5502745186d79c14dee1471f2fd638ac1b9aa0bb Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 17:14:16 +0200 Subject: [PATCH 059/111] Add direct chunk write lib --- sf-writer/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/sf-writer/CMakeLists.txt b/sf-writer/CMakeLists.txt index d75df61..b591255 100644 --- a/sf-writer/CMakeLists.txt +++ b/sf-writer/CMakeLists.txt @@ -14,6 +14,7 @@ target_link_libraries(sf-writer sf-replay-lib zmq hdf5 + hdf5_hl hdf5_cpp pthread ) From 0656c95beac0f3a5cb77e0c421bf673efcf8816c Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 17:15:53 +0200 Subject: [PATCH 060/111] Adjust chunking size to min size --- sf-writer/src/JFH5Writer.cpp | 55 +++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/sf-writer/src/JFH5Writer.cpp b/sf-writer/src/JFH5Writer.cpp index d2a5231..9eb61ec 100644 --- a/sf-writer/src/JFH5Writer.cpp +++ b/sf-writer/src/JFH5Writer.cpp @@ -1,6 +1,7 @@ #include "JFH5Writer.hpp" #include #include +#include //extern "C" @@ -32,8 +33,9 @@ JFH5Writer::JFH5Writer(const std::string& output_file, H5::DataSpace image_dataspace(3, image_dataset_dims); + auto chunk_size = min(n_images_, BUFFER_BLOCK_SIZE); hsize_t image_dataset_chunking[3] = - {1, n_modules * MODULE_Y_SIZE, MODULE_X_SIZE}; + {chunk_size, n_modules * MODULE_Y_SIZE, MODULE_X_SIZE}; H5::DSetCreatPropList image_dataset_properties; image_dataset_properties.setChunk(3, image_dataset_chunking); @@ -136,28 +138,37 @@ void JFH5Writer::write( throw runtime_error("Received unexpected block for stop_pulse_id."); } - hsize_t b_i_dims[3] = {BUFFER_BLOCK_SIZE, - MODULE_Y_SIZE * n_modules_, - MODULE_X_SIZE}; - H5::DataSpace b_i_space(3, b_i_dims); - hsize_t b_i_count[] = {n_images_to_copy, - MODULE_Y_SIZE * n_modules_, - MODULE_X_SIZE}; - hsize_t b_i_start[] = {n_images_offset, 0, 0}; - b_i_space.selectHyperslab(H5S_SELECT_SET, b_i_count, b_i_start); +// hsize_t b_i_dims[3] = {BUFFER_BLOCK_SIZE, +// MODULE_Y_SIZE * n_modules_, +// MODULE_X_SIZE}; +// H5::DataSpace b_i_space(3, b_i_dims); +// hsize_t b_i_count[] = {n_images_to_copy, +// MODULE_Y_SIZE * n_modules_, +// MODULE_X_SIZE}; +// hsize_t b_i_start[] = {n_images_offset, 0, 0}; +// b_i_space.selectHyperslab(H5S_SELECT_SET, b_i_count, b_i_start); +// +// hsize_t f_i_dims[3] = {n_images_, +// MODULE_Y_SIZE * n_modules_, +// MODULE_X_SIZE}; +// H5::DataSpace f_i_space(3, f_i_dims); +// hsize_t f_i_count[] = {n_images_to_copy, +// MODULE_Y_SIZE * n_modules_, +// MODULE_X_SIZE}; +// hsize_t f_i_start[] = {current_write_index_, 0, 0}; +// f_i_space.selectHyperslab(H5S_SELECT_SET, f_i_count, f_i_start); +// +// image_dataset_.write( +// data, H5::PredType::NATIVE_UINT16, b_i_space, f_i_space); - hsize_t f_i_dims[3] = {n_images_, - MODULE_Y_SIZE * n_modules_, - MODULE_X_SIZE}; - H5::DataSpace f_i_space(3, f_i_dims); - hsize_t f_i_count[] = {n_images_to_copy, - MODULE_Y_SIZE * n_modules_, - MODULE_X_SIZE}; - hsize_t f_i_start[] = {current_write_index_, 0, 0}; - f_i_space.selectHyperslab(H5S_SELECT_SET, f_i_count, f_i_start); - - image_dataset_.write( - data, H5::PredType::NATIVE_UINT16, b_i_space, f_i_space); + hsize_t offset[] = {current_write_index_, 0, 0}; + H5DOwrite_chunk( + image_dataset_.getId(), + H5P_DEFAULT, + 0, + offset, + n_images_to_copy * MODULE_N_BYTES * n_modules_, + data); // pulse_id { From 7ef67df64496f8121d5ce5a2cba7713ed1aaf6bf Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 17:16:20 +0200 Subject: [PATCH 061/111] Removal of commented out code --- sf-writer/src/JFH5Writer.cpp | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/sf-writer/src/JFH5Writer.cpp b/sf-writer/src/JFH5Writer.cpp index 9eb61ec..1fdb94a 100644 --- a/sf-writer/src/JFH5Writer.cpp +++ b/sf-writer/src/JFH5Writer.cpp @@ -138,29 +138,6 @@ void JFH5Writer::write( throw runtime_error("Received unexpected block for stop_pulse_id."); } -// hsize_t b_i_dims[3] = {BUFFER_BLOCK_SIZE, -// MODULE_Y_SIZE * n_modules_, -// MODULE_X_SIZE}; -// H5::DataSpace b_i_space(3, b_i_dims); -// hsize_t b_i_count[] = {n_images_to_copy, -// MODULE_Y_SIZE * n_modules_, -// MODULE_X_SIZE}; -// hsize_t b_i_start[] = {n_images_offset, 0, 0}; -// b_i_space.selectHyperslab(H5S_SELECT_SET, b_i_count, b_i_start); -// -// hsize_t f_i_dims[3] = {n_images_, -// MODULE_Y_SIZE * n_modules_, -// MODULE_X_SIZE}; -// H5::DataSpace f_i_space(3, f_i_dims); -// hsize_t f_i_count[] = {n_images_to_copy, -// MODULE_Y_SIZE * n_modules_, -// MODULE_X_SIZE}; -// hsize_t f_i_start[] = {current_write_index_, 0, 0}; -// f_i_space.selectHyperslab(H5S_SELECT_SET, f_i_count, f_i_start); -// -// image_dataset_.write( -// data, H5::PredType::NATIVE_UINT16, b_i_space, f_i_space); - hsize_t offset[] = {current_write_index_, 0, 0}; H5DOwrite_chunk( image_dataset_.getId(), From 8c0fd094760cb66d73ce8909a2aea8a7eef458ff Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 18:23:47 +0200 Subject: [PATCH 062/111] Move BufferBinaryReader to sf_writer --- sf-writer/CMakeLists.txt | 1 - {sf-replay => sf-writer}/include/BufferBinaryReader.hpp | 0 {sf-replay => sf-writer}/src/BufferBinaryReader.cpp | 0 sf-writer/src/main.cpp | 2 +- 4 files changed, 1 insertion(+), 2 deletions(-) rename {sf-replay => sf-writer}/include/BufferBinaryReader.hpp (100%) rename {sf-replay => sf-writer}/src/BufferBinaryReader.cpp (100%) diff --git a/sf-writer/CMakeLists.txt b/sf-writer/CMakeLists.txt index b591255..5b520f5 100644 --- a/sf-writer/CMakeLists.txt +++ b/sf-writer/CMakeLists.txt @@ -11,7 +11,6 @@ add_executable(sf-writer src/main.cpp) set_target_properties(sf-writer PROPERTIES OUTPUT_NAME sf_writer) target_link_libraries(sf-writer sf-writer-lib - sf-replay-lib zmq hdf5 hdf5_hl diff --git a/sf-replay/include/BufferBinaryReader.hpp b/sf-writer/include/BufferBinaryReader.hpp similarity index 100% rename from sf-replay/include/BufferBinaryReader.hpp rename to sf-writer/include/BufferBinaryReader.hpp diff --git a/sf-replay/src/BufferBinaryReader.cpp b/sf-writer/src/BufferBinaryReader.cpp similarity index 100% rename from sf-replay/src/BufferBinaryReader.cpp rename to sf-writer/src/BufferBinaryReader.cpp diff --git a/sf-writer/src/main.cpp b/sf-writer/src/main.cpp index 56bb458..e946937 100644 --- a/sf-writer/src/main.cpp +++ b/sf-writer/src/main.cpp @@ -10,7 +10,7 @@ #include "bitshuffle/bitshuffle.h" #include "JFH5Writer.hpp" #include "ImageAssembler.hpp" -#include "../../sf-replay/include/BufferBinaryReader.hpp" +#include "BufferBinaryReader.hpp" using namespace std; using namespace core_buffer; From 90534583a86ffaafb35d8b6bdd46bc8565c35e30 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 18:25:33 +0200 Subject: [PATCH 063/111] Remove sf_replay as its not needed anymore --- CMakeLists.txt | 1 - sf-replay/CMakeLists.txt | 21 ---- sf-replay/include/BlockZmqSender.hpp | 23 ----- sf-replay/src/BlockZmqSender.cpp | 47 --------- sf-replay/src/main.cpp | 137 ------------------------- sf-replay/test/CMakeLists.txt | 10 -- sf-replay/test/main.cpp | 9 -- sf-replay/test/test_ReplayH5Reader.cpp | 73 ------------- 8 files changed, 321 deletions(-) delete mode 100644 sf-replay/CMakeLists.txt delete mode 100644 sf-replay/include/BlockZmqSender.hpp delete mode 100644 sf-replay/src/BlockZmqSender.cpp delete mode 100644 sf-replay/src/main.cpp delete mode 100644 sf-replay/test/CMakeLists.txt delete mode 100644 sf-replay/test/main.cpp delete mode 100644 sf-replay/test/test_ReplayH5Reader.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index aed6e98..44e8243 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -30,6 +30,5 @@ add_subdirectory( add_subdirectory("core-buffer") add_subdirectory("sf-buffer") -add_subdirectory("sf-replay") add_subdirectory("sf-stream") add_subdirectory("sf-writer") \ No newline at end of file diff --git a/sf-replay/CMakeLists.txt b/sf-replay/CMakeLists.txt deleted file mode 100644 index 51b15c6..0000000 --- a/sf-replay/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -file(GLOB SOURCES - src/*.cpp) - -add_library(sf-replay-lib STATIC ${SOURCES}) -target_include_directories(sf-replay-lib PUBLIC include/) -target_link_libraries(sf-replay-lib - external - core-buffer-lib) - -add_executable(sf-replay src/main.cpp) -set_target_properties(sf-replay PROPERTIES OUTPUT_NAME sf_replay) -target_link_libraries(sf-replay - core-buffer-lib - sf-replay-lib - zmq - hdf5 - hdf5_cpp - pthread) - -enable_testing() -add_subdirectory(test/) \ No newline at end of file diff --git a/sf-replay/include/BlockZmqSender.hpp b/sf-replay/include/BlockZmqSender.hpp deleted file mode 100644 index e4eb813..0000000 --- a/sf-replay/include/BlockZmqSender.hpp +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef SF_DAQ_BUFFER_BLOCKZMQSENDER_HPP -#define SF_DAQ_BUFFER_BLOCKZMQSENDER_HPP - -#include -#include -#include - -class BlockZmqSender { - - void* ctx_; - void* socket_; - -public: - BlockZmqSender(const std::string& ipc_id, const int source_id); - virtual ~BlockZmqSender(); - - void close(); - - void send(const BufferBinaryBlock* block_data); -}; - - -#endif //SF_DAQ_BUFFER_BLOCKZMQSENDER_HPP diff --git a/sf-replay/src/BlockZmqSender.cpp b/sf-replay/src/BlockZmqSender.cpp deleted file mode 100644 index fc6667f..0000000 --- a/sf-replay/src/BlockZmqSender.cpp +++ /dev/null @@ -1,47 +0,0 @@ -#include "BlockZmqSender.hpp" - -#include -#include - -#include "buffer_config.hpp" - -using namespace std; -using namespace core_buffer; - - -BlockZmqSender::BlockZmqSender(const string& ipc_id, const int source_id) -{ - auto ipc_base = REPLAY_STREAM_IPC_URL + ipc_id + "-"; - stringstream ipc_stream; - ipc_stream << ipc_base << source_id; - const auto ipc_address = ipc_stream.str(); - - ctx_ = zmq_ctx_new(); - socket_ = zmq_socket(ctx_, ZMQ_PUSH); - - const int sndhwm = REPLAY_SNDHWM; - if (zmq_setsockopt(socket_, ZMQ_SNDHWM, &sndhwm, sizeof(sndhwm)) != 0) - throw runtime_error(zmq_strerror (errno)); - - const int linger_ms = -1; - if (zmq_setsockopt(socket_, ZMQ_LINGER, &linger_ms, sizeof(linger_ms)) != 0) - throw runtime_error(zmq_strerror (errno)); - - if (zmq_bind(socket_, ipc_address.c_str()) != 0) - throw runtime_error(zmq_strerror (errno)); -} - -BlockZmqSender::~BlockZmqSender() -{ - close(); -} - -void BlockZmqSender::close() { - zmq_close(socket_); - zmq_ctx_destroy(ctx_); -} - -void BlockZmqSender::send(const BufferBinaryBlock* block_data) -{ - zmq_send(socket_, block_data, sizeof(BufferBinaryBlock), ZMQ_SNDMORE); -} diff --git a/sf-replay/src/main.cpp b/sf-replay/src/main.cpp deleted file mode 100644 index 5b96e46..0000000 --- a/sf-replay/src/main.cpp +++ /dev/null @@ -1,137 +0,0 @@ -#include -#include - -#include "FastQueue.hpp" -#include "buffer_config.hpp" -#include "BufferBinaryReader.hpp" -#include "BlockZmqSender.hpp" - -using namespace std; -using namespace core_buffer; -using namespace chrono; - -void sf_replay ( - const string device, - const string channel_name, - FastQueue& queue, - const uint64_t start_pulse_id, - const uint64_t stop_pulse_id - ) -{ - BufferBinaryReader block_reader(device, channel_name); - - uint64_t start_block = start_pulse_id / BUFFER_BLOCK_SIZE; - uint64_t stop_block = stop_pulse_id / BUFFER_BLOCK_SIZE; - - // "<= stop_block" because we include the stop_block in the transfer. - for (uint64_t curr_block=start_block; - curr_block <= stop_block; - curr_block++) { - - int slot_id; - while((slot_id = queue.reserve()) == -1) { - this_thread::sleep_for(chrono::milliseconds( - RB_READ_RETRY_INTERVAL_MS)); - } - - auto start_time = steady_clock::now(); - - auto block_buffer = queue.get_metadata_buffer(slot_id); - - block_reader.get_block(curr_block, block_buffer); - - auto end_time = steady_clock::now(); - uint64_t read_us_duration = duration_cast( - end_time-start_time).count(); - - queue.commit(); - - // TODO: Proper statistics - cout << "sf_replay:avg_read_us "; - cout << read_us_duration / BUFFER_BLOCK_SIZE << endl; - } -} - -int main (int argc, char *argv[]) { - - if (argc != 7) { - cout << endl; - cout << "Usage: sf_replay [ipc_id] [device]"; - cout << " [channel_name] [source_id] [start_pulse_id] [stop_pulse_id]"; - cout << endl; - cout << "\tdevice: Name of detector." << endl; - cout << "\tchannel_name: M00-M31 for JF16M." << endl; - cout << "\tsource_id: Module index" << endl; - cout << "\tstart_pulse_id: Start pulse_id of retrieval." << endl; - cout << "\tstop_pulse_id: Stop pulse_id of retrieval." << endl; - cout << endl; - - exit(-1); - } - - const string ipc_id = string(argv[1]); - const string device = string(argv[2]); - const string channel_name = string(argv[3]); - const auto source_id = atoi(argv[4]); - const auto start_pulse_id = (uint64_t) atoll(argv[5]); - const auto stop_pulse_id = (uint64_t) atoll(argv[6]); - - // 0 bytes for data since everything is in the header. - FastQueue queue(0, REPLAY_FASTQUEUE_N_SLOTS); - - thread file_read_thread(sf_replay, - device, channel_name, ref(queue), - start_pulse_id, stop_pulse_id); - - uint64_t send_us = 0; - uint64_t max_send_us = 0; - uint64_t n_stats = 0; - - BlockZmqSender sender(ipc_id, source_id); - - uint64_t start_block = start_pulse_id / BUFFER_BLOCK_SIZE; - uint64_t stop_block = stop_pulse_id / BUFFER_BLOCK_SIZE; - - // "<= stop_block" because we include the stop_block in the transfer. - for (uint64_t curr_block=start_block; - curr_block <= stop_block; - curr_block++) { - - int slot_id; - while((slot_id = queue.read()) == -1) { - this_thread::sleep_for(chrono::milliseconds( - RB_READ_RETRY_INTERVAL_MS)); - } - - auto block_buffer = queue.get_metadata_buffer(slot_id); - - auto start_time = steady_clock::now(); - - sender.send(block_buffer); - - auto end_time = steady_clock::now(); - uint64_t send_us_duration = - duration_cast(end_time-start_time).count(); - - queue.release(); - - // TODO: Proper statistics - n_stats++; - - send_us += send_us_duration; - max_send_us = max(max_send_us, send_us_duration); - - if (n_stats == STATS_MODULO) { - cout << "sf_replay:avg_send_us " << send_us / STATS_MODULO; - cout << " sf_replay:max_send_us " << max_send_us; - cout << endl; - - n_stats = 0; - send_us = 0; - max_send_us = 0; - } - } - - file_read_thread.join(); - return 0; -} diff --git a/sf-replay/test/CMakeLists.txt b/sf-replay/test/CMakeLists.txt deleted file mode 100644 index 45588cf..0000000 --- a/sf-replay/test/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -add_executable(sf-replay-tests main.cpp) - -target_link_libraries(sf-replay-tests - core-buffer-lib - sf-buffer-lib - sf-replay-lib - hdf5 - hdf5_cpp - gtest - ) diff --git a/sf-replay/test/main.cpp b/sf-replay/test/main.cpp deleted file mode 100644 index d40a5df..0000000 --- a/sf-replay/test/main.cpp +++ /dev/null @@ -1,9 +0,0 @@ -#include "gtest/gtest.h" -#include "test_ReplayH5Reader.cpp" - -using namespace std; - -int main(int argc, char **argv) { - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/sf-replay/test/test_ReplayH5Reader.cpp b/sf-replay/test/test_ReplayH5Reader.cpp deleted file mode 100644 index 17be0a0..0000000 --- a/sf-replay/test/test_ReplayH5Reader.cpp +++ /dev/null @@ -1,73 +0,0 @@ -#include -#include - -#include "ReplayH5Reader.hpp" -#include "BufferH5Writer.hpp" - -using namespace std; -using namespace core_buffer; - -TEST(ReplayH5Reader, basic_interaction) -{ - auto root_folder = "."; - auto device_name = "fast_device"; - size_t pulse_id = 65; - uint16_t source_id = 124; - - // This 2 must be compatible by design. - BufferH5Writer writer(root_folder, device_name); - ReplayH5Reader reader(root_folder, device_name); - - ModuleFrame w_metadata; - ModuleFrame* r_metadata; - auto w_frame_buffer = make_unique(MODULE_N_PIXELS); - char* r_frame_buffer; - - // Setup test values. - w_metadata.pulse_id = pulse_id; - w_metadata.frame_index = 2; - w_metadata.daq_rec = 3; - w_metadata.n_received_packets = 128; - w_metadata.module_id = source_id; - - for (size_t i=0; ipulse_id, pulse_id); - ASSERT_EQ(r_metadata->module_id, source_id); - ASSERT_EQ(r_metadata->frame_index, 2); - ASSERT_EQ(r_metadata->daq_rec, 3); - ASSERT_EQ(r_metadata->n_received_packets, 128); - - // Data as well. - auto offset = MODULE_N_PIXELS * (pulse_id-1); - for (size_t i=0; ipulse_id, 0); - ASSERT_EQ(r_metadata->frame_index, 0); - ASSERT_EQ(r_metadata->daq_rec, 0); - ASSERT_EQ(r_metadata->n_received_packets, 0); - ASSERT_EQ(r_metadata->module_id, 0); - } - - reader.close_file(); -} From e8454ee5950107e1bfa76fad6667ce9a490ed1ee Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 18:27:43 +0200 Subject: [PATCH 064/111] Rollback direct chunk writing --- sf-writer/include/WriterZmqReceiver.hpp | 34 ------ sf-writer/src/JFH5Writer.cpp | 39 ++++-- sf-writer/src/WriterZmqReceiver.cpp | 138 ---------------------- sf-writer/test/test_WriterZmqReceiver.cpp | 80 ------------- 4 files changed, 31 insertions(+), 260 deletions(-) delete mode 100644 sf-writer/include/WriterZmqReceiver.hpp delete mode 100644 sf-writer/src/WriterZmqReceiver.cpp delete mode 100644 sf-writer/test/test_WriterZmqReceiver.cpp diff --git a/sf-writer/include/WriterZmqReceiver.hpp b/sf-writer/include/WriterZmqReceiver.hpp deleted file mode 100644 index 96bc16c..0000000 --- a/sf-writer/include/WriterZmqReceiver.hpp +++ /dev/null @@ -1,34 +0,0 @@ -#ifndef SF_DAQ_BUFFER_WRITERZMQRECEIVER_HPP -#define SF_DAQ_BUFFER_WRITERZMQRECEIVER_HPP - -#include -#include "JFH5Writer.hpp" -#include -#include - - -class WriterZmqReceiver { - - const size_t n_modules_; - std::vector sockets_; - const uint64_t stop_pulse_id_; - - ModuleFrame f_meta_; - -public: - WriterZmqReceiver( - void *ctx, - const std::string& ipc_prefix, - const size_t n_modules, - const uint64_t stop_pulse_id); - - virtual ~WriterZmqReceiver(); - - void get_next_buffer( - const uint64_t start_pulse_id, - ImageMetadataBlock* i_meta, - char* image_buffer); -}; - - -#endif //SF_DAQ_BUFFER_WRITERZMQRECEIVER_HPP diff --git a/sf-writer/src/JFH5Writer.cpp b/sf-writer/src/JFH5Writer.cpp index 1fdb94a..33bfe9f 100644 --- a/sf-writer/src/JFH5Writer.cpp +++ b/sf-writer/src/JFH5Writer.cpp @@ -138,14 +138,37 @@ void JFH5Writer::write( throw runtime_error("Received unexpected block for stop_pulse_id."); } - hsize_t offset[] = {current_write_index_, 0, 0}; - H5DOwrite_chunk( - image_dataset_.getId(), - H5P_DEFAULT, - 0, - offset, - n_images_to_copy * MODULE_N_BYTES * n_modules_, - data); + hsize_t b_i_dims[3] = {BUFFER_BLOCK_SIZE, + MODULE_Y_SIZE * n_modules_, + MODULE_X_SIZE}; + H5::DataSpace b_i_space(3, b_i_dims); + hsize_t b_i_count[] = {n_images_to_copy, + MODULE_Y_SIZE * n_modules_, + MODULE_X_SIZE}; + hsize_t b_i_start[] = {current_write_index_, 0, 0}; + b_i_space.selectHyperslab(H5S_SELECT_SET, b_i_count, b_i_start); + + hsize_t f_i_dims[3] = {n_images_, + MODULE_Y_SIZE * n_modules_, + MODULE_X_SIZE}; + H5::DataSpace f_i_space(3, f_i_dims); + hsize_t f_i_count[] = {n_images_to_copy, + MODULE_Y_SIZE * n_modules_, + MODULE_X_SIZE}; + hsize_t f_i_start[] = {current_write_index_, 0, 0}; + f_i_space.selectHyperslab(H5S_SELECT_SET, f_i_count, f_i_start); + + image_dataset_.write( + data, H5::PredType::NATIVE_UINT16, b_i_space, f_i_space); + +// hsize_t offset[] = {current_write_index_, 0, 0}; +// H5DOwrite_chunk( +// image_dataset_.getId(), +// H5P_DEFAULT, +// 0, +// offset, +// n_images_to_copy * MODULE_N_BYTES * n_modules_, +// data); // pulse_id { diff --git a/sf-writer/src/WriterZmqReceiver.cpp b/sf-writer/src/WriterZmqReceiver.cpp deleted file mode 100644 index 0d3d8f8..0000000 --- a/sf-writer/src/WriterZmqReceiver.cpp +++ /dev/null @@ -1,138 +0,0 @@ -#include "WriterZmqReceiver.hpp" -#include "zmq.h" -#include "date.h" -#include -#include - -using namespace std; -using namespace core_buffer; - -WriterZmqReceiver::WriterZmqReceiver( - void *ctx, - const string &ipc_prefix, - const size_t n_modules, - const uint64_t stop_pulse_id_) : - n_modules_(n_modules), - sockets_(n_modules), - stop_pulse_id_(stop_pulse_id_) -{ - - for (size_t i = 0; i < n_modules; i++) { - sockets_[i] = zmq_socket(ctx, ZMQ_PULL); - - int rcvhwm = WRITER_RCVHWM; - if (zmq_setsockopt(sockets_[i], ZMQ_RCVHWM, &rcvhwm, - sizeof(rcvhwm)) != 0) { - throw runtime_error(zmq_strerror(errno)); - } - int linger = 0; - if (zmq_setsockopt(sockets_[i], ZMQ_LINGER, &linger, - sizeof(linger)) != 0) { - throw runtime_error(zmq_strerror(errno)); - } - - stringstream ipc_addr; - ipc_addr << ipc_prefix << i; - const auto ipc = ipc_addr.str(); - - if (zmq_connect(sockets_[i], ipc.c_str()) != 0) { - throw runtime_error(zmq_strerror(errno)); - } - } -} - -WriterZmqReceiver::~WriterZmqReceiver() -{ - for (size_t i = 0; i < n_modules_; i++) { - zmq_close(sockets_[i]); - } -} - -void WriterZmqReceiver::get_next_buffer( - const uint64_t start_pulse_id, - ImageMetadataBlock* i_meta, - char* image_buffer) -{ - auto n_images_in_buffer = WRITER_DATA_CACHE_N_IMAGES; - auto images_left = stop_pulse_id_ - start_pulse_id + 1; - if (images_left < n_images_in_buffer) { - n_images_in_buffer = images_left; - } - -// i_meta->n_images = (uint16_t)n_images_in_buffer; - - for (uint64_t i_pulse=0; i_pulsepulse_id[i_pulse] = pulse_id; - i_meta->is_good_image[i_pulse] = 1; - i_meta->frame_index[i_pulse] = 0; - i_meta->daq_rec[i_pulse] = 0; - - for (size_t i_module = 0; i_module < n_modules_; i_module++) { - - auto n_bytes_metadata = zmq_recv( - sockets_[i_module], &f_meta_, sizeof(f_meta_), 0); - - if (n_bytes_metadata != sizeof(f_meta_)) { - throw runtime_error("Wrong number of metadata bytes."); - } - - if (f_meta_.pulse_id == 0) { - i_meta->is_good_image[i_pulse] = 0; - - } else { - if (!pulse_id_initialized) { - // Init the image metadata with the first valid frame. - pulse_id_initialized = true; - - i_meta->frame_index[i_pulse] = f_meta_.frame_index; - i_meta->daq_rec[i_pulse] = f_meta_.daq_rec; - } - - if (f_meta_.pulse_id != i_meta->pulse_id[i_pulse]) { - stringstream err_msg; - - err_msg << "[WriterZmqReceiver::get_next_buffer]"; - err_msg << " Read unexpected pulse_id. "; - err_msg << " Expected " << pulse_id; - err_msg << " received "; - err_msg << f_meta_.pulse_id; - err_msg << " from i_module " << i_module << endl; - - throw runtime_error(err_msg.str()); - } - } - - // Once the image is not good, we don't care to re-flag it. - if (i_meta->is_good_image[i_pulse] == 1) { - - if (f_meta_.frame_index != i_meta->frame_index[i_pulse]) { - i_meta->is_good_image[i_pulse] = 0; - } - - if (f_meta_.daq_rec != i_meta->daq_rec[i_pulse]) { - i_meta->is_good_image[i_pulse] = 0; - } - - if (f_meta_.n_received_packets != JF_N_PACKETS_PER_FRAME) { - i_meta->is_good_image[i_pulse] = 0; - } - } - - auto pulse_offset = i_pulse * n_modules_ * MODULE_N_BYTES ; - auto module_offset = i_module * MODULE_N_BYTES; - - auto n_bytes_image = zmq_recv( - sockets_[i_module], - (image_buffer + pulse_offset + module_offset), - MODULE_N_BYTES, 0); - - if (n_bytes_image != MODULE_N_BYTES) { - throw runtime_error("Wrong number of data bytes."); - } - } - } -} diff --git a/sf-writer/test/test_WriterZmqReceiver.cpp b/sf-writer/test/test_WriterZmqReceiver.cpp deleted file mode 100644 index 402e409..0000000 --- a/sf-writer/test/test_WriterZmqReceiver.cpp +++ /dev/null @@ -1,80 +0,0 @@ -#include -#include "WriterZmqReceiver.hpp" -#include "bitshuffle/bitshuffle.h" -#include -#include -#include "buffer_config.hpp" -#include "zmq.h" - -using namespace std; -using namespace core_buffer; - -TEST(WriterZmqReceiver, basic_test) -{ - size_t n_modules = 4; - uint64_t pulse_id = 12345; - - auto ctx = zmq_ctx_new(); - zmq_ctx_set (ctx, ZMQ_IO_THREADS, 1); - - void* sockets[n_modules]; - for (size_t i = 0; i < n_modules; i++) { - sockets[i] = zmq_socket(ctx, ZMQ_PUSH); - - int linger = 0; - if (zmq_setsockopt(sockets[i], ZMQ_LINGER, &linger, - sizeof(linger)) != 0) { - throw runtime_error(zmq_strerror(errno)); - } - - stringstream ipc_addr; - ipc_addr << REPLAY_STREAM_IPC_URL << i; - const auto ipc = ipc_addr.str(); - - if (zmq_bind(sockets[i], ipc.c_str()) != 0) { - throw runtime_error(zmq_strerror(errno)); - } - } - this_thread::sleep_for(chrono::milliseconds(100)); - - WriterZmqReceiver receiver(ctx, REPLAY_STREAM_IPC_URL, n_modules); - this_thread::sleep_for(chrono::milliseconds(100)); - - size_t compressed_frame_size = 5000; - auto frame_buffer = make_unique(compressed_frame_size); - - ImageMetadata image_metadata; - auto compress_size = bshuf_compress_lz4_bound( - MODULE_N_PIXELS, PIXEL_N_BYTES, MODULE_N_PIXELS); - auto image_buffer = make_unique(compress_size * n_modules); - - for (size_t i = 0; i < n_modules; i++) { - - ReplayModuleFrameBuffer frame_metadata; - frame_metadata.metadata.pulse_id = pulse_id; - frame_metadata.metadata.frame_index = pulse_id + 100; - frame_metadata.metadata.n_received_packets = 128; - frame_metadata.metadata.daq_rec = 4; - - frame_metadata.is_frame_present = 1; - frame_metadata.data_n_bytes = compressed_frame_size; - - zmq_send(sockets[i], - &frame_metadata, - sizeof(ReplayModuleFrameBuffer), - ZMQ_SNDMORE); - - zmq_send(sockets[i], - (char*)(frame_buffer.get()), - compressed_frame_size, - 0); - } - - receiver.get_next_buffer(pulse_id, &image_metadata, image_buffer.get()); - EXPECT_EQ(pulse_id, image_metadata.pulse_id); - EXPECT_EQ(image_metadata.is_good_frame, 1); - EXPECT_EQ(image_metadata.daq_rec, 4); - EXPECT_EQ(image_metadata.data_n_bytes, - 5000*n_modules); -// 5000*n_modules+BSHUF_LZ4_HEADER_BYTES); -} \ No newline at end of file From cfbcbf4c2826cf96b4170edd159539dec55e0ea4 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 18:30:22 +0200 Subject: [PATCH 065/111] Copy complete buffers to file --- sf-writer/src/JFH5Writer.cpp | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/sf-writer/src/JFH5Writer.cpp b/sf-writer/src/JFH5Writer.cpp index 33bfe9f..baa82e9 100644 --- a/sf-writer/src/JFH5Writer.cpp +++ b/sf-writer/src/JFH5Writer.cpp @@ -97,16 +97,27 @@ JFH5Writer::~JFH5Writer() void JFH5Writer::close_file() { + hsize_t b_m_dims[2] = {n_images_, 1}; + H5::DataSpace b_m_space (2, b_m_dims); - pulse_id_dataset_.write(b_pulse_id_, H5::PredType::NATIVE_UINT64); + hsize_t f_m_dims[] = {n_images_, 1}; + H5::DataSpace f_m_space(2, f_m_dims); - frame_index_dataset_.write(b_frame_index_, - H5::PredType::NATIVE_UINT64); + pulse_id_dataset_.write( + b_pulse_id_, H5::PredType::NATIVE_UINT64, + b_m_space, f_m_space); - daq_rec_dataset_.write(b_daq_rec_, H5::PredType::NATIVE_UINT32); + frame_index_dataset_.write( + b_frame_index_, H5::PredType::NATIVE_UINT64, + b_m_space, f_m_space); - is_good_frame_dataset_.write(b_is_good_frame_, - H5::PredType::NATIVE_UINT8); + daq_rec_dataset_.write( + b_daq_rec_, H5::PredType::NATIVE_UINT32, + b_m_space, f_m_space); + + is_good_frame_dataset_.write( + b_is_good_frame_, H5::PredType::NATIVE_UINT8, + b_m_space, f_m_space); image_dataset_.close(); pulse_id_dataset_.close(); From 11df261a89b4db416490ea30aaf6708fdd9990db Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 18:33:32 +0200 Subject: [PATCH 066/111] Fix buffer offset --- sf-writer/src/JFH5Writer.cpp | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/sf-writer/src/JFH5Writer.cpp b/sf-writer/src/JFH5Writer.cpp index baa82e9..9ac58c0 100644 --- a/sf-writer/src/JFH5Writer.cpp +++ b/sf-writer/src/JFH5Writer.cpp @@ -156,7 +156,7 @@ void JFH5Writer::write( hsize_t b_i_count[] = {n_images_to_copy, MODULE_Y_SIZE * n_modules_, MODULE_X_SIZE}; - hsize_t b_i_start[] = {current_write_index_, 0, 0}; + hsize_t b_i_start[] = {n_images_offset, 0, 0}; b_i_space.selectHyperslab(H5S_SELECT_SET, b_i_count, b_i_start); hsize_t f_i_dims[3] = {n_images_, @@ -172,14 +172,6 @@ void JFH5Writer::write( image_dataset_.write( data, H5::PredType::NATIVE_UINT16, b_i_space, f_i_space); -// hsize_t offset[] = {current_write_index_, 0, 0}; -// H5DOwrite_chunk( -// image_dataset_.getId(), -// H5P_DEFAULT, -// 0, -// offset, -// n_images_to_copy * MODULE_N_BYTES * n_modules_, -// data); // pulse_id { From eb07d91f7b5f5f7e5cfdf198f7aaebf46a6bbcc3 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 18:40:43 +0200 Subject: [PATCH 067/111] Cleanup JF writer --- sf-writer/include/JFH5Writer.hpp | 5 --- sf-writer/src/JFH5Writer.cpp | 57 +++++++++++++++----------------- 2 files changed, 26 insertions(+), 36 deletions(-) diff --git a/sf-writer/include/JFH5Writer.hpp b/sf-writer/include/JFH5Writer.hpp index 6d51341..cc5fbc7 100644 --- a/sf-writer/include/JFH5Writer.hpp +++ b/sf-writer/include/JFH5Writer.hpp @@ -18,11 +18,6 @@ class JFH5Writer { H5::H5File file_; H5::DataSet image_dataset_; - H5::DataSet pulse_id_dataset_; - H5::DataSet frame_index_dataset_; - H5::DataSet daq_rec_dataset_; - H5::DataSet is_good_frame_dataset_; - public: JFH5Writer(const std::string& output_file, diff --git a/sf-writer/src/JFH5Writer.cpp b/sf-writer/src/JFH5Writer.cpp index 9ac58c0..7823ea2 100644 --- a/sf-writer/src/JFH5Writer.cpp +++ b/sf-writer/src/JFH5Writer.cpp @@ -56,28 +56,7 @@ JFH5Writer::JFH5Writer(const std::string& output_file, image_dataspace, image_dataset_properties); - hsize_t metadata_dataset_dims[] = {n_images_, 1}; - H5::DataSpace metadata_dataspace(2, metadata_dataset_dims); - pulse_id_dataset_ = file_.createDataSet( - "pulse_id", - H5::PredType::NATIVE_UINT64, - metadata_dataspace); - - frame_index_dataset_ = file_.createDataSet( - "frame_index", - H5::PredType::NATIVE_UINT64, - metadata_dataspace); - - daq_rec_dataset_ = file_.createDataSet( - "daq_rec", - H5::PredType::NATIVE_UINT32, - metadata_dataspace); - - is_good_frame_dataset_ = file_.createDataSet( - "is_good_frame", - H5::PredType::NATIVE_UINT8, - metadata_dataspace); b_pulse_id_ = new uint64_t[n_images_]; b_frame_index_= new uint64_t[n_images_]; @@ -97,33 +76,49 @@ JFH5Writer::~JFH5Writer() void JFH5Writer::close_file() { + image_dataset_.close(); + hsize_t b_m_dims[2] = {n_images_, 1}; H5::DataSpace b_m_space (2, b_m_dims); hsize_t f_m_dims[] = {n_images_, 1}; H5::DataSpace f_m_space(2, f_m_dims); - pulse_id_dataset_.write( + auto pulse_id_dataset = file_.createDataSet( + "pulse_id", + H5::PredType::NATIVE_UINT64, + f_m_space); + pulse_id_dataset.write( b_pulse_id_, H5::PredType::NATIVE_UINT64, b_m_space, f_m_space); + pulse_id_dataset.close(); - frame_index_dataset_.write( + auto frame_index_dataset = file_.createDataSet( + "frame_index", + H5::PredType::NATIVE_UINT64, + f_m_space); + frame_index_dataset.write( b_frame_index_, H5::PredType::NATIVE_UINT64, b_m_space, f_m_space); + frame_index_dataset.close(); - daq_rec_dataset_.write( + auto daq_rec_dataset = file_.createDataSet( + "daq_rec", + H5::PredType::NATIVE_UINT32, + f_m_space); + daq_rec_dataset.write( b_daq_rec_, H5::PredType::NATIVE_UINT32, b_m_space, f_m_space); + daq_rec_dataset.close(); - is_good_frame_dataset_.write( + auto is_good_frame_dataset = file_.createDataSet( + "is_good_frame", + H5::PredType::NATIVE_UINT8, + f_m_space); + is_good_frame_dataset.write( b_is_good_frame_, H5::PredType::NATIVE_UINT8, b_m_space, f_m_space); - - image_dataset_.close(); - pulse_id_dataset_.close(); - frame_index_dataset_.close(); - daq_rec_dataset_.close(); - is_good_frame_dataset_.close(); + is_good_frame_dataset.close(); file_.close(); } From 4f044d6b69a7f9667f046d9ff7c4e1deb351696a Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 18:43:24 +0200 Subject: [PATCH 068/111] Add check if file is already closed --- sf-writer/src/JFH5Writer.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sf-writer/src/JFH5Writer.cpp b/sf-writer/src/JFH5Writer.cpp index 7823ea2..44588af 100644 --- a/sf-writer/src/JFH5Writer.cpp +++ b/sf-writer/src/JFH5Writer.cpp @@ -76,6 +76,10 @@ JFH5Writer::~JFH5Writer() void JFH5Writer::close_file() { + if (file_.getId() == -1) { + return; + } + image_dataset_.close(); hsize_t b_m_dims[2] = {n_images_, 1}; From 7ca8630681632f288caa79021e0692a7d490aca2 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Wed, 27 May 2020 18:53:13 +0200 Subject: [PATCH 069/111] Internalize file closing --- sf-writer/include/JFH5Writer.hpp | 4 +++- sf-writer/src/main.cpp | 2 -- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sf-writer/include/JFH5Writer.hpp b/sf-writer/include/JFH5Writer.hpp index cc5fbc7..7432122 100644 --- a/sf-writer/include/JFH5Writer.hpp +++ b/sf-writer/include/JFH5Writer.hpp @@ -19,6 +19,8 @@ class JFH5Writer { H5::DataSet image_dataset_; + void close_file(); + public: JFH5Writer(const std::string& output_file, const uint64_t start_pulse_id, @@ -26,7 +28,7 @@ public: const size_t n_modules); ~JFH5Writer(); void write(const ImageMetadataBlock* metadata, const char* data); - void close_file(); + uint64_t* b_pulse_id_; uint64_t* b_frame_index_; diff --git a/sf-writer/src/main.cpp b/sf-writer/src/main.cpp index e946937..0c25efb 100644 --- a/sf-writer/src/main.cpp +++ b/sf-writer/src/main.cpp @@ -143,8 +143,6 @@ int main (int argc, char *argv[]) cout << write_us_duration / BUFFER_BLOCK_SIZE << endl; } - writer.close_file(); - for (auto& reading_thread :reading_threads) { if (reading_thread.joinable()) { reading_thread.join(); From dce0cc162895a4db9661c9b8414ecf7d0649121b Mon Sep 17 00:00:00 2001 From: Dmitry Ozerov Date: Tue, 2 Jun 2020 09:43:46 +0200 Subject: [PATCH 070/111] new pedestal for Bernina 16M and json library in makefile --- sf-stream/CMakeLists.txt | 5 +++-- sf-stream/src/main.cpp | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/sf-stream/CMakeLists.txt b/sf-stream/CMakeLists.txt index 216e2dd..5ea58fa 100644 --- a/sf-stream/CMakeLists.txt +++ b/sf-stream/CMakeLists.txt @@ -12,7 +12,8 @@ set_target_properties(sf-stream PROPERTIES OUTPUT_NAME sf_stream) target_link_libraries(sf-stream sf-stream-lib zmq - pthread) + pthread + jsoncpp) enable_testing() -add_subdirectory(test/) \ No newline at end of file +add_subdirectory(test/) diff --git a/sf-stream/src/main.cpp b/sf-stream/src/main.cpp index 2c236ef..0221b16 100644 --- a/sf-stream/src/main.cpp +++ b/sf-stream/src/main.cpp @@ -120,7 +120,7 @@ int main (int argc, char *argv[]) header["pulse_id"] = (Json::Value::UInt64)pulse_id; //this needs to be re-read from external source - header["pedestal_file"] = "/sf/bernina/data/p17534/res/JF_pedestals/pedestal_20200423_1018.JF07T32V01.res.h5"; + header["pedestal_file"] = "/sf/bernina/data/p17534/res/JF_pedestals/pedestal_20200529_1408.JF07T32V01.res.h5"; header["gain_file"] = "/sf/bernina/config/jungfrau/gainMaps/JF07T32V01/gains.h5"; header["number_frames_expected"] = 10000; From c0ef36562870f382a1630fcd996ac0bb9b4f9cea Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 2 Jun 2020 10:20:57 +0200 Subject: [PATCH 071/111] Replace CppJSON with rapidjson --- sf-stream/CMakeLists.txt | 1 + sf-stream/src/main.cpp | 79 ++++++++++++++++++++++++++-------------- 2 files changed, 52 insertions(+), 28 deletions(-) diff --git a/sf-stream/CMakeLists.txt b/sf-stream/CMakeLists.txt index 216e2dd..5add224 100644 --- a/sf-stream/CMakeLists.txt +++ b/sf-stream/CMakeLists.txt @@ -10,6 +10,7 @@ target_link_libraries(sf-stream-lib add_executable(sf-stream src/main.cpp) set_target_properties(sf-stream PROPERTIES OUTPUT_NAME sf_stream) target_link_libraries(sf-stream + external sf-stream-lib zmq pthread) diff --git a/sf-stream/src/main.cpp b/sf-stream/src/main.cpp index 2c236ef..6ce5598 100644 --- a/sf-stream/src/main.cpp +++ b/sf-stream/src/main.cpp @@ -10,8 +10,11 @@ #include #include #include -#include "date.h" -#include +#include "rapidjson/document.h" +#include "rapidjson/stringbuffer.h" +#include "rapidjson/writer.h" + + using namespace std; using namespace core_buffer; @@ -59,8 +62,13 @@ int main (int argc, char *argv[]) } uint16_t data_empty [] = { 0, 0, 0, 0}; - Json::Value header; - Json::StreamWriterBuilder builder; + + rapidjson::Document header(rapidjson::kObjectType); + auto& header_alloc = header.GetAllocator(); + + rapidjson::StringBuffer header_buffer; + rapidjson::Writer header_writer(header_buffer); + header.Accept(header_writer); // TODO: Remove stats trash. int stats_counter = 0; @@ -112,39 +120,52 @@ int main (int argc, char *argv[]) } } - //Here we need to send to streamvis and live analysis metadata(probably need to operate still on them) and data(not every frame) + // TODO: Here we need to send to streamvis and live analysis metadata(probably need to operate still on them) and data(not every frame) - header["frame"] = (Json::Value::UInt64)frame_index; - header["is_good_frame"] = is_good_frame; - header["daq_rec"] = (Json::Value::UInt64)daq_rec; - header["pulse_id"] = (Json::Value::UInt64)pulse_id; + header.AddMember("frame", frame_index, header_alloc); + header.AddMember("is_good_frame", is_good_frame, header_alloc); + header.AddMember("daq_rec", daq_rec, header_alloc); + header.AddMember("pulse_id", pulse_id, header_alloc); - //this needs to be re-read from external source - header["pedestal_file"] = "/sf/bernina/data/p17534/res/JF_pedestals/pedestal_20200423_1018.JF07T32V01.res.h5"; - header["gain_file"] = "/sf/bernina/config/jungfrau/gainMaps/JF07T32V01/gains.h5"; + // TODO: this needs to be re-read from external source + header.AddMember("pedestal_file", "/sf/bernina/data/p17534/res/JF_pedestals/pedestal_20200423_1018.JF07T32V01.res.h5", header_alloc); + header.AddMember("gain_file", "/sf/bernina/config/jungfrau/gainMaps/JF07T32V01/gains.h5", header_alloc); - header["number_frames_expected"] = 10000; - header["run_name"] = to_string(uint64_t(pulse_id/10000)*10000); + header.AddMember("number_frames_expected", 10000, header_alloc); - // detector name should come as parameter to sf_stream - header["detector_name"] = "JF07T32V01"; + rapidjson::Value run_name; + run_name.SetString( + to_string(uint64_t(pulse_id/10000)*10000).c_str(), + header_alloc); + header.AddMember("run_name", run_name, header_alloc); - header["htype"] = "array-1.0"; - header["type"] = "uint16"; + // TODO: Detector name should come as parameter to sf_stream + header.AddMember("number_frames_expected", "JF07T32V01", header_alloc); + + header.AddMember("htype", "array-1.0", header_alloc); + header.AddMember("type", "uint16", header_alloc); + + // To be retrieved and filled with correct values down. + auto shape_value = rapidjson::Value(rapidjson::kArrayType); + shape_value.PushBack((uint64_t)0, header_alloc); + shape_value.PushBack((uint64_t)0, header_alloc); + header.AddMember("shape", shape_value, header_alloc); int send_streamvis = 0; if ( reduction_factor_streamvis > 1 ) { send_streamvis = rand() % reduction_factor_streamvis; } if ( send_streamvis == 0 ) { - header["shape"][0] = 16384; - header["shape"][1] = 1024; + auto& shape = header["shape"]; + shape[0] = 16384; + shape[1] = 1024; } else{ - header["shape"][0] = 2; - header["shape"][1] = 2; + auto& shape = header["shape"]; + shape[0] = 2; + shape[1] = 2; } - string text_header = Json::writeString(builder, header); + string text_header = header_buffer.GetString(); zmq_send(socket_streamvis, text_header.c_str(), @@ -169,14 +190,16 @@ int main (int argc, char *argv[]) send_live_analysis = rand() % reduction_factor_live_analysis; } if ( send_live_analysis == 0 ) { - header["shape"][0] = 16384; - header["shape"][1] = 1024; + auto& shape = header["shape"]; + shape[0] = 16384; + shape[1] = 1024; } else{ - header["shape"][0] = 2; - header["shape"][1] = 2; + auto& shape = header["shape"]; + shape[0] = 2; + shape[1] = 2; } - text_header = Json::writeString(builder, header); + text_header = header_buffer.GetString(); zmq_send(socket_live, text_header.c_str(), From 3fc9f38e537563d068b7c93b4c2d0c6336c0acd1 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 2 Jun 2020 10:26:13 +0200 Subject: [PATCH 072/111] Improved pedestal situation --- sf-stream/src/main.cpp | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/sf-stream/src/main.cpp b/sf-stream/src/main.cpp index 6ce5598..e977c45 100644 --- a/sf-stream/src/main.cpp +++ b/sf-stream/src/main.cpp @@ -19,6 +19,13 @@ using namespace std; using namespace core_buffer; +// TODO: this needs to be re-read from external source +const string PEDE_FILENAME = + "/sf/bernina/data/p17534/res/JF_pedestals/pedestal_20200529_1408.JF07T32V01.res.h5"; +const string GAIN_FILENAME = + "/sf/bernina/config/jungfrau/gainMaps/JF07T32V01/gains.h5"; + + int main (int argc, char *argv[]) { if (argc != 5) { @@ -127,9 +134,13 @@ int main (int argc, char *argv[]) header.AddMember("daq_rec", daq_rec, header_alloc); header.AddMember("pulse_id", pulse_id, header_alloc); - // TODO: this needs to be re-read from external source - header.AddMember("pedestal_file", "/sf/bernina/data/p17534/res/JF_pedestals/pedestal_20200423_1018.JF07T32V01.res.h5", header_alloc); - header.AddMember("gain_file", "/sf/bernina/config/jungfrau/gainMaps/JF07T32V01/gains.h5", header_alloc); + rapidjson::Value pedestal_file; + pedestal_file.SetString(PEDE_FILENAME.c_str(), header_alloc); + header.AddMember("pedestal_file", pedestal_file, header_alloc); + + rapidjson::Value gain_file; + gain_file.SetString(PEDE_FILENAME.c_str(), header_alloc); + header.AddMember("gain_file", gain_file, header_alloc); header.AddMember("number_frames_expected", 10000, header_alloc); From f9152fa86159ac695ddb1ab230395b1ee2ec5196 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 2 Jun 2020 10:48:21 +0200 Subject: [PATCH 073/111] Remove manual writer tests --- sf-writer/test/CMakeLists.txt | 11 -- sf-writer/test/manual/test_sf_writer_recv.cpp | 130 ------------------ 2 files changed, 141 deletions(-) delete mode 100644 sf-writer/test/manual/test_sf_writer_recv.cpp diff --git a/sf-writer/test/CMakeLists.txt b/sf-writer/test/CMakeLists.txt index 7666c3e..13dd5c0 100644 --- a/sf-writer/test/CMakeLists.txt +++ b/sf-writer/test/CMakeLists.txt @@ -7,14 +7,3 @@ target_link_libraries(sf-writer-tests zmq gtest ) - -add_executable(sf-writer-recv manual/test_sf_writer_recv.cpp) -set_target_properties(sf-writer-recv PROPERTIES OUTPUT_NAME sf_writer) -set_target_properties(sf-writer-recv PROPERTIES EXCLUDE_FROM_ALL TRUE) -target_link_libraries(sf-writer-recv - sf-writer-lib - zmq - hdf5 - hdf5_cpp - pthread - ) \ No newline at end of file diff --git a/sf-writer/test/manual/test_sf_writer_recv.cpp b/sf-writer/test/manual/test_sf_writer_recv.cpp deleted file mode 100644 index b78c48f..0000000 --- a/sf-writer/test/manual/test_sf_writer_recv.cpp +++ /dev/null @@ -1,130 +0,0 @@ -#include -#include "buffer_config.hpp" -#include "zmq.h" -#include -#include -#include -#include "JFH5Writer.hpp" -#include -#include -#include "date.h" -#include "bitshuffle/bitshuffle.h" -#include "WriterZmqReceiver.hpp" - -using namespace std; -using namespace core_buffer; - -void receive_replay( - void* ctx, - const string ipc_prefix, - const size_t n_modules, - FastQueue& queue, - const uint64_t start_pulse_id, - const uint64_t stop_pulse_id) -{ - try { - WriterZmqReceiver receiver(ctx, ipc_prefix, n_modules, stop_pulse_id); - - int slot_id; - while((slot_id = queue.reserve()) == -1) { - this_thread::sleep_for(chrono::milliseconds( - RB_READ_RETRY_INTERVAL_MS)); - } - - uint64_t pulse_id = start_pulse_id; - // "<= stop_pulse_id" because we include the last pulse_id. - while(pulse_id <= stop_pulse_id) { - auto start_time = chrono::steady_clock::now(); - - auto image_metadata = queue.get_metadata_buffer(slot_id); - auto image_buffer = queue.get_data_buffer(slot_id); - - receiver.get_next_buffer(pulse_id, image_metadata, image_buffer); - - pulse_id += image_metadata->n_images; - - auto end_time = chrono::steady_clock::now(); - auto read_us_duration = chrono::duration_cast( - end_time-start_time).count(); - - cout << "sf_writer::avg_read_us "; - cout << read_us_duration / image_metadata->n_images << endl; - } - - queue.commit(); - - } catch (const std::exception& e) { - using namespace date; - using namespace chrono; - - cout << "[" << system_clock::now() << "]"; - cout << "[sf_writer::receive_replay]"; - cout << " Stopped because of exception: " << endl; - cout << e.what() << endl; - - throw; - } -} - -int main (int argc, char *argv[]) -{ - if (argc != 5) { - cout << endl; - cout << "Usage: sf_writer "; - cout << " [ipc_id] [output_file] [start_pulse_id] [stop_pulse_id]"; - cout << endl; - cout << "\tipc_id: Unique identifier for ipc." << endl; - cout << "\toutput_file: Complete path to the output file." << endl; - cout << "\tstart_pulse_id: Start pulse_id of retrieval." << endl; - cout << "\tstop_pulse_id: Stop pulse_id of retrieval." << endl; - cout << endl; - - exit(-1); - } - - const string ipc_id = string(argv[1]); - string output_file = string(argv[2]); - uint64_t start_pulse_id = (uint64_t) atoll(argv[3]); - uint64_t stop_pulse_id = (uint64_t) atoll(argv[4]); - - size_t n_modules = 32; - - FastQueue queue( - MODULE_N_BYTES * n_modules * WRITER_DATA_CACHE_N_IMAGES, - WRITER_FASTQUEUE_N_SLOTS); - - auto ctx = zmq_ctx_new(); - zmq_ctx_set (ctx, ZMQ_IO_THREADS, WRITER_ZMQ_IO_THREADS); - - auto ipc_base = REPLAY_STREAM_IPC_URL + ipc_id + "-"; - - receive_replay( - ctx, ipc_base, n_modules,queue, start_pulse_id, stop_pulse_id); - -// -// -// auto current_pulse_id = start_pulse_id; -// // "<= stop_pulse_id" because we include the last pulse_id. -// while (current_pulse_id <= stop_pulse_id) { -// -// int slot_id; -// while((slot_id = queue.read()) == -1) { -// this_thread::sleep_for(chrono::milliseconds( -// RB_READ_RETRY_INTERVAL_MS)); -// } -// -// auto metadata = queue.get_metadata_buffer(slot_id); -// -// for (int i_pulse=0; i_pulse < metadata->n_images; i_pulse++) { -// cout << "Written image " << metadata->pulse_id[i_pulse] << endl; -// -// } -// -// queue.release(); -// current_pulse_id += metadata->n_images; -// } -// -// //wait till receive thread is finished -// replay_receive_thread.join(); - return 0; -} From 78a6da4a82357c83739491137492c08dadb0e4cc Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 2 Jun 2020 10:48:44 +0200 Subject: [PATCH 074/111] Remove non-existing test --- sf-writer/test/main.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/sf-writer/test/main.cpp b/sf-writer/test/main.cpp index 7078be5..296a468 100644 --- a/sf-writer/test/main.cpp +++ b/sf-writer/test/main.cpp @@ -1,5 +1,4 @@ #include "gtest/gtest.h" -#include "test_WriterZmqReceiver.cpp" #include "test_WriterH5Writer.cpp" using namespace std; From 95be4ae7597b9debf548a3626971bd984b878137 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 2 Jun 2020 10:51:53 +0200 Subject: [PATCH 075/111] Update terminology definition --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5811367..50b4124 100644 --- a/README.md +++ b/README.md @@ -124,4 +124,6 @@ In order to unify the way we write code and talk about concept the following terminology definitions should be followed: - frame (data from a single module) -- image (data of the assembled image) \ No newline at end of file +- image (data of the assembled image) +- start_pulse_id and stop_pulse_id (not end_pulse_id) is used to determine the +inclusive range (both start and stop pulse_id are included) of pulses. \ No newline at end of file From 68bf6e988e017c3b4e1aa2313ad450b946c75ce3 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 2 Jun 2020 10:54:07 +0200 Subject: [PATCH 076/111] Rename struct according to terminology --- core-buffer/include/formats.hpp | 4 ++-- sf-writer/src/ImageAssembler.cpp | 4 ++-- sf-writer/src/JFH5Writer.cpp | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/core-buffer/include/formats.hpp b/core-buffer/include/formats.hpp index 72222b0..8dec7c9 100644 --- a/core-buffer/include/formats.hpp +++ b/core-buffer/include/formats.hpp @@ -10,8 +10,8 @@ struct ImageMetadataBlock uint64_t frame_index[core_buffer::BUFFER_BLOCK_SIZE]; uint32_t daq_rec[core_buffer::BUFFER_BLOCK_SIZE]; uint8_t is_good_image[core_buffer::BUFFER_BLOCK_SIZE]; - uint64_t block_first_pulse_id; - uint64_t block_last_pulse_id; + uint64_t block_start_pulse_id; + uint64_t block_stop_pulse_id; }; const char BUFFER_FORMAT_START_BYTE = 0xBE; diff --git a/sf-writer/src/ImageAssembler.cpp b/sf-writer/src/ImageAssembler.cpp index fe271c4..5f638fd 100644 --- a/sf-writer/src/ImageAssembler.cpp +++ b/sf-writer/src/ImageAssembler.cpp @@ -42,9 +42,9 @@ void ImageAssembler::process( // TODO: Temp workaround. Make proper initialization. if (i_module == 0) { - metadata_buffer_[slot_id].block_first_pulse_id = + metadata_buffer_[slot_id].block_start_pulse_id = block_buffer->frame[0].metadata.pulse_id; - metadata_buffer_[slot_id].block_last_pulse_id = + metadata_buffer_[slot_id].block_stop_pulse_id = block_buffer->frame[BUFFER_BLOCK_SIZE-1].metadata.pulse_id; } diff --git a/sf-writer/src/JFH5Writer.cpp b/sf-writer/src/JFH5Writer.cpp index 44588af..f798b64 100644 --- a/sf-writer/src/JFH5Writer.cpp +++ b/sf-writer/src/JFH5Writer.cpp @@ -131,8 +131,8 @@ void JFH5Writer::write( const ImageMetadataBlock* metadata, const char* data) { size_t n_images_offset = 0; - if (start_pulse_id_ > metadata->block_first_pulse_id) { - n_images_offset = start_pulse_id_ - metadata->block_first_pulse_id; + if (start_pulse_id_ > metadata->block_start_pulse_id) { + n_images_offset = start_pulse_id_ - metadata->block_start_pulse_id; } if (n_images_offset > BUFFER_BLOCK_SIZE) { @@ -140,8 +140,8 @@ void JFH5Writer::write( } size_t n_images_to_copy = BUFFER_BLOCK_SIZE - n_images_offset; - if (stop_pulse_id_ < metadata->block_last_pulse_id) { - n_images_to_copy -= metadata->block_last_pulse_id - stop_pulse_id_; + if (stop_pulse_id_ < metadata->block_stop_pulse_id) { + n_images_to_copy -= metadata->block_stop_pulse_id - stop_pulse_id_; } if (n_images_to_copy < 1) { From ecf5f92fc896452cdd1dd4f68d6dcb041f500874 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 2 Jun 2020 10:58:18 +0200 Subject: [PATCH 077/111] Fix basic writer test --- sf-writer/test/test_WriterH5Writer.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/sf-writer/test/test_WriterH5Writer.cpp b/sf-writer/test/test_WriterH5Writer.cpp index abcc4be..d3850f3 100644 --- a/sf-writer/test/test_WriterH5Writer.cpp +++ b/sf-writer/test/test_WriterH5Writer.cpp @@ -1,26 +1,27 @@ +#include #include "JFH5Writer.hpp" #include "gtest/gtest.h" #include "bitshuffle/bitshuffle.h" - +using namespace std; using namespace core_buffer; TEST(WriterH5Writer, basic_interaction) { size_t n_modules = 2; - size_t n_frames = 5; + uint64_t start_pulse_id = 1; + uint64_t end_pulse_id = 5; - auto data = make_unique(n_modules*MODULE_N_BYTES); + auto data = make_unique(n_modules*MODULE_N_BYTES*BUFFER_BLOCK_SIZE); auto metadata = make_shared(); // Needed by writer. - metadata->data_n_bytes[0] = 500; - metadata->n_pulses_in_buffer = 1; + metadata->block_start_pulse_id = 0; + metadata->block_stop_pulse_id = BUFFER_BLOCK_SIZE - 1; - JFH5Writer writer("ignore.h5", n_frames, n_modules); + JFH5Writer writer("ignore.h5", start_pulse_id, end_pulse_id, n_modules); writer.write(metadata.get(), data.get()); - writer.close_file(); } TEST(WriterH5Writer, test_compression) From 7a45802909062368579d91ba5e6e359ed2d01fa1 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 2 Jun 2020 11:32:01 +0200 Subject: [PATCH 078/111] Add basic writing test --- sf-writer/test/test_WriterH5Writer.cpp | 153 ++++++++++++++----------- 1 file changed, 86 insertions(+), 67 deletions(-) diff --git a/sf-writer/test/test_WriterH5Writer.cpp b/sf-writer/test/test_WriterH5Writer.cpp index d3850f3..c786747 100644 --- a/sf-writer/test/test_WriterH5Writer.cpp +++ b/sf-writer/test/test_WriterH5Writer.cpp @@ -11,7 +11,7 @@ TEST(WriterH5Writer, basic_interaction) { size_t n_modules = 2; uint64_t start_pulse_id = 1; - uint64_t end_pulse_id = 5; + uint64_t stop_pulse_id = 5; auto data = make_unique(n_modules*MODULE_N_BYTES*BUFFER_BLOCK_SIZE); auto metadata = make_shared(); @@ -20,74 +20,93 @@ TEST(WriterH5Writer, basic_interaction) metadata->block_start_pulse_id = 0; metadata->block_stop_pulse_id = BUFFER_BLOCK_SIZE - 1; - JFH5Writer writer("ignore.h5", start_pulse_id, end_pulse_id, n_modules); + JFH5Writer writer("ignore.h5", start_pulse_id, stop_pulse_id, n_modules); writer.write(metadata.get(), data.get()); } -TEST(WriterH5Writer, test_compression) +TEST(WriterH5Writer, test_writing) { -// size_t n_modules = 2; -// size_t n_frames = 2; -// -// auto comp_buffer_size = bshuf_compress_lz4_bound( -// MODULE_N_PIXELS, PIXEL_N_BYTES, MODULE_N_PIXELS); -// -// auto f_raw_buffer = make_unique(MODULE_N_PIXELS); -// auto f_comp_buffer = make_unique(comp_buffer_size); -// -// auto i_comp_buffer = make_unique( -// (comp_buffer_size * n_modules) + BSHUF_LZ4_HEADER_BYTES); -// auto i_raw_buffer = make_unique( -// MODULE_N_PIXELS * n_modules * n_frames); -// -// bshuf_write_uint64_BE(&i_comp_buffer[0], -// MODULE_N_BYTES * n_modules); -// bshuf_write_uint32_BE(&i_comp_buffer[8], -// MODULE_N_PIXELS * PIXEL_N_BYTES); -// -// size_t total_compressed_size = BSHUF_LZ4_HEADER_BYTES; -// for (int i_module=0; i_module(); -// metadata->data_n_bytes = total_compressed_size; -// -// metadata->is_good_frame = 1; -// metadata->frame_index = 3; -// metadata->pulse_id = 3; -// metadata->daq_rec = 3; -// -// auto result = bshuf_decompress_lz4( -// &i_comp_buffer[12], &i_raw_buffer[0], -// MODULE_N_PIXELS*n_modules, PIXEL_N_BYTES, MODULE_N_PIXELS); -// -// WriterH5Writer writer("ignore.h5", n_frames, n_modules); -// writer.write(metadata.get(), &i_comp_buffer[0]); -// writer.close_file(); -// -// H5::H5File reader("ignore.h5", H5F_ACC_RDONLY); -// auto image_dataset = reader.openDataSet("image"); -// image_dataset.read(&i_raw_buffer[0], H5::PredType::NATIVE_UINT16); -// -// for (int i_module=0; i_module(); + metadata->block_start_pulse_id = 0; + metadata->block_stop_pulse_id = BUFFER_BLOCK_SIZE - 1; + + for (uint64_t pulse_id=start_pulse_id; + pulse_id<=stop_pulse_id; + pulse_id++) { + + metadata->pulse_id[pulse_id] = pulse_id; + metadata->frame_index[pulse_id] = pulse_id + 10; + metadata->daq_rec[pulse_id] = pulse_id + 100; + metadata->is_good_image[pulse_id] = 1; + } + + + auto image_buffer = make_unique( + MODULE_N_PIXELS * n_modules * BUFFER_BLOCK_SIZE); + + for (int i_block=0; i_block<=BUFFER_BLOCK_SIZE; i_block++) { + for (int i_module=0; i_module(n_images); + auto pulse_id_dataset = reader.openDataSet("pulse_id"); + pulse_id_dataset.read(&pulse_id_data[0], H5::PredType::NATIVE_UINT64); + + auto frame_index_data = make_unique(n_images); + auto frame_index_dataset = reader.openDataSet("frame_index"); + frame_index_dataset.read(&frame_index_data[0], H5::PredType::NATIVE_UINT64); + + auto daq_rec_data = make_unique(n_images); + auto daq_rec_dataset = reader.openDataSet("daq_rec"); + daq_rec_dataset.read(&daq_rec_data[0], H5::PredType::NATIVE_UINT32); + + auto is_good_frame_data = make_unique(n_images); + auto is_good_frame_dataset = reader.openDataSet("is_good_frame"); + is_good_frame_dataset.read( + &is_good_frame_data[0], H5::PredType::NATIVE_UINT8); + + for (uint64_t pulse_id=start_pulse_id; + pulse_id<=stop_pulse_id; + pulse_id++) { + + ASSERT_EQ(pulse_id_data[pulse_id - start_pulse_id], pulse_id); + ASSERT_EQ(frame_index_data[pulse_id - start_pulse_id], pulse_id + 10); + ASSERT_EQ(daq_rec_data[pulse_id - start_pulse_id], pulse_id + 100); + ASSERT_EQ(is_good_frame_data[pulse_id - start_pulse_id], 1); + } } \ No newline at end of file From 042726002c09a6392e1e7ff04d8a7db1bb8418c2 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 2 Jun 2020 11:36:33 +0200 Subject: [PATCH 079/111] Change new function call with old --- sf-buffer/src/WriterUtils.cpp | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/sf-buffer/src/WriterUtils.cpp b/sf-buffer/src/WriterUtils.cpp index 4adfeba..41e5767 100644 --- a/sf-buffer/src/WriterUtils.cpp +++ b/sf-buffer/src/WriterUtils.cpp @@ -45,17 +45,7 @@ void WriterUtils::create_destination_folder(const string& output_file) string output_folder(output_file.substr(0, file_separator_index)); - if(!filesystem::create_directories(output_folder)) { - stringstream err_msg; - - using namespace date; - using namespace chrono; - err_msg << "[" << system_clock::now() << "]"; - err_msg << "[WriterUtils::create_destination_folder]"; - err_msg << " Cannot create directory "; - err_msg << output_folder << endl; - - throw runtime_error(err_msg.str()); - } + string create_folder_command("mkdir -p " + output_folder); + system(create_folder_command.c_str()); } } From 0c5337db6bd60034b5abfee2833b3c82d172a167 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 2 Jun 2020 11:37:43 +0200 Subject: [PATCH 080/111] Add comment for exchanging the system call --- sf-buffer/src/WriterUtils.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/sf-buffer/src/WriterUtils.cpp b/sf-buffer/src/WriterUtils.cpp index 41e5767..9b8d51a 100644 --- a/sf-buffer/src/WriterUtils.cpp +++ b/sf-buffer/src/WriterUtils.cpp @@ -45,6 +45,7 @@ void WriterUtils::create_destination_folder(const string& output_file) string output_folder(output_file.substr(0, file_separator_index)); + // TODO: filesystem::create_directories(output_folder) string create_folder_command("mkdir -p " + output_folder); system(create_folder_command.c_str()); } From 2ab097e6557d182d508652e126c897b16b2039ee Mon Sep 17 00:00:00 2001 From: Dmitry Ozerov Date: Tue, 2 Jun 2020 11:58:27 +0200 Subject: [PATCH 081/111] make possible to run multiple detectors(sf-buffer/sf-stream uses separate streams), for sf-stream read config parameters from json file --- scripts/JF07-buffer-worker.sh | 6 +---- scripts/JF07-stream.sh | 2 +- sf-buffer/CMakeLists.txt | 2 +- sf-buffer/src/main.cpp | 17 +++++++----- sf-stream/src/main.cpp | 51 ++++++++++++++++++----------------- 5 files changed, 40 insertions(+), 38 deletions(-) diff --git a/scripts/JF07-buffer-worker.sh b/scripts/JF07-buffer-worker.sh index e9302fc..5d744e1 100644 --- a/scripts/JF07-buffer-worker.sh +++ b/scripts/JF07-buffer-worker.sh @@ -21,11 +21,7 @@ coreAssociatedBuffer=(4 4 4 4 5 5 5 5 6 6 6 6 7 7 7 7 8 8 8 8 9 9 9 9 10 10 10 1 #coreAssociatedBuffer=(1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32) initialUDPport=50100 -# strange that it doesn't work to add 08 or 09 (value too great for base (error token is "09")) port=$((${initialUDPport}+10#${M})) -#port=`expr ${initialUDPport} + ${M}` DETECTOR=JF07T32V01 -echo ${port} -#taskset -c ${coreAssociatedBuffer[10#${M}]} /usr/bin/sf_buffer M${M} ${port} /gpfs/photonics/swissfel/buffer/${DETECTOR} > /gpfs/photonics/swissfel/buffer/${port}.log -taskset -c ${coreAssociatedBuffer[10#${M}]} /usr/bin/sf_buffer M${M} ${port} /gpfs/photonics/swissfel/buffer/${DETECTOR} ${M} +taskset -c ${coreAssociatedBuffer[10#${M}]} /usr/bin/sf_buffer ${DETECTOR} M${M} ${port} /gpfs/photonics/swissfel/buffer/${DETECTOR} ${M} diff --git a/scripts/JF07-stream.sh b/scripts/JF07-stream.sh index 6cbabc1..bcaf1d4 100644 --- a/scripts/JF07-stream.sh +++ b/scripts/JF07-stream.sh @@ -2,4 +2,4 @@ coreAssociated="20,21,22,23" -taskset -c ${coreAssociated} /usr/bin/sf_stream tcp://129.129.241.42:9007 25 tcp://192.168.30.29:9107 10 +taskset -c ${coreAssociated} /usr/bin/sf_stream /gpfs/photonics/swissfel/buffer/config/stream-JF07.json diff --git a/sf-buffer/CMakeLists.txt b/sf-buffer/CMakeLists.txt index c4c74ed..a47f0b7 100644 --- a/sf-buffer/CMakeLists.txt +++ b/sf-buffer/CMakeLists.txt @@ -17,4 +17,4 @@ target_link_libraries(sf-buffer hdf5_cpp) enable_testing() -add_subdirectory(test/) \ No newline at end of file +add_subdirectory(test/) diff --git a/sf-buffer/src/main.cpp b/sf-buffer/src/main.cpp index b6aadc6..a3bf997 100644 --- a/sf-buffer/src/main.cpp +++ b/sf-buffer/src/main.cpp @@ -17,11 +17,12 @@ using namespace std; using namespace core_buffer; int main (int argc, char *argv[]) { - if (argc != 5) { + if (argc != 6) { cout << endl; - cout << "Usage: sf_buffer [device_name] [udp_port] [root_folder]"; + cout << "Usage: sf_buffer [detector_name] [device_name] [udp_port] [root_folder]"; cout << "[source_id]"; cout << endl; + cout << "\tdetector_name: Detector name, example JF07T32V01" << endl; cout << "\tdevice_name: Name to write to disk."; cout << "\tudp_port: UDP port to connect to." << endl; cout << "\troot_folder: FS root folder." << endl; @@ -31,13 +32,15 @@ int main (int argc, char *argv[]) { exit(-1); } - string device_name = string(argv[1]); - int udp_port = atoi(argv[2]); - string root_folder = string(argv[3]); - int source_id = atoi(argv[4]); + string detector_name = string(argv[1]); + string device_name = string(argv[2]); + int udp_port = atoi(argv[3]); + string root_folder = string(argv[4]); + int source_id = atoi(argv[5]); stringstream ipc_stream; - ipc_stream << BUFFER_LIVE_IPC_URL << source_id; + string LIVE_IPC_URL = BUFFER_LIVE_IPC_URL + detector_name + "-"; + ipc_stream << LIVE_IPC_URL << source_id; const auto ipc_address = ipc_stream.str(); auto ctx = zmq_ctx_new(); diff --git a/sf-stream/src/main.cpp b/sf-stream/src/main.cpp index e977c45..257bd25 100644 --- a/sf-stream/src/main.cpp +++ b/sf-stream/src/main.cpp @@ -13,42 +13,42 @@ #include "rapidjson/document.h" #include "rapidjson/stringbuffer.h" #include "rapidjson/writer.h" - - +#include +#include using namespace std; using namespace core_buffer; -// TODO: this needs to be re-read from external source -const string PEDE_FILENAME = - "/sf/bernina/data/p17534/res/JF_pedestals/pedestal_20200529_1408.JF07T32V01.res.h5"; -const string GAIN_FILENAME = - "/sf/bernina/config/jungfrau/gainMaps/JF07T32V01/gains.h5"; - - int main (int argc, char *argv[]) { - if (argc != 5) { + if (argc != 2) { cout << endl; cout << "Usage: sf_stream "; - cout << " [streamvis_address] [reduction_factor_streamvis]"; - cout << " [live_analysis_address] [reduction_factor_live_analysis]"; + cout << " [config_json_file]"; cout << endl; - cout << "\tstreamvis_address: address to streamvis, example tcp://129.129.241.42:9007" << endl; - cout << "\treduction_factor_streamvis: 1 out of N (example 10) images to send to streamvis. For remaining send metadata." << endl; - cout << "\tlive_analysis_address: address to live_analysis, example tcp://129.129.241.42:9107" << endl; - cout << "\treduction_factor_live_analysis: 1 out of N (example 10) images to send to live analysis. For remaining send metadata. N<=1 - send every image" << endl; + cout << "\tconfig_json_file: json file with the configuration parameters(detector name, number of modules, pedestal and gain files" << endl; cout << endl; exit(-1); } - string streamvis_address = string(argv[1]); - int reduction_factor_streamvis = (int) atoll(argv[2]); - string live_analysis_address = string(argv[3]); - int reduction_factor_live_analysis = (uint64_t) atoll(argv[4]); + string config_json_file = string(argv[1]); + + ifstream ifs(config_json_file); + rapidjson::IStreamWrapper isw(ifs); + rapidjson::Document config_parameters; + config_parameters.ParseStream(isw); + + string streamvis_address = config_parameters["streamvis_stream"].GetString(); + int reduction_factor_streamvis = config_parameters["streamvis_rate"].GetInt(); + string live_analysis_address = config_parameters["live_stream"].GetString(); + int reduction_factor_live_analysis = config_parameters["live_rate"].GetInt(); + + const string PEDE_FILENAME = config_parameters["pedestal_file"].GetString(); + const string GAIN_FILENAME = config_parameters["gain_file"].GetString(); + const string DETECTOR_NAME = config_parameters["detector_name"].GetString(); + size_t n_modules = config_parameters["n_modules"].GetInt(); - size_t n_modules = 32; FastQueue queue( n_modules * MODULE_N_BYTES, STREAM_FASTQUEUE_SLOTS); @@ -56,7 +56,9 @@ int main (int argc, char *argv[]) auto ctx = zmq_ctx_new(); zmq_ctx_set (ctx, ZMQ_IO_THREADS, STREAM_ZMQ_IO_THREADS); - LiveRecvModule recv_module(queue, n_modules, ctx, BUFFER_LIVE_IPC_URL); + const string LIVE_IPC_URL = BUFFER_LIVE_IPC_URL+DETECTOR_NAME+"-"; + + LiveRecvModule recv_module(queue, n_modules, ctx, LIVE_IPC_URL); // 0mq sockets to streamvis and live analysis void *socket_streamvis = zmq_socket(ctx, ZMQ_PUB); @@ -150,8 +152,9 @@ int main (int argc, char *argv[]) header_alloc); header.AddMember("run_name", run_name, header_alloc); - // TODO: Detector name should come as parameter to sf_stream - header.AddMember("number_frames_expected", "JF07T32V01", header_alloc); + rapidjson::Value detector_name; + detector_name.SetString(DETECTOR_NAME.c_str(), header_alloc); + header.AddMember("detector_name", detector_name, header_alloc); header.AddMember("htype", "array-1.0", header_alloc); header.AddMember("type", "uint16", header_alloc); From 684108d833ee92f15c1b746cf139929da2a31735 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 2 Jun 2020 12:18:08 +0200 Subject: [PATCH 082/111] Fixed writing JSON error --- sf-stream/src/main.cpp | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/sf-stream/src/main.cpp b/sf-stream/src/main.cpp index e977c45..6ecd3b5 100644 --- a/sf-stream/src/main.cpp +++ b/sf-stream/src/main.cpp @@ -72,10 +72,7 @@ int main (int argc, char *argv[]) rapidjson::Document header(rapidjson::kObjectType); auto& header_alloc = header.GetAllocator(); - - rapidjson::StringBuffer header_buffer; - rapidjson::Writer header_writer(header_buffer); - header.Accept(header_writer); + string text_header; // TODO: Remove stats trash. int stats_counter = 0; @@ -176,7 +173,14 @@ int main (int argc, char *argv[]) shape[1] = 2; } - string text_header = header_buffer.GetString(); + + { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + header.Accept(writer); + + text_header = buffer.GetString(); + } zmq_send(socket_streamvis, text_header.c_str(), @@ -210,7 +214,13 @@ int main (int argc, char *argv[]) shape[1] = 2; } - text_header = header_buffer.GetString(); + { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + header.Accept(writer); + + text_header = buffer.GetString(); + } zmq_send(socket_live, text_header.c_str(), From fc93b90164b5255376df545aceeb8acd784fff7c Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 2 Jun 2020 12:46:54 +0200 Subject: [PATCH 083/111] Move initialization in loop --- sf-stream/src/main.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/sf-stream/src/main.cpp b/sf-stream/src/main.cpp index 2adb6c9..6515044 100644 --- a/sf-stream/src/main.cpp +++ b/sf-stream/src/main.cpp @@ -72,9 +72,7 @@ int main (int argc, char *argv[]) uint16_t data_empty [] = { 0, 0, 0, 0}; - rapidjson::Document header(rapidjson::kObjectType); - auto& header_alloc = header.GetAllocator(); - string text_header; + // TODO: Remove stats trash. int stats_counter = 0; @@ -84,6 +82,10 @@ int main (int argc, char *argv[]) while (true) { + rapidjson::Document header(rapidjson::kObjectType); + auto& header_alloc = header.GetAllocator(); + string text_header; + auto start_time = chrono::steady_clock::now(); auto slot_id = queue.read(); @@ -176,7 +178,6 @@ int main (int argc, char *argv[]) shape[1] = 2; } - { rapidjson::StringBuffer buffer; rapidjson::Writer writer(buffer); From 6ab6c03aa6e059da06cfff6059d54194948066fa Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 2 Jun 2020 12:55:08 +0200 Subject: [PATCH 084/111] Move to using block_id instead of slot_id --- sf-writer/include/ImageAssembler.hpp | 8 ++++---- sf-writer/src/ImageAssembler.cpp | 30 ++++++++++++++-------------- sf-writer/src/main.cpp | 16 ++++++--------- 3 files changed, 25 insertions(+), 29 deletions(-) diff --git a/sf-writer/include/ImageAssembler.hpp b/sf-writer/include/ImageAssembler.hpp index 87f1070..9aaec79 100644 --- a/sf-writer/include/ImageAssembler.hpp +++ b/sf-writer/include/ImageAssembler.hpp @@ -20,14 +20,14 @@ public: virtual ~ImageAssembler(); - bool is_slot_free(const int slot_id); - bool is_slot_full(const int slot_id); + bool is_slot_free(const int bunch_id); + bool is_slot_full(const int bunch_id); - void process(int slot_id, + void process(int bunch_id, const int i_module, const BufferBinaryBlock* block_buffer); - void free_slot(const int slot_id); + void free_slot(const int bunch_id); ImageMetadataBlock* get_metadata_buffer(const int slot_id); char* get_data_buffer(const int slot_id); diff --git a/sf-writer/src/ImageAssembler.cpp b/sf-writer/src/ImageAssembler.cpp index 5f638fd..c62d994 100644 --- a/sf-writer/src/ImageAssembler.cpp +++ b/sf-writer/src/ImageAssembler.cpp @@ -23,32 +23,32 @@ ImageAssembler::~ImageAssembler() delete[] metadata_buffer_; } -bool ImageAssembler::is_slot_free(const int slot_id) +bool ImageAssembler::is_slot_free(const int bunch_id) { - return buffer_status_[slot_id % IA_N_SLOTS] > 0; + return buffer_status_[bunch_id % IA_N_SLOTS] > 0; } -bool ImageAssembler::is_slot_full(const int slot_id) +bool ImageAssembler::is_slot_full(const int bunch_id) { - return buffer_status_[slot_id % IA_N_SLOTS] == 0; + return buffer_status_[bunch_id % IA_N_SLOTS] == 0; } void ImageAssembler::process( - int slot_id, + int bunch_id, const int i_module, const BufferBinaryBlock* block_buffer) { - slot_id %= IA_N_SLOTS; + bunch_id %= IA_N_SLOTS; // TODO: Temp workaround. Make proper initialization. if (i_module == 0) { - metadata_buffer_[slot_id].block_start_pulse_id = + metadata_buffer_[bunch_id].block_start_pulse_id = block_buffer->frame[0].metadata.pulse_id; - metadata_buffer_[slot_id].block_stop_pulse_id = + metadata_buffer_[bunch_id].block_stop_pulse_id = block_buffer->frame[BUFFER_BLOCK_SIZE-1].metadata.pulse_id; } - size_t slot_offset = slot_id * image_buffer_slot_n_bytes_; + size_t slot_offset = bunch_id * image_buffer_slot_n_bytes_; size_t module_image_offset = i_module * MODULE_N_BYTES; for (size_t i_pulse=0; i_pulse < BUFFER_BLOCK_SIZE; i_pulse++) { @@ -61,23 +61,23 @@ void ImageAssembler::process( // TODO: Temp workaround. We need to synchronize this access. if (i_module == 0) { - metadata_buffer_[slot_id].pulse_id[i_pulse] = + metadata_buffer_[bunch_id].pulse_id[i_pulse] = block_buffer->frame[i_pulse].metadata.pulse_id; - metadata_buffer_[slot_id].frame_index[i_pulse] = + metadata_buffer_[bunch_id].frame_index[i_pulse] = block_buffer->frame[i_pulse].metadata.frame_index; - metadata_buffer_[slot_id].daq_rec[i_pulse] = + metadata_buffer_[bunch_id].daq_rec[i_pulse] = block_buffer->frame[i_pulse].metadata.daq_rec; // metadata_buffer_[slot_id].is_good_image[i_pulse] = // block_buffer->frame[i_pulse].is_good_image.pulse_id; } } - buffer_status_[slot_id]--; + buffer_status_[bunch_id]--; } -void ImageAssembler::free_slot(const int slot_id) +void ImageAssembler::free_slot(const int bunch_id) { - buffer_status_[slot_id % IA_N_SLOTS] = n_modules_; + buffer_status_[bunch_id % IA_N_SLOTS] = n_modules_; } ImageMetadataBlock* ImageAssembler::get_metadata_buffer(const int slot_id) diff --git a/sf-writer/src/main.cpp b/sf-writer/src/main.cpp index 0c25efb..f5c5195 100644 --- a/sf-writer/src/main.cpp +++ b/sf-writer/src/main.cpp @@ -26,10 +26,9 @@ void read_buffer( BufferBinaryReader block_reader(device, channel_name); auto block_buffer = new BufferBinaryBlock(); - int slot_id = 0; for (uint64_t block_id:buffer_blocks) { - while(!image_assembler.is_slot_free(slot_id)) { + while(!image_assembler.is_slot_free(block_id)) { this_thread::sleep_for(chrono::milliseconds( WRITER_IMAGE_ASSEMBLER_RETRY_MS)); } @@ -44,8 +43,7 @@ void read_buffer( start_time = steady_clock::now(); - image_assembler.process(slot_id, i_module, block_buffer); - slot_id++; + image_assembler.process(block_id, i_module, block_buffer); end_time = steady_clock::now(); uint64_t compose_us_duration = duration_cast( @@ -117,16 +115,15 @@ int main (int argc, char *argv[]) JFH5Writer writer(output_file, start_pulse_id, stop_pulse_id, n_modules); - int slot_id = 0; for (uint64_t block_id:buffer_blocks) { - while(!image_assembler.is_slot_full(slot_id)) { + while(!image_assembler.is_slot_full(block_id)) { this_thread::sleep_for(chrono::milliseconds( WRITER_IMAGE_ASSEMBLER_RETRY_MS)); } - auto metadata = image_assembler.get_metadata_buffer(slot_id); - auto data = image_assembler.get_data_buffer(slot_id); + auto metadata = image_assembler.get_metadata_buffer(block_id); + auto data = image_assembler.get_data_buffer(block_id); auto start_time = steady_clock::now(); @@ -136,8 +133,7 @@ int main (int argc, char *argv[]) auto write_us_duration = chrono::duration_cast( end_time-start_time).count(); - image_assembler.free_slot(slot_id); - slot_id++; + image_assembler.free_slot(block_id); cout << "sf_writer:avg_write_us "; cout << write_us_duration / BUFFER_BLOCK_SIZE << endl; From a01eae404257d2f7ce74e64cdd27ddc109a65b05 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 2 Jun 2020 12:58:28 +0200 Subject: [PATCH 085/111] Use proper atomic functions --- sf-writer/src/ImageAssembler.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sf-writer/src/ImageAssembler.cpp b/sf-writer/src/ImageAssembler.cpp index c62d994..90c4d6f 100644 --- a/sf-writer/src/ImageAssembler.cpp +++ b/sf-writer/src/ImageAssembler.cpp @@ -25,12 +25,12 @@ ImageAssembler::~ImageAssembler() bool ImageAssembler::is_slot_free(const int bunch_id) { - return buffer_status_[bunch_id % IA_N_SLOTS] > 0; + return buffer_status_[bunch_id % IA_N_SLOTS].load() > 0; } bool ImageAssembler::is_slot_full(const int bunch_id) { - return buffer_status_[bunch_id % IA_N_SLOTS] == 0; + return buffer_status_[bunch_id % IA_N_SLOTS].load() == 0; } void ImageAssembler::process( @@ -72,12 +72,12 @@ void ImageAssembler::process( } } - buffer_status_[bunch_id]--; + buffer_status_[bunch_id].fetch_sub(1); } void ImageAssembler::free_slot(const int bunch_id) { - buffer_status_[bunch_id % IA_N_SLOTS] = n_modules_; + buffer_status_[bunch_id % IA_N_SLOTS].store(n_modules_); } ImageMetadataBlock* ImageAssembler::get_metadata_buffer(const int slot_id) From 3c428bae5c1046834c3b98784a6c1539546bb044 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 2 Jun 2020 12:58:38 +0200 Subject: [PATCH 086/111] Remove unused code --- sf-writer/src/main.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/sf-writer/src/main.cpp b/sf-writer/src/main.cpp index f5c5195..76f4f79 100644 --- a/sf-writer/src/main.cpp +++ b/sf-writer/src/main.cpp @@ -82,7 +82,6 @@ int main (int argc, char *argv[]) uint64_t start_block = start_pulse_id / BUFFER_BLOCK_SIZE; uint64_t stop_block = stop_pulse_id / BUFFER_BLOCK_SIZE; - auto n_blocks = stop_block - start_block + 1; ImageAssembler image_assembler(n_modules); From c300a8dea6d77022666466a2dd9e27c4a7067d9f Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 2 Jun 2020 13:00:06 +0200 Subject: [PATCH 087/111] Renaming parameters --- sf-writer/include/ImageAssembler.hpp | 4 ++-- sf-writer/src/ImageAssembler.cpp | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sf-writer/include/ImageAssembler.hpp b/sf-writer/include/ImageAssembler.hpp index 9aaec79..2c176a5 100644 --- a/sf-writer/include/ImageAssembler.hpp +++ b/sf-writer/include/ImageAssembler.hpp @@ -29,8 +29,8 @@ public: void free_slot(const int bunch_id); - ImageMetadataBlock* get_metadata_buffer(const int slot_id); - char* get_data_buffer(const int slot_id); + ImageMetadataBlock* get_metadata_buffer(const int bunch_id); + char* get_data_buffer(const int bunch_id); }; diff --git a/sf-writer/src/ImageAssembler.cpp b/sf-writer/src/ImageAssembler.cpp index 90c4d6f..31914fd 100644 --- a/sf-writer/src/ImageAssembler.cpp +++ b/sf-writer/src/ImageAssembler.cpp @@ -80,13 +80,13 @@ void ImageAssembler::free_slot(const int bunch_id) buffer_status_[bunch_id % IA_N_SLOTS].store(n_modules_); } -ImageMetadataBlock* ImageAssembler::get_metadata_buffer(const int slot_id) +ImageMetadataBlock* ImageAssembler::get_metadata_buffer(const int bunch_id) { - return &(metadata_buffer_[slot_id % IA_N_SLOTS]); + return &(metadata_buffer_[bunch_id % IA_N_SLOTS]); } -char* ImageAssembler::get_data_buffer(const int slot_id) +char* ImageAssembler::get_data_buffer(const int bunch_id) { return image_buffer_ + - ((slot_id % IA_N_SLOTS) * image_buffer_slot_n_bytes_); + ((bunch_id % IA_N_SLOTS) * image_buffer_slot_n_bytes_); } From 8cec014743180aaf763365144b9962203844f3df Mon Sep 17 00:00:00 2001 From: Dmitry Ozerov Date: Tue, 2 Jun 2020 13:03:38 +0200 Subject: [PATCH 088/111] propagate properly gain file to the streamvis --- sf-stream/src/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sf-stream/src/main.cpp b/sf-stream/src/main.cpp index 6515044..b0a9632 100644 --- a/sf-stream/src/main.cpp +++ b/sf-stream/src/main.cpp @@ -140,7 +140,7 @@ int main (int argc, char *argv[]) header.AddMember("pedestal_file", pedestal_file, header_alloc); rapidjson::Value gain_file; - gain_file.SetString(PEDE_FILENAME.c_str(), header_alloc); + gain_file.SetString(GAIN_FILENAME.c_str(), header_alloc); header.AddMember("gain_file", gain_file, header_alloc); header.AddMember("number_frames_expected", 10000, header_alloc); From a80c0f1c3c707852ee093b1ecd459282e8c99bbc Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 2 Jun 2020 13:05:19 +0200 Subject: [PATCH 089/111] Finish slot_id calculation --- sf-writer/include/ImageAssembler.hpp | 3 +- sf-writer/src/ImageAssembler.cpp | 41 +++++++++++++++++----------- 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/sf-writer/include/ImageAssembler.hpp b/sf-writer/include/ImageAssembler.hpp index 2c176a5..b4a12d1 100644 --- a/sf-writer/include/ImageAssembler.hpp +++ b/sf-writer/include/ImageAssembler.hpp @@ -12,7 +12,8 @@ class ImageAssembler { const size_t image_buffer_slot_n_bytes_; char* image_buffer_; - ImageMetadataBlock* metadata_buffer_; + ImageMetadataBlock* image_metadata_buffer_; + ModuleFrame* frame_metadata_buffer_; std::atomic_int* buffer_status_; public: diff --git a/sf-writer/src/ImageAssembler.cpp b/sf-writer/src/ImageAssembler.cpp index 31914fd..81abff9 100644 --- a/sf-writer/src/ImageAssembler.cpp +++ b/sf-writer/src/ImageAssembler.cpp @@ -9,7 +9,9 @@ ImageAssembler::ImageAssembler(const size_t n_modules) : image_buffer_slot_n_bytes_(BUFFER_BLOCK_SIZE * MODULE_N_BYTES * n_modules_) { image_buffer_ = new char[IA_N_SLOTS * image_buffer_slot_n_bytes_]; - metadata_buffer_ = new ImageMetadataBlock[IA_N_SLOTS]; + image_metadata_buffer_ = new ImageMetadataBlock[IA_N_SLOTS]; + frame_metadata_buffer_ = + new ModuleFrame[IA_N_SLOTS * BUFFER_BLOCK_SIZE * n_modules]; buffer_status_ = new atomic_int[IA_N_SLOTS]; for (size_t i=0; i 0; + auto slot_id = bunch_id % IA_N_SLOTS; + return buffer_status_[slot_id].load() > 0; } bool ImageAssembler::is_slot_full(const int bunch_id) { - return buffer_status_[bunch_id % IA_N_SLOTS].load() == 0; + auto slot_id = bunch_id % IA_N_SLOTS; + return buffer_status_[slot_id].load() == 0; } void ImageAssembler::process( @@ -38,17 +42,20 @@ void ImageAssembler::process( const int i_module, const BufferBinaryBlock* block_buffer) { - bunch_id %= IA_N_SLOTS; + auto slot_id = bunch_id % IA_N_SLOTS; // TODO: Temp workaround. Make proper initialization. if (i_module == 0) { - metadata_buffer_[bunch_id].block_start_pulse_id = + image_metadata_buffer_[slot_id].block_start_pulse_id = block_buffer->frame[0].metadata.pulse_id; - metadata_buffer_[bunch_id].block_stop_pulse_id = + image_metadata_buffer_[slot_id].block_stop_pulse_id = block_buffer->frame[BUFFER_BLOCK_SIZE-1].metadata.pulse_id; } - size_t slot_offset = bunch_id * image_buffer_slot_n_bytes_; + auto metadata_offset = + frame_metadata_buffer_[metadata_offset] = 0; + + size_t slot_offset = slot_id * image_buffer_slot_n_bytes_; size_t module_image_offset = i_module * MODULE_N_BYTES; for (size_t i_pulse=0; i_pulse < BUFFER_BLOCK_SIZE; i_pulse++) { @@ -61,32 +68,34 @@ void ImageAssembler::process( // TODO: Temp workaround. We need to synchronize this access. if (i_module == 0) { - metadata_buffer_[bunch_id].pulse_id[i_pulse] = + image_metadata_buffer_[slot_id].pulse_id[i_pulse] = block_buffer->frame[i_pulse].metadata.pulse_id; - metadata_buffer_[bunch_id].frame_index[i_pulse] = + image_metadata_buffer_[slot_id].frame_index[i_pulse] = block_buffer->frame[i_pulse].metadata.frame_index; - metadata_buffer_[bunch_id].daq_rec[i_pulse] = + image_metadata_buffer_[slot_id].daq_rec[i_pulse] = block_buffer->frame[i_pulse].metadata.daq_rec; // metadata_buffer_[slot_id].is_good_image[i_pulse] = // block_buffer->frame[i_pulse].is_good_image.pulse_id; } } - buffer_status_[bunch_id].fetch_sub(1); + buffer_status_[slot_id].fetch_sub(1); } void ImageAssembler::free_slot(const int bunch_id) { - buffer_status_[bunch_id % IA_N_SLOTS].store(n_modules_); + auto slot_id = bunch_id % IA_N_SLOTS; + buffer_status_[slot_id].store(n_modules_); } ImageMetadataBlock* ImageAssembler::get_metadata_buffer(const int bunch_id) { - return &(metadata_buffer_[bunch_id % IA_N_SLOTS]); + auto slot_id = bunch_id % IA_N_SLOTS; + return &(image_metadata_buffer_[slot_id]); } char* ImageAssembler::get_data_buffer(const int bunch_id) { - return image_buffer_ + - ((bunch_id % IA_N_SLOTS) * image_buffer_slot_n_bytes_); + auto slot_id = bunch_id % IA_N_SLOTS; + return image_buffer_ + (slot_id * image_buffer_slot_n_bytes_); } From 2a0e90bd920fc7ff6bbb62d53df2a7321f21f1a7 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 2 Jun 2020 13:22:40 +0200 Subject: [PATCH 090/111] Image assembly process improvements --- sf-writer/src/ImageAssembler.cpp | 42 +++++++++++--------------------- 1 file changed, 14 insertions(+), 28 deletions(-) diff --git a/sf-writer/src/ImageAssembler.cpp b/sf-writer/src/ImageAssembler.cpp index 81abff9..cb1eb14 100644 --- a/sf-writer/src/ImageAssembler.cpp +++ b/sf-writer/src/ImageAssembler.cpp @@ -11,7 +11,7 @@ ImageAssembler::ImageAssembler(const size_t n_modules) : image_buffer_ = new char[IA_N_SLOTS * image_buffer_slot_n_bytes_]; image_metadata_buffer_ = new ImageMetadataBlock[IA_N_SLOTS]; frame_metadata_buffer_ = - new ModuleFrame[IA_N_SLOTS * BUFFER_BLOCK_SIZE * n_modules]; + new ModuleFrame[IA_N_SLOTS * n_modules * BUFFER_BLOCK_SIZE]; buffer_status_ = new atomic_int[IA_N_SLOTS]; for (size_t i=0; iframe[0].metadata.pulse_id; - image_metadata_buffer_[slot_id].block_stop_pulse_id = - block_buffer->frame[BUFFER_BLOCK_SIZE-1].metadata.pulse_id; - } + size_t slot_i_offset = slot_id * image_buffer_slot_n_bytes_; + size_t module_i_offset = i_module * MODULE_N_BYTES; - auto metadata_offset = - frame_metadata_buffer_[metadata_offset] = 0; - - size_t slot_offset = slot_id * image_buffer_slot_n_bytes_; - size_t module_image_offset = i_module * MODULE_N_BYTES; + size_t slot_m_offset = slot_id * n_modules_ * BUFFER_BLOCK_SIZE; + size_t module_m_offset = i_module * n_modules_; for (size_t i_pulse=0; i_pulse < BUFFER_BLOCK_SIZE; i_pulse++) { - size_t image_offset = i_pulse * MODULE_N_BYTES * n_modules_; - + size_t metadata_offset = slot_m_offset + module_m_offset + i_pulse; memcpy( - image_buffer_ + slot_offset + image_offset + module_image_offset, + &(frame_metadata_buffer_[metadata_offset]), + &(block_buffer->frame[i_pulse].metadata), + sizeof(ModuleFrame)); + + size_t image_offset = slot_i_offset + module_i_offset; + image_offset += i_pulse * MODULE_N_BYTES * n_modules_; + memcpy( + image_buffer_ + image_offset, &(block_buffer->frame[i_pulse].data[0]), MODULE_N_BYTES); - - // TODO: Temp workaround. We need to synchronize this access. - if (i_module == 0) { - image_metadata_buffer_[slot_id].pulse_id[i_pulse] = - block_buffer->frame[i_pulse].metadata.pulse_id; - image_metadata_buffer_[slot_id].frame_index[i_pulse] = - block_buffer->frame[i_pulse].metadata.frame_index; - image_metadata_buffer_[slot_id].daq_rec[i_pulse] = - block_buffer->frame[i_pulse].metadata.daq_rec; -// metadata_buffer_[slot_id].is_good_image[i_pulse] = -// block_buffer->frame[i_pulse].is_good_image.pulse_id; - } } buffer_status_[slot_id].fetch_sub(1); From 6a7d2730f92a60e5fbbbd4c448b4b20a73b64a8a Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 2 Jun 2020 13:39:06 +0200 Subject: [PATCH 091/111] Fix offsets in ImageAssembler --- sf-writer/src/ImageAssembler.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sf-writer/src/ImageAssembler.cpp b/sf-writer/src/ImageAssembler.cpp index cb1eb14..ba08458 100644 --- a/sf-writer/src/ImageAssembler.cpp +++ b/sf-writer/src/ImageAssembler.cpp @@ -44,14 +44,17 @@ void ImageAssembler::process( { auto slot_id = bunch_id % IA_N_SLOTS; + // TODO: This offsets are not really readable. size_t slot_i_offset = slot_id * image_buffer_slot_n_bytes_; size_t module_i_offset = i_module * MODULE_N_BYTES; - size_t slot_m_offset = slot_id * n_modules_ * BUFFER_BLOCK_SIZE; - size_t module_m_offset = i_module * n_modules_; + size_t n_metadata_in_slot = n_modules_ * BUFFER_BLOCK_SIZE; + size_t slot_m_offset = slot_id * n_metadata_in_slot; + size_t module_m_offset = i_module; for (size_t i_pulse=0; i_pulse < BUFFER_BLOCK_SIZE; i_pulse++) { - size_t metadata_offset = slot_m_offset + module_m_offset + i_pulse; + size_t metadata_offset = slot_m_offset + module_m_offset; + metadata_offset += i_pulse * n_modules_; memcpy( &(frame_metadata_buffer_[metadata_offset]), &(block_buffer->frame[i_pulse].metadata), From d0b04fd2ea4f37116ba29478b55d221d1c67894b Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 2 Jun 2020 13:54:15 +0200 Subject: [PATCH 092/111] Basic interaction test for buffer --- sf-writer/test/main.cpp | 1 + sf-writer/test/test_ImageAssembler.cpp | 32 ++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 sf-writer/test/test_ImageAssembler.cpp diff --git a/sf-writer/test/main.cpp b/sf-writer/test/main.cpp index 296a468..94aeb99 100644 --- a/sf-writer/test/main.cpp +++ b/sf-writer/test/main.cpp @@ -1,5 +1,6 @@ #include "gtest/gtest.h" #include "test_WriterH5Writer.cpp" +#include "test_ImageAssembler.cpp" using namespace std; diff --git a/sf-writer/test/test_ImageAssembler.cpp b/sf-writer/test/test_ImageAssembler.cpp new file mode 100644 index 0000000..0071015 --- /dev/null +++ b/sf-writer/test/test_ImageAssembler.cpp @@ -0,0 +1,32 @@ +#include + +#include "ImageAssembler.hpp" +#include "gtest/gtest.h" + +using namespace std; +using namespace core_buffer; + +TEST(ImageAssembler, basic_interaction) +{ + size_t n_modules = 3; + uint64_t bunch_id = 0; + + ImageAssembler assembler(n_modules); + + ASSERT_EQ(assembler.is_slot_free(bunch_id), true); + + auto buffer_block = make_unique(); + auto buffer_ptr = buffer_block.get(); + + for (size_t i_module=0; i_module < n_modules; i_module++) { + assembler.process(bunch_id, i_module, buffer_ptr); + } + + ASSERT_EQ(assembler.is_slot_full(bunch_id), true); + + auto metadata = assembler.get_metadata_buffer(bunch_id); + auto data = assembler.get_data_buffer(bunch_id); + + assembler.free_slot(bunch_id); + ASSERT_EQ(assembler.is_slot_free(bunch_id), true); +} From 1e4be89185f6d9275510b2086059645cbd04caf7 Mon Sep 17 00:00:00 2001 From: babic_a Date: Tue, 2 Jun 2020 13:56:23 +0200 Subject: [PATCH 093/111] systemd scripts for JF01 and JF13 --- scripts/JF01-buffer-worker.sh | 20 ++++++++ scripts/JF01-buffer-worker@.service | 16 +++++++ ...F07-replay.service => JF01-buffer.service} | 4 +- scripts/JF01-stream.service | 15 ++++++ scripts/JF01-stream.sh | 5 ++ scripts/JF07-replay-worker.sh | 26 ----------- scripts/JF07-replay-worker@.service | 16 ------- scripts/JF13-buffer-worker.sh | 20 ++++++++ scripts/JF13-buffer-worker@.service | 16 +++++++ scripts/JF13-buffer.service | 10 ++++ scripts/JF13-stream.service | 15 ++++++ scripts/JF13-stream.sh | 5 ++ scripts/start_detector.sh | 46 +++++++++++++++++++ sf-stream/src/main.cpp | 4 +- 14 files changed, 172 insertions(+), 46 deletions(-) create mode 100644 scripts/JF01-buffer-worker.sh create mode 100644 scripts/JF01-buffer-worker@.service rename scripts/{JF07-replay.service => JF01-buffer.service} (70%) create mode 100644 scripts/JF01-stream.service create mode 100644 scripts/JF01-stream.sh delete mode 100644 scripts/JF07-replay-worker.sh delete mode 100644 scripts/JF07-replay-worker@.service create mode 100644 scripts/JF13-buffer-worker.sh create mode 100644 scripts/JF13-buffer-worker@.service create mode 100644 scripts/JF13-buffer.service create mode 100644 scripts/JF13-stream.service create mode 100644 scripts/JF13-stream.sh create mode 100755 scripts/start_detector.sh diff --git a/scripts/JF01-buffer-worker.sh b/scripts/JF01-buffer-worker.sh new file mode 100644 index 0000000..b907af5 --- /dev/null +++ b/scripts/JF01-buffer-worker.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +if [ $# != 1 ] +then + systemctl start JF01-buffer-worker@{00..02} + exit +fi + +M=$1 + +# Add ourselves to the user cpuset. +# echo $$ > /sys/fs/cgroup/cpuset/user/tasks + +coreAssociatedBuffer=(12 12 12) + +initialUDPport=50010 +port=$((${initialUDPport}+10#${M})) +DETECTOR=JF01T03V01 + +taskset -c ${coreAssociatedBuffer[10#${M}]} /usr/bin/sf_buffer ${DETECTOR} M${M} ${port} /gpfs/photonics/swissfel/buffer/${DETECTOR} ${M} diff --git a/scripts/JF01-buffer-worker@.service b/scripts/JF01-buffer-worker@.service new file mode 100644 index 0000000..b8b6a97 --- /dev/null +++ b/scripts/JF01-buffer-worker@.service @@ -0,0 +1,16 @@ +[Unit] +Description=JF01 UDP2buffer worker instance as a service, instance %i +Requires=JF01-buffer.service +Before=JF01-buffer.service +BindsTo=JF01-buffer.service + +[Service] +PermissionsStartOnly=true +Type=idle +User=root +ExecStart=/usr/bin/sh /home/writer/git/sf_daq_buffer/scripts/JF01-buffer-worker.sh %i +TimeoutStartSec=10 +RestartSec=10 + +[Install] +WantedBy=JF01-buffer.service diff --git a/scripts/JF07-replay.service b/scripts/JF01-buffer.service similarity index 70% rename from scripts/JF07-replay.service rename to scripts/JF01-buffer.service index 288b922..efdc14d 100644 --- a/scripts/JF07-replay.service +++ b/scripts/JF01-buffer.service @@ -1,9 +1,9 @@ [Unit] -Description=All replay instances of JF07 +Description=All UDP-buffer instances of JF01 [Service] Type=oneshot -ExecStart=/usr/bin/sh /home/writer/git/sf_daq_buffer/scripts/JF07-replay-worker.sh +ExecStart=/usr/bin/sh /home/writer/git/sf_daq_buffer/scripts/JF01-buffer-worker.sh RemainAfterExit=yes [Install] diff --git a/scripts/JF01-stream.service b/scripts/JF01-stream.service new file mode 100644 index 0000000..79e7b1b --- /dev/null +++ b/scripts/JF01-stream.service @@ -0,0 +1,15 @@ +[Unit] +Description=stream service (to streamvis and live analysis) of JF01 + +[Service] +PermissionsStartOnly=true +Type=idle +User=root +ExecStart=/usr/bin/sh /home/writer/git/sf_daq_buffer/scripts/JF01-stream.sh +TimeoutStartSec=10 +Restart=on-failure +RestartSec=10 + +[Install] +WantedBy=multi-user.target + diff --git a/scripts/JF01-stream.sh b/scripts/JF01-stream.sh new file mode 100644 index 0000000..c7d1724 --- /dev/null +++ b/scripts/JF01-stream.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +coreAssociated="24" + +taskset -c ${coreAssociated} /usr/bin/sf_stream /gpfs/photonics/swissfel/buffer/config/stream-JF01.json diff --git a/scripts/JF07-replay-worker.sh b/scripts/JF07-replay-worker.sh deleted file mode 100644 index bb51829..0000000 --- a/scripts/JF07-replay-worker.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -if [ $# != 1 ] -then - systemctl start JF07-replay-worker@{00..32} - exit -fi - -M=$1 - -#8 replay workers per core, last (stream to visualisation) worker occupies 4 -coreAssociated=(20 20 20 20 20 20 20 20 21 21 21 21 21 21 21 21 22 22 22 22 22 22 22 22 23 23 23 23 23 23 23 23 24,25,26,27) - -latest_file=`cat /gpfs/photonics/swissfel/buffer/JF07T32V01/M00/LATEST` -last_pulse_id=`basename ${latest_file} | sed 's/.h5//'` -first_pulse_id=$((${last_pulse_id}-360000)) - -echo "First/last pulse_id : ${first_pulse_id} ${last_pulse_id}" - -if [ ${M} == 32 ] -then -# taskset -c ${coreAssociated[10#${M}]} /usr/bin/sf_writer /gpfs/photonics/swissfel/buffer/test.${first_pulse_id}-${last_pulse_id}.h5 ${first_pulse_id} ${last_pulse_id} - taskset -c ${coreAssociated[10#${M}]} /usr/bin/sf_stream tcp://129.129.241.42:9007 10 tcp://192.168.30.29:9107 30 -else - taskset -c ${coreAssociated[10#${M}]} /usr/bin/sf_replay /gpfs/photonics/swissfel/buffer/JF07T32V01 M${M} ${M} ${first_pulse_id} ${last_pulse_id} -fi diff --git a/scripts/JF07-replay-worker@.service b/scripts/JF07-replay-worker@.service deleted file mode 100644 index 42e8dda..0000000 --- a/scripts/JF07-replay-worker@.service +++ /dev/null @@ -1,16 +0,0 @@ -[Unit] -Description=JF07 replay worker instance as a service, instance %i -Requires=JF07-replay.service -Before=JF07-replay.service -BindsTo=JF07-replay.service - -[Service] -PermissionsStartOnly=true -Type=idle -User=root -ExecStart=/usr/bin/sh /home/writer/git/sf_daq_buffer/scripts/JF07-replay-worker.sh %i -TimeoutStartSec=10 -RestartSec=10 - -[Install] -WantedBy=JF07-replay.service diff --git a/scripts/JF13-buffer-worker.sh b/scripts/JF13-buffer-worker.sh new file mode 100644 index 0000000..c269179 --- /dev/null +++ b/scripts/JF13-buffer-worker.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +if [ $# != 1 ] +then + systemctl start JF13-buffer-worker@00 + exit +fi + +M=$1 + +# Add ourselves to the user cpuset. +# echo $$ > /sys/fs/cgroup/cpuset/user/tasks + +coreAssociatedBuffer=(13) + +initialUDPport=50190 +port=$((${initialUDPport}+10#${M})) +DETECTOR=JF13T01V01 + +taskset -c ${coreAssociatedBuffer[10#${M}]} /usr/bin/sf_buffer ${DETECTOR} M${M} ${port} /gpfs/photonics/swissfel/buffer/${DETECTOR} ${M} diff --git a/scripts/JF13-buffer-worker@.service b/scripts/JF13-buffer-worker@.service new file mode 100644 index 0000000..0ccd3d5 --- /dev/null +++ b/scripts/JF13-buffer-worker@.service @@ -0,0 +1,16 @@ +[Unit] +Description=JF13 UDP2buffer worker instance as a service, instance %i +Requires=JF13-buffer.service +Before=JF13-buffer.service +BindsTo=JF13-buffer.service + +[Service] +PermissionsStartOnly=true +Type=idle +User=root +ExecStart=/usr/bin/sh /home/writer/git/sf_daq_buffer/scripts/JF13-buffer-worker.sh %i +TimeoutStartSec=10 +RestartSec=10 + +[Install] +WantedBy=JF13-buffer.service diff --git a/scripts/JF13-buffer.service b/scripts/JF13-buffer.service new file mode 100644 index 0000000..2c435ce --- /dev/null +++ b/scripts/JF13-buffer.service @@ -0,0 +1,10 @@ +[Unit] +Description=All UDP-buffer instances of JF13 + +[Service] +Type=oneshot +ExecStart=/usr/bin/sh /home/writer/git/sf_daq_buffer/scripts/JF13-buffer-worker.sh +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target diff --git a/scripts/JF13-stream.service b/scripts/JF13-stream.service new file mode 100644 index 0000000..7e0a492 --- /dev/null +++ b/scripts/JF13-stream.service @@ -0,0 +1,15 @@ +[Unit] +Description=stream service (to streamvis and live analysis) of JF13 + +[Service] +PermissionsStartOnly=true +Type=idle +User=root +ExecStart=/usr/bin/sh /home/writer/git/sf_daq_buffer/scripts/JF13-stream.sh +TimeoutStartSec=10 +Restart=on-failure +RestartSec=10 + +[Install] +WantedBy=multi-user.target + diff --git a/scripts/JF13-stream.sh b/scripts/JF13-stream.sh new file mode 100644 index 0000000..3b8e226 --- /dev/null +++ b/scripts/JF13-stream.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +coreAssociated="25" + +taskset -c ${coreAssociated} /usr/bin/sf_stream /gpfs/photonics/swissfel/buffer/config/stream-JF13.json diff --git a/scripts/start_detector.sh b/scripts/start_detector.sh new file mode 100755 index 0000000..482d7d4 --- /dev/null +++ b/scripts/start_detector.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +if [ $# -lt 1 ] +then + echo "Usage : $0 DETECTOR_NAME " + echo " DETECTOR_NAME: JF07 or JF01..." + echo " number_of_cycles : optional, default 100" + exit +fi + +DETECTOR=$1 +case ${DETECTOR} in +'JF01') + D=1 + ;; +'JF07') + D=7 + ;; +'JF13') + D=13 + ;; +*) + echo "Unsupported detector" + exit + ;; +esac + +n_cycles=100 +if [ $# == 2 ] +then + n_cycles=$2 +fi + +export PATH=/home/dbe/miniconda3/bin:$PATH +source deactivate +source activate dia + +sls_detector_put ${D}-timing trigger +sls_detector_put ${D}-cycles ${n_cycles} +sls_detector_put ${D}-exptime 5e-06 +sls_detector_put ${D}-frames 1 +sls_detector_put ${D}-dr 16 +#sls_detector_put ${D}-clearbit to 0x5d 0 # normal mode, not highG0 +sls_detector_put ${D}-status start + +echo "Now start trigger" diff --git a/sf-stream/src/main.cpp b/sf-stream/src/main.cpp index b0a9632..d9a4b45 100644 --- a/sf-stream/src/main.cpp +++ b/sf-stream/src/main.cpp @@ -170,7 +170,7 @@ int main (int argc, char *argv[]) } if ( send_streamvis == 0 ) { auto& shape = header["shape"]; - shape[0] = 16384; + shape[0] = n_modules*512; shape[1] = 1024; } else{ auto& shape = header["shape"]; @@ -210,7 +210,7 @@ int main (int argc, char *argv[]) } if ( send_live_analysis == 0 ) { auto& shape = header["shape"]; - shape[0] = 16384; + shape[0] = n_modules*512; shape[1] = 1024; } else{ auto& shape = header["shape"]; From b1cba5633f9174fba772e20197d920ab45afe778 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 2 Jun 2020 14:05:32 +0200 Subject: [PATCH 094/111] Partial implementation of image verification --- sf-writer/include/ImageAssembler.hpp | 2 +- sf-writer/src/ImageAssembler.cpp | 21 ++++++++++++++++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/sf-writer/include/ImageAssembler.hpp b/sf-writer/include/ImageAssembler.hpp index b4a12d1..83e1f67 100644 --- a/sf-writer/include/ImageAssembler.hpp +++ b/sf-writer/include/ImageAssembler.hpp @@ -12,7 +12,7 @@ class ImageAssembler { const size_t image_buffer_slot_n_bytes_; char* image_buffer_; - ImageMetadataBlock* image_metadata_buffer_; + ImageMetadataBlock* meta_buffer_; ModuleFrame* frame_metadata_buffer_; std::atomic_int* buffer_status_; diff --git a/sf-writer/src/ImageAssembler.cpp b/sf-writer/src/ImageAssembler.cpp index ba08458..0582c53 100644 --- a/sf-writer/src/ImageAssembler.cpp +++ b/sf-writer/src/ImageAssembler.cpp @@ -9,7 +9,7 @@ ImageAssembler::ImageAssembler(const size_t n_modules) : image_buffer_slot_n_bytes_(BUFFER_BLOCK_SIZE * MODULE_N_BYTES * n_modules_) { image_buffer_ = new char[IA_N_SLOTS * image_buffer_slot_n_bytes_]; - image_metadata_buffer_ = new ImageMetadataBlock[IA_N_SLOTS]; + meta_buffer_ = new ImageMetadataBlock[IA_N_SLOTS]; frame_metadata_buffer_ = new ModuleFrame[IA_N_SLOTS * n_modules * BUFFER_BLOCK_SIZE]; buffer_status_ = new atomic_int[IA_N_SLOTS]; @@ -22,7 +22,7 @@ ImageAssembler::ImageAssembler(const size_t n_modules) : ImageAssembler::~ImageAssembler() { delete[] image_buffer_; - delete[] image_metadata_buffer_; + delete[] meta_buffer_; } bool ImageAssembler::is_slot_free(const int bunch_id) @@ -80,7 +80,22 @@ void ImageAssembler::free_slot(const int bunch_id) ImageMetadataBlock* ImageAssembler::get_metadata_buffer(const int bunch_id) { auto slot_id = bunch_id % IA_N_SLOTS; - return &(image_metadata_buffer_[slot_id]); + + for (size_t i_pulse=0; i_pulse < BUFFER_BLOCK_SIZE; i_pulse++) { + auto is_not_init = false; + + for (size_t i_module=0; i_module < n_modules_; i_module++) { + +// +// if (is_not_init) { +// if () +// } + } + + meta_buffer_[slot_id].pulse_id[i_pulse] + } + + return &(meta_buffer_[slot_id]); } char* ImageAssembler::get_data_buffer(const int bunch_id) From e5157979cad746419eb10acfde2d5170f1cc9aee Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 2 Jun 2020 14:06:47 +0200 Subject: [PATCH 095/111] Change bunch_id to uint64_t --- sf-writer/include/ImageAssembler.hpp | 12 ++++++------ sf-writer/src/ImageAssembler.cpp | 14 +++++++------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/sf-writer/include/ImageAssembler.hpp b/sf-writer/include/ImageAssembler.hpp index 83e1f67..38ae694 100644 --- a/sf-writer/include/ImageAssembler.hpp +++ b/sf-writer/include/ImageAssembler.hpp @@ -21,17 +21,17 @@ public: virtual ~ImageAssembler(); - bool is_slot_free(const int bunch_id); - bool is_slot_full(const int bunch_id); + bool is_slot_free(const uint64_t bunch_id); + bool is_slot_full(const uint64_t bunch_id); - void process(int bunch_id, + void process(uint64_t bunch_id, const int i_module, const BufferBinaryBlock* block_buffer); - void free_slot(const int bunch_id); + void free_slot(const uint64_t bunch_id); - ImageMetadataBlock* get_metadata_buffer(const int bunch_id); - char* get_data_buffer(const int bunch_id); + ImageMetadataBlock* get_metadata_buffer(const uint64_t bunch_id); + char* get_data_buffer(const uint64_t bunch_id); }; diff --git a/sf-writer/src/ImageAssembler.cpp b/sf-writer/src/ImageAssembler.cpp index 0582c53..3f09064 100644 --- a/sf-writer/src/ImageAssembler.cpp +++ b/sf-writer/src/ImageAssembler.cpp @@ -25,20 +25,20 @@ ImageAssembler::~ImageAssembler() delete[] meta_buffer_; } -bool ImageAssembler::is_slot_free(const int bunch_id) +bool ImageAssembler::is_slot_free(const uint64_t bunch_id) { auto slot_id = bunch_id % IA_N_SLOTS; return buffer_status_[slot_id].load() > 0; } -bool ImageAssembler::is_slot_full(const int bunch_id) +bool ImageAssembler::is_slot_full(const uint64_t bunch_id) { auto slot_id = bunch_id % IA_N_SLOTS; return buffer_status_[slot_id].load() == 0; } void ImageAssembler::process( - int bunch_id, + const uint64_t bunch_id, const int i_module, const BufferBinaryBlock* block_buffer) { @@ -71,13 +71,13 @@ void ImageAssembler::process( buffer_status_[slot_id].fetch_sub(1); } -void ImageAssembler::free_slot(const int bunch_id) +void ImageAssembler::free_slot(const uint64_t bunch_id) { auto slot_id = bunch_id % IA_N_SLOTS; buffer_status_[slot_id].store(n_modules_); } -ImageMetadataBlock* ImageAssembler::get_metadata_buffer(const int bunch_id) +ImageMetadataBlock* ImageAssembler::get_metadata_buffer(const uint64_t bunch_id) { auto slot_id = bunch_id % IA_N_SLOTS; @@ -92,13 +92,13 @@ ImageMetadataBlock* ImageAssembler::get_metadata_buffer(const int bunch_id) // } } - meta_buffer_[slot_id].pulse_id[i_pulse] + meta_buffer_[slot_id].pulse_id[i_pulse]; } return &(meta_buffer_[slot_id]); } -char* ImageAssembler::get_data_buffer(const int bunch_id) +char* ImageAssembler::get_data_buffer(const uint64_t bunch_id) { auto slot_id = bunch_id % IA_N_SLOTS; return image_buffer_ + (slot_id * image_buffer_slot_n_bytes_); From 37d3794b81488d80568340a010b21f7b4b3e0403 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 2 Jun 2020 14:29:59 +0200 Subject: [PATCH 096/111] Improve the loop a bit --- sf-writer/src/ImageAssembler.cpp | 44 ++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/sf-writer/src/ImageAssembler.cpp b/sf-writer/src/ImageAssembler.cpp index 3f09064..290d8d5 100644 --- a/sf-writer/src/ImageAssembler.cpp +++ b/sf-writer/src/ImageAssembler.cpp @@ -37,35 +37,53 @@ bool ImageAssembler::is_slot_full(const uint64_t bunch_id) return buffer_status_[slot_id].load() == 0; } +inline size_t ImageAssembler::get_data_offset( + const uint64_t slot_id, const int i_module) +{ + size_t slot_i_offset = slot_id * image_buffer_slot_n_bytes_; + size_t module_i_offset = i_module * MODULE_N_BYTES; + + return slot_i_offset + module_i_offset; +} + +inline size_t ImageAssembler::get_metadata_offset( + const uint64_t slot_id, const int i_module) +{ + size_t n_metadata_in_slot = n_modules_ * BUFFER_BLOCK_SIZE; + size_t slot_m_offset = slot_id * n_metadata_in_slot; + size_t module_m_offset = i_module; + + return slot_m_offset + module_m_offset; +} + void ImageAssembler::process( const uint64_t bunch_id, const int i_module, const BufferBinaryBlock* block_buffer) { - auto slot_id = bunch_id % IA_N_SLOTS; + const auto slot_id = bunch_id % IA_N_SLOTS; - // TODO: This offsets are not really readable. - size_t slot_i_offset = slot_id * image_buffer_slot_n_bytes_; - size_t module_i_offset = i_module * MODULE_N_BYTES; + const auto image_offset = get_data_offset(slot_id, i_module); + const auto image_offset_step = MODULE_N_BYTES * n_modules_; - size_t n_metadata_in_slot = n_modules_ * BUFFER_BLOCK_SIZE; - size_t slot_m_offset = slot_id * n_metadata_in_slot; - size_t module_m_offset = i_module; + const auto meta_offset = get_metadata_offset(slot_id, i_module); + const auto meta_offset_step = 1; for (size_t i_pulse=0; i_pulse < BUFFER_BLOCK_SIZE; i_pulse++) { - size_t metadata_offset = slot_m_offset + module_m_offset; - metadata_offset += i_pulse * n_modules_; + memcpy( - &(frame_metadata_buffer_[metadata_offset]), + &(frame_metadata_buffer_[meta_offset]), &(block_buffer->frame[i_pulse].metadata), sizeof(ModuleFrame)); - size_t image_offset = slot_i_offset + module_i_offset; - image_offset += i_pulse * MODULE_N_BYTES * n_modules_; + meta_offset += meta_offset_step; + memcpy( image_buffer_ + image_offset, &(block_buffer->frame[i_pulse].data[0]), MODULE_N_BYTES); + + image_offset += image_offset_step; } buffer_status_[slot_id].fetch_sub(1); @@ -85,7 +103,7 @@ ImageMetadataBlock* ImageAssembler::get_metadata_buffer(const uint64_t bunch_id) auto is_not_init = false; for (size_t i_module=0; i_module < n_modules_; i_module++) { - + auto metadata_offset = get_metadata_offset(slot_id, i_module); // // if (is_not_init) { // if () From 1e12c38ead792a114a45b3608677581ca81339e3 Mon Sep 17 00:00:00 2001 From: Dmitry Ozerov Date: Tue, 2 Jun 2020 14:34:07 +0200 Subject: [PATCH 097/111] cleanup script for the buffer --- scripts/delete_old_files_in_buffer.sh | 30 +++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100755 scripts/delete_old_files_in_buffer.sh diff --git a/scripts/delete_old_files_in_buffer.sh b/scripts/delete_old_files_in_buffer.sh new file mode 100755 index 0000000..1d0db5a --- /dev/null +++ b/scripts/delete_old_files_in_buffer.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +hours=5 +threshold=80 + +if [ $# = 1 ] +then + threshold=$1 +fi +if [ $# = 2 ] +then + hours=$2 +fi + +df -h | grep BUFFER > /dev/null +if [ $? != 0 ] +then + # BUFFER is not present + exit +fi + +occupancy=`df -h /gpfs/photonics/swissfel/buffer | grep BUFFER | awk '{print $5}' | sed 's/%//'` +if [ ${occupancy} -lt ${threshold} ] +then +# echo OK, not action + exit +fi + +#find /gpfs/photonics/swissfel/buffer/JF* -type f -mmin +$((${hours}*60)) -delete +find /gpfs/photonics/swissfel/buffer/JF*/M* -type d -mmin +$((${hours}*60)) -delete From 4c49d4611fbe872fdc5712dd8da481412dd50448 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 2 Jun 2020 14:41:01 +0200 Subject: [PATCH 098/111] Rename variable to new meta standard --- sf-writer/include/ImageAssembler.hpp | 2 +- sf-writer/src/ImageAssembler.cpp | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/sf-writer/include/ImageAssembler.hpp b/sf-writer/include/ImageAssembler.hpp index 38ae694..ada474e 100644 --- a/sf-writer/include/ImageAssembler.hpp +++ b/sf-writer/include/ImageAssembler.hpp @@ -13,7 +13,7 @@ class ImageAssembler { char* image_buffer_; ImageMetadataBlock* meta_buffer_; - ModuleFrame* frame_metadata_buffer_; + ModuleFrame* frame_meta_buffer_; std::atomic_int* buffer_status_; public: diff --git a/sf-writer/src/ImageAssembler.cpp b/sf-writer/src/ImageAssembler.cpp index 290d8d5..9773630 100644 --- a/sf-writer/src/ImageAssembler.cpp +++ b/sf-writer/src/ImageAssembler.cpp @@ -10,7 +10,7 @@ ImageAssembler::ImageAssembler(const size_t n_modules) : { image_buffer_ = new char[IA_N_SLOTS * image_buffer_slot_n_bytes_]; meta_buffer_ = new ImageMetadataBlock[IA_N_SLOTS]; - frame_metadata_buffer_ = + frame_meta_buffer_ = new ModuleFrame[IA_N_SLOTS * n_modules * BUFFER_BLOCK_SIZE]; buffer_status_ = new atomic_int[IA_N_SLOTS]; @@ -37,7 +37,7 @@ bool ImageAssembler::is_slot_full(const uint64_t bunch_id) return buffer_status_[slot_id].load() == 0; } -inline size_t ImageAssembler::get_data_offset( +size_t ImageAssembler::get_data_offset( const uint64_t slot_id, const int i_module) { size_t slot_i_offset = slot_id * image_buffer_slot_n_bytes_; @@ -46,7 +46,7 @@ inline size_t ImageAssembler::get_data_offset( return slot_i_offset + module_i_offset; } -inline size_t ImageAssembler::get_metadata_offset( +size_t ImageAssembler::get_metadata_offset( const uint64_t slot_id, const int i_module) { size_t n_metadata_in_slot = n_modules_ * BUFFER_BLOCK_SIZE; @@ -67,12 +67,12 @@ void ImageAssembler::process( const auto image_offset_step = MODULE_N_BYTES * n_modules_; const auto meta_offset = get_metadata_offset(slot_id, i_module); - const auto meta_offset_step = 1; + const auto meta_offset_step = n_modules_; for (size_t i_pulse=0; i_pulse < BUFFER_BLOCK_SIZE; i_pulse++) { memcpy( - &(frame_metadata_buffer_[meta_offset]), + &(frame_meta_buffer_[meta_offset]), &(block_buffer->frame[i_pulse].metadata), sizeof(ModuleFrame)); @@ -101,7 +101,7 @@ ImageMetadataBlock* ImageAssembler::get_metadata_buffer(const uint64_t bunch_id) for (size_t i_pulse=0; i_pulse < BUFFER_BLOCK_SIZE; i_pulse++) { auto is_not_init = false; - + meta for (size_t i_module=0; i_module < n_modules_; i_module++) { auto metadata_offset = get_metadata_offset(slot_id, i_module); // From 374fc62d1ee75b86a8f76ef2c78544704922e630 Mon Sep 17 00:00:00 2001 From: Dmitry Ozerov Date: Tue, 2 Jun 2020 13:56:23 +0200 Subject: [PATCH 099/111] systemd scripts for JF01 and JF13 --- scripts/JF01-buffer-worker.sh | 20 ++++++++ scripts/JF01-buffer-worker@.service | 16 +++++++ ...F07-replay.service => JF01-buffer.service} | 4 +- scripts/JF01-stream.service | 15 ++++++ scripts/JF01-stream.sh | 5 ++ scripts/JF07-replay-worker.sh | 26 ----------- scripts/JF07-replay-worker@.service | 16 ------- scripts/JF13-buffer-worker.sh | 20 ++++++++ scripts/JF13-buffer-worker@.service | 16 +++++++ scripts/JF13-buffer.service | 10 ++++ scripts/JF13-stream.service | 15 ++++++ scripts/JF13-stream.sh | 5 ++ scripts/start_detector.sh | 46 +++++++++++++++++++ sf-stream/src/main.cpp | 4 +- 14 files changed, 172 insertions(+), 46 deletions(-) create mode 100644 scripts/JF01-buffer-worker.sh create mode 100644 scripts/JF01-buffer-worker@.service rename scripts/{JF07-replay.service => JF01-buffer.service} (70%) create mode 100644 scripts/JF01-stream.service create mode 100644 scripts/JF01-stream.sh delete mode 100644 scripts/JF07-replay-worker.sh delete mode 100644 scripts/JF07-replay-worker@.service create mode 100644 scripts/JF13-buffer-worker.sh create mode 100644 scripts/JF13-buffer-worker@.service create mode 100644 scripts/JF13-buffer.service create mode 100644 scripts/JF13-stream.service create mode 100644 scripts/JF13-stream.sh create mode 100755 scripts/start_detector.sh diff --git a/scripts/JF01-buffer-worker.sh b/scripts/JF01-buffer-worker.sh new file mode 100644 index 0000000..b907af5 --- /dev/null +++ b/scripts/JF01-buffer-worker.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +if [ $# != 1 ] +then + systemctl start JF01-buffer-worker@{00..02} + exit +fi + +M=$1 + +# Add ourselves to the user cpuset. +# echo $$ > /sys/fs/cgroup/cpuset/user/tasks + +coreAssociatedBuffer=(12 12 12) + +initialUDPport=50010 +port=$((${initialUDPport}+10#${M})) +DETECTOR=JF01T03V01 + +taskset -c ${coreAssociatedBuffer[10#${M}]} /usr/bin/sf_buffer ${DETECTOR} M${M} ${port} /gpfs/photonics/swissfel/buffer/${DETECTOR} ${M} diff --git a/scripts/JF01-buffer-worker@.service b/scripts/JF01-buffer-worker@.service new file mode 100644 index 0000000..b8b6a97 --- /dev/null +++ b/scripts/JF01-buffer-worker@.service @@ -0,0 +1,16 @@ +[Unit] +Description=JF01 UDP2buffer worker instance as a service, instance %i +Requires=JF01-buffer.service +Before=JF01-buffer.service +BindsTo=JF01-buffer.service + +[Service] +PermissionsStartOnly=true +Type=idle +User=root +ExecStart=/usr/bin/sh /home/writer/git/sf_daq_buffer/scripts/JF01-buffer-worker.sh %i +TimeoutStartSec=10 +RestartSec=10 + +[Install] +WantedBy=JF01-buffer.service diff --git a/scripts/JF07-replay.service b/scripts/JF01-buffer.service similarity index 70% rename from scripts/JF07-replay.service rename to scripts/JF01-buffer.service index 288b922..efdc14d 100644 --- a/scripts/JF07-replay.service +++ b/scripts/JF01-buffer.service @@ -1,9 +1,9 @@ [Unit] -Description=All replay instances of JF07 +Description=All UDP-buffer instances of JF01 [Service] Type=oneshot -ExecStart=/usr/bin/sh /home/writer/git/sf_daq_buffer/scripts/JF07-replay-worker.sh +ExecStart=/usr/bin/sh /home/writer/git/sf_daq_buffer/scripts/JF01-buffer-worker.sh RemainAfterExit=yes [Install] diff --git a/scripts/JF01-stream.service b/scripts/JF01-stream.service new file mode 100644 index 0000000..79e7b1b --- /dev/null +++ b/scripts/JF01-stream.service @@ -0,0 +1,15 @@ +[Unit] +Description=stream service (to streamvis and live analysis) of JF01 + +[Service] +PermissionsStartOnly=true +Type=idle +User=root +ExecStart=/usr/bin/sh /home/writer/git/sf_daq_buffer/scripts/JF01-stream.sh +TimeoutStartSec=10 +Restart=on-failure +RestartSec=10 + +[Install] +WantedBy=multi-user.target + diff --git a/scripts/JF01-stream.sh b/scripts/JF01-stream.sh new file mode 100644 index 0000000..c7d1724 --- /dev/null +++ b/scripts/JF01-stream.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +coreAssociated="24" + +taskset -c ${coreAssociated} /usr/bin/sf_stream /gpfs/photonics/swissfel/buffer/config/stream-JF01.json diff --git a/scripts/JF07-replay-worker.sh b/scripts/JF07-replay-worker.sh deleted file mode 100644 index bb51829..0000000 --- a/scripts/JF07-replay-worker.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -if [ $# != 1 ] -then - systemctl start JF07-replay-worker@{00..32} - exit -fi - -M=$1 - -#8 replay workers per core, last (stream to visualisation) worker occupies 4 -coreAssociated=(20 20 20 20 20 20 20 20 21 21 21 21 21 21 21 21 22 22 22 22 22 22 22 22 23 23 23 23 23 23 23 23 24,25,26,27) - -latest_file=`cat /gpfs/photonics/swissfel/buffer/JF07T32V01/M00/LATEST` -last_pulse_id=`basename ${latest_file} | sed 's/.h5//'` -first_pulse_id=$((${last_pulse_id}-360000)) - -echo "First/last pulse_id : ${first_pulse_id} ${last_pulse_id}" - -if [ ${M} == 32 ] -then -# taskset -c ${coreAssociated[10#${M}]} /usr/bin/sf_writer /gpfs/photonics/swissfel/buffer/test.${first_pulse_id}-${last_pulse_id}.h5 ${first_pulse_id} ${last_pulse_id} - taskset -c ${coreAssociated[10#${M}]} /usr/bin/sf_stream tcp://129.129.241.42:9007 10 tcp://192.168.30.29:9107 30 -else - taskset -c ${coreAssociated[10#${M}]} /usr/bin/sf_replay /gpfs/photonics/swissfel/buffer/JF07T32V01 M${M} ${M} ${first_pulse_id} ${last_pulse_id} -fi diff --git a/scripts/JF07-replay-worker@.service b/scripts/JF07-replay-worker@.service deleted file mode 100644 index 42e8dda..0000000 --- a/scripts/JF07-replay-worker@.service +++ /dev/null @@ -1,16 +0,0 @@ -[Unit] -Description=JF07 replay worker instance as a service, instance %i -Requires=JF07-replay.service -Before=JF07-replay.service -BindsTo=JF07-replay.service - -[Service] -PermissionsStartOnly=true -Type=idle -User=root -ExecStart=/usr/bin/sh /home/writer/git/sf_daq_buffer/scripts/JF07-replay-worker.sh %i -TimeoutStartSec=10 -RestartSec=10 - -[Install] -WantedBy=JF07-replay.service diff --git a/scripts/JF13-buffer-worker.sh b/scripts/JF13-buffer-worker.sh new file mode 100644 index 0000000..c269179 --- /dev/null +++ b/scripts/JF13-buffer-worker.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +if [ $# != 1 ] +then + systemctl start JF13-buffer-worker@00 + exit +fi + +M=$1 + +# Add ourselves to the user cpuset. +# echo $$ > /sys/fs/cgroup/cpuset/user/tasks + +coreAssociatedBuffer=(13) + +initialUDPport=50190 +port=$((${initialUDPport}+10#${M})) +DETECTOR=JF13T01V01 + +taskset -c ${coreAssociatedBuffer[10#${M}]} /usr/bin/sf_buffer ${DETECTOR} M${M} ${port} /gpfs/photonics/swissfel/buffer/${DETECTOR} ${M} diff --git a/scripts/JF13-buffer-worker@.service b/scripts/JF13-buffer-worker@.service new file mode 100644 index 0000000..0ccd3d5 --- /dev/null +++ b/scripts/JF13-buffer-worker@.service @@ -0,0 +1,16 @@ +[Unit] +Description=JF13 UDP2buffer worker instance as a service, instance %i +Requires=JF13-buffer.service +Before=JF13-buffer.service +BindsTo=JF13-buffer.service + +[Service] +PermissionsStartOnly=true +Type=idle +User=root +ExecStart=/usr/bin/sh /home/writer/git/sf_daq_buffer/scripts/JF13-buffer-worker.sh %i +TimeoutStartSec=10 +RestartSec=10 + +[Install] +WantedBy=JF13-buffer.service diff --git a/scripts/JF13-buffer.service b/scripts/JF13-buffer.service new file mode 100644 index 0000000..2c435ce --- /dev/null +++ b/scripts/JF13-buffer.service @@ -0,0 +1,10 @@ +[Unit] +Description=All UDP-buffer instances of JF13 + +[Service] +Type=oneshot +ExecStart=/usr/bin/sh /home/writer/git/sf_daq_buffer/scripts/JF13-buffer-worker.sh +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target diff --git a/scripts/JF13-stream.service b/scripts/JF13-stream.service new file mode 100644 index 0000000..7e0a492 --- /dev/null +++ b/scripts/JF13-stream.service @@ -0,0 +1,15 @@ +[Unit] +Description=stream service (to streamvis and live analysis) of JF13 + +[Service] +PermissionsStartOnly=true +Type=idle +User=root +ExecStart=/usr/bin/sh /home/writer/git/sf_daq_buffer/scripts/JF13-stream.sh +TimeoutStartSec=10 +Restart=on-failure +RestartSec=10 + +[Install] +WantedBy=multi-user.target + diff --git a/scripts/JF13-stream.sh b/scripts/JF13-stream.sh new file mode 100644 index 0000000..3b8e226 --- /dev/null +++ b/scripts/JF13-stream.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +coreAssociated="25" + +taskset -c ${coreAssociated} /usr/bin/sf_stream /gpfs/photonics/swissfel/buffer/config/stream-JF13.json diff --git a/scripts/start_detector.sh b/scripts/start_detector.sh new file mode 100755 index 0000000..482d7d4 --- /dev/null +++ b/scripts/start_detector.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +if [ $# -lt 1 ] +then + echo "Usage : $0 DETECTOR_NAME " + echo " DETECTOR_NAME: JF07 or JF01..." + echo " number_of_cycles : optional, default 100" + exit +fi + +DETECTOR=$1 +case ${DETECTOR} in +'JF01') + D=1 + ;; +'JF07') + D=7 + ;; +'JF13') + D=13 + ;; +*) + echo "Unsupported detector" + exit + ;; +esac + +n_cycles=100 +if [ $# == 2 ] +then + n_cycles=$2 +fi + +export PATH=/home/dbe/miniconda3/bin:$PATH +source deactivate +source activate dia + +sls_detector_put ${D}-timing trigger +sls_detector_put ${D}-cycles ${n_cycles} +sls_detector_put ${D}-exptime 5e-06 +sls_detector_put ${D}-frames 1 +sls_detector_put ${D}-dr 16 +#sls_detector_put ${D}-clearbit to 0x5d 0 # normal mode, not highG0 +sls_detector_put ${D}-status start + +echo "Now start trigger" diff --git a/sf-stream/src/main.cpp b/sf-stream/src/main.cpp index b0a9632..d9a4b45 100644 --- a/sf-stream/src/main.cpp +++ b/sf-stream/src/main.cpp @@ -170,7 +170,7 @@ int main (int argc, char *argv[]) } if ( send_streamvis == 0 ) { auto& shape = header["shape"]; - shape[0] = 16384; + shape[0] = n_modules*512; shape[1] = 1024; } else{ auto& shape = header["shape"]; @@ -210,7 +210,7 @@ int main (int argc, char *argv[]) } if ( send_live_analysis == 0 ) { auto& shape = header["shape"]; - shape[0] = 16384; + shape[0] = n_modules*512; shape[1] = 1024; } else{ auto& shape = header["shape"]; From 1a9caf08ee030c7e7ef948e8da3eda02378131f0 Mon Sep 17 00:00:00 2001 From: Dmitry Ozerov Date: Tue, 2 Jun 2020 14:34:07 +0200 Subject: [PATCH 100/111] cleanup script for the buffer --- scripts/delete_old_files_in_buffer.sh | 30 +++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100755 scripts/delete_old_files_in_buffer.sh diff --git a/scripts/delete_old_files_in_buffer.sh b/scripts/delete_old_files_in_buffer.sh new file mode 100755 index 0000000..1d0db5a --- /dev/null +++ b/scripts/delete_old_files_in_buffer.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +hours=5 +threshold=80 + +if [ $# = 1 ] +then + threshold=$1 +fi +if [ $# = 2 ] +then + hours=$2 +fi + +df -h | grep BUFFER > /dev/null +if [ $? != 0 ] +then + # BUFFER is not present + exit +fi + +occupancy=`df -h /gpfs/photonics/swissfel/buffer | grep BUFFER | awk '{print $5}' | sed 's/%//'` +if [ ${occupancy} -lt ${threshold} ] +then +# echo OK, not action + exit +fi + +#find /gpfs/photonics/swissfel/buffer/JF* -type f -mmin +$((${hours}*60)) -delete +find /gpfs/photonics/swissfel/buffer/JF*/M* -type d -mmin +$((${hours}*60)) -delete From 477efdddbecfff8a69f62cd54d244e702892600b Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 2 Jun 2020 15:44:51 +0200 Subject: [PATCH 101/111] Finished implementation of ImageAssembler --- sf-writer/include/ImageAssembler.hpp | 3 ++ sf-writer/src/ImageAssembler.cpp | 68 ++++++++++++++++++++++------ 2 files changed, 58 insertions(+), 13 deletions(-) diff --git a/sf-writer/include/ImageAssembler.hpp b/sf-writer/include/ImageAssembler.hpp index ada474e..4aa13c9 100644 --- a/sf-writer/include/ImageAssembler.hpp +++ b/sf-writer/include/ImageAssembler.hpp @@ -16,6 +16,9 @@ class ImageAssembler { ModuleFrame* frame_meta_buffer_; std::atomic_int* buffer_status_; + size_t get_data_offset(const uint64_t slot_id, const int i_module); + size_t get_metadata_offset(const uint64_t slot_id, const int i_module); + public: ImageAssembler(const size_t n_modules); diff --git a/sf-writer/src/ImageAssembler.cpp b/sf-writer/src/ImageAssembler.cpp index 9773630..153c7d7 100644 --- a/sf-writer/src/ImageAssembler.cpp +++ b/sf-writer/src/ImageAssembler.cpp @@ -63,10 +63,10 @@ void ImageAssembler::process( { const auto slot_id = bunch_id % IA_N_SLOTS; - const auto image_offset = get_data_offset(slot_id, i_module); + auto image_offset = get_data_offset(slot_id, i_module); const auto image_offset_step = MODULE_N_BYTES * n_modules_; - const auto meta_offset = get_metadata_offset(slot_id, i_module); + auto meta_offset = get_metadata_offset(slot_id, i_module); const auto meta_offset_step = n_modules_; for (size_t i_pulse=0; i_pulse < BUFFER_BLOCK_SIZE; i_pulse++) { @@ -97,20 +97,62 @@ void ImageAssembler::free_slot(const uint64_t bunch_id) ImageMetadataBlock* ImageAssembler::get_metadata_buffer(const uint64_t bunch_id) { - auto slot_id = bunch_id % IA_N_SLOTS; + const auto slot_id = bunch_id % IA_N_SLOTS; + + auto& image_pulse_id = meta_buffer_[slot_id].pulse_id; + auto& image_frame_index = meta_buffer_[slot_id].frame_index; + auto& image_daq_rec = meta_buffer_[slot_id].daq_rec; + auto& image_is_good_frame = meta_buffer_[slot_id].is_good_image; + + auto meta_offset = get_metadata_offset(slot_id, 0); + const auto meta_offset_step = 1; for (size_t i_pulse=0; i_pulse < BUFFER_BLOCK_SIZE; i_pulse++) { - auto is_not_init = false; - meta - for (size_t i_module=0; i_module < n_modules_; i_module++) { - auto metadata_offset = get_metadata_offset(slot_id, i_module); -// -// if (is_not_init) { -// if () -// } - } - meta_buffer_[slot_id].pulse_id[i_pulse]; + auto is_pulse_init = false; + image_is_good_frame[i_pulse] = 1; + + for (size_t i_module=0; i_module < n_modules_; i_module++) { + + auto& frame_meta = frame_meta_buffer_[meta_offset]; + auto is_good_frame = + frame_meta.n_received_packets == JF_N_PACKETS_PER_FRAME; + + if (!is_good_frame) { + image_is_good_frame[i_pulse] = 0; + // TODO: Update meta_offset only once in the loop. + meta_offset += meta_offset_step; + continue; + } + + if (!is_pulse_init) { + image_pulse_id[i_pulse] = frame_meta.pulse_id; + image_frame_index[i_pulse] = frame_meta.frame_index; + image_daq_rec[i_pulse] = frame_meta.daq_rec; + + is_pulse_init = true; + } + + if (image_is_good_frame[i_pulse] == 1) { + if (image_pulse_id[i_pulse] != frame_meta.pulse_id) { + image_is_good_frame[i_pulse] = 0; + } + + if (image_frame_index[i_pulse] != frame_meta.frame_index) { + image_is_good_frame[i_pulse] = 0; + } + + if (image_daq_rec[i_pulse] != frame_meta.daq_rec) { + image_is_good_frame[i_pulse] = 0; + } + + if (frame_meta.n_received_packets != JF_N_PACKETS_PER_FRAME) { + image_is_good_frame[i_pulse] = 0; + } + } + + meta_offset += meta_offset_step; + } } return &(meta_buffer_[slot_id]); From 2b75770aca2994f72c3cf24a855312d504de87c0 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 2 Jun 2020 15:46:42 +0200 Subject: [PATCH 102/111] Add test to check the image assembly metadata calculation --- sf-writer/test/test_ImageAssembler.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sf-writer/test/test_ImageAssembler.cpp b/sf-writer/test/test_ImageAssembler.cpp index 0071015..78916bb 100644 --- a/sf-writer/test/test_ImageAssembler.cpp +++ b/sf-writer/test/test_ImageAssembler.cpp @@ -29,4 +29,8 @@ TEST(ImageAssembler, basic_interaction) assembler.free_slot(bunch_id); ASSERT_EQ(assembler.is_slot_free(bunch_id), true); -} + + for (size_t i_pulse = 0; i_pulse < BUFFER_BLOCK_SIZE; i_pulse++) { + ASSERT_EQ(metadata->is_good_image[i_pulse], 0); + } +} \ No newline at end of file From 6904684d48773132c02346c92fbdba9b45ae6b27 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 2 Jun 2020 16:48:24 +0200 Subject: [PATCH 103/111] Reconstruction unit test --- sf-writer/test/test_ImageAssembler.cpp | 53 +++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/sf-writer/test/test_ImageAssembler.cpp b/sf-writer/test/test_ImageAssembler.cpp index 78916bb..c9bab06 100644 --- a/sf-writer/test/test_ImageAssembler.cpp +++ b/sf-writer/test/test_ImageAssembler.cpp @@ -33,4 +33,55 @@ TEST(ImageAssembler, basic_interaction) for (size_t i_pulse = 0; i_pulse < BUFFER_BLOCK_SIZE; i_pulse++) { ASSERT_EQ(metadata->is_good_image[i_pulse], 0); } -} \ No newline at end of file +} + +TEST(ImageAssembler, reconstruction) +{ + size_t n_modules = 2; + uint64_t bunch_id = 0; + + ImageAssembler assembler(n_modules); + + ASSERT_EQ(assembler.is_slot_free(bunch_id), true); + + auto buffer_block = make_unique(); + auto buffer_ptr = buffer_block.get(); + + for (size_t i_module=0; i_module < n_modules; i_module++) { + + for (size_t i_pulse=0; i_pulse < BUFFER_BLOCK_SIZE; i_pulse++) { + auto& frame_meta = buffer_block->frame[i_pulse].metadata; + + frame_meta.pulse_id = 100 + i_pulse; + frame_meta.daq_rec = 1000 + i_pulse; + frame_meta.frame_index = 10000 + i_pulse; + frame_meta.n_received_packets = JF_N_PACKETS_PER_FRAME; + + for (size_t i_pixel=0; i_pixel < MODULE_N_PIXELS; i_pixel++) { + buffer_block->frame[i_pulse].data[i_pixel] = + (i_module * 10) + (i_pixel % 100); + } + } + + assembler.process(bunch_id, i_module, buffer_ptr); + } + + ASSERT_EQ(assembler.is_slot_full(bunch_id), true); + + auto metadata = assembler.get_metadata_buffer(bunch_id); + auto data = assembler.get_data_buffer(bunch_id); + + assembler.free_slot(bunch_id); + ASSERT_EQ(assembler.is_slot_free(bunch_id), true); + + for (size_t i_pulse = 0; i_pulse < BUFFER_BLOCK_SIZE; i_pulse++) { + ASSERT_EQ(metadata->pulse_id[i_pulse], 100 + i_pulse); + ASSERT_EQ(metadata->daq_rec[i_pulse], 1000 + i_pulse); + ASSERT_EQ(metadata->frame_index[i_pulse], 10000 + i_pulse); + ASSERT_EQ(metadata->is_good_image[i_pulse], 1); + + for (size_t i_module=0; i_module < n_modules; i_module++) { + // TODO: Check assembled image. + } + } +} From dbc07b6ad694d11799b09e8f243156130b3cb00f Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 2 Jun 2020 17:26:46 +0200 Subject: [PATCH 104/111] Add possibility to link direct chunk write --- sf-writer/test/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/sf-writer/test/CMakeLists.txt b/sf-writer/test/CMakeLists.txt index 13dd5c0..2b9f4f0 100644 --- a/sf-writer/test/CMakeLists.txt +++ b/sf-writer/test/CMakeLists.txt @@ -3,6 +3,7 @@ add_executable(sf-writer-tests main.cpp) target_link_libraries(sf-writer-tests sf-writer-lib hdf5 + hdf5_hl hdf5_cpp zmq gtest From ec0f763b704e40627a48a833176f8fe001b95bfe Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 2 Jun 2020 17:27:00 +0200 Subject: [PATCH 105/111] Change writer to do direct chunk write --- sf-writer/src/JFH5Writer.cpp | 58 +++++++++++++++++------------------- 1 file changed, 27 insertions(+), 31 deletions(-) diff --git a/sf-writer/src/JFH5Writer.cpp b/sf-writer/src/JFH5Writer.cpp index f798b64..8e2a817 100644 --- a/sf-writer/src/JFH5Writer.cpp +++ b/sf-writer/src/JFH5Writer.cpp @@ -3,12 +3,12 @@ #include #include - -//extern "C" -//{ +extern "C" +{ // #include "H5DOpublic.h" // #include -//} + #include "H5DOpublic.h" +} using namespace std; using namespace core_buffer; @@ -33,9 +33,9 @@ JFH5Writer::JFH5Writer(const std::string& output_file, H5::DataSpace image_dataspace(3, image_dataset_dims); - auto chunk_size = min(n_images_, BUFFER_BLOCK_SIZE); +// auto chunk_size = min(n_images_, BUFFER_BLOCK_SIZE); hsize_t image_dataset_chunking[3] = - {chunk_size, n_modules * MODULE_Y_SIZE, MODULE_X_SIZE}; + {1, n_modules * MODULE_Y_SIZE, MODULE_X_SIZE}; H5::DSetCreatPropList image_dataset_properties; image_dataset_properties.setChunk(3, image_dataset_chunking); @@ -148,30 +148,6 @@ void JFH5Writer::write( throw runtime_error("Received unexpected block for stop_pulse_id."); } - hsize_t b_i_dims[3] = {BUFFER_BLOCK_SIZE, - MODULE_Y_SIZE * n_modules_, - MODULE_X_SIZE}; - H5::DataSpace b_i_space(3, b_i_dims); - hsize_t b_i_count[] = {n_images_to_copy, - MODULE_Y_SIZE * n_modules_, - MODULE_X_SIZE}; - hsize_t b_i_start[] = {n_images_offset, 0, 0}; - b_i_space.selectHyperslab(H5S_SELECT_SET, b_i_count, b_i_start); - - hsize_t f_i_dims[3] = {n_images_, - MODULE_Y_SIZE * n_modules_, - MODULE_X_SIZE}; - H5::DataSpace f_i_space(3, f_i_dims); - hsize_t f_i_count[] = {n_images_to_copy, - MODULE_Y_SIZE * n_modules_, - MODULE_X_SIZE}; - hsize_t f_i_start[] = {current_write_index_, 0, 0}; - f_i_space.selectHyperslab(H5S_SELECT_SET, f_i_count, f_i_start); - - image_dataset_.write( - data, H5::PredType::NATIVE_UINT16, b_i_space, f_i_space); - - // pulse_id { auto b_current_ptr = b_pulse_id_ + current_write_index_; @@ -204,5 +180,25 @@ void JFH5Writer::write( sizeof(uint8_t) * n_images_to_copy); } - current_write_index_ += n_images_to_copy; + // images - direct chunk write one by one. + for (size_t i_pulse=n_images_offset; + i_pulse Date: Tue, 2 Jun 2020 17:31:05 +0200 Subject: [PATCH 106/111] Fix writer implementation --- sf-writer/src/JFH5Writer.cpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/sf-writer/src/JFH5Writer.cpp b/sf-writer/src/JFH5Writer.cpp index 8e2a817..da808f1 100644 --- a/sf-writer/src/JFH5Writer.cpp +++ b/sf-writer/src/JFH5Writer.cpp @@ -56,8 +56,6 @@ JFH5Writer::JFH5Writer(const std::string& output_file, image_dataspace, image_dataset_properties); - - b_pulse_id_ = new uint64_t[n_images_]; b_frame_index_= new uint64_t[n_images_]; b_daq_rec_ = new uint32_t[n_images_]; @@ -181,11 +179,11 @@ void JFH5Writer::write( } // images - direct chunk write one by one. - for (size_t i_pulse=n_images_offset; - i_pulse Date: Tue, 2 Jun 2020 17:37:20 +0200 Subject: [PATCH 107/111] Add reconstruction block start and stop pulse_id --- sf-writer/src/ImageAssembler.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sf-writer/src/ImageAssembler.cpp b/sf-writer/src/ImageAssembler.cpp index 153c7d7..3e45fcf 100644 --- a/sf-writer/src/ImageAssembler.cpp +++ b/sf-writer/src/ImageAssembler.cpp @@ -107,6 +107,10 @@ ImageMetadataBlock* ImageAssembler::get_metadata_buffer(const uint64_t bunch_id) auto meta_offset = get_metadata_offset(slot_id, 0); const auto meta_offset_step = 1; + meta_buffer_->block_start_pulse_id = bunch_id * BUFFER_BLOCK_SIZE; + meta_buffer_->block_stop_pulse_id = + meta_buffer_->block_start_pulse_id + BUFFER_BLOCK_SIZE - 1; + for (size_t i_pulse=0; i_pulse < BUFFER_BLOCK_SIZE; i_pulse++) { auto is_pulse_init = false; From c41c9809e02dd133e586f10d36a36d825866c410 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 2 Jun 2020 17:37:38 +0200 Subject: [PATCH 108/111] Add test for start and stop block pulse_id --- sf-writer/test/test_ImageAssembler.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sf-writer/test/test_ImageAssembler.cpp b/sf-writer/test/test_ImageAssembler.cpp index c9bab06..fb98fba 100644 --- a/sf-writer/test/test_ImageAssembler.cpp +++ b/sf-writer/test/test_ImageAssembler.cpp @@ -74,6 +74,9 @@ TEST(ImageAssembler, reconstruction) assembler.free_slot(bunch_id); ASSERT_EQ(assembler.is_slot_free(bunch_id), true); + ASSERT_EQ(metadata->block_start_pulse_id, 0); + ASSERT_EQ(metadata->block_stop_pulse_id, BUFFER_BLOCK_SIZE-1); + for (size_t i_pulse = 0; i_pulse < BUFFER_BLOCK_SIZE; i_pulse++) { ASSERT_EQ(metadata->pulse_id[i_pulse], 100 + i_pulse); ASSERT_EQ(metadata->daq_rec[i_pulse], 1000 + i_pulse); From acd14f67c67801529bfc738d7318eb1b57b745cb Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 2 Jun 2020 17:48:23 +0200 Subject: [PATCH 109/111] Fix bug in image assembler --- sf-writer/src/ImageAssembler.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sf-writer/src/ImageAssembler.cpp b/sf-writer/src/ImageAssembler.cpp index 3e45fcf..438aa78 100644 --- a/sf-writer/src/ImageAssembler.cpp +++ b/sf-writer/src/ImageAssembler.cpp @@ -107,8 +107,9 @@ ImageMetadataBlock* ImageAssembler::get_metadata_buffer(const uint64_t bunch_id) auto meta_offset = get_metadata_offset(slot_id, 0); const auto meta_offset_step = 1; - meta_buffer_->block_start_pulse_id = bunch_id * BUFFER_BLOCK_SIZE; - meta_buffer_->block_stop_pulse_id = + meta_buffer_[slot_id].block_start_pulse_id = + bunch_id * BUFFER_BLOCK_SIZE; + meta_buffer_[slot_id].block_stop_pulse_id = meta_buffer_->block_start_pulse_id + BUFFER_BLOCK_SIZE - 1; for (size_t i_pulse=0; i_pulse < BUFFER_BLOCK_SIZE; i_pulse++) { From 1af9d10683d9a3bc54cc19be00783f32ade0f1b4 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 2 Jun 2020 17:52:38 +0200 Subject: [PATCH 110/111] Reverted back to non chunk writer --- sf-writer/src/JFH5Writer.cpp | 65 +++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/sf-writer/src/JFH5Writer.cpp b/sf-writer/src/JFH5Writer.cpp index da808f1..17a2110 100644 --- a/sf-writer/src/JFH5Writer.cpp +++ b/sf-writer/src/JFH5Writer.cpp @@ -3,12 +3,12 @@ #include #include -extern "C" -{ + +//extern "C" +//{ // #include "H5DOpublic.h" // #include - #include "H5DOpublic.h" -} +//} using namespace std; using namespace core_buffer; @@ -34,10 +34,10 @@ JFH5Writer::JFH5Writer(const std::string& output_file, H5::DataSpace image_dataspace(3, image_dataset_dims); // auto chunk_size = min(n_images_, BUFFER_BLOCK_SIZE); - hsize_t image_dataset_chunking[3] = - {1, n_modules * MODULE_Y_SIZE, MODULE_X_SIZE}; - H5::DSetCreatPropList image_dataset_properties; - image_dataset_properties.setChunk(3, image_dataset_chunking); +// hsize_t image_dataset_chunking[3] = +// {chunk_size, n_modules * MODULE_Y_SIZE, MODULE_X_SIZE}; +// H5::DSetCreatPropList image_dataset_properties; +// image_dataset_properties.setChunk(3, image_dataset_chunking); // // block_size, compression type // uint compression_prop[] = @@ -53,8 +53,7 @@ JFH5Writer::JFH5Writer(const std::string& output_file, image_dataset_ = file_.createDataSet( "image", H5::PredType::NATIVE_UINT16, - image_dataspace, - image_dataset_properties); + image_dataspace); b_pulse_id_ = new uint64_t[n_images_]; b_frame_index_= new uint64_t[n_images_]; @@ -146,6 +145,30 @@ void JFH5Writer::write( throw runtime_error("Received unexpected block for stop_pulse_id."); } + hsize_t b_i_dims[3] = {BUFFER_BLOCK_SIZE, + MODULE_Y_SIZE * n_modules_, + MODULE_X_SIZE}; + H5::DataSpace b_i_space(3, b_i_dims); + hsize_t b_i_count[] = {n_images_to_copy, + MODULE_Y_SIZE * n_modules_, + MODULE_X_SIZE}; + hsize_t b_i_start[] = {n_images_offset, 0, 0}; + b_i_space.selectHyperslab(H5S_SELECT_SET, b_i_count, b_i_start); + + hsize_t f_i_dims[3] = {n_images_, + MODULE_Y_SIZE * n_modules_, + MODULE_X_SIZE}; + H5::DataSpace f_i_space(3, f_i_dims); + hsize_t f_i_count[] = {n_images_to_copy, + MODULE_Y_SIZE * n_modules_, + MODULE_X_SIZE}; + hsize_t f_i_start[] = {current_write_index_, 0, 0}; + f_i_space.selectHyperslab(H5S_SELECT_SET, f_i_count, f_i_start); + + image_dataset_.write( + data, H5::PredType::NATIVE_UINT16, b_i_space, f_i_space); + + // pulse_id { auto b_current_ptr = b_pulse_id_ + current_write_index_; @@ -178,25 +201,5 @@ void JFH5Writer::write( sizeof(uint8_t) * n_images_to_copy); } - // images - direct chunk write one by one. - for (size_t i_image=n_images_offset; - i_image < n_images_to_copy; - i_image++) { - - auto image_ptr = data + (i_image * n_modules_ * MODULE_N_BYTES); - hsize_t offset[] = {current_write_index_, 0, 0}; - - if(H5DOwrite_chunk(image_dataset_.getId(), H5P_DEFAULT, 0, - offset, MODULE_N_BYTES*n_modules_, data)) { - - stringstream error_message; - error_message << "[JFH5Writer::write]"; - error_message << "Error while writing image chunk to file."; - error_message << "Offset " << current_write_index_ << endl; - - throw invalid_argument(error_message.str()); - } - - current_write_index_++; - } + current_write_index_ += n_images_to_copy; } From 9c32529638e47749f6f2efd674234d5954333f9a Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 2 Jun 2020 18:12:13 +0200 Subject: [PATCH 111/111] Make block start and stop pulse_id more readable --- sf-writer/src/ImageAssembler.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/sf-writer/src/ImageAssembler.cpp b/sf-writer/src/ImageAssembler.cpp index 438aa78..73a0820 100644 --- a/sf-writer/src/ImageAssembler.cpp +++ b/sf-writer/src/ImageAssembler.cpp @@ -107,10 +107,11 @@ ImageMetadataBlock* ImageAssembler::get_metadata_buffer(const uint64_t bunch_id) auto meta_offset = get_metadata_offset(slot_id, 0); const auto meta_offset_step = 1; - meta_buffer_[slot_id].block_start_pulse_id = - bunch_id * BUFFER_BLOCK_SIZE; - meta_buffer_[slot_id].block_stop_pulse_id = - meta_buffer_->block_start_pulse_id + BUFFER_BLOCK_SIZE - 1; + uint64_t start_pulse_id = bunch_id * BUFFER_BLOCK_SIZE; + meta_buffer_[slot_id].block_start_pulse_id = start_pulse_id; + + uint64_t stop_pulse_id = start_pulse_id + BUFFER_BLOCK_SIZE - 1; + meta_buffer_[slot_id].block_stop_pulse_id = stop_pulse_id; for (size_t i_pulse=0; i_pulse < BUFFER_BLOCK_SIZE; i_pulse++) {