From d1ab066eac662df1a052b9bbb472ca3a73fee5b1 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 6 Jul 2021 14:48:45 +0200 Subject: [PATCH 1/3] Moved live writer to std-det-writer --- CMakeLists.txt | 2 +- jf-live-writer/CMakeLists.txt | 38 ------------------- std-det-writer/CMakeLists.txt | 31 +++++++++++++++ {jf-live-writer => std-det-writer}/README.md | 2 +- .../debug.Dockerfile | 4 +- .../include/JFH5Writer.hpp | 0 std-det-writer/include/StdDetWriterConfig.hpp | 33 ++++++++++++++++ .../include/WriterStats.hpp | 0 .../include/broker_format.hpp | 0 .../include/live_writer_config.hpp | 0 .../src/JFH5Writer.cpp | 0 .../src/WriterStats.cpp | 0 .../src/main.cpp | 20 ++++++---- .../test/CMakeLists.txt | 0 .../test/main.cpp | 0 15 files changed, 80 insertions(+), 50 deletions(-) delete mode 100644 jf-live-writer/CMakeLists.txt create mode 100644 std-det-writer/CMakeLists.txt rename {jf-live-writer => std-det-writer}/README.md (99%) rename {jf-live-writer => std-det-writer}/debug.Dockerfile (71%) rename {jf-live-writer => std-det-writer}/include/JFH5Writer.hpp (100%) create mode 100644 std-det-writer/include/StdDetWriterConfig.hpp rename {jf-live-writer => std-det-writer}/include/WriterStats.hpp (100%) rename {jf-live-writer => std-det-writer}/include/broker_format.hpp (100%) rename {jf-live-writer => std-det-writer}/include/live_writer_config.hpp (100%) rename {jf-live-writer => std-det-writer}/src/JFH5Writer.cpp (100%) rename {jf-live-writer => std-det-writer}/src/WriterStats.cpp (100%) rename {jf-live-writer => std-det-writer}/src/main.cpp (87%) rename {jf-live-writer => std-det-writer}/test/CMakeLists.txt (100%) rename {jf-live-writer => std-det-writer}/test/main.cpp (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 19194fc..7d4662c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -38,5 +38,5 @@ add_subdirectory("std-udp-sync") add_subdirectory("jf-assembler") #add_subdirectory("sf-stream") #add_subdirectory("sf-writer") -add_subdirectory("jf-live-writer") +add_subdirectory("std-det-writer") diff --git a/jf-live-writer/CMakeLists.txt b/jf-live-writer/CMakeLists.txt deleted file mode 100644 index f338d37..0000000 --- a/jf-live-writer/CMakeLists.txt +++ /dev/null @@ -1,38 +0,0 @@ -find_package(MPI REQUIRED) -# Because of openmpi. -add_definitions(-DOMPI_SKIP_MPICXX) - -file(GLOB SOURCES - src/*.cpp) - -add_library(jf-live-writer-lib STATIC ${SOURCES}) -target_include_directories(jf-live-writer-lib - PUBLIC include/ - SYSTEM ${MPI_INCLUDE_PATH}) - -target_link_libraries(jf-live-writer-lib - external - core-buffer-lib - ${MPI_LIBRARIES} - ) - -add_executable(jf-live-writer src/main.cpp) - - -if (USE_EIGER) - set (LIB_NAME_UDP_RECV "eiger_live_writer") -else() - set (LIB_NAME_UDP_RECV "jf_live_writer") -endif() - -set_target_properties(jf-live-writer PROPERTIES OUTPUT_NAME ${LIB_NAME_UDP_RECV}) - -target_link_libraries(jf-live-writer - jf-live-writer-lib - zmq - hdf5 - rt - ) - -enable_testing() -add_subdirectory(test/) \ No newline at end of file diff --git a/std-det-writer/CMakeLists.txt b/std-det-writer/CMakeLists.txt new file mode 100644 index 0000000..c01a581 --- /dev/null +++ b/std-det-writer/CMakeLists.txt @@ -0,0 +1,31 @@ +find_package(MPI REQUIRED) +# Because of openmpi. +add_definitions(-DOMPI_SKIP_MPICXX) + +file(GLOB SOURCES + src/*.cpp) + +add_library(std-det-writer-lib STATIC ${SOURCES}) +target_include_directories(std-det-writer-lib + PUBLIC include/ + SYSTEM ${MPI_INCLUDE_PATH}) + +target_link_libraries(std-det-writer-lib + external + core-buffer-lib + ${MPI_LIBRARIES} + ) + +add_executable(std-det-writer src/main.cpp) + +set_target_properties(std-det-writer PROPERTIES OUTPUT_NAME std_det_writer) + +target_link_libraries(std-det-writer + std-det-writer-lib + zmq + hdf5 + rt + ) + +enable_testing() +add_subdirectory(test/) \ No newline at end of file diff --git a/jf-live-writer/README.md b/std-det-writer/README.md similarity index 99% rename from jf-live-writer/README.md rename to std-det-writer/README.md index 21dd53e..e7ddf89 100644 --- a/jf-live-writer/README.md +++ b/std-det-writer/README.md @@ -1,4 +1,4 @@ -# jf-live-writer +# std-det-writer This component is a PHDF5 based MPI writer for high performance detectors that need more than 2GB/s of write speed. It parallelizes the HDF5 writing to diff --git a/jf-live-writer/debug.Dockerfile b/std-det-writer/debug.Dockerfile similarity index 71% rename from jf-live-writer/debug.Dockerfile rename to std-det-writer/debug.Dockerfile index 1a8c53f..eb0f094 100644 --- a/jf-live-writer/debug.Dockerfile +++ b/std-det-writer/debug.Dockerfile @@ -4,7 +4,7 @@ COPY . /sf_daq_buffer/ RUN mkdir /sf_daq_buffer/build && \ cd /sf_daq_buffer/build && \ - cmake3 -DBUILD_JF_LIVE_WRITER=ON .. && \ - make jf-live-writer + cmake3 .. && \ + make std-det-writer WORKDIR /sf_daq_buffer/build diff --git a/jf-live-writer/include/JFH5Writer.hpp b/std-det-writer/include/JFH5Writer.hpp similarity index 100% rename from jf-live-writer/include/JFH5Writer.hpp rename to std-det-writer/include/JFH5Writer.hpp diff --git a/std-det-writer/include/StdDetWriterConfig.hpp b/std-det-writer/include/StdDetWriterConfig.hpp new file mode 100644 index 0000000..847439b --- /dev/null +++ b/std-det-writer/include/StdDetWriterConfig.hpp @@ -0,0 +1,33 @@ +#ifndef SF_DAQ_BUFFER_UDPRECVCONFIG_HPP +#define SF_DAQ_BUFFER_UDPRECVCONFIG_HPP + + +#include +#include +#include +#include +#include + +struct UdpRecvConfig { + static UdpRecvConfig from_json_file(const std::string& filename) { + std::ifstream ifs(filename); + rapidjson::IStreamWrapper isw(ifs); + rapidjson::Document config_parameters; + config_parameters.ParseStream(isw); + + return { + config_parameters["detector_name"].GetString(), + config_parameters["detector_type"].GetString(), + config_parameters["n_modules"].GetInt(), + config_parameters["start_udp_port"].GetInt(), + }; + } + + const std::string detector_name; + const std::string detector_type; + const int n_modules; + const int start_udp_port; +}; + + +#endif //SF_DAQ_BUFFER_UDPRECVCONFIG_HPP diff --git a/jf-live-writer/include/WriterStats.hpp b/std-det-writer/include/WriterStats.hpp similarity index 100% rename from jf-live-writer/include/WriterStats.hpp rename to std-det-writer/include/WriterStats.hpp diff --git a/jf-live-writer/include/broker_format.hpp b/std-det-writer/include/broker_format.hpp similarity index 100% rename from jf-live-writer/include/broker_format.hpp rename to std-det-writer/include/broker_format.hpp diff --git a/jf-live-writer/include/live_writer_config.hpp b/std-det-writer/include/live_writer_config.hpp similarity index 100% rename from jf-live-writer/include/live_writer_config.hpp rename to std-det-writer/include/live_writer_config.hpp diff --git a/jf-live-writer/src/JFH5Writer.cpp b/std-det-writer/src/JFH5Writer.cpp similarity index 100% rename from jf-live-writer/src/JFH5Writer.cpp rename to std-det-writer/src/JFH5Writer.cpp diff --git a/jf-live-writer/src/WriterStats.cpp b/std-det-writer/src/WriterStats.cpp similarity index 100% rename from jf-live-writer/src/WriterStats.cpp rename to std-det-writer/src/WriterStats.cpp diff --git a/jf-live-writer/src/main.cpp b/std-det-writer/src/main.cpp similarity index 87% rename from jf-live-writer/src/main.cpp rename to std-det-writer/src/main.cpp index e2940e7..4af58a7 100644 --- a/jf-live-writer/src/main.cpp +++ b/std-det-writer/src/main.cpp @@ -40,7 +40,9 @@ int main (int argc, char *argv[]) auto receiver = BufferUtils::connect_socket( ctx, config.detector_name, "writer-agent"); - RamBuffer ram_buffer(config.detector_name, config.n_modules); + const size_t IMAGE_N_BYTES = 12; + RamBuffer image_buffer(config.detector_name + "_assembler", + sizeof(ImageMetadata), IMAGE_N_BYTES, 1, RAM_BUFFER_N_SLOTS); JFH5Writer writer(config); WriterStats stats(config.detector_name); @@ -60,9 +62,17 @@ int main (int argc, char *argv[]) stats.start_run(meta); } + // i_image == n_images -> end of run. + if (meta.i_image == meta.n_images) { + writer.close_run(); + + stats.end_run(); + continue; + } + // Fair distribution of images among writers. if (meta.i_image % n_writers == i_writer) { - char* data = ram_buffer.get_slot_data(meta.image_metadata.pulse_id); + char* data = ram_buffer.get_slot_data(meta.image_metadata.id); stats.start_image_write(); writer.write_data(meta.run_id, meta.i_image, data); @@ -74,11 +84,5 @@ int main (int argc, char *argv[]) writer.write_meta(meta.run_id, meta.i_image, meta.image_metadata); } - // i_image + 1 == meta.n_images -> we received the last image. - if (meta.i_image+1 == meta.n_images) { - writer.close_run(); - - stats.end_run(); - } } } diff --git a/jf-live-writer/test/CMakeLists.txt b/std-det-writer/test/CMakeLists.txt similarity index 100% rename from jf-live-writer/test/CMakeLists.txt rename to std-det-writer/test/CMakeLists.txt diff --git a/jf-live-writer/test/main.cpp b/std-det-writer/test/main.cpp similarity index 100% rename from jf-live-writer/test/main.cpp rename to std-det-writer/test/main.cpp From b7997a0006c52d173544791497ec18a069eeef9a Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 6 Jul 2021 15:21:10 +0200 Subject: [PATCH 2/3] Remove unused datasets --- std-det-writer/include/JFH5Writer.hpp | 8 ++-- std-det-writer/src/JFH5Writer.cpp | 53 ++++++++------------------- 2 files changed, 19 insertions(+), 42 deletions(-) diff --git a/std-det-writer/include/JFH5Writer.hpp b/std-det-writer/include/JFH5Writer.hpp index f62306e..f413792 100644 --- a/std-det-writer/include/JFH5Writer.hpp +++ b/std-det-writer/include/JFH5Writer.hpp @@ -26,11 +26,9 @@ class JFH5Writer { // Open file specific variables. hid_t file_id_ = -1; - hid_t image_dataset_id_ = -1; - hid_t pulse_dataset_id_= -1; - hid_t frame_dataset_id_ = -1; - hid_t daq_rec_dataset_id_ = -1; - hid_t is_good_dataset_id_ = -1; + hid_t image_data_dataset_ = -1; + hid_t image_id_dataset_ = -1; + hid_t status_dataset_ = -1; static hid_t get_datatype(int bits_per_pixel); void open_file(const std::string& output_file, uint32_t n_images); diff --git a/std-det-writer/src/JFH5Writer.cpp b/std-det-writer/src/JFH5Writer.cpp index 7034bd7..811060f 100644 --- a/std-det-writer/src/JFH5Writer.cpp +++ b/std-det-writer/src/JFH5Writer.cpp @@ -149,10 +149,10 @@ void JFH5Writer::open_file(const string& output_file, const uint32_t n_images) // throw runtime_error("Cannot set compression filter on dataset."); // } - image_dataset_id_ = H5Dcreate( + image_data_dataset_ = H5Dcreate( data_group_id, "data", get_datatype(bits_per_pixel_), image_space_id, H5P_DEFAULT, dcpl_id, H5P_DEFAULT); - if (image_dataset_id_ < 0) { + if (image_data_dataset_ < 0) { throw runtime_error("Cannot create image dataset."); } @@ -174,10 +174,8 @@ void JFH5Writer::open_file(const string& output_file, const uint32_t n_images) return dataset_id; }; - pulse_dataset_id_ = create_meta_dataset("pulse_id", H5T_NATIVE_UINT64); - frame_dataset_id_ = create_meta_dataset("frame_index", H5T_NATIVE_UINT64); - daq_rec_dataset_id_ = create_meta_dataset("daq_rec", H5T_NATIVE_UINT32); - is_good_dataset_id_ = create_meta_dataset("is_good_frame", H5T_NATIVE_UINT8); + image_id_dataset_ = create_meta_dataset("image_id", H5T_NATIVE_UINT64); + status_dataset_ = create_meta_dataset("status", H5T_NATIVE_UINT64); H5Sclose(meta_space_id); H5Sclose(image_space_id); @@ -191,20 +189,11 @@ void JFH5Writer::close_file() return; } - H5Dclose(image_dataset_id_); - image_dataset_id_ = -1; + H5Dclose(image_id_dataset_); + image_id_dataset_ = -1; - H5Dclose(pulse_dataset_id_); - pulse_dataset_id_ = -1; - - H5Dclose(frame_dataset_id_); - frame_dataset_id_ = -1; - - H5Dclose(daq_rec_dataset_id_); - daq_rec_dataset_id_ = -1; - - H5Dclose(is_good_dataset_id_); - is_good_dataset_id_ = -1; + H5Dclose(status_dataset_); + status_dataset_ = -1; H5Fclose(file_id_); file_id_ = -1; @@ -223,7 +212,7 @@ void JFH5Writer::write_data( throw runtime_error("Cannot create image ram dataspace."); } - auto file_ds = H5Dget_space(image_dataset_id_); + auto file_ds = H5Dget_space(image_data_dataset_); if (file_ds < 0) { throw runtime_error("Cannot get image dataset file dataspace."); } @@ -237,8 +226,8 @@ void JFH5Writer::write_data( throw runtime_error("Cannot select image dataset file hyperslab."); } - if (H5Dwrite(image_dataset_id_, get_datatype(bits_per_pixel_), - ram_ds, file_ds, H5P_DEFAULT, data) < 0) { + if (H5Dwrite(image_data_dataset_, get_datatype(bits_per_pixel_), + ram_ds, file_ds, H5P_DEFAULT, data) < 0) { throw runtime_error("Cannot write data to image dataset."); } @@ -259,7 +248,7 @@ void JFH5Writer::write_meta( throw runtime_error("Cannot create metadata ram dataspace."); } - auto file_ds = H5Dget_space(pulse_dataset_id_); + auto file_ds = H5Dget_space(image_id_dataset_); if (file_ds < 0) { throw runtime_error("Cannot get metadata dataset file dataspace."); } @@ -273,23 +262,13 @@ void JFH5Writer::write_meta( throw runtime_error("Cannot select metadata dataset file hyperslab."); } - if (H5Dwrite(pulse_dataset_id_, H5T_NATIVE_UINT64, - ram_ds, file_ds, H5P_DEFAULT, &(meta.pulse_id)) < 0) { + if (H5Dwrite(image_id_dataset_, H5T_NATIVE_UINT64, + ram_ds, file_ds, H5P_DEFAULT, &(meta.id)) < 0) { throw runtime_error("Cannot write data to pulse_id dataset."); } - if (H5Dwrite(frame_dataset_id_, H5T_NATIVE_UINT64, - ram_ds, file_ds, H5P_DEFAULT, &(meta.frame_index)) < 0) { - throw runtime_error("Cannot write data to frame_index dataset."); - } - - if (H5Dwrite(daq_rec_dataset_id_, H5T_NATIVE_UINT32, - ram_ds, file_ds, H5P_DEFAULT, &(meta.daq_rec)) < 0) { - throw runtime_error("Cannot write data to daq_rec dataset."); - } - - if (H5Dwrite(is_good_dataset_id_, H5T_NATIVE_UINT32, - ram_ds, file_ds, H5P_DEFAULT, &(meta.is_good_image)) < 0) { + if (H5Dwrite(status_dataset_, H5T_NATIVE_UINT64, + ram_ds, file_ds, H5P_DEFAULT, &(meta.status)) < 0) { throw runtime_error("Cannot write data to is_good_image dataset."); } From 23a26fdea6791df58982a73212ee9ea70dc6ad37 Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Tue, 6 Jul 2021 15:41:09 +0200 Subject: [PATCH 3/3] Remove config object from h5 writer --- std-det-writer/include/JFH5Writer.hpp | 5 +++-- std-det-writer/src/JFH5Writer.cpp | 7 ++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/std-det-writer/include/JFH5Writer.hpp b/std-det-writer/include/JFH5Writer.hpp index f413792..b979285 100644 --- a/std-det-writer/include/JFH5Writer.hpp +++ b/std-det-writer/include/JFH5Writer.hpp @@ -12,8 +12,8 @@ extern "C" { class JFH5Writer { - const std::string root_folder_; const std::string detector_name_; + const std::string root_folder_; static const int64_t NO_RUN_ID = -1; @@ -35,7 +35,8 @@ class JFH5Writer { void close_file(); public: - explicit JFH5Writer(const BufferUtils::DetectorConfig& config); + explicit JFH5Writer( + const std::string detector_name, const std::string root_folder); ~JFH5Writer(); void open_run(int64_t run_id, diff --git a/std-det-writer/src/JFH5Writer.cpp b/std-det-writer/src/JFH5Writer.cpp index 811060f..3e58a1d 100644 --- a/std-det-writer/src/JFH5Writer.cpp +++ b/std-det-writer/src/JFH5Writer.cpp @@ -15,9 +15,10 @@ using namespace std; using namespace buffer_config; using namespace live_writer_config; -JFH5Writer::JFH5Writer(const BufferUtils::DetectorConfig& config): - root_folder_(config.buffer_folder), - detector_name_(config.detector_name) +JFH5Writer::JFH5Writer( + const std::string detector_name, const std::string root_folder): + detector_name_(detector_name), + root_folder_(root_folder) { }