From 50ef79e621d0331496d43273c6c05438711a40b0 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Wed, 27 Nov 2024 11:46:58 +0100 Subject: [PATCH] version 1.0.0-rc.26 quick fixes --- common/StatusVector.h | 4 ++++ docs/CHANGELOG.md | 3 +++ frame_serialize/JFJochMessages.h | 2 +- image_pusher/TestImagePusher.cpp | 6 ++++-- receiver/JFJochReceiverPlots.cpp | 3 ++- tests/StatusVectorTest.cpp | 21 +++++++++++++++++++++ writer/CMakeLists.txt | 2 ++ writer/HDF5DataFile.cpp | 2 ++ writer/HDF5DataFilePluginAzInt.cpp | 4 ++-- writer/HDF5DataFilePluginBkg.cpp | 16 ++++++++++++++++ writer/HDF5DataFilePluginBkg.h | 21 +++++++++++++++++++++ writer/HDF5NXmx.cpp | 12 +++++++++--- 12 files changed, 87 insertions(+), 9 deletions(-) create mode 100644 writer/HDF5DataFilePluginBkg.cpp create mode 100644 writer/HDF5DataFilePluginBkg.h diff --git a/common/StatusVector.h b/common/StatusVector.h index 43301550..c4dfbbf2 100644 --- a/common/StatusVector.h +++ b/common/StatusVector.h @@ -28,6 +28,10 @@ public: std::unique_lock ul(m); content.clear(); present.clear(); + + mean = NAN; + count = 0; + sum = 0; } void AddElement(uint32_t id, std::optional val) { diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 0440e0be..3623d703 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -6,10 +6,13 @@ * jfjoch_broker: add crystal lattice plots * jfjoch_broker: remove empty bins from the plots * jfjoch_broker: Fix bugs in ModuleSummation and MXAnalyzer for CPU "long" summation +* jfjoch_broker: Fix bug when mean background estimation / indexing rate where affected by previous experiment * jfjoch_writer: fix missing "-w" parameter * jfjoch_writer: temporary files have ".tmp" suffix * jfjoch_writer: refactor logic for watermarks * jfjoch_writer: report on internal FIFO utilization +* jfjoch_writer: clean-up naming for azimuthal integration and background estimate +* jfjoch_writer: write final background estimate and indexing rate in the master file * tools/: remove unnecessary tools, make naming consistent * CBOR: Add indexing rate and background estimate to end message * CBOR: Clean-up documentation diff --git a/frame_serialize/JFJochMessages.h b/frame_serialize/JFJochMessages.h index 516aea5e..0b1b11b5 100644 --- a/frame_serialize/JFJochMessages.h +++ b/frame_serialize/JFJochMessages.h @@ -51,7 +51,7 @@ struct DataMessage { std::optional spot_count_in_rings = 0; std::vector az_int_profile; - float bkg_estimate; + std::optional bkg_estimate; bool indexing_result; std::vector indexing_lattice; diff --git a/image_pusher/TestImagePusher.cpp b/image_pusher/TestImagePusher.cpp index af767650..d187459d 100644 --- a/image_pusher/TestImagePusher.cpp +++ b/image_pusher/TestImagePusher.cpp @@ -165,7 +165,9 @@ bool TestImagePusher::CheckImage(const DiffractionExperiment &x, const std::vect logger.Info("Mean conversion error: {:.3f}", result); } } - if (no_errors && data_message) - logger.Info("Background estimate {} Spot count {}", data_message->bkg_estimate, data_message->spots.size()); + if (no_errors && data_message && data_message->bkg_estimate.has_value()) + logger.Info("Background estimate {} Spot count {}", + data_message->bkg_estimate.value(), + data_message->spots.size()); return no_errors; } diff --git a/receiver/JFJochReceiverPlots.cpp b/receiver/JFJochReceiverPlots.cpp index 66e150be..f10244a4 100644 --- a/receiver/JFJochReceiverPlots.cpp +++ b/receiver/JFJochReceiverPlots.cpp @@ -31,7 +31,8 @@ void JFJochReceiverPlots::Setup(const DiffractionExperiment &experiment, const A } void JFJochReceiverPlots::Add(const DataMessage &msg, const AzimuthalIntegrationProfile &profile) { - bkg_estimate.AddElement(msg.number, msg.bkg_estimate); + if ( msg.bkg_estimate.has_value()) + bkg_estimate.AddElement(msg.number, msg.bkg_estimate.value()); spot_count.AddElement("Spots (crystal)", msg.number, msg.spots.size()); spot_count.AddElement("Spots (rings)", msg.number, msg.spot_count_in_rings); diff --git a/tests/StatusVectorTest.cpp b/tests/StatusVectorTest.cpp index adae0741..945ffd9c 100644 --- a/tests/StatusVectorTest.cpp +++ b/tests/StatusVectorTest.cpp @@ -173,3 +173,24 @@ TEST_CASE("StatusMultiVector","[StatusMultiVector]") { REQUIRE(ret[1].x[1] == 1.0f); REQUIRE(ret[1].y[1] == 4.0f); } + +TEST_CASE("StatusVector_Clear","[StatusVector]") { + StatusVector status_vector; + status_vector.AddElement(5, 800); + status_vector.AddElement(8, 1000); + REQUIRE(status_vector.Mean() == 900); + + status_vector.Clear(); + REQUIRE(!std::isfinite(status_vector.Mean())); + + status_vector.AddElement(50, 200); + status_vector.AddElement(80, 100); + REQUIRE(status_vector.Mean() == 150); + + auto plot = status_vector.GetMeanPlot(1); + + REQUIRE(plot.size() == 1); + REQUIRE(plot[0].x.size() == 2); + REQUIRE(plot[0].x[0] == 50); + REQUIRE(plot[0].x[1] == 80); +} \ No newline at end of file diff --git a/writer/CMakeLists.txt b/writer/CMakeLists.txt index caa2381b..a64246f3 100644 --- a/writer/CMakeLists.txt +++ b/writer/CMakeLists.txt @@ -21,6 +21,8 @@ ADD_LIBRARY(JFJochWriter STATIC HDF5DataFilePluginAzInt.h HDF5DataFilePluginJUNGFRAU.cpp HDF5DataFilePluginJUNGFRAU.h + HDF5DataFilePluginBkg.cpp + HDF5DataFilePluginBkg.h ) TARGET_LINK_LIBRARIES(JFJochWriter JFJochZMQ JFJochLogger JFJochHDF5Wrappers CBORStream2FrameSerialize) diff --git a/writer/HDF5DataFile.cpp b/writer/HDF5DataFile.cpp index ea05e8fe..0374817e 100644 --- a/writer/HDF5DataFile.cpp +++ b/writer/HDF5DataFile.cpp @@ -8,6 +8,7 @@ #include "../compression/JFJochCompressor.h" #include "HDF5DataFilePluginAzInt.h" +#include "HDF5DataFilePluginBkg.h" #include "HDF5DataFilePluginMX.h" #include "HDF5DataFilePluginXFEL.h" #include "HDF5DataFilePluginJUNGFRAU.h" @@ -26,6 +27,7 @@ HDF5DataFile::HDF5DataFile(const StartMessage &msg, uint64_t in_file_number) { image_low = file_number * msg.images_per_file; tmp_filename = fmt::format("{}.{:x}.tmp", filename, std::chrono::system_clock::now().time_since_epoch().count()); + plugins.emplace_back(std::make_unique()); plugins.emplace_back(std::make_unique()); plugins.emplace_back(std::make_unique()); plugins.emplace_back(std::make_unique(msg.az_int_bin_to_q)); diff --git a/writer/HDF5DataFilePluginAzInt.cpp b/writer/HDF5DataFilePluginAzInt.cpp index ac374d84..091a28fb 100644 --- a/writer/HDF5DataFilePluginAzInt.cpp +++ b/writer/HDF5DataFilePluginAzInt.cpp @@ -13,7 +13,7 @@ void HDF5DataFilePluginAzInt::OpenFile(HDF5File &data_file, const DataMessage &m return; HDF5Group(data_file, "/entry/azint").NXClass("NXcollection"); - data_file.SaveVector("/entry/azint/binToQ", az_int_bin_to_q); + data_file.SaveVector("/entry/azint/bin_to_q", az_int_bin_to_q); az_int.reserve(RESERVE_IMAGES * az_int_bin_to_q.size()); } @@ -35,5 +35,5 @@ void HDF5DataFilePluginAzInt::Write(const DataMessage &msg, uint64_t image_numbe void HDF5DataFilePluginAzInt::WriteFinal(HDF5File &data_file) { if (!az_int.empty()) - data_file.SaveVector("/entry/azint/result", az_int, {(hsize_t) (max_image_number + 1), az_int_bin_to_q.size()}); + data_file.SaveVector("/entry/azint/image", az_int, {(hsize_t) (max_image_number + 1), az_int_bin_to_q.size()}); } diff --git a/writer/HDF5DataFilePluginBkg.cpp b/writer/HDF5DataFilePluginBkg.cpp new file mode 100644 index 00000000..659d7893 --- /dev/null +++ b/writer/HDF5DataFilePluginBkg.cpp @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: 2024 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include "HDF5DataFilePluginBkg.h" + +void HDF5DataFilePluginBkg::OpenFile(HDF5File &data_file, const DataMessage &msg) {} + +void HDF5DataFilePluginBkg::Write(const DataMessage &msg, uint64_t image_number) { + if (msg.bkg_estimate) + bkg_estimate[image_number] = msg.bkg_estimate.value(); +} + +void HDF5DataFilePluginBkg::WriteFinal(HDF5File &data_file) { + if (!bkg_estimate.empty()) + data_file.SaveVector("/entry/bkg_estimate", bkg_estimate.vec()); +} diff --git a/writer/HDF5DataFilePluginBkg.h b/writer/HDF5DataFilePluginBkg.h new file mode 100644 index 00000000..a6a3d97f --- /dev/null +++ b/writer/HDF5DataFilePluginBkg.h @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: 2024 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#ifndef HDF5DATAFILEPLUGINBKG_H +#define HDF5DATAFILEPLUGINBKG_H + + +#include "HDF5DataFilePlugin.h" +#include "../common/AutoIncrVector.h" + +class HDF5DataFilePluginBkg : public HDF5DataFilePlugin { + AutoIncrVector bkg_estimate; +public: + void OpenFile(HDF5File &data_file, const DataMessage& msg) override; + void Write(const DataMessage& msg, uint64_t image_number) override; + void WriteFinal(HDF5File &data_file) override; +}; + + + +#endif //HDF5DATAFILEPLUGINBKG_H diff --git a/writer/HDF5NXmx.cpp b/writer/HDF5NXmx.cpp index 81b9057d..9407e1f2 100644 --- a/writer/HDF5NXmx.cpp +++ b/writer/HDF5NXmx.cpp @@ -301,11 +301,12 @@ void NXmx::SaveCBORImage(const std::string &hdf5_path, const CompressedImage &im void NXmx::AzimuthalIntegration(const StartMessage &start, const EndMessage &end) { if (!start.az_int_bin_to_q.empty()) { - HDF5Group rad_int_group(*hdf5_file, "/entry/result/azimIntegration"); - rad_int_group.SaveVector("bin_to_q", start.az_int_bin_to_q); + HDF5Group az_int_group(*hdf5_file, "/entry/azint"); + az_int_group.NXClass("NXcollection"); + az_int_group.SaveVector("bin_to_q", start.az_int_bin_to_q); for (const auto &[x,y] : end.az_int_result) - rad_int_group.SaveVector(x, y); + az_int_group.SaveVector(x, y); } } @@ -335,6 +336,11 @@ void NXmx::Finalize(const EndMessage &end) { Sample(start_message, end); AzimuthalIntegration(start_message, end); ADUHistogram(end); + + if (end.indexing_rate) + SaveScalar(*hdf5_file, "/entry/result/indexing_rate", end.indexing_rate.value()); + if (end.bkg_estimate) + SaveScalar(*hdf5_file, "/entry/result/bkg_estimate", end.bkg_estimate.value()); } void NXmx::UserData(const StartMessage &start) {