diff --git a/frame_serialize/CBORMessages.h b/frame_serialize/CBORMessages.h index 4d34c4c2..ac86c954 100644 --- a/frame_serialize/CBORMessages.h +++ b/frame_serialize/CBORMessages.h @@ -48,8 +48,6 @@ struct DataMessage { std::string series_unique_id; uint64_t series_id; - - std::vector adu_histogram; }; struct GoniometerAxis { @@ -153,6 +151,8 @@ struct EndMessage { uint64_t series_id; std::map> rad_int_result; + std::map> adu_histogram; + uint64_t adu_histogram_bin_width; }; #endif //JUNGFRAUJOCH_CBORMESSAGES_H diff --git a/frame_serialize/JFJochFrameDeserializer.cpp b/frame_serialize/JFJochFrameDeserializer.cpp index c884fc75..cfb5b7f2 100644 --- a/frame_serialize/JFJochFrameDeserializer.cpp +++ b/frame_serialize/JFJochFrameDeserializer.cpp @@ -407,8 +407,6 @@ void JFJochFrameDeserializer::ProcessImageMessageUserDataElement(CborValue &valu data_message.storage_cell = GetCBORUInt(map_value) & UINT32_MAX; else if (key == "bunch_id") data_message.bunch_id = GetCBORUInt(map_value); - else if (key == "adu_histogram") - GetCBORUInt64Array(map_value, data_message.adu_histogram); else cbor_value_advance(&map_value); } @@ -484,6 +482,17 @@ void JFJochFrameDeserializer::ProcessRadIntResultElement(CborValue &value) { cborErr(cbor_value_leave_container(&value, &map_value)); } +void JFJochFrameDeserializer::ProcessADUHistogramElement(CborValue &value) { + CborValue map_value; + cborErr(cbor_value_enter_container(&value, &map_value)); + while (! cbor_value_at_end(&map_value)) { + auto key = GetCBORString(map_value); + std::vector val; + GetCBORUInt64Array(map_value, val); + end_message.adu_histogram[key] = val; + } + cborErr(cbor_value_leave_container(&value, &map_value)); +} void JFJochFrameDeserializer::ProcessChannels(CborValue &value) { if (!cbor_value_is_array(&value)) @@ -676,6 +685,10 @@ void JFJochFrameDeserializer::ProcessEndMessageUserDataElement(CborValue &value) end_message.write_master_file = GetCBORBool(map_value); else if (key == "rad_int_result") ProcessRadIntResultElement(map_value); + else if (key == "adu_histogram") + ProcessADUHistogramElement(map_value); + else if (key == "adu_histogram_bin_width") + end_message.adu_histogram_bin_width = GetCBORUInt(map_value); else cbor_value_advance(&map_value); } diff --git a/frame_serialize/JFJochFrameDeserializer.h b/frame_serialize/JFJochFrameDeserializer.h index df79cf29..d1288a52 100644 --- a/frame_serialize/JFJochFrameDeserializer.h +++ b/frame_serialize/JFJochFrameDeserializer.h @@ -38,6 +38,7 @@ private: void ProcessImageData(CborValue &value); void ProcessPixelMaskElement(CborValue &value); void ProcessRadIntResultElement(CborValue &value); + void ProcessADUHistogramElement(CborValue &value); void ProcessUnitCellElement(CborValue &value); void ProcessStartMessageUserDataElement(CborValue &value); void ProcessImageMessageUserDataElement(CborValue &value); diff --git a/frame_serialize/JFJochFrameSerializer.cpp b/frame_serialize/JFJochFrameSerializer.cpp index cf51d881..f50871d6 100644 --- a/frame_serialize/JFJochFrameSerializer.cpp +++ b/frame_serialize/JFJochFrameSerializer.cpp @@ -157,6 +157,16 @@ inline void CBOR_ENC_RAD_INT_RESULT(CborEncoder &encoder, const char* key, cborErr(cbor_encoder_close_container(&encoder, &mapEncoder)); } +inline void CBOR_ENC_ADU_HIST(CborEncoder &encoder, const char* key, + const std::map> &adu_histogram) { + CborEncoder mapEncoder; + cborErr(cbor_encode_text_stringz(&encoder, key)); + cborErr(cbor_encoder_create_map(&encoder, &mapEncoder, adu_histogram.size())); + for (auto &[map_key, map_val]: adu_histogram) + CBOR_ENC(mapEncoder, map_key.c_str(), map_val); + cborErr(cbor_encoder_close_container(&encoder, &mapEncoder)); +} + inline void CBOR_ENC(CborEncoder &encoder, const char* key, const std::vector& spots) { CborEncoder arrayEncoder; @@ -389,13 +399,15 @@ void JFJochFrameSerializer::SerializeSequenceEnd(const EndMessage& message) { CBOR_ENC(mapEncoder, "end_date", message.end_date); cborErr(cbor_encode_text_stringz(&mapEncoder, "user_data")); - cborErr(cbor_encoder_create_map(&mapEncoder, &userDataMapEncoder, 6)); + cborErr(cbor_encoder_create_map(&mapEncoder, &userDataMapEncoder, 8)); CBOR_ENC(userDataMapEncoder, "magic_number", user_data_magic_number); CBOR_ENC(userDataMapEncoder, "number_of_images", message.number_of_images); CBOR_ENC(userDataMapEncoder, "max_receiver_delay", message.max_receiver_delay); CBOR_ENC(userDataMapEncoder, "receiver_efficiency", message.efficiency); CBOR_ENC(userDataMapEncoder, "write_master_file", message.write_master_file); CBOR_ENC_RAD_INT_RESULT(userDataMapEncoder, "rad_int_result", message.rad_int_result); + CBOR_ENC_ADU_HIST(userDataMapEncoder, "adu_histogram", message.adu_histogram); + CBOR_ENC(userDataMapEncoder, "adu_histogram_bin_width", message.adu_histogram_bin_width); cborErr(cbor_encoder_close_container(&mapEncoder, &userDataMapEncoder)); cborErr(cbor_encoder_close_container(&encoder, &mapEncoder)); @@ -422,7 +434,7 @@ void JFJochFrameSerializer::SerializeImage(const DataMessage& message) { message.timestamp_base); cborErr(cbor_encode_text_stringz(&mapEncoder, "user_data")); - cborErr(cbor_encoder_create_map(&mapEncoder, &userDataMapEncoder, 11)); + cborErr(cbor_encoder_create_map(&mapEncoder, &userDataMapEncoder, 10)); CBOR_ENC(userDataMapEncoder, "magic_number", user_data_magic_number); CBOR_ENC(userDataMapEncoder, "spots", message.spots); @@ -434,7 +446,6 @@ void JFJochFrameSerializer::SerializeImage(const DataMessage& message) { CBOR_ENC(userDataMapEncoder, "receiver_available_send_buffers", message.receiver_available_send_buffers); CBOR_ENC(userDataMapEncoder, "receiver_aq_dev_delay", message.receiver_aq_dev_delay); CBOR_ENC(userDataMapEncoder, "storage_cell", (uint64_t) message.storage_cell); - CBOR_ENC(userDataMapEncoder, "adu_histogram", message.adu_histogram); cborErr(cbor_encoder_close_container(&mapEncoder, &userDataMapEncoder)); diff --git a/receiver/JFJochReceiver.cpp b/receiver/JFJochReceiver.cpp index 6838e6cd..fc79b35e 100644 --- a/receiver/JFJochReceiver.cpp +++ b/receiver/JFJochReceiver.cpp @@ -41,7 +41,8 @@ JFJochReceiver::JFJochReceiver(const JFJochProtoBuf::ReceiverInput &settings, frame_transformation_ready((experiment.GetImageNum() > 0) ? frame_transformation_nthreads : 0), send_buffer_count(in_send_buffer_count), indexing_solution_per_file(experiment.GetDataFileCount()), - numa_policy(in_numa_policy) + numa_policy(in_numa_policy), + adu_histogram_module(experiment.GetModulesNum()) { send_buffer = (uint8_t *) malloc(send_buffer_size * send_buffer_count); @@ -345,7 +346,9 @@ void JFJochReceiver::MeasurePedestalThread(uint16_t data_stream, uint16_t module void JFJochReceiver::MiniSummationThread(int d, int m, size_t image_number, bool &send_image, FrameTransformation &transformation, DataMessage &message, std::vector &conversion, - ADUHistogram& adu_histogram, RadialIntegrationProfile *profile) { + RadialIntegrationProfile *profile) { + size_t module_abs_number = experiment.GetFirstModuleOfDataStream(d) + m; + for (int j = 0; j < experiment.GetSummation(); j++) { size_t frame_number = image_number * experiment.GetSummation() + j; acquisition_device[d]->Counters().WaitForFrame(frame_number + 2); @@ -371,8 +374,8 @@ void JFJochReceiver::MiniSummationThread(int d, int m, size_t image_number, bool if (profile) profile->Add(*acquisition_device[d]->GetDeviceOutput(frame_number, m)); - adu_histogram.Add(*acquisition_device[d]->GetDeviceOutput(frame_number, m)); adu_histogram_total.Add(*acquisition_device[d]->GetDeviceOutput(frame_number, m)); + adu_histogram_module[module_abs_number].Add(*acquisition_device[d]->GetDeviceOutput(frame_number, m)); if (experiment.GetConversionOnCPU()) { auto &conv = conversion.at(experiment.GetFirstModuleOfDataStream(d) + m); @@ -438,7 +441,6 @@ void JFJochReceiver::FrameTransformationThread() { message.timestamp_base = 10*1000*1000; message.exptime_base = 10*1000*1000; message.indexing_result = 0; - ADUHistogram adu_histogram; bool send_preview = false; bool send_bkg_estimate = false; @@ -468,7 +470,7 @@ void JFJochReceiver::FrameTransformationThread() { this, d, m, image_number, std::ref(send_image), std::ref(transformation), std::ref(message), std::ref(*conversion), - std::ref(adu_histogram), rad_int_profile_image.get())); + rad_int_profile_image.get())); message.receiver_aq_dev_delay = max_delay; } else { for (int j = 0; j < experiment.GetSummation(); j++) { @@ -502,8 +504,10 @@ void JFJochReceiver::FrameTransformationThread() { } else transformation.ProcessModule(src, m, d); - adu_histogram.Add(*acquisition_device[d]->GetDeviceOutput(frame_number, m)); + size_t module_abs_number = experiment.GetFirstModuleOfDataStream(d) + m; + adu_histogram_module[module_abs_number].Add(*acquisition_device[d]->GetDeviceOutput(frame_number, m)); adu_histogram_total.Add(*acquisition_device[d]->GetDeviceOutput(frame_number, m)); + if (rad_int_profile_image) rad_int_profile_image->Add(*acquisition_device[d]->GetDeviceOutput(frame_number, m)); @@ -579,8 +583,6 @@ void JFJochReceiver::FrameTransformationThread() { *rad_int_profile_per_file[image_number % experiment.GetDataFileCount()] += *rad_int_profile_image; } - message.adu_histogram = adu_histogram.GetHistogram(); - if (send_preview) preview_publisher->Publish(experiment, transformation.GetPreview16BitImage(), @@ -721,6 +723,11 @@ void JFJochReceiver::FinalizeMeasurement() { message.rad_int_result["file" + std::to_string(i)] = rad_int_profile_per_file[i]->GetResult(); + message.adu_histogram["total"] = adu_histogram_total.GetHistogram(); + for (int i = 0; i < adu_histogram_module.size(); i++) + message.adu_histogram["module" + std::to_string(i)] + = adu_histogram_module[i].GetHistogram(); + image_pusher.EndDataCollection(message); logger.Info("Disconnected from writers"); } diff --git a/receiver/JFJochReceiver.h b/receiver/JFJochReceiver.h index 71d0f4ce..06d0d43d 100644 --- a/receiver/JFJochReceiver.h +++ b/receiver/JFJochReceiver.h @@ -44,6 +44,7 @@ class JFJochReceiver { std::vector> data_acquisition_futures; ADUHistogram adu_histogram_total; + std::vector adu_histogram_module; std::unique_ptr rad_int_mapping; std::unique_ptr rad_int_profile; @@ -114,7 +115,7 @@ class JFJochReceiver { void MiniSummationThread(int d, int m, size_t image_number, bool &send_image, FrameTransformation &transformation, DataMessage &message, std::vector &conversion, - ADUHistogram& adu_histogram, RadialIntegrationProfile *profile); + RadialIntegrationProfile *profile); void Cancel(const JFJochException &e); void FinalizeMeasurement(); JFJochProtoBuf::DataProcessingSettings GetDataProcessingSettings(); diff --git a/tests/CBORTest.cpp b/tests/CBORTest.cpp index 0720dfab..96a1b121 100644 --- a/tests/CBORTest.cpp +++ b/tests/CBORTest.cpp @@ -287,6 +287,41 @@ JFJochFrameSerializer serializer(buffer.data(), buffer.size()); CHECK(message.rad_int_result["file0"] == output_message.rad_int_result["file0"]); } +TEST_CASE("CBORSerialize_End_ADUHistogram", "[CBOR]") { + std::vector buffer(8*1024*1024); + JFJochFrameSerializer serializer(buffer.data(), buffer.size()); + + EndMessage message { + .number_of_images = 57789, + .max_receiver_delay = 3456, + .efficiency = 0.99, + .write_master_file = true, + .end_date = "ccc", + .series_unique_id = "bla5", + .series_id = 45676782, + .adu_histogram_bin_width = 55 + }; + + message.adu_histogram["avg"] = {11, 12, 13}; + message.adu_histogram["file0"] = {56, 75, 34}; + + REQUIRE_NOTHROW(serializer.SerializeSequenceEnd(message)); + + JFJochFrameDeserializer deserializer; + REQUIRE_NOTHROW(deserializer.Process(buffer.data(), serializer.GetBufferSize())); + REQUIRE(deserializer.GetType() == JFJochFrameDeserializer::Type::END); + + EndMessage output_message{}; + REQUIRE_NOTHROW(output_message = deserializer.GetEndMessage()); + REQUIRE(output_message.adu_histogram.size() == 2); + REQUIRE(output_message.adu_histogram.contains("avg")); + REQUIRE(output_message.adu_histogram.contains("file0")); + CHECK(message.adu_histogram["avg"] == output_message.adu_histogram["avg"]); + CHECK(message.adu_histogram["file0"] == output_message.adu_histogram["file0"]); + CHECK(message.adu_histogram_bin_width == output_message.adu_histogram_bin_width); +} + + TEST_CASE("CBORSerialize_Image", "[CBOR]") { std::vector buffer(8*1024*1024); JFJochFrameSerializer serializer(buffer.data(), buffer.size()); diff --git a/writer/HDF5DataFile.cpp b/writer/HDF5DataFile.cpp index d0a285fb..8dd48ccc 100644 --- a/writer/HDF5DataFile.cpp +++ b/writer/HDF5DataFile.cpp @@ -155,9 +155,6 @@ void HDF5DataFile::Write(const DataMessage &msg, uint64_t image_number) { if (!msg.rad_int_profile.empty() && (msg.rad_int_profile.size() == rad_int_bin_to_q.size())) rad_int_group->SaveVector("img" + std::to_string(image_number), msg.rad_int_profile); - - if (!msg.adu_histogram.empty()) - adu_histo_group->SaveVector("img" + std::to_string(image_number), msg.adu_histogram); } HDF5DataFileStatistics HDF5DataFile::GetStatistics() const { diff --git a/writer/HDF5NXmx.cpp b/writer/HDF5NXmx.cpp index 2e326259..f92f348f 100644 --- a/writer/HDF5NXmx.cpp +++ b/writer/HDF5NXmx.cpp @@ -18,6 +18,7 @@ void HDF5Metadata::NXmx( const StartMessage &start, const EndMessage &end) { hdf5_file.Attr("HDF5_Version", hdf5_version()); HDF5Group(hdf5_file, "/entry").NXClass("NXentry").SaveScalar("definition", "NXmx"); + HDF5Group(hdf5_file, "/entry/result").NXClass("NXcollection"); LinkToData(&hdf5_file, start, end); Facility(&hdf5_file, start, end); @@ -28,6 +29,7 @@ void HDF5Metadata::NXmx( const StartMessage &start, const EndMessage &end) { Sample(&hdf5_file, start, end); Calibration(&hdf5_file, start, end); RadInt(&hdf5_file, start, end); + ADUHistogram(&hdf5_file, end); } void HDF5Metadata::Time(HDF5File *hdf5_file, const StartMessage &start, const EndMessage &end) { @@ -278,7 +280,6 @@ std::string HDF5Metadata::DataFileName(const std::string &prefix, int64_t file_n void HDF5Metadata::RadInt(HDF5File *hdf5_file, const StartMessage &start, const EndMessage &end) { if (!start.rad_int_bin_to_q.empty()) { - HDF5Group(*hdf5_file, "/entry/result").NXClass("NXcollection"); HDF5Group rad_int_group(*hdf5_file, "/entry/result/rad_int"); rad_int_group.SaveVector("bin_to_q", start.rad_int_bin_to_q); @@ -289,3 +290,11 @@ void HDF5Metadata::RadInt(HDF5File *hdf5_file, const StartMessage &start, const rad_int_group.SaveVector(x, y); } } + +void HDF5Metadata::ADUHistogram(HDF5File *hdf5_file, const EndMessage &end) { + HDF5Group adu_histo_group(*hdf5_file, "/entry/result/adu_histogram"); + + adu_histo_group.SaveScalar("bin_width", end.adu_histogram_bin_width); + for (const auto &[x,y] : end.adu_histogram) + adu_histo_group.SaveVector(x, y); +} \ No newline at end of file diff --git a/writer/HDF5NXmx.h b/writer/HDF5NXmx.h index 580897bd..d9756c09 100644 --- a/writer/HDF5NXmx.h +++ b/writer/HDF5NXmx.h @@ -27,6 +27,7 @@ namespace HDF5Metadata { void SaveCBORImage(HDF5File *hdf5_file, const std::string& hdf5_path, const CBORImage &image); void RadInt(HDF5File *hdf5_file, const StartMessage &start, const EndMessage &end); + void ADUHistogram(HDF5File *hdf5_file, const EndMessage &end); std::string DataFileName(const std::string& prefix, int64_t file_number); }