diff --git a/frame_serialize/ImageMessage.h b/frame_serialize/ImageMessage.h index 41233356..f6b579dd 100644 --- a/frame_serialize/ImageMessage.h +++ b/frame_serialize/ImageMessage.h @@ -30,6 +30,7 @@ struct DataMessage { uint64_t bunch_id; uint32_t jf_info; + uint64_t timestamp; }; #endif //JUNGFRAUJOCH_IMAGEMESSAGE_H diff --git a/frame_serialize/JFJochFrameDeserializer.cpp b/frame_serialize/JFJochFrameDeserializer.cpp index 8e4b3694..5e05ee27 100644 --- a/frame_serialize/JFJochFrameDeserializer.cpp +++ b/frame_serialize/JFJochFrameDeserializer.cpp @@ -342,6 +342,8 @@ bool JFJochFrameDeserializer::ProcessImageMapElement(CborValue &value) { data_message.jf_info = GetCBORUInt(value) & UINT32_MAX; else if (key == "bunch_id") data_message.bunch_id = GetCBORUInt(value); + else if (key == "timestamp") + data_message.timestamp = GetCBORUInt(value); else cbor_value_advance(&value); return true; diff --git a/frame_serialize/JFJochFrameSerializer.cpp b/frame_serialize/JFJochFrameSerializer.cpp index 67d3bb16..3460848b 100644 --- a/frame_serialize/JFJochFrameSerializer.cpp +++ b/frame_serialize/JFJochFrameSerializer.cpp @@ -363,7 +363,7 @@ void JFJochFrameSerializer::SerializeImage(const DataMessage& message) { CborEncoder encoder, mapEncoder; cbor_encoder_init(&encoder, buffer.data(), buffer.size(), 0); - cborErr(cbor_encoder_create_map(&encoder, &mapEncoder, 8)); + cborErr(cbor_encoder_create_map(&encoder, &mapEncoder, 9)); CBOR_ENC(mapEncoder, "type", "image"); CBOR_ENC(mapEncoder, "image_id", message.number); @@ -372,7 +372,8 @@ void JFJochFrameSerializer::SerializeImage(const DataMessage& message) { CBOR_ENC(mapEncoder, "rad_int_profile", message.rad_int_profile); CBOR_ENC(mapEncoder, "indexing_result", message.indexing_result); CBOR_ENC(mapEncoder, "bunch_id", message.bunch_id); - CBOR_ENC(mapEncoder, "jf_info", (int64_t) message.jf_info); + CBOR_ENC(mapEncoder, "jf_info", (uint64_t) message.jf_info); + CBOR_ENC(mapEncoder, "timestamp", message.timestamp); cborErr(cbor_encoder_close_container(&encoder, &mapEncoder)); buffer.resize(cbor_encoder_get_buffer_size(&encoder, buffer.data())); diff --git a/receiver/JFJochReceiver.cpp b/receiver/JFJochReceiver.cpp index 4391e673..8ed847b6 100644 --- a/receiver/JFJochReceiver.cpp +++ b/receiver/JFJochReceiver.cpp @@ -96,7 +96,7 @@ JFJochReceiver::JFJochReceiver(const JFJochProtoBuf::ReceiverInput &settings, logger.Info("Data acquisition devices ready"); - if ((experiment.GetDetectorMode() == DetectorMode::PedestalG0) + if ((experiment.GetDetectorMode() == DetectorMode::PedestalG0) || (experiment.GetDetectorMode() == DetectorMode::PedestalG1) || (experiment.GetDetectorMode() == DetectorMode::PedestalG2)) { @@ -327,6 +327,13 @@ int64_t JFJochReceiver::FrameTransformationThread() { if (acquisition_device[d]->IsFullModuleCollected(frame_number, m)) { src = acquisition_device[d]->GetFrameBuffer(frame_number, m); + + if (!send_image) { + // the information is for first module/frame that was collected in full + message.bunch_id = acquisition_device[d]->GetBunchID(frame_number, m); + message.jf_info = acquisition_device[d]->GetJFInfo(frame_number, m); + message.timestamp = acquisition_device[d]->GetTimestamp(frame_number, m); + } send_image = true; } else src = acquisition_device[d]->GetErrorFrameBuffer(); @@ -440,9 +447,9 @@ void JFJochReceiver::GetStatistics(JFJochProtoBuf::ReceiverOutput &ret) const { if (experiment.GetImageNum() > 0) { ret.set_compressed_ratio( static_cast (images_sent - * experiment.GetPixelDepth() - * experiment.GetModulesNum() - * RAW_MODULE_SIZE) + * experiment.GetPixelDepth() + * experiment.GetModulesNum() + * RAW_MODULE_SIZE) / static_cast (compressed_size)); } diff --git a/receiver/host/AcquisitionDevice.cpp b/receiver/host/AcquisitionDevice.cpp index c653eeb6..d460ad4b 100644 --- a/receiver/host/AcquisitionDevice.cpp +++ b/receiver/host/AcquisitionDevice.cpp @@ -451,3 +451,15 @@ ActionConfig AcquisitionDevice::ReadActionRegister() { int32_t AcquisitionDevice::GetNUMANode() const { return -1; } + +uint64_t AcquisitionDevice::GetBunchID(size_t curr_frame, uint16_t module_number) const { + return counters.GetBunchID(curr_frame, module_number); +} + +uint32_t AcquisitionDevice::GetJFInfo(size_t curr_frame, uint16_t module_number) const { + return counters.GetJFInfo(curr_frame, module_number); +} + +uint32_t AcquisitionDevice::GetTimestamp(size_t curr_frame, uint16_t module_number) const { + return counters.GetTimestamp(curr_frame, module_number); +} \ No newline at end of file diff --git a/receiver/host/AcquisitionDevice.h b/receiver/host/AcquisitionDevice.h index 503b08a6..91e55788 100644 --- a/receiver/host/AcquisitionDevice.h +++ b/receiver/host/AcquisitionDevice.h @@ -118,6 +118,9 @@ public: void WaitForFrame(size_t curr_frame, uint16_t module_number = UINT16_MAX) const; int64_t CalculateDelay(size_t curr_frame, uint16_t module_number = UINT16_MAX) const; // mutex acquired indirectly uint64_t GetBufferHandle(size_t frame, uint8_t module_number) const; + uint64_t GetBunchID(size_t curr_frame, uint16_t module_number) const; + uint32_t GetJFInfo(size_t curr_frame, uint16_t module_number) const; + uint32_t GetTimestamp(size_t curr_frame, uint16_t module_number) const; bool IsFullModuleCollected(size_t frame, uint8_t module_number) const; ActionConfig ReadActionRegister(); diff --git a/receiver/host/AcquisitionOnlineCounters.cpp b/receiver/host/AcquisitionOnlineCounters.cpp index 87eab205..5378ec10 100644 --- a/receiver/host/AcquisitionOnlineCounters.cpp +++ b/receiver/host/AcquisitionOnlineCounters.cpp @@ -30,6 +30,9 @@ void AcquisitionOnlineCounters::Reset(const DiffractionExperiment &experiment, u handle_for_frame = std::vector((expected_frames+1) * nmodules, HandleNotFound); full_module_collected = std::vector(expected_frames * nmodules); + bunch_id = std::vector(expected_frames * nmodules); + jf_info = std::vector(expected_frames * nmodules); + timestamp = std::vector(expected_frames * nmodules); } void AcquisitionOnlineCounters::UpdateCounters(const Completion *c) { @@ -53,6 +56,9 @@ void AcquisitionOnlineCounters::UpdateCounters(const Completion *c) { full_module_collected.at(c->frame_number * nmodules + c->module) = (c->packet_count == expected_packets_per_module); handle_for_frame.at(c->frame_number * nmodules + c->module) = c->handle; + bunch_id.at(c->frame_number * nmodules + c->module) = c->bunchid; + jf_info.at(c->frame_number * nmodules + c->module) = c->debug; + timestamp.at(c->frame_number * nmodules + c->module) = c->timestamp; } data_updated.notify_all(); @@ -64,52 +70,52 @@ void AcquisitionOnlineCounters::SetAcquisitionFinished() { data_updated.notify_all(); } -uint64_t AcquisitionOnlineCounters::GetBufferHandle(size_t frame, uint16_t module) const { +uint64_t AcquisitionOnlineCounters::GetBufferHandle(size_t frame, uint16_t module_number) const { if (frame >= expected_frames) throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, "Wrong frame number: " + std::to_string(frame)); - if (module >= nmodules) + if (module_number >= nmodules) throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, "Wrong module number"); - return handle_for_frame.at(frame * nmodules + module); + return handle_for_frame.at(frame * nmodules + module_number); } -uint64_t AcquisitionOnlineCounters::GetBufferHandleAndClear(size_t frame, uint16_t module) { +uint64_t AcquisitionOnlineCounters::GetBufferHandleAndClear(size_t frame, uint16_t module_number) { std::unique_lock ul(m); if (frame >= expected_frames) throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, "Wrong frame number: " + std::to_string(frame)); - if (module >= nmodules) + if (module_number >= nmodules) throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, "Wrong module number"); - uint64_t ret_val = handle_for_frame.at(frame * nmodules + module); - handle_for_frame.at(frame * nmodules + module) = HandleNotFound; + uint64_t ret_val = handle_for_frame.at(frame * nmodules + module_number); + handle_for_frame.at(frame * nmodules + module_number) = HandleNotFound; return ret_val; } -uint64_t AcquisitionOnlineCounters::GetHead(uint16_t module) const { - if (module >= max_modules) +uint64_t AcquisitionOnlineCounters::GetHead(uint16_t module_number) const { + if (module_number >= max_modules) throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, "Module out of bounds"); - return head[module]; + return head[module_number]; } uint64_t AcquisitionOnlineCounters::GetSlowestHead() const { return slowest_head; } -void AcquisitionOnlineCounters::WaitForFrame(size_t curr_frame, uint16_t module) const { - uint64_t slowest_head_tmp = (module == UINT16_MAX) ? GetSlowestHead() : GetHead(module); +void AcquisitionOnlineCounters::WaitForFrame(size_t curr_frame, uint16_t module_number) const { + uint64_t slowest_head_tmp = (module_number == UINT16_MAX) ? GetSlowestHead() : GetHead(module_number); while (!acquisition_finished && (slowest_head_tmp < curr_frame)) { std::this_thread::sleep_for(std::chrono::microseconds(100)); - slowest_head_tmp = (module == UINT16_MAX) ? GetSlowestHead() : GetHead(module); + slowest_head_tmp = (module_number == UINT16_MAX) ? GetSlowestHead() : GetHead(module_number); } } -int64_t AcquisitionOnlineCounters::CalculateDelay(size_t curr_frame, uint16_t module) const { +int64_t AcquisitionOnlineCounters::CalculateDelay(size_t curr_frame, uint16_t module_number) const { uint64_t slowest_head_tmp; - if (module == UINT16_MAX) + if (module_number == UINT16_MAX) slowest_head_tmp = GetSlowestHead(); else - slowest_head_tmp = GetHead(module); + slowest_head_tmp = GetHead(module_number); return slowest_head_tmp - curr_frame; } @@ -118,13 +124,41 @@ bool AcquisitionOnlineCounters::IsAcquisitionFinished() const { return acquisition_finished; } -bool AcquisitionOnlineCounters::IsFullModuleCollected(size_t frame, uint16_t module) const { +bool AcquisitionOnlineCounters::IsFullModuleCollected(size_t frame, uint16_t module_number) const { if (frame >= expected_frames) throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, "Wrong frame number: " + std::to_string(frame)); - if (module >= nmodules) + if (module_number >= nmodules) throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, "Wrong module number"); - return full_module_collected[frame * nmodules + module]; + return full_module_collected[frame * nmodules + module_number]; } + +uint64_t AcquisitionOnlineCounters::GetBunchID(size_t frame, uint16_t module_number) const { + if (frame >= expected_frames) + throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, "Wrong frame number: " + std::to_string(frame)); + + if (module_number >= nmodules) + throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, "Wrong module number"); + + return bunch_id[frame * nmodules + module_number]; +} + +uint32_t AcquisitionOnlineCounters::GetJFInfo(size_t frame, uint16_t module_number) const { + if (frame >= expected_frames) + throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, "Wrong frame number: " + std::to_string(frame)); + + if (module_number >= nmodules) + throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, "Wrong module number"); + return jf_info[frame * nmodules + module_number]; +} + +uint64_t AcquisitionOnlineCounters::GetTimestamp(size_t frame, uint16_t module_number) const { + if (frame >= expected_frames) + throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, "Wrong frame number: " + std::to_string(frame)); + + if (module_number >= nmodules) + throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, "Wrong module number"); + return timestamp[frame * nmodules + module_number]; +} \ No newline at end of file diff --git a/receiver/host/AcquisitionOnlineCounters.h b/receiver/host/AcquisitionOnlineCounters.h index 009017d5..c47ca713 100644 --- a/receiver/host/AcquisitionOnlineCounters.h +++ b/receiver/host/AcquisitionOnlineCounters.h @@ -25,6 +25,10 @@ class AcquisitionOnlineCounters { std::vector handle_for_frame; std::vector full_module_collected; // vector of bool might have weird concurrency behavior + std::vector bunch_id; + std::vector jf_info; + std::vector timestamp; + uint64_t slowest_head; std::vector head; bool acquisition_finished; @@ -38,15 +42,19 @@ public: void UpdateCounters(const Completion *c); void SetAcquisitionFinished(); - uint64_t GetHead(uint16_t module) const; + uint64_t GetHead(uint16_t module_number) const; uint64_t GetSlowestHead() const; - void WaitForFrame(size_t curr_frame, uint16_t module = UINT16_MAX) const; - int64_t CalculateDelay(size_t curr_frame, uint16_t module = UINT16_MAX) const; // mutex acquired indirectly - uint64_t GetBufferHandle(size_t frame, uint16_t module) const; - uint64_t GetBufferHandleAndClear(size_t frame, uint16_t module); - bool IsFullModuleCollected(size_t frame, uint16_t module) const; + void WaitForFrame(size_t curr_frame, uint16_t module_number = UINT16_MAX) const; + int64_t CalculateDelay(size_t curr_frame, uint16_t module_number = UINT16_MAX) const; // mutex acquired indirectly + uint64_t GetBufferHandle(size_t frame, uint16_t module_number) const; + uint64_t GetBufferHandleAndClear(size_t frame, uint16_t module_number); + bool IsFullModuleCollected(size_t frame, uint16_t module_number) const; bool IsAcquisitionFinished() const; + + uint64_t GetBunchID(size_t curr_frame, uint16_t module_number) const; + uint32_t GetJFInfo(size_t curr_frame, uint16_t module_number) const; + uint64_t GetTimestamp(size_t curr_frame, uint16_t module_number) const; }; diff --git a/tests/AcquisitionCountersTest.cpp b/tests/AcquisitionCountersTest.cpp index 0428ce0c..08d30516 100644 --- a/tests/AcquisitionCountersTest.cpp +++ b/tests/AcquisitionCountersTest.cpp @@ -22,6 +22,9 @@ TEST_CASE("AcquisitionDeviceCountersTest","[AcquisitionDeviceCounters]") { c.frame_number = 32; c.module = 1; c.handle = 17; + c.bunchid = 1ul<<56; + c.debug = 1u<<31; + c.timestamp = 456; counters.UpdateCounters(&c); REQUIRE(counters.GetSlowestHead() == 0); @@ -32,6 +35,10 @@ TEST_CASE("AcquisitionDeviceCountersTest","[AcquisitionDeviceCounters]") { REQUIRE(counters.CalculateDelay(33, 1) == -1); REQUIRE(counters.GetBufferHandle(32, 1) == 17); REQUIRE(counters.GetBufferHandle(32, 0) == AcquisitionOnlineCounters::HandleNotFound); + REQUIRE(counters.GetBunchID(32, 1) == c.bunchid); + REQUIRE(counters.GetJFInfo(32, 1) == c.debug); + REQUIRE(counters.GetTimestamp(32, 1) == c.timestamp); + c.frame_number = 15; c.module = 0; counters.UpdateCounters(&c); diff --git a/tests/CBORTest.cpp b/tests/CBORTest.cpp index 1f10d1c9..4d16edc7 100644 --- a/tests/CBORTest.cpp +++ b/tests/CBORTest.cpp @@ -173,7 +173,8 @@ TEST_CASE("CBORSerialize_Image", "[CBOR]") { .spots = spots, .indexing_result = 1, .bunch_id = UINT64_MAX, - .jf_info = UINT32_MAX + .jf_info = UINT32_MAX, + .timestamp = 1ul<<27 | 1ul <<35 }; REQUIRE_NOTHROW(serializer.SerializeImage(message)); @@ -197,6 +198,7 @@ TEST_CASE("CBORSerialize_Image", "[CBOR]") { REQUIRE(image_array.bunch_id == message.bunch_id); REQUIRE(image_array.jf_info == message.jf_info); + REQUIRE(image_array.timestamp == message.timestamp); } TEST_CASE("CBORSerialize_Image_2", "[CBOR]") { diff --git a/writer/HDF5DataFile.cpp b/writer/HDF5DataFile.cpp index e0f1199c..b8f4bd22 100644 --- a/writer/HDF5DataFile.cpp +++ b/writer/HDF5DataFile.cpp @@ -35,6 +35,12 @@ HDF5DataFile::~HDF5DataFile() { group.SaveVector("peakYPosRaw", spot_y, dims, CompressionAlgorithm::BSHUF_LZ4); group.SaveVector("peakTotalIntensity", spot_intensity, dims, CompressionAlgorithm::BSHUF_LZ4); group.SaveVector("indexingResult", indexing_result); + + HDF5Group group_exp(*data_file, "/entry/jungfrau"); + group_exp.NXClass("NXcollection"); + group_exp.SaveVector("bunch_id", bunch_id); + group_exp.SaveVector("info", jf_info); + group_exp.SaveVector("timestamp", timestamp); } data_file.reset(); @@ -53,6 +59,9 @@ void HDF5DataFile::CreateFile() { spot_y.resize(max_spots); spot_intensity.resize(max_spots); indexing_result.resize(1); + bunch_id.resize(1); + jf_info.resize(1); + timestamp.resize(1); } void HDF5DataFile::Write(const DataMessage &msg, uint64_t image_number) { @@ -68,6 +77,9 @@ void HDF5DataFile::Write(const DataMessage &msg, uint64_t image_number) { spot_y.resize(max_spots * (max_image_number + 1)); spot_intensity.resize(max_spots * (max_image_number + 1)); indexing_result.resize(max_image_number + 1); + bunch_id.resize(max_image_number + 1); + jf_info.resize(max_image_number + 1); + timestamp.resize(max_image_number + 1); } nimages++; @@ -84,6 +96,9 @@ void HDF5DataFile::Write(const DataMessage &msg, uint64_t image_number) { } indexing_result[image_number] = msg.indexing_result; + bunch_id[image_number] = msg.bunch_id; + jf_info[image_number] = msg.jf_info; + timestamp[image_number] = msg.timestamp; } size_t HDF5DataFile::GetMaxImageNumber() const { diff --git a/writer/HDF5DataFile.h b/writer/HDF5DataFile.h index 09431d8f..7f1e8fab 100644 --- a/writer/HDF5DataFile.h +++ b/writer/HDF5DataFile.h @@ -28,6 +28,10 @@ class HDF5DataFile { HDF5DataType data_type; std::string image_units; + std::vector bunch_id; + std::vector jf_info; + std::vector timestamp; + std::vector spot_x; std::vector spot_y; std::vector spot_intensity;