diff --git a/common/JFJochMessages.h b/common/JFJochMessages.h index 3d25da43..87a15bdc 100644 --- a/common/JFJochMessages.h +++ b/common/JFJochMessages.h @@ -1,9 +1,6 @@ // SPDX-FileCopyrightText: 2025 Filip Leonarski, Paul Scherrer Institute // SPDX-License-Identifier: GPL-3.0-only -// SPDX-FileCopyrightText: 2024 Filip Leonarski, Paul Scherrer Institute -// SPDX-License-Identifier: GPL-3.0-only - #ifndef JUNGFRAUJOCH_JFJOCHMESSAGES_H #define JUNGFRAUJOCH_JFJOCHMESSAGES_H @@ -136,7 +133,9 @@ struct DataMessage { std::optional jf_info; std::optional receiver_aq_dev_delay; - std::optional receiver_free_send_buf; + std::optional receiver_buf_available; + std::optional receiver_buf_in_sending; + std::optional receiver_buf_in_preparation; std::optional storage_cell; std::optional xfel_pulse_id; diff --git a/docs/CBOR.md b/docs/CBOR.md index 4e0c6735..64c7f995 100644 --- a/docs/CBOR.md +++ b/docs/CBOR.md @@ -116,88 +116,90 @@ See [DECTRIS documentation](https://github.com/dectris/documentation/tree/main/s ## Image message -| Field name | Type | Description | Present in DECTRIS format | Optional | -|----------------------------|----------------------|-----------------------------------------------------------------------------------------------------------------------------------|:-------------------------:|:--------:| -| type | String | value "image" | X | | -| magic_number | uint64 | Number used to describe version of the Jungfraujoch data interface - to allow to detect inconsistency between sender and receiver | | | -| series_unique_id | string | Unique text ID of the series (run_name parameter) | X | | -| series_id | uint64 | Unique numeric ID of the series (run_number parameter) | X | | -| image_id | uint64 | Number of image within the series; for MX lossy compression this is sequential excluding removed frames | X | | -| original_image_id | uint64 | Number of image within the series; for MX lossy compression this includes removed frames in the count | | | -| real_time | Rational | Exposure time | X | | -| start_time | Rational | Exposure start time (highly approximate) | X | | -| end_time | Rational | Exposure end time (highly approximate) | X | | -| spots | Array(object) | Spots: | | | -| - x | float | observed position in x (pixels) | | | -| - y | float | observed position in y (pixels) | | | -| - I | float | intensity (photons) | | | -| - maxc | int64 | max count (photons) | | | -| - ice_ring | bool | spot in resolution range for ice rings | | | -| - indexed | bool | indexed solution | | | -| reflections | Array(object) | Reflections: | | | -| - h | int64 | Miller index | | | -| - k | int64 | Miller index | | | -| - l | int64 | Miller index | | | -| - x | float | position in x (pixels) | | | -| - y | float | position in y (pixels) | | | -| - d | float | resolution \[Angstrom\] | | | -| - I | float | integrated intensity (photons) | | | -| - bkg | float | mean background value (photons) | | | -| - sigma | float | standard deviation, estimated from counting statistics (photons) | | | -| - image | float | image number (present for each spot) | | | -| - dist_ewald | float | distance to Ewald sphere (present only for indexed spots) | | | -| - rlp | float | Reciprocal Lorentz and polarization corrections | | | -| - partiality | float | Partiality of the reflection | | | -| spot_count | uint64 | Spot count | | | -| spot_count_ice_rings | uint64 | Number of spots within identified rings (experimental) | | | -| spot_count_low_res | uint64 | Number of spots in low resolution (prior to filtering) | | | -| spot_count_indexed | uint64 | Number of spots which fit indexing solution within a given tolerance | | | -| az_int_profile | Array(float) | Azimuthal integration results, use az_int_bin_to_q from start message for legend | | | -| | | NaN is used for empty bins and has to be taken care by the receiver | | | -| indexing_result | bool | Indexing successful | | | -| indexing_lattice | Array(9 * float) | Indexing result real lattice; present only if indexed | | X | -| indexing_unit_cell | object | Indexing result unit cell: a, b, c \[angstrom\] and alpha, beta, gamma \[degree\]; present only if indexed | | X | -| | | Unit cell is redundant to lattice - yet to simplify downstream programs to analyze results, both are provided | | | -| profile_radius | float | Profile radius of the image - describes distance of observed reflections from the Ewald sphere \[Angstrom^-1\] | | | -| mosaicity | float | Angular range of spots in image from a rotation scan \[degree\] | | | -| b_factor | float | Estimated B-factor (Angstrom^2) | | | -| indexing_time | float | Time spent on indexing \[s\] | | | -| processing_time | float | Total processing time \[s\] | | | -| xfel_pulse_id | uint64 | Bunch ID (for pulsed source, e.g., SwissFEL) | | X | -| xfel_event_code | uint64 | Event code (for pulsed source, e.g., SwissFEL) | | X | -| lattice_type | object | Bravais lattice classification of the indexing result (present only if available) | | X | -| - centering | string | One-letter centering code: P, A, B, C, I, F, or R | | | -| - niggli_class | int64 | Integer identifier for the Niggli-reduced Bravais class | | | -| - system | string | Crystal system: triclinic, monoclinic, orthorhombic, tetragonal, trigonal, hexagonal, cubic | | | -| jf_info | uint64 | Detector info field | | | -| receiver_aq_dev_delay | uint64 | Receiver internal delay | | | -| receiver_free_send_buf | uint64 | Receiver internal number of available send buffers | | | -| storage_cell | uint64 | Storage cell number | | | -| saturated_pixel_count | uint64 | Saturated pixel count | | | -| pixel_sum | uint64 | Sum of all pixels, excl. error and saturation | | | -| error_pixel_count | uint64 | Error pixel count | | | -| strong_pixel_count | uint64 | Strong pixel count (first stage of spot finding) | | | -| min_viable_pixel_value | int64 | Minimal pixel value, excl. error and saturation | | | -| max_viable_pixel_value | int64 | Maximal pixel value, excl. error and saturation | | | -| resolution_estimate | float | Diffraction resolution estimation \[Angstrom\] | | X | -| data_collection_efficiency | float | Image collection efficiency \[\] | | | -| packets_expected | uint64 | Number of packets expected per image (in units of 2 kB) | | | -| packets_received | uint64 | Number of packets received per image (in units of 2 kB) | | | -| bkg_estimate | float | Mean value for pixels in resolution range from 3.0 to 5.0 A \[photons\] | | | -| beam_center_x | float | Beam center X from post-indexing refinement \[pixel\] | | X | -| beam_center_y | float | Beam center Y from post-indexing refinement \[pixel\] | | X | -| beam_corr_x | float | Beam center correction X applied during processing \[pixel\] | | X | -| beam_corr_y | float | Beam center correction Y applied during processing \[pixel\] | | X | -| adu_histogram | Array(uint64) | ADU histogram | | | -| roi_integrals | object | Results of ROI calculation | | X | -| - sum | int64 | Sum of pixels in ROI area \[photons\] | | | -| - sum_square | int64 | Sum of squares of pixels in ROI area \[photons\] | | | -| - pixels | uint64 | Valid pixels in ROI area | | | -| - max_count | int64 | Highest count in ROI area \[photons\] | | | -| - x_weighted_sum | int64 | ROI pixel X position multiplied by photon count \[photons * pixels\] | | | -| - y_weighted_sum | int64 | ROI pixel Y position multiplied by photon count \[photons * pixels\] | | | -| user_data | string | Optional user defined text information - this is image_appendix serialized to JSON format | X | | -| data | Map(string -> Image) | Image | X | | +| Field name | Type | Description | Present in DECTRIS format | Optional | +|-----------------------------|----------------------|-----------------------------------------------------------------------------------------------------------------------------------|:-------------------------:|:--------:| +| type | String | value "image" | X | | +| magic_number | uint64 | Number used to describe version of the Jungfraujoch data interface - to allow to detect inconsistency between sender and receiver | | | +| series_unique_id | string | Unique text ID of the series (run_name parameter) | X | | +| series_id | uint64 | Unique numeric ID of the series (run_number parameter) | X | | +| image_id | uint64 | Number of image within the series; for MX lossy compression this is sequential excluding removed frames | X | | +| original_image_id | uint64 | Number of image within the series; for MX lossy compression this includes removed frames in the count | | | +| real_time | Rational | Exposure time | X | | +| start_time | Rational | Exposure start time (highly approximate) | X | | +| end_time | Rational | Exposure end time (highly approximate) | X | | +| spots | Array(object) | Spots: | | | +| - x | float | observed position in x (pixels) | | | +| - y | float | observed position in y (pixels) | | | +| - I | float | intensity (photons) | | | +| - maxc | int64 | max count (photons) | | | +| - ice_ring | bool | spot in resolution range for ice rings | | | +| - indexed | bool | indexed solution | | | +| reflections | Array(object) | Reflections: | | | +| - h | int64 | Miller index | | | +| - k | int64 | Miller index | | | +| - l | int64 | Miller index | | | +| - x | float | position in x (pixels) | | | +| - y | float | position in y (pixels) | | | +| - d | float | resolution \[Angstrom\] | | | +| - I | float | integrated intensity (photons) | | | +| - bkg | float | mean background value (photons) | | | +| - sigma | float | standard deviation, estimated from counting statistics (photons) | | | +| - image | float | image number (present for each spot) | | | +| - dist_ewald | float | distance to Ewald sphere (present only for indexed spots) | | | +| - rlp | float | Reciprocal Lorentz and polarization corrections | | | +| - partiality | float | Partiality of the reflection | | | +| spot_count | uint64 | Spot count | | | +| spot_count_ice_rings | uint64 | Number of spots within identified rings (experimental) | | | +| spot_count_low_res | uint64 | Number of spots in low resolution (prior to filtering) | | | +| spot_count_indexed | uint64 | Number of spots which fit indexing solution within a given tolerance | | | +| az_int_profile | Array(float) | Azimuthal integration results, use az_int_bin_to_q from start message for legend | | | +| | | NaN is used for empty bins and has to be taken care by the receiver | | | +| indexing_result | bool | Indexing successful | | | +| indexing_lattice | Array(9 * float) | Indexing result real lattice; present only if indexed | | X | +| indexing_unit_cell | object | Indexing result unit cell: a, b, c \[angstrom\] and alpha, beta, gamma \[degree\]; present only if indexed | | X | +| | | Unit cell is redundant to lattice - yet to simplify downstream programs to analyze results, both are provided | | | +| profile_radius | float | Profile radius of the image - describes distance of observed reflections from the Ewald sphere \[Angstrom^-1\] | | | +| mosaicity | float | Angular range of spots in image from a rotation scan \[degree\] | | | +| b_factor | float | Estimated B-factor (Angstrom^2) | | | +| indexing_time | float | Time spent on indexing \[s\] | | | +| processing_time | float | Total processing time \[s\] | | | +| xfel_pulse_id | uint64 | Bunch ID (for pulsed source, e.g., SwissFEL) | | X | +| xfel_event_code | uint64 | Event code (for pulsed source, e.g., SwissFEL) | | X | +| lattice_type | object | Bravais lattice classification of the indexing result (present only if available) | | X | +| - centering | string | One-letter centering code: P, A, B, C, I, F, or R | | | +| - niggli_class | int64 | Integer identifier for the Niggli-reduced Bravais class | | | +| - system | string | Crystal system: triclinic, monoclinic, orthorhombic, tetragonal, trigonal, hexagonal, cubic | | | +| jf_info | uint64 | Detector info field | | | +| receiver_aq_dev_delay | uint64 | Receiver internal delay | | | +| receiver_free_send_buf | uint64 | Receiver internal number of available buffer locations | | | +| receiver_buf_in_sending | uint64 | Receiver internal number of buffer locations currently in sending/writing | | | +| receiver_buf_in_preparation | uint64 | Receiver internal number of buffer locations currently in processing | | | +| storage_cell | uint64 | Storage cell number | | | +| saturated_pixel_count | uint64 | Saturated pixel count | | | +| pixel_sum | uint64 | Sum of all pixels, excl. error and saturation | | | +| error_pixel_count | uint64 | Error pixel count | | | +| strong_pixel_count | uint64 | Strong pixel count (first stage of spot finding) | | | +| min_viable_pixel_value | int64 | Minimal pixel value, excl. error and saturation | | | +| max_viable_pixel_value | int64 | Maximal pixel value, excl. error and saturation | | | +| resolution_estimate | float | Diffraction resolution estimation \[Angstrom\] | | X | +| data_collection_efficiency | float | Image collection efficiency \[\] | | | +| packets_expected | uint64 | Number of packets expected per image (in units of 2 kB) | | | +| packets_received | uint64 | Number of packets received per image (in units of 2 kB) | | | +| bkg_estimate | float | Mean value for pixels in resolution range from 3.0 to 5.0 A \[photons\] | | | +| beam_center_x | float | Beam center X from post-indexing refinement \[pixel\] | | X | +| beam_center_y | float | Beam center Y from post-indexing refinement \[pixel\] | | X | +| beam_corr_x | float | Beam center correction X applied during processing \[pixel\] | | X | +| beam_corr_y | float | Beam center correction Y applied during processing \[pixel\] | | X | +| adu_histogram | Array(uint64) | ADU histogram | | | +| roi_integrals | object | Results of ROI calculation | | X | +| - sum | int64 | Sum of pixels in ROI area \[photons\] | | | +| - sum_square | int64 | Sum of squares of pixels in ROI area \[photons\] | | | +| - pixels | uint64 | Valid pixels in ROI area | | | +| - max_count | int64 | Highest count in ROI area \[photons\] | | | +| - x_weighted_sum | int64 | ROI pixel X position multiplied by photon count \[photons * pixels\] | | | +| - y_weighted_sum | int64 | ROI pixel Y position multiplied by photon count \[photons * pixels\] | | | +| user_data | string | Optional user defined text information - this is image_appendix serialized to JSON format | X | | +| data | Map(string -> Image) | Image | X | | ## Metadata message diff --git a/frame_serialize/CBORStream2Deserializer.cpp b/frame_serialize/CBORStream2Deserializer.cpp index 036daf00..e4d1fa7d 100644 --- a/frame_serialize/CBORStream2Deserializer.cpp +++ b/frame_serialize/CBORStream2Deserializer.cpp @@ -694,7 +694,11 @@ namespace { else if (key == "receiver_aq_dev_delay") message.receiver_aq_dev_delay = GetCBORInt(value); else if (key == "receiver_free_send_buf") - message.receiver_free_send_buf = GetCBORInt(value); + message.receiver_buf_available = GetCBORInt(value); + else if (key == "receiver_buf_in_sending") + message.receiver_buf_in_sending = GetCBORInt(value); + else if (key == "receiver_buf_in_preparation") + message.receiver_buf_in_preparation = GetCBORInt(value); else if (key == "storage_cell") message.storage_cell = GetCBORUInt(value) & UINT32_MAX; else if (key == "xfel_pulse_id") diff --git a/frame_serialize/CBORStream2Serializer.cpp b/frame_serialize/CBORStream2Serializer.cpp index 97a2f8bd..00be8a98 100644 --- a/frame_serialize/CBORStream2Serializer.cpp +++ b/frame_serialize/CBORStream2Serializer.cpp @@ -712,7 +712,9 @@ void CBORStream2Serializer::SerializeImageInternal(CborEncoder &mapEncoder, cons CBOR_ENC(mapEncoder, "lattice_type", message.lattice_type.value()); CBOR_ENC(mapEncoder, "jf_info", message.jf_info); CBOR_ENC(mapEncoder, "receiver_aq_dev_delay", message.receiver_aq_dev_delay); - CBOR_ENC(mapEncoder, "receiver_free_send_buf", message.receiver_free_send_buf); + CBOR_ENC(mapEncoder, "receiver_free_send_buf", message.receiver_buf_available); + CBOR_ENC(mapEncoder, "receiver_buf_in_sending", message.receiver_buf_in_preparation); + CBOR_ENC(mapEncoder, "receiver_buf_in_preparation", message.receiver_buf_in_preparation); CBOR_ENC(mapEncoder, "storage_cell", message.storage_cell); CBOR_ENC(mapEncoder, "saturated_pixel_count", message.saturated_pixel_count); CBOR_ENC(mapEncoder, "pixel_sum", message.pixel_sum); diff --git a/receiver/JFJochReceiverFPGA.cpp b/receiver/JFJochReceiverFPGA.cpp index 5e6faa63..68e632c2 100644 --- a/receiver/JFJochReceiverFPGA.cpp +++ b/receiver/JFJochReceiverFPGA.cpp @@ -426,7 +426,10 @@ void JFJochReceiverFPGA::FrameTransformationThread(uint32_t threadid) { CompressionAlgorithm::NO_COMPRESSION); analyzer->Process(message, local_spot_finding_settings); - message.receiver_free_send_buf = image_buffer.GetAvailSlots(); + auto status = image_buffer.GetStatus(); + message.receiver_buf_available = status.available_slots; + message.receiver_buf_in_preparation = status.preparation_slots; + message.receiver_buf_in_sending = status.sending_slots; message.az_int_profile = az_int_profile_image.GetResult(); message.bkg_estimate = az_int_profile_image.GetBkgEstimate(experiment.GetAzimuthalIntegrationSettings()); diff --git a/receiver/JFJochReceiverLite.cpp b/receiver/JFJochReceiverLite.cpp index 56c008f1..e3595733 100644 --- a/receiver/JFJochReceiverLite.cpp +++ b/receiver/JFJochReceiverLite.cpp @@ -286,7 +286,7 @@ void JFJochReceiverLite::DataAnalysisThread(uint32_t id) { data_msg.user_data = experiment.GetImageAppendix(); data_msg.run_number = experiment.GetRunNumber(); data_msg.run_name = experiment.GetRunName(); - data_msg.receiver_free_send_buf = image_buffer.GetAvailSlots(); + data_msg.receiver_buf_available = image_buffer.GetAvailSlots(); data_msg.receiver_aq_dev_delay = image_puller.GetCurrentFifoUtilization(); saturated_pixels.Add(data_msg.saturated_pixel_count); diff --git a/receiver/JFJochReceiverPlots.cpp b/receiver/JFJochReceiverPlots.cpp index 5578770d..79e66a62 100644 --- a/receiver/JFJochReceiverPlots.cpp +++ b/receiver/JFJochReceiverPlots.cpp @@ -77,7 +77,9 @@ void JFJochReceiverPlots::Setup(const DiffractionExperiment &experiment, const A saturated_pixels.Clear(r); strong_pixels.Clear(r); receiver_delay.Clear(r); - receiver_free_send_buf.Clear(r); + receiver_buf_available.Clear(r); + receiver_buf_in_preparation.Clear(r); + receiver_buf_in_sending.Clear(r); image_collection_efficiency.Clear(r); { @@ -122,7 +124,9 @@ void JFJochReceiverPlots::Add(const DataMessage &msg, const AzimuthalIntegration packets_received.AddElement(msg.number, msg.packets_received); image_collection_efficiency.AddElement(msg.number, msg.image_collection_efficiency); receiver_delay.AddElement(msg.number, msg.receiver_aq_dev_delay); - receiver_free_send_buf.AddElement(msg.number, msg.receiver_free_send_buf); + receiver_buf_available.AddElement(msg.number, msg.receiver_buf_available); + receiver_buf_in_sending.AddElement(msg.number, msg.receiver_buf_in_sending); + receiver_buf_in_preparation.AddElement(msg.number, msg.receiver_buf_in_preparation); max_value.AddElement(msg.number, msg.max_viable_pixel_value); indexing_time.AddElement(msg.number, msg.indexing_time_s); @@ -267,9 +271,18 @@ MultiLinePlot JFJochReceiverPlots::GetPlots(const PlotRequest &request) { case PlotType::ReceiverDelay: ret = receiver_delay.GetMeanPlot(nbins, start, incr, request.fill_value); break; - case PlotType::ReceiverFreeSendBuf: - ret = receiver_free_send_buf.GetMeanPlot(nbins, start, incr, request.fill_value); + case PlotType::ReceiverFreeSendBuf: { + auto available = receiver_buf_available.GetMeanPerBin(nbins, start, incr, request.fill_value); + auto sending = receiver_buf_in_sending.GetMeanPerBin(nbins, start, incr, request.fill_value); + auto preparation = receiver_buf_in_preparation.GetMeanPerBin(nbins, start, incr, request.fill_value); + available.title = "available"; + sending.title = "sending"; + preparation.title = "preparation"; + ret.AddPlot(available); + ret.AddPlot(sending); + ret.AddPlot(preparation); break; + } case PlotType::StrongPixels: ret = strong_pixels.GetMeanPlot(nbins, start, incr, request.fill_value); break; @@ -431,7 +444,7 @@ void JFJochReceiverPlots::GetPlotRaw(std::vector &v, PlotType type, const v = receiver_delay.ExportArray(); break; case PlotType::ReceiverFreeSendBuf: - v = receiver_free_send_buf.ExportArray(); + v = receiver_buf_available.ExportArray(); break; case PlotType::StrongPixels: v = strong_pixels.ExportArray(); diff --git a/receiver/JFJochReceiverPlots.h b/receiver/JFJochReceiverPlots.h index d7989467..0dacc0ba 100644 --- a/receiver/JFJochReceiverPlots.h +++ b/receiver/JFJochReceiverPlots.h @@ -45,7 +45,9 @@ class JFJochReceiverPlots { StatusVector saturated_pixels; StatusVector strong_pixels; StatusVector receiver_delay; - StatusVector receiver_free_send_buf; + StatusVector receiver_buf_available; + StatusVector receiver_buf_in_sending; + StatusVector receiver_buf_in_preparation; StatusVector image_collection_efficiency; StatusVector packets_received; StatusVector max_value; diff --git a/tests/DiffractionExperimentTest.cpp b/tests/DiffractionExperimentTest.cpp index bc37b269..d406acfc 100644 --- a/tests/DiffractionExperimentTest.cpp +++ b/tests/DiffractionExperimentTest.cpp @@ -591,7 +591,7 @@ TEST_CASE("DiffractionExperiment_ImageTime_EIGER","[DiffractionExperiment]") { DiffractionExperiment x(DetEIGER(3)); DatasetSettings s; x.FrameTime(583us); - REQUIRE(x.GetDetectorSetup().GetReadOutTime() == 3us); + REQUIRE(x.GetDetectorSetup().GetReadOutTime() == 20us); s.ImageTime(5ms); x.ImportDatasetSettings(s); diff --git a/writer/HDF5DataFilePluginDetector.cpp b/writer/HDF5DataFilePluginDetector.cpp index 315874c6..ef3f86fc 100644 --- a/writer/HDF5DataFilePluginDetector.cpp +++ b/writer/HDF5DataFilePluginDetector.cpp @@ -12,8 +12,8 @@ void HDF5DataFilePluginDetector::Write(const DataMessage &msg, uint64_t image_nu storage_cell[image_number] = msg.storage_cell.value(); if (msg.receiver_aq_dev_delay.has_value()) receiver_aq_dev_delay[image_number] = msg.receiver_aq_dev_delay.value(); - if (msg.receiver_free_send_buf.has_value()) - receiver_free_buffers[image_number] = msg.receiver_free_send_buf.value(); + if (msg.receiver_buf_available.has_value()) + receiver_free_buffers[image_number] = msg.receiver_buf_available.value(); if (msg.packets_received.has_value()) packets_received[image_number] = msg.packets_received.value();