From 85a169ec19077fc973cb9e857c86d33b2f519d35 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Sun, 14 Apr 2024 21:47:06 +0200 Subject: [PATCH] Refactor how frames are sent from receiver --- broker/JFJochBrokerHttp.cpp | 10 + broker/JFJochBrokerHttp.h | 3 + broker/JFJochBrokerParser.h | 1 - broker/gen/api/DefaultApi.cpp | 34 + broker/gen/api/DefaultApi.h | 9 + broker/gen/model/Dataset_settings.h | 4 +- broker/jfjoch_api.yaml | 24 + broker/jfjoch_broker.cpp | 10 +- broker/redoc-static.html | 28 +- common/CMakeLists.txt | 18 +- common/DiffractionExperiment.cpp | 4 + common/DiffractionExperiment.h | 1 + common/Plot.h | 2 +- common/SendBuffer.cpp | 71 ++ common/SendBuffer.h | 28 + common/SendBufferControl.cpp | 36 + common/SendBufferControl.h | 23 + common/ThreadSafeFIFO.h | 15 + common/ZeroCopyReturnValue.h | 11 +- common/time_utc.h | 19 + frame_serialize/CBORStream2Deserializer.cpp | 81 +- frame_serialize/CBORStream2Deserializer.h | 9 +- frame_serialize/CBORStream2Serializer.cpp | 43 +- frame_serialize/CBORStream2Serializer.h | 4 + frame_serialize/CMakeLists.txt | 1 - frame_serialize/ImagePusher.h | 7 +- frame_serialize/JFJochMessages.h | 26 +- frame_serialize/README.md | 13 +- frame_serialize/TestImagePusher.cpp | 41 +- frame_serialize/TestImagePusher.h | 7 +- .../ZMQStream2PreviewPublisher.cpp | 33 - frame_serialize/ZMQStream2PreviewPublisher.h | 21 - frame_serialize/ZMQStream2Pusher.cpp | 27 +- frame_serialize/ZMQStream2Pusher.h | 16 +- frame_serialize/ZMQStream2PusherGroup.cpp | 36 +- frame_serialize/ZMQStream2PusherGroup.h | 6 +- frontend_ui/package-lock.json | 769 ++++---------- frontend_ui/package.json | 6 +- frontend_ui/src/App.tsx | 3 +- .../src/components/DataProcessingPlot.tsx | 10 +- .../src/components/DataProcessingPlots.tsx | 4 + .../src/openapi/models/dataset_settings.ts | 6 +- .../src/openapi/services/DefaultService.ts | 21 + python/__init__.py | 1 - python/jungfraujoch.py | 173 ---- python/jungfraujoch_metadata.py | 121 --- python/preview.py | 99 -- python/preview_albula.py | 123 --- python/preview_w_adxv/.gitkeep | 0 python/preview_w_adxv/README.md | 28 - python/preview_w_adxv/adxvSocketh5.py | 192 ---- python/preview_w_adxv/preview.py | 168 --- python/preview_w_adxv/spot_type.txt | 2 - python/preview_w_adxv/start_adxv.sh | 29 - receiver/JFJochReceiver.cpp | 155 ++- receiver/JFJochReceiver.h | 14 +- receiver/JFJochReceiverPlots.cpp | 9 +- receiver/JFJochReceiverPlots.h | 1 + receiver/JFJochReceiverService.cpp | 17 +- receiver/JFJochReceiverService.h | 7 +- receiver/JFJochReceiverTest.cpp | 14 +- receiver/JFJochReceiverTest.h | 8 +- receiver/jfjoch_action_test.cpp | 27 +- tests/CBORTest.cpp | 292 +----- tests/CMakeLists.txt | 3 +- tests/HDF5WritingTest.cpp | 20 +- tests/JFJochReceiverProcessingTest.cpp | 107 +- tests/SendBufferTest.cpp | 70 ++ tests/ThreadSafeFIFOSetTest.cpp | 23 + tests/ZMQImagePusherTest.cpp | 70 +- tests/stream2.c | 956 ------------------ tests/stream2.h | 213 ---- tools/CMakeLists.txt | 3 - tools/HDF5DatasetWriteTest.cpp | 5 +- tools/jfjoch_preview_test.cpp | 74 -- tools/jfjoch_spot_finding_test.cpp | 2 +- tools/jfjoch_writer_test.cpp | 6 +- writer/CMakeLists.txt | 2 - writer/HDF5ImagePusher.cpp | 46 - writer/HDF5ImagePusher.h | 21 - writer/HDF5NXmx.cpp | 306 +++--- writer/HDF5NXmx.h | 45 +- writer/StreamWriter.cpp | 14 +- writer/ZMQImagePuller.cpp | 32 +- writer/ZMQImagePuller.h | 2 + 85 files changed, 1385 insertions(+), 3656 deletions(-) create mode 100644 common/SendBuffer.cpp create mode 100644 common/SendBuffer.h create mode 100644 common/SendBufferControl.cpp create mode 100644 common/SendBufferControl.h create mode 100644 common/time_utc.h delete mode 100644 frame_serialize/ZMQStream2PreviewPublisher.cpp delete mode 100644 frame_serialize/ZMQStream2PreviewPublisher.h delete mode 100644 python/__init__.py delete mode 100644 python/jungfraujoch.py delete mode 100644 python/jungfraujoch_metadata.py delete mode 100644 python/preview.py delete mode 100644 python/preview_albula.py delete mode 100644 python/preview_w_adxv/.gitkeep delete mode 100644 python/preview_w_adxv/README.md delete mode 100644 python/preview_w_adxv/adxvSocketh5.py delete mode 100644 python/preview_w_adxv/preview.py delete mode 100644 python/preview_w_adxv/spot_type.txt delete mode 100644 python/preview_w_adxv/start_adxv.sh create mode 100644 tests/SendBufferTest.cpp delete mode 100644 tests/stream2.c delete mode 100644 tests/stream2.h delete mode 100644 tools/jfjoch_preview_test.cpp delete mode 100644 writer/HDF5ImagePusher.cpp delete mode 100644 writer/HDF5ImagePusher.h diff --git a/broker/JFJochBrokerHttp.cpp b/broker/JFJochBrokerHttp.cpp index 2680ade4..b364f60d 100644 --- a/broker/JFJochBrokerHttp.cpp +++ b/broker/JFJochBrokerHttp.cpp @@ -679,6 +679,16 @@ void JFJochBrokerHttp::plot_receiver_delay_post(const org::openapitools::server: ProcessOutput(Convert(plot), response); } +void JFJochBrokerHttp::plot_receiver_free_send_buffers_post( + const org::openapitools::server::model::Plot_request &plotRequest, + Pistache::Http::ResponseWriter &response) { + PlotRequest req{.type = PlotType::ReceiverFreeSendBuf, .binning = 0}; + if (plotRequest.binningIsSet()) + req.binning = plotRequest.getBinning(); + auto plot = state_machine.GetPlots(req); + ProcessOutput(Convert(plot), response); +} + void JFJochBrokerHttp::plot_roi_sum_post(const org::openapitools::server::model::Plot_request &plotRequest, Pistache::Http::ResponseWriter &response) { PlotRequest req{.type = PlotType::ROISum, .binning = 0}; diff --git a/broker/JFJochBrokerHttp.h b/broker/JFJochBrokerHttp.h index 63fc1fa3..d5da8c77 100644 --- a/broker/JFJochBrokerHttp.h +++ b/broker/JFJochBrokerHttp.h @@ -48,6 +48,9 @@ class JFJochBrokerHttp : public org::openapitools::server::api::DefaultApi { void plot_strong_pixel_post(const org::openapitools::server::model::Plot_request &plotRequest, Pistache::Http::ResponseWriter &response) override; + void plot_receiver_free_send_buffers_post(const org::openapitools::server::model::Plot_request &plotRequest, + Pistache::Http::ResponseWriter &response) override; + void plot_receiver_delay_post(const org::openapitools::server::model::Plot_request &plotRequest, Pistache::Http::ResponseWriter &response) override; diff --git a/broker/JFJochBrokerParser.h b/broker/JFJochBrokerParser.h index 8315c3e6..56617938 100644 --- a/broker/JFJochBrokerParser.h +++ b/broker/JFJochBrokerParser.h @@ -7,7 +7,6 @@ #include "../common/DiffractionExperiment.h" #include "JFJochBrokerHttp.h" #include "../acquisition_device/AcquisitionDeviceGroup.h" -#include "../frame_serialize/ZMQStream2PreviewPublisher.h" DetectorGeometry ParseStandardDetectorGeometry(const nlohmann::json &j); DetectorGeometry ParseCustomDetectorGeometry(const nlohmann::json &j); diff --git a/broker/gen/api/DefaultApi.cpp b/broker/gen/api/DefaultApi.cpp index 826d356e..5310ccfa 100644 --- a/broker/gen/api/DefaultApi.cpp +++ b/broker/gen/api/DefaultApi.cpp @@ -55,6 +55,7 @@ void DefaultApi::setupRoutes() { Routes::Get(*router, base + "/plot/rad_int", Routes::bind(&DefaultApi::plot_rad_int_get_handler, this)); Routes::Get(*router, base + "/plot/rad_int_per_file", Routes::bind(&DefaultApi::plot_rad_int_per_file_get_handler, this)); Routes::Post(*router, base + "/plot/receiver_delay", Routes::bind(&DefaultApi::plot_receiver_delay_post_handler, this)); + Routes::Post(*router, base + "/plot/receiver_free_send_buffers", Routes::bind(&DefaultApi::plot_receiver_free_send_buffers_post_handler, this)); Routes::Get(*router, base + "/plot/resolution_estimate_histogram", Routes::bind(&DefaultApi::plot_resolution_estimate_histogram_get_handler, this)); Routes::Post(*router, base + "/plot/roi_max_count", Routes::bind(&DefaultApi::plot_roi_max_count_post_handler, this)); Routes::Post(*router, base + "/plot/roi_sum", Routes::bind(&DefaultApi::plot_roi_sum_post_handler, this)); @@ -655,6 +656,39 @@ void DefaultApi::plot_receiver_delay_post_handler(const Pistache::Rest::Request response.send(Pistache::Http::Code::Internal_Server_Error, e.what()); } +} +void DefaultApi::plot_receiver_free_send_buffers_post_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response) { + try { + + + // Getting the body param + + Plot_request plotRequest; + + try { + nlohmann::json::parse(request.body()).get_to(plotRequest); + plotRequest.validate(); + } catch (std::exception &e) { + const std::pair errorInfo = this->handleParsingException(e); + response.send(errorInfo.first, errorInfo.second); + return; + } + + try { + this->plot_receiver_free_send_buffers_post(plotRequest, response); + } catch (Pistache::Http::HttpError &e) { + response.send(static_cast(e.code()), e.what()); + return; + } catch (std::exception &e) { + const std::pair errorInfo = this->handleOperationException(e); + response.send(errorInfo.first, errorInfo.second); + return; + } + + } catch (std::exception &e) { + response.send(Pistache::Http::Code::Internal_Server_Error, e.what()); + } + } void DefaultApi::plot_resolution_estimate_histogram_get_handler(const Pistache::Rest::Request &, Pistache::Http::ResponseWriter response) { try { diff --git a/broker/gen/api/DefaultApi.h b/broker/gen/api/DefaultApi.h index f3862d8a..29e8914d 100644 --- a/broker/gen/api/DefaultApi.h +++ b/broker/gen/api/DefaultApi.h @@ -80,6 +80,7 @@ private: void plot_rad_int_get_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response); void plot_rad_int_per_file_get_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response); void plot_receiver_delay_post_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response); + void plot_receiver_free_send_buffers_post_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response); void plot_resolution_estimate_histogram_get_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response); void plot_roi_max_count_post_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response); void plot_roi_sum_post_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response); @@ -284,6 +285,14 @@ private: /// (optional) virtual void plot_receiver_delay_post(const org::openapitools::server::model::Plot_request &plotRequest, Pistache::Http::ResponseWriter &response) = 0; /// + /// Generate receiver free send buffer plot + /// + /// + /// Amount of send buffers available during frame processing - used for internal debugging; binning is configurable + /// + /// (optional) + virtual void plot_receiver_free_send_buffers_post(const org::openapitools::server::model::Plot_request &plotRequest, Pistache::Http::ResponseWriter &response) = 0; + /// /// Generate resolution estimate histogram /// /// diff --git a/broker/gen/model/Dataset_settings.h b/broker/gen/model/Dataset_settings.h index 87a24421..4b2070aa 100644 --- a/broker/gen/model/Dataset_settings.h +++ b/broker/gen/model/Dataset_settings.h @@ -61,7 +61,7 @@ public: /// Dataset_settings members /// - /// For standard synchrotron data collection - this is number of images collected per one TTL trigger For XFEL mode - this number is ignored and set to 1 For storage cell mode - this number is ignored and set to number of storage cells + /// For standard synchrotron data collection - this is number of images collected per one TTL trigger For XFEL (pulsed source) - this number is ignored and set to 1 For storage cell mode - this number is ignored and set to number of storage cells /// int64_t getImagesPerTrigger() const; void setImagesPerTrigger(int64_t const value); @@ -75,7 +75,7 @@ public: bool ntriggerIsSet() const; void unsetNtrigger(); /// - /// FPGA frame summation. For summation above two 32-bit pixel format will be used, unless explicitly specified. + /// FPGA frame summation. For summation above two 32-bit pixel format will be used, unless explicitly specified. Frame summation factor applies only to conversion mode (assumed as 1 for raw data). In XFEL mode: summation happens for frames collected with multiple triggers. Ignored for storage cells (assumed as 1). /// int64_t getSummation() const; void setSummation(int64_t const value); diff --git a/broker/jfjoch_api.yaml b/broker/jfjoch_api.yaml index 36a270d4..5fb64385 100644 --- a/broker/jfjoch_api.yaml +++ b/broker/jfjoch_api.yaml @@ -1345,6 +1345,30 @@ paths: schema: type: string description: Exception error + /plot/receiver_free_send_buffers: + post: + summary: Generate receiver free send buffer plot + description: Amount of send buffers available during frame processing - used for internal debugging; binning is configurable + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/plot_request' + responses: + "200": + description: Everything OK + content: + application/json: + schema: + $ref: '#/components/schemas/plots' + "400": + description: Input parsing or validation error + content: + text/plain: + schema: + type: string + description: + Exception error /plot/image_collection_efficiency: post: summary: Generate image collection efficiency plot diff --git a/broker/jfjoch_broker.cpp b/broker/jfjoch_broker.cpp index b93d84a7..14a0260f 100644 --- a/broker/jfjoch_broker.cpp +++ b/broker/jfjoch_broker.cpp @@ -71,7 +71,6 @@ int main (int argc, char **argv) { std::unique_ptr receiver; std::unique_ptr image_pusher; - std::unique_ptr preview_publisher; ZMQContext context; @@ -90,13 +89,8 @@ int main (int argc, char **argv) { zmq_send_watermark, zmq_send_buffer_size); - receiver = std::make_unique(aq_devices, logger, *image_pusher); - - std::string zmq_preview_addr = ParseString(input, "zmq_preview_addr"); - if (!zmq_preview_addr.empty()) { - preview_publisher = std::make_unique(context, zmq_preview_addr); - receiver->PreviewPublisher(preview_publisher.get()); - } + int32_t send_buffer_size_MiB = ParseInt32(input, "send_buffer_size_MiB", 2048); + receiver = std::make_unique(aq_devices, logger, *image_pusher, send_buffer_size_MiB); std::string numa_policy = ParseString(input, "numa_policy"); if (!numa_policy.empty()) diff --git a/broker/redoc-static.html b/broker/redoc-static.html index d71d84f8..c21316ef 100644 --- a/broker/redoc-static.html +++ b/broker/redoc-static.html @@ -341,7 +341,7 @@ data-styled.g137[id="sc-cMdfCE"]{content:"dvQijr,"}/*!sc*/ -
API docs by Redocly
API docs by Redocly

Everything OK

Request samples

Content type
application/json
{
  • "binning": 0
}

Response samples

Content type
application/json
{
  • "title": "string",
  • "plot": [
    ]
}

Generate image collection efficiency plot

Request samples

Content type
application/json
{
  • "binning": 0
}

Response samples

Content type
application/json
{
  • "title": "string",
  • "plot": [
    ]
}

Generate receiver free send buffer plot

Amount of send buffers available during frame processing - used for internal debugging; binning is configurable

+
Request Body schema: application/json
binning
integer <int64>

Responses

Request samples

Content type
application/json
{
  • "binning": 0
}

Response samples

Content type
application/json
{
  • "title": "string",
  • "plot": [
    ]
}

Generate image collection efficiency plot

Ratio of collected and expected packets per image; binning is configurable

Request Body schema: application/json
binning
integer <int64>

Responses

Request samples

Content type
application/json
{
  • "binning": 0
}

Response samples

Content type
application/json
{
  • "title": "string",
  • "plot": [
    ]
}

Generate indexing rate per file

Request samples

Content type
application/json
{
  • "binning": 0
}

Response samples

Content type
application/json
{
  • "title": "string",
  • "plot": [
    ]
}

Generate indexing rate per file

Indexing rate per each of data files; useful for example for time resolved data

Responses

Response samples

Content type
application/json
{
  • "title": "string",
  • "plot": [
    ]
}

Generate resolution estimate histogram

Response samples

Content type
application/json
{
  • "title": "string",
  • "plot": [
    ]
}

Generate resolution estimate histogram

Generate histogram of crystal resolutions from 1.0 to 5.0 A based on ML model

Responses

Response samples

Content type
application/json
{
  • "title": "string",
  • "plot": [
    ]
}

Generate radial integration profile

Response samples

Content type
application/json
{
  • "title": "string",
  • "plot": [
    ]
}

Generate radial integration profile

Generate average radial integration profile

Responses

Response samples

Content type
application/json
{
  • "title": "string",
  • "plot": [
    ]
}

Generate radial integration profiles per file

Response samples

Content type
application/json
{
  • "title": "string",
  • "plot": [
    ]
}

Generate radial integration profiles per file

Radial integration plots for both the whole dataset and per file; useful for time-resolved measurements

Responses

Response samples

Content type
application/json
{
  • "title": "string",
  • "plot": [
    ]
}

Get data collection statistics

Response samples

Content type
application/json
{
  • "title": "string",
  • "plot": [
    ]
}

Get data collection statistics

Results of the last data collection

Responses

Response samples

Content type
application/json
{
  • "file_prefix": "string",
  • "images_expected": 0,
  • "images_collected": 0,
  • "images_sent": 0,
  • "max_image_number_sent": 0,
  • "collection_efficiency": 1,
  • "compression_ratio": 5.3,
  • "cancelled": true,
  • "max_receiver_delay": 0,
  • "indexing_rate": 0.1,
  • "detector_width": 0,
  • "detector_height": 0,
  • "detector_pixel_depth": 2,
  • "bkg_estimate": 0.1
}

Get calibration statistics

Response samples

Content type
application/json
{
  • "file_prefix": "string",
  • "images_expected": 0,
  • "images_collected": 0,
  • "images_sent": 0,
  • "max_image_number_sent": 0,
  • "collection_efficiency": 1,
  • "compression_ratio": 5.3,
  • "cancelled": true,
  • "max_receiver_delay": 0,
  • "indexing_rate": 0.1,
  • "detector_width": 0,
  • "detector_height": 0,
  • "detector_pixel_depth": 2,
  • "bkg_estimate": 0.1
}

Get calibration statistics

Statistics are provided for each module/storage cell separately

Responses

Response samples

Content type
application/json
[
  • {
    }
]

Get last preview image in JPEG format using custom settings

Request Body schema: application/json
saturation
required
integer <int64> [ 0 .. 65535 ]

Response samples

Content type
application/json
[
  • {
    }
]

Get last preview image in JPEG format using custom settings

Request Body schema: application/json
saturation
required
integer <int64> [ 0 .. 65535 ]

Saturation value to set contrast in the preview image

show_spots
boolean
Default: true

Show spot finding results on the image

@@ -741,7 +747,7 @@ Changing detector will set detector to Inactive state and will requ " class="sc-iKOmoZ sc-cCzLxZ WVNwY VEBGS sc-ckdEwu LxEPk">

No preview image recorded so far

Request samples

Content type
application/json
{
  • "saturation": 65535,
  • "show_spots": true,
  • "show_roi": false,
  • "jpeg_quality": 100,
  • "show_indexed": true
}

Response samples

Content type
application/json
{
  • "msg": "Detector in wrong state",
  • "reason": "WrongDAQState"
}

Get last preview image in JPEG format using default settings

Responses

Request samples

Content type
application/json
{
  • "saturation": 65535,
  • "show_spots": true,
  • "show_roi": false,
  • "jpeg_quality": 100,
  • "show_indexed": true
}

Response samples

Content type
application/json
{
  • "msg": "Detector in wrong state",
  • "reason": "WrongDAQState"
}

Get last preview image in JPEG format using default settings

Responses