From f5f86d9ab6dee9fea2bd497a53dfccb73831671f Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Sat, 27 Jan 2024 21:23:56 +0100 Subject: [PATCH] Modifications in preparation to MAX IV experiment --- .gitlab-ci.yml | 25 +- CMakeLists.txt | 10 +- README.md | 2 + acquisition_device/AcquisitionCounters.cpp | 3 +- acquisition_device/AcquisitionDevice.cpp | 65 +- acquisition_device/AcquisitionDevice.h | 15 +- acquisition_device/AcquisitionDeviceGroup.cpp | 10 + acquisition_device/AcquisitionDeviceGroup.h | 3 +- acquisition_device/CMakeLists.txt | 2 +- acquisition_device/FPGAAcquisitionDevice.cpp | 219 ++++--- acquisition_device/FPGAAcquisitionDevice.h | 9 +- acquisition_device/HLSSimulatedDevice.cpp | 424 +++++++------ acquisition_device/HLSSimulatedDevice.h | 15 +- acquisition_device/PCIExpressDevice.cpp | 27 +- acquisition_device/PCIExpressDevice.h | 5 +- broker/CMakeLists.txt | 2 - broker/JFJochBrokerHttp.cpp | 109 +++- broker/JFJochBrokerHttp.h | 28 +- broker/JFJochBrokerParser.cpp | 17 +- broker/JFJochBrokerParser.h | 2 +- broker/JFJochServices.cpp | 47 +- broker/JFJochServices.h | 2 +- broker/JFJochStateMachine.cpp | 215 +++++-- broker/JFJochStateMachine.h | 19 +- broker/gen/api/DefaultApi.cpp | 225 ++++++- broker/gen/api/DefaultApi.h | 62 +- broker/gen/model/Broker_status.cpp | 50 +- broker/gen/model/Broker_status.h | 9 - broker/gen/model/Dataset_settings.cpp | 116 +++- broker/gen/model/Dataset_settings.h | 38 ++ .../model/Dataset_settings_roi_sum_area.cpp | 133 ++++ .../gen/model/Dataset_settings_roi_sum_area.h | 97 +++ broker/gen/model/Detector_settings.cpp | 62 +- broker/gen/model/Detector_settings.h | 24 +- broker/gen/model/Measurement_statistics.cpp | 60 +- broker/gen/model/Measurement_statistics.h | 18 + broker/gen/model/Rotation_axis.cpp | 179 ++++++ broker/gen/model/Rotation_axis.h | 95 +++ broker/jfjoch_api.yaml | 218 ++++++- broker/jfjoch_broker.cpp | 33 +- broker/redoc-static.html | 84 ++- common/ADUHistogram.cpp | 16 +- common/ADUHistogram.h | 2 + common/CMakeLists.txt | 4 +- common/Definitions.h | 58 +- common/DetectorGeometry.cpp | 22 + common/DetectorGeometry.h | 4 + common/DiffractionExperiment.cpp | 198 +++++- common/DiffractionExperiment.h | 59 +- common/DiffractionSpot.cpp | 7 + common/DiffractionSpot.h | 1 + common/Plot.h | 3 +- common/RawToConvertedGeometry.h | 12 + common/StatusVector.h | 1 + common/ZMQWrappers.cpp | 43 +- common/ZMQWrappers.h | 18 +- common/to_fixed.h | 18 - detector_control/DetectorWrapper.cpp | 34 +- etc/CMakeLists.txt | 1 - etc/broker.json | 3 +- export_images/CMakeLists.txt | 16 + export_images/WriteJPEG.cpp | 41 ++ export_images/WriteJPEG.h | 7 + export_images/WriteTIFF.cpp | 60 ++ export_images/WriteTIFF.h | 12 + fpga/README.md | 10 +- fpga/hdl/action_config.v | 56 +- fpga/hls/CMakeLists.txt | 4 +- fpga/hls/adu_histo.cpp | 36 +- fpga/hls/arp.cpp | 1 - fpga/hls/data_collection_fsm.cpp | 37 +- fpga/hls/ethernet.cpp | 7 +- fpga/hls/frame_generator.cpp | 22 +- fpga/hls/frame_summation.cpp | 20 +- fpga/hls/frame_summation_reorder_compl.cpp | 9 +- fpga/hls/frame_summation_tb.cpp | 3 +- fpga/hls/frame_summation_tb_2.cpp | 3 +- fpga/hls/hls_jfjoch.h | 50 +- fpga/hls/host_writer.cpp | 125 +++- fpga/hls/integration.cpp | 20 +- fpga/hls/integration_tb.cpp | 4 +- fpga/hls/ipv4.cpp | 53 +- fpga/hls/jf_conversion.cpp | 48 +- fpga/hls/load_calibration.cpp | 235 ++++--- fpga/hls/load_from_hbm.cpp | 18 +- fpga/hls/mask_missing.cpp | 10 +- fpga/hls/pedestal.cpp | 17 +- fpga/hls/save_to_hbm.cpp | 24 +- fpga/hls/sls_detector.cpp | 12 +- fpga/hls/spot_finder.cpp | 59 +- fpga/hls/spot_finder_tb.cpp | 9 +- fpga/hls/stream_24bit_conv.cpp | 23 +- fpga/hls/stream_merge.cpp | 6 +- fpga/hls/timer.cpp | 11 +- fpga/hls/udp.cpp | 8 +- fpga/host_library/CMakeLists.txt | 3 +- fpga/host_library/DeviceOutput.h | 51 -- fpga/host_library/JungfraujochDevice.cpp | 32 +- fpga/host_library/JungfraujochDevice.h | 13 +- fpga/host_library/README.md | 31 +- fpga/host_library/jfjoch_pcie_status.cpp | 4 + fpga/include/jfjoch_fpga.h | 229 +++++++ fpga/pcie_driver/ActionConfig.h | 110 ---- fpga/pcie_driver/jfjoch_drv.c | 2 +- fpga/pcie_driver/jfjoch_drv.h | 16 +- fpga/pcie_driver/jfjoch_function.c | 97 ++- fpga/pcie_driver/jfjoch_int.c | 15 +- fpga/pcie_driver/jfjoch_ioctl.c | 22 +- fpga/pcie_driver/jfjoch_ioctl.h | 7 +- fpga/pcie_driver/jfjoch_sysfs.c | 33 +- fpga/scripts/hbm_cache.tcl | 4 + fpga/scripts/image_processing.tcl | 12 +- fpga/scripts/jfjoch.tcl | 24 +- fpga/scripts/mac_100g_pcie.tcl | 39 +- fpga/scripts/mac_4x10g.tcl | 2 +- fpga/scripts/network_stack.tcl | 8 +- fpga/scripts/setup_action.sh | 8 +- frame_serialize/CBORStream2Deserializer.cpp | 374 +++++------ frame_serialize/CBORStream2Deserializer.h | 5 +- frame_serialize/CBORStream2Serializer.cpp | 265 ++++---- frame_serialize/CMakeLists.txt | 8 +- frame_serialize/ImagePusher.h | 4 +- frame_serialize/JFJochMessages.h | 66 +- frame_serialize/README.md | 153 ++++- frame_serialize/TestImagePusher.cpp | 9 +- frame_serialize/TestImagePusher.h | 5 +- frame_serialize/ZMQBsreadImagePusher.cpp | 9 +- frame_serialize/ZMQBsreadImagePusher.h | 4 +- frame_serialize/ZMQPreviewPublisher.cpp | 51 -- frame_serialize/ZMQPreviewPublisher.h | 28 - .../ZMQStream2PreviewPublisher.cpp | 33 + frame_serialize/ZMQStream2PreviewPublisher.h | 21 + frame_serialize/ZMQStream2Pusher.cpp | 97 ++- frame_serialize/ZMQStream2Pusher.h | 26 +- frame_serialize/ZMQStream2PusherGroup.cpp | 61 ++ frame_serialize/ZMQStream2PusherGroup.h | 27 + frontend_ui/public/bootstrap.min.css | 7 + frontend_ui/public/bootstrap.min.css.map | 1 + frontend_ui/public/index.html | 4 +- frontend_ui/src/components/Calibration.tsx | 4 +- .../src/components/DataProcessingPlot.tsx | 53 +- .../src/components/DataProcessingPlots.tsx | 80 ++- .../src/components/DetectorSelection.tsx | 2 +- .../src/components/DetectorSettings.tsx | 30 +- .../src/components/MeasurementStatistics.tsx | 38 +- .../RadialIntegrationProfilePlots.tsx | 4 +- frontend_ui/src/components/StatusBar.tsx | 19 +- frontend_ui/src/openapi/index.ts | 1 + .../src/openapi/models/broker_status.ts | 1 - .../src/openapi/models/dataset_settings.ts | 20 + .../src/openapi/models/detector_settings.ts | 17 + .../openapi/models/measurement_statistics.ts | 2 + .../src/openapi/models/rotation_axis.ts | 23 + .../src/openapi/services/DefaultService.ts | 123 +++- ...ng.cpp => AzimuthalIntegrationMapping.cpp} | 26 +- ...apping.h => AzimuthalIntegrationMapping.h} | 12 +- ...le.cpp => AzimuthalIntegrationProfile.cpp} | 36 +- image_analysis/AzimuthalIntegrationProfile.h | 31 + image_analysis/CMakeLists.txt | 4 +- image_analysis/CrystalLattice.cpp | 3 +- image_analysis/CrystalLattice.h | 1 + image_analysis/RadialIntegrationProfile.h | 28 - image_analysis/StrongPixelSet.cpp | 55 +- image_analysis/StrongPixelSet.h | 10 +- jungfrau/JFConversionFloatingPoint.cpp | 70 +-- jungfrau/JFConversionFloatingPoint.h | 4 +- jungfrau/JFModuleGainCalibration.cpp | 73 ++- jungfrau/JFModuleGainCalibration.h | 21 +- jungfrau/JFModulePedestal.cpp | 7 +- jungfrau/JFModulePedestal.h | 1 - python/preview.py | 122 ++-- python/preview_albula.py | 123 ++++ python/preview_w_adxv/.gitkeep | 0 python/preview_w_adxv/README.md | 28 + python/preview_w_adxv/adxvSocketh5.py | 192 ++++++ python/preview_w_adxv/preview.py | 168 +++++ python/preview_w_adxv/spot_type.txt | 2 + python/preview_w_adxv/start_adxv.sh | 29 + receiver/CMakeLists.txt | 10 +- {common => receiver}/FrameTransformation.cpp | 78 ++- {common => receiver}/FrameTransformation.h | 11 +- receiver/ImageMetadata.cpp | 45 ++ receiver/ImageMetadata.h | 33 + receiver/JFJochReceiver.cpp | 584 ++++++++---------- receiver/JFJochReceiver.h | 69 ++- receiver/JFJochReceiverPlots.cpp | 106 ++++ receiver/JFJochReceiverPlots.h | 44 ++ receiver/JFJochReceiverService.cpp | 13 +- receiver/JFJochReceiverService.h | 4 +- receiver/JFJochReceiverTest.cpp | 13 +- receiver/JFJochReceiverTest.h | 4 +- receiver/PreviewCounter.cpp | 17 + receiver/PreviewCounter.h | 18 + receiver/jfjoch_action_test.cpp | 6 +- ...nTest.cpp => AzimuthalIntegrationTest.cpp} | 58 +- tests/CBORTest.cpp | 112 ++-- tests/CMakeLists.txt | 12 +- tests/DetectorGeometryTest.cpp | 26 + tests/DiffractionExperimentTest.cpp | 103 ++- tests/DiffractionGeometryTest.cpp | 5 + tests/DiffractionSpotTest.cpp | 21 + tests/FPGAFrameGeneratorTest.cpp | 26 + tests/FPGAHostWriterTest.cpp | 64 ++ tests/FPGAIntegrationTest.cpp | 262 ++++++-- tests/FPGANetworkTest.cpp | 48 +- tests/FPGASpotFindingUnitTest.cpp | 8 +- tests/FPGAUnitTest.h | 6 +- tests/FrameTransformationTest.cpp | 112 +++- tests/HDF5WritingTest.cpp | 50 +- tests/JFJochReceiverIntegrationTest.cpp | 106 +++- tests/PreviewCounterTest.cpp | 16 + tests/RawToConvertedGeometryTest.cpp | 9 + tests/SpotAnalyzeUnitTest.cpp | 56 -- tests/StreamWriterTest.cpp | 12 +- tests/StrongPixelSetTest.cpp | 130 ++++ tests/TIFFTest.cpp | 32 + tests/ZMQImagePusherTest.cpp | 58 +- tests/ZMQPreviewPublisherTest.cpp | 79 --- tools/CMakeLists.txt | 17 +- tools/CompressionBenchmark.cpp | 2 +- tools/HDF5DatasetWriteTest.cpp | 8 +- tools/SpotFindingPerformanceTest.cpp | 58 ++ tools/gain_file_statistics.cpp | 25 + tools/jfjoch_preview_test.cpp | 80 +++ tools/jfjoch_writer_test.cpp | 8 +- writer/CMakeLists.txt | 18 +- writer/HDF5DataFile.cpp | 168 +++-- writer/HDF5DataFile.h | 20 +- writer/HDF5NXmx.cpp | 48 +- writer/HDF5NXmx.h | 2 +- writer/HDF5Objects.cpp | 8 +- writer/HDF5Objects.h | 5 +- writer/HDF5Writer.cpp | 2 +- writer/JFJochWriterHttp.cpp | 53 ++ writer/JFJochWriterHttp.h | 28 + writer/StreamWriter.cpp | 22 +- writer/StreamWriter.h | 7 +- writer/ZMQImagePuller.cpp | 22 +- writer/ZMQImagePuller.h | 5 +- writer/gen/api/DefaultApi.cpp | 149 +++++ writer/gen/api/DefaultApi.h | 102 +++ writer/gen/model/Helpers.cpp | 148 +++++ writer/gen/model/Helpers.h | 136 ++++ writer/gen/model/Writer_statistics.cpp | 164 +++++ writer/gen/model/Writer_statistics.h | 96 +++ .../_wait_till_done_get_500_response.cpp | 91 +++ .../model/_wait_till_done_get_500_response.h | 77 +++ writer/jfjoch_writer_http.cpp | 75 +++ writer/redoc-static.html | 341 ++++++++++ writer/writer_api.yaml | 93 +++ 250 files changed, 9363 insertions(+), 3022 deletions(-) create mode 100644 broker/gen/model/Dataset_settings_roi_sum_area.cpp create mode 100644 broker/gen/model/Dataset_settings_roi_sum_area.h create mode 100644 broker/gen/model/Rotation_axis.cpp create mode 100644 broker/gen/model/Rotation_axis.h delete mode 100644 common/to_fixed.h delete mode 100644 etc/CMakeLists.txt create mode 100644 export_images/CMakeLists.txt create mode 100644 export_images/WriteJPEG.cpp create mode 100644 export_images/WriteJPEG.h create mode 100644 export_images/WriteTIFF.cpp create mode 100644 export_images/WriteTIFF.h delete mode 100644 fpga/host_library/DeviceOutput.h create mode 100644 fpga/include/jfjoch_fpga.h delete mode 100644 fpga/pcie_driver/ActionConfig.h delete mode 100644 frame_serialize/ZMQPreviewPublisher.cpp delete mode 100644 frame_serialize/ZMQPreviewPublisher.h create mode 100644 frame_serialize/ZMQStream2PreviewPublisher.cpp create mode 100644 frame_serialize/ZMQStream2PreviewPublisher.h create mode 100644 frame_serialize/ZMQStream2PusherGroup.cpp create mode 100644 frame_serialize/ZMQStream2PusherGroup.h create mode 100644 frontend_ui/public/bootstrap.min.css create mode 100644 frontend_ui/public/bootstrap.min.css.map create mode 100644 frontend_ui/src/openapi/models/rotation_axis.ts rename image_analysis/{RadialIntegrationMapping.cpp => AzimuthalIntegrationMapping.cpp} (70%) rename image_analysis/{RadialIntegrationMapping.h => AzimuthalIntegrationMapping.h} (59%) rename image_analysis/{RadialIntegrationProfile.cpp => AzimuthalIntegrationProfile.cpp} (63%) create mode 100644 image_analysis/AzimuthalIntegrationProfile.h delete mode 100644 image_analysis/RadialIntegrationProfile.h create mode 100644 python/preview_albula.py create mode 100644 python/preview_w_adxv/.gitkeep create mode 100644 python/preview_w_adxv/README.md create mode 100644 python/preview_w_adxv/adxvSocketh5.py create mode 100644 python/preview_w_adxv/preview.py create mode 100644 python/preview_w_adxv/spot_type.txt create mode 100644 python/preview_w_adxv/start_adxv.sh rename {common => receiver}/FrameTransformation.cpp (63%) rename {common => receiver}/FrameTransformation.h (65%) create mode 100644 receiver/ImageMetadata.cpp create mode 100644 receiver/ImageMetadata.h create mode 100644 receiver/JFJochReceiverPlots.cpp create mode 100644 receiver/JFJochReceiverPlots.h create mode 100644 receiver/PreviewCounter.cpp create mode 100644 receiver/PreviewCounter.h rename tests/{RadialIntegrationTest.cpp => AzimuthalIntegrationTest.cpp} (62%) create mode 100644 tests/DiffractionSpotTest.cpp create mode 100644 tests/FPGAFrameGeneratorTest.cpp create mode 100644 tests/FPGAHostWriterTest.cpp create mode 100644 tests/PreviewCounterTest.cpp delete mode 100644 tests/SpotAnalyzeUnitTest.cpp create mode 100644 tests/StrongPixelSetTest.cpp create mode 100644 tests/TIFFTest.cpp delete mode 100644 tests/ZMQPreviewPublisherTest.cpp create mode 100644 tools/SpotFindingPerformanceTest.cpp create mode 100644 tools/gain_file_statistics.cpp create mode 100644 tools/jfjoch_preview_test.cpp create mode 100644 writer/JFJochWriterHttp.cpp create mode 100644 writer/JFJochWriterHttp.h create mode 100644 writer/gen/api/DefaultApi.cpp create mode 100644 writer/gen/api/DefaultApi.h create mode 100644 writer/gen/model/Helpers.cpp create mode 100644 writer/gen/model/Helpers.h create mode 100644 writer/gen/model/Writer_statistics.cpp create mode 100644 writer/gen/model/Writer_statistics.h create mode 100644 writer/gen/model/_wait_till_done_get_500_response.cpp create mode 100644 writer/gen/model/_wait_till_done_get_500_response.h create mode 100644 writer/jfjoch_writer_http.cpp create mode 100644 writer/redoc-static.html create mode 100644 writer/writer_api.yaml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5b262f05..8d63d01d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,6 +20,23 @@ build:x86:gcc: - cmake -DCMAKE_BUILD_TYPE=Release .. - make -j48 jfjoch +build:x86:gcc_writer: + stage: build + variables: + GIT_SUBMODULE_STRATEGY: recursive + CC: gcc + CXX: g++ + tags: + - gcc + - x86 + needs: [] + script: + - mkdir build + - cd build + - source /opt/rh/gcc-toolset-12/enable + - cmake -DCMAKE_BUILD_TYPE=Release -DJFJOCH_WRITER_ONLY=ON .. + - make -j48 jfjoch + build:x86:driver: stage: build variables: @@ -49,7 +66,7 @@ build:x86:vitis_hls: - fpga/scripts/* - fpga/xdc/* - fpga/microblaze/* - - common/Definitions.h + - fpga/include/jfjoch_fpga.h script: - source /opt/Xilinx/Vitis_HLS/2022.1/settings64.sh - mkdir build @@ -69,10 +86,10 @@ build:x86:frontend: - npm install - npm run build - cd build - - zip ../../frontend.zip * + - zip -r ../../frontend.${CI_COMMIT_SHORT_SHA}.zip * artifacts: paths: - - frontend.zip + - frontend.${CI_COMMIT_SHORT_SHA}.zip expire_in: 1 week test:x86:gcc: @@ -193,7 +210,7 @@ synthesis:vivado_pcie_100g: - fpga/hdl/* - fpga/scripts/* - fpga/xdc/* - - common/Definitions.h + - fpga/include/jfjoch_fpga.h tags: - vivado artifacts: diff --git a/CMakeLists.txt b/CMakeLists.txt index c6702818..e21724ce 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.18) +CMAKE_MINIMUM_REQUIRED(VERSION 3.19) PROJECT(Jungfraujoch VERSION 1.0 LANGUAGES C CXX) @@ -40,12 +40,13 @@ ADD_SUBDIRECTORY(common) ADD_SUBDIRECTORY(writer) ADD_SUBDIRECTORY(frame_serialize) +ADD_SUBDIRECTORY(broker/pistache) + IF (JFJOCH_WRITER_ONLY) MESSAGE(STATUS "Compiling HDF5 writer only") - SET(jfjoch_executables jfjoch_writer) + SET(jfjoch_executables jfjoch_writer jfjoch_writer_http) ELSE() ADD_SUBDIRECTORY(broker) - ADD_SUBDIRECTORY(etc) ADD_SUBDIRECTORY(fpga) ADD_SUBDIRECTORY(acquisition_device) ADD_SUBDIRECTORY(receiver) @@ -53,7 +54,8 @@ ELSE() ADD_SUBDIRECTORY(detector_control) ADD_SUBDIRECTORY(tests) ADD_SUBDIRECTORY(tools) - SET(jfjoch_executables jfjoch_broker jfjoch_writer CatchTest CompressionBenchmark HDF5DatasetWriteTest jfjoch_udp_simulator sls_detector_put sls_detector_get) +# ADD_SUBDIRECTORY(export_images) + SET(jfjoch_executables jfjoch_broker jfjoch_writer jfjoch_writer_http CatchTest CompressionBenchmark HDF5DatasetWriteTest jfjoch_udp_simulator sls_detector_put sls_detector_get) ENDIF() ADD_CUSTOM_COMMAND(OUTPUT frontend_ui/build/index.html diff --git a/README.md b/README.md index 556480d7..0bd17b1b 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,8 @@ Required: * HDF5 library version 1.10 or newer * ZeroMQ library * OpenSSL library +* TIFF library + C++ binding (optional) +* JPEG library (optional) Optional: * CUDA compiler version 11 or newer - image analysis features won't work without it diff --git a/acquisition_device/AcquisitionCounters.cpp b/acquisition_device/AcquisitionCounters.cpp index 47f7cbaa..5c9e7002 100644 --- a/acquisition_device/AcquisitionCounters.cpp +++ b/acquisition_device/AcquisitionCounters.cpp @@ -54,7 +54,8 @@ void AcquisitionCounters::UpdateCounters(const Completion *c) { if (c->module_number >= nmodules) throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, - "UpdateCounters wrong module number: " + std::to_string(c->module_number) + " for frame " + std::to_string(c->frame_number)); + "UpdateCounters wrong module number: " + std::to_string(c->module_number) + " for frame " + std::to_string(c->frame_number) + + " and handle " + std::to_string(c->handle)); if (c->frame_number >= expected_frames) throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "UpdateCounters frame number is out of bounds"); diff --git a/acquisition_device/AcquisitionDevice.cpp b/acquisition_device/AcquisitionDevice.cpp index 962aafd8..13640f15 100644 --- a/acquisition_device/AcquisitionDevice.cpp +++ b/acquisition_device/AcquisitionDevice.cpp @@ -32,8 +32,7 @@ void *mmap_acquisition_buffer(size_t size, int16_t numa_node) { return ret; } -AcquisitionDevice::AcquisitionDevice(uint16_t in_data_stream) : -buffer_err(FPGA_BUFFER_LOCATION_SIZE) { +AcquisitionDevice::AcquisitionDevice(uint16_t in_data_stream) { logger = nullptr; data_stream = in_data_stream; } @@ -53,13 +52,6 @@ void AcquisitionDevice::StartAction(const DiffractionExperiment &experiment, uin throw(JFJochException(JFJochExceptionCategory::InputParameterAboveMax, "Number of modules exceeds max possible for FPGA")); - for (int i = 0; i < RAW_MODULE_SIZE; i++) { - if (experiment.GetDetectorMode() == DetectorMode::Conversion) - buffer_err[i] = PIXEL_OUT_LOST; - else - buffer_err[i] = -1; - } - counters.Reset(experiment, data_stream); expected_frames = experiment.GetFrameNum() / experiment.GetSummation(); @@ -78,13 +70,25 @@ void AcquisitionDevice::StartAction(const DiffractionExperiment &experiment, uin StartSendingWorkRequests(); start_time = std::chrono::system_clock::now(); + + if (experiment.IsUsingInternalPacketGen()) + RunInternalGenerator(experiment); } void AcquisitionDevice::WaitForActionComplete(bool pedestal_mode) { auto c = work_completion_queue.GetBlocking(); while (c.type != Completion::Type::End) { - auto output = GetDeviceOutput(c.handle); + DeviceOutput* output; + + try { + output = GetDeviceOutput(c.handle); + } catch (const JFJochException &e) { + if (logger) + logger->ErrorException(e); + continue; + } + c.module_number = output->module_statistics.module_number; c.packet_count = output->module_statistics.packet_count; c.frame_number = output->module_statistics.frame_number; @@ -101,11 +105,22 @@ void AcquisitionDevice::WaitForActionComplete(bool pedestal_mode) { data_stream, c.frame_number, c.module_number, c.handle); SendWorkRequest(c.handle); } else if (pedestal_mode && !c.pedestal) { + try { counters.UpdateCounters(&c); + } catch (const JFJochException &e) { + if (logger) + logger->ErrorException(e); + } SendWorkRequest(c.handle); - } else - counters.UpdateCounters(&c); - + } else { + try { + counters.UpdateCounters(&c); + } catch (const JFJochException &e) { + if (logger) + logger->ErrorException(e); + SendWorkRequest(c.handle); + } + } if (logger != nullptr) logger->Debug("Data stream {} completion frame number {} module {} handle {}", data_stream, c.frame_number, c.module_number, c.handle); @@ -134,7 +149,7 @@ const DeviceOutput *AcquisitionDevice::GetDeviceOutput(size_t frame_number, uint if (handle != HandleNotValid) return GetDeviceOutput(handle); else - return (DeviceOutput *) GetErrorFrameBuffer(); + throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, "Frame not collected"); } const DeviceOutput *AcquisitionDevice::GetDeviceOutputPedestal(size_t storage_cell, uint16_t module_number) const { @@ -142,11 +157,7 @@ const DeviceOutput *AcquisitionDevice::GetDeviceOutputPedestal(size_t storage_ce if (handle != HandleNotValid) return GetDeviceOutput(handle); else - return (DeviceOutput *) GetErrorFrameBuffer(); -} - -const int16_t *AcquisitionDevice::GetErrorFrameBuffer() const { - return buffer_err.data(); + throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, "Frame not collected"); } const DeviceOutput *AcquisitionDevice::GetDeviceOutput(size_t handle) const { @@ -172,9 +183,9 @@ void AcquisitionDevice::InitializeIntegrationMap(const DiffractionExperiment &ex const std::vector &v, const std::vector &weights) {} -void AcquisitionDevice::MapBuffersStandard(size_t c2h_buffer_count, size_t h2c_buffer_count, int16_t numa_node) { +void AcquisitionDevice::MapBuffersStandard(size_t c2h_buffer_count, int16_t numa_node) { try { - for (int i = 0; i < std::max(c2h_buffer_count, h2c_buffer_count); i++) + for (int i = 0; i < c2h_buffer_count; i++) buffer_device.emplace_back((DeviceOutput *) mmap_acquisition_buffer(FPGA_BUFFER_LOCATION_SIZE, numa_node)); } catch (const JFJochException &e) { UnmapBuffers(); @@ -267,3 +278,15 @@ void AcquisitionDevice::SetDefaultDataSource(AcquisitionDeviceSource id) { AcquisitionDeviceSource AcquisitionDevice::GetDataSource() { return AcquisitionDeviceSource::NONE; } + +void AcquisitionDevice::RunInternalGenerator(const DiffractionExperiment &experiment) { + FrameGeneratorConfig config{}; + config.frames = experiment.GetFrameNum() + DELAY_FRAMES_STOP_AND_QUIT + 1; + config.modules = experiment.GetModulesNum(data_stream); + config.bunchid = INT_PKT_GEN_BUNCHID; + config.exptime = INT_PKT_GEN_EXPTTIME; + config.debug = INT_PKT_GEN_DEBUG; + config.dest_mac_addr = MacAddressFromStr(GetMACAddress()); + config.dest_ipv4_addr = IPv4AddressFromStr(GetIPv4Address()); + HW_RunInternalGenerator(config); +} \ No newline at end of file diff --git a/acquisition_device/AcquisitionDevice.h b/acquisition_device/AcquisitionDevice.h index cce6ac49..fc8177d7 100644 --- a/acquisition_device/AcquisitionDevice.h +++ b/acquisition_device/AcquisitionDevice.h @@ -14,11 +14,11 @@ #include "../common/ThreadSafeFIFO.h" #include "../jungfrau/JFCalibration.h" -#include "../fpga/pcie_driver/ActionConfig.h" #include "AcquisitionCounters.h" #include "Completion.h" -#include "../fpga/host_library/DeviceOutput.h" +#include "../fpga/include/jfjoch_fpga.h" +#include "../common/NetworkAddressConvert.h" struct AcquisitionDeviceStatistics { uint64_t good_packets; @@ -34,8 +34,6 @@ struct AcquisitionDeviceStatistics { enum class AcquisitionDeviceSource {MAC_100G, MAC_4x10G, FRAME_GENERATOR, NONE}; class AcquisitionDevice { - std::vector buffer_err; - std::chrono::time_point start_time; std::chrono::time_point end_time; @@ -58,13 +56,13 @@ protected: uint32_t max_modules = 1; uint64_t mac_addr; uint32_t ipv4_addr; - std::atomic work_completion_count; explicit AcquisitionDevice(uint16_t data_stream); void UnmapBuffers(); - void MapBuffersStandard(size_t c2h_buffer_count, size_t h2c_buffer_count, int16_t numa_node); + void MapBuffersStandard(size_t c2h_buffer_count, int16_t numa_node); const DeviceOutput *GetDeviceOutput(size_t handle) const; DeviceOutput *GetDeviceOutput(size_t handle); + virtual void HW_RunInternalGenerator(const FrameGeneratorConfig& config) = 0; public: AcquisitionDevice(const AcquisitionDevice &other) = delete; AcquisitionDevice &operator=(const AcquisitionDevice &other) = delete; @@ -89,9 +87,7 @@ public: AcquisitionDeviceStatistics GetStatistics() const; const DeviceOutput *GetDeviceOutput(size_t frame_number, uint16_t module_number) const; const DeviceOutput *GetDeviceOutputPedestal(size_t frame_number, uint16_t module_number) const; - void FrameBufferRelease(size_t frame_number, uint16_t module_number); - const int16_t *GetErrorFrameBuffer() const; // Calibration virtual void InitializeCalibration(const DiffractionExperiment &experiment, const JFCalibration &calib); @@ -100,7 +96,7 @@ public: const std::vector &weights); const AcquisitionCounters& Counters() const; - virtual void SetSpotFinderParameters(int16_t count_threshold, double snr_threshold) {}; + virtual void SetSpotFinderParameters(int32_t count_threshold, float snr_threshold) {}; virtual std::string GetIPv4Address() const; virtual void SetIPv4Address(uint32_t ipv4_addr_network_order); virtual void SetMACAddress(uint64_t mac_addr_network_order); @@ -116,6 +112,7 @@ public: AcquisitionDeviceNetConfig GetNetConfig() const; virtual uint32_t GetExpectedDescriptorsPerModule() const = 0; + void RunInternalGenerator(const DiffractionExperiment& experiment); }; diff --git a/acquisition_device/AcquisitionDeviceGroup.cpp b/acquisition_device/AcquisitionDeviceGroup.cpp index 5a0178b4..86d363fb 100644 --- a/acquisition_device/AcquisitionDeviceGroup.cpp +++ b/acquisition_device/AcquisitionDeviceGroup.cpp @@ -33,4 +33,14 @@ std::vector AcquisitionDeviceGroup::GetNetworkConfig for (const auto &i: aq_devices) ret.push_back(i->GetNetConfig()); return ret; +} + +void AcquisitionDeviceGroup::SetDefaultDataSource(AcquisitionDeviceSource id) { + for (auto &i: aq_devices) + i->SetDefaultDataSource(id); +} + +void AcquisitionDeviceGroup::EnableLogging(Logger *logger) { + for (auto &i: aq_devices) + i->EnableLogging(logger); } \ No newline at end of file diff --git a/acquisition_device/AcquisitionDeviceGroup.h b/acquisition_device/AcquisitionDeviceGroup.h index 5040c0c7..a528c0a1 100644 --- a/acquisition_device/AcquisitionDeviceGroup.h +++ b/acquisition_device/AcquisitionDeviceGroup.h @@ -15,8 +15,9 @@ public: void Add(std::unique_ptr &&device); void AddPCIeDevice(const std::string &device_name); void AddHLSDevice(int64_t buffer_size_modules); - void AddMockDevice(int64_t buffer_size_modules); std::vector GetNetworkConfig(); + void SetDefaultDataSource(AcquisitionDeviceSource id); + void EnableLogging(Logger *logger); }; #endif //JUNGFRAUJOCH_ACQUISITIONDEVICEGROUP_H diff --git a/acquisition_device/CMakeLists.txt b/acquisition_device/CMakeLists.txt index f0d51fba..a1d99752 100644 --- a/acquisition_device/CMakeLists.txt +++ b/acquisition_device/CMakeLists.txt @@ -2,7 +2,7 @@ ADD_LIBRARY(JungfraujochAcqusitionDevice STATIC AcquisitionDevice.cpp AcquisitionDevice.h AcquisitionCounters.cpp AcquisitionCounters.h HLSSimulatedDevice.cpp HLSSimulatedDevice.h - Completion.cpp Completion.h ../fpga/pcie_driver/ActionConfig.h + Completion.cpp Completion.h ../fpga/include/jfjoch_fpga.h PCIExpressDevice.cpp PCIExpressDevice.h FPGAAcquisitionDevice.cpp FPGAAcquisitionDevice.h AcquisitionDeviceGroup.cpp diff --git a/acquisition_device/FPGAAcquisitionDevice.cpp b/acquisition_device/FPGAAcquisitionDevice.cpp index 84757505..59a7dace 100644 --- a/acquisition_device/FPGAAcquisitionDevice.cpp +++ b/acquisition_device/FPGAAcquisitionDevice.cpp @@ -1,9 +1,7 @@ // Copyright (2019-2023) Paul Scherrer Institute -#include "FPGAAcquisitionDevice.h" #include -#include -#include "../common/to_fixed.h" +#include "FPGAAcquisitionDevice.h" void FPGAAcquisitionDevice::StartSendingWorkRequests() { stop_work_requests = false; @@ -27,26 +25,47 @@ void FPGAAcquisitionDevice::Finalize() { void FPGAAcquisitionDevice::ReadWorkCompletionThread() { uint32_t values; + if (logger) + logger->Debug("Starting read work completion thread for stream {}", data_stream); Completion c{}; bool quit_loop = false; do { - while (!HW_ReadMailbox(&values)) - std::this_thread::sleep_for(std::chrono::microseconds(10)); - - c = parse_hw_completion(values); - if (c.data_collection_id == data_collection_id) { - work_completion_queue.PutBlocking(c); - if (c.type == Completion::Type::End) - quit_loop = true; - } else if (logger) { - if (c.type == Completion::Type::Start) - logger->Warning("Stream {} Start completion with wrong data collection ID", data_stream); - else - logger->Warning("Stream {} Image completion with wrong data collection ID frame {} module {}", - data_stream, c.frame_number, c.module_number); + bool read; + try { + read = HW_ReadMailbox(&values); + } catch (const JFJochException& e) { + if (logger) + logger->ErrorException(e); + read = false; } + if (read) { + c = parse_hw_completion(values); + if (logger) { + logger->Debug("Stream {} Completion data collection ID {} type {} frame {} module {}", + data_stream, + c.data_collection_id, + (int)c.type, + c.frame_number, + c.module_number); + } + if (c.data_collection_id == data_collection_id) { + work_completion_queue.PutBlocking(c); + if (c.type == Completion::Type::End) + quit_loop = true; + } else if (logger) { + if (c.type == Completion::Type::Start) + logger->Warning("Stream {} Start completion with wrong data collection ID", data_stream); + else + logger->Warning("Stream {} Image completion with wrong data collection ID frame {} module {}", + data_stream, c.frame_number, c.module_number); + } + } else + std::this_thread::sleep_for(std::chrono::microseconds(10)); } while (!quit_loop); + + if (logger) + logger->Debug("Done read work completion thread for stream {}", data_stream); } void FPGAAcquisitionDevice::SendWorkRequestThread() { @@ -73,6 +92,7 @@ void FPGAAcquisitionDevice::InitializeIntegrationMap(const DiffractionExperiment const std::vector &v, const std::vector &weights) { auto offset = experiment.GetFirstModuleOfDataStream(data_stream); + size_t modules = experiment.GetModulesNum(data_stream); if (v.size() != experiment.GetModulesNum() * RAW_MODULE_SIZE) throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, @@ -82,22 +102,21 @@ void FPGAAcquisitionDevice::InitializeIntegrationMap(const DiffractionExperiment throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Mismatch regarding weights array"); - size_t modules = experiment.GetModulesNum(data_stream); - - if (modules > 2 * buffer_device.size()) - throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, - "Not enough host/FPGA buffers to load all integration map values"); - - for (int m = 0; m < modules; m++) - memcpy(buffer_device[m], v.data() + (offset + m) * RAW_MODULE_SIZE, RAW_MODULE_SIZE * sizeof(uint16_t)); - - - for (int m = 0; m < modules; m++) { - for (int i = 0; i < RAW_MODULE_SIZE; i++) { - buffer_device[modules + m]->pixels[i] = to_fixed(weights[(offset + m) * RAW_MODULE_SIZE + i], 15); - } + for (uint32_t m = 0; m < modules; m++) { + memcpy(buffer_device[0]->pixels, v.data() + (offset + m) * RAW_MODULE_SIZE, RAW_MODULE_SIZE * sizeof(uint16_t)); + buffer_device[0]->module_statistics.module_number = m; + buffer_device[0]->module_statistics.load_calibration_destination = LOAD_CALIBRATION_DEST_INTEGRATION_MAP; + LoadCalibration(0); + } + + for (uint32_t m = 0; m < modules; m++) { + for (int i = 0; i < RAW_MODULE_SIZE; i++) + ((float *)buffer_device[0]->pixels)[i] = weights[(offset + m) * RAW_MODULE_SIZE + i]; + + buffer_device[0]->module_statistics.module_number = m; + buffer_device[0]->module_statistics.load_calibration_destination = LOAD_CALIBRATION_DEST_INTEGRATION_WEIGHTS; + LoadCalibration(0); } - HW_LoadIntegrationMap(modules); } void FPGAAcquisitionDevice::SetInternalGeneratorFrameForAllModules(const std::vector &v) { @@ -111,14 +130,13 @@ void FPGAAcquisitionDevice::SetInternalGeneratorFrameForAllModules(const std::ve v.data(), RAW_MODULE_SIZE * sizeof(uint16_t)); } - if (max_modules > buffer_device.size()) - throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, - "Not enough host/FPGA buffers to load all integration map values"); + memcpy(buffer_device[0]->pixels, internal_pkt_gen_frame.data(), RAW_MODULE_SIZE * sizeof(uint16_t)); - for (int m = 0; m < max_modules; m++) - memcpy(buffer_device[m], internal_pkt_gen_frame.data() + m * RAW_MODULE_SIZE, RAW_MODULE_SIZE * sizeof(uint16_t)); - - HW_LoadInternalGeneratorFrame(max_modules); + buffer_device[0]->module_statistics.load_calibration_destination = LOAD_CALIBRATION_DEST_FRAME_GEN; + for (uint32_t m = 0; m < max_modules; m++) { + buffer_device[0]->module_statistics.module_number = m; + LoadCalibration(0); + } } @@ -140,10 +158,13 @@ void FPGAAcquisitionDevice::SetInternalGeneratorFrame(const std::vectormodule_statistics.module_number = m; + buffer_device[0]->module_statistics.load_calibration_destination = LOAD_CALIBRATION_DEST_FRAME_GEN; + LoadCalibration(0); + } } void FPGAAcquisitionDevice::InitializeCalibration(const DiffractionExperiment &experiment, const JFCalibration &calib) { @@ -160,58 +181,87 @@ void FPGAAcquisitionDevice::InitializeCalibration(const DiffractionExperiment &e size_t modules = experiment.GetModulesNum(data_stream); size_t storage_cells = experiment.GetStorageCellNumber(); - if (modules * (3 + 3 * storage_cells) > buffer_device.size()) - throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, - "Not enough host/FPGA buffers to load all calibration constants"); + for (uint32_t m = 0; m < modules; m++) { + if (experiment.IsFixedGainG1()) { + calib.GainCalibration(m).ExportFixedG1(buffer_device[0]); + buffer_device[0]->module_statistics.module_number = m; + buffer_device[0]->module_statistics.load_calibration_destination = LOAD_CALIBRATION_DEST_GAIN_G1; + LoadCalibration(0); + } else { + if (experiment.IsUsingGainHG0()) + calib.GainCalibration(m).ExportHG0(buffer_device[0]); + else + calib.GainCalibration(m).ExportG0(buffer_device[0]); + buffer_device[0]->module_statistics.module_number = m; + buffer_device[0]->module_statistics.load_calibration_destination = LOAD_CALIBRATION_DEST_GAIN_G0; + LoadCalibration(0); - for (int m = 0; m < modules; m++) { - calib.GainCalibration(m).ExportG0((uint16_t *) buffer_device[m]->pixels); - calib.GainCalibration(m).ExportG1((uint16_t *) buffer_device[m + modules]->pixels); - calib.GainCalibration(m).ExportG2((uint16_t *) buffer_device[m + modules * 2]->pixels); + calib.GainCalibration(m).ExportG1(buffer_device[0]); + buffer_device[0]->module_statistics.module_number = m; + buffer_device[0]->module_statistics.load_calibration_destination = LOAD_CALIBRATION_DEST_GAIN_G1; + LoadCalibration(0); + + calib.GainCalibration(m).ExportG2(buffer_device[0]); + buffer_device[0]->module_statistics.module_number = m; + buffer_device[0]->module_statistics.load_calibration_destination = LOAD_CALIBRATION_DEST_GAIN_G2; + LoadCalibration(0); + } } for (int s = 0; s < storage_cells; s++) { auto mask = calib.CalculateMask(experiment, s); for (int m = 0; m < modules; m++) { - auto pedestal_g0 = calib.Pedestal(offset + m, 0, s).GetPedestal(); - auto pedestal_g1 = calib.Pedestal(offset + m, 1, s).GetPedestal(); - auto pedestal_g2 = calib.Pedestal(offset + m, 2, s).GetPedestal(); - for (int i = 0; i < RAW_MODULE_SIZE; i++) { - if (experiment.GetApplyPixelMaskInFPGA() && (mask[(offset + m) * RAW_MODULE_SIZE + i] != 0)) { - buffer_device[(3 + 0 * storage_cells + s) * modules + m]->pixels[i] = 16384; - buffer_device[(3 + 1 * storage_cells + s) * modules + m]->pixels[i] = 16384; - buffer_device[(3 + 2 * storage_cells + s) * modules + m]->pixels[i] = 16384; - } else { - ((uint16_t *) buffer_device[(3 + 0 * storage_cells + s) * modules + m]->pixels)[i] = pedestal_g0[i]; - ((uint16_t *) buffer_device[(3 + 1 * storage_cells + s) * modules + m]->pixels)[i] = pedestal_g1[i]; - ((uint16_t *) buffer_device[(3 + 2 * storage_cells + s) * modules + m]->pixels)[i] = pedestal_g2[i]; - } - } + auto pedestal_g1 = calib.Pedestal(offset + m, 1, s).GetPedestal(); + for (int i = 0; i < RAW_MODULE_SIZE; i++) { + if (experiment.GetApplyPixelMaskInFPGA() && (mask[(offset + m) * RAW_MODULE_SIZE + i] != 0)) + buffer_device[0]->pixels[i] = 16384; + else + ((uint16_t *) buffer_device[0]->pixels)[i] = pedestal_g1[i]; + } + buffer_device[0]->module_statistics.module_number = m + modules * s; + buffer_device[0]->module_statistics.load_calibration_destination = LOAD_CALIBRATION_DEST_PEDESTAL_G1; + LoadCalibration(0); + + if (!experiment.IsFixedGainG1()) { + auto pedestal_g0 = calib.Pedestal(offset + m, 0, s).GetPedestal(); + for (int i = 0; i < RAW_MODULE_SIZE; i++) { + if (experiment.GetApplyPixelMaskInFPGA() && (mask[(offset + m) * RAW_MODULE_SIZE + i] != 0)) + buffer_device[0]->pixels[i] = 16384; + else + ((uint16_t *) buffer_device[0]->pixels)[i] = pedestal_g0[i]; + } + buffer_device[0]->module_statistics.module_number = m + modules * s; + buffer_device[0]->module_statistics.load_calibration_destination = LOAD_CALIBRATION_DEST_PEDESTAL_G0; + LoadCalibration(0); + + auto pedestal_g2 = calib.Pedestal(offset + m, 2, s).GetPedestal(); + for (int i = 0; i < RAW_MODULE_SIZE; i++) { + if (experiment.GetApplyPixelMaskInFPGA() && (mask[(offset + m) * RAW_MODULE_SIZE + i] != 0)) + buffer_device[0]->pixels[i] = 16384; + else + ((uint16_t *) buffer_device[0]->pixels)[i] = pedestal_g2[i]; + } + buffer_device[0]->module_statistics.module_number = m + modules * s; + buffer_device[0]->module_statistics.load_calibration_destination = LOAD_CALIBRATION_DEST_PEDESTAL_G2; + LoadCalibration(0); + } } } - HW_LoadCalibration(modules, storage_cells); -} - -union float_uint32 { - float f; - uint32_t u; -}; - -inline uint32_t float2uint(float f) { - float_uint32 fu; - fu.f = f; - return fu.u; } void FPGAAcquisitionDevice::FillActionRegister(const DiffractionExperiment& x, DataCollectionConfig &job) { std::random_device rd; std::uniform_int_distribution dist; + + // Avoid data_collection_id of DATA_COLLECTION_ID_PURGE - this is reserved for cleaning unused work requests from the FPGA data_collection_id = dist(rd); + while (data_collection_id == DATA_COLLECTION_ID_PURGE) + data_collection_id = dist(rd); job.nmodules = x.GetModulesNum(data_stream) - 1; job.nframes = x.GetFrameNum(); - job.one_over_energy = float2uint(1 / x.GetPhotonEnergy_keV()); + job.energy_kev = x.GetPhotonEnergy_keV(); job.nstorage_cells = x.GetStorageCellNumber() - 1; job.mode = data_collection_id << 16; job.nsummation = x.GetSummation() - 1; @@ -235,6 +285,9 @@ void FPGAAcquisitionDevice::FillActionRegister(const DiffractionExperiment& x, D break; } + if (x.IsFixedGainG1()) + job.mode |= MODE_FIXG1; + if (!x.IsPixelSigned()) job.mode |= MODE_UNSIGNED; if (x.GetPixelDepth() == 4) @@ -290,17 +343,17 @@ FPGAAcquisitionDevice::FPGAAcquisitionDevice(uint16_t data_stream) internal_pkt_gen_frame(RAW_MODULE_SIZE * MAX_MODULES_FPGA) { } -void FPGAAcquisitionDevice::SetSpotFinderParameters(int16_t count_threshold, double snr_threshold) { +void FPGAAcquisitionDevice::SetSpotFinderParameters(int32_t count_threshold, float snr_threshold) { if (snr_threshold < 0) { if (logger) logger->Warning("Trying to set SNR threshold below zero: {}", snr_threshold ); snr_threshold = 0; - } else if (snr_threshold > 64) { + } else if (snr_threshold > 16) { if (logger) logger->Warning("Trying to set SNR threshold too high: {}", snr_threshold ); - snr_threshold = 64; + snr_threshold = 16; } - SpotFinderParameters params{.count_threshold = count_threshold, .snr_threshold = to_fixed(snr_threshold, 2)}; + SpotFinderParameters params{.count_threshold = count_threshold, .snr_threshold = snr_threshold}; HW_SetSpotFinderParameters(params); } @@ -342,3 +395,7 @@ void FPGAAcquisitionDevice::SetDataSource(AcquisitionDeviceSource id) { uint32_t FPGAAcquisitionDevice::GetExpectedDescriptorsPerModule() const { return expected_descriptors_per_module; } + +void FPGAAcquisitionDevice::LoadCalibration(uint32_t handle) { + HW_LoadCalibration(LoadCalibrationConfig{.handle = handle}); +} \ No newline at end of file diff --git a/acquisition_device/FPGAAcquisitionDevice.h b/acquisition_device/FPGAAcquisitionDevice.h index 6f0e7946..84c79ec6 100644 --- a/acquisition_device/FPGAAcquisitionDevice.h +++ b/acquisition_device/FPGAAcquisitionDevice.h @@ -4,7 +4,7 @@ #define JUNGFRAUJOCH_FPGAACQUISITIONDEVICE_H #include "AcquisitionDevice.h" -#include "../fpga/pcie_driver/ActionConfig.h" +#include "../fpga/include/jfjoch_fpga.h" class FPGAAcquisitionDevice : public AcquisitionDevice { uint16_t data_collection_id = 0; @@ -29,9 +29,8 @@ class FPGAAcquisitionDevice : public AcquisitionDevice { volatile bool stop_work_requests = false; void SendWorkRequestThread(); - virtual void HW_LoadCalibration(uint32_t modules, uint32_t storage_cells) = 0; - virtual void HW_LoadIntegrationMap(uint32_t modules) = 0; - virtual void HW_LoadInternalGeneratorFrame(uint32_t modules) = 0; + virtual void HW_LoadCalibration(const LoadCalibrationConfig &config) = 0; + void LoadCalibration(uint32_t handle); virtual bool HW_ReadMailbox(uint32_t *values) = 0; virtual bool HW_SendWorkRequest(uint32_t handle) = 0; virtual void HW_SetSpotFinderParameters(const SpotFinderParameters ¶ms) = 0; @@ -57,7 +56,7 @@ public: void SetInternalGeneratorFrameForAllModules(const std::vector &v); void SetInternalGeneratorFrame(); std::vector GetInternalGeneratorFrame() const override; - void SetSpotFinderParameters(int16_t count_threshold, double snr_threshold) override; + void SetSpotFinderParameters(int32_t count_threshold, float snr_threshold) override; AcquisitionDeviceSource GetDataSource() override; void SetDefaultDataSource(AcquisitionDeviceSource id) override; uint32_t GetExpectedDescriptorsPerModule() const override; diff --git a/acquisition_device/HLSSimulatedDevice.cpp b/acquisition_device/HLSSimulatedDevice.cpp index 2bd44099..e1a6b71e 100644 --- a/acquisition_device/HLSSimulatedDevice.cpp +++ b/acquisition_device/HLSSimulatedDevice.cpp @@ -42,8 +42,7 @@ HLSSimulatedDevice::HLSSimulatedDevice(uint16_t data_stream, size_t in_frame_buf max_modules = MAX_MODULES_FPGA; - MapBuffersStandard(in_frame_buffer_size_modules, - (3 + 3 * 16) * max_modules + 2, numa_node); + MapBuffersStandard(in_frame_buffer_size_modules, numa_node); auto in_mem_location32 = (uint32_t *) dma_address_table.data(); @@ -142,35 +141,40 @@ void HLSSimulatedDevice::FPGA_StartAction(const DiffractionExperiment &experimen if (action_thread.joinable()) action_thread.join(); + run_counter += 1; run_data_collection = 1; cancel_data_collection = 0; idle = false; + + while (!din_frame_generator.empty()) + din_frame_generator.read(); + datamover_out.ClearCompletedDescriptors(); - if (experiment.IsUsingInternalPacketGen()) { - data_source = STREAM_MERGE_SRC_FRAME_GEN; - auto ret = frame_generator(din_frame_generator, - hbm.data(), - hbm.data(), - hbm_if_size, - experiment.GetFrameNum() + DELAY_FRAMES_STOP_AND_QUIT + 1, - experiment.GetModulesNum(data_stream), - mac_addr, - mac_addr, - ipv4_addr, - ipv4_addr, - INT_PKT_GEN_BUNCHID, - INT_PKT_GEN_EXPTTIME, - INT_PKT_GEN_DEBUG, - cancel_data_collection); - if (ret) - throw JFJochException(JFJochExceptionCategory::AcquisitionDeviceError, - "Error running internal packet generator"); - } else { - data_source = STREAM_MERGE_SRC_100G; - } + action_thread = std::thread(&HLSSimulatedDevice::HLSMainThread, this ); } +void HLSSimulatedDevice::FrameGeneratorFuture(FrameGeneratorConfig config) { + frame_generator(din_frame_generator, + hbm.data(), + hbm.data(), + hbm_if_size, + config.frames, + config.modules, + mac_addr, + config.dest_mac_addr, + ipv4_addr, + config.dest_ipv4_addr, + config.bunchid, + config.exptime, + config.debug, + cancel_data_collection); +} + +void HLSSimulatedDevice::HW_RunInternalGenerator(const FrameGeneratorConfig &config) { + frame_generator_future = std::async(std::launch::async, &HLSSimulatedDevice::FrameGeneratorFuture, this, config); +} + void HLSSimulatedDevice::FPGA_EndAction() { if (action_thread.joinable()) action_thread.join(); @@ -189,10 +193,11 @@ bool HLSSimulatedDevice::HW_ReadMailbox(uint32_t *values) { values[0] = tmp; // equivalent to driver functionality if (ret) { + uint32_t data_collection_id = (values[0] >> 16) & 0xFFFF; uint32_t handle = values[0] & 0xFFFF; if (handle == HANDLE_START) completion_count = 0; - else if (handle != HANDLE_END) { + else if ((handle != HANDLE_END) && (data_collection_id != DATA_COLLECTION_ID_PURGE)) { completion_count++; while (completion_count * DMA_DESCRIPTORS_PER_MODULE > datamover_out.GetCompletedDescriptors()) std::this_thread::sleep_for(std::chrono::milliseconds(1)); @@ -215,6 +220,12 @@ bool HLSSimulatedDevice::HW_SendWorkRequest(uint32_t handle) { return true; } +inline uint32_t float2uint(float f) { + float_uint32 fu; + fu.f = f; + return fu.u; +} + void HLSSimulatedDevice::HLSMainThread() { uint64_t counter_hbm; uint64_t counter_host; @@ -273,85 +284,135 @@ void HLSSimulatedDevice::HLSMainThread() { hls::stream> spot_finder_result_1; hls::stream > udp_metadata; - ap_uint<1> idle_data_collection; + volatile ap_uint<1> idle_data_collection = 1; + ap_uint<1> load_to_hbm_idle; + ap_uint<1> save_to_hbm_idle; + ap_uint<1> integration_idle; + ap_uint<1> stream_conv_idle; + ap_uint<1> frame_summation_idle; - if (data_source == STREAM_MERGE_SRC_FRAME_GEN) { - while (!din_frame_generator.empty()) - stream_merge(din_eth, din_eth_4x10G, din_frame_generator, network0, data_source); - } else if (data_source == STREAM_MERGE_SRC_100G) { - while (!din_eth.empty()) - stream_merge(din_eth, din_eth_4x10G, din_frame_generator, network0, data_source); - } else if (data_source == STREAM_MERGE_SRC_4x10G) { - while (!din_eth_4x10G.empty()) - stream_merge(din_eth, din_eth_4x10G, din_frame_generator, network0, data_source); - } + volatile bool done = false; + volatile bool udp_done = false; // done AND udp_idle + volatile bool sls_done = false; // done AND sls_idle - while(!network0.empty()) - ethernet(network0, ip1, arp1, mac_addr, eth_packets, clear_counters); + // Sent gratuitous ARP message + arp(arp1, dout_eth, mac_addr, ipv4_addr, 1, 1); - while(!ip1.empty()) - ipv4(ip1, udp1, icmp1, ipv4_addr); + Logger logger_hls("HLS"); - arp(arp1, - dout_eth, - mac_addr, - ipv4_addr, - 1, run_data_collection); + volatile rcv_state_t state = RCV_INIT; + // Start data collection + data_collection_fsm(raw0, raw1, + addr0, addr1, + run_data_collection, + cancel_data_collection, + idle_data_collection, + cfg.mode, + float2uint(cfg.energy_kev), + cfg.nframes, + cfg.nmodules, + cfg.nstorage_cells, + cfg.nsummation , state); - while (!arp1.empty()) { - arp(arp1, - dout_eth, - mac_addr, - ipv4_addr, - 1, run_data_collection); - } + run_data_collection = 0; + data_collection_fsm(raw0, raw1, + addr0, addr1, + run_data_collection, + cancel_data_collection, + idle_data_collection, + cfg.mode, + float2uint(cfg.energy_kev), + cfg.nframes, + cfg.nmodules, + cfg.nstorage_cells, + cfg.nsummation, state); - // reset static counter - arp(arp1, - dout_eth, - mac_addr, - ipv4_addr, - 0, run_data_collection); + hls_cores.emplace_back([&] { + while (!udp_done) { + if (din_eth.empty() && din_eth_4x10G.empty() && din_frame_generator.empty()) + std::this_thread::sleep_for(std::chrono::microseconds(10)); + else + stream_merge(din_eth, din_eth_4x10G, din_frame_generator, network0, data_source); + } + logger_hls.Info("Stream_merge done"); + }); - while(!icmp1.empty()) - icmp(icmp1, dout_eth, icmp_packets, clear_counters); + hls_cores.emplace_back([&] { + while (!udp_done) { + if (network0.empty()) + std::this_thread::sleep_for(std::chrono::microseconds(10)); + else + ethernet(network0, ip1, arp1, mac_addr, eth_packets, clear_counters); + } + logger_hls.Info("ethernet done"); + }); - while (!udp1.empty()) - udp(udp1, udp2, udp_metadata, udp_packets, clear_counters); + hls_cores.emplace_back([&] { + while (!udp_done) { + if (ip1.empty()) + std::this_thread::sleep_for(std::chrono::microseconds(10)); + else + ipv4(ip1, udp1, icmp1, ipv4_addr); + } + logger_hls.Info("ipv4 done"); + }); - while (!udp2.empty()) - sls_detector(udp2, udp_metadata, raw0, addr0, sls_packets, udp_eth_err, udp_len_err, clear_counters); + hls_cores.emplace_back([&] { + ap_uint<1> udp_idle = 1; + while (!done || !udp_idle) { + if (udp1.empty()) + std::this_thread::sleep_for(std::chrono::microseconds(10)); + else + udp(udp1, udp2, udp_metadata, udp_packets, clear_counters, udp_idle); + } + udp_done = true; + logger_hls.Info("udp done"); + }); + + hls_cores.emplace_back([&] { + ap_uint<1> sls_idle = 1; + while (!done || !sls_idle) { + if (udp2.empty()) + std::this_thread::sleep_for(std::chrono::microseconds (10)); + else + sls_detector(udp2, udp_metadata, raw0, addr0, sls_packets, udp_eth_err, udp_len_err, clear_counters, sls_idle); + } + sls_done = true; + logger_hls.Info("sls_detector done"); + }); // 1. Parse incoming UDP packets - idle_data_collection = 0; hls_cores.emplace_back([&] { - while ((idle_data_collection == 0) || (!raw0.empty())) { + while ((state != RCV_WAIT_FOR_START) || (idle_data_collection.read() == 0) || (!raw0.empty())) { data_collection_fsm(raw0, raw1, addr0, addr1, run_data_collection, cancel_data_collection, idle_data_collection, cfg.mode, - cfg.one_over_energy, + float2uint(cfg.energy_kev), cfg.nframes, cfg.nmodules, cfg.nstorage_cells, - cfg.nsummation); - run_data_collection = 0; + cfg.nsummation, + state); } + done = true; + + logger_hls.Info("data collection done"); }); // 2. Cache images in HBM hls_cores.emplace_back([&] { save_to_hbm(raw1, converted_1, addr1, compl0, hbm_handles, datamover_out_hbm_0.GetDataStream(), datamover_out_hbm_1.GetDataStream(), datamover_out_hbm_0.GetCtrlStream(), datamover_out_hbm_1.GetCtrlStream(), - hbm_if_size);}); + save_to_hbm_idle, hbm_if_size);}); hls_cores.emplace_back([&] {frame_summation_reorder_compl(converted_1, converted_2, compl0, compl1);}); hls_cores.emplace_back([&] { load_from_hbm(converted_2, converted_3, compl1, compl2, hbm_handles, datamover_in_hbm_0.GetDataStream(), datamover_in_hbm_1.GetDataStream(), datamover_in_hbm_0.GetCtrlStream(), datamover_in_hbm_1.GetCtrlStream(), - hbm_if_size);}); + load_to_hbm_idle, hbm_if_size);}); hls_cores.emplace_back([&] { pedestal(converted_3, converted_3a, compl2, compl2a, hbm.data(), @@ -393,126 +454,139 @@ void HLSSimulatedDevice::HLSMainThread() { hbm_if_size); }); // 6. Frame summation - hls_cores.emplace_back([&] { frame_summation(converted_6, stream_768_0, compl5, compl6);}); + hls_cores.emplace_back([&] { frame_summation(converted_6, stream_768_0, compl5, compl6, frame_summation_idle);}); // 7. Integration of pixels hls_cores.emplace_back([&] { integration(stream_768_0, stream_768_1, integration_result_0, compl6, compl7, - hbm.data(), hbm.data(), hbm.data(), hbm.data(), hbm_if_size);}); + hbm.data(), hbm.data(), hbm.data(), hbm.data(), integration_idle, hbm_if_size);}); hls_cores.emplace_back([&] { axis_64_to_512(integration_result_0, integration_result_1);}); // 8. Spot finding - hls_cores.emplace_back([&] { spot_finder(stream_768_1, stream_768_2, spot_finder_result_0, count_threshold, snr_threshold);}); + ap_uint<32> tmp_snr_threshold = float2uint(snr_threshold); + hls_cores.emplace_back([&] { spot_finder(stream_768_1, stream_768_2, spot_finder_result_0, + count_threshold,tmp_snr_threshold);}); hls_cores.emplace_back([&] { axis_32_to_512(spot_finder_result_0, spot_finder_result_1);}); // 9. Reduce/extend 24-bit stream - hls_cores.emplace_back([&] { stream_24bit_conv(stream_768_2, converted_7);}); + hls_cores.emplace_back([&] { stream_24bit_conv(stream_768_2, converted_7, stream_conv_idle);}); // 10. Prepare data to write to host memory hls_cores.emplace_back([&] { + ap_uint<3> state; host_writer(converted_7, adu_histo_result, integration_result_1, spot_finder_result_1, compl7, datamover_out.GetDataStream(), datamover_out.GetCtrlStream(), work_request_stream, completion_stream, - dma_address_table.data(), packets_processed, host_writer_idle);}); + dma_address_table.data(), packets_processed, host_writer_idle, cancel_data_collection, state);}); for (auto &i : hls_cores) i.join(); - if (!din_eth.empty()) - throw std::runtime_error("din_eth queue not empty"); + if (frame_generator_future.valid()) + frame_generator_future.get(); - if (!addr1.empty()) - throw std::runtime_error("Addr1 queue not empty"); + // reset static counter + arp(arp1, dout_eth, mac_addr, ipv4_addr, 0, 1); + try { + while (!din_eth.empty()) + din_eth.read(); - if (!addr2.empty()) - throw std::runtime_error("Addr2 queue not empty"); + if (!addr1.empty()) + throw std::runtime_error("Addr1 queue not empty"); - if (!addr3.empty()) - throw std::runtime_error("Addr3 queue not empty"); + if (!addr2.empty()) + throw std::runtime_error("Addr2 queue not empty"); - if (!raw1.empty()) - throw std::runtime_error("Raw1 queue not empty"); + if (!addr3.empty()) + throw std::runtime_error("Addr3 queue not empty"); - if (!raw2.empty()) - throw std::runtime_error("Raw2 queue not empty"); + if (!raw1.empty()) + throw std::runtime_error("Raw1 queue not empty"); - if (!raw3.empty()) - throw std::runtime_error("Raw3 queue not empty"); + if (!raw2.empty()) + throw std::runtime_error("Raw2 queue not empty"); - if (!converted_0.empty()) - throw std::runtime_error("Converted_0 queue not empty"); + if (!raw3.empty()) + throw std::runtime_error("Raw3 queue not empty"); - if (!converted_1.empty()) - throw std::runtime_error("Converted_1 queue not empty"); + if (!converted_0.empty()) + throw std::runtime_error("Converted_0 queue not empty"); - if (!converted_2.empty()) - throw std::runtime_error("Converted_2 queue not empty"); + if (!converted_1.empty()) + throw std::runtime_error("Converted_1 queue not empty"); - if (!converted_3.empty()) - throw std::runtime_error("Converted_3 queue not empty"); + if (!converted_2.empty()) + throw std::runtime_error("Converted_2 queue not empty"); - if (!converted_4.empty()) - throw std::runtime_error("Converted_4 queue not empty"); + if (!converted_3.empty()) + throw std::runtime_error("Converted_3 queue not empty"); - if (!converted_5.empty()) - throw std::runtime_error("Converted_5 queue not empty"); + if (!converted_4.empty()) + throw std::runtime_error("Converted_4 queue not empty"); - if (!converted_6.empty()) - throw std::runtime_error("Converted_6 queue not empty"); + if (!converted_5.empty()) + throw std::runtime_error("Converted_5 queue not empty"); - if (!converted_7.empty()) - throw std::runtime_error("Converted_7 queue not empty"); + if (!converted_6.empty()) + throw std::runtime_error("Converted_6 queue not empty"); - if (!converted_8.empty()) - throw std::runtime_error("Converted_8 queue not empty"); + if (!converted_7.empty()) + throw std::runtime_error("Converted_7 queue not empty"); - if (!compl0.empty()) - throw std::runtime_error("Compl0 queue not empty"); + if (!converted_8.empty()) + throw std::runtime_error("Converted_8 queue not empty"); - if (!compl1.empty()) - throw std::runtime_error("Compl1 queue not empty"); + if (!compl0.empty()) + throw std::runtime_error("Compl0 queue not empty"); - if (!compl2.empty()) - throw std::runtime_error("Compl2 queue not empty"); + if (!compl1.empty()) + throw std::runtime_error("Compl1 queue not empty"); - if (!compl3.empty()) - throw std::runtime_error("Compl3 queue not empty"); + if (!compl2.empty()) + throw std::runtime_error("Compl2 queue not empty"); - if (!compl4.empty()) - throw std::runtime_error("Compl4 queue not empty"); + if (!compl3.empty()) + throw std::runtime_error("Compl3 queue not empty"); - if (!compl5.empty()) - throw std::runtime_error("Compl5 queue not empty"); + if (!compl4.empty()) + throw std::runtime_error("Compl4 queue not empty"); - if (!compl6.empty()) - throw std::runtime_error("Compl6 queue not empty"); + if (!compl5.empty()) + throw std::runtime_error("Compl5 queue not empty"); - if (!compl7.empty()) - throw std::runtime_error("Compl7 queue not empty"); + if (!compl6.empty()) + throw std::runtime_error("Compl6 queue not empty"); - if (!stream_768_0.empty()) - throw std::runtime_error("Stream_768_0 queue not empty"); + if (!compl7.empty()) + throw std::runtime_error("Compl7 queue not empty"); - if (!stream_768_1.empty()) - throw std::runtime_error("Stream_768_1 queue not empty"); + if (!stream_768_0.empty()) + throw std::runtime_error("Stream_768_0 queue not empty"); - if (!stream_768_2.empty()) - throw std::runtime_error("Stream_768_2 queue not empty"); + if (!stream_768_1.empty()) + throw std::runtime_error("Stream_768_1 queue not empty"); - if (!hbm_handles.empty()) - throw std::runtime_error("Handles queue not empty"); + if (!stream_768_2.empty()) + throw std::runtime_error("Stream_768_2 queue not empty"); - if (!integration_result_0.empty()) - throw std::runtime_error("Integration result queue not empty"); + if (!hbm_handles.empty()) + throw std::runtime_error("Handles queue not empty"); - if (!integration_result_1.empty()) - throw std::runtime_error("Integration result queue not empty"); + if (!integration_result_0.empty()) + throw std::runtime_error("Integration result queue not empty"); - if (!datamover_in.GetDataStream().empty()) - throw std::runtime_error("Datamover queue is not empty"); + if (!integration_result_1.empty()) + throw std::runtime_error("Integration result queue not empty"); - while (!datamover_out.IsIdle()) - std::this_thread::sleep_for(std::chrono::milliseconds(100)); + if (!datamover_in.GetDataStream().empty()) + throw std::runtime_error("Datamover queue is not empty"); + while (!datamover_out.IsIdle()) + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } catch (const std::runtime_error &e) { + if (logger) + logger->ErrorException(e); + throw e; + } if (logger) logger->Info("Packets Eth {} UDP {} SLS {} Proc {}", eth_packets, udp_packets, sls_packets, packets_processed); @@ -525,77 +599,19 @@ void HLSSimulatedDevice::HW_GetStatus(DataCollectionStatus *status) const { status->ctrl_reg = ap_uint<1>(host_writer_idle) ? (1 << 4) : 0; status->max_modules = max_modules; status->hbm_size_bytes = hbm_if_size; + status->run_counter = run_counter; } -void HLSSimulatedDevice::HW_LoadCalibration(uint32_t modules, uint32_t storage_cells) { - - if (logger) - logger->Info("Load calibration start"); - - int ret = load_calibration(hbm.data(), - hbm.data(), - modules, - storage_cells, +void HLSSimulatedDevice::HW_LoadCalibration(const LoadCalibrationConfig &config) { + int ret = load_calibration(hbm.data(), hbm.data(), + config, hbm_if_size, - LOAD_CALIBRATION_DEST_CALIB, datamover_in.GetCtrlStream(), datamover_in.GetDataStream(), dma_address_table.data()); if (ret) throw JFJochException(JFJochExceptionCategory::AcquisitionDeviceError, - "Error in loading calibration"); - if (logger) - logger->Info("Load calibration done"); - - if (!datamover_in.GetDataStream().empty()) - throw std::runtime_error("Datamover queue is not empty"); -} - -void HLSSimulatedDevice::HW_LoadIntegrationMap(uint32_t modules) { - if (logger) - logger->Info("Load calibration start"); - - int ret = load_calibration(hbm.data(), - hbm.data(), - modules, - 0, - hbm_if_size, - LOAD_CALIBRATION_DEST_INTEGRATION, - datamover_in.GetCtrlStream(), - datamover_in.GetDataStream(), - dma_address_table.data()); - - if (ret) - throw JFJochException(JFJochExceptionCategory::AcquisitionDeviceError, - "Error in loading calibration"); - - if (logger) - logger->Info("Load integration_map"); - - if (!datamover_in.GetDataStream().empty()) - throw std::runtime_error("Datamover queue is not empty"); -} - -void HLSSimulatedDevice::HW_LoadInternalGeneratorFrame(uint32_t modules) { - if (logger) - logger->Info("Load to frame generator"); - - int ret = load_calibration(hbm.data(), - hbm.data(), - modules, - 0, - hbm_if_size, - LOAD_CALIBRATION_DEST_FRAME_GEN, - datamover_in.GetCtrlStream(), - datamover_in.GetDataStream(), - dma_address_table.data()); - - if (ret) - throw JFJochException(JFJochExceptionCategory::AcquisitionDeviceError, - "Error in loading calibration"); - - if (logger) - logger->Info("Load to frame generator done"); + "Error in loading calibration " + std::to_string(ret)); if (!datamover_in.GetDataStream().empty()) throw std::runtime_error("Datamover queue is not empty"); @@ -612,4 +628,4 @@ void HLSSimulatedDevice::HW_SetDataSource(uint32_t val) { uint32_t HLSSimulatedDevice::HW_GetDataSource() { return data_source; -} \ No newline at end of file +} diff --git a/acquisition_device/HLSSimulatedDevice.h b/acquisition_device/HLSSimulatedDevice.h index 358457e1..02ebd3ec 100644 --- a/acquisition_device/HLSSimulatedDevice.h +++ b/acquisition_device/HLSSimulatedDevice.h @@ -19,9 +19,11 @@ class HLSSimulatedDevice : public FPGAAcquisitionDevice { AXI_STREAM din_frame_generator; AXI_STREAM dout_eth; + uint32_t run_counter = 0; + ap_uint<2> data_source = STREAM_MERGE_SRC_100G; - ap_int<16> count_threshold = INT16_MAX; - ap_uint<8> snr_threshold = 0; + ap_int<32> count_threshold = -1; + float snr_threshold = 0; DataCollectionConfig cfg; @@ -37,6 +39,7 @@ class HLSSimulatedDevice : public FPGAAcquisitionDevice { uint32_t completion_count; std::thread action_thread; + std::future frame_generator_future; Datamover<512> datamover_in; Datamover<512> datamover_out; @@ -59,14 +62,14 @@ class HLSSimulatedDevice : public FPGAAcquisitionDevice { bool HW_IsIdle() const override; bool HW_ReadMailbox(uint32_t *values) override; bool HW_SendWorkRequest(uint32_t handle) override; - void HW_LoadCalibration(uint32_t modules, uint32_t storage_cells) override; - void HW_LoadIntegrationMap(uint32_t modules) override; - void HW_LoadInternalGeneratorFrame(uint32_t modules) override; + void HW_LoadCalibration(const LoadCalibrationConfig &config) override; void HW_GetStatus(DataCollectionStatus *status) const override; void HW_SetSpotFinderParameters(const SpotFinderParameters ¶ms) override; uint32_t HW_GetDataSource() override; void HW_SetDataSource(uint32_t val) override; - void HLSMainThread() ; + void HW_RunInternalGenerator(const FrameGeneratorConfig &config) override; + void FrameGeneratorFuture(FrameGeneratorConfig config); + void HLSMainThread(); public: HLSSimulatedDevice(uint16_t data_stream, size_t in_frame_buffer_size_modules, int16_t numa_node = -1); ~HLSSimulatedDevice() override; diff --git a/acquisition_device/PCIExpressDevice.cpp b/acquisition_device/PCIExpressDevice.cpp index e9e75dcc..7f5dc257 100644 --- a/acquisition_device/PCIExpressDevice.cpp +++ b/acquisition_device/PCIExpressDevice.cpp @@ -44,15 +44,10 @@ bool PCIExpressDevice::HW_SendWorkRequest(uint32_t handle) { void PCIExpressDevice::FPGA_StartAction(const DiffractionExperiment &experiment) { dev.Start(); - if (experiment.IsUsingInternalPacketGen()) { - FrameGeneratorConfig config{}; - config.frames = experiment.GetFrameNum() + DELAY_FRAMES_STOP_AND_QUIT + 1; - config.modules = experiment.GetModulesNum(data_stream); +} - config.dest_mac_addr = dev.GetMACAddress(); - config.dest_ipv4_addr = dev.GetIPv4Address(); - dev.RunFrameGenerator(config); - } +void PCIExpressDevice::HW_RunInternalGenerator(const FrameGeneratorConfig &config) { + dev.RunFrameGenerator(config); } void PCIExpressDevice::FPGA_EndAction() { @@ -107,18 +102,6 @@ std::string PCIExpressDevice::GetIPv4Address() const { return IPv4AddressToStr(dev.GetIPv4Address()); } -void PCIExpressDevice::HW_LoadCalibration(uint32_t in_modules, uint32_t in_storage_cells) { - dev.LoadCalibration(in_modules, in_storage_cells); -} - -void PCIExpressDevice::HW_LoadIntegrationMap(uint32_t in_modules) { - dev.LoadIntegrationMap(in_modules); -} - -void PCIExpressDevice::HW_LoadInternalGeneratorFrame(uint32_t in_modules) { - dev.LoadInternalGeneratorFrame(in_modules); -} - void PCIExpressDevice::HW_SetSpotFinderParameters(const SpotFinderParameters ¶ms) { dev.SetSpotFinderParameters(params); } @@ -139,3 +122,7 @@ void PCIExpressDevice::HW_SetDataSource(uint32_t val) { return dev.SetDataSource(val); } +void PCIExpressDevice::HW_LoadCalibration(const LoadCalibrationConfig &config) { + dev.LoadCalibration(config); +} + diff --git a/acquisition_device/PCIExpressDevice.h b/acquisition_device/PCIExpressDevice.h index ef6e0e23..5cb3711d 100644 --- a/acquisition_device/PCIExpressDevice.h +++ b/acquisition_device/PCIExpressDevice.h @@ -9,20 +9,19 @@ class PCIExpressDevice : public FPGAAcquisitionDevice { JungfraujochDevice dev; + void HW_LoadCalibration(const LoadCalibrationConfig &config) override; bool HW_ReadMailbox(uint32_t *values) override; bool HW_SendWorkRequest(uint32_t handle) override; void FPGA_StartAction(const DiffractionExperiment &experiment) override; bool HW_IsIdle() const final; void HW_WriteActionRegister(const DataCollectionConfig *job) override; void HW_ReadActionRegister(DataCollectionConfig *job) override; - void HW_LoadCalibration(uint32_t modules, uint32_t storage_cells) override; - void HW_LoadIntegrationMap(uint32_t modules) override; - void HW_LoadInternalGeneratorFrame(uint32_t modules) override; void HW_SetSpotFinderParameters(const SpotFinderParameters ¶ms) override; uint32_t HW_GetDataSource() override; void HW_SetDataSource(uint32_t val) override; void FPGA_EndAction() override; uint32_t GetNumKernelBuffers() const; + void HW_RunInternalGenerator(const FrameGeneratorConfig &config) override; public: explicit PCIExpressDevice(uint16_t data_stream); PCIExpressDevice(uint16_t data_stream, uint16_t pci_slot); diff --git a/broker/CMakeLists.txt b/broker/CMakeLists.txt index c419de6b..6cc36bdb 100644 --- a/broker/CMakeLists.txt +++ b/broker/CMakeLists.txt @@ -1,5 +1,3 @@ -ADD_SUBDIRECTORY(pistache) - AUX_SOURCE_DIRECTORY(gen/model MODEL_SOURCES) ADD_LIBRARY(JFJochAPI STATIC ${MODEL_SOURCES} gen/api/DefaultApi.cpp gen/api/DefaultApi.h) diff --git a/broker/JFJochBrokerHttp.cpp b/broker/JFJochBrokerHttp.cpp index a3a9a682..1b4c248b 100644 --- a/broker/JFJochBrokerHttp.cpp +++ b/broker/JFJochBrokerHttp.cpp @@ -38,22 +38,29 @@ inline org::openapitools::server::model::Spot_finding_settings Convert(const Spo inline org::openapitools::server::model::Measurement_statistics Convert(const MeasurementStatistics &input) { org::openapitools::server::model::Measurement_statistics ret{}; - ret.setFilePrefix(input.file_prefix); + if (!input.file_prefix.empty()) + ret.setFilePrefix(input.file_prefix); + + ret.setImagesExpected(input.images_expected); ret.setImagesCollected(input.images_collected); + ret.setImagesSent(input.images_sent); ret.setMaxImageNumberSent(input.max_image_number_sent); - ret.setCollectionEfficiency(input.collection_efficiency); + if (input.collection_efficiency >= 0.0) + ret.setCollectionEfficiency(input.collection_efficiency); ret.setCompressionRatio(input.compression_ratio); ret.setCancelled(input.cancelled); ret.setMaxReceiverDelay(input.max_receive_delay); - ret.setIndexingRate(input.indexing_rate); - ret.setDetectorWidth(input.detector_width); ret.setDetectorHeight(input.detector_height); ret.setDetectorPixelDepth(input.detector_pixel_depth); - ret.setBkgEstimate(input.bkg_estimate); + if (input.indexing_rate >= 0.0) + ret.setIndexingRate(input.indexing_rate); + + if (input.bkg_estimate >= 0.0) + ret.setBkgEstimate(input.bkg_estimate); return ret; } @@ -69,6 +76,8 @@ inline DetectorSettings Convert(const org::openapitools::server::model::Detector ret.use_internal_packet_generator = input.isInternalFrameGenerator(); ret.collect_raw_data = input.isCollectRawData(); + ret.fixed_gain_g1 = input.isFixedGainG1(); + ret.use_gain_hg0 = input.isUseGainHg0(); if (input.pedestalG0FramesIsSet()) ret.pedestal_g0_frames = input.getPedestalG0Frames(); @@ -91,7 +100,8 @@ inline org::openapitools::server::model::Detector_settings Convert(const Detecto ret.setStorageCellCount(input.storage_cell_count); ret.setInternalFrameGenerator(input.use_internal_packet_generator); ret.setCollectRawData(input.collect_raw_data); - + ret.setFixedGainG1(input.fixed_gain_g1); + ret.setUseGainHg0(input.use_gain_hg0); ret.setPedestalG0Frames(input.pedestal_g0_frames.value()); ret.setPedestalG1Frames(input.pedestal_g1_frames.value()); ret.setPedestalG2Frames(input.pedestal_g2_frames.value()); @@ -125,10 +135,8 @@ inline org::openapitools::server::model::Broker_status Convert(const BrokerStatu } if (input.progress.has_value()) ret.setProgress(input.progress.value()); - if (input.indexing_rate.has_value()) + if (input.indexing_rate.has_value() && (input.indexing_rate >= 0.0)) ret.setIndexingRate(input.indexing_rate.value()); - if (input.receiver_send_buffers_avail.has_value()) - ret.setReceiverSendBuffersAvail(input.receiver_send_buffers_avail.value()); return ret; } @@ -274,22 +282,32 @@ inline DatasetSettings Convert(const org::openapitools::server::model::Dataset_s if (input.transmissionIsSet()) ret.attenuator_transmission = input.getTransmission(); + if (input.omegaIsSet()) { + ret.omega_step = input.getOmega().getStep(); + ret.omega_start = input.getOmega().getStart(); + if (input.getOmega().getVector().size() == 3) { + auto v = input.getOmega().getVector(); + ret.omega_axis = Coord(v[0], v[1], v[2]); + } else + ret.omega_axis = Coord(0, 0, 0); + } + + if (input.roiSumAreaIsSet()) { + ret.roi_sum_area = ROIRectangle{ + .x_min = static_cast(input.getRoiSumArea().getXMin()), + .x_max = static_cast(input.getRoiSumArea().getXMax()), + .y_min = static_cast(input.getRoiSumArea().getYMin()), + .y_max = static_cast(input.getRoiSumArea().getYMax()) + }; + } ret.space_group_number = input.spaceGroupNumberIsSet() ? input.getSpaceGroupNumber() : 0; ret.sample_name = input.getSampleName(); - + ret.header_appendix = input.getHeaderAppendix(); + ret.image_appendix = input.getImageAppendix(); ret.save_calibration = input.saveCalibrationIsSet() && input.isSaveCalibration(); return ret; } -template -void ProcessOutput(const T& output, Pistache::Http::ResponseWriter &response) { - std::stringstream s; - assert(output.validate(s)); - - nlohmann::json j; - to_json(j, output); - response.send(Pistache::Http::Code::Ok, j.dump(), MIME(Application, Json)); -} JFJochBrokerHttp::JFJochBrokerHttp(const DiffractionExperiment &experiment, std::shared_ptr &rtr) : DefaultApi(rtr) { @@ -314,6 +332,7 @@ JFJochServices &JFJochBrokerHttp::Services() { void JFJochBrokerHttp::cancel_post(Pistache::Http::ResponseWriter &response) { state_machine.Cancel(); + response.send(Pistache::Http::Code::Ok); } void JFJochBrokerHttp::deactivate_post(Pistache::Http::ResponseWriter &response) { @@ -408,8 +427,11 @@ void JFJochBrokerHttp::config_spot_finding_put( response.send(Pistache::Http::Code::Ok); } -void JFJochBrokerHttp::plot_adu_histogram_get(Pistache::Http::ResponseWriter &response) { - PlotRequest req{.type = PlotType::ADUHistorgram}; +void JFJochBrokerHttp::plot_saturated_pixel_post(const org::openapitools::server::model::Plot_request &plotRequest, + Pistache::Http::ResponseWriter &response) { + PlotRequest req{.type = PlotType::SaturatedPixels, .binning = 0}; + if (plotRequest.binningIsSet()) + req.binning = plotRequest.getBinning(); auto plot = state_machine.GetPlots(req); ProcessOutput(Convert(plot), response); } @@ -539,4 +561,49 @@ void JFJochBrokerHttp::GetStaticFile(const Pistache::Rest::Request &request, Pis JFJochBrokerHttp &JFJochBrokerHttp::FrontendDirectory(const std::string &directory) { frontend_directory = directory; return *this; +} + +void JFJochBrokerHttp::plot_error_pixel_post(const org::openapitools::server::model::Plot_request &plotRequest, + Pistache::Http::ResponseWriter &response) { + PlotRequest req{.type = PlotType::ErrorPixels, .binning = 0}; + if (plotRequest.binningIsSet()) + req.binning = plotRequest.getBinning(); + auto plot = state_machine.GetPlots(req); + ProcessOutput(Convert(plot), response); +} + +void JFJochBrokerHttp::plot_strong_pixel_post(const org::openapitools::server::model::Plot_request &plotRequest, + Pistache::Http::ResponseWriter &response) { + PlotRequest req{.type = PlotType::StrongPixels, .binning = 0}; + if (plotRequest.binningIsSet()) + req.binning = plotRequest.getBinning(); + auto plot = state_machine.GetPlots(req); + ProcessOutput(Convert(plot), response); +} + +void JFJochBrokerHttp::plot_image_collection_efficiency_post( + const org::openapitools::server::model::Plot_request &plotRequest, Pistache::Http::ResponseWriter &response) { + PlotRequest req{.type = PlotType::ImageCollectionEfficiency, .binning = 0}; + if (plotRequest.binningIsSet()) + req.binning = plotRequest.getBinning(); + auto plot = state_machine.GetPlots(req); + ProcessOutput(Convert(plot), response); +} + +void JFJochBrokerHttp::plot_receiver_delay_post(const org::openapitools::server::model::Plot_request &plotRequest, + Pistache::Http::ResponseWriter &response) { + PlotRequest req{.type = PlotType::ReceiverDelay, .binning = 0}; + if (plotRequest.binningIsSet()) + req.binning = plotRequest.getBinning(); + auto plot = state_machine.GetPlots(req); + ProcessOutput(Convert(plot), response); +} + +void JFJochBrokerHttp::plot_roi_sum_post(const org::openapitools::server::model::Plot_request &plotRequest, + Pistache::Http::ResponseWriter &response) { + PlotRequest req{.type = PlotType::ROISum, .binning = 0}; + if (plotRequest.binningIsSet()) + req.binning = plotRequest.getBinning(); + auto plot = state_machine.GetPlots(req); + ProcessOutput(Convert(plot), response); } \ No newline at end of file diff --git a/broker/JFJochBrokerHttp.h b/broker/JFJochBrokerHttp.h index 0440166e..f830d5c2 100644 --- a/broker/JFJochBrokerHttp.h +++ b/broker/JFJochBrokerHttp.h @@ -38,7 +38,21 @@ class JFJochBrokerHttp : public org::openapitools::server::api::DefaultApi { void config_spot_finding_put(const org::openapitools::server::model::Spot_finding_settings &spotFindingSettings, Pistache::Http::ResponseWriter &response) override; - void plot_adu_histogram_get(Pistache::Http::ResponseWriter &response) override; + void plot_roi_sum_post(const org::openapitools::server::model::Plot_request &plotRequest, + Pistache::Http::ResponseWriter &response) override; + + void plot_saturated_pixel_post(const org::openapitools::server::model::Plot_request &plotRequest, + Pistache::Http::ResponseWriter &response) override; + void plot_error_pixel_post(const org::openapitools::server::model::Plot_request &plotRequest, + Pistache::Http::ResponseWriter &response) override; + void plot_strong_pixel_post(const org::openapitools::server::model::Plot_request &plotRequest, + Pistache::Http::ResponseWriter &response) override; + + void plot_receiver_delay_post(const org::openapitools::server::model::Plot_request &plotRequest, + Pistache::Http::ResponseWriter &response) override; + + void plot_image_collection_efficiency_post(const org::openapitools::server::model::Plot_request &plotRequest, + Pistache::Http::ResponseWriter &response) override; void plot_bkg_estimate_post(const org::openapitools::server::model::Plot_request &plotRequest, Pistache::Http::ResponseWriter &response) override; @@ -77,6 +91,18 @@ class JFJochBrokerHttp : public org::openapitools::server::api::DefaultApi { void GetStaticFile(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response); std::pair handleOperationException(const std::exception &ex) const noexcept override; + template + void ProcessOutput(const T& output, Pistache::Http::ResponseWriter &response) { + std::stringstream s; + if(!output.validate(s)) { + logger.Error(s.str()); + response.send(Pistache::Http::Code::Internal_Server_Error, s.str(), MIME(Text, Plain)); + } + + nlohmann::json j; + to_json(j, output); + response.send(Pistache::Http::Code::Ok, j.dump(), MIME(Application, Json)); + } public: JFJochBrokerHttp(const DiffractionExperiment& experiment, std::shared_ptr &rtr); void AddDetectorSetup(const DetectorSetup &setup); diff --git a/broker/JFJochBrokerParser.cpp b/broker/JFJochBrokerParser.cpp index 00ebdd02..63f4d6d9 100644 --- a/broker/JFJochBrokerParser.cpp +++ b/broker/JFJochBrokerParser.cpp @@ -246,6 +246,16 @@ void ParseFacilityConfiguration(const nlohmann::json &input, const std::string& experiment.InstrumentName(GET_STR(j, "instrument_name")); experiment.InstrumentNameShort(GET_STR(j, "instrument_name_short")); + if (j.contains("omega_axis")) { + if (j["omega_axis"].is_array() && (j["omega_axis"].size() == 3)) + experiment.DefaultOmegaAxis(Coord(j["omega_axis"][0].get(), + j["omega_axis"][1].get(), + j["omega_axis"][2].get())); + else + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, + "omega_axis must be float array of 3"); + } + experiment.PedestalG0Frames(GET_I64(j, "pedestal_g0_frames")); experiment.PedestalG1Frames(GET_I64(j, "pedestal_g1_frames")); experiment.PedestalG2Frames(GET_I64(j, "pedestal_g2_frames")); @@ -264,7 +274,7 @@ void ParseAcquisitionDeviceGroup(const nlohmann::json &input, const std::string& const auto &j = input[tag]; std::string dev_type = GET_STR(j, "type"); - + bool use_4x10g = GET_BOOL(j, "use_4x10g", false); int64_t buffer_size = GET_I64(j, "buffer_size", 1024); int64_t dev_count = GET_I64(j, "count", 1); @@ -284,6 +294,11 @@ void ParseAcquisitionDeviceGroup(const nlohmann::json &input, const std::string& } else throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Device type unknown"); + if (use_4x10g) + aq_devices.SetDefaultDataSource(AcquisitionDeviceSource::MAC_4x10G); + else + aq_devices.SetDefaultDataSource(AcquisitionDeviceSource::MAC_100G); + std::vector ipv4_addr = GET_STR_ARR(j, "ipv4_addr"); if (ipv4_addr.size() != aq_devices.size()) throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, diff --git a/broker/JFJochBrokerParser.h b/broker/JFJochBrokerParser.h index e5b02de7..4fa7bd4d 100644 --- a/broker/JFJochBrokerParser.h +++ b/broker/JFJochBrokerParser.h @@ -7,7 +7,7 @@ #include "../common/DiffractionExperiment.h" #include "JFJochBrokerHttp.h" #include "../acquisition_device/AcquisitionDeviceGroup.h" -#include "../frame_serialize/ZMQPreviewPublisher.h" +#include "../frame_serialize/ZMQStream2PreviewPublisher.h" DetectorGeometry ParseStandardDetectorGeometry(const nlohmann::json &j); DetectorGeometry ParseCustomDetectorGeometry(const nlohmann::json &j); diff --git a/broker/JFJochServices.cpp b/broker/JFJochServices.cpp index a6b677e9..07293fe5 100644 --- a/broker/JFJochServices.cpp +++ b/broker/JFJochServices.cpp @@ -14,12 +14,13 @@ void JFJochServices::Start(const DiffractionExperiment& experiment, const JFCali receiver->Start(experiment, &calibration); else receiver->Start(experiment, nullptr); + + if (detector && !experiment.IsUsingInternalPacketGen()) { + logger.Info(" ... detector start"); + detector->Start(experiment); + } } - if (detector && !experiment.IsUsingInternalPacketGen()) { - logger.Info(" ... detector start"); - detector->Start(experiment); - } logger.Info(" Done!"); } @@ -37,7 +38,7 @@ void JFJochServices::On(const DiffractionExperiment &x) { logger.Info(" ... done"); } -JFJochServicesOutput JFJochServices::Stop(const JFCalibration &calibration) { +JFJochServicesOutput JFJochServices::Stop() { JFJochServicesOutput ret; std::unique_ptr exception; @@ -49,8 +50,8 @@ JFJochServicesOutput JFJochServices::Stop(const JFCalibration &calibration) { logger.Info(" ... Receiver efficiency: {} % Max delay: {} Compression ratio {}x", static_cast(ret.receiver_output.efficiency * 100.0), - ret.receiver_output.max_receive_delay, - static_cast(std::round(ret.receiver_output.compressed_ratio))); + ret.receiver_output.status.max_receive_delay, + static_cast(std::round(ret.receiver_output.status.compressed_ratio))); if (ret.receiver_output.efficiency < 1.0) { for (int i = 0; i < ret.receiver_output.received_packets.size(); i++) { if (ret.receiver_output.received_packets[i] != ret.receiver_output.expected_packets[i]) @@ -63,20 +64,25 @@ JFJochServicesOutput JFJochServices::Stop(const JFCalibration &calibration) { exception = std::make_unique(e); } logger.Info("Receiver finished with success"); - } - if (detector) { - logger.Info("Stopping detector"); - try { + if (detector) { + logger.Info("Stopping detector"); + try { - detector->Stop(); - logger.Info(" ... done"); - } catch (JFJochException &e) { - logger.Error(" ... finished with error {}", e.what()); - exception = std::make_unique(e); + detector->Stop(); + logger.Info(" ... done"); + } catch (JFJochException &e) { + logger.Error(" ... finished with error {}", e.what()); + exception = std::make_unique(e); + } } + } else { + logger.Info("No receiver - sleeping for 30 seconds"); + std::this_thread::sleep_for(std::chrono::seconds(30)); + logger.Info("Sleep done"); } + if (exception) throw JFJochException(*exception); @@ -84,10 +90,11 @@ JFJochServicesOutput JFJochServices::Stop(const JFCalibration &calibration) { } void JFJochServices::Cancel() { - if (detector) - detector->Stop(); - if (receiver != nullptr) + if (receiver != nullptr) { + if (detector) + detector->Stop(); receiver->Cancel(); + } } JFJochServices &JFJochServices::Receiver(JFJochReceiverService *input) { @@ -135,6 +142,6 @@ void JFJochServices::SetSpotFindingSettings(const SpotFindingSettings &settings) } void JFJochServices::Trigger() { - if (detector) + if (detector && (receiver != nullptr)) detector->Trigger(); } diff --git a/broker/JFJochServices.h b/broker/JFJochServices.h index 856c52c7..f9f12cc6 100644 --- a/broker/JFJochServices.h +++ b/broker/JFJochServices.h @@ -23,7 +23,7 @@ public: void On(const DiffractionExperiment& experiment); void Off(); void Start(const DiffractionExperiment& experiment, const JFCalibration &calibration); - JFJochServicesOutput Stop(const JFCalibration &calibration); + JFJochServicesOutput Stop(); void Cancel(); void Trigger(); diff --git a/broker/JFJochStateMachine.cpp b/broker/JFJochStateMachine.cpp index 6f4b2955..55f5edee 100644 --- a/broker/JFJochStateMachine.cpp +++ b/broker/JFJochStateMachine.cpp @@ -24,11 +24,20 @@ void LoadDatasetSettings(DiffractionExperiment& experiment, const DatasetSetting experiment.SampleName(settings.sample_name); experiment.Compression(settings.compression); experiment.SaveCalibration(settings.save_calibration); + experiment.OmegaStart(settings.omega_start).OmegaStep(settings.omega_step); + if (settings.omega_axis.Length() == 0.0f) + experiment.OmegaAxis(); + else + experiment.OmegaAxis(settings.omega_axis); + + experiment.ImageAppendix(settings.image_appendix); + experiment.HeaderAppendix(settings.header_appendix); if (settings.summation == 0) experiment.Summation(1); else experiment.Summation(settings.summation); experiment.FPGAOutputMode(settings.fpga_pixel_output); + experiment.ROISummation(settings.roi_sum_area); } catch (...) { experiment = tmp; throw; @@ -52,6 +61,9 @@ void ApplyDetectorSettings(DiffractionExperiment& experiment, const DetectorSett else experiment.Mode(DetectorMode::Conversion); + experiment.UsingGainHG0(settings.use_gain_hg0); + experiment.FixedGainG1(settings.fixed_gain_g1); + if (settings.pedestal_g0_frames) experiment.PedestalG0Frames(settings.pedestal_g0_frames.value()); if (settings.pedestal_g1_frames) @@ -76,9 +88,9 @@ void ApplyRadialIntegrationSettings(DiffractionExperiment& experiment, const Rad experiment.ApplyPolarizationCorr(false); experiment.ApplySolidAngleCorr(settings.solid_angle_correction); - experiment.LowQForRadialInt_recipA(settings.low_q_recipA); - experiment.HighQForRadialInt_recipA(settings.high_q_recipA); - experiment.QSpacingForRadialInt_recipA(settings.q_spacing); + experiment.LowQForAzimInt_recipA(settings.low_q_recipA); + experiment.HighQForAzimInt_recipA(settings.high_q_recipA); + experiment.QSpacingForAzimInt_recipA(settings.q_spacing); } catch (...) { experiment = tmp; throw; @@ -91,32 +103,36 @@ data_processing_settings(DiffractionExperiment::DefaultDataProcessingSettings()) } -void JFJochStateMachine::ImportPedestalG0(const JFJochReceiverOutput &receiver_output) { +bool JFJochStateMachine::ImportPedestalG0(const JFJochReceiverOutput &receiver_output) { if (receiver_output.pedestal_result.empty()) - return; + return false; if (receiver_output.pedestal_result.size() != experiment.GetModulesNum() * experiment.GetStorageCellNumber()) throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Mismatch in pedestal output"); + size_t gain_level = experiment.IsFixedGainG1() ? 1 : 0; + for (int s = 0; s < experiment.GetStorageCellNumber(); s++) { for (int module = 0; module < experiment.GetModulesNum(); module++) - calibration->Pedestal(module, 0, s) + calibration->Pedestal(module, gain_level, s) = receiver_output.pedestal_result[module + s * experiment.GetModulesNum()]; } SetCalibrationStatistics(calibration->GetModuleStatistics()); + return true; } -void JFJochStateMachine::ImportPedestal(const JFJochReceiverOutput &receiver_output, size_t gain_level, - size_t storage_cell) { +bool JFJochStateMachine::ImportPedestalG1G2(const JFJochReceiverOutput &receiver_output, size_t gain_level, + size_t storage_cell) { if (receiver_output.pedestal_result.empty()) - return; + return false; - if (receiver_output.pedestal_result.size() != experiment.GetModulesNum() * experiment.GetStorageCellNumber()) + if (receiver_output.pedestal_result.size() != experiment.GetModulesNum()) throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Mismatch in pedestal output"); for (int i = 0; i < receiver_output.pedestal_result.size(); i++) calibration->Pedestal(i, gain_level, storage_cell) = receiver_output.pedestal_result[i]; SetCalibrationStatistics(calibration->GetModuleStatistics()); + return true; } void JFJochStateMachine::TakePedestalInternalAll(std::unique_lock &ul) { @@ -135,10 +151,11 @@ void JFJochStateMachine::TakePedestalInternalAll(std::unique_lock &u try { TakePedestalInternalG0(ul); - - for (int i = 0; i < experiment.GetStorageCellNumber(); i++) { - TakePedestalInternalG1(ul, i); - TakePedestalInternalG2(ul, i); + if (!experiment.IsFixedGainG1()) { + for (int i = 0; i < experiment.GetStorageCellNumber(); i++) { + TakePedestalInternalG1(ul, i); + TakePedestalInternalG2(ul, i); + } } } catch (const std::exception &e) { logger.Error("Pedestal sequence error {}", e.what()); @@ -148,33 +165,45 @@ void JFJochStateMachine::TakePedestalInternalAll(std::unique_lock &u logger.Info("Pedestal sequence done"); } + void JFJochStateMachine::TakePedestalInternalG0(std::unique_lock &ul) { - state = JFJochState::Pedestal; DiffractionExperiment local_experiment(experiment); - local_experiment.Mode(DetectorMode::PedestalG0); + if (local_experiment.IsFixedGainG1()) + local_experiment.Mode(DetectorMode::PedestalG1); + else + local_experiment.Mode(DetectorMode::PedestalG0); if (local_experiment.GetStorageCellNumber() == 1) local_experiment.StorageCellStart(15); else local_experiment.StorageCellStart(0); - if (!cancel_sequence && (local_experiment.GetPedestalG0Frames() > 0)) { - services.Start(local_experiment, *calibration); - services.Trigger(); - ul.unlock(); - // Allow to cancel/abort during the pedestal data collection - // Must ensure that while state is Pedestal, nothing can take lock for longer time, to avoid deadlock - auto pedestal_output = services.Stop(*calibration); - ul.lock(); - - // SetFullMeasurementOutput(pedestal_output); - ImportPedestalG0(pedestal_output.receiver_output); + if (cancel_sequence) { + state = JFJochState::Inactive; + return; } - state = JFJochState::Idle; + if (local_experiment.GetPedestalG0Frames() == 0) { + state = JFJochState::Idle; + return; + } + + state = JFJochState::Pedestal; + services.Start(local_experiment, *calibration); + services.Trigger(); + ul.unlock(); + // Allow to cancel/abort during the pedestal data collection + // Must ensure that while state is Pedestal, nothing can take lock for longer time, to avoid deadlock + auto pedestal_output = services.Stop(); + ul.lock(); + + SetFullMeasurementOutput(pedestal_output); + if (ImportPedestalG0(pedestal_output.receiver_output)) + state = JFJochState::Idle; + else + state = JFJochState::Inactive; } void JFJochStateMachine::TakePedestalInternalG1(std::unique_lock &ul, int32_t storage_cell) { - state = JFJochState::Pedestal; DiffractionExperiment local_experiment(experiment); local_experiment.Mode(DetectorMode::PedestalG1); @@ -183,23 +212,35 @@ void JFJochStateMachine::TakePedestalInternalG1(std::unique_lock &ul else local_experiment.StorageCellStart(15); - if (!cancel_sequence && (local_experiment.GetPedestalG1Frames() > 0)) { - services.Start(local_experiment, *calibration); - services.Trigger(); - ul.unlock(); - // Allow to cancel/abort during the pedestal data collection - // Must ensure that while state is Pedestal, nothing can take lock for longer time, to avoid deadlock - auto pedestal_output = services.Stop(*calibration); - ul.lock(); - // SetFullMeasurementOutput(pedestal_output); - ImportPedestal(pedestal_output.receiver_output, 1, storage_cell); + if (cancel_sequence) { + state = JFJochState::Inactive; + return; } - state = JFJochState::Idle; + + if (local_experiment.GetPedestalG1Frames() == 0) { + state = JFJochState::Idle; + return; + } + + state = JFJochState::Pedestal; + services.Start(local_experiment, *calibration); + services.Trigger(); + ul.unlock(); + // Allow to cancel/abort during the pedestal data collection + // Must ensure that while state is Pedestal, nothing can take lock for longer time, to avoid deadlock + auto pedestal_output = services.Stop(); + ul.lock(); + + SetFullMeasurementOutput(pedestal_output); + if (ImportPedestalG1G2(pedestal_output.receiver_output, 1, storage_cell)) + state = JFJochState::Idle; + else + state = JFJochState::Inactive; + } void JFJochStateMachine::TakePedestalInternalG2(std::unique_lock &ul, int32_t storage_cell) { - state = JFJochState::Pedestal; DiffractionExperiment local_experiment(experiment); local_experiment.Mode(DetectorMode::PedestalG2); @@ -208,19 +249,30 @@ void JFJochStateMachine::TakePedestalInternalG2(std::unique_lock &ul else local_experiment.StorageCellStart(15); - if (!cancel_sequence && (local_experiment.GetPedestalG2Frames() > 0)) { - services.Start(local_experiment, *calibration); - services.Trigger(); - ul.unlock(); - // Allow to cancel/abort during the pedestal data collection - // Must ensure that while state is Pedestal, nothing can take lock for longer time, to avoid deadlock - auto pedestal_output = services.Stop(*calibration); - ul.lock(); - - // SetFullMeasurementOutput(pedestal_output); - ImportPedestal(pedestal_output.receiver_output, 2, storage_cell); + if (cancel_sequence) { + state = JFJochState::Inactive; + return; } - state = JFJochState::Idle; + + if (local_experiment.GetPedestalG2Frames() == 0) { + state = JFJochState::Idle; + return; + } + + state = JFJochState::Pedestal; + services.Start(local_experiment, *calibration); + services.Trigger(); + ul.unlock(); + // Allow to cancel/abort during the pedestal data collection + // Must ensure that while state is Pedestal, nothing can take lock for longer time, to avoid deadlock + auto pedestal_output = services.Stop(); + ul.lock(); + + SetFullMeasurementOutput(pedestal_output); + if (ImportPedestalG1G2(pedestal_output.receiver_output, 2, storage_cell)) + state = JFJochState::Idle; + else + state = JFJochState::Inactive; } void JFJochStateMachine::Initialize() { @@ -292,6 +344,8 @@ void JFJochStateMachine::Start(const DatasetSettings& settings) { else experiment.StorageCellStart(0); + experiment.IncrementSeriesID(); + try { state = JFJochState::Busy; services.SetSpotFindingSettings(GetSpotFindingSettings()); @@ -321,7 +375,7 @@ c.wait(ul, [&] { return !IsRunning(); }); void JFJochStateMachine::MeasurementThread() { try { - auto tmp_output = services.Stop(*calibration); + auto tmp_output = services.Stop(); SetFullMeasurementOutput(tmp_output); { std::unique_lock ul(m); @@ -377,15 +431,17 @@ void JFJochStateMachine::SetFullMeasurementOutput(const JFJochServicesOutput &ou tmp.detector_width = experiment.GetXPixelsNum(); tmp.detector_height = experiment.GetYPixelsNum(); tmp.detector_pixel_depth = experiment.GetPixelDepth(); + tmp.images_expected = experiment.GetImageNum(); - tmp.compression_ratio = output.receiver_output.compressed_ratio; + tmp.compression_ratio = output.receiver_output.status.compressed_ratio; tmp.collection_efficiency = output.receiver_output.efficiency; - tmp.images_collected = output.receiver_output.images_sent; - tmp.cancelled = output.receiver_output.cancelled; - tmp.max_image_number_sent = output.receiver_output.max_image_number_sent; - tmp.max_receive_delay = output.receiver_output.max_receive_delay; - tmp.indexing_rate = output.receiver_output.indexing_rate; - tmp.bkg_estimate = output.receiver_output.bkg_estimate; + tmp.images_collected = output.receiver_output.status.images_collected; + tmp.images_sent = output.receiver_output.status.images_sent; + tmp.cancelled = output.receiver_output.status.cancelled; + tmp.max_image_number_sent = output.receiver_output.status.max_image_number_sent; + tmp.max_receive_delay = output.receiver_output.status.max_receive_delay; + tmp.indexing_rate = output.receiver_output.status.indexing_rate; + tmp.bkg_estimate = output.receiver_output.status.bkg_estimate; measurement_statistics = tmp; } @@ -398,6 +454,7 @@ void JFJochStateMachine::ClearAndSetMeasurementStatistics() { tmp.detector_height = experiment.GetXPixelsNum(); tmp.detector_width = experiment.GetYPixelsNum(); tmp.detector_pixel_depth = experiment.GetPixelDepth(); + tmp.images_expected = experiment.GetImageNum(); measurement_statistics = tmp; } @@ -407,8 +464,31 @@ void JFJochStateMachine::ClearMeasurementStatistics() { } std::optional JFJochStateMachine::GetMeasurementStatistics() const { - std::unique_lock ul(last_receiver_output_mutex); - return measurement_statistics; + + auto rcv_status = services.GetReceiverStatus(); + if (rcv_status) { + MeasurementStatistics tmp; + + tmp.file_prefix = experiment.GetFilePrefix(); + tmp.detector_width = experiment.GetXPixelsNum(); + tmp.detector_height = experiment.GetYPixelsNum(); + tmp.detector_pixel_depth = experiment.GetPixelDepth(); + tmp.images_expected = experiment.GetImageNum(); + + tmp.compression_ratio = rcv_status->compressed_ratio; + tmp.collection_efficiency = -1.0; + tmp.images_collected = rcv_status->images_collected; + tmp.images_sent = rcv_status->images_sent; + tmp.cancelled = rcv_status->cancelled; + tmp.max_image_number_sent = rcv_status->max_image_number_sent; + tmp.max_receive_delay = rcv_status->max_receive_delay; + tmp.indexing_rate = rcv_status->indexing_rate; + tmp.bkg_estimate = rcv_status->bkg_estimate; + return tmp; + } else { + std::unique_lock ul(last_receiver_output_mutex); + return measurement_statistics; + } } std::vector JFJochStateMachine::GetCalibrationStatistics() const { @@ -434,6 +514,8 @@ DetectorSettings JFJochStateMachine::GetDetectorSettings() const { ret.pedestal_g1_frames = experiment.GetPedestalG1Frames(); ret.pedestal_g2_frames = experiment.GetPedestalG2Frames(); ret.storage_cell_delay_ns = experiment.GetStorageCellDelay().count(); + ret.fixed_gain_g1 = experiment.IsFixedGainG1(); + ret.use_gain_hg0 = experiment.IsUsingGainHG0(); return ret; } @@ -468,7 +550,6 @@ BrokerStatus JFJochStateMachine::GetStatus() const { if (rcv_status) { ret.progress = rcv_status.value().progress; ret.indexing_rate = rcv_status.value().indexing_rate; - ret.receiver_send_buffers_avail = rcv_status.value().send_buffers_avail; } } catch (JFJochException &e) {} // ignore exception in getting receiver status (don't really care, e.g. if receiver is down) @@ -563,9 +644,9 @@ RadialIntegrationSettings JFJochStateMachine::GetRadialIntegrationSettings() con ret.polarization_factor = experiment.GetPolarizationFactor(); ret.solid_angle_correction = experiment.GetApplySolidAngleCorr(); - ret.q_spacing = experiment.GetQSpacingForRadialInt_recipA(); - ret.low_q_recipA = experiment.GetLowQForRadialInt_recipA(); - ret.high_q_recipA = experiment.GetHighQForRadialInt_recipA(); + ret.q_spacing = experiment.GetQSpacingForAzimInt_recipA(); + ret.low_q_recipA = experiment.GetLowQForAzimInt_recipA(); + ret.high_q_recipA = experiment.GetHighQForAzimInt_recipA(); return ret; } diff --git a/broker/JFJochStateMachine.h b/broker/JFJochStateMachine.h index c6f65502..fb0a8286 100644 --- a/broker/JFJochStateMachine.h +++ b/broker/JFJochStateMachine.h @@ -20,7 +20,6 @@ struct BrokerStatus { JFJochState broker_state; std::optional progress; std::optional indexing_rate; - std::optional receiver_send_buffers_avail; }; struct DetectorListElement { @@ -37,7 +36,9 @@ struct DetectorList { struct MeasurementStatistics { std::string file_prefix; + int64_t images_expected; int64_t images_collected; + int64_t images_sent; int64_t max_image_number_sent; float collection_efficiency; float compression_ratio; @@ -78,6 +79,15 @@ struct DatasetSettings { bool save_calibration; std::optional total_flux; std::optional attenuator_transmission; + + std::optional omega_step; + float omega_start; + Coord omega_axis; + + std::string image_appendix; + std::string header_appendix; + + std::optional roi_sum_area; }; struct DetectorSettings { @@ -87,7 +97,8 @@ struct DetectorSettings { int64_t storage_cell_count; bool use_internal_packet_generator; bool collect_raw_data; - + bool use_gain_hg0; + bool fixed_gain_g1; std::optional pedestal_g0_frames; std::optional pedestal_g1_frames; std::optional pedestal_g2_frames; @@ -141,8 +152,8 @@ class JFJochStateMachine { void MeasurementThread(); void PedestalThread(std::unique_lock ul); void InitializeThread(std::unique_lock ul); - void ImportPedestal(const JFJochReceiverOutput &receiver_output, size_t gain_level, size_t storage_cell = 0); - void ImportPedestalG0(const JFJochReceiverOutput &receiver_output); + bool ImportPedestalG1G2(const JFJochReceiverOutput &receiver_output, size_t gain_level, size_t storage_cell = 0); + bool ImportPedestalG0(const JFJochReceiverOutput &receiver_output); bool IsRunning() const; // Is state Busy/Pedestal/Measure std::optional CheckError(); diff --git a/broker/gen/api/DefaultApi.cpp b/broker/gen/api/DefaultApi.cpp index f5b03537..90e2f895 100644 --- a/broker/gen/api/DefaultApi.cpp +++ b/broker/gen/api/DefaultApi.cpp @@ -45,13 +45,18 @@ void DefaultApi::setupRoutes() { Routes::Post(*router, base + "/deactivate", Routes::bind(&DefaultApi::deactivate_post_handler, this)); Routes::Post(*router, base + "/initialize", Routes::bind(&DefaultApi::initialize_post_handler, this)); Routes::Post(*router, base + "/pedestal", Routes::bind(&DefaultApi::pedestal_post_handler, this)); - Routes::Get(*router, base + "/plot/adu_histogram", Routes::bind(&DefaultApi::plot_adu_histogram_get_handler, this)); Routes::Post(*router, base + "/plot/bkg_estimate", Routes::bind(&DefaultApi::plot_bkg_estimate_post_handler, this)); + Routes::Post(*router, base + "/plot/error_pixel", Routes::bind(&DefaultApi::plot_error_pixel_post_handler, this)); + Routes::Post(*router, base + "/plot/image_collection_efficiency", Routes::bind(&DefaultApi::plot_image_collection_efficiency_post_handler, this)); Routes::Get(*router, base + "/plot/indexing_rate_per_file", Routes::bind(&DefaultApi::plot_indexing_rate_per_file_get_handler, this)); Routes::Post(*router, base + "/plot/indexing_rate", Routes::bind(&DefaultApi::plot_indexing_rate_post_handler, this)); Routes::Get(*router, base + "/plot/rad_int", Routes::bind(&DefaultApi::plot_rad_int_get_handler, this)); Routes::Get(*router, base + "/plot/rad_int_per_file", Routes::bind(&DefaultApi::plot_rad_int_per_file_get_handler, this)); + Routes::Post(*router, base + "/plot/receiver_delay", Routes::bind(&DefaultApi::plot_receiver_delay_post_handler, this)); + Routes::Post(*router, base + "/plot/roi_sum", Routes::bind(&DefaultApi::plot_roi_sum_post_handler, this)); + Routes::Post(*router, base + "/plot/saturated_pixel", Routes::bind(&DefaultApi::plot_saturated_pixel_post_handler, this)); Routes::Post(*router, base + "/plot/spot_count", Routes::bind(&DefaultApi::plot_spot_count_post_handler, this)); + Routes::Post(*router, base + "/plot/strong_pixel", Routes::bind(&DefaultApi::plot_strong_pixel_post_handler, this)); Routes::Post(*router, base + "/start", Routes::bind(&DefaultApi::start_post_handler, this)); Routes::Get(*router, base + "/statistics/calibration", Routes::bind(&DefaultApi::statistics_calibration_get_handler, this)); Routes::Get(*router, base + "/statistics/data_collection", Routes::bind(&DefaultApi::statistics_data_collection_get_handler, this)); @@ -372,26 +377,6 @@ void DefaultApi::pedestal_post_handler(const Pistache::Rest::Request &, Pistache response.send(Pistache::Http::Code::Internal_Server_Error, e.what()); } -} -void DefaultApi::plot_adu_histogram_get_handler(const Pistache::Rest::Request &, Pistache::Http::ResponseWriter response) { - try { - - - try { - this->plot_adu_histogram_get(response); - } catch (Pistache::Http::HttpError &e) { - response.send(static_cast(e.code()), e.what()); - return; - } catch (std::exception &e) { - const std::pair errorInfo = this->handleOperationException(e); - response.send(errorInfo.first, errorInfo.second); - return; - } - - } catch (std::exception &e) { - response.send(Pistache::Http::Code::Internal_Server_Error, e.what()); - } - } void DefaultApi::plot_bkg_estimate_post_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response) { try { @@ -425,6 +410,72 @@ void DefaultApi::plot_bkg_estimate_post_handler(const Pistache::Rest::Request &r response.send(Pistache::Http::Code::Internal_Server_Error, e.what()); } +} +void DefaultApi::plot_error_pixel_post_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response) { + try { + + + // Getting the body param + + Plot_request plotRequest; + + try { + nlohmann::json::parse(request.body()).get_to(plotRequest); + plotRequest.validate(); + } catch (std::exception &e) { + const std::pair errorInfo = this->handleParsingException(e); + response.send(errorInfo.first, errorInfo.second); + return; + } + + try { + this->plot_error_pixel_post(plotRequest, response); + } catch (Pistache::Http::HttpError &e) { + response.send(static_cast(e.code()), e.what()); + return; + } catch (std::exception &e) { + const std::pair errorInfo = this->handleOperationException(e); + response.send(errorInfo.first, errorInfo.second); + return; + } + + } catch (std::exception &e) { + response.send(Pistache::Http::Code::Internal_Server_Error, e.what()); + } + +} +void DefaultApi::plot_image_collection_efficiency_post_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response) { + try { + + + // Getting the body param + + Plot_request plotRequest; + + try { + nlohmann::json::parse(request.body()).get_to(plotRequest); + plotRequest.validate(); + } catch (std::exception &e) { + const std::pair errorInfo = this->handleParsingException(e); + response.send(errorInfo.first, errorInfo.second); + return; + } + + try { + this->plot_image_collection_efficiency_post(plotRequest, response); + } catch (Pistache::Http::HttpError &e) { + response.send(static_cast(e.code()), e.what()); + return; + } catch (std::exception &e) { + const std::pair errorInfo = this->handleOperationException(e); + response.send(errorInfo.first, errorInfo.second); + return; + } + + } catch (std::exception &e) { + response.send(Pistache::Http::Code::Internal_Server_Error, e.what()); + } + } void DefaultApi::plot_indexing_rate_per_file_get_handler(const Pistache::Rest::Request &, Pistache::Http::ResponseWriter response) { try { @@ -518,6 +569,105 @@ void DefaultApi::plot_rad_int_per_file_get_handler(const Pistache::Rest::Request response.send(Pistache::Http::Code::Internal_Server_Error, e.what()); } +} +void DefaultApi::plot_receiver_delay_post_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response) { + try { + + + // Getting the body param + + Plot_request plotRequest; + + try { + nlohmann::json::parse(request.body()).get_to(plotRequest); + plotRequest.validate(); + } catch (std::exception &e) { + const std::pair errorInfo = this->handleParsingException(e); + response.send(errorInfo.first, errorInfo.second); + return; + } + + try { + this->plot_receiver_delay_post(plotRequest, response); + } catch (Pistache::Http::HttpError &e) { + response.send(static_cast(e.code()), e.what()); + return; + } catch (std::exception &e) { + const std::pair errorInfo = this->handleOperationException(e); + response.send(errorInfo.first, errorInfo.second); + return; + } + + } catch (std::exception &e) { + response.send(Pistache::Http::Code::Internal_Server_Error, e.what()); + } + +} +void DefaultApi::plot_roi_sum_post_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response) { + try { + + + // Getting the body param + + Plot_request plotRequest; + + try { + nlohmann::json::parse(request.body()).get_to(plotRequest); + plotRequest.validate(); + } catch (std::exception &e) { + const std::pair errorInfo = this->handleParsingException(e); + response.send(errorInfo.first, errorInfo.second); + return; + } + + try { + this->plot_roi_sum_post(plotRequest, response); + } catch (Pistache::Http::HttpError &e) { + response.send(static_cast(e.code()), e.what()); + return; + } catch (std::exception &e) { + const std::pair errorInfo = this->handleOperationException(e); + response.send(errorInfo.first, errorInfo.second); + return; + } + + } catch (std::exception &e) { + response.send(Pistache::Http::Code::Internal_Server_Error, e.what()); + } + +} +void DefaultApi::plot_saturated_pixel_post_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response) { + try { + + + // Getting the body param + + Plot_request plotRequest; + + try { + nlohmann::json::parse(request.body()).get_to(plotRequest); + plotRequest.validate(); + } catch (std::exception &e) { + const std::pair errorInfo = this->handleParsingException(e); + response.send(errorInfo.first, errorInfo.second); + return; + } + + try { + this->plot_saturated_pixel_post(plotRequest, response); + } catch (Pistache::Http::HttpError &e) { + response.send(static_cast(e.code()), e.what()); + return; + } catch (std::exception &e) { + const std::pair errorInfo = this->handleOperationException(e); + response.send(errorInfo.first, errorInfo.second); + return; + } + + } catch (std::exception &e) { + response.send(Pistache::Http::Code::Internal_Server_Error, e.what()); + } + } void DefaultApi::plot_spot_count_post_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response) { try { @@ -551,6 +701,39 @@ void DefaultApi::plot_spot_count_post_handler(const Pistache::Rest::Request &req response.send(Pistache::Http::Code::Internal_Server_Error, e.what()); } +} +void DefaultApi::plot_strong_pixel_post_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response) { + try { + + + // Getting the body param + + Plot_request plotRequest; + + try { + nlohmann::json::parse(request.body()).get_to(plotRequest); + plotRequest.validate(); + } catch (std::exception &e) { + const std::pair errorInfo = this->handleParsingException(e); + response.send(errorInfo.first, errorInfo.second); + return; + } + + try { + this->plot_strong_pixel_post(plotRequest, response); + } catch (Pistache::Http::HttpError &e) { + response.send(static_cast(e.code()), e.what()); + return; + } catch (std::exception &e) { + const std::pair errorInfo = this->handleOperationException(e); + response.send(errorInfo.first, errorInfo.second); + return; + } + + } catch (std::exception &e) { + response.send(Pistache::Http::Code::Internal_Server_Error, e.what()); + } + } void DefaultApi::start_post_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response) { try { diff --git a/broker/gen/api/DefaultApi.h b/broker/gen/api/DefaultApi.h index 68838c02..10689b46 100644 --- a/broker/gen/api/DefaultApi.h +++ b/broker/gen/api/DefaultApi.h @@ -67,13 +67,18 @@ private: void deactivate_post_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response); void initialize_post_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response); void pedestal_post_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response); - void plot_adu_histogram_get_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response); void plot_bkg_estimate_post_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response); + void plot_error_pixel_post_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response); + void plot_image_collection_efficiency_post_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response); void plot_indexing_rate_per_file_get_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response); void plot_indexing_rate_post_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response); void plot_rad_int_get_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response); void plot_rad_int_per_file_get_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response); + void plot_receiver_delay_post_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response); + void plot_roi_sum_post_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response); + void plot_saturated_pixel_post_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response); void plot_spot_count_post_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response); + void plot_strong_pixel_post_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response); void start_post_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response); void statistics_calibration_get_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response); void statistics_data_collection_get_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response); @@ -187,13 +192,6 @@ private: /// virtual void pedestal_post(Pistache::Http::ResponseWriter &response) = 0; /// - /// Generate ADU histogram - /// - /// - /// ADU histogram for all images within current data collection - /// - virtual void plot_adu_histogram_get(Pistache::Http::ResponseWriter &response) = 0; - /// /// Generate background estimate plot /// /// @@ -202,6 +200,22 @@ private: /// (optional) virtual void plot_bkg_estimate_post(const org::openapitools::server::model::Plot_request &plotRequest, Pistache::Http::ResponseWriter &response) = 0; /// + /// Generate error pixels plot + /// + /// + /// Count of error pixels per image; binning is configurable + /// + /// (optional) + virtual void plot_error_pixel_post(const org::openapitools::server::model::Plot_request &plotRequest, Pistache::Http::ResponseWriter &response) = 0; + /// + /// Generate image collection efficiency plot + /// + /// + /// Ratio of collected and expected packets per image; binning is configurable + /// + /// (optional) + virtual void plot_image_collection_efficiency_post(const org::openapitools::server::model::Plot_request &plotRequest, Pistache::Http::ResponseWriter &response) = 0; + /// /// Generate indexing rate per file /// /// @@ -231,6 +245,30 @@ private: /// virtual void plot_rad_int_per_file_get(Pistache::Http::ResponseWriter &response) = 0; /// + /// Generate receiver delay plot + /// + /// + /// Amount of frames the receiver is behind the FPGA for each image - used for internal debugging; binning is configurable + /// + /// (optional) + virtual void plot_receiver_delay_post(const org::openapitools::server::model::Plot_request &plotRequest, Pistache::Http::ResponseWriter &response) = 0; + /// + /// Generate ROI sum plot + /// + /// + /// Sum of ROI rectangle per image; binning is configurable + /// + /// (optional) + virtual void plot_roi_sum_post(const org::openapitools::server::model::Plot_request &plotRequest, Pistache::Http::ResponseWriter &response) = 0; + /// + /// Generate saturated pixels plot + /// + /// + /// Count of saturated pixels per image; binning is configurable + /// + /// (optional) + virtual void plot_saturated_pixel_post(const org::openapitools::server::model::Plot_request &plotRequest, Pistache::Http::ResponseWriter &response) = 0; + /// /// Generate spot count plot /// /// @@ -239,6 +277,14 @@ private: /// (optional) virtual void plot_spot_count_post(const org::openapitools::server::model::Plot_request &plotRequest, Pistache::Http::ResponseWriter &response) = 0; /// + /// Generate strong pixels plot + /// + /// + /// Count of strong pixels per image (from spot finding); binning is configurable + /// + /// (optional) + virtual void plot_strong_pixel_post(const org::openapitools::server::model::Plot_request &plotRequest, Pistache::Http::ResponseWriter &response) = 0; + /// /// Start detector /// /// diff --git a/broker/gen/model/Broker_status.cpp b/broker/gen/model/Broker_status.cpp index e45c04b8..9aab6e4a 100644 --- a/broker/gen/model/Broker_status.cpp +++ b/broker/gen/model/Broker_status.cpp @@ -26,8 +26,6 @@ Broker_status::Broker_status() m_ProgressIsSet = false; m_Indexing_rate = 0.0f; m_Indexing_rateIsSet = false; - m_Receiver_send_buffers_avail = 0.0f; - m_Receiver_send_buffers_availIsSet = false; } @@ -76,25 +74,6 @@ bool Broker_status::validate(std::stringstream& msg, const std::string& pathPref const std::string currentValuePath = _pathPrefix + ".indexingRate"; - if (value < static_cast(0.0)) - { - success = false; - msg << currentValuePath << ": must be greater than or equal to 0.0;"; - } - if (value > static_cast(1.0)) - { - success = false; - msg << currentValuePath << ": must be less than or equal to 1.0;"; - } - - } - - if (receiverSendBuffersAvailIsSet()) - { - const float& value = m_Receiver_send_buffers_avail; - const std::string currentValuePath = _pathPrefix + ".receiverSendBuffersAvail"; - - if (value < static_cast(0.0)) { success = false; @@ -123,10 +102,7 @@ bool Broker_status::operator==(const Broker_status& rhs) const ((!progressIsSet() && !rhs.progressIsSet()) || (progressIsSet() && rhs.progressIsSet() && getProgress() == rhs.getProgress())) && - ((!indexingRateIsSet() && !rhs.indexingRateIsSet()) || (indexingRateIsSet() && rhs.indexingRateIsSet() && getIndexingRate() == rhs.getIndexingRate())) && - - - ((!receiverSendBuffersAvailIsSet() && !rhs.receiverSendBuffersAvailIsSet()) || (receiverSendBuffersAvailIsSet() && rhs.receiverSendBuffersAvailIsSet() && getReceiverSendBuffersAvail() == rhs.getReceiverSendBuffersAvail())) + ((!indexingRateIsSet() && !rhs.indexingRateIsSet()) || (indexingRateIsSet() && rhs.indexingRateIsSet() && getIndexingRate() == rhs.getIndexingRate())) ; } @@ -144,8 +120,6 @@ void to_json(nlohmann::json& j, const Broker_status& o) j["progress"] = o.m_Progress; if(o.indexingRateIsSet()) j["indexing_rate"] = o.m_Indexing_rate; - if(o.receiverSendBuffersAvailIsSet()) - j["receiver_send_buffers_avail"] = o.m_Receiver_send_buffers_avail; } @@ -162,11 +136,6 @@ void from_json(const nlohmann::json& j, Broker_status& o) j.at("indexing_rate").get_to(o.m_Indexing_rate); o.m_Indexing_rateIsSet = true; } - if(j.find("receiver_send_buffers_avail") != j.end()) - { - j.at("receiver_send_buffers_avail").get_to(o.m_Receiver_send_buffers_avail); - o.m_Receiver_send_buffers_availIsSet = true; - } } @@ -212,23 +181,6 @@ void Broker_status::unsetIndexing_rate() { m_Indexing_rateIsSet = false; } -float Broker_status::getReceiverSendBuffersAvail() const -{ - return m_Receiver_send_buffers_avail; -} -void Broker_status::setReceiverSendBuffersAvail(float const value) -{ - m_Receiver_send_buffers_avail = value; - m_Receiver_send_buffers_availIsSet = true; -} -bool Broker_status::receiverSendBuffersAvailIsSet() const -{ - return m_Receiver_send_buffers_availIsSet; -} -void Broker_status::unsetReceiver_send_buffers_avail() -{ - m_Receiver_send_buffers_availIsSet = false; -} } // namespace org::openapitools::server::model diff --git a/broker/gen/model/Broker_status.h b/broker/gen/model/Broker_status.h index d901f459..1f32a14a 100644 --- a/broker/gen/model/Broker_status.h +++ b/broker/gen/model/Broker_status.h @@ -77,13 +77,6 @@ public: void setIndexingRate(float const value); bool indexingRateIsSet() const; void unsetIndexing_rate(); - /// - /// - /// - float getReceiverSendBuffersAvail() const; - void setReceiverSendBuffersAvail(float const value); - bool receiverSendBuffersAvailIsSet() const; - void unsetReceiver_send_buffers_avail(); friend void to_json(nlohmann::json& j, const Broker_status& o); friend void from_json(const nlohmann::json& j, Broker_status& o); @@ -94,8 +87,6 @@ protected: bool m_ProgressIsSet; float m_Indexing_rate; bool m_Indexing_rateIsSet; - float m_Receiver_send_buffers_avail; - bool m_Receiver_send_buffers_availIsSet; }; diff --git a/broker/gen/model/Dataset_settings.cpp b/broker/gen/model/Dataset_settings.cpp index 002835af..57f8c2f7 100644 --- a/broker/gen/model/Dataset_settings.cpp +++ b/broker/gen/model/Dataset_settings.cpp @@ -47,6 +47,12 @@ Dataset_settings::Dataset_settings() m_Total_fluxIsSet = false; m_Transmission = 0.0f; m_TransmissionIsSet = false; + m_OmegaIsSet = false; + m_Header_appendix = ""; + m_Header_appendixIsSet = false; + m_Image_appendix = ""; + m_Image_appendixIsSet = false; + m_Roi_sum_areaIsSet = false; m_Unit_cellIsSet = false; } @@ -197,7 +203,7 @@ bool Dataset_settings::validate(std::stringstream& msg, const std::string& pathP } } - + return success; } @@ -255,6 +261,18 @@ bool Dataset_settings::operator==(const Dataset_settings& rhs) const ((!transmissionIsSet() && !rhs.transmissionIsSet()) || (transmissionIsSet() && rhs.transmissionIsSet() && getTransmission() == rhs.getTransmission())) && + ((!omegaIsSet() && !rhs.omegaIsSet()) || (omegaIsSet() && rhs.omegaIsSet() && getOmega() == rhs.getOmega())) && + + + ((!headerAppendixIsSet() && !rhs.headerAppendixIsSet()) || (headerAppendixIsSet() && rhs.headerAppendixIsSet() && getHeaderAppendix() == rhs.getHeaderAppendix())) && + + + ((!imageAppendixIsSet() && !rhs.imageAppendixIsSet()) || (imageAppendixIsSet() && rhs.imageAppendixIsSet() && getImageAppendix() == rhs.getImageAppendix())) && + + + ((!roiSumAreaIsSet() && !rhs.roiSumAreaIsSet()) || (roiSumAreaIsSet() && rhs.roiSumAreaIsSet() && getRoiSumArea() == rhs.getRoiSumArea())) && + + ((!unitCellIsSet() && !rhs.unitCellIsSet()) || (unitCellIsSet() && rhs.unitCellIsSet() && getUnitCell() == rhs.getUnitCell())) ; @@ -294,6 +312,14 @@ void to_json(nlohmann::json& j, const Dataset_settings& o) j["total_flux"] = o.m_Total_flux; if(o.transmissionIsSet()) j["transmission"] = o.m_Transmission; + if(o.omegaIsSet()) + j["omega"] = o.m_Omega; + if(o.headerAppendixIsSet()) + j["header_appendix"] = o.m_Header_appendix; + if(o.imageAppendixIsSet()) + j["image_appendix"] = o.m_Image_appendix; + if(o.roiSumAreaIsSet()) + j["roi_sum_area"] = o.m_Roi_sum_area; if(o.unitCellIsSet()) j["unit_cell"] = o.m_Unit_cell; @@ -357,6 +383,26 @@ void from_json(const nlohmann::json& j, Dataset_settings& o) j.at("transmission").get_to(o.m_Transmission); o.m_TransmissionIsSet = true; } + if(j.find("omega") != j.end()) + { + j.at("omega").get_to(o.m_Omega); + o.m_OmegaIsSet = true; + } + if(j.find("header_appendix") != j.end()) + { + j.at("header_appendix").get_to(o.m_Header_appendix); + o.m_Header_appendixIsSet = true; + } + if(j.find("image_appendix") != j.end()) + { + j.at("image_appendix").get_to(o.m_Image_appendix); + o.m_Image_appendixIsSet = true; + } + if(j.find("roi_sum_area") != j.end()) + { + j.at("roi_sum_area").get_to(o.m_Roi_sum_area); + o.m_Roi_sum_areaIsSet = true; + } if(j.find("unit_cell") != j.end()) { j.at("unit_cell").get_to(o.m_Unit_cell); @@ -583,6 +629,74 @@ void Dataset_settings::unsetTransmission() { m_TransmissionIsSet = false; } +org::openapitools::server::model::Rotation_axis Dataset_settings::getOmega() const +{ + return m_Omega; +} +void Dataset_settings::setOmega(org::openapitools::server::model::Rotation_axis const& value) +{ + m_Omega = value; + m_OmegaIsSet = true; +} +bool Dataset_settings::omegaIsSet() const +{ + return m_OmegaIsSet; +} +void Dataset_settings::unsetOmega() +{ + m_OmegaIsSet = false; +} +std::string Dataset_settings::getHeaderAppendix() const +{ + return m_Header_appendix; +} +void Dataset_settings::setHeaderAppendix(std::string const& value) +{ + m_Header_appendix = value; + m_Header_appendixIsSet = true; +} +bool Dataset_settings::headerAppendixIsSet() const +{ + return m_Header_appendixIsSet; +} +void Dataset_settings::unsetHeader_appendix() +{ + m_Header_appendixIsSet = false; +} +std::string Dataset_settings::getImageAppendix() const +{ + return m_Image_appendix; +} +void Dataset_settings::setImageAppendix(std::string const& value) +{ + m_Image_appendix = value; + m_Image_appendixIsSet = true; +} +bool Dataset_settings::imageAppendixIsSet() const +{ + return m_Image_appendixIsSet; +} +void Dataset_settings::unsetImage_appendix() +{ + m_Image_appendixIsSet = false; +} +org::openapitools::server::model::Dataset_settings_roi_sum_area Dataset_settings::getRoiSumArea() const +{ + return m_Roi_sum_area; +} +void Dataset_settings::setRoiSumArea(org::openapitools::server::model::Dataset_settings_roi_sum_area const& value) +{ + m_Roi_sum_area = value; + m_Roi_sum_areaIsSet = true; +} +bool Dataset_settings::roiSumAreaIsSet() const +{ + return m_Roi_sum_areaIsSet; +} +void Dataset_settings::unsetRoi_sum_area() +{ + m_Roi_sum_areaIsSet = false; +} org::openapitools::server::model::Dataset_settings_unit_cell Dataset_settings::getUnitCell() const { return m_Unit_cell; diff --git a/broker/gen/model/Dataset_settings.h b/broker/gen/model/Dataset_settings.h index 2542232e..b977c6e6 100644 --- a/broker/gen/model/Dataset_settings.h +++ b/broker/gen/model/Dataset_settings.h @@ -19,7 +19,9 @@ #define Dataset_settings_H_ +#include "Rotation_axis.h" #include +#include "Dataset_settings_roi_sum_area.h" #include "Dataset_settings_unit_cell.h" #include @@ -162,6 +164,34 @@ public: /// /// /// + org::openapitools::server::model::Rotation_axis getOmega() const; + void setOmega(org::openapitools::server::model::Rotation_axis const& value); + bool omegaIsSet() const; + void unsetOmega(); + /// + /// Header appendix, added as user_data to start message + /// + std::string getHeaderAppendix() const; + void setHeaderAppendix(std::string const& value); + bool headerAppendixIsSet() const; + void unsetHeader_appendix(); + /// + /// Image appendix, added as user_data to image message + /// + std::string getImageAppendix() const; + void setImageAppendix(std::string const& value); + bool imageAppendixIsSet() const; + void unsetImage_appendix(); + /// + /// + /// + org::openapitools::server::model::Dataset_settings_roi_sum_area getRoiSumArea() const; + void setRoiSumArea(org::openapitools::server::model::Dataset_settings_roi_sum_area const& value); + bool roiSumAreaIsSet() const; + void unsetRoi_sum_area(); + /// + /// + /// org::openapitools::server::model::Dataset_settings_unit_cell getUnitCell() const; void setUnitCell(org::openapitools::server::model::Dataset_settings_unit_cell const& value); bool unitCellIsSet() const; @@ -202,6 +232,14 @@ protected: bool m_Total_fluxIsSet; float m_Transmission; bool m_TransmissionIsSet; + org::openapitools::server::model::Rotation_axis m_Omega; + bool m_OmegaIsSet; + std::string m_Header_appendix; + bool m_Header_appendixIsSet; + std::string m_Image_appendix; + bool m_Image_appendixIsSet; + org::openapitools::server::model::Dataset_settings_roi_sum_area m_Roi_sum_area; + bool m_Roi_sum_areaIsSet; org::openapitools::server::model::Dataset_settings_unit_cell m_Unit_cell; bool m_Unit_cellIsSet; diff --git a/broker/gen/model/Dataset_settings_roi_sum_area.cpp b/broker/gen/model/Dataset_settings_roi_sum_area.cpp new file mode 100644 index 00000000..5f4256e7 --- /dev/null +++ b/broker/gen/model/Dataset_settings_roi_sum_area.cpp @@ -0,0 +1,133 @@ +/** +* Jungfraujoch +* Jungfraujoch Broker Web API +* +* The version of the OpenAPI document: 1.0.0 +* +* +* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). +* https://openapi-generator.tech +* Do not edit the class manually. +*/ + + +#include "Dataset_settings_roi_sum_area.h" +#include "Helpers.h" + +#include + +namespace org::openapitools::server::model +{ + +Dataset_settings_roi_sum_area::Dataset_settings_roi_sum_area() +{ + m_X_min = 0L; + m_X_max = 0L; + m_Y_min = 0L; + m_Y_max = 0L; + +} + +void Dataset_settings_roi_sum_area::validate() const +{ + std::stringstream msg; + if (!validate(msg)) + { + throw org::openapitools::server::helpers::ValidationException(msg.str()); + } +} + +bool Dataset_settings_roi_sum_area::validate(std::stringstream& msg) const +{ + return validate(msg, ""); +} + +bool Dataset_settings_roi_sum_area::validate(std::stringstream& msg, const std::string& pathPrefix) const +{ + bool success = true; + const std::string _pathPrefix = pathPrefix.empty() ? "Dataset_settings_roi_sum_area" : pathPrefix; + + + return success; +} + +bool Dataset_settings_roi_sum_area::operator==(const Dataset_settings_roi_sum_area& rhs) const +{ + return + + + (getXMin() == rhs.getXMin()) + && + + (getXMax() == rhs.getXMax()) + && + + (getYMin() == rhs.getYMin()) + && + + (getYMax() == rhs.getYMax()) + + + ; +} + +bool Dataset_settings_roi_sum_area::operator!=(const Dataset_settings_roi_sum_area& rhs) const +{ + return !(*this == rhs); +} + +void to_json(nlohmann::json& j, const Dataset_settings_roi_sum_area& o) +{ + j = nlohmann::json(); + j["x_min"] = o.m_X_min; + j["x_max"] = o.m_X_max; + j["y_min"] = o.m_Y_min; + j["y_max"] = o.m_Y_max; + +} + +void from_json(const nlohmann::json& j, Dataset_settings_roi_sum_area& o) +{ + j.at("x_min").get_to(o.m_X_min); + j.at("x_max").get_to(o.m_X_max); + j.at("y_min").get_to(o.m_Y_min); + j.at("y_max").get_to(o.m_Y_max); + +} + +int64_t Dataset_settings_roi_sum_area::getXMin() const +{ + return m_X_min; +} +void Dataset_settings_roi_sum_area::setXMin(int64_t const value) +{ + m_X_min = value; +} +int64_t Dataset_settings_roi_sum_area::getXMax() const +{ + return m_X_max; +} +void Dataset_settings_roi_sum_area::setXMax(int64_t const value) +{ + m_X_max = value; +} +int64_t Dataset_settings_roi_sum_area::getYMin() const +{ + return m_Y_min; +} +void Dataset_settings_roi_sum_area::setYMin(int64_t const value) +{ + m_Y_min = value; +} +int64_t Dataset_settings_roi_sum_area::getYMax() const +{ + return m_Y_max; +} +void Dataset_settings_roi_sum_area::setYMax(int64_t const value) +{ + m_Y_max = value; +} + + +} // namespace org::openapitools::server::model + diff --git a/broker/gen/model/Dataset_settings_roi_sum_area.h b/broker/gen/model/Dataset_settings_roi_sum_area.h new file mode 100644 index 00000000..79041795 --- /dev/null +++ b/broker/gen/model/Dataset_settings_roi_sum_area.h @@ -0,0 +1,97 @@ +/** +* Jungfraujoch +* Jungfraujoch Broker Web API +* +* The version of the OpenAPI document: 1.0.0 +* +* +* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). +* https://openapi-generator.tech +* Do not edit the class manually. +*/ +/* + * Dataset_settings_roi_sum_area.h + * + * Rectangle for ROI summation + */ + +#ifndef Dataset_settings_roi_sum_area_H_ +#define Dataset_settings_roi_sum_area_H_ + + +#include + +namespace org::openapitools::server::model +{ + +/// +/// Rectangle for ROI summation +/// +class Dataset_settings_roi_sum_area +{ +public: + Dataset_settings_roi_sum_area(); + virtual ~Dataset_settings_roi_sum_area() = default; + + + /// + /// Validate the current data in the model. Throws a ValidationException on failure. + /// + void validate() const; + + /// + /// Validate the current data in the model. Returns false on error and writes an error + /// message into the given stringstream. + /// + bool validate(std::stringstream& msg) const; + + /// + /// Helper overload for validate. Used when one model stores another model and calls it's validate. + /// Not meant to be called outside that case. + /// + bool validate(std::stringstream& msg, const std::string& pathPrefix) const; + + bool operator==(const Dataset_settings_roi_sum_area& rhs) const; + bool operator!=(const Dataset_settings_roi_sum_area& rhs) const; + + ///////////////////////////////////////////// + /// Dataset_settings_roi_sum_area members + + /// + /// + /// + int64_t getXMin() const; + void setXMin(int64_t const value); + /// + /// + /// + int64_t getXMax() const; + void setXMax(int64_t const value); + /// + /// + /// + int64_t getYMin() const; + void setYMin(int64_t const value); + /// + /// + /// + int64_t getYMax() const; + void setYMax(int64_t const value); + + friend void to_json(nlohmann::json& j, const Dataset_settings_roi_sum_area& o); + friend void from_json(const nlohmann::json& j, Dataset_settings_roi_sum_area& o); +protected: + int64_t m_X_min; + + int64_t m_X_max; + + int64_t m_Y_min; + + int64_t m_Y_max; + + +}; + +} // namespace org::openapitools::server::model + +#endif /* Dataset_settings_roi_sum_area_H_ */ diff --git a/broker/gen/model/Detector_settings.cpp b/broker/gen/model/Detector_settings.cpp index 84a8b967..2c4190f9 100644 --- a/broker/gen/model/Detector_settings.cpp +++ b/broker/gen/model/Detector_settings.cpp @@ -38,6 +38,10 @@ Detector_settings::Detector_settings() m_Pedestal_g2_framesIsSet = false; m_Storage_cell_delay_ns = 0L; m_Storage_cell_delay_nsIsSet = false; + m_Fixed_gain_g1 = false; + m_Fixed_gain_g1IsSet = false; + m_Use_gain_hg0 = false; + m_Use_gain_hg0IsSet = false; } @@ -135,7 +139,7 @@ bool Detector_settings::validate(std::stringstream& msg, const std::string& path } } - + return success; } @@ -169,7 +173,13 @@ bool Detector_settings::operator==(const Detector_settings& rhs) const ((!pedestalG2FramesIsSet() && !rhs.pedestalG2FramesIsSet()) || (pedestalG2FramesIsSet() && rhs.pedestalG2FramesIsSet() && getPedestalG2Frames() == rhs.getPedestalG2Frames())) && - ((!storageCellDelayNsIsSet() && !rhs.storageCellDelayNsIsSet()) || (storageCellDelayNsIsSet() && rhs.storageCellDelayNsIsSet() && getStorageCellDelayNs() == rhs.getStorageCellDelayNs())) + ((!storageCellDelayNsIsSet() && !rhs.storageCellDelayNsIsSet()) || (storageCellDelayNsIsSet() && rhs.storageCellDelayNsIsSet() && getStorageCellDelayNs() == rhs.getStorageCellDelayNs())) && + + + ((!fixedGainG1IsSet() && !rhs.fixedGainG1IsSet()) || (fixedGainG1IsSet() && rhs.fixedGainG1IsSet() && isFixedGainG1() == rhs.isFixedGainG1())) && + + + ((!useGainHg0IsSet() && !rhs.useGainHg0IsSet()) || (useGainHg0IsSet() && rhs.useGainHg0IsSet() && isUseGainHg0() == rhs.isUseGainHg0())) ; } @@ -199,6 +209,10 @@ void to_json(nlohmann::json& j, const Detector_settings& o) j["pedestal_g2_frames"] = o.m_Pedestal_g2_frames; if(o.storageCellDelayNsIsSet()) j["storage_cell_delay_ns"] = o.m_Storage_cell_delay_ns; + if(o.fixedGainG1IsSet()) + j["fixed_gain_g1"] = o.m_Fixed_gain_g1; + if(o.useGainHg0IsSet()) + j["use_gain_hg0"] = o.m_Use_gain_hg0; } @@ -245,6 +259,16 @@ void from_json(const nlohmann::json& j, Detector_settings& o) j.at("storage_cell_delay_ns").get_to(o.m_Storage_cell_delay_ns); o.m_Storage_cell_delay_nsIsSet = true; } + if(j.find("fixed_gain_g1") != j.end()) + { + j.at("fixed_gain_g1").get_to(o.m_Fixed_gain_g1); + o.m_Fixed_gain_g1IsSet = true; + } + if(j.find("use_gain_hg0") != j.end()) + { + j.at("use_gain_hg0").get_to(o.m_Use_gain_hg0); + o.m_Use_gain_hg0IsSet = true; + } } @@ -392,6 +416,40 @@ void Detector_settings::unsetStorage_cell_delay_ns() { m_Storage_cell_delay_nsIsSet = false; } +bool Detector_settings::isFixedGainG1() const +{ + return m_Fixed_gain_g1; +} +void Detector_settings::setFixedGainG1(bool const value) +{ + m_Fixed_gain_g1 = value; + m_Fixed_gain_g1IsSet = true; +} +bool Detector_settings::fixedGainG1IsSet() const +{ + return m_Fixed_gain_g1IsSet; +} +void Detector_settings::unsetFixed_gain_g1() +{ + m_Fixed_gain_g1IsSet = false; +} +bool Detector_settings::isUseGainHg0() const +{ + return m_Use_gain_hg0; +} +void Detector_settings::setUseGainHg0(bool const value) +{ + m_Use_gain_hg0 = value; + m_Use_gain_hg0IsSet = true; +} +bool Detector_settings::useGainHg0IsSet() const +{ + return m_Use_gain_hg0IsSet; +} +void Detector_settings::unsetUse_gain_hg0() +{ + m_Use_gain_hg0IsSet = false; +} } // namespace org::openapitools::server::model diff --git a/broker/gen/model/Detector_settings.h b/broker/gen/model/Detector_settings.h index 4e789cbb..59bc6fe9 100644 --- a/broker/gen/model/Detector_settings.h +++ b/broker/gen/model/Detector_settings.h @@ -77,14 +77,14 @@ public: bool storageCellCountIsSet() const; void unsetStorage_cell_count(); /// - /// + /// Use internal frame generator in FPGA instead of getting data from a real detector /// bool isInternalFrameGenerator() const; void setInternalFrameGenerator(bool const value); bool internalFrameGeneratorIsSet() const; void unsetInternal_frame_generator(); /// - /// + /// Turn off conversion of pixel read-out to photon count /// bool isCollectRawData() const; void setCollectRawData(bool const value); @@ -112,12 +112,26 @@ public: bool pedestalG2FramesIsSet() const; void unsetPedestal_g2_frames(); /// - /// + /// Delay between two storage cells /// int64_t getStorageCellDelayNs() const; void setStorageCellDelayNs(int64_t const value); bool storageCellDelayNsIsSet() const; void unsetStorage_cell_delay_ns(); + /// + /// Fix gain to G1 (can be useful for storage cells) + /// + bool isFixedGainG1() const; + void setFixedGainG1(bool const value); + bool fixedGainG1IsSet() const; + void unsetFixed_gain_g1(); + /// + /// Use high G0 (for low energy applications) + /// + bool isUseGainHg0() const; + void setUseGainHg0(bool const value); + bool useGainHg0IsSet() const; + void unsetUse_gain_hg0(); friend void to_json(nlohmann::json& j, const Detector_settings& o); friend void from_json(const nlohmann::json& j, Detector_settings& o); @@ -140,6 +154,10 @@ protected: bool m_Pedestal_g2_framesIsSet; int64_t m_Storage_cell_delay_ns; bool m_Storage_cell_delay_nsIsSet; + bool m_Fixed_gain_g1; + bool m_Fixed_gain_g1IsSet; + bool m_Use_gain_hg0; + bool m_Use_gain_hg0IsSet; }; diff --git a/broker/gen/model/Measurement_statistics.cpp b/broker/gen/model/Measurement_statistics.cpp index 5b49009d..9eb47844 100644 --- a/broker/gen/model/Measurement_statistics.cpp +++ b/broker/gen/model/Measurement_statistics.cpp @@ -23,8 +23,12 @@ Measurement_statistics::Measurement_statistics() { m_File_prefix = ""; m_File_prefixIsSet = false; + m_Images_expected = 0L; + m_Images_expectedIsSet = false; m_Images_collected = 0L; m_Images_collectedIsSet = false; + m_Images_sent = 0L; + m_Images_sentIsSet = false; m_Max_image_number_sent = 0L; m_Max_image_number_sentIsSet = false; m_Collection_efficiency = 0.0f; @@ -67,7 +71,7 @@ bool Measurement_statistics::validate(std::stringstream& msg, const std::string& bool success = true; const std::string _pathPrefix = pathPrefix.empty() ? "Measurement_statistics" : pathPrefix; - + if (collectionEfficiencyIsSet()) { const float& value = m_Collection_efficiency; @@ -113,9 +117,15 @@ bool Measurement_statistics::operator==(const Measurement_statistics& rhs) const ((!filePrefixIsSet() && !rhs.filePrefixIsSet()) || (filePrefixIsSet() && rhs.filePrefixIsSet() && getFilePrefix() == rhs.getFilePrefix())) && + ((!imagesExpectedIsSet() && !rhs.imagesExpectedIsSet()) || (imagesExpectedIsSet() && rhs.imagesExpectedIsSet() && getImagesExpected() == rhs.getImagesExpected())) && + + ((!imagesCollectedIsSet() && !rhs.imagesCollectedIsSet()) || (imagesCollectedIsSet() && rhs.imagesCollectedIsSet() && getImagesCollected() == rhs.getImagesCollected())) && + ((!imagesSentIsSet() && !rhs.imagesSentIsSet()) || (imagesSentIsSet() && rhs.imagesSentIsSet() && getImagesSent() == rhs.getImagesSent())) && + + ((!maxImageNumberSentIsSet() && !rhs.maxImageNumberSentIsSet()) || (maxImageNumberSentIsSet() && rhs.maxImageNumberSentIsSet() && getMaxImageNumberSent() == rhs.getMaxImageNumberSent())) && @@ -158,8 +168,12 @@ void to_json(nlohmann::json& j, const Measurement_statistics& o) j = nlohmann::json(); if(o.filePrefixIsSet()) j["file_prefix"] = o.m_File_prefix; + if(o.imagesExpectedIsSet()) + j["images_expected"] = o.m_Images_expected; if(o.imagesCollectedIsSet()) j["images_collected"] = o.m_Images_collected; + if(o.imagesSentIsSet()) + j["images_sent"] = o.m_Images_sent; if(o.maxImageNumberSentIsSet()) j["max_image_number_sent"] = o.m_Max_image_number_sent; if(o.collectionEfficiencyIsSet()) @@ -190,11 +204,21 @@ void from_json(const nlohmann::json& j, Measurement_statistics& o) j.at("file_prefix").get_to(o.m_File_prefix); o.m_File_prefixIsSet = true; } + if(j.find("images_expected") != j.end()) + { + j.at("images_expected").get_to(o.m_Images_expected); + o.m_Images_expectedIsSet = true; + } if(j.find("images_collected") != j.end()) { j.at("images_collected").get_to(o.m_Images_collected); o.m_Images_collectedIsSet = true; } + if(j.find("images_sent") != j.end()) + { + j.at("images_sent").get_to(o.m_Images_sent); + o.m_Images_sentIsSet = true; + } if(j.find("max_image_number_sent") != j.end()) { j.at("max_image_number_sent").get_to(o.m_Max_image_number_sent); @@ -265,6 +289,23 @@ void Measurement_statistics::unsetFile_prefix() { m_File_prefixIsSet = false; } +int64_t Measurement_statistics::getImagesExpected() const +{ + return m_Images_expected; +} +void Measurement_statistics::setImagesExpected(int64_t const value) +{ + m_Images_expected = value; + m_Images_expectedIsSet = true; +} +bool Measurement_statistics::imagesExpectedIsSet() const +{ + return m_Images_expectedIsSet; +} +void Measurement_statistics::unsetImages_expected() +{ + m_Images_expectedIsSet = false; +} int64_t Measurement_statistics::getImagesCollected() const { return m_Images_collected; @@ -282,6 +323,23 @@ void Measurement_statistics::unsetImages_collected() { m_Images_collectedIsSet = false; } +int64_t Measurement_statistics::getImagesSent() const +{ + return m_Images_sent; +} +void Measurement_statistics::setImagesSent(int64_t const value) +{ + m_Images_sent = value; + m_Images_sentIsSet = true; +} +bool Measurement_statistics::imagesSentIsSet() const +{ + return m_Images_sentIsSet; +} +void Measurement_statistics::unsetImages_sent() +{ + m_Images_sentIsSet = false; +} int64_t Measurement_statistics::getMaxImageNumberSent() const { return m_Max_image_number_sent; diff --git a/broker/gen/model/Measurement_statistics.h b/broker/gen/model/Measurement_statistics.h index 3c63aa4a..70a1fb99 100644 --- a/broker/gen/model/Measurement_statistics.h +++ b/broker/gen/model/Measurement_statistics.h @@ -68,6 +68,13 @@ public: /// /// /// + int64_t getImagesExpected() const; + void setImagesExpected(int64_t const value); + bool imagesExpectedIsSet() const; + void unsetImages_expected(); + /// + /// + /// int64_t getImagesCollected() const; void setImagesCollected(int64_t const value); bool imagesCollectedIsSet() const; @@ -75,6 +82,13 @@ public: /// /// /// + int64_t getImagesSent() const; + void setImagesSent(int64_t const value); + bool imagesSentIsSet() const; + void unsetImages_sent(); + /// + /// + /// int64_t getMaxImageNumberSent() const; void setMaxImageNumberSent(int64_t const value); bool maxImageNumberSentIsSet() const; @@ -148,8 +162,12 @@ public: protected: std::string m_File_prefix; bool m_File_prefixIsSet; + int64_t m_Images_expected; + bool m_Images_expectedIsSet; int64_t m_Images_collected; bool m_Images_collectedIsSet; + int64_t m_Images_sent; + bool m_Images_sentIsSet; int64_t m_Max_image_number_sent; bool m_Max_image_number_sentIsSet; float m_Collection_efficiency; diff --git a/broker/gen/model/Rotation_axis.cpp b/broker/gen/model/Rotation_axis.cpp new file mode 100644 index 00000000..e180df07 --- /dev/null +++ b/broker/gen/model/Rotation_axis.cpp @@ -0,0 +1,179 @@ +/** +* Jungfraujoch +* Jungfraujoch Broker Web API +* +* The version of the OpenAPI document: 1.0.0 +* +* +* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). +* https://openapi-generator.tech +* Do not edit the class manually. +*/ + + +#include "Rotation_axis.h" +#include "Helpers.h" + +#include + +namespace org::openapitools::server::model +{ + +Rotation_axis::Rotation_axis() +{ + m_Step = 0.0f; + m_Start = 0.0f; + m_StartIsSet = false; + m_VectorIsSet = false; + +} + +void Rotation_axis::validate() const +{ + std::stringstream msg; + if (!validate(msg)) + { + throw org::openapitools::server::helpers::ValidationException(msg.str()); + } +} + +bool Rotation_axis::validate(std::stringstream& msg) const +{ + return validate(msg, ""); +} + +bool Rotation_axis::validate(std::stringstream& msg, const std::string& pathPrefix) const +{ + bool success = true; + const std::string _pathPrefix = pathPrefix.empty() ? "Rotation_axis" : pathPrefix; + + + if (vectorIsSet()) + { + const std::vector& value = m_Vector; + const std::string currentValuePath = _pathPrefix + ".vector"; + + + if (value.size() < 3) + { + success = false; + msg << currentValuePath << ": must have at least 3 elements;"; + } + if (value.size() > 3) + { + success = false; + msg << currentValuePath << ": must have at most 3 elements;"; + } + { // Recursive validation of array elements + const std::string oldValuePath = currentValuePath; + int i = 0; + for (const float& value : value) + { + const std::string currentValuePath = oldValuePath + "[" + std::to_string(i) + "]"; + + + + i++; + } + } + + } + + return success; +} + +bool Rotation_axis::operator==(const Rotation_axis& rhs) const +{ + return + + + (getStep() == rhs.getStep()) + && + + + ((!startIsSet() && !rhs.startIsSet()) || (startIsSet() && rhs.startIsSet() && getStart() == rhs.getStart())) && + + + ((!vectorIsSet() && !rhs.vectorIsSet()) || (vectorIsSet() && rhs.vectorIsSet() && getVector() == rhs.getVector())) + + ; +} + +bool Rotation_axis::operator!=(const Rotation_axis& rhs) const +{ + return !(*this == rhs); +} + +void to_json(nlohmann::json& j, const Rotation_axis& o) +{ + j = nlohmann::json(); + j["step"] = o.m_Step; + if(o.startIsSet()) + j["start"] = o.m_Start; + if(o.vectorIsSet() || !o.m_Vector.empty()) + j["vector"] = o.m_Vector; + +} + +void from_json(const nlohmann::json& j, Rotation_axis& o) +{ + j.at("step").get_to(o.m_Step); + if(j.find("start") != j.end()) + { + j.at("start").get_to(o.m_Start); + o.m_StartIsSet = true; + } + if(j.find("vector") != j.end()) + { + j.at("vector").get_to(o.m_Vector); + o.m_VectorIsSet = true; + } + +} + +float Rotation_axis::getStep() const +{ + return m_Step; +} +void Rotation_axis::setStep(float const value) +{ + m_Step = value; +} +float Rotation_axis::getStart() const +{ + return m_Start; +} +void Rotation_axis::setStart(float const value) +{ + m_Start = value; + m_StartIsSet = true; +} +bool Rotation_axis::startIsSet() const +{ + return m_StartIsSet; +} +void Rotation_axis::unsetStart() +{ + m_StartIsSet = false; +} +std::vector Rotation_axis::getVector() const +{ + return m_Vector; +} +void Rotation_axis::setVector(std::vector const value) +{ + m_Vector = value; + m_VectorIsSet = true; +} +bool Rotation_axis::vectorIsSet() const +{ + return m_VectorIsSet; +} +void Rotation_axis::unsetVector() +{ + m_VectorIsSet = false; +} + + +} // namespace org::openapitools::server::model + diff --git a/broker/gen/model/Rotation_axis.h b/broker/gen/model/Rotation_axis.h new file mode 100644 index 00000000..9de7ba0a --- /dev/null +++ b/broker/gen/model/Rotation_axis.h @@ -0,0 +1,95 @@ +/** +* Jungfraujoch +* Jungfraujoch Broker Web API +* +* The version of the OpenAPI document: 1.0.0 +* +* +* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). +* https://openapi-generator.tech +* Do not edit the class manually. +*/ +/* + * Rotation_axis.h + * + * Definition of a crystal rotation axis + */ + +#ifndef Rotation_axis_H_ +#define Rotation_axis_H_ + + +#include +#include + +namespace org::openapitools::server::model +{ + +/// +/// Definition of a crystal rotation axis +/// +class Rotation_axis +{ +public: + Rotation_axis(); + virtual ~Rotation_axis() = default; + + + /// + /// Validate the current data in the model. Throws a ValidationException on failure. + /// + void validate() const; + + /// + /// Validate the current data in the model. Returns false on error and writes an error + /// message into the given stringstream. + /// + bool validate(std::stringstream& msg) const; + + /// + /// Helper overload for validate. Used when one model stores another model and calls it's validate. + /// Not meant to be called outside that case. + /// + bool validate(std::stringstream& msg, const std::string& pathPrefix) const; + + bool operator==(const Rotation_axis& rhs) const; + bool operator!=(const Rotation_axis& rhs) const; + + ///////////////////////////////////////////// + /// Rotation_axis members + + /// + /// Angle step in degrees + /// + float getStep() const; + void setStep(float const value); + /// + /// Start angle in degrees + /// + float getStart() const; + void setStart(float const value); + bool startIsSet() const; + void unsetStart(); + /// + /// Rotation axis - vector with only zeros means using default value (set in a configuration file) + /// + std::vector getVector() const; + void setVector(std::vector const value); + bool vectorIsSet() const; + void unsetVector(); + + friend void to_json(nlohmann::json& j, const Rotation_axis& o); + friend void from_json(const nlohmann::json& j, Rotation_axis& o); +protected: + float m_Step; + + float m_Start; + bool m_StartIsSet; + std::vector m_Vector; + bool m_VectorIsSet; + +}; + +} // namespace org::openapitools::server::model + +#endif /* Rotation_axis_H_ */ diff --git a/broker/jfjoch_api.yaml b/broker/jfjoch_api.yaml index bfbbee68..eaed6961 100644 --- a/broker/jfjoch_api.yaml +++ b/broker/jfjoch_api.yaml @@ -5,6 +5,32 @@ info: version: 1.0.0 components: schemas: + rotation_axis: + description: Definition of a crystal rotation axis + type: object + required: + - step + properties: + step: + type: number + format: float + example: 0.1 + description: Angle step in degrees + start: + type: number + format: float + example: 50 + description: Start angle in degrees + default: 0 + vector: + type: array + description: Rotation axis - vector with only zeros means using default value (set in a configuration file) + default: [ 0, 0, 0 ] + items: + type: number + format: float + minItems: 3 + maxItems: 3 dataset_settings: type: object required: @@ -115,6 +141,39 @@ components: description: | /entry/instrument/attenuator/attenuator_transmission Transmission of attenuator (filter) [no units] + omega: + $ref: "#/components/schemas/rotation_axis" + header_appendix: + type: string + description: Header appendix, added as user_data to start message + image_appendix: + type: string + description: Image appendix, added as user_data to image message + roi_sum_area: + type: object + description: Rectangle for ROI summation + required: + - x_min + - x_max + - y_min + - y_max + properties: + x_min: + type: integer + format: int64 + example: 12 + x_max: + type: integer + format: int64 + example: 14 + y_min: + type: integer + format: int64 + example: 25 + y_max: + type: integer + format: int64 + example: 32 unit_cell: type: object description: Units of angstrom and degree @@ -150,7 +209,6 @@ components: type: number format: float example: 90 - detector_settings: type: object required: @@ -174,9 +232,11 @@ components: internal_frame_generator: type: boolean default: false + description: Use internal frame generator in FPGA instead of getting data from a real detector collect_raw_data: type: boolean default: false + description: Turn off conversion of pixel read-out to photon count pedestal_g0_frames: type: integer format: int64 @@ -192,6 +252,15 @@ components: storage_cell_delay_ns: type: integer format: int64 + description: Delay between two storage cells + fixed_gain_g1: + type: boolean + default: false + description: Fix gain to G1 (can be useful for storage cells) + use_gain_hg0: + type: boolean + default: false + description: Use high G0 (for low energy applications) spot_finding_settings: type: object required: @@ -308,9 +377,15 @@ components: properties: file_prefix: type: string + images_expected: + type: integer + format: int64 images_collected: type: integer format: int64 + images_sent: + type: integer + format: int64 max_image_number_sent: type: integer format: int64 @@ -374,12 +449,6 @@ components: example: 0.10 minimum: 0.0 maximum: 1.0 - receiver_send_buffers_avail: - type: number - format: float - example: 0.8 - minimum: 0.0 - maximum: 1.0 plot_request: type: object properties: @@ -828,10 +897,15 @@ paths: schema: type: string description: Exception error - /plot/adu_histogram: - get: - summary: Generate ADU histogram - description: ADU histogram for all images within current data collection + /plot/saturated_pixel: + post: + summary: Generate saturated pixels plot + description: Count of saturated pixels per image; binning is configurable + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/plot_request' responses: "200": description: Everything OK @@ -839,6 +913,128 @@ paths: application/json: schema: $ref: '#/components/schemas/plot' + "400": + description: Input parsing or validation error + content: + text/plain: + schema: + type: string + description: Exception error + /plot/error_pixel: + post: + summary: Generate error pixels plot + description: Count of error pixels per image; binning is configurable + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/plot_request' + responses: + "200": + description: Everything OK + content: + application/json: + schema: + $ref: '#/components/schemas/plot' + "400": + description: Input parsing or validation error + content: + text/plain: + schema: + type: string + description: Exception error + /plot/strong_pixel: + post: + summary: Generate strong pixels plot + description: Count of strong pixels per image (from spot finding); binning is configurable + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/plot_request' + responses: + "200": + description: Everything OK + content: + application/json: + schema: + $ref: '#/components/schemas/plot' + "400": + description: Input parsing or validation error + content: + text/plain: + schema: + type: string + description: Exception error + /plot/roi_sum: + post: + summary: Generate ROI sum plot + description: Sum of ROI rectangle per image; binning is configurable + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/plot_request' + responses: + "200": + description: Everything OK + content: + application/json: + schema: + $ref: '#/components/schemas/plot' + "400": + description: Input parsing or validation error + content: + text/plain: + schema: + type: string + description: Exception error + /plot/receiver_delay: + post: + summary: Generate receiver delay plot + description: Amount of frames the receiver is behind the FPGA for each image - used for internal debugging; binning is configurable + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/plot_request' + responses: + "200": + description: Everything OK + content: + application/json: + schema: + $ref: '#/components/schemas/plot' + "400": + description: Input parsing or validation error + content: + text/plain: + schema: + type: string + description: Exception error + /plot/image_collection_efficiency: + post: + summary: Generate image collection efficiency plot + description: Ratio of collected and expected packets per image; binning is configurable + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/plot_request' + responses: + "200": + description: Everything OK + content: + application/json: + schema: + $ref: '#/components/schemas/plot' + "400": + description: Input parsing or validation error + content: + text/plain: + schema: + type: string + description: Exception error /plot/indexing_rate_per_file: get: summary: Generate indexing rate per file diff --git a/broker/jfjoch_broker.cpp b/broker/jfjoch_broker.cpp index 8149912a..eabaf2ee 100644 --- a/broker/jfjoch_broker.cpp +++ b/broker/jfjoch_broker.cpp @@ -12,7 +12,7 @@ #include "JFJochBrokerHttp.h" #include "JFJochBrokerParser.h" -#include "../frame_serialize/ZMQStream2Pusher.h" +#include "../frame_serialize/ZMQStream2PusherGroup.h" static Pistache::Http::Endpoint *httpEndpoint; @@ -70,8 +70,8 @@ int main (int argc, char **argv) { } std::unique_ptr receiver; - std::unique_ptr image_pusher; - std::unique_ptr preview_publisher; + std::unique_ptr image_pusher; + std::unique_ptr preview_publisher; ZMQContext context; @@ -82,12 +82,12 @@ int main (int argc, char **argv) { ParseAcquisitionDeviceGroup(input, "devices", aq_devices); if (aq_devices.size() > 0) { experiment.DataStreams(aq_devices.size()); - image_pusher = std::make_unique(context, ParseStringArray(input, "zmq_image_addr")); + image_pusher = std::make_unique(context, ParseStringArray(input, "zmq_image_addr")); receiver = std::make_unique(aq_devices, logger, *image_pusher); std::string zmq_preview_addr = ParseString(input, "zmq_preview_addr"); if (!zmq_preview_addr.empty()) { - preview_publisher = std::make_unique(context, zmq_preview_addr); + preview_publisher = std::make_unique(context, zmq_preview_addr); receiver->PreviewPublisher(preview_publisher.get()); } @@ -101,27 +101,38 @@ int main (int argc, char **argv) { ParseFacilityConfiguration(input, "cfg", experiment); + logger.Info("Source {} Instrument {} Default rotation axis {:.2f},{:.2f},{:.2f}", + experiment.GetSourceName(), experiment.GetInstrumentName(), + experiment.GetDefaultOmegaAxis().x, + experiment.GetDefaultOmegaAxis().y, + experiment.GetDefaultOmegaAxis().z); + Pistache::Address addr(Pistache::Ipv4::any(), Pistache::Port(http_port)); httpEndpoint = new Pistache::Http::Endpoint((addr)); - std::vector sigs{SIGQUIT, SIGINT, SIGTERM, SIGHUP}; - setUpUnixSignals(sigs); - auto router = std::make_shared(); auto opts = Pistache::Http::Endpoint::options().threads(8); opts.flags(Pistache::Tcp::Options::ReuseAddr); httpEndpoint->init(opts); + std::vector sigs{SIGQUIT, SIGINT, SIGTERM, SIGHUP}; + setUpUnixSignals(sigs); + JFJochBrokerHttp broker(experiment, router); broker.FrontendDirectory(input["frontend_directory"]); ParseDetectorSetup(input, "detectors", broker); - if (receiver) - broker.Services().Receiver(receiver.get()); + if (input["verbose"].is_boolean()) { + logger.Verbose(input["verbose"]); + aq_devices.EnableLogging(&logger); + } - // broker.Services().Detector(); + if (receiver) { + broker.Services().Receiver(receiver.get()); + broker.Services().Detector(); + } httpEndpoint->setHandler(router->handler()); httpEndpoint->serve(); diff --git a/broker/redoc-static.html b/broker/redoc-static.html index 6ca64223..c0c7cb39 100644 --- a/broker/redoc-static.html +++ b/broker/redoc-static.html @@ -225,6 +225,8 @@ data-styled.g58[id="sc-gTgyBf"]{content:"ihUpGF,"}/*!sc*/ data-styled.g59[id="sc-laRQQM"]{content:"cREGdc,"}/*!sc*/ .iPuJRj{color:#666;}/*!sc*/ data-styled.g60[id="sc-iNqNmV"]{content:"iPuJRj,"}/*!sc*/ +.KUVXc{color:#666;word-break:break-word;}/*!sc*/ +data-styled.g61[id="sc-jeGSXX"]{content:"KUVXc,"}/*!sc*/ .giWqTo{color:#d41f1c;font-size:0.9em;font-weight:normal;margin-left:20px;line-height:1;}/*!sc*/ data-styled.g62[id="sc-eJMOVy"]{content:"giWqTo,"}/*!sc*/ .cEfEOv{border-radius:2px;word-break:break-word;background-color:rgba(51,51,51,0.05);color:rgba(51,51,51,0.9);padding:0 5px;border:1px solid rgba(51,51,51,0.1);font-family:Courier,monospace;}/*!sc*/ @@ -339,7 +341,7 @@ data-styled.g137[id="sc-cKZHtR"]{content:"lcoFQc,"}/*!sc*/ -
API docs by Redocly
API docs by Redocly

/entry/instrument/attenuator/attenuator_transmission Transmission of attenuator (filter) [no units]

+
object (rotation_axis)

Definition of a crystal rotation axis

+
header_appendix
string

Header appendix, added as user_data to start message

+
image_appendix
string

Image appendix, added as user_data to image message

+
object

Rectangle for ROI summation

object

Units of angstrom and degree

Responses

Request samples

Content type
application/json
{
  • "images_per_trigger": 0,
  • "ntrigger": 1,
  • "summation": 1,
  • "beam_x_pxl": 0,
  • "beam_y_pxl": 0,
  • "detector_distance_mm": 0,
  • "photon_energy_keV": 0,
  • "file_prefix": "",
  • "data_file_count": 1,
  • "space_group_number": 0,
  • "sample_name": "string",
  • "save_calibration": false,
  • "fpga_output": "auto",
  • "compression": "bslz4",
  • "total_flux": 0,
  • "transmission": 1,
  • "unit_cell": {
    }
}

Response samples

Content type
application/json
{
  • "msg": "Detector in wrong state",
  • "reason": "WrongDAQState"
}

Wait for acquisition done

Request samples

Content type
application/json
{
  • "images_per_trigger": 0,
  • "ntrigger": 1,
  • "summation": 1,
  • "beam_x_pxl": 0,
  • "beam_y_pxl": 0,
  • "detector_distance_mm": 0,
  • "photon_energy_keV": 0,
  • "file_prefix": "",
  • "data_file_count": 1,
  • "space_group_number": 0,
  • "sample_name": "string",
  • "save_calibration": false,
  • "fpga_output": "auto",
  • "compression": "bslz4",
  • "total_flux": 0,
  • "transmission": 1,
  • "omega": {
    },
  • "header_appendix": "string",
  • "image_appendix": "string",
  • "roi_sum_area": {
    },
  • "unit_cell": {
    }
}

Response samples

Content type
application/json
{
  • "msg": "Detector in wrong state",
  • "reason": "WrongDAQState"
}

Wait for acquisition done

Block execution of external script till initialization, data collection or pedestal is finished. @@ -491,17 +501,27 @@ If detector is in Inactive or Error states, new settin " class="sc-iJuWdM sc-cBNeRQ gpkGbA dujygE">

Interval between consecutive frames.

count_time_us
integer <int64>

Integration time of the detector. If not provided count time will be set to maximum value for a given frame time.

-
storage_cell_count
integer <int64> [ 1 .. 16 ]
Default: 1
internal_frame_generator
boolean
Default: false
collect_raw_data
boolean
Default: false
pedestal_g0_frames
integer <int64> >= 0
pedestal_g1_frames
integer <int64> >= 0
pedestal_g2_frames
integer <int64> >= 0
storage_cell_delay_ns
integer <int64>

Responses

Request samples

Content type
application/json
{
  • "frame_time_us": 450,
  • "count_time_us": 0,
  • "storage_cell_count": 1,
  • "internal_frame_generator": false,
  • "collect_raw_data": false,
  • "pedestal_g0_frames": 0,
  • "pedestal_g1_frames": 0,
  • "pedestal_g2_frames": 0,
  • "storage_cell_delay_ns": 0
}

Response samples

Content type
application/json
{
  • "msg": "Detector in wrong state",
  • "reason": "WrongDAQState"
}

Get detector configuration

Request samples

Content type
application/json
{
  • "frame_time_us": 450,
  • "count_time_us": 0,
  • "storage_cell_count": 1,
  • "internal_frame_generator": false,
  • "collect_raw_data": false,
  • "pedestal_g0_frames": 0,
  • "pedestal_g1_frames": 0,
  • "pedestal_g2_frames": 0,
  • "storage_cell_delay_ns": 0,
  • "fixed_gain_g1": false,
  • "use_gain_hg0": false
}

Response samples

Content type
application/json
{
  • "msg": "Detector in wrong state",
  • "reason": "WrongDAQState"
}

Get detector configuration

Can be done anytime

Responses

Response samples

Content type
application/json
{
  • "frame_time_us": 450,
  • "count_time_us": 0,
  • "storage_cell_count": 1,
  • "internal_frame_generator": false,
  • "collect_raw_data": false,
  • "pedestal_g0_frames": 0,
  • "pedestal_g1_frames": 0,
  • "pedestal_g2_frames": 0,
  • "storage_cell_delay_ns": 0
}

Configure spot finding

Response samples

Content type
application/json
{
  • "frame_time_us": 450,
  • "count_time_us": 0,
  • "storage_cell_count": 1,
  • "internal_frame_generator": false,
  • "collect_raw_data": false,
  • "pedestal_g0_frames": 0,
  • "pedestal_g1_frames": 0,
  • "pedestal_g2_frames": 0,
  • "storage_cell_delay_ns": 0,
  • "fixed_gain_g1": false,
  • "use_gain_hg0": false
}

Configure spot finding

Can be done anytime, also while data collection is running

Request Body schema: application/json
signal_to_noise_threshold
required
number <float> >= 0
photon_count_threshold
required
integer <int64> >= 0
min_pix_per_spot
required
integer <int64> >= 1
max_pix_per_spot
required
integer <int64> >= 1
high_resolution_limit
required
number <float>
low_resolution_limit
required
number <float>
preview_indexed_only
boolean
Default: false

Responses

Responses

Response samples

Content type
application/json
{
  • "state": "Inactive",
  • "progress": 1,
  • "indexing_rate": 0.1,
  • "receiver_send_buffers_avail": 0.8
}

Generate background estimate plot

Response samples

Content type
application/json
{
  • "state": "Inactive",
  • "progress": 1,
  • "indexing_rate": 0.1
}

Generate background estimate plot

Mean intensity for d = 3 - 5 A per image; binning is configurable

Request Body schema: application/json
binning
integer <int64>

Responses

Request samples

Content type
application/json
{
  • "binning": 0
}

Response samples

Content type
application/json
{
  • "x": [
    ],
  • "y": [
    ]
}

Generate ADU histogram

ADU histogram for all images within current data collection

-

Responses

Request samples

Content type
application/json
{
  • "binning": 0
}

Response samples

Content type
application/json
{
  • "x": [
    ],
  • "y": [
    ]
}

Generate saturated pixels plot

Count of saturated pixels per image; binning is configurable

+
Request Body schema: application/json
binning
integer <int64>

Responses

Response samples

Content type
application/json
{
  • "x": [
    ],
  • "y": [
    ]
}

Generate indexing rate per file

400

Input parsing or validation error

+

Request samples

Content type
application/json
{
  • "binning": 0
}

Response samples

Content type
application/json
{
  • "x": [
    ],
  • "y": [
    ]
}

Generate error pixels plot

Count of error pixels per image; binning is configurable

+
Request Body schema: application/json
binning
integer <int64>

Responses

Request samples

Content type
application/json
{
  • "binning": 0
}

Response samples

Content type
application/json
{
  • "x": [
    ],
  • "y": [
    ]
}

Generate strong pixels plot

Count of strong pixels per image (from spot finding); binning is configurable

+
Request Body schema: application/json
binning
integer <int64>

Responses

Request samples

Content type
application/json
{
  • "binning": 0
}

Response samples

Content type
application/json
{
  • "x": [
    ],
  • "y": [
    ]
}

Generate ROI sum plot

Sum of ROI rectangle per image; binning is configurable

+
Request Body schema: application/json
binning
integer <int64>

Responses

Request samples

Content type
application/json
{
  • "binning": 0
}

Response samples

Content type
application/json
{
  • "x": [
    ],
  • "y": [
    ]
}

Generate receiver delay plot

Amount of frames the receiver is behind the FPGA for each image - used for internal debugging; binning is configurable

+
Request Body schema: application/json
binning
integer <int64>

Responses

Request samples

Content type
application/json
{
  • "binning": 0
}

Response samples

Content type
application/json
{
  • "x": [
    ],
  • "y": [
    ]
}

Generate image collection efficiency plot

Ratio of collected and expected packets per image; binning is configurable

+
Request Body schema: application/json
binning
integer <int64>

Responses

Request samples

Content type
application/json
{
  • "binning": 0
}

Response samples

Content type
application/json
{
  • "x": [
    ],
  • "y": [
    ]
}

Generate indexing rate per file

Indexing rate per each of data files; useful for example for time resolved data

Responses

Response samples

Content type
application/json
{
  • "x": [
    ],
  • "y": [
    ]
}

Generate radial integration profile

Response samples

Content type
application/json
{
  • "x": [
    ],
  • "y": [
    ]
}

Generate radial integration profile

Generate average radial integration profile

Responses

Response samples

Content type
application/json
{
  • "x": [
    ],
  • "y": [
    ]
}

Generate radial integration profiles per file

Response samples

Content type
application/json
{
  • "x": [
    ],
  • "y": [
    ]
}

Generate radial integration profiles per file

Radial integration plots for both the whole dataset and per file; useful for time-resolved measurements

Responses

Response samples

Content type
application/json
[
  • {
    }
]

Get data collection statistics

Response samples

Content type
application/json
[
  • {
    }
]

Get data collection statistics

Results of the last data collection

Responses

Response samples

Content type
application/json
{
  • "file_prefix": "string",
  • "images_collected": 0,
  • "max_image_number_sent": 0,
  • "collection_efficiency": 1,
  • "compression_ratio": 5.3,
  • "cancelled": true,
  • "max_receiver_delay": 0,
  • "indexing_rate": 0,
  • "detector_width": 0,
  • "detector_height": 0,
  • "detector_pixel_depth": 2,
  • "bkg_estimate": 0
}

Get calibration statistics

Response samples

Content type
application/json
{
  • "file_prefix": "string",
  • "images_expected": 0,
  • "images_collected": 0,
  • "images_sent": 0,
  • "max_image_number_sent": 0,
  • "collection_efficiency": 1,
  • "compression_ratio": 5.3,
  • "cancelled": true,
  • "max_receiver_delay": 0,
  • "indexing_rate": 0,
  • "detector_width": 0,
  • "detector_height": 0,
  • "detector_pixel_depth": 2,
  • "bkg_estimate": 0
}

Get calibration statistics

Statistics are provided for each module/storage cell separately

Responses

Response samples

Content type
application/json
[
  • {
    }
]
+

Response samples

Content type
application/json
[
  • {
    }
]
+ + + + + +
API docs by Redocly

Jungfraujoch writer (1.0.0)

Download OpenAPI specification:Download

Jungfraujoch Writer Web API

+

Start writer

Responses

Will timeout after 5 seconds +If multiple parallel calls are made - only one will result in success +

Wait for writing done

+

Responses

Response samples

Content type
application/json
{
  • "nimages": 0,
  • "performance_MBs": 0,
  • "performance_Hz": 0
}

Cancel running data collection

It only instructs writer to cancel, but doesn't wait for cancellation actually happening. +It still requires to call /wait_till_done

+

Responses

Check if detector is done (similar to wait_till_do

Check if detector is done (similar to wait_till_done, but no timeout)

+

Responses

Response samples

Content type
application/json
{
  • "nimages": 0,
  • "performance_MBs": 0,
  • "performance_Hz": 0
}
+ + + + diff --git a/writer/writer_api.yaml b/writer/writer_api.yaml new file mode 100644 index 00000000..5245139f --- /dev/null +++ b/writer/writer_api.yaml @@ -0,0 +1,93 @@ +openapi: 3.0.3 +info: + title: Jungfraujoch writer + description: Jungfraujoch Writer Web API + version: 1.0.0 +components: + schemas: + writer_statistics: + type: object + properties: + nimages: + type: integer + format: uint64 + description: Number of images written + performance_MBs: + type: number + format: float + description: Performance in MB/s + performance_Hz: + type: number + format: float + description: Performance in images/s +paths: + /start: + post: + summary: Start writer + responses: + "200": + description: Initialization started + /wait_till_done: + get: + description: Wait for writing done + summary: | + Will timeout after 5 seconds + If multiple parallel calls are made - only one will result in success + responses: + "200": + description: Statistics of the last measurement + content: + application/json: + schema: + $ref: '#/components/schemas/writer_statistics' + "404": + description: No measurement running or finished recently + "500": + description: Measurement resulted with error + content: + application/json: + schema: + type: object + required: + - msg + properties: + msg: + type: string + description: Error message + "504": + description: 5 second timeout reached, need to restart operation + /cancel: + post: + summary: Cancel running data collection + description: | + It only instructs writer to cancel, but doesn't wait for cancellation actually happening. + It still requires to call `/wait_till_done` + responses: + "200": + description: Cancel message acknowledged + /check_if_done: + get: + description: Check if detector is done (similar to wait_till_done, but no timeout) + responses: + "200": + description: Statistics of the last measurement + content: + application/json: + schema: + $ref: '#/components/schemas/writer_statistics' + "404": + description: No measurement running or finished recently + "500": + description: Measurement resulted with error + content: + application/json: + schema: + type: object + required: + - msg + properties: + msg: + type: string + description: Error message + "504": + description: 5 second timeout reached, need to restart operation