From f7069ebf5e0d46222c8510b924a9e00816dd67df Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Thu, 16 Nov 2023 13:57:15 +0100 Subject: [PATCH] JFJochStateMachine: Reduce reliance on ProtoBuf (work in progress) --- broker/JFJochBroker.cpp | 104 ++++++++++++++++++++++-- broker/JFJochStateMachine.cpp | 122 ++++------------------------ broker/JFJochStateMachine.h | 14 ++-- tests/JFJochFullIntegrationTest.cpp | 38 ++++----- 4 files changed, 139 insertions(+), 139 deletions(-) diff --git a/broker/JFJochBroker.cpp b/broker/JFJochBroker.cpp index 26f6f65b..e04c8bf5 100644 --- a/broker/JFJochBroker.cpp +++ b/broker/JFJochBroker.cpp @@ -4,6 +4,100 @@ #define GRPC_RUN(x) try { {x;} return grpc::Status::OK; } catch (const std::exception& e) { logger.ErrorException(e); return {grpc::StatusCode::ABORTED, e.what()}; } + +inline DataProcessingSettings Convert(const JFJochProtoBuf::DataProcessingSettings &input) { + DataProcessingSettings ret; + ret.signal_to_noise_threshold = input.signal_to_noise_threshold(); + ret.photon_count_threshold = input.photon_count_threshold(); + ret.min_pix_per_spot = input.min_pix_per_spot(); + ret.max_pix_per_spot = input.max_pix_per_spot(); + ret.local_bkg_size = input.local_bkg_size(); + ret.high_resolution_limit = input.high_resolution_limit(); + ret.low_resolution_limit = input.low_resolution_limit(); + ret.bkg_estimate_low_q = input.bkg_estimate_low_q(); + ret.bkg_estimate_high_q = input.bkg_estimate_high_q(); + ret.preview_indexed_only = input.preview_indexed_only(); + return ret; +} + +inline JFJochProtoBuf::DataProcessingSettings Convert(const DataProcessingSettings &input) { + JFJochProtoBuf::DataProcessingSettings ret; + ret.set_signal_to_noise_threshold(input.signal_to_noise_threshold); + ret.set_photon_count_threshold(input.photon_count_threshold); + ret.set_min_pix_per_spot(input.min_pix_per_spot); + ret.set_max_pix_per_spot(input.max_pix_per_spot); + ret.set_local_bkg_size(input.local_bkg_size); + ret.set_high_resolution_limit(input.high_resolution_limit); + ret.set_low_resolution_limit(input.low_resolution_limit); + ret.set_bkg_estimate_low_q(input.bkg_estimate_low_q); + ret.set_bkg_estimate_high_q(input.bkg_estimate_high_q); + ret.set_preview_indexed_only(input.preview_indexed_only); + return ret; +} + +inline PlotRequest Convert(const JFJochProtoBuf::PlotRequest& request) { + PlotRequest ret; + ret.binning = request.binning(); + switch (request.type()) { + case JFJochProtoBuf::BKG_ESTIMATE: + ret.type = PlotType::BkgEstimate; + break; + case JFJochProtoBuf::RAD_INT: + ret.type = PlotType::RadInt; + break; + case JFJochProtoBuf::SPOT_COUNT: + ret.type = PlotType::SpotCount; + break; + case JFJochProtoBuf::INDEXING_RATE: + ret.type = PlotType::IndexingRate; + break; + case JFJochProtoBuf::INDEXING_RATE_PER_FILE: + ret.type = PlotType::IndexingRatePerFile; + break; + default: + case JFJochProtoBuf::ADU_HISTOGRAM: + ret.type = PlotType::ADUHistorgram; + break; + } + return ret; +} + +inline JFJochProtoBuf::Plot Convert(const Plot& input) { + JFJochProtoBuf::Plot output; + if (!input.x.empty()) + *output.mutable_x() = {input.x.begin(), input.x.end()}; + if (!input.y.empty()) + *output.mutable_y() = {input.y.begin(), input.y.end()}; + return output; +} + +inline JFJochProtoBuf::RadialIntegrationProfiles Convert(const RadialIntegrationProfiles& input) { + JFJochProtoBuf::RadialIntegrationProfiles output; + for (const auto &i: input.profiles) { + auto tmp = output.add_profiles(); + tmp->set_title(i.title); + *tmp->mutable_plot() = Convert(i.plot); + } + return output; +} + +inline JFJochProtoBuf::JFCalibrationStatistics Convert(const std::vector& input) { + JFJochProtoBuf::JFCalibrationStatistics output; + for (const auto& i: input) { + auto ret = output.add_module_statistics(); + ret->set_module_number(i.module_number); + ret->set_masked_pixels(i.masked_pixels); + ret->set_storage_cell_number(i.storage_cell_number); + ret->set_gain_g0_mean(i.gain_g0_mean); + ret->set_gain_g1_mean(i.gain_g1_mean); + ret->set_gain_g2_mean(i.gain_g2_mean); + ret->set_pedestal_g0_mean(i.pedestal_g0_mean); + ret->set_pedestal_g1_mean(i.pedestal_g1_mean); + ret->set_pedestal_g2_mean(i.pedestal_g2_mean); + } + return output; +} + JFJochBroker::JFJochBroker(const DiffractionExperiment &experiment) { state_machine.NotThreadSafe_Experiment() = experiment; } @@ -50,12 +144,12 @@ grpc::Status JFJochBroker::GetStatus(grpc::ServerContext *context, const JFJochP grpc::Status JFJochBroker::GetPlots(grpc::ServerContext *context, const JFJochProtoBuf::PlotRequest *request, JFJochProtoBuf::Plot *response) { - GRPC_RUN( *response = state_machine.GetPlots(*request) ); + GRPC_RUN( *response = Convert(state_machine.GetPlots(Convert(*request))) ); } grpc::Status JFJochBroker::GetRadialIntegrationProfiles(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request, JFJochProtoBuf::RadialIntegrationProfiles *response) { - GRPC_RUN( *response = state_machine.GetRadialIntegrationProfiles() ); + GRPC_RUN( *response = Convert(state_machine.GetRadialIntegrationProfiles()) ); } grpc::Status JFJochBroker::GetPedestalG0(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request, @@ -85,7 +179,7 @@ grpc::Status JFJochBroker::GetDetectorSettings(grpc::ServerContext *context, con grpc::Status JFJochBroker::GetCalibrationStatistics(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request, JFJochProtoBuf::JFCalibrationStatistics *response) { - GRPC_RUN( *response = state_machine.GetCalibrationStatistics() ); + GRPC_RUN( *response = Convert(state_machine.GetCalibrationStatistics()) ); } grpc::Status JFJochBroker::PutDetectorSettings(grpc::ServerContext *context, @@ -96,13 +190,13 @@ grpc::Status JFJochBroker::PutDetectorSettings(grpc::ServerContext *context, grpc::Status JFJochBroker::GetDataProcessingSettings(::grpc::ServerContext *context, const ::JFJochProtoBuf::Empty *request, ::JFJochProtoBuf::DataProcessingSettings *response) { - GRPC_RUN( *response = state_machine.GetDataProcessingSettings() ); + GRPC_RUN( *response = Convert(state_machine.GetDataProcessingSettings()) ); } grpc::Status JFJochBroker::PutDataProcessingSettings(::grpc::ServerContext *context, const ::JFJochProtoBuf::DataProcessingSettings *request, ::JFJochProtoBuf::Empty *response) { - GRPC_RUN( state_machine.SetDataProcessingSettings(*request) ); + GRPC_RUN( state_machine.SetDataProcessingSettings(Convert(*request)) ); } grpc::Status JFJochBroker::GetMeasurementStatistics(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request, diff --git a/broker/JFJochStateMachine.cpp b/broker/JFJochStateMachine.cpp index d6371561..80b892de 100644 --- a/broker/JFJochStateMachine.cpp +++ b/broker/JFJochStateMachine.cpp @@ -4,99 +4,6 @@ #include "JFJochStateMachine.h" -inline DataProcessingSettings Convert(const JFJochProtoBuf::DataProcessingSettings &input) { - DataProcessingSettings ret; - ret.signal_to_noise_threshold = input.signal_to_noise_threshold(); - ret.photon_count_threshold = input.photon_count_threshold(); - ret.min_pix_per_spot = input.min_pix_per_spot(); - ret.max_pix_per_spot = input.max_pix_per_spot(); - ret.local_bkg_size = input.local_bkg_size(); - ret.high_resolution_limit = input.high_resolution_limit(); - ret.low_resolution_limit = input.low_resolution_limit(); - ret.bkg_estimate_low_q = input.bkg_estimate_low_q(); - ret.bkg_estimate_high_q = input.bkg_estimate_high_q(); - ret.preview_indexed_only = input.preview_indexed_only(); - return ret; -} - -inline JFJochProtoBuf::DataProcessingSettings Convert(const DataProcessingSettings &input) { - JFJochProtoBuf::DataProcessingSettings ret; - ret.set_signal_to_noise_threshold(input.signal_to_noise_threshold); - ret.set_photon_count_threshold(input.photon_count_threshold); - ret.set_min_pix_per_spot(input.min_pix_per_spot); - ret.set_max_pix_per_spot(input.max_pix_per_spot); - ret.set_local_bkg_size(input.local_bkg_size); - ret.set_high_resolution_limit(input.high_resolution_limit); - ret.set_low_resolution_limit(input.low_resolution_limit); - ret.set_bkg_estimate_low_q(input.bkg_estimate_low_q); - ret.set_bkg_estimate_high_q(input.bkg_estimate_high_q); - ret.set_preview_indexed_only(input.preview_indexed_only); - return ret; -} - -inline PlotRequest Convert(const JFJochProtoBuf::PlotRequest& request) { - PlotRequest ret; - ret.binning = request.binning(); - switch (request.type()) { - case JFJochProtoBuf::BKG_ESTIMATE: - ret.type = PlotType::BkgEstimate; - break; - case JFJochProtoBuf::RAD_INT: - ret.type = PlotType::RadInt; - break; - case JFJochProtoBuf::SPOT_COUNT: - ret.type = PlotType::SpotCount; - break; - case JFJochProtoBuf::INDEXING_RATE: - ret.type = PlotType::IndexingRate; - break; - case JFJochProtoBuf::INDEXING_RATE_PER_FILE: - ret.type = PlotType::IndexingRatePerFile; - break; - default: - case JFJochProtoBuf::ADU_HISTOGRAM: - ret.type = PlotType::ADUHistorgram; - break; - } - return ret; -} - -inline JFJochProtoBuf::Plot Convert(const Plot& input) { - JFJochProtoBuf::Plot output; - if (!input.x.empty()) - *output.mutable_x() = {input.x.begin(), input.x.end()}; - if (!input.y.empty()) - *output.mutable_y() = {input.y.begin(), input.y.end()}; - return output; -} - -inline JFJochProtoBuf::RadialIntegrationProfiles Convert(const RadialIntegrationProfiles& input) { - JFJochProtoBuf::RadialIntegrationProfiles output; - for (const auto &i: input.profiles) { - auto tmp = output.add_profiles(); - tmp->set_title(i.title); - *tmp->mutable_plot() = Convert(i.plot); - } - return output; -} - -inline JFJochProtoBuf::JFCalibrationStatistics Convert(const std::vector& input) { - JFJochProtoBuf::JFCalibrationStatistics output; - for (const auto& i: input) { - auto ret = output.add_module_statistics(); - ret->set_module_number(i.module_number); - ret->set_masked_pixels(i.masked_pixels); - ret->set_storage_cell_number(i.storage_cell_number); - ret->set_gain_g0_mean(i.gain_g0_mean); - ret->set_gain_g1_mean(i.gain_g1_mean); - ret->set_gain_g2_mean(i.gain_g2_mean); - ret->set_pedestal_g0_mean(i.pedestal_g0_mean); - ret->set_pedestal_g1_mean(i.pedestal_g1_mean); - ret->set_pedestal_g2_mean(i.pedestal_g2_mean); - } - return output; -} - void LoadDatasetSettings(DiffractionExperiment& experiment, const JFJochProtoBuf::DatasetSettings &settings) { // Save DatasetSettings - if something goes wrong, restore old settings auto tmp = experiment; @@ -211,14 +118,14 @@ void JFJochStateMachine::ImportPedestalG0(const JFJochReceiverOutput &receiver_o calibration->Pedestal(module, 0, s) = receiver_output.pedestal_result[module + s * experiment.GetModulesNum()]; } - SetCalibrationStatistics(Convert(calibration->GetModuleStatistics())); + SetCalibrationStatistics(calibration->GetModuleStatistics()); } void JFJochStateMachine::ImportPedestal(const JFJochReceiverOutput &receiver_output, size_t gain_level, size_t storage_cell) { for (int i = 0; i < receiver_output.pedestal_result.size(); i++) calibration->Pedestal(i, gain_level, storage_cell) = receiver_output.pedestal_result[i]; - SetCalibrationStatistics(Convert(calibration->GetModuleStatistics())); + SetCalibrationStatistics(calibration->GetModuleStatistics()); } void JFJochStateMachine::TakePedestalInternalAll(std::unique_lock &ul) { @@ -543,12 +450,12 @@ void JFJochStateMachine::LoadMask(const std::vector &vec, uint32_t bit calibration->LoadMask(experiment, vec, bit); } -JFJochProtoBuf::JFCalibrationStatistics JFJochStateMachine::GetCalibrationStatistics() const { +std::vector JFJochStateMachine::GetCalibrationStatistics() const { std::unique_lock ul(calibration_statistics_mutex); return calibration_statistics; } -void JFJochStateMachine::SetCalibrationStatistics(const JFJochProtoBuf::JFCalibrationStatistics &input) { +void JFJochStateMachine::SetCalibrationStatistics(const std::vector &input) { std::unique_lock ul(calibration_statistics_mutex); calibration_statistics = input; } @@ -687,25 +594,24 @@ JFJochProtoBuf::BrokerStatus JFJochStateMachine::GetStatus() const { return ret; } -JFJochProtoBuf::Plot JFJochStateMachine::GetPlots(const JFJochProtoBuf::PlotRequest &request) const { - return Convert(services.GetPlots(Convert(request))); +Plot JFJochStateMachine::GetPlots(const PlotRequest &request) const { + return services.GetPlots(request); } -JFJochProtoBuf::RadialIntegrationProfiles JFJochStateMachine::GetRadialIntegrationProfiles() const { - return Convert(services.GetRadialIntegrationProfiles()); +RadialIntegrationProfiles JFJochStateMachine::GetRadialIntegrationProfiles() const { + return services.GetRadialIntegrationProfiles(); } -void JFJochStateMachine::SetDataProcessingSettings(const JFJochProtoBuf::DataProcessingSettings &settings) { +void JFJochStateMachine::SetDataProcessingSettings(const DataProcessingSettings &settings) { std::unique_lock ul(data_processing_settings_mutex); - auto tmp = Convert(settings); - DiffractionExperiment::CheckDataProcessingSettings(tmp); - data_processing_settings = tmp; - services.SetDataProcessingSettings(tmp); + DiffractionExperiment::CheckDataProcessingSettings(settings); + data_processing_settings = settings; + services.SetDataProcessingSettings(settings); } -JFJochProtoBuf::DataProcessingSettings JFJochStateMachine::GetDataProcessingSettings() const { +DataProcessingSettings JFJochStateMachine::GetDataProcessingSettings() const { std::unique_lock ul(data_processing_settings_mutex); - return Convert(data_processing_settings); + return data_processing_settings; } DataProcessingSettings JFJochStateMachine::GetDataAnalysisSettings() const { diff --git a/broker/JFJochStateMachine.h b/broker/JFJochStateMachine.h index c48bc93f..cc4d1582 100644 --- a/broker/JFJochStateMachine.h +++ b/broker/JFJochStateMachine.h @@ -39,7 +39,7 @@ class JFJochStateMachine { std::future measurement; mutable std::mutex calibration_statistics_mutex; - JFJochProtoBuf::JFCalibrationStatistics calibration_statistics; + std::vector calibration_statistics; mutable std::mutex last_receiver_output_mutex; std::optional measurement_statistics; @@ -75,26 +75,26 @@ public: void Cancel(); void LoadMask(const std::vector &vec, uint32_t bit); - void SetCalibrationStatistics(const JFJochProtoBuf::JFCalibrationStatistics &input); + void SetCalibrationStatistics(const std::vector &input); JFJochProtoBuf::DetectorSettings GetDetectorSettings() const; void SetDetectorSettings(const JFJochProtoBuf::DetectorSettings& settings); // return by value to ensure thread safety std::optional GetMeasurementStatistics() const; - JFJochProtoBuf::JFCalibrationStatistics GetCalibrationStatistics() const; + std::vector GetCalibrationStatistics() const; JFJochProtoBuf::BrokerStatus GetStatus() const; - JFJochProtoBuf::Plot GetPlots(const JFJochProtoBuf::PlotRequest &request) const; - JFJochProtoBuf::RadialIntegrationProfiles GetRadialIntegrationProfiles() const; + Plot GetPlots(const PlotRequest &request) const; + RadialIntegrationProfiles GetRadialIntegrationProfiles() const; JFJochProtoBuf::Image GetNeXusMask() const; JFJochProtoBuf::Image GetPedestalG0() const; JFJochProtoBuf::Image GetPedestalG1() const; JFJochProtoBuf::Image GetPedestalG2() const; - void SetDataProcessingSettings(const JFJochProtoBuf::DataProcessingSettings& settings); - JFJochProtoBuf::DataProcessingSettings GetDataProcessingSettings() const; + void SetDataProcessingSettings(const DataProcessingSettings& settings); + DataProcessingSettings GetDataProcessingSettings() const; DataProcessingSettings GetDataAnalysisSettings() const; JFJochState GetState() const; diff --git a/tests/JFJochFullIntegrationTest.cpp b/tests/JFJochFullIntegrationTest.cpp index e6d26561..e271dee4 100644 --- a/tests/JFJochFullIntegrationTest.cpp +++ b/tests/JFJochFullIntegrationTest.cpp @@ -792,14 +792,14 @@ TEST_CASE("JFJochIntegrationTest_ZMQ_lysozyme_spot", "[JFJochReceiver]") { setup.mutable_unit_cell()->set_gamma(90.0); setup.set_data_file_count(1); - JFJochProtoBuf::DataProcessingSettings settings; - settings.set_signal_to_noise_threshold(4); - settings.set_photon_count_threshold(5); - settings.set_min_pix_per_spot(3); - settings.set_max_pix_per_spot(200); - settings.set_low_resolution_limit(80.0); - settings.set_high_resolution_limit(2.5); - settings.set_local_bkg_size(5); + DataProcessingSettings settings; + settings.signal_to_noise_threshold = 4; + settings.photon_count_threshold = 5; + settings.min_pix_per_spot = 3; + settings.max_pix_per_spot = 200; + settings.low_resolution_limit = 80.0; + settings.high_resolution_limit = 2.5; + settings.local_bkg_size = 5; REQUIRE_NOTHROW(state_machine.SetDataProcessingSettings(settings)); @@ -1028,20 +1028,20 @@ TEST_CASE("JFJochIntegrationTest_ZMQ_lysozyme_rad_int", "[JFJochReceiver]") { logger.Info("Stopped measurement"); auto rad_int = state_machine.GetRadialIntegrationProfiles(); - REQUIRE(rad_int.profiles_size() == 4+1); - auto &plot_map = rad_int.profiles(); + REQUIRE(rad_int.profiles.size() == 4+1); + auto &plot_map = rad_int.profiles; - CHECK(plot_map[1].title() == "file0"); - CHECK(plot_map[1].plot().x_size() == 3); - CHECK(plot_map[1].plot().y_size() == 3); + CHECK(plot_map[1].title == "file0"); + CHECK(plot_map[1].plot.x.size() == 3); + CHECK(plot_map[1].plot.y.size() == 3); - CHECK(plot_map[4].title() == "file3"); - REQUIRE(plot_map[4].plot().x_size() == 3); - CHECK(plot_map[4].plot().x(0) == Approx(1.0)); - CHECK(plot_map[4].plot().y_size() == 3); + CHECK(plot_map[4].title == "file3"); + REQUIRE(plot_map[4].plot.x.size() == 3); + CHECK(plot_map[4].plot.x[0] == Approx(1.0)); + CHECK(plot_map[4].plot.y.size() == 3); - CHECK(plot_map[0].title() == "dataset"); - CHECK(plot_map[0].plot().x_size() == 3); + CHECK(plot_map[0].title == "dataset"); + CHECK(plot_map[0].plot.x.size() == 3); REQUIRE_NOTHROW(writer_future.get()); }