diff --git a/broker/JFJochBrokerHttp.cpp b/broker/JFJochBrokerHttp.cpp index dbb9196b6..2fdd17090 100644 --- a/broker/JFJochBrokerHttp.cpp +++ b/broker/JFJochBrokerHttp.cpp @@ -204,6 +204,13 @@ void JFJochBrokerHttp::register_routes(httplib::Server &server) { server.Put("/config/indexing", bind_json(&JFJochBrokerHttp::config_indexing_put, Indexing_settings{})); server.Get("/config/bragg_integration", bind_noarg(&JFJochBrokerHttp::config_bragg_integration_get)); server.Put("/config/bragg_integration", bind_json(&JFJochBrokerHttp::config_bragg_integration_put, Bragg_integration_settings{})); + server.Get("/config/analysis", bind_noarg(&JFJochBrokerHttp::config_analysis_get)); + server.Put("/config/analysis", bind_json(&JFJochBrokerHttp::config_analysis_put, Analysis_settings{})); + server.Get("/config/grid_scan_analysis", bind_noarg(&JFJochBrokerHttp::config_grid_scan_analysis_get)); + server.Put("/config/grid_scan_analysis", + bind_json(&JFJochBrokerHttp::config_grid_scan_analysis_put, Grid_scan_analysis_settings{})); + server.Get("/config/calibration", bind_noarg(&JFJochBrokerHttp::config_calibration_get)); + server.Put("/config/calibration", bind_json(&JFJochBrokerHttp::config_calibration_put, Calibration_settings{})); server.Get("/config/instrument", bind_noarg(&JFJochBrokerHttp::config_instrument_get)); server.Put("/config/instrument", bind_json(&JFJochBrokerHttp::config_instrument_put, Instrument_metadata{})); server.Put("/config/internal_generator_image", [this](const httplib::Request &req, httplib::Response &res) { @@ -601,6 +608,9 @@ void JFJochBrokerHttp::statistics_get(httplib::Response &response) { statistics.setBuffer(Convert(state_machine.GetImageBufferStatus())); statistics.setIndexing(Convert(state_machine.GetIndexingSettings())); statistics.setBraggIntegration(Convert(state_machine.GetBraggIntegrationSettings())); + statistics.setAnalysis(Convert(state_machine.GetAnalysisSettings())); + statistics.setGridScanAnalysis(Convert(state_machine.GetGridScanAnalysisSettings())); + statistics.setCalibrationSettings(Convert(state_machine.GetCalibrationSettings())); statistics.setDarkMask(Convert(state_machine.GetDarkMaskSettings())); statistics.setImagePusher(Convert(state_machine.GetImagePusherStatus())); @@ -845,6 +855,36 @@ void JFJochBrokerHttp::config_bragg_integration_put(const Bragg_integration_sett response.status = 200; } +void JFJochBrokerHttp::config_analysis_get(httplib::Response &response) { + ProcessOutput(Convert(state_machine.GetAnalysisSettings()), response); +} + +void JFJochBrokerHttp::config_analysis_put(const Analysis_settings &analysisSettings, + httplib::Response &response) { + state_machine.SetAnalysisSettings(Convert(analysisSettings)); + response.status = 200; +} + +void JFJochBrokerHttp::config_grid_scan_analysis_get(httplib::Response &response) { + ProcessOutput(Convert(state_machine.GetGridScanAnalysisSettings()), response); +} + +void JFJochBrokerHttp::config_grid_scan_analysis_put(const Grid_scan_analysis_settings &gridScanAnalysisSettings, + httplib::Response &response) { + state_machine.SetGridScanAnalysisSettings(Convert(gridScanAnalysisSettings)); + response.status = 200; +} + +void JFJochBrokerHttp::config_calibration_get(httplib::Response &response) { + ProcessOutput(Convert(state_machine.GetCalibrationSettings()), response); +} + +void JFJochBrokerHttp::config_calibration_put(const Calibration_settings &calibrationSettings, + httplib::Response &response) { + state_machine.SetCalibrationSettings(Convert(calibrationSettings)); + response.status = 200; +} + void JFJochBrokerHttp::result_scan_get(httplib::Response &response) { auto ret = state_machine.GetScanResult(); if (ret.has_value()) diff --git a/broker/JFJochBrokerHttp.h b/broker/JFJochBrokerHttp.h index 6d6ddc859..a3b52f311 100644 --- a/broker/JFJochBrokerHttp.h +++ b/broker/JFJochBrokerHttp.h @@ -181,6 +181,16 @@ class JFJochBrokerHttp { void config_bragg_integration_get(httplib::Response &response); void config_bragg_integration_put(const org::openapitools::server::model::Bragg_integration_settings &braggIntegrationSettings, httplib::Response &response); + void config_analysis_get(httplib::Response &response); + void config_analysis_put(const org::openapitools::server::model::Analysis_settings &analysisSettings, + httplib::Response &response); + void config_grid_scan_analysis_get(httplib::Response &response); + void config_grid_scan_analysis_put( + const org::openapitools::server::model::Grid_scan_analysis_settings &gridScanAnalysisSettings, + httplib::Response &response); + void config_calibration_get(httplib::Response &response); + void config_calibration_put(const org::openapitools::server::model::Calibration_settings &calibrationSettings, + httplib::Response &response); void config_dark_mask_get(httplib::Response &response); void config_dark_mask_put(const org::openapitools::server::model::Dark_mask_settings &darkMaskSettings, diff --git a/broker/JFJochBrokerParser.cpp b/broker/JFJochBrokerParser.cpp index 7f4efcfbc..b93d474ec 100644 --- a/broker/JFJochBrokerParser.cpp +++ b/broker/JFJochBrokerParser.cpp @@ -188,6 +188,20 @@ void ParseFacilityConfiguration(const org::openapitools::server::model::Jfjoch_s experiment.ImportBraggIntegrationSettings( BraggIntegrationSettings().MaxHKL(BRAGG_ONLINE_DEFAULT_MAX_HKL)); + if (j.analysisIsSet()) { + auto analysis = Convert(j.getAnalysis()); + CheckAnalysisSettingsOnline(analysis); + experiment.ImportAnalysisSettings(analysis); + } + + if (j.gridScanAnalysisIsSet()) + experiment.ImportGridScanAnalysisSettings(Convert(j.getGridScanAnalysis())); + + // After the mode, so that selecting PowderCalibration in the same file still lands the azimuthal + // integration the fit needs. + if (j.calibrationSettingsIsSet()) + experiment.ImportCalibrationSettings(Convert(j.getCalibrationSettings())); + if (j.darkMaskIsSet()) experiment.ImportDarkMaskSettings(Convert(j.getDarkMask())); } diff --git a/broker/JFJochStateMachine.cpp b/broker/JFJochStateMachine.cpp index 018904ceb..e17bd9e8d 100644 --- a/broker/JFJochStateMachine.cpp +++ b/broker/JFJochStateMachine.cpp @@ -1210,6 +1210,64 @@ void JFJochStateMachine::SetBraggIntegrationSettings(const BraggIntegrationSetti experiment.ImportBraggIntegrationSettings(input); } +void CheckAnalysisSettingsOnline(const AnalysisSettings &settings) { + // The REST schema has no mx_rotation, so this can only arrive from a caller built against a wider + // vocabulary than this API offers. Refuse it rather than accept it and quietly run stills: an + // analysis silently replaced by a different one is indistinguishable from the one that was asked + // for, and jfjoch_broker has no rotation analysis path to run. + if (settings.GetMode() == AnalysisMode::MXRotation) + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, + "Rotation MX analysis is not available online: jfjoch_broker has no rotation " + "analysis path. Collect with mx_stills, azint or none, and process the " + "stored file with rugnux"); +} + +AnalysisSettings JFJochStateMachine::GetAnalysisSettings() const { + std::unique_lock ul(experiment_indexing_settings_mutex); + return experiment.GetAnalysisSettings(); +} + +void JFJochStateMachine::SetAnalysisSettings(const AnalysisSettings &input) { + std::unique_lock ul(m); + if (IsRunning()) + throw WrongDAQStateException("Cannot change analysis settings during data collection"); + + CheckAnalysisSettingsOnline(input); + + // The analysis engines read the mode off the experiment when they are built at the start of the + // next run, so importing it here is all that is needed. + std::unique_lock ul2(experiment_indexing_settings_mutex); + experiment.ImportAnalysisSettings(input); +} + +GridScanAnalysisSettings JFJochStateMachine::GetGridScanAnalysisSettings() const { + std::unique_lock ul(experiment_indexing_settings_mutex); + return experiment.GetGridScanAnalysisSettings(); +} + +void JFJochStateMachine::SetGridScanAnalysisSettings(const GridScanAnalysisSettings &input) { + std::unique_lock ul(m); + if (IsRunning()) + throw WrongDAQStateException("Cannot change grid scan analysis settings during data collection"); + + std::unique_lock ul2(experiment_indexing_settings_mutex); + experiment.ImportGridScanAnalysisSettings(input); +} + +CalibrationSettings JFJochStateMachine::GetCalibrationSettings() const { + std::unique_lock ul(experiment_indexing_settings_mutex); + return experiment.GetCalibrationSettings(); +} + +void JFJochStateMachine::SetCalibrationSettings(const CalibrationSettings &input) { + std::unique_lock ul(m); + if (IsRunning()) + throw WrongDAQStateException("Cannot change calibration settings during data collection"); + + std::unique_lock ul2(experiment_indexing_settings_mutex); + experiment.ImportCalibrationSettings(input); +} + std::optional JFJochStateMachine::GetScanResult() const { std::unique_lock ul(m); if (IsRunning()) diff --git a/broker/JFJochStateMachine.h b/broker/JFJochStateMachine.h index 6ebea0938..46adcb983 100644 --- a/broker/JFJochStateMachine.h +++ b/broker/JFJochStateMachine.h @@ -72,6 +72,11 @@ struct MeasurementStatistics { std::optional roi_beam_sum; }; +// What jfjoch_broker will accept as an analysis mode. Both routes to the setting - the /config/analysis +// PUT and the facility configuration file - call it, so a config file cannot pin a mode the API would +// refuse and leave the broker in a state no client could have set. Throws InputParameterInvalid. +void CheckAnalysisSettingsOnline(const AnalysisSettings &settings); + class JFJochStateMachine { Logger &logger; JFJochServices &services; @@ -244,6 +249,15 @@ public: void SetBraggIntegrationSettings(const BraggIntegrationSettings &input); BraggIntegrationSettings GetBraggIntegrationSettings() const; + + void SetAnalysisSettings(const AnalysisSettings &input); + AnalysisSettings GetAnalysisSettings() const; + + void SetGridScanAnalysisSettings(const GridScanAnalysisSettings &input); + GridScanAnalysisSettings GetGridScanAnalysisSettings() const; + + void SetCalibrationSettings(const CalibrationSettings &input); + CalibrationSettings GetCalibrationSettings() const; PixelMaskStatistics GetPixelMaskStatistics() const; void GetStartMessageFromBuffer(std::vector &v); diff --git a/broker/OpenAPIConvert.cpp b/broker/OpenAPIConvert.cpp index 978feaa42..c87b96d7c 100644 --- a/broker/OpenAPIConvert.cpp +++ b/broker/OpenAPIConvert.cpp @@ -2,6 +2,7 @@ // SPDX-License-Identifier: GPL-3.0-only #include "../common/JFJochMath.h" +#include "../common/GitInfo.h" #include "OpenAPIConvert.h" // From https://en.cppreference.com/w/cpp/string/byte/tolower @@ -1124,6 +1125,204 @@ org::openapitools::server::model::Bragg_integration_settings Convert(const Bragg return ret; } +AnalysisSettings Convert(const org::openapitools::server::model::Analysis_settings &input) { + AnalysisSettings ret; + // No default: the compiler must object here when a mode is added to the API, rather than let the + // new value fall through to whatever the last branch was. MXRotation is not in the API at all - + // jfjoch_broker has no rotation analysis path - so there is no case for it. + switch (input.getMode().getValue()) { + case org::openapitools::server::model::Analysis_mode::eAnalysis_mode::INVALID_VALUE_OPENAPI_GENERATED: + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Unknown analysis mode"); + case org::openapitools::server::model::Analysis_mode::eAnalysis_mode::NONE: + ret.Mode(AnalysisMode::None); + break; + case org::openapitools::server::model::Analysis_mode::eAnalysis_mode::MXSTILLS: + ret.Mode(AnalysisMode::MXStills); + break; + case org::openapitools::server::model::Analysis_mode::eAnalysis_mode::AZINT: + ret.Mode(AnalysisMode::Azint); + break; + case org::openapitools::server::model::Analysis_mode::eAnalysis_mode::GRID: + ret.Mode(AnalysisMode::Grid); + break; + case org::openapitools::server::model::Analysis_mode::eAnalysis_mode::POWDERCALIBRATION: + ret.Mode(AnalysisMode::PowderCalibration); + break; + } + + return ret; +} + +org::openapitools::server::model::Analysis_settings Convert(const AnalysisSettings &input) { + org::openapitools::server::model::Analysis_settings ret; + org::openapitools::server::model::Analysis_mode tmp; + switch (input.GetMode()) { + case AnalysisMode::None: + tmp.setValue(org::openapitools::server::model::Analysis_mode::eAnalysis_mode::NONE); + break; + case AnalysisMode::MXStills: + tmp.setValue(org::openapitools::server::model::Analysis_mode::eAnalysis_mode::MXSTILLS); + break; + case AnalysisMode::Azint: + tmp.setValue(org::openapitools::server::model::Analysis_mode::eAnalysis_mode::AZINT); + break; + case AnalysisMode::Grid: + tmp.setValue(org::openapitools::server::model::Analysis_mode::eAnalysis_mode::GRID); + break; + case AnalysisMode::PowderCalibration: + tmp.setValue(org::openapitools::server::model::Analysis_mode::eAnalysis_mode::POWDERCALIBRATION); + break; + case AnalysisMode::MXRotation: + // Unreachable through the API: the setter refuses it, so it can never be stored. Reported + // rather than silently rewritten to stills, which would be the broker claiming to run an + // analysis it does not have. + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, + "Rotation MX analysis cannot be reported by this API"); + } + ret.setMode(tmp); + return ret; +} + +GridScanAnalysisSettings Convert(const org::openapitools::server::model::Grid_scan_analysis_settings &input) { + GridScanAnalysisSettings ret; + ret.ProteinScoreThreshold(input.getProteinScoreThreshold()); + ret.MinBlobCells(input.getMinBlobCells()); + ret.DecisiveSingleCellScore(input.getDecisiveSingleCellScore()); + ret.MaxCrystals(input.getMaxCrystals()); + ret.Indexing(input.isIndexing()); + return ret; +} + +org::openapitools::server::model::Grid_scan_analysis_settings Convert(const GridScanAnalysisSettings &input) { + org::openapitools::server::model::Grid_scan_analysis_settings ret; + ret.setProteinScoreThreshold(input.GetProteinScoreThreshold()); + ret.setMinBlobCells(input.GetMinBlobCells()); + ret.setDecisiveSingleCellScore(input.GetDecisiveSingleCellScore()); + ret.setMaxCrystals(input.GetMaxCrystals()); + ret.setIndexing(input.IsIndexing()); + return ret; +} + +CalibrationSettings Convert(const org::openapitools::server::model::Calibration_settings &input) { + CalibrationSettings ret; + // No default: the compiler must object here when a source is added to the API. + switch (input.getMethod().getValue()) { + case org::openapitools::server::model::Calibration_method::eCalibration_method::INVALID_VALUE_OPENAPI_GENERATED: + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Unknown calibration method"); + case org::openapitools::server::model::Calibration_method::eCalibration_method::RINGS: + ret.Method(CalibrationMethod::Rings); + break; + case org::openapitools::server::model::Calibration_method::eCalibration_method::SPOTS: + ret.Method(CalibrationMethod::Spots); + break; + } + + if (input.calibrantIsSet()) + ret.Calibrant(input.getCalibrant()); + + return ret; +} + +org::openapitools::server::model::Calibration_settings Convert(const CalibrationSettings &input) { + org::openapitools::server::model::Calibration_settings ret; + org::openapitools::server::model::Calibration_method tmp; + switch (input.GetMethod()) { + case CalibrationMethod::Rings: + tmp.setValue(org::openapitools::server::model::Calibration_method::eCalibration_method::RINGS); + break; + case CalibrationMethod::Spots: + tmp.setValue(org::openapitools::server::model::Calibration_method::eCalibration_method::SPOTS); + break; + } + ret.setMethod(tmp); + if (!input.GetCalibrant().empty()) + ret.setCalibrant(input.GetCalibrant()); + return ret; +} + +org::openapitools::server::model::Powder_calibration_output Convert(const CalibrationResult &input) { + org::openapitools::server::model::Powder_calibration_output ret; + const auto &g = input.geometry; + + // The geometry under this API's own property names, so it can be PUT back without translating a + // field. Same content as the .json rugnux writes (WriteCalibrationJson). + org::openapitools::server::model::Dataset_settings settings; + settings.setBeamXPxl(g.GetBeamX_pxl()); + settings.setBeamYPxl(g.GetBeamY_pxl()); + settings.setDetectorDistanceMm(g.GetDetectorDistance_mm()); + settings.setIncidentEnergyKeV(WVL_1A_IN_KEV / g.GetWavelength_A()); + // The rotations go in together or not at all: omitting them would state a FLAT detector, since the + // schema's default is 0.0, rather than leave the tilt unstated. + if (g.GetPoniRot1_rad() != 0.0f || g.GetPoniRot2_rad() != 0.0f || g.GetPoniRot3_rad() != 0.0f) { + settings.setPoniRot1Rad(g.GetPoniRot1_rad()); + settings.setPoniRot2Rad(g.GetPoniRot2_rad()); + settings.setPoniRot3Rad(g.GetPoniRot3_rad()); + } + ret.setDatasetSettings(settings); + + org::openapitools::server::model::Powder_calibration_quality quality; + // The verdict first: a caller that reads nothing else must still not mistake a non-fit for a fit. + quality.setConverged(input.converged); + if (!input.converged) + quality.setNotConvergedReason(input.reason); + quality.setRingPoints(input.ring_points); + quality.setRmsRadialPxl(input.rms_radial_pxl); + quality.setBeamSigmaPxl(input.beam_sigma_pxl); + // Where the beam actually lands, which is not the PONI above once the detector is tilted. + const auto [direct_x, direct_y] = g.GetDirectBeam_pxl(); + quality.setDirectBeamXPxl(direct_x); + quality.setDirectBeamYPxl(direct_y); + quality.setHeaderDistanceMm(input.header_distance_mm); + quality.setTiltRefined(input.tilt_refined); + quality.setTiltSignificance(input.tilt_significance); + if (input.seed_distance_mm > 0.0f) + quality.setRingSeedDistanceMm(input.seed_distance_mm); + + if (input.uncertainty.valid) { + org::openapitools::server::model::Powder_calibration_fit_sigma sigma; + sigma.setBeamXPxl(input.uncertainty.sigma_beam_x_pxl); + sigma.setBeamYPxl(input.uncertainty.sigma_beam_y_pxl); + sigma.setDetectorDistanceMm(input.uncertainty.sigma_distance_mm); + if (input.uncertainty.sigma_rot1_rad > 0.0 || input.uncertainty.sigma_rot2_rad > 0.0) { + sigma.setPoniRot1Rad(input.uncertainty.sigma_rot1_rad); + sigma.setPoniRot2Rad(input.uncertainty.sigma_rot2_rad); + sigma.setCorrelationBeamXRot1(input.uncertainty.corr_beam_x_rot1); + sigma.setCorrelationBeamYRot2(input.uncertainty.corr_beam_y_rot2); + } + quality.setFitSigma(sigma); + } + + if (input.spots_geometry) { + org::openapitools::server::model::Powder_calibration_spot_check cross; + cross.setBeamXPxl(input.spots_geometry->GetBeamX_pxl()); + cross.setBeamYPxl(input.spots_geometry->GetBeamY_pxl()); + cross.setDetectorDistanceMm(input.spots_geometry->GetDetectorDistance_mm()); + cross.setDisagreementPxl(input.spots_disagreement_pxl); + quality.setSpotCrossCheck(cross); + } + + ret.setCalibration(quality); + ret.setJfjochVersion(jfjoch_version()); + return ret; +} + +org::openapitools::server::model::Grid_scan_crystal Convert(const GridScanCrystal &input) { + org::openapitools::server::model::Grid_scan_crystal ret; + ret.setNx(input.nx); + ret.setNy(input.ny); + ret.setXUm(input.x_um); + ret.setYUm(input.y_um); + ret.setImageNumber(input.image_number); + ret.setMajorUm(input.major_um); + ret.setMinorUm(input.minor_um); + ret.setAngleDeg(input.angle_deg); + ret.setScore(input.score); + ret.setIceScore(input.ice_score); + ret.setResA(input.res_A); + ret.setNImages(input.n_images); + return ret; +} + org::openapitools::server::model::Scan_result Convert(const ScanResult& input) { org::openapitools::server::model::Scan_result ret; ret.setFilePrefix(input.file_prefix); @@ -1204,6 +1403,22 @@ org::openapitools::server::model::Scan_result Convert(const ScanResult& input) { } if (input.rotation_crystal_system && input.rotation_centering) ret.setRotationBravais(BravaisSymbol(*input.rotation_crystal_system, *input.rotation_centering)); + + // The two mode-specific payloads. Optional and independent, exactly as the rotation_* members + // above are: present when that mode ran, absent otherwise. + if (input.grid) { + org::openapitools::server::model::Grid_scan_result grid; + std::vector crystals; + for (const auto &c : input.grid->crystals) + crystals.emplace_back(Convert(c)); + grid.setCrystals(crystals); + grid.setBeamSizeXUm(input.grid->beam_size_x_um); + grid.setBeamSizeYUm(input.grid->beam_size_y_um); + ret.setGrid(grid); + } + if (input.calibration) + ret.setCalibration(Convert(input.calibration.value())); + return ret; } diff --git a/broker/OpenAPIConvert.h b/broker/OpenAPIConvert.h index fd36ea1da..e7e6e98f3 100644 --- a/broker/OpenAPIConvert.h +++ b/broker/OpenAPIConvert.h @@ -29,6 +29,11 @@ #include "gen/model/Grid_scan.h" #include "gen/model/Indexing_settings.h" #include "gen/model/Bragg_integration_settings.h" +#include "gen/model/Analysis_settings.h" +#include "gen/model/Grid_scan_analysis_settings.h" +#include "gen/model/Calibration_settings.h" +#include "gen/model/Grid_scan_result.h" +#include "gen/model/Powder_calibration_output.h" #include "gen/model/Scan_result.h" #include "../common/JFJochMessages.h" @@ -40,6 +45,9 @@ #include "../jungfrau/JFCalibration.h" #include "../common/InstrumentMetadata.h" #include "../common/ScanResult.h" +#include "../common/AnalysisSettings.h" +#include "../common/GridScanAnalysisSettings.h" +#include "../common/CalibrationSettings.h" #include "../image_pusher/ImagePusher.h" SpotFindingSettings Convert(const org::openapitools::server::model::Spot_finding_settings &input); @@ -51,6 +59,18 @@ org::openapitools::server::model::Indexing_settings Convert(const IndexingSettin BraggIntegrationSettings Convert(const org::openapitools::server::model::Bragg_integration_settings &input); org::openapitools::server::model::Bragg_integration_settings Convert(const BraggIntegrationSettings &input); +AnalysisSettings Convert(const org::openapitools::server::model::Analysis_settings &input); +org::openapitools::server::model::Analysis_settings Convert(const AnalysisSettings &input); + +GridScanAnalysisSettings Convert(const org::openapitools::server::model::Grid_scan_analysis_settings &input); +org::openapitools::server::model::Grid_scan_analysis_settings Convert(const GridScanAnalysisSettings &input); + +CalibrationSettings Convert(const org::openapitools::server::model::Calibration_settings &input); +org::openapitools::server::model::Calibration_settings Convert(const CalibrationSettings &input); + +org::openapitools::server::model::Grid_scan_crystal Convert(const GridScanCrystal &input); +org::openapitools::server::model::Powder_calibration_output Convert(const CalibrationResult &input); + org::openapitools::server::model::Measurement_statistics Convert(const MeasurementStatistics &input); DetectorSettings Convert(const org::openapitools::server::model::Detector_settings &input); diff --git a/broker/jfjoch_api.yaml b/broker/jfjoch_api.yaml index da6a44297..6adf5c891 100644 --- a/broker/jfjoch_api.yaml +++ b/broker/jfjoch_api.yaml @@ -618,7 +618,14 @@ components: $ref: '#/components/schemas/unit_cell' spot_finding: type: boolean - description: Enable spot finding and save spots + deprecated: true + description: | + Enable spot finding and save spots. + + DEPRECATED - use the analysis mode instead (/config/analysis). The mode takes precedence: a + mode that analyses no spots (None, Azint, PowderCalibration) switches spot finding off + whatever this says, and under a mode that does find spots this remains a finer control that + can still turn it off. It is kept while callers move over. default: true smargon: type: object @@ -1074,9 +1081,15 @@ components: enable: type: boolean default: true + deprecated: true description: | Enable spot finding. This is temporary setting, i.e. can be changed anytime during data collection. Even if disabled spot finding information will still be send and written, though always with zero spots. + + DEPRECATED - use the analysis mode instead (/config/analysis). The mode takes precedence: a + mode that analyses no spots (None, Azint, PowderCalibration) switches spot finding off + whatever this says, and under a mode that does find spots this remains a finer control that + can still turn it off. It is kept while callers move over. indexing: type: boolean default: true @@ -1597,6 +1610,89 @@ components: type: array items: $ref: '#/components/schemas/plot' + grid_scan_crystal: + type: object + description: One crystal found in a grid scan. + required: + - nx + - ny + - score + properties: + nx: + type: number + format: float + description: Centre, fractional grid coordinate along the fast grid axis + ny: + type: number + format: float + description: Centre, fractional grid coordinate along the slow grid axis + x_um: + type: number + format: float + description: Centre, signed offset along the fast grid axis, in micrometres + y_um: + type: number + format: float + description: Centre, signed offset along the slow grid axis, in micrometres + image_number: + type: integer + format: int64 + description: The grid point at the centre, as an image ordinal + major_um: + type: number + format: float + description: Extent along the crystal's major principal axis, in micrometres + minor_um: + type: number + format: float + description: Extent along its minor principal axis, in micrometres + angle_deg: + type: number + format: float + description: | + Direction of the major axis from the +x grid axis, counter-clockwise, in degrees. This is + an AXIS, not a direction: it lives in [0, 180) and wraps, so 179 and 0 are adjacent. + Averaging two of these arithmetically across the wrap turns two nearly parallel needles + into a right angle - combine them on the doubled angle (mean of 2*theta, halved), or not + at all. + score: + type: number + format: float + description: Diffraction score the crystal was ranked on + ice_score: + type: number + format: float + description: Ice-ring score over the crystal's grid points + res_A: + type: number + format: float + description: Resolution estimate over the crystal's grid points, in Angstrom + n_images: + type: integer + format: int64 + description: Grid points that fell in this crystal + grid_scan_result: + type: object + description: | + What a grid scan found. The list is sorted by score, best first, and may hold any number of + crystals - a consumer must not assume at most one. + required: + - crystals + properties: + crystals: + type: array + items: + $ref: '#/components/schemas/grid_scan_crystal' + beam_size_x_um: + type: number + format: float + description: | + The beam the crystal extents were measured with, along the fast grid axis. Those extents + still contain it, so this says what a consumer has to take back out. + beam_size_y_um: + type: number + format: float + description: The same along the slow grid axis scan_result: type: object description: Results of a scan @@ -1614,6 +1710,10 @@ components: description: | Bravais lattice type of the global rotation-indexing solution: crystal-system letter + centering, e.g. "tP", "oC", "cF", "hR", "mP". Empty if not determined. + grid: + $ref: '#/components/schemas/grid_scan_result' + calibration: + $ref: '#/components/schemas/powder_calibration_output' images: type: array items: @@ -1983,6 +2083,12 @@ components: $ref: '#/components/schemas/indexing_settings' bragg_integration: $ref: '#/components/schemas/bragg_integration_settings' + analysis: + $ref: '#/components/schemas/analysis_settings' + grid_scan_analysis: + $ref: '#/components/schemas/grid_scan_analysis_settings' + calibration_settings: + $ref: '#/components/schemas/calibration_settings' image_pusher: $ref: '#/components/schemas/image_pusher_status' error_message: @@ -2585,6 +2691,137 @@ components: type: string example: 10.1.1.7 description: IPv4 address of the block device + analysis_mode: + type: string + description: | + What analysis jfjoch_broker runs over the images. One setting replaces the several independent + switches - spot finding on/off in two places, indexing on/off, rotation on/off - that used to be + composed at each site into an answer nobody had written down. + None - images are received, written and streamed; nothing is analysed. + MXStills - spot finding, per-image indexing, refinement and integration. The default, and what + an out-of-the-box broker has always done. + Azint - azimuthal integration only; no spots are found. + Grid - grid scan: per-image scoring and crystal selection. + PowderCalibration - detector geometry from a calibrant's powder rings. Forces azimuthal + integration onto the CPU, since the FPGA core cannot hold enough bins for the sectored + profile a ring fit needs; a calibration exposure is a few images at a few Hz, so the cost + does not matter. + + Rotation MX analysis is deliberately absent: jfjoch_broker has no rotation analysis path, so + this API cannot express it. Rotation data is collected here and processed offline with rugnux. + enum: + - "None" + - "MXStills" + - "Azint" + - "Grid" + - "PowderCalibration" + default: "MXStills" + analysis_settings: + type: object + description: | + What every analysis method shares, and nothing else. Each method's own parameters have their + own settings object and their own endpoint - grid_scan_analysis_settings + (/config/grid_scan_analysis), calibration_settings (/config/calibration) - so that adding a + parameter to one method does not widen the structure every method reads. + + Persistent: it survives a data collection, unlike the per-dataset settings given to /start. + required: + - mode + properties: + mode: + $ref: '#/components/schemas/analysis_mode' + calibration_method: + type: string + description: | + How the powder rings the detector geometry is fitted to are measured. + Rings - the run-summed (q x azimuth) azimuthal profile: the ring measured at every azimuth. + Spots - the pooled per-image spot lists: the ring sampled wherever the spot finder bit. + enum: + - "Rings" + - "Spots" + default: "Rings" + calibration_settings: + type: object + description: | + Settings for analysis mode PowderCalibration and nothing else. + + Selecting that mode also moves azimuthal integration onto the CPU and, where fewer than four + azimuthal sectors were asked for, sets a usable count. Ring positions are read off a + (q x azimuth) profile, and the FPGA integration core holds 2048 bins in total - at 32 sectors + that leaves 64 q bins, far too coarse to locate a ring. The frame rate this costs is not frame + rate a calibration exposure needs. + required: + - method + properties: + calibrant: + type: string + description: | + Which calibrant's ring d-spacings the geometry is fitted against. Empty selects the + processing program's own default rather than meaning "no calibrant". + method: + $ref: '#/components/schemas/calibration_method' + grid_scan_analysis_settings: + type: object + description: | + Settings for analysis mode Grid and nothing else: how the per-image protein score is turned + into a list of crystals, and whether the raster indexes as well as scores. + required: + - protein_score_threshold + - min_blob_cells + - decisive_single_cell_score + - max_crystals + - indexing + properties: + protein_score_threshold: + type: number + format: float + minimum: 0 + maximum: 1 + default: 0.5 + description: | + A raster cell counts as protein above this. The per-image protein score saturates, so this + only has to separate "something diffracted here" from "nothing did". + min_blob_cells: + type: integer + format: int64 + minimum: 1 + default: 3 + description: | + How many cells above that threshold make a shape rather than a coincidence. Two cells can + be the two ends of a single hit lying on a cell boundary; three is the smallest patch that + is not. + decisive_single_cell_score: + type: number + format: float + minimum: 0 + maximum: 1 + default: 0.9 + description: | + ...unless one cell on its own is decisive. The rule above is about coincidences, and a + lone cell scoring near the top of a saturating score is not one - a crystal smaller than + the grid step lights exactly one cell, and refusing it would lose precisely the samples a + fine raster is run to find. Set well above protein_score_threshold: this admits the + obvious case, it does not lower the general threshold by the back door. + max_crystals: + type: integer + format: int64 + minimum: 1 + default: 10 + description: | + Most crystals reported. A raster over a loop full of shards can label dozens of blobs, and + past the first few the list is no longer a ranking anyone acts on. Crystals are sorted by + score, so this keeps the best. + indexing: + type: boolean + default: true + description: | + Whether each raster cell is indexed as well as scored. On by default: a raster runs at up + to 100 Hz, which the FFT indexer keeps up with, and it is additive - blobs are still found + on the protein score, so indexing changes nothing about which cells are called crystals + and only adds what was found in them. The lattice count per cell is the cheapest + multi-lattice or cracked-crystal signal there is, and on a fixed-target serial experiment + with a known cell a raster that indexes is most of the measurement. Turn it off for a very + large raster where the GPU is the constraint. bragg_integration_settings: type: object description: "Settings for Bragg spot integration" @@ -2637,6 +2874,12 @@ components: $ref: '#/components/schemas/indexing_settings' bragg_integration: $ref: '#/components/schemas/bragg_integration_settings' + analysis: + $ref: '#/components/schemas/analysis_settings' + grid_scan_analysis: + $ref: '#/components/schemas/grid_scan_analysis_settings' + calibration_settings: + $ref: '#/components/schemas/calibration_settings' detector_settings: $ref: '#/components/schemas/detector_settings' azim_int: @@ -3017,6 +3260,117 @@ paths: application/json: schema: $ref: '#/components/schemas/bragg_integration_settings' + /config/analysis: + put: + summary: Change analysis settings + description: | + This can only be done when detector is `Idle`, `Error` or `Inactive` states. + The mode is persistent - it is not reset by a data collection. + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/analysis_settings' + responses: + "200": + description: Everything OK + "400": + description: Input parsing or validation error + content: + text/plain: + schema: + type: string + description: Exception error + "500": + description: Error within Jungfraujoch code - see output message. + content: + application/json: + schema: + $ref: '#/components/schemas/error_message' + get: + summary: Get analysis configuration + description: Can be done anytime + responses: + "200": + description: Everything OK + content: + application/json: + schema: + $ref: '#/components/schemas/analysis_settings' + /config/grid_scan_analysis: + put: + summary: Change grid scan analysis settings + description: | + This can only be done when detector is `Idle`, `Error` or `Inactive` states. + Persistent - not reset by a data collection. + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/grid_scan_analysis_settings' + responses: + "200": + description: Everything OK + "400": + description: Input parsing or validation error + content: + text/plain: + schema: + type: string + description: Exception error + "500": + description: Error within Jungfraujoch code - see output message. + content: + application/json: + schema: + $ref: '#/components/schemas/error_message' + get: + summary: Get grid scan analysis configuration + description: Can be done anytime + responses: + "200": + description: Everything OK + content: + application/json: + schema: + $ref: '#/components/schemas/grid_scan_analysis_settings' + /config/calibration: + put: + summary: Change powder calibration settings + description: | + This can only be done when detector is `Idle`, `Error` or `Inactive` states. + Persistent - not reset by a data collection. + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/calibration_settings' + responses: + "200": + description: Everything OK + "400": + description: Input parsing or validation error + content: + text/plain: + schema: + type: string + description: Exception error + "500": + description: Error within Jungfraujoch code - see output message. + content: + application/json: + schema: + $ref: '#/components/schemas/error_message' + get: + summary: Get powder calibration configuration + description: Can be done anytime + responses: + "200": + description: Everything OK + content: + application/json: + schema: + $ref: '#/components/schemas/calibration_settings' /config/file_writer: put: summary: Change file writer settings diff --git a/common/AnalysisSettings.cpp b/common/AnalysisSettings.cpp new file mode 100644 index 000000000..1e1be5a89 --- /dev/null +++ b/common/AnalysisSettings.cpp @@ -0,0 +1,98 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include "AnalysisSettings.h" +#include "GridScanAnalysisSettings.h" + +// The mode-to-analysis table. One place, one row per mode, and every gate in the pipeline reads it - +// so what a mode does can be read off here instead of being reconstructed from `if (mode == ...)` +// scattered through the engines. +// +// spots index bragg merge score azint +// None - - - - - - +// MXRotation x x x x x x +// MXStills x x x x x x +// Azint - - - - - x +// Grid x * - - x x * = default; a setting +// PowderCalibration x - - - - x +// +// Two rows want a word. +// +// PowderCalibration keeps spot finding. The geometry is fitted either to the ring arcs of the summed +// (q x azimuth) profile or to the POOLED SPOTS (rugnux --calibration rings|spots), and the second of +// those is a spot finder run - so the mode must not decide this; whoever picks the source does. +// +// Grid indexes by default, but that entry is not the mode's to fix: it is +// GridScanAnalysisSettings::indexing, and what stands here is only what that setting defaults to. A +// raster runs at up to 100 Hz, which the FFT indexer keeps up with, and a fixed-target serial +// experiment with a known cell wants ffbidx on every cell - so whether a raster indexes is a property +// of the experiment, not of rasters. DiffractionExperiment::GetAnalysisStages substitutes the +// configured value. +// +// The switch has no default: adding a mode must be a compile error here rather than a silent +// fall-through to whatever the last row happened to be. +AnalysisStages AnalysisModeStages(AnalysisMode mode) { + switch (mode) { + case AnalysisMode::None: + return {false, false, false, false, false, false}; + case AnalysisMode::MXRotation: + case AnalysisMode::MXStills: + return {true, true, true, true, true, true}; + case AnalysisMode::Azint: + return {false, false, false, false, false, true}; + case AnalysisMode::Grid: + return {true, GridScanAnalysisSettings().IsIndexing(), false, false, true, true}; + case AnalysisMode::PowderCalibration: + return {true, false, false, false, false, true}; + } + return {false, false, false, false, false, false}; +} + +bool AnalysisModeIsMX(AnalysisMode mode) { + switch (mode) { + case AnalysisMode::MXRotation: + case AnalysisMode::MXStills: + return true; + case AnalysisMode::None: + case AnalysisMode::Azint: + case AnalysisMode::Grid: + case AnalysisMode::PowderCalibration: + return false; + } + return false; +} + +std::string AnalysisModeName(AnalysisMode mode) { + switch (mode) { + case AnalysisMode::None: return "none"; + case AnalysisMode::MXRotation: return "mx_rotation"; + case AnalysisMode::MXStills: return "mx_stills"; + case AnalysisMode::Azint: return "azint"; + case AnalysisMode::Grid: return "grid"; + case AnalysisMode::PowderCalibration: return "powder_calibration"; + } + return "none"; +} + +std::optional AnalysisModeFromName(std::string_view name) { + if (name == "none") return AnalysisMode::None; + if (name == "mx_rotation") return AnalysisMode::MXRotation; + if (name == "mx_stills") return AnalysisMode::MXStills; + if (name == "azint") return AnalysisMode::Azint; + if (name == "grid") return AnalysisMode::Grid; + if (name == "powder_calibration") return AnalysisMode::PowderCalibration; + return {}; +} + +AnalysisSettings &AnalysisSettings::Mode(AnalysisMode input) { + mode = input; + return *this; +} + +AnalysisMode AnalysisSettings::GetMode() const { + return mode; +} + +bool AnalysisSettings::IsMX() const { + return AnalysisModeIsMX(mode); +} diff --git a/common/AnalysisSettings.h b/common/AnalysisSettings.h new file mode 100644 index 000000000..095ba8e74 --- /dev/null +++ b/common/AnalysisSettings.h @@ -0,0 +1,79 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#pragma once + +#include +#include +#include + +// What analysis runs over the images - the one place the question is asked, shared by jfjoch_broker, +// rugnux and jfjoch_viewer. Before this, the answer was composed at each site out of a detector type, +// two independent "spot finding off" switches and a rotation flag, so what was configured and what +// actually ran were different things. +// +// There is deliberately no Auto value. GetIndexingAlgorithm() resolves Auto at read time against the +// GPU count and the unit cell, which is exactly why an indexing setting cannot be read back off the +// configuration; the mode getter is a plain accessor and stays one. +// +// MXStills is the default: None would silently switch analysis off on every deployment whose +// configuration predates this field. +// +// MXRotation is not offered by the REST API - jfjoch_broker has no rotation analysis path, so +// broker/jfjoch_api.yaml simply cannot express it and JFJochStateMachine refuses it if it arrives by +// any other route. Rotation data is processed offline with rugnux. +enum class AnalysisMode { + None, // images are received, written and streamed; nothing is analysed + MXRotation, // spot finding + rotation (two-pass) indexing + refinement + integration + MXStills, // spot finding + per-image indexing + refinement + integration + Azint, // azimuthal integration only + Grid, // grid scan: per-image scoring and crystal selection + PowderCalibration // detector geometry from a calibrant's powder rings +}; + +// Which stages of the pipeline a mode runs. This is the whole point of the mode: it does not label a +// run, it decides what happens in it, and every gate in the pipeline reads this rather than testing +// the mode itself. The table is in AnalysisSettings.cpp - modes are the rows, stages the columns. +struct AnalysisStages { + bool spot_finding; + // Fixed by the mode everywhere except Grid, where it is a setting + // (GridScanAnalysisSettings::indexing). AnalysisModeStages carries the grid DEFAULT; + // DiffractionExperiment::GetAnalysisStages is where the configured value takes over, and that is + // what every gate reads. + bool indexing; + // Bragg prediction and integration. It follows indexing in every engine - nothing is predicted + // without a lattice - so this never says yes where indexing says no. + bool bragg_integration; + bool scale_merge; + // Per-image protein / ice scoring: what a grid scan ranks its grid points on. + bool scoring; + bool azimuthal_integration; +}; + +AnalysisStages AnalysisModeStages(AnalysisMode mode); + +// True for the two MX modes, i.e. the modes that index and integrate Bragg reflections. +bool AnalysisModeIsMX(AnalysisMode mode); + +// The wire spelling of a mode: the same token in the CBOR stream, the HDF5 master and the rugnux +// --mode option, so a value can be carried between them without a translation table per hop. (The +// OpenAPI enum spells its values the way its neighbouring settings enums do, and OpenAPIConvert +// translates, as it does for every other enum in the API.) +std::string AnalysisModeName(AnalysisMode mode); +std::optional AnalysisModeFromName(std::string_view name); + +// What every analysis method shares, and nothing else. Each method's own knobs live in its own class +// beside this one - GridScanAnalysisSettings, CalibrationSettings - so that adding a parameter to one +// method cannot widen the structure every method reads. +// +// Persistent, not per-dataset: it sits on DiffractionExperiment outside the DatasetSettings member, +// which is the one thing a /start replaces wholesale. The per-method classes are bound the same way. +class AnalysisSettings { + AnalysisMode mode = AnalysisMode::MXStills; +public: + AnalysisSettings& Mode(AnalysisMode input); + + [[nodiscard]] AnalysisMode GetMode() const; + + [[nodiscard]] bool IsMX() const; +}; diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index 7cc3ac5c9..4923d02a4 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -123,6 +123,13 @@ ADD_LIBRARY(JFJochCommon STATIC ScanResultGenerator.h BraggIntegrationSettings.cpp BraggIntegrationSettings.h + AnalysisSettings.cpp + AnalysisSettings.h + GridScanAnalysisSettings.cpp + GridScanAnalysisSettings.h + CalibrationSettings.cpp + CalibrationSettings.h + GridScanResult.h SpotToSave.cpp XrayFluorescenceSpectrum.cpp XrayFluorescenceSpectrum.h diff --git a/common/CalibrationSettings.cpp b/common/CalibrationSettings.cpp new file mode 100644 index 000000000..929bee5d7 --- /dev/null +++ b/common/CalibrationSettings.cpp @@ -0,0 +1,28 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include "CalibrationSettings.h" + +CalibrationSettings &CalibrationSettings::Calibrant(const std::string &input) { + calibrant = input; + return *this; +} + +CalibrationSettings &CalibrationSettings::Method(CalibrationMethod input) { + method = input; + return *this; +} + +const std::string &CalibrationSettings::GetCalibrant() const { + return calibrant; +} + +CalibrationMethod CalibrationSettings::GetMethod() const { + return method; +} + +void CalibrationSettings::ApplyToAzimuthalIntegration(AzimuthalIntegrationSettings &azint) const { + azint.ForceCPUinFPGAWorkflow(true); + if (azint.GetAzimuthalBinCount() < 4) + azint.AzimuthalBinCount(CALIBRATION_AZIM_BINS_DEFAULT); +} diff --git a/common/CalibrationSettings.h b/common/CalibrationSettings.h new file mode 100644 index 000000000..42ef5229d --- /dev/null +++ b/common/CalibrationSettings.h @@ -0,0 +1,53 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#pragma once + +#include + +#include "AzimuthalIntegrationSettings.h" + +// How the powder rings the detector geometry is fitted to are measured (rugnux --calibration). +// Lives here rather than beside the fit itself so that the setting and the code that consumes it are +// not two different vocabularies; image_analysis/geom_refinement/PowderCalibration.h includes this. +enum class CalibrationMethod { + Rings, // the run-summed (q x azimuth) azimuthal profile: the ring measured at every azimuth + Spots // the pooled per-image spot lists: the ring sampled wherever the spot finder bit +}; + +// Azimuthal sectors a powder calibration integrates into when nothing usable was asked for. A plain +// radial profile has averaged every ring over every direction, so nothing is left to say where its +// centre is (RingsFromAzimuthalProfile refuses below 4), and the azimuthal default of one sector is +// exactly that. Sweeping the count on a LaB6 exposure and on a crystal's ice rings, 32 and above +// agree to about 0.1 px with a flat residual, while 8 sectors is visibly coarser (0.3 px away, twice +// the residual). +constexpr int CALIBRATION_AZIM_BINS_DEFAULT = 32; + +// Settings for AnalysisMode::PowderCalibration and nothing else. Per-method by the same rule as +// GridScanAnalysisSettings: AnalysisSettings carries what every method shares, each method carries +// its own knobs. +class CalibrationSettings { + // Which calibrant's ring d-spacings the geometry is fitted against. Empty means the front end's + // own default (rugnux --calibrant) rather than "no calibrant". + std::string calibrant; + CalibrationMethod method = CalibrationMethod::Rings; +public: + CalibrationSettings& Calibrant(const std::string &input); + CalibrationSettings& Method(CalibrationMethod input); + + [[nodiscard]] const std::string &GetCalibrant() const; + [[nodiscard]] CalibrationMethod GetMethod() const; + + // The azimuthal integration a powder calibration needs, applied to whatever is configured. + // + // Ring positions are read off a (q x azimuth) profile, so the integration needs azimuthal sectors + // AND enough q bins under each of them to place a ring. The FPGA integration core holds + // FPGA_INTEGRATION_BIN_COUNT = 2048 bins in TOTAL, which at 32 sectors leaves 64 q bins - far too + // coarse to locate a ring - so the integration is moved onto the CPU, where that limit does not + // apply. This is not a defect on the FPGA path to be fixed back onto it: the core is sized for a + // detector running at full rate, and a calibration exposure is a few images at a few Hz, so the + // frame rate this costs is not frame rate this mode needs. + // + // Only supplies a sector count where fewer than four were asked for; an explicit count stands. + void ApplyToAzimuthalIntegration(AzimuthalIntegrationSettings &azint) const; +}; diff --git a/common/DatasetSettings.h b/common/DatasetSettings.h index b85e13eee..0caea99f9 100644 --- a/common/DatasetSettings.h +++ b/common/DatasetSettings.h @@ -67,6 +67,9 @@ class DatasetSettings { float poni_rot_2_rad; float poni_rot_3_rad; + // DEPRECATED - the analysis mode (AnalysisSettings) is the switch now. It still gates spot finding + // beneath a mode that finds spots; see DiffractionExperiment::IsSpotFindingEnabled for the + // precedence between the three switches that answer this one question. bool spot_finding_enable; int64_t max_spot_count; diff --git a/common/DiffractionExperiment.cpp b/common/DiffractionExperiment.cpp index 19cc77b5c..7e740b8a6 100644 --- a/common/DiffractionExperiment.cpp +++ b/common/DiffractionExperiment.cpp @@ -706,6 +706,7 @@ void DiffractionExperiment::FillMessage(StartMessage &message) const { // signed ones reserve INTx_MIN as the marker (GetUnderflow), so the lowest valid is one above. message.underload_value = IsPixelSigned() ? GetUnderflow() + 1 : 0; message.indexing_algorithm = GetIndexingAlgorithm(); + message.analysis_mode = analysis.GetMode(); message.images_per_trigger = dataset.GetImageNumPerTrigger(); if (GetDetectorType() == DetectorType::JUNGFRAU) { @@ -1072,7 +1073,13 @@ bool DiffractionExperiment::IsPulsedSource() const { } bool DiffractionExperiment::IsSpotFindingEnabled() const { - return dataset.IsSpotFindingEnabled() && !IsPedestalRun(); + // Three switches answer one question, and this is where the precedence between them is fixed. The + // analysis mode is the authority: a mode that analyses no spots (none, azint, powder_calibration) + // switches spot finding off whatever else is set. Under a mode that does find spots, the two older + // switches - the per-dataset dataset_settings.spot_finding here and the persistent + // SpotFindingSettings::enable read in MXAnalysisAfterFPGA - remain the finer control beneath it and + // can still turn it off. Both are DEPRECATED and go away once their callers have moved to the mode. + return GetAnalysisStages().spot_finding && dataset.IsSpotFindingEnabled() && !IsPedestalRun(); } float DiffractionExperiment::GetPhotonEnergyForConversion_keV() const { @@ -1766,6 +1773,58 @@ BraggIntegrationSettings DiffractionExperiment::GetBraggIntegrationSettings() co return bragg_integration_settings; } +DiffractionExperiment &DiffractionExperiment::ImportAnalysisSettings(const AnalysisSettings &input) { + analysis = input; + + // Powder calibration cannot be integrated on the FPGA; the rule and the reason live with the + // calibration settings. Applied from both imports, so it does not matter which of the mode and + // the calibration settings is set first. + if (analysis.GetMode() == AnalysisMode::PowderCalibration) + calibration.ApplyToAzimuthalIntegration(az_integration_settings); + + return *this; +} + +AnalysisSettings DiffractionExperiment::GetAnalysisSettings() const { + return analysis; +} + +DiffractionExperiment & +DiffractionExperiment::ImportGridScanAnalysisSettings(const GridScanAnalysisSettings &input) { + grid_scan_analysis = input; + return *this; +} + +GridScanAnalysisSettings DiffractionExperiment::GetGridScanAnalysisSettings() const { + return grid_scan_analysis; +} + +DiffractionExperiment &DiffractionExperiment::ImportCalibrationSettings(const CalibrationSettings &input) { + calibration = input; + + if (analysis.GetMode() == AnalysisMode::PowderCalibration) + calibration.ApplyToAzimuthalIntegration(az_integration_settings); + + return *this; +} + +CalibrationSettings DiffractionExperiment::GetCalibrationSettings() const { + return calibration; +} + +AnalysisMode DiffractionExperiment::GetAnalysisMode() const { + return analysis.GetMode(); +} + +AnalysisStages DiffractionExperiment::GetAnalysisStages() const { + auto stages = AnalysisModeStages(analysis.GetMode()); + // Every stage is the mode's to decide except one: whether a raster indexes is a setting, and this + // is where the configured value replaces the default the table carries. + if (analysis.GetMode() == AnalysisMode::Grid) + stages.indexing = grid_scan_analysis.IsIndexing(); + return stages; +} + DiffractionExperiment &DiffractionExperiment::ImportScalingSettings(const ScalingSettings &input) { scaling_settings = input; return *this; diff --git a/common/DiffractionExperiment.h b/common/DiffractionExperiment.h index 03515c17c..3a9757b8d 100644 --- a/common/DiffractionExperiment.h +++ b/common/DiffractionExperiment.h @@ -37,6 +37,9 @@ constexpr int64_t SaturationLimitFromValue(int64_t saturation_value) { return sa #include "CompressedImage.h" #include "IndexingSettings.h" #include "BraggIntegrationSettings.h" +#include "AnalysisSettings.h" +#include "GridScanAnalysisSettings.h" +#include "CalibrationSettings.h" #include "ScalingSettings.h" #include @@ -84,6 +87,12 @@ class DiffractionExperiment { FileWriterSettings file_writer; IndexingSettings indexing; BraggIntegrationSettings bragg_integration_settings; + // Persistent, deliberately outside the DatasetSettings member above: a /start replaces `dataset` + // wholesale, and the analysis mode is a property of how the instrument is set up, not of one run. + // One shared group plus one per analysis method, bound the same way. + AnalysisSettings analysis; + GridScanAnalysisSettings grid_scan_analysis; + CalibrationSettings calibration; ScalingSettings scaling_settings; DarkMaskSettings dark_mask_settings; @@ -199,6 +208,21 @@ public: DiffractionExperiment& ImportBraggIntegrationSettings(const BraggIntegrationSettings& input); BraggIntegrationSettings GetBraggIntegrationSettings() const; + DiffractionExperiment& ImportAnalysisSettings(const AnalysisSettings& input); + AnalysisSettings GetAnalysisSettings() const; + + DiffractionExperiment& ImportGridScanAnalysisSettings(const GridScanAnalysisSettings& input); + GridScanAnalysisSettings GetGridScanAnalysisSettings() const; + + // Under AnalysisMode::PowderCalibration this also rewrites the azimuthal integration the fit + // needs - see CalibrationSettings::ApplyToAzimuthalIntegration for what and why. + DiffractionExperiment& ImportCalibrationSettings(const CalibrationSettings& input); + CalibrationSettings GetCalibrationSettings() const; + [[nodiscard]] AnalysisMode GetAnalysisMode() const; + // What this experiment's mode runs. Every gate in the pipeline reads this; the table is in + // common/AnalysisSettings.cpp. + [[nodiscard]] AnalysisStages GetAnalysisStages() const; + DiffractionExperiment& ImportFileWriterSettings(const FileWriterSettings& input); FileWriterSettings GetFileWriterSettings() const; diff --git a/common/GridScanAnalysisSettings.cpp b/common/GridScanAnalysisSettings.cpp new file mode 100644 index 000000000..b90f98b33 --- /dev/null +++ b/common/GridScanAnalysisSettings.cpp @@ -0,0 +1,49 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include "GridScanAnalysisSettings.h" + +GridScanAnalysisSettings &GridScanAnalysisSettings::ProteinScoreThreshold(float input) { + protein_score_threshold = input; + return *this; +} + +GridScanAnalysisSettings &GridScanAnalysisSettings::MinBlobCells(int64_t input) { + min_blob_cells = input; + return *this; +} + +GridScanAnalysisSettings &GridScanAnalysisSettings::DecisiveSingleCellScore(float input) { + decisive_single_cell_score = input; + return *this; +} + +GridScanAnalysisSettings &GridScanAnalysisSettings::MaxCrystals(int64_t input) { + max_crystals = input; + return *this; +} + +GridScanAnalysisSettings &GridScanAnalysisSettings::Indexing(bool input) { + indexing = input; + return *this; +} + +float GridScanAnalysisSettings::GetProteinScoreThreshold() const { + return protein_score_threshold; +} + +int64_t GridScanAnalysisSettings::GetMinBlobCells() const { + return min_blob_cells; +} + +float GridScanAnalysisSettings::GetDecisiveSingleCellScore() const { + return decisive_single_cell_score; +} + +int64_t GridScanAnalysisSettings::GetMaxCrystals() const { + return max_crystals; +} + +bool GridScanAnalysisSettings::IsIndexing() const { + return indexing; +} diff --git a/common/GridScanAnalysisSettings.h b/common/GridScanAnalysisSettings.h new file mode 100644 index 000000000..3221e8e31 --- /dev/null +++ b/common/GridScanAnalysisSettings.h @@ -0,0 +1,63 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#pragma once + +#include + +// Settings for AnalysisMode::Grid and nothing else. Per-method by the same rule as +// CalibrationSettings: AnalysisSettings carries what every method shares, each method carries its +// own knobs. +// +// THIS IS THE SINGLE HOME FOR THESE VALUES. AnalyzeGridScan +// (image_analysis/grid_scan_analysis/AnalyzeGridScan.h) is to take this object as one argument - +// +// GridScanResult AnalyzeGridScan(const ScanResult &scan, const GridScanSettings &grid, +// float beam_size_x_um, float beam_size_y_um, +// const GridScanAnalysisSettings &settings = {}); +// +// - rather than loose parameters carrying defaults of their own, so that a knob has one default and +// one place where it is documented. PROTEIN_SCORE_THRESHOLD_DEFAULT and MIN_BLOB_CELLS_DEFAULT are +// replaced by the two members below and must not be reintroduced beside that header. +// +// The beam size stays a separate argument on purpose: it is measured (from the file, or --beam-size), +// not configured, and rugnux reports where it came from beside the crystals it sized. +class GridScanAnalysisSettings { + // A cell counts as protein above this. The per-image protein score saturates, so this only has + // to separate "something diffracted here" from "nothing did". + float protein_score_threshold = 0.5f; + // How many cells above that threshold make a shape rather than a coincidence. Two cells can be + // the two ends of a single hit lying on a cell boundary; three is the smallest patch that is not. + int64_t min_blob_cells = 3; + // ...unless one cell on its own is decisive. The rule above is about coincidences, and a lone + // cell scoring near the top of a saturating score is not one - a crystal smaller than the grid + // step lights exactly one cell, and refusing it would lose precisely the samples a fine raster is + // run to find. Well above protein_score_threshold on purpose: this admits the obvious case, it + // does not lower the general threshold by the back door. + float decisive_single_cell_score = 0.9f; + // Most crystals reported. A raster over a loop full of shards can label dozens of blobs, and past + // the first few the list is no longer a ranking anyone acts on - the DAQ collects from the top of + // it. Crystals are sorted by score, so this keeps the best. + int64_t max_crystals = 10; + // Whether each raster cell is indexed as well as scored. + // + // On by default. It is affordable - a raster runs at up to 100 Hz, which the FFT indexer keeps up + // with - and it is additive: the blobs are still found on the protein score, so indexing changes + // nothing about which cells are called crystals and only adds what was found in them. The lattice + // count per cell is the cheapest multi-lattice or cracked-crystal signal there is, and on a + // fixed-target serial experiment with a known cell (-C, ffbidx) a raster that indexes is most of + // the measurement. Turn it off for a very large raster where the GPU is the constraint. + bool indexing = true; +public: + GridScanAnalysisSettings& ProteinScoreThreshold(float input); + GridScanAnalysisSettings& MinBlobCells(int64_t input); + GridScanAnalysisSettings& DecisiveSingleCellScore(float input); + GridScanAnalysisSettings& MaxCrystals(int64_t input); + GridScanAnalysisSettings& Indexing(bool input); + + [[nodiscard]] float GetProteinScoreThreshold() const; + [[nodiscard]] int64_t GetMinBlobCells() const; + [[nodiscard]] float GetDecisiveSingleCellScore() const; + [[nodiscard]] int64_t GetMaxCrystals() const; + [[nodiscard]] bool IsIndexing() const; +}; diff --git a/common/GridScanResult.h b/common/GridScanResult.h new file mode 100644 index 000000000..99c8e3035 --- /dev/null +++ b/common/GridScanResult.h @@ -0,0 +1,53 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#pragma once + +// PLACEHOLDER - TO BE REPLACED AT INTEGRATION. The real GridScanCrystal/GridScanResult live in +// image_analysis/grid_scan_analysis/GridScanResult.h, alongside the analysis that owns the ranking, +// clustering and crystal selection producing them. This file carries only the shape the rest of the +// system needs, so that the /result/scan payload, the CBOR END block and the HDF5 master could land +// without waiting for it; the field list has been checked against the real header and agrees with it. +// +// To integrate: delete this file and its line in common/CMakeLists.txt, and point the two includes - +// common/ScanResult.h and common/JFJochMessages.h - at the real header. Nothing else refers to these +// two structs. The grid_scan_crystal / grid_scan_result schemas in broker/jfjoch_api.yaml are the +// same placeholder and go the same way. + +#include +#include +#include + +// One crystal picked out of a grid scan. Positions are in the grid's own frame: nx/ny are fractional +// grid coordinates and x_um/y_um the same point as a signed offset along the two grid axes, so a +// consumer can address the crystal either by grid step or by stage motion without redoing the +// arithmetic. +struct GridScanCrystal { + float nx = 0.0f; // centre, fractional grid coordinate along the fast grid axis + float ny = 0.0f; // centre, fractional grid coordinate along the slow grid axis + float x_um = 0.0f; // centre, signed offset along the fast grid axis + float y_um = 0.0f; // centre, signed offset along the slow grid axis + int64_t image_number = -1; // the grid point at the centre, as an image ordinal + float major_um = 0.0f; // extent along the crystal's own major principal axis + float minor_um = 0.0f; // extent along its minor principal axis + // Direction of the major axis from the +x grid axis, counter-clockwise. An AXIS, not a direction: + // it lives in [0, 180) and wraps, so 179 deg and 0 deg are adjacent. Averaging two such angles + // arithmetically across the wrap turns two nearly parallel needles into a right angle - combine + // them on the doubled angle (mean of 2*theta, halved) or not at all. + float angle_deg = 0.0f; + float score = 0.0f; + float ice_score = 0.0f; + float res_A = NAN; // NaN where no resolution was measured in the blob + int64_t n_images = 0; // grid points that fell in this crystal +}; + +// The crystals a raster found, sorted by score descending. Today the analysis returns zero or one, but +// N is the design intent: nothing may assume at most one entry. +struct GridScanResult { + std::vector crystals; + + // The beam the extents above were measured with, along the grid axes. Those extents still contain + // it, so this says what a consumer has to take back out. + float beam_size_x_um = 0.0f; + float beam_size_y_um = 0.0f; +}; diff --git a/common/JFJochMessages.h b/common/JFJochMessages.h index 26ec7521d..0bf2f0dbe 100644 --- a/common/JFJochMessages.h +++ b/common/JFJochMessages.h @@ -22,6 +22,8 @@ #include "Reflection.h" #include "CrystalLattice.h" #include "IndexingSettings.h" +#include "AnalysisSettings.h" +#include "GridScanResult.h" #include "XrayFluorescenceSpectrum.h" #include "../gemmi_gph/gemmi/symmetry.hpp" @@ -363,6 +365,11 @@ struct StartMessage { IndexingAlgorithmEnum indexing_algorithm; GeomRefinementAlgorithmEnum geom_refinement_algorithm; + // Which analysis produced this stream, so a written file records it. Absent in a stream written + // before the mode existed, which is why it is optional rather than defaulted: naming a mode for a + // file that never carried one would be an invention, not provenance. + std::optional analysis_mode; + std::optional poni_rot1; std::optional poni_rot2; std::optional poni_rot3; @@ -416,6 +423,9 @@ struct EndMessage { std::optional rotation_lattice_type; std::optional rotation_lattice; std::vector rotation_extra_lattices; + // The crystals a grid scan found, best score first. Empty when the run was not a raster, or when + // the raster found nothing; a raster can find any number, so nothing here caps it at one. + std::vector grid_crystals; std::optional unit_cell; // Space group determined by the offline analysis (overrides the start message when writing the // master, since it is only known after merging). Both spellings are carried and a reader prefers diff --git a/common/ScanResult.h b/common/ScanResult.h index 02ce9af4b..db092cf94 100644 --- a/common/ScanResult.h +++ b/common/ScanResult.h @@ -11,6 +11,8 @@ #include "CrystalLattice.h" #include "UnitCell.h" +#include "GridScanResult.h" +#include "../image_analysis/geom_refinement/PowderCalibration.h" // CalibrationResult struct ScanResultElem { int64_t number = -1; @@ -57,6 +59,13 @@ struct ScanResult { // Bravais lattice type of the global rotation-indexing solution (from RotationIndexer). std::optional rotation_crystal_system; std::optional rotation_centering; + + // What the run produced beyond the per-image list, one member per analysis mode that has an + // answer of its own. Same idiom as the three rotation_* members above: the payload of a mode is a + // top-level optional, present when that mode ran and absent otherwise, so images stays required + // and every existing /result/scan client is untouched. + std::optional grid; // AnalysisMode::Grid + std::optional calibration; // AnalysisMode::PowderCalibration }; // Two-letter Bravais lattice symbol: crystal-system letter + centering, e.g. "tP", "oC", "cF", "hR". diff --git a/docs/CBOR.md b/docs/CBOR.md index f3cf80a99..024a97bc6 100644 --- a/docs/CBOR.md +++ b/docs/CBOR.md @@ -119,6 +119,7 @@ There are minor differences at the moment: | - sample_temperature_K | float | Sample temperature \[K\] | | | - detect_ice_rings | bool | Ice ring detection feature is enabled | | | - indexing_algorithm | string | Indexing algorithm used on-the-fly; allowed values: ffbidx, fft, fftw, none | | +| - analysis_mode | string | Analysis that produced the stream; allowed values: none, mx_rotation, mx_stills, azint, grid, powder_calibration. Absent in a stream written before this key existed | | | - geom_refinement_algorithm | string | Post-indexing detector geometry refinement algorithm; allowed values: none, beam_center | | | - poni_rot1 | float | Tilt of the detector rot1 according to PyFAI PONI convention \[rad\] | | | - poni_rot2 | float | Tilt of the detector rot2 according to PyFAI PONI convention \[rad\] | | @@ -316,6 +317,16 @@ See [DECTRIS documentation](https://github.com/dectris/documentation/tree/main/s | - system | string | Crystal system: triclinic, monoclinic, orthorhombic, tetragonal, trigonal, hexagonal, cubic | | | rotation_lattice | Array(9 * float) | Real-space lattice basis, flattened 3x3 in row-major order | | | rotation_extra_lattices | Array(Array(9*float)) | Additional indexed lattices (orientation variants); present only if found | | +| grid_crystals | Array(Map) | Crystals found by a grid scan, best score first; present only for a raster that found any. Any number of entries - a consumer must not assume at most one | | +| - nx, ny | float | Crystal centre as a fractional grid coordinate, along the fast and the slow grid axis | | +| - x_um, y_um | float | The same centre as a signed offset along those two axes \[um\] | | +| - image_number | int64 | Image ordinal of the grid point at the centre | | +| - major_um, minor_um | float | Extent along the crystal's major and minor principal axes \[um\] | | +| - angle_deg | float | Major axis from the +x grid axis, counter-clockwise \[deg\]. An AXIS, not a direction: in \[0, 180) and wrapping, so 179 and 0 are adjacent and these do not average arithmetically | | +| - score | float | Diffraction score the crystal was ranked on | | +| - ice_score | float | Ice-ring score over the crystal's grid points | | +| - res_A | float | Resolution estimate over the crystal's grid points \[A\] | | +| - n_images | int64 | Grid points that fell in this crystal | | | data_collection_efficiency_image | Array(float) | Per-image data collection efficiency. Missing values are encoded as 0 or 1 depending on producer context | | | spot_count | Array(int32) | Per-image spot count | | | spot_count_ice_ring | Array(int32) | Per-image number of spots within identified ice-ring resolution ranges | | diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 26b7ebd27..7cb37d363 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -3,6 +3,14 @@ ### 1.0.0-rc.167 +* jfjoch_broker takes an explicit analysis mode (`/config/analysis`, the `analysis` block of the configuration file, and the Analysis panel of the web frontend): `MXStills` (the default), `Azint`, `Grid`, `PowderCalibration` or `None`. The mode decides what runs and takes precedence over `dataset_settings.spot_finding` and `spot_finding_settings.enable`, which are now deprecated. +* `PowderCalibration` moves azimuthal integration onto the CPU and sets a sector count of its own, since the FPGA integration core cannot hold enough bins for a sectored profile; a calibration exposure then runs at a few Hz. +* Each analysis method has its own settings and its own endpoint: `/config/grid_scan_analysis` (protein-score threshold, minimum cells per crystal, decisive single-cell score, maximum crystals reported, and whether the raster indexes) and `/config/calibration` (calibrant and ring source). `/config/analysis` now carries the mode alone. +* A grid scan indexes each raster cell by default; turn it off with `indexing` in `/config/grid_scan_analysis`. +* Rotation MX analysis is not offered online: it is absent from the REST API and refused by the broker. Rotation data is collected with jfjoch_broker and processed with rugnux. +* `rugnux --mode` accepts `mx_rotation` and `mx_stills` beside `mx`, which continues to choose between them from the goniometer. +* The stream and the written file record which analysis produced them, as `analysis_mode` in the CBOR start message and `/entry/MX/analysis_mode` in the HDF5 master. +* The crystals a grid scan found travel through the CBOR end message (`grid_crystals`), the HDF5 master (`/entry/MX/crystals`) and `GET /result/scan`, which also carries a powder-calibration result. * `rugnux --model` reports CC(model, data) - the correlation of the merged intensities with the placed, scaled model - by resolution shell, on the same shells as CC1/2, with the reflection count and a significance for each. * `rugnux --model` fits the model's scale, anisotropic B and bulk-solvent parameters on the working reflections only, so the R-free it reports is measured against a model no free reflection helped scale. * The bulk-solvent parameters of `rugnux --model` are searched over their physically meaningful range instead of being fitted without bounds, so a model is never scaled with a solvent term that has silently switched itself off. diff --git a/docs/HDF5.md b/docs/HDF5.md index 1267fab6c..e913aa517 100644 --- a/docs/HDF5.md +++ b/docs/HDF5.md @@ -363,6 +363,7 @@ variants. | Dataset | Units | Meaning | |---------|-------|---------| +| `analysis_mode` | | which analysis produced the file: `none` / `mx_rotation` / `mx_stills` / `azint` / `grid` / `powder_calibration`. Absent in a file written before this dataset existed | | `indexing_algorithm` | | `FFBIDX` / `FFT (CUDA)` / `FFT (FFTW)` | | `geom_refinement_algorithm` | | e.g. `beam_center` | | `rotationLatticeIndexed` | Å | whole-run rotation-indexing lattice (`[9]`) | @@ -379,6 +380,23 @@ variants. | `indexedLatticeCount` | | per-image lattice count summary (master). *Note: data files use `indexingLatticeCount`; readers accept either.* | | `reindexMatrix` | | change of basis from the setting the per-image data are in to the setting of `/entry/sample/unit_cell` (`[9]`, `int32`, flattened 3×3, row major) — see below | +**Grid-scan crystals** (master, `/entry/MX/crystals`, written when a raster found any). Parallel 1-D +datasets, all of length *N*, one per field, sorted by `score` descending — crystals are numbered +rather than named, so a group per crystal would put the index in the path. A raster can find any +number of them: nothing may assume at most one. + +| Dataset | Units | Meaning | +|---------|-------|---------| +| `nx`, `ny` | | crystal centre as a fractional grid coordinate, along the fast and the slow grid axis | +| `xUm`, `yUm` | µm | the same centre as a signed offset along those two axes | +| `imageNumber` | | image ordinal of the grid point at the centre | +| `majorUm`, `minorUm` | µm | extent along the crystal's major and minor principal axes | +| `angleDeg` | ° | major axis from the +x grid axis, counter-clockwise. An **axis, not a direction**: it lies in [0, 180) and wraps, so 179° and 0° are adjacent. Two of these do not average arithmetically — combine them on the doubled angle (mean of 2θ, halved), or a pair of nearly parallel needles comes out as a right angle | +| `score` | | diffraction score the crystal was ranked on | +| `iceScore` | ratio | ice-ring score over the crystal's grid points | +| `resA` | Å | resolution estimate over the crystal's grid points | +| `nImages` | | grid points that fell in this crystal | + **Reindex matrix.** The per-image `h`, `k`, `l` and `latticeIndexed` are written as each image is processed, in the setting that image was *indexed* in. The space group is only settled afterwards, by the merge, and settling it can re-seat the lattice into the group's conventional setting — so diff --git a/frame_serialize/CBORStream2Deserializer.cpp b/frame_serialize/CBORStream2Deserializer.cpp index ec4d6577a..9511e1a9b 100644 --- a/frame_serialize/CBORStream2Deserializer.cpp +++ b/frame_serialize/CBORStream2Deserializer.cpp @@ -595,6 +595,33 @@ namespace { return lm; } + GridScanCrystal GetCBORGridScanCrystal(CborValue &value) { + GridScanCrystal c{}; + CborValue map_value; + cborErr(cbor_value_enter_container(&value, &map_value)); + while (!cbor_value_at_end(&map_value)) { + auto key = GetCBORString(map_value); + if (key == "nx") c.nx = GetCBORFloat(map_value); + else if (key == "ny") c.ny = GetCBORFloat(map_value); + else if (key == "x_um") c.x_um = GetCBORFloat(map_value); + else if (key == "y_um") c.y_um = GetCBORFloat(map_value); + else if (key == "image_number") c.image_number = GetCBORInt(map_value); + else if (key == "major_um") c.major_um = GetCBORFloat(map_value); + else if (key == "minor_um") c.minor_um = GetCBORFloat(map_value); + else if (key == "angle_deg") c.angle_deg = GetCBORFloat(map_value); + else if (key == "score") c.score = GetCBORFloat(map_value); + else if (key == "ice_score") c.ice_score = GetCBORFloat(map_value); + else if (key == "res_A") c.res_A = GetCBORFloat(map_value); + else if (key == "n_images") c.n_images = GetCBORInt(map_value); + else + // A field this build does not know. Step over its value, or the loop would read the + // payload as the next key and never reach the end of the map. + cbor_value_advance(&map_value); + } + cborErr(cbor_value_leave_container(&value, &map_value)); + return c; + } + XrayFluorescenceSpectrum GetCBORFluorescenceSpectrum(CborValue &value) { std::vector data, energy; CborValue map_value; @@ -1204,6 +1231,8 @@ namespace { message.poni_rot3 = j["poni_rot3"]; if (j.contains("detect_ice_rings")) message.detect_ice_rings = j["detect_ice_rings"]; + if (j.contains("analysis_mode")) + message.analysis_mode = AnalysisModeFromName(j["analysis_mode"].get()); if (j.contains("images_per_trigger")) message.images_per_trigger = j["images_per_trigger"]; if (j.contains("indexing_algorithm")) { @@ -1566,7 +1595,14 @@ namespace { message.rotation_lattice = CrystalLattice(tmp); } else if (key == "rotation_lattice_type") message.rotation_lattice_type = GetCBORLatticeMessage(value); - else if (key == "rotation_extra_lattices") { + else if (key == "grid_crystals") { + size_t array_len = GetCBORArrayLen(value); + CborValue array_value; + cborErr(cbor_value_enter_container(&value, &array_value)); + for (size_t i = 0; i < array_len; i++) + message.grid_crystals.push_back(GetCBORGridScanCrystal(array_value)); + cborErr(cbor_value_leave_container(&value, &array_value)); + } else if (key == "rotation_extra_lattices") { size_t array_len = GetCBORArrayLen(value); CborValue array_value; cborErr(cbor_value_enter_container(&value, &array_value)); diff --git a/frame_serialize/CBORStream2Serializer.cpp b/frame_serialize/CBORStream2Serializer.cpp index 9a6a8f8e4..92498ed2f 100644 --- a/frame_serialize/CBORStream2Serializer.cpp +++ b/frame_serialize/CBORStream2Serializer.cpp @@ -302,6 +302,39 @@ inline void CBOR_ENC(CborEncoder &encoder, const char* key, const std::vector& crystals) { + CborEncoder arrayEncoder; + + cborErr(cbor_encode_text_stringz(&encoder, key)); + cborErr(cbor_encoder_create_array(&encoder, &arrayEncoder, crystals.size())); + + for (const auto &c: crystals) + CBOR_ENC(arrayEncoder, c); + + cborErr(cbor_encoder_close_container(&encoder, &arrayEncoder)); +} + inline void CBOR_ENC(CborEncoder &encoder, const char* key, const std::vector& refs) { CborEncoder arrayEncoder, mapEncoder; @@ -629,6 +662,11 @@ inline void CBOR_ENC_START_USER_DATA(CborEncoder& encoder, const char* key, if (message.detect_ice_rings.has_value()) j["detect_ice_rings"] = message.detect_ice_rings.value(); + // Which analysis produced this stream. One shared spelling table (AnalysisModeName), so the token + // in the stream is the same one the HDF5 master, the rugnux --mode option and the OpenAPI enum use. + if (message.analysis_mode.has_value()) + j["analysis_mode"] = AnalysisModeName(message.analysis_mode.value()); + switch(message.indexing_algorithm) { case IndexingAlgorithmEnum::FFBIDX: j["indexing_algorithm"] = "ffbidx"; @@ -817,6 +855,8 @@ void CBORStream2Serializer::SerializeSequenceEnd(const EndMessage& message) { CBOR_ENC_FLOAT_ARRAY_NOKEY(arrayEncoder, el.GetVector()); cborErr(cbor_encoder_close_container(&mapEncoder, &arrayEncoder)); } + if (!message.grid_crystals.empty()) + CBOR_ENC(mapEncoder, "grid_crystals", message.grid_crystals); CBOR_ENC(mapEncoder, "data_collection_efficiency_image", message.data_collection_efficiency); CBOR_ENC(mapEncoder, "spot_count", message.spot_count); CBOR_ENC(mapEncoder, "spot_count_ice_ring", message.spot_count_ice_ring); diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index c3f749f13..b350d69f9 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -43,6 +43,9 @@ import MaskVisualization from "./components/MaskVisualization"; import DarkMaskSettings from "./components/DarkMaskSettings"; import IndexingSettings from "./components/IndexingSettings"; import BraggIntegrationSettings from "./components/BraggIntegrationSettings"; +import AnalysisSettings from "./components/AnalysisSettings"; +import GridScanAnalysisSettings from "./components/GridScanAnalysisSettings"; +import CalibrationSettings from "./components/CalibrationSettings"; import AzIntSettings from "./components/AzIntSettings"; import ROI from "./components/ROI"; import FileWriterSettings from "./components/FileWriterSettings"; @@ -211,9 +214,12 @@ function App() { { id: 'processing', label: 'On-the-fly processing', icon: , render: () => ( + + + {/* ROI holds three wide side-by-side tables — keep it full width. */} diff --git a/frontend/src/components/AnalysisSettings.tsx b/frontend/src/components/AnalysisSettings.tsx new file mode 100644 index 000000000..27a488ddb --- /dev/null +++ b/frontend/src/components/AnalysisSettings.tsx @@ -0,0 +1,71 @@ +import {memo, useEffect, useState} from 'react'; + +import FormControl from "@mui/material/FormControl"; +import InputLabel from "@mui/material/InputLabel"; +import Select, {SelectChangeEvent} from "@mui/material/Select"; +import MenuItem from "@mui/material/MenuItem"; +import {analysis_mode, analysis_settings} from "../client"; +import {putConfigAnalysisMutation} from "../client/@tanstack/react-query.gen"; +import _ from "lodash"; +import SettingsPanel from "./SettingsPanel"; +import {useUpload} from "./useUpload"; + +type MyProps = { + s?: analysis_settings +} + +// MXStills, not None: a broker whose configuration predates this setting keeps analysing. +const default_analysis_settings: analysis_settings = { + mode: analysis_mode.MX_STILLS +}; + +function AnalysisSettings({s: serverS}: MyProps) { + const [s, setS] = useState(default_analysis_settings); + const [lastDownloadedS, setLastDownloadedS] = useState(default_analysis_settings); + const {submit, pending, snackbar} = useUpload(putConfigAnalysisMutation()); + + useEffect(() => { + if ((serverS !== undefined) && !_.isEqual(serverS, lastDownloadedS)) { + setS(serverS); + setLastDownloadedS(serverS); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [serverS]); + + const handleModeChange = (event: SelectChangeEvent) => { + setS(prev => ({...prev, mode: event.target.value as analysis_mode})); + }; + + const dirty = !_.isEqual(s, lastDownloadedS); + + return ( + submit(s)} uploadDisabled={pending} snackbar={snackbar}> + Analysis mode + + Analysis mode + {/* Only what the API can express. Rotation MX is deliberately not here: jfjoch_broker + has no rotation analysis path, so the setting must be unaskable rather than asked + for and then refused. Rotation data is collected here and processed with rugnux. */} + + + The mode decides what runs, and takes precedence over the spot-finding switches. Rotation + analysis is not available online - collect the sweep here and process the stored file with + rugnux. Grid scan and powder calibration have settings of their own, in the panels beside + this one. + + + ); +} + +export default memo(AnalysisSettings); diff --git a/frontend/src/components/CalibrationSettings.tsx b/frontend/src/components/CalibrationSettings.tsx new file mode 100644 index 000000000..375e3a794 --- /dev/null +++ b/frontend/src/components/CalibrationSettings.tsx @@ -0,0 +1,69 @@ +import {memo, useEffect, useState} from 'react'; + +import FormControl from "@mui/material/FormControl"; +import InputLabel from "@mui/material/InputLabel"; +import Select, {SelectChangeEvent} from "@mui/material/Select"; +import MenuItem from "@mui/material/MenuItem"; +import TextField from "@mui/material/TextField"; +import {calibration_method, calibration_settings} from "../client"; +import {putConfigCalibrationMutation} from "../client/@tanstack/react-query.gen"; +import _ from "lodash"; +import SettingsPanel from "./SettingsPanel"; +import {useUpload} from "./useUpload"; + +type MyProps = { + s?: calibration_settings +} + +const default_calibration_settings: calibration_settings = { + method: calibration_method.RINGS +}; + +function CalibrationSettings({s: serverS}: MyProps) { + const [s, setS] = useState(default_calibration_settings); + const [lastDownloadedS, setLastDownloadedS] = useState(default_calibration_settings); + const {submit, pending, snackbar} = useUpload(putConfigCalibrationMutation()); + + useEffect(() => { + if ((serverS !== undefined) && !_.isEqual(serverS, lastDownloadedS)) { + setS(serverS); + setLastDownloadedS(serverS); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [serverS]); + + const handleMethodChange = (event: SelectChangeEvent) => { + setS(prev => ({...prev, method: event.target.value as calibration_method})); + }; + + const dirty = !_.isEqual(s, lastDownloadedS); + + return ( + submit(s)} uploadDisabled={pending} snackbar={snackbar}> + Calibrant + + setS(prev => ({...prev, calibrant: e.target.value}))} + fullWidth/> + + Which standard's ring d-spacings the geometry is fitted against. Empty selects the + processing program's own default rather than meaning "no calibrant". + + Ring source + + Ring source + + + Selecting the powder calibration analysis mode also moves azimuthal integration onto the + CPU and gives it enough azimuthal sectors to locate a ring, so it runs at a few Hz — which + is all a calibration exposure needs. + + ); +} + +export default memo(CalibrationSettings); diff --git a/frontend/src/components/GridScanAnalysisSettings.tsx b/frontend/src/components/GridScanAnalysisSettings.tsx new file mode 100644 index 000000000..ebc658dbd --- /dev/null +++ b/frontend/src/components/GridScanAnalysisSettings.tsx @@ -0,0 +1,133 @@ +import {memo, useEffect, useState} from 'react'; + +import {Checkbox, FormControl, FormControlLabel} from '@mui/material'; +import {grid_scan_analysis_settings} from "../client"; +import {putConfigGridScanAnalysisMutation} from "../client/@tanstack/react-query.gen"; +import _ from "lodash"; +import NumberTextField from "./NumberTextField"; +import SettingsPanel from "./SettingsPanel"; +import {useUpload} from "./useUpload"; + +type MyProps = { + s?: grid_scan_analysis_settings +} + +const default_grid_scan_analysis_settings: grid_scan_analysis_settings = { + protein_score_threshold: 0.5, + min_blob_cells: 3, + decisive_single_cell_score: 0.9, + max_crystals: 10, + indexing: true +}; + +function GridScanAnalysisSettings({s: serverS}: MyProps) { + const [s, setS] = useState(default_grid_scan_analysis_settings); + const [lastDownloadedS, setLastDownloadedS] = + useState(default_grid_scan_analysis_settings); + const [downloadCounter, setDownloadCounter] = useState(0); + const [thresholdErr, setThresholdErr] = useState(false); + const [minCellsErr, setMinCellsErr] = useState(false); + const [decisiveErr, setDecisiveErr] = useState(false); + const [maxCrystalsErr, setMaxCrystalsErr] = useState(false); + const {submit, pending, snackbar} = useUpload(putConfigGridScanAnalysisMutation()); + + useEffect(() => { + if ((serverS !== undefined) && !_.isEqual(serverS, lastDownloadedS)) { + setS(serverS); + setLastDownloadedS(serverS); + setDownloadCounter(c => c + 1); + setThresholdErr(false); + setMinCellsErr(false); + setDecisiveErr(false); + setMaxCrystalsErr(false); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [serverS]); + + const hasError = () => thresholdErr || minCellsErr || decisiveErr || maxCrystalsErr; + const dirty = !_.isEqual(s, lastDownloadedS); + + return ( + submit(s)} uploadDisabled={pending || hasError()} snackbar={snackbar}> + Which cells are a crystal + + { + setS(prev => ({...prev, protein_score_threshold: val})); + setThresholdErr(err); + }} + float decimals={2} + fullWidth/> + + The per-image protein score saturates, so this only has to separate "something diffracted + here" from "nothing did". + + + { + setS(prev => ({...prev, min_blob_cells: val})); + setMinCellsErr(err); + }} + fullWidth/> + + Two cells can be the two ends of a single hit on a cell boundary; three is the smallest + patch that is a shape rather than a coincidence. + + + { + setS(prev => ({...prev, decisive_single_cell_score: val})); + setDecisiveErr(err); + }} + float decimals={2} + fullWidth/> + + One cell scoring this high is a crystal on its own — a crystal smaller than the grid step + lights exactly one cell. Keep it well above the threshold above. + + Reporting + + { + setS(prev => ({...prev, max_crystals: val})); + setMaxCrystalsErr(err); + }} + fullWidth/> + + Crystals are sorted by score, so this keeps the best. + + Indexing + setS(prev => ({...prev, indexing: e.target.checked}))}/>} + label="Index each raster cell"/> + Additive: crystals are still found on the protein score, so indexing only adds what was + found in them. Turn it off for a very large raster where the GPU is the constraint. + + ); +} + +export default memo(GridScanAnalysisSettings); diff --git a/image_analysis/IndexAndRefine.cpp b/image_analysis/IndexAndRefine.cpp index 67c969705..e7b879968 100644 --- a/image_analysis/IndexAndRefine.cpp +++ b/image_analysis/IndexAndRefine.cpp @@ -604,7 +604,11 @@ void IndexAndRefine::QuickPredictAndIntegrate(DataMessage &msg, std::optional IndexAndRefine::DetermineRefineAnalyze(DataMessage &msg, const SpotFindingSettings &spot_finding_settings) { - if (!indexer_ || !spot_finding_settings.indexing) + // The analysis mode decides first and the deprecated per-run switch second: a mode that does not + // index (none, azint, grid, powder_calibration) does not index whatever the switch says. One gate + // for all three frontends - nothing predicts or integrates without a lattice, so this also stops + // Bragg prediction and integration. + if (!indexer_ || !experiment.GetAnalysisStages().indexing || !spot_finding_settings.indexing) return std::nullopt; IndexingOutcome outcome(experiment); diff --git a/image_analysis/MXAnalysisAfterFPGA.cpp b/image_analysis/MXAnalysisAfterFPGA.cpp index 5c8150b12..39f6ae121 100644 --- a/image_analysis/MXAnalysisAfterFPGA.cpp +++ b/image_analysis/MXAnalysisAfterFPGA.cpp @@ -40,7 +40,11 @@ MXAnalysisAfterFPGA::MXAnalysisAfterFPGA(const DiffractionExperiment &in_experim if (experiment.IsSpotFindingEnabled()) find_spots = true; - if (experiment.GetAzimuthalIntegrationSettings().IsForceCPUinFPGAWorkflow()) + // AnalysisMode::None analyses nothing at all, azimuthal integration included; every other mode + // keeps it (see the table in common/AnalysisSettings.cpp). Note this only governs the CPU engine - + // on the FPGA path the integration is a stage of the hardware pipeline and is not switched here. + if (experiment.GetAnalysisStages().azimuthal_integration + && experiment.GetAzimuthalIntegrationSettings().IsForceCPUinFPGAWorkflow()) cpu_azint = std::make_unique(integration); } diff --git a/image_analysis/geom_refinement/PowderCalibration.h b/image_analysis/geom_refinement/PowderCalibration.h index 6e00452d5..5f34b5a9f 100644 --- a/image_analysis/geom_refinement/PowderCalibration.h +++ b/image_analysis/geom_refinement/PowderCalibration.h @@ -8,18 +8,13 @@ #include #include "../../common/AzimuthalIntegrationMapping.h" +#include "../../common/CalibrationSettings.h" // CalibrationMethod #include "../../common/DiffractionExperiment.h" #include "../../common/DiffractionGeometry.h" #include "../../common/SpotToSave.h" #include "PowderAutoSeed.h" #include "RingOptimizer.h" // RingFitUncertainty -// How the powder rings the detector geometry is fitted to are measured (rugnux --calibration). -enum class CalibrationMethod { - Rings, // the run-summed (q x azimuth) azimuthal profile: the ring measured at every azimuth - Spots // the pooled per-image spot lists: the ring sampled wherever the spot finder bit -}; - struct CalibrationResult { DiffractionGeometry geometry; // the fitted geometry size_t ring_points = 0; // ring measurements the fit used diff --git a/image_analysis/spot_finding/SpotFindingSettings.h b/image_analysis/spot_finding/SpotFindingSettings.h index 897408920..411036e7c 100644 --- a/image_analysis/spot_finding/SpotFindingSettings.h +++ b/image_analysis/spot_finding/SpotFindingSettings.h @@ -58,6 +58,9 @@ constexpr bool SpotShapeAccepted(int64_t pixel_count, int64_t bbox_side) { } struct SpotFindingSettings { + // DEPRECATED - the analysis mode (common/AnalysisSettings.h) is the switch now. It still gates spot + // finding beneath a mode that finds spots; see DiffractionExperiment::IsSpotFindingEnabled for the + // precedence between the three switches that answer this one question. bool enable = true; float signal_to_noise_threshold = 4.0; // STRONG_PIXEL in XDS int64_t photon_count_threshold = 10; // Threshold in photon counts diff --git a/reader/HDF5MetadataSource.cpp b/reader/HDF5MetadataSource.cpp index 71fee6092..392b86aaf 100644 --- a/reader/HDF5MetadataSource.cpp +++ b/reader/HDF5MetadataSource.cpp @@ -798,6 +798,53 @@ HDF5MetadataSource::OpenResult HDF5MetadataSource::Open(const std::string &filen } if (master_file->Exists("/entry/MX")) { + // The analysis that wrote the file, kept as provenance only. It is deliberately not pushed + // onto the experiment: what a stored file was produced by is not what the next run should + // do, and letting it decide would be exactly the implicitness the mode exists to remove. + dataset->file_analysis_mode = + AnalysisModeFromName(master_file->GetString("/entry/MX/analysis_mode", "")); + + // Grid-scan crystals: parallel 1-D datasets, one per field, all of length N. nx sets N and + // the rest are read into that length, so a file written by a build that had one field + // fewer still re-opens with the crystals it does have. + if (master_file->Exists("/entry/MX/crystals/nx")) { + auto nx = master_file->ReadOptVector("/entry/MX/crystals/nx"); + auto ny = master_file->ReadOptVector("/entry/MX/crystals/ny"); + auto x_um = master_file->ReadOptVector("/entry/MX/crystals/xUm"); + auto y_um = master_file->ReadOptVector("/entry/MX/crystals/yUm"); + auto image_number = master_file->ReadOptVector("/entry/MX/crystals/imageNumber"); + auto major_um = master_file->ReadOptVector("/entry/MX/crystals/majorUm"); + auto minor_um = master_file->ReadOptVector("/entry/MX/crystals/minorUm"); + auto angle_deg = master_file->ReadOptVector("/entry/MX/crystals/angleDeg"); + auto score = master_file->ReadOptVector("/entry/MX/crystals/score"); + auto ice_score = master_file->ReadOptVector("/entry/MX/crystals/iceScore"); + auto res_A = master_file->ReadOptVector("/entry/MX/crystals/resA"); + auto n_images = master_file->ReadOptVector("/entry/MX/crystals/nImages"); + + const size_t n = nx.size(); + auto f = [n](const std::vector &v, size_t i) { return v.size() == n ? v[i] : 0.0f; }; + auto i64 = [n](const std::vector &v, size_t i) { + return v.size() == n ? v[i] : int64_t(0); + }; + + dataset->grid_crystals.resize(n); + for (size_t i = 0; i < n; i++) { + auto &c = dataset->grid_crystals[i]; + c.nx = nx[i]; + c.ny = f(ny, i); + c.x_um = f(x_um, i); + c.y_um = f(y_um, i); + c.image_number = i64(image_number, i); + c.major_um = f(major_um, i); + c.minor_um = f(minor_um, i); + c.angle_deg = f(angle_deg, i); + c.score = f(score, i); + c.ice_score = f(ice_score, i); + c.res_A = f(res_A, i); + c.n_images = i64(n_images, i); + } + } + auto indexing = master_file->GetString("/entry/MX/indexing_algorithm", "none"); if (indexing == "fft" || indexing == "FFT (CUDA)" || indexing == "FFT (FFTW)") dataset->experiment.IndexingAlgorithm(IndexingAlgorithmEnum::FFT); diff --git a/reader/JFJochReaderDataset.h b/reader/JFJochReaderDataset.h index 8bf91072c..7380dd183 100644 --- a/reader/JFJochReaderDataset.h +++ b/reader/JFJochReaderDataset.h @@ -31,6 +31,15 @@ struct JFJochReaderDataset { // defaults ice handling by geometry when the file is silent (see rugnux_cli). std::optional file_detect_ice_rings; + // The analysis mode the master file records (/entry/MX/analysis_mode) - provenance, not a request: + // it says what produced the file, and nothing reads it to decide what to do next. Absent for a file + // written before the mode existed. + std::optional file_analysis_mode; + + // The crystals a grid scan found (/entry/MX/crystals), best score first. Empty for a file that is + // not a raster, or whose raster found nothing. + std::vector grid_crystals; + std::string jfjoch_release; // Change of basis (3x3 integers, row major) from the setting the per-image reflections and diff --git a/rugnux/Rugnux.cpp b/rugnux/Rugnux.cpp index 3fd6e078f..b74ad8792 100644 --- a/rugnux/Rugnux.cpp +++ b/rugnux/Rugnux.cpp @@ -489,6 +489,10 @@ Rugnux::Rugnux(JFJochReader &reader, DiffractionExperiment experiment, : reader_(reader), experiment_(std::move(experiment)), pixel_mask_(std::move(pixel_mask)), config_(std::move(config)), user_fixed_sg_(experiment_.GetGemmiSpaceGroup()) { + // The run's analysis mode belongs on the experiment as well as on the config: that is what the + // written file records, and what the pipeline's own gates (spot finding, indexing) read. + experiment_.ImportAnalysisSettings(experiment_.GetAnalysisSettings().Mode(config_.mode)); + // Bit 9 describes where THIS run found the beam stop, so a mask read back from a file that // already carries one starts clear; the user mask (bit 8) is left as it was loaded. pixel_mask_.ClearBeamStopMask(experiment_); @@ -1541,7 +1545,7 @@ ProcessResult Rugnux::RunAllPasses(RugnuxObserver *observer) { // (refined-geometry) pass writes merged files, under the plain "_*" name. The first pass still runs // under a "_01" prefix so that the _process.h5 it writes, when one is asked for, does not collide // with the second pass's - the ice-ring flags and geometry stored in each are the ones that pass used. - if (config_.mode == ProcessMode::FullAnalysis && config_.rotation_postrefine_geometry + if (AnalysisModeIsMX(config_.mode) && config_.rotation_postrefine_geometry && experiment_.IsRotationIndexing()) { Logger logger("Rugnux"); const std::string base_prefix = config_.output_prefix; @@ -1993,13 +1997,13 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b return result; } - const bool full = (config_.mode == ProcessMode::FullAnalysis); + const bool full = AnalysisModeIsMX(config_.mode); // Powder calibration runs the same per-image engine as the full analysis (spot finding lives inside // MXAnalysisWithoutFPGA) with indexing switched off in its settings. By rings it needs only the // azimuthal profile and the azint worker would do - but the spots are worth their half second even // then, because the circle through them places the beam centre without reference to the header at // all, which is the one hypothesis that survives a header the profile cannot correct. - const bool calibration = (config_.mode == ProcessMode::Calibration); + const bool calibration = (config_.mode == AnalysisMode::PowderCalibration); const bool calibration_spots = calibration && config_.calibration_method == CalibrationMethod::Spots; const bool per_image_analysis = full || calibration; const bool write_files = write_output && !config_.output_prefix.empty(); diff --git a/rugnux/Rugnux.h b/rugnux/Rugnux.h index de46a641d..cc67524ea 100644 --- a/rugnux/Rugnux.h +++ b/rugnux/Rugnux.h @@ -36,14 +36,12 @@ class JFJochHDF5Reader; // an optional scaling/merging post-pass, and the _process.h5 output. The detector geometry and all // algorithm settings are configured on the DiffractionExperiment by the caller; ProcessConfig only // carries run control. Cancellable from any thread (e.g. SIGINT or a GUI button) via Cancel(). -enum class ProcessMode { - AzimuthalIntegration, // preprocess + azimuthal integration only (rugnux --mode=azint) - FullAnalysis, // spot finding + indexing + refinement + integration (rugnux --mode=mx) - Calibration // detector geometry from powder rings (rugnux --mode=calibration) -}; - struct ProcessConfig { - ProcessMode mode = ProcessMode::FullAnalysis; + // The shared analysis vocabulary (common/AnalysisSettings.h), not a rugnux one: the broker, the + // viewer and rugnux all say what analysis runs the same way. rugnux's own --mode spellings are a + // layer above this (RugnuxMode in rugnux_cli.cpp); AnalysisMode::None never reaches here, since a + // rugnux run that analyses nothing has no purpose. + AnalysisMode mode = AnalysisMode::MXStills; int start_image = 0; int end_image = -1; // -1 => to the end of the dataset diff --git a/rugnux/RugnuxCommandLine.cpp b/rugnux/RugnuxCommandLine.cpp index 2f83c90a1..d16c8c14b 100644 --- a/rugnux/RugnuxCommandLine.cpp +++ b/rugnux/RugnuxCommandLine.cpp @@ -54,8 +54,8 @@ std::string RugnuxCommandLine(const ProcessConfig &config, const std::string &input_file, const std::string &calibrant_name) { std::vector args; - const bool azint = (config.mode == ProcessMode::AzimuthalIntegration); - const bool calibration = (config.mode == ProcessMode::Calibration); + const bool azint = (config.mode == AnalysisMode::Azint); + const bool calibration = (config.mode == AnalysisMode::PowderCalibration); args.emplace_back("rugnux"); if (azint) { args.emplace_back("--mode"); diff --git a/rugnux/RugnuxCommandLine.h b/rugnux/RugnuxCommandLine.h index f387a1686..384db0eae 100644 --- a/rugnux/RugnuxCommandLine.h +++ b/rugnux/RugnuxCommandLine.h @@ -5,14 +5,14 @@ #include -#include "Rugnux.h" // ProcessConfig, ProcessMode +#include "Rugnux.h" // ProcessConfig, AnalysisMode class DiffractionExperiment; // Reconstruct an equivalent rugnux command line (including --mode azint / calibration) for a // configured run, so a job set up in the GUI can be handed off to a cluster. Covers the settings that // matter for the run, not every obscure flag; geometry is taken from the input file, so geometry -// overrides are not emitted. calibrant_name is only used by ProcessMode::Calibration - the config +// overrides are not emitted. calibrant_name is only used by AnalysisMode::PowderCalibration - the config // carries the calibrant's rings, and --calibrant takes the name they were resolved from. std::string RugnuxCommandLine(const ProcessConfig &config, const DiffractionExperiment &experiment, diff --git a/rugnux/rugnux_cli.cpp b/rugnux/rugnux_cli.cpp index c6a177586..e7f2a2351 100644 --- a/rugnux/rugnux_cli.cpp +++ b/rugnux/rugnux_cli.cpp @@ -22,6 +22,7 @@ #include "../reader/JFJochCBFReader.h" #include "../common/Logger.h" #include "../common/Definitions.h" +#include "../common/CalibrationSettings.h" #include "../common/DiffractionExperiment.h" #include "../common/PixelMask.h" #include "../common/print_license.h" @@ -74,14 +75,6 @@ constexpr double RELATIVE_B_DEFAULT_DEG = 10.0; // mutually exclusive by construction. enum class RugnuxMode { MX, Azint, Scale, Calibration, Raster }; -// Azimuthal sectors used by --mode calibration --calibration rings when --azim-phi-bins was not given. -// The default of 1 is a plain radial profile, in which a ring has been averaged over every direction so -// nothing is left to say where its centre is (RingsFromAzimuthalProfile refuses below 4). Sweeping the -// count on a LaB6 exposure and on a crystal's ice rings, 32 and above agree to about 0.1 px and the -// residual is flat, while 8 sectors is visibly coarser (0.3 px away, twice the residual). Going higher -// buys nothing and runs into the 65534-bin limit on q x azimuth, so 32 is the default. -constexpr int CALIBRATION_AZIM_BINS_DEFAULT = 32; - // --mode raster: what a grid scan needs off the per-image loop. AnalyzeGridScan reads elem.number as // the image's place in the grid, so it is the file's OWN image number that goes there - the loop's // ordinal counts only the images -s/-e/--stride selected and means nothing to a grid. @@ -117,7 +110,9 @@ void print_usage() { std::cout << " Mode" << std::endl; std::cout << " --mode What this run does (default: mx)" << std::endl; - std::cout << " mx Full analysis - spot finding, indexing, integration and merging" << std::endl; + std::cout << " mx Full analysis - spot finding, indexing, integration and merging. Rotation or stills is chosen from the goniometer" << std::endl; + std::cout << " mx_rotation As mx, forced to rotation (two-pass indexing); same as -R" << std::endl; + std::cout << " mx_stills As mx, forced to per-image stills; same as --force-still" << std::endl; std::cout << " azint Only azimuthal integration (no spot finding/indexing); writes _process.h5" << std::endl; std::cout << " scale Only re-scale/merge the already-integrated reflections in (no re-integration)" << std::endl; std::cout << " calibration Determine the detector geometry from powder rings; writes .poni for pyFAI and .json, whose dataset_settings member is a jfjoch_broker dataset_settings body as it stands" << std::endl; @@ -1116,12 +1111,17 @@ static int RunRugnux(int argc, char **argv) { case OPT_MODE: { const std::string m = optarg ? optarg : ""; if (m == "mx") mode = RugnuxMode::MX; + // The two MX modes spelled out. mx picks between them from the goniometer; these say which + // one, and are the same tokens the shared AnalysisMode uses. mx_stills is what --force-still + // has always done, and mx_rotation what -R does, so they are spellings and not new switches. + else if (m == "mx_stills") { mode = RugnuxMode::MX; force_still = true; } + else if (m == "mx_rotation") { mode = RugnuxMode::MX; rotation_indexing = true; two_pass_rotation = true; } else if (m == "azint") mode = RugnuxMode::Azint; else if (m == "scale") mode = RugnuxMode::Scale; else if (m == "calibration") mode = RugnuxMode::Calibration; else if (m == "raster") mode = RugnuxMode::Raster; else { - logger.Error("Invalid --mode: {} (expected mx|azint|scale|calibration|raster)", m); + logger.Error("Invalid --mode: {} (expected mx|mx_rotation|mx_stills|azint|scale|calibration|raster)", m); return 1; } break; @@ -2115,7 +2115,7 @@ static int RunRugnux(int argc, char **argv) { if (polarization_factor) experiment.PolarizationFactor(polarization_factor.value()); ProcessConfig config; - config.mode = ProcessMode::AzimuthalIntegration; + config.mode = AnalysisMode::Azint; config.start_image = start_image; config.end_image = end_image; config.stride = image_stride; @@ -2264,8 +2264,13 @@ static int RunRugnux(int argc, char **argv) { ApplyRugnuxExperimentDefaults(experiment); if (polarization_factor) experiment.PolarizationFactor(polarization_factor.value()); + // The calibrant and the ring source on the experiment as well as on the config: one home for + // them, and what the written file records the calibration as having used. + experiment.ImportCalibrationSettings( + CalibrationSettings().Calibrant(calibrant).Method(calibration_method)); + ProcessConfig config; - config.mode = ProcessMode::Calibration; + config.mode = AnalysisMode::PowderCalibration; config.calibration_method = calibration_method; config.calibration_refine_tilt = calibration_refine_tilt; // -C wins over --calibrant: a cell given on the command line IS the standard, and the built-in @@ -2841,7 +2846,9 @@ static int RunRugnux(int argc, char **argv) { // Run the shared full-analysis workflow (rotation indexing + scaling/merging live in // Rugnux; the experiment above carries all algorithm settings). ProcessConfig config; - config.mode = ProcessMode::FullAnalysis; + // Rotation vs stills has been settled above (goniometer autodetect, --force-still, -R), so the + // shared mode can state which of the two MX analyses this run is rather than leaving it implicit. + config.mode = rotation_indexing ? AnalysisMode::MXRotation : AnalysisMode::MXStills; config.start_image = start_image; config.end_image = end_image; config.stride = image_stride; diff --git a/tests/AnalysisSettingsTest.cpp b/tests/AnalysisSettingsTest.cpp new file mode 100644 index 000000000..4595e45d1 --- /dev/null +++ b/tests/AnalysisSettingsTest.cpp @@ -0,0 +1,280 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include + +#include "../common/AnalysisSettings.h" +#include "../common/DiffractionExperiment.h" +#include "../frame_serialize/CBORStream2Serializer.h" +#include "../frame_serialize/CBORStream2Deserializer.h" +#include "../writer/FileWriter.h" +#include "../reader/JFJochHDF5Reader.h" + +TEST_CASE("AnalysisMode_Default", "[AnalysisMode]") { + // MXStills, not None: a deployment whose configuration predates the mode must keep analysing. + CHECK(AnalysisSettings().GetMode() == AnalysisMode::MXStills); + CHECK(DiffractionExperiment().GetAnalysisMode() == AnalysisMode::MXStills); +} + +TEST_CASE("AnalysisMode_Names", "[AnalysisMode]") { + for (auto mode : {AnalysisMode::None, AnalysisMode::MXRotation, AnalysisMode::MXStills, + AnalysisMode::Azint, AnalysisMode::Grid, AnalysisMode::PowderCalibration}) { + auto back = AnalysisModeFromName(AnalysisModeName(mode)); + REQUIRE(back.has_value()); + CHECK(*back == mode); + } + CHECK_FALSE(AnalysisModeFromName("").has_value()); + CHECK_FALSE(AnalysisModeFromName("mx").has_value()); +} + +TEST_CASE("AnalysisMode_Stages", "[AnalysisMode]") { + // The table itself. Bragg integration must never be on where indexing is off - nothing is + // predicted without a lattice. + for (auto mode : {AnalysisMode::None, AnalysisMode::MXRotation, AnalysisMode::MXStills, + AnalysisMode::Azint, AnalysisMode::Grid, AnalysisMode::PowderCalibration}) { + const auto s = AnalysisModeStages(mode); + CHECK((!s.bragg_integration || s.indexing)); + CHECK((!s.scale_merge || s.bragg_integration)); + } + + CHECK(AnalysisModeStages(AnalysisMode::MXStills).indexing); + CHECK(AnalysisModeStages(AnalysisMode::MXRotation).indexing); + + const auto none = AnalysisModeStages(AnalysisMode::None); + CHECK_FALSE(none.spot_finding); + CHECK_FALSE(none.azimuthal_integration); + + const auto azint = AnalysisModeStages(AnalysisMode::Azint); + CHECK_FALSE(azint.spot_finding); + CHECK_FALSE(azint.indexing); + CHECK_FALSE(azint.scoring); + CHECK(azint.azimuthal_integration); + + // Calibration keeps the spot finder: --calibration spots fits the pooled spots. + const auto calib = AnalysisModeStages(AnalysisMode::PowderCalibration); + CHECK(calib.spot_finding); + CHECK_FALSE(calib.indexing); + + // Grid indexing is a setting, so the table only carries its default. + const auto grid = AnalysisModeStages(AnalysisMode::Grid); + CHECK(grid.spot_finding); + CHECK(grid.scoring); + CHECK(grid.indexing == GridScanAnalysisSettings().IsIndexing()); +} + +TEST_CASE("AnalysisMode_GridIndexingIsASetting", "[AnalysisMode]") { + // The one stage the mode does not fix. The configured value must reach the gates, which read + // GetAnalysisStages and not the table. + DiffractionExperiment experiment; + experiment.ImportAnalysisSettings(AnalysisSettings().Mode(AnalysisMode::Grid)); + CHECK(experiment.GetAnalysisStages().indexing); // the default is on + + experiment.ImportGridScanAnalysisSettings(GridScanAnalysisSettings().Indexing(false)); + CHECK_FALSE(experiment.GetAnalysisStages().indexing); + + experiment.ImportGridScanAnalysisSettings(GridScanAnalysisSettings().Indexing(true)); + CHECK(experiment.GetAnalysisStages().indexing); + + // ...and only under Grid. No other mode's indexing answer moves with it. + experiment.ImportAnalysisSettings(AnalysisSettings().Mode(AnalysisMode::Azint)); + experiment.ImportGridScanAnalysisSettings(GridScanAnalysisSettings().Indexing(true)); + CHECK_FALSE(experiment.GetAnalysisStages().indexing); +} + +TEST_CASE("AnalysisMode_GridScanSettingsDefaults", "[AnalysisMode]") { + const GridScanAnalysisSettings s; + // A single cell is admitted only well above the threshold that admits a cell into a blob; + // otherwise the decisive rule would just be a lower general threshold. + CHECK(s.GetDecisiveSingleCellScore() > s.GetProteinScoreThreshold()); + CHECK(s.GetMinBlobCells() >= 1); + CHECK(s.GetMaxCrystals() >= 1); + + GridScanAnalysisSettings set; + set.ProteinScoreThreshold(0.7f).MinBlobCells(5).DecisiveSingleCellScore(0.95f) + .MaxCrystals(3).Indexing(false); + CHECK(set.GetProteinScoreThreshold() == Catch::Approx(0.7)); + CHECK(set.GetMinBlobCells() == 5); + CHECK(set.GetDecisiveSingleCellScore() == Catch::Approx(0.95)); + CHECK(set.GetMaxCrystals() == 3); + CHECK_FALSE(set.IsIndexing()); +} + +TEST_CASE("AnalysisMode_PrecedenceOverSpotFindingSwitch", "[AnalysisMode]") { + DiffractionExperiment experiment; + DatasetSettings dataset; + dataset.SpotFindingEnable(true).MaxSpotCount(500); + experiment.ImportDatasetSettings(dataset); + + CHECK(experiment.IsSpotFindingEnabled()); + + // A mode that analyses no spots wins over the deprecated per-dataset switch, and takes the spot + // budget with it. + experiment.ImportAnalysisSettings(AnalysisSettings().Mode(AnalysisMode::Azint)); + CHECK_FALSE(experiment.IsSpotFindingEnabled()); + CHECK(experiment.GetMaxSpotCount() == 0); + + // Under a mode that does find spots, the deprecated switch is still able to turn it off. + experiment.ImportAnalysisSettings(AnalysisSettings().Mode(AnalysisMode::MXStills)); + CHECK(experiment.IsSpotFindingEnabled()); + dataset.SpotFindingEnable(false); + experiment.ImportDatasetSettings(dataset); + CHECK_FALSE(experiment.IsSpotFindingEnabled()); +} + +TEST_CASE("AnalysisMode_PowderCalibrationForcesCPUAzInt", "[AnalysisMode]") { + // The FPGA integration core holds 2048 bins in total, so a sectored profile cannot be built there. + DiffractionExperiment experiment; + REQUIRE_FALSE(experiment.GetAzimuthalIntegrationSettings().IsForceCPUinFPGAWorkflow()); + + experiment.ImportAnalysisSettings(AnalysisSettings().Mode(AnalysisMode::PowderCalibration)); + CHECK(experiment.GetAzimuthalIntegrationSettings().IsForceCPUinFPGAWorkflow()); + CHECK(experiment.GetAzimuthalIntegrationSettings().GetAzimuthalBinCount() + == CALIBRATION_AZIM_BINS_DEFAULT); + + // An explicit sector count stands; the mode only supplies one where none is usable. + DiffractionExperiment explicit_bins; + explicit_bins.ImportAzimuthalIntegrationSettings( + AzimuthalIntegrationSettings().AzimuthalBinCount(64)); + explicit_bins.ImportAnalysisSettings(AnalysisSettings().Mode(AnalysisMode::PowderCalibration)); + CHECK(explicit_bins.GetAzimuthalIntegrationSettings().GetAzimuthalBinCount() == 64); + + // The coupling is applied from both imports, so the order the two are set in does not matter. + DiffractionExperiment settings_first; + settings_first.ImportCalibrationSettings(CalibrationSettings().Calibrant("lab6")); + CHECK_FALSE(settings_first.GetAzimuthalIntegrationSettings().IsForceCPUinFPGAWorkflow()); + settings_first.ImportAnalysisSettings(AnalysisSettings().Mode(AnalysisMode::PowderCalibration)); + CHECK(settings_first.GetAzimuthalIntegrationSettings().IsForceCPUinFPGAWorkflow()); + + DiffractionExperiment mode_first; + mode_first.ImportAnalysisSettings(AnalysisSettings().Mode(AnalysisMode::PowderCalibration)); + mode_first.ImportCalibrationSettings(CalibrationSettings().Calibrant("lab6")); + CHECK(mode_first.GetAzimuthalIntegrationSettings().IsForceCPUinFPGAWorkflow()); + CHECK(mode_first.GetCalibrationSettings().GetCalibrant() == "lab6"); + + // A mode that is not a calibration is left alone. + DiffractionExperiment stills; + stills.ImportCalibrationSettings(CalibrationSettings().Method(CalibrationMethod::Spots)); + CHECK_FALSE(stills.GetAzimuthalIntegrationSettings().IsForceCPUinFPGAWorkflow()); + CHECK(stills.GetCalibrationSettings().GetMethod() == CalibrationMethod::Spots); +} + +TEST_CASE("AnalysisMode_CBORStartRoundTrip", "[AnalysisMode][CBOR]") { + std::vector buffer(1024 * 1024); + CBORStream2Serializer serializer(buffer.data(), buffer.size()); + + DiffractionExperiment experiment; + experiment.ImportAnalysisSettings(AnalysisSettings().Mode(AnalysisMode::Grid)); + + StartMessage message{}; + experiment.FillMessage(message); + REQUIRE(message.analysis_mode.has_value()); + REQUIRE_NOTHROW(serializer.SerializeSequenceStart(message)); + + auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize()); + REQUIRE(deserialized); + REQUIRE(deserialized->start_message); + REQUIRE(deserialized->start_message->analysis_mode.has_value()); + CHECK(*deserialized->start_message->analysis_mode == AnalysisMode::Grid); +} + +TEST_CASE("AnalysisMode_CBORStartAbsentMode", "[AnalysisMode][CBOR]") { + // A stream written before the mode existed says nothing, and is read back as saying nothing - + // naming a mode for it would be an invention rather than provenance. + std::vector buffer(1024 * 1024); + CBORStream2Serializer serializer(buffer.data(), buffer.size()); + + StartMessage message{}; + REQUIRE_NOTHROW(serializer.SerializeSequenceStart(message)); + + auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize()); + REQUIRE(deserialized); + REQUIRE(deserialized->start_message); + CHECK_FALSE(deserialized->start_message->analysis_mode.has_value()); +} + +TEST_CASE("AnalysisMode_CBORGridCrystals", "[AnalysisMode][CBOR]") { + std::vector buffer(1024 * 1024); + CBORStream2Serializer serializer(buffer.data(), buffer.size()); + + EndMessage message{}; + message.max_image_number = 0; + // Two crystals, not one: a raster can find any number and nothing may assume at most one. + message.grid_crystals.push_back(GridScanCrystal{ + .nx = 3.5f, .ny = 7.25f, .x_um = -12.5f, .y_um = 40.0f, .image_number = 143, + .major_um = 22.5f, .minor_um = 8.0f, .angle_deg = 179.5f, .score = 0.92f, + .ice_score = 0.03f, .res_A = 1.85f, .n_images = 17}); + message.grid_crystals.push_back(GridScanCrystal{ + .nx = 11.0f, .ny = 2.0f, .x_um = 60.0f, .y_um = -30.0f, .image_number = 44, + .major_um = 9.0f, .minor_um = 7.5f, .angle_deg = 0.5f, .score = 0.41f, + .ice_score = 0.30f, .res_A = 3.2f, .n_images = 4}); + REQUIRE_NOTHROW(serializer.SerializeSequenceEnd(message)); + + auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize()); + REQUIRE(deserialized); + REQUIRE(deserialized->end_message); + const auto &out = deserialized->end_message->grid_crystals; + REQUIRE(out.size() == 2); + CHECK(out[0].nx == Catch::Approx(3.5)); + CHECK(out[0].image_number == 143); + CHECK(out[0].major_um == Catch::Approx(22.5)); + CHECK(out[0].angle_deg == Catch::Approx(179.5)); + CHECK(out[0].res_A == Catch::Approx(1.85)); + CHECK(out[0].n_images == 17); + CHECK(out[1].score == Catch::Approx(0.41)); + CHECK(out[1].ice_score == Catch::Approx(0.30)); + CHECK(out[1].y_um == Catch::Approx(-30.0)); +} + +TEST_CASE("AnalysisMode_HDF5MasterRoundTrip", "[AnalysisMode][HDF5][Full]") { + // The mode and the crystal list are dataset-wide metadata: written to the master, read back from + // it, so a stored raster re-opens knowing what produced it and what it found. + DiffractionExperiment x(DetJF(1)); + x.FilePrefix("test_analysis_mode").ImagesPerTrigger(1).OverwriteExistingFiles(true); + x.BeamX_pxl(100).BeamY_pxl(200).DetectorDistance_mm(150) + .IncidentEnergy_keV(WVL_1A_IN_KEV).PixelSigned(false).BitDepthImage(16) + .FrameTime(std::chrono::microseconds(500), std::chrono::microseconds(10)); + x.ImportAnalysisSettings(AnalysisSettings().Mode(AnalysisMode::Grid)); + RegisterHDF5Filter(); + + std::vector image(x.GetPixelsNum(), 0); + { + StartMessage start_message; + x.FillMessage(start_message); + + FileWriter file_set(start_message); + + DataMessage message{}; + message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum()); + message.number = 0; + REQUIRE_NOTHROW(file_set.WriteHDF5(message)); + + EndMessage end_message; + end_message.max_image_number = 1; + end_message.grid_crystals.push_back(GridScanCrystal{ + .nx = 3.5f, .ny = 7.25f, .x_um = -12.5f, .y_um = 40.0f, .image_number = 143, + .major_um = 22.5f, .minor_um = 8.0f, .angle_deg = 179.5f, .score = 0.92f, + .ice_score = 0.03f, .res_A = 1.85f, .n_images = 17}); + end_message.grid_crystals.push_back(GridScanCrystal{ + .nx = 11.0f, .ny = 2.0f, .x_um = 60.0f, .y_um = -30.0f, .image_number = 44, + .major_um = 9.0f, .minor_um = 7.5f, .angle_deg = 0.5f, .score = 0.41f, + .ice_score = 0.30f, .res_A = 3.2f, .n_images = 4}); + REQUIRE_NOTHROW(file_set.WriteHDF5(end_message)); + file_set.Finalize(); + } + { + JFJochHDF5Reader reader; + reader.ReadFile("test_analysis_mode_master.h5"); + auto dataset = reader.GetDataset(); + REQUIRE(dataset->file_analysis_mode.has_value()); + CHECK(*dataset->file_analysis_mode == AnalysisMode::Grid); + REQUIRE(dataset->grid_crystals.size() == 2); + CHECK(dataset->grid_crystals[0].nx == Catch::Approx(3.5)); + CHECK(dataset->grid_crystals[0].image_number == 143); + CHECK(dataset->grid_crystals[0].angle_deg == Catch::Approx(179.5)); + CHECK(dataset->grid_crystals[0].n_images == 17); + CHECK(dataset->grid_crystals[1].score == Catch::Approx(0.41)); + CHECK(dataset->grid_crystals[1].res_A == Catch::Approx(3.2)); + } + remove("test_analysis_mode_master.h5"); + remove("test_analysis_mode_data_000001.h5"); +} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 2158bb6d9..adc528c44 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,5 +1,6 @@ ADD_EXECUTABLE(jfjoch_test DiffractionExperimentTest.cpp + AnalysisSettingsTest.cpp RawToConvertedGeometryTest.cpp ../common/RawToConvertedGeometry.h ../common/DiffractionExperiment.h diff --git a/tests/RugnuxLargeTest.cpp b/tests/RugnuxLargeTest.cpp index d8b995877..fc09a1a54 100644 --- a/tests/RugnuxLargeTest.cpp +++ b/tests/RugnuxLargeTest.cpp @@ -59,7 +59,7 @@ TEST_CASE("Rugnux_Rotation", "[large]") { experiment.ImportIndexingSettings(indexing); ProcessConfig config; - config.mode = ProcessMode::FullAnalysis; + config.mode = AnalysisMode::MXStills; config.nthreads = default_threads(); config.spot_finding = DiffractionExperiment::DefaultDataProcessingSettings(); config.spot_finding.indexing = true; diff --git a/tests/RugnuxTest.cpp b/tests/RugnuxTest.cpp index 105705e36..71e6098f3 100644 --- a/tests/RugnuxTest.cpp +++ b/tests/RugnuxTest.cpp @@ -55,7 +55,7 @@ TEST_CASE("Rugnux_AzInt", "[HDF5][Full]") { REQUIRE(dataset); ProcessConfig config; - config.mode = ProcessMode::AzimuthalIntegration; + config.mode = AnalysisMode::Azint; config.nthreads = 2; config.output_prefix = "process_azint_out"; @@ -94,7 +94,7 @@ TEST_CASE("Rugnux_NoOutput", "[HDF5][Full]") { // Empty output prefix => process without writing any file. ProcessConfig config; - config.mode = ProcessMode::AzimuthalIntegration; + config.mode = AnalysisMode::Azint; config.nthreads = 3; Rugnux process(reader, dataset->experiment, *dataset->pixel_mask, config); @@ -118,7 +118,7 @@ TEST_CASE("Rugnux_Cancel", "[HDF5][Full]") { auto dataset = reader.GetDataset(); ProcessConfig config; - config.mode = ProcessMode::AzimuthalIntegration; + config.mode = AnalysisMode::Azint; config.nthreads = 2; Rugnux process(reader, dataset->experiment, *dataset->pixel_mask, config); @@ -144,7 +144,7 @@ TEST_CASE("RugnuxCommandLine_Full", "[process]") { x.SpaceGroupNumber(96); ProcessConfig config; - config.mode = ProcessMode::FullAnalysis; + config.mode = AnalysisMode::MXStills; config.nthreads = 8; config.output_prefix = "run1"; config.end_image = 500; @@ -174,7 +174,7 @@ TEST_CASE("RugnuxCommandLine_AzInt", "[process]") { x.ImportAzimuthalIntegrationSettings(a); ProcessConfig config; - config.mode = ProcessMode::AzimuthalIntegration; + config.mode = AnalysisMode::Azint; config.nthreads = 2; config.output_prefix = "az"; diff --git a/viewer/widgets/JFJochViewerSettingsDock.cpp b/viewer/widgets/JFJochViewerSettingsDock.cpp index 8a7e52f02..4f8511821 100644 --- a/viewer/widgets/JFJochViewerSettingsDock.cpp +++ b/viewer/widgets/JFJochViewerSettingsDock.cpp @@ -126,8 +126,8 @@ JFJochViewerSettingsDock::JFJochViewerSettingsDock(const SpotFindingSettings &sp stack->addWidget(new QWidget(stack)); // AzInt: the shared section below is the whole page stack->addWidget(BuildCalibrationPage()); connect(group, &QButtonGroup::idClicked, this, [this, stack](int id) { - mode_ = id == 1 ? ProcessMode::AzimuthalIntegration - : id == 2 ? ProcessMode::Calibration : ProcessMode::FullAnalysis; + mode_ = id == 1 ? AnalysisMode::Azint + : id == 2 ? AnalysisMode::PowderCalibration : AnalysisMode::MXStills; stack->setCurrentIndex(id); azintSection_->setVisible(id != 0); }); diff --git a/viewer/widgets/JFJochViewerSettingsDock.h b/viewer/widgets/JFJochViewerSettingsDock.h index 639846ebe..888815ba1 100644 --- a/viewer/widgets/JFJochViewerSettingsDock.h +++ b/viewer/widgets/JFJochViewerSettingsDock.h @@ -14,7 +14,7 @@ #include "../../image_analysis/spot_finding/SpotFindingSettings.h" #include "../../reader/JFJochReaderDataset.h" #include "../../reader/JFJochReaderImage.h" -#include "../../rugnux/Rugnux.h" // ProcessMode +#include "../../rugnux/Rugnux.h" // AnalysisMode #include "../ReferenceMtzInfo.h" #include "PowderCalibrationWidget.h" // CalibrationSelection @@ -58,8 +58,8 @@ signals: void referenceSelected(QString path, QString column); // user picked a reference MTZ / column void reanalyzeImage(bool armed); // "Analyze image" toggle // "Analyze dataset". The mode is the selected page; the calibrant selection is only read for - // ProcessMode::Calibration and is carried along so the job needs no second copy of the controls. - void analyzeDataset(ProcessMode mode, CalibrationSelection calibration); + // AnalysisMode::PowderCalibration and is carried along so the job needs no second copy of the controls. + void analyzeDataset(AnalysisMode mode, CalibrationSelection calibration); private: SpotFindingSettings spot_; @@ -75,7 +75,7 @@ private: // turns the pair into that field, and per-image only ever applies to stills. bool adaptive_min_pix_ = false; int64_t min_pix_value_ = 2; - ProcessMode mode_ = ProcessMode::FullAnalysis; // the selected page, and what "Analyze dataset" runs + AnalysisMode mode_ = AnalysisMode::MXStills; // the selected page, and what "Analyze dataset" runs // "Analyze dataset" hero button, disabled while a live HTTP source is connected. QPushButton *analyzeDatasetBtn_ = nullptr; diff --git a/viewer/windows/JFJochProcessingJobsWindow.cpp b/viewer/windows/JFJochProcessingJobsWindow.cpp index 2c7e0cb44..6edf8e569 100644 --- a/viewer/windows/JFJochProcessingJobsWindow.cpp +++ b/viewer/windows/JFJochProcessingJobsWindow.cpp @@ -55,19 +55,21 @@ namespace { } // Short name of a run mode, for the dialog title, the Mode column and the run label. - const char *mode_name(ProcessMode mode) { + const char *mode_name(AnalysisMode mode) { switch (mode) { - case ProcessMode::AzimuthalIntegration: return "AzInt"; - case ProcessMode::Calibration: return "Calib"; - case ProcessMode::FullAnalysis: - default: return "Full"; + case AnalysisMode::Azint: return "AzInt"; + case AnalysisMode::PowderCalibration: return "Calib"; + case AnalysisMode::Grid: return "Grid"; + case AnalysisMode::None: return "None"; + case AnalysisMode::MXRotation: + case AnalysisMode::MXStills: + default: return "Full"; } } // Azimuthal sectors a calibration by rings falls back to when the panel has too few of them: with - // one sector the profile has averaged the ring over every direction and cannot locate it. Same - // value, for the same reason, as rugnux --mode calibration. - constexpr int CALIBRATION_AZIM_BINS_DEFAULT = 32; + // one sector the profile has averaged the ring over every direction and cannot locate it. The + // shared default (common/AnalysisSettings.h) - the same value, for the same reason, everywhere. } JFJochProcessingJobsWindow::JFJochProcessingJobsWindow(JFJochImageReadingWorker *worker, QWidget *parent) @@ -141,8 +143,8 @@ int JFJochProcessingJobsWindow::askJob(const ReprocessingInputs &inputs, JobSpec QDialog dlg(window()); // centre on the main window, not inside the processing dock // The kind of job comes from the panel's MX / AzInt / Calib toggle, so the dialog only collects // the run options. - const bool azint = spec.mode == ProcessMode::AzimuthalIntegration; - const bool calibration = spec.mode == ProcessMode::Calibration; + const bool azint = spec.mode == AnalysisMode::Azint; + const bool calibration = spec.mode == AnalysisMode::PowderCalibration; dlg.setWindowTitle(calibration ? "New detector-calibration job" : azint ? "New azimuthal-integration job" : "New full-analysis job"); @@ -208,7 +210,7 @@ int JFJochProcessingJobsWindow::askJob(const ReprocessingInputs &inputs, JobSpec // cell from the strongest indexed frames, then the main pass re-indexes with it. It is stills-only // and anchors on a known cell, so offer it only for a stills-with-cell run; default it on there, to // match the rugnux CLI (a no-op for rotation / de-novo stills). Stills mode = rotation indexing off. - const bool stills_with_cell = spec.mode == ProcessMode::FullAnalysis + const bool stills_with_cell = AnalysisModeIsMX(spec.mode) && !inputs.experiment.GetIndexingSettings().GetRotationIndexing() && inputs.experiment.GetUnitCell().has_value(); auto *refine_geometry = new QCheckBox("Refine geometry (stills)", &dlg); @@ -236,7 +238,7 @@ int JFJochProcessingJobsWindow::askJob(const ReprocessingInputs &inputs, JobSpec // cell + rotation axis from the whole sweep, then re-integrates at that geometry (only the refined pass // is written, as the canonical _* output). Rotation-only; // default on there, to match the rugnux CLI. - const bool rotation_run = spec.mode == ProcessMode::FullAnalysis + const bool rotation_run = AnalysisModeIsMX(spec.mode) && inputs.experiment.GetIndexingSettings().GetRotationIndexing(); auto *postrefine = new QCheckBox("Post-refine geometry (rotation, two-pass)", &dlg); postrefine->setChecked(rotation_run); @@ -361,7 +363,7 @@ ProcessConfig JFJochProcessingJobsWindow::buildConfig(const JobSpec &spec, const // Files land at output_prefix; leave it empty (write nothing, stats only) when neither output is // wanted. The two flags then select which files are actually written there. A calibration always // keeps the prefix - the .poni written next to it is the whole point of the run. - config.output_prefix = (spec.save_h5 || spec.save_merged || spec.mode == ProcessMode::Calibration) + config.output_prefix = (spec.save_h5 || spec.save_merged || spec.mode == AnalysisMode::PowderCalibration) ? spec.prefix.toStdString() : std::string(); config.write_process_h5 = spec.save_h5; config.write_merged = spec.save_merged; @@ -376,7 +378,7 @@ ProcessConfig JFJochProcessingJobsWindow::buildConfig(const JobSpec &spec, const // (--no-fit-spindle is the deviation), so the dialog's single switch sets both. config.estimate_beam_center = spec.estimate_beam_center; config.fit_spindle = spec.estimate_beam_center; - if (spec.mode == ProcessMode::Calibration) { + if (spec.mode == AnalysisMode::PowderCalibration) { config.calibration_method = spec.calibration.method; config.calibration_refine_tilt = spec.calibration.refine_tilt; config.calibrant_ring_q = spec.calibration.ring_q; @@ -385,7 +387,7 @@ ProcessConfig JFJochProcessingJobsWindow::buildConfig(const JobSpec &spec, const config.spot_finding.enable = true; config.spot_finding.indexing = false; } - if (spec.mode == ProcessMode::FullAnalysis) { + if (AnalysisModeIsMX(spec.mode)) { // Rotation indexing follows the panel's rotation axis (= the experiment's indexing // setting); a rotation run uses 60 first-pass images to find the lattice. config.rotation_indexing = inputs.experiment.GetIndexingSettings().GetRotationIndexing(); @@ -404,7 +406,7 @@ ProcessConfig JFJochProcessingJobsWindow::buildConfig(const JobSpec &spec, const return config; } -void JFJochProcessingJobsWindow::newJob(ProcessMode mode, CalibrationSelection calibration) { +void JFJochProcessingJobsWindow::newJob(AnalysisMode mode, CalibrationSelection calibration) { const ReprocessingInputs inputs = worker_->GetReprocessingInputs(); if (!inputs.valid) { QMessageBox::information(this, "Processing", "Open a file first (processing is not available for live HTTP data)."); @@ -439,13 +441,13 @@ void JFJochProcessingJobsWindow::newJob(ProcessMode mode, CalibrationSelection c // Calibrating from the run-summed profile needs the profile to be binned in azimuth; the panel's // bin count is an azimuthal-integration setting and defaults to a plain radial profile, which // carries no information about where the ring centre is. Same fallback as the rugnux CLI. - if (spec.mode == ProcessMode::Calibration && spec.calibration.method == CalibrationMethod::Rings + if (spec.mode == AnalysisMode::PowderCalibration && spec.calibration.method == CalibrationMethod::Rings && experiment.GetAzimuthalIntegrationSettings().GetAzimuthalBinCount() < 4) { AzimuthalIntegrationSettings azint = experiment.GetAzimuthalIntegrationSettings(); azint.AzimuthalBinCount(CALIBRATION_AZIM_BINS_DEFAULT); experiment.ImportAzimuthalIntegrationSettings(azint); } - if (spec.mode == ProcessMode::FullAnalysis && spec.scaling) { + if (AnalysisModeIsMX(spec.mode) && spec.scaling) { ScalingSettings scaling = RugnuxDefaultScalingSettings(config.rotation_indexing); // Keep what the settings dock does expose; take the rest from the shared defaults. const auto &dock = inputs.experiment.GetScalingSettings(); @@ -487,7 +489,7 @@ void JFJochProcessingJobsWindow::newJob(ProcessMode mode, CalibrationSelection c info.label = label; if (spec.save_h5) info.snapshot_path = QString::fromStdString(config.output_prefix) + "_process.h5"; - if (spec.mode == ProcessMode::Calibration) { + if (spec.mode == AnalysisMode::PowderCalibration) { info.poni_path = spec.prefix + ".poni"; info.calibration_header = experiment.GetDiffractionGeometry(); calibration_experiment_ = experiment; diff --git a/viewer/windows/JFJochProcessingJobsWindow.h b/viewer/windows/JFJochProcessingJobsWindow.h index 5d33fa5d3..f6f3dd72c 100644 --- a/viewer/windows/JFJochProcessingJobsWindow.h +++ b/viewer/windows/JFJochProcessingJobsWindow.h @@ -45,8 +45,8 @@ public slots: void clearJobs(); // reset the table on a new file (re-adds the Original row) void setActiveRun(QString active_id); // bold the row of the run currently shown in the plots // The panel's "Analyze dataset" action; the mode is the panel's selected page. The calibrant - // selection is only used by ProcessMode::Calibration. - void newJob(ProcessMode mode = ProcessMode::FullAnalysis, CalibrationSelection calibration = {}); + // selection is only used by AnalysisMode::PowderCalibration. + void newJob(AnalysisMode mode = AnalysisMode::MXStills, CalibrationSelection calibration = {}); private slots: void cancelJob(); @@ -80,7 +80,7 @@ private: QToolButton *graph_btn = nullptr; // per-row "show statistics" button (enabled once stats exist) }; struct JobSpec { - ProcessMode mode = ProcessMode::FullAnalysis; + AnalysisMode mode = AnalysisMode::MXStills; int start_image = 0; int end_image = 0; // 0 == to the end int threads = 4; diff --git a/writer/HDF5NXmx.cpp b/writer/HDF5NXmx.cpp index 85765e5a4..08377874a 100644 --- a/writer/HDF5NXmx.cpp +++ b/writer/HDF5NXmx.cpp @@ -496,6 +496,12 @@ void NXmx::Detector(const StartMessage &start, const EndMessage &end) { void NXmx::MX(const StartMessage &start) { HDF5Group(*hdf5_file, "/entry/MX").NXClass("NXcollection"); + + // Which analysis produced this file, in the same spelling the CBOR stream and the rugnux --mode + // option use. Absent when the stream carried no mode, i.e. was written before the field existed. + if (start.analysis_mode.has_value()) + hdf5_file->SaveScalar("/entry/MX/analysis_mode", AnalysisModeName(start.analysis_mode.value())); + switch (start.indexing_algorithm) { case IndexingAlgorithmEnum::FFBIDX: hdf5_file->SaveScalar("/entry/MX/indexing_algorithm", "FFBIDX"); @@ -1165,6 +1171,47 @@ void NXmx::Finalize(const EndMessage &end) { if (end.rotation_lattice_type) SaveScalar(*hdf5_file, "/entry/MX/rotationLatticeNiggliClass", end.rotation_lattice_type->niggli_class); + // The crystals a grid scan found: dataset-wide metadata, so it goes in the master. Written as + // parallel 1-D datasets of length N under /entry/MX/crystals, one per field, because crystals + // are numbered rather than named - a group per crystal would put the index in the path and make + // the list unreadable without walking it. Sorted by score, best first, as it arrives. + if (!end.grid_crystals.empty()) { + const size_t n = end.grid_crystals.size(); + std::vector nx(n), ny(n), x_um(n), y_um(n), major_um(n), minor_um(n), + angle_deg(n), score(n), ice_score(n), res_A(n); + std::vector image_number(n), n_images(n); + for (size_t i = 0; i < n; i++) { + const auto &c = end.grid_crystals[i]; + nx[i] = c.nx; + ny[i] = c.ny; + x_um[i] = c.x_um; + y_um[i] = c.y_um; + image_number[i] = c.image_number; + major_um[i] = c.major_um; + minor_um[i] = c.minor_um; + angle_deg[i] = c.angle_deg; + score[i] = c.score; + ice_score[i] = c.ice_score; + res_A[i] = c.res_A; + n_images[i] = c.n_images; + } + HDF5Group(*hdf5_file, "/entry/MX/crystals").NXClass("NXcollection"); + SaveVector(*hdf5_file, "/entry/MX/crystals/nx", nx); + SaveVector(*hdf5_file, "/entry/MX/crystals/ny", ny); + SaveVector(*hdf5_file, "/entry/MX/crystals/xUm", x_um)->Units("um"); + SaveVector(*hdf5_file, "/entry/MX/crystals/yUm", y_um)->Units("um"); + SaveVector(*hdf5_file, "/entry/MX/crystals/imageNumber", image_number); + SaveVector(*hdf5_file, "/entry/MX/crystals/majorUm", major_um)->Units("um"); + SaveVector(*hdf5_file, "/entry/MX/crystals/minorUm", minor_um)->Units("um"); + // An axis, not a direction: in [0, 180), so 179 deg and 0 deg are adjacent and these do + // not average arithmetically. + SaveVector(*hdf5_file, "/entry/MX/crystals/angleDeg", angle_deg)->Units("deg"); + SaveVector(*hdf5_file, "/entry/MX/crystals/score", score); + SaveVector(*hdf5_file, "/entry/MX/crystals/iceScore", ice_score); + SaveVector(*hdf5_file, "/entry/MX/crystals/resA", res_A)->Units("Angstrom"); + SaveVector(*hdf5_file, "/entry/MX/crystals/nImages", n_images); + } + // The setting the per-image reflections and lattices were written in is the one they were // indexed in, which is not always the setting of the cell above: the space group is chosen // after the images have gone to file, and choosing it can re-seat the lattice. Write the