Merge branch 'rc188-report-counts' into rc188-int

This commit is contained in:
leonarski_f committed 2026-11-19 15:37:18 +01:00
commit ed753654fb
50 files changed
+474 -81

No files matched your search

+1
View File
@@ -958,6 +958,7 @@ PlotType ConvertPlotType(const std::optional<std::string>& input) {
if (input == "image_scale_cc") return PlotType::ImageScaleCC;
if (input == "compression_ratio") return PlotType::CompressionRatio;
if (input == "indexing_lattice_count") return PlotType::IndexingLatticeCount;
if (input == "image_merged") return PlotType::ImageMerged;
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"Plot type not recognized");
}
+1
View File
@@ -130,6 +130,7 @@ components:
- image_scale_cc
- compression_ratio
- ice_ring_score
- image_merged
roi:
in: query
name: roi
+1 -1
View File
@@ -542,7 +542,7 @@ This format doesn&#x27;t transmit information about X-axis, only values, so it i
<script type="module">
import { hydrate } from "https://cdn.redocly.com/redoc/v3.1.0/bundles/redoc.standalone.js";
const __redoc_definition = {"openapi":"3.0.3","info":{"title":"Jungfraujoch","description":"API to control Jungfraujoch developed by the Paul Scherrer Institute (Switzerland).\nJungfraujoch is a data acquisition and analysis system for pixel array detectors, primarly PSI JUNGFRAU.\nJungfraujoch uses FPGA boards to acquire data at high data rates.\n\n# License Clarification\n\nWhile this API definition is licensed under GPL-3.0, **the GPL copyleft provisions do not apply**\nwhen this file is used solely to generate OpenAPI clients or when implementing applications that\ninteract with the API. Generated client code and applications using this API definition are not\nsubject to the GPL license requirements and may be distributed under terms of your choosing.\n\nThis exception is similar in spirit to the Linux Kernel's approach to userspace API headers and\nthe GCC Runtime Library Exception. The Linux Kernel developers have explicitly stated that\nuser programs that merely use the kernel interfaces (syscalls, ioctl definitions, etc.) are not\nderivative works of the kernel and are not subject to the terms of the GPL.\n\nThis exception is intended to allow wider use of this API specification without imposing GPL\nrequirements on applications that merely interact with the API, regardless of whether they\ncommunicate through network calls or other mechanisms.\n","version":"1.0.0-rc.187","contact":{"name":"Filip Leonarski (Paul Scherrer Institute)","email":"filip.leonarski@psi.ch"},"license":{"name":"GPL-3.0","url":"https://www.gnu.org/licenses/gpl-3.0.html"}},"servers":[{"url":"http://localhost:5232","description":"Test Jungfraujoch system"}],"components":{"securitySchemes":{"bearerAuth":{"type":"http","scheme":"bearer","description":"Per-dataset secret. `/start` may carry one or more equivalent `tokens`; while the current\ndataset has any, the operations marked with this scheme require `Authorization: Bearer \u003ctoken>`\nand answer 401 otherwise. A dataset started without `tokens` is not protected and the header is ignored.\n"}},"parameters":{"binning":{"in":"query","name":"binning","required":false,"schema":{"default":1,"type":"integer"},"description":"Binning of frames for the plot (0 = default binning)"},"experimental_coord":{"in":"query","name":"experimental_coord","required":false,"schema":{"type":"boolean","default":false},"description":"If measurement has goniometer axis defined, plot X-axis will represent rotation angle\nIf measurement has grid scan defined, plot X-axis and Y-axis will represent grid position, Z will be used as the final value\nFor still measurement the number is ignored\n"},"azint_unit":{"in":"query","name":"azint_unit","required":false,"schema":{"type":"string","enum":["Q_recipA","d_A","two_theta_deg"],"default":"Q_recipA"},"description":"Unit used for azim int.\n"},"fill_value":{"in":"query","name":"fill","required":false,"schema":{"type":"number","format":"float"},"description":"Fill value for elements that were missed during data collection\n"},"plot_type":{"in":"query","name":"type","required":true,"description":"Type of requested plot","schema":{"type":"string","enum":["bkg_estimate","spindle_blind_fraction","azint","azint_1d","spot_count","spot_count_low_res","spot_count_indexed","spot_count_ice","indexing_rate","indexing_lattice_count","indexing_unit_cell_length","indexing_unit_cell_angle","profile_radius","mosaicity","b_factor","error_pixels","saturated_pixels","image_collection_efficiency","receiver_delay","receiver_free_send_buf","strong_pixels","roi_sum","roi_mean","roi_max_count","roi_pixels","roi_weighted_x","roi_weighted_y","packets_received","max_pixel_value","resolution_estimate","pixel_sum","processing_time","beam_center_x","beam_center_y","integrated_reflections","image_scale_factor","image_scale_cc","compression_ratio","ice_ring_score"]}},"roi":{"in":"query","name":"roi","required":false,"schema":{"type":"string","minLength":1},"description":"Name of ROI for which plot is requested"},"image_id":{"in":"query","name":"id","required":false,"schema":{"type":"integer","format":"int64","default":-1,"minimum":-2},"description":"Image ID in the image buffer. Special values: -1 - last image in the buffer, -2: last indexed image in the buffer"},"saturation":{"in":"query","name":"saturation","required":false,"schema":{"type":"number","format":"float","minimum":-32767,"maximum":32767},"description":"Saturation value to set contrast in the preview image; if not provided, then autocontrast procedure is used"},"background":{"in":"query","name":"background","required":false,"schema":{"type":"number","format":"float","default":0,"minimum":-32767,"maximum":32767},"description":"Background value to set contrast in the preview image"},"show_spots":{"in":"query","name":"show_spots","schema":{"type":"boolean","default":true},"description":"Show spot finding results on the image"},"show_predictions":{"in":"query","name":"show_predictions","schema":{"type":"boolean","default":false}Line truncated
const __redoc_definition = {"openapi":"3.0.3","info":{"title":"Jungfraujoch","description":"API to control Jungfraujoch developed by the Paul Scherrer Institute (Switzerland).\nJungfraujoch is a data acquisition and analysis system for pixel array detectors, primarly PSI JUNGFRAU.\nJungfraujoch uses FPGA boards to acquire data at high data rates.\n\n# License Clarification\n\nWhile this API definition is licensed under GPL-3.0, **the GPL copyleft provisions do not apply**\nwhen this file is used solely to generate OpenAPI clients or when implementing applications that\ninteract with the API. Generated client code and applications using this API definition are not\nsubject to the GPL license requirements and may be distributed under terms of your choosing.\n\nThis exception is similar in spirit to the Linux Kernel's approach to userspace API headers and\nthe GCC Runtime Library Exception. The Linux Kernel developers have explicitly stated that\nuser programs that merely use the kernel interfaces (syscalls, ioctl definitions, etc.) are not\nderivative works of the kernel and are not subject to the terms of the GPL.\n\nThis exception is intended to allow wider use of this API specification without imposing GPL\nrequirements on applications that merely interact with the API, regardless of whether they\ncommunicate through network calls or other mechanisms.\n","version":"1.0.0-rc.187","contact":{"name":"Filip Leonarski (Paul Scherrer Institute)","email":"filip.leonarski@psi.ch"},"license":{"name":"GPL-3.0","url":"https://www.gnu.org/licenses/gpl-3.0.html"}},"servers":[{"url":"http://localhost:5232","description":"Test Jungfraujoch system"}],"components":{"securitySchemes":{"bearerAuth":{"type":"http","scheme":"bearer","description":"Per-dataset secret. `/start` may carry one or more equivalent `tokens`; while the current\ndataset has any, the operations marked with this scheme require `Authorization: Bearer \u003ctoken>`\nand answer 401 otherwise. A dataset started without `tokens` is not protected and the header is ignored.\n"}},"parameters":{"binning":{"in":"query","name":"binning","required":false,"schema":{"default":1,"type":"integer"},"description":"Binning of frames for the plot (0 = default binning)"},"experimental_coord":{"in":"query","name":"experimental_coord","required":false,"schema":{"type":"boolean","default":false},"description":"If measurement has goniometer axis defined, plot X-axis will represent rotation angle\nIf measurement has grid scan defined, plot X-axis and Y-axis will represent grid position, Z will be used as the final value\nFor still measurement the number is ignored\n"},"azint_unit":{"in":"query","name":"azint_unit","required":false,"schema":{"type":"string","enum":["Q_recipA","d_A","two_theta_deg"],"default":"Q_recipA"},"description":"Unit used for azim int.\n"},"fill_value":{"in":"query","name":"fill","required":false,"schema":{"type":"number","format":"float"},"description":"Fill value for elements that were missed during data collection\n"},"plot_type":{"in":"query","name":"type","required":true,"description":"Type of requested plot","schema":{"type":"string","enum":["bkg_estimate","spindle_blind_fraction","azint","azint_1d","spot_count","spot_count_low_res","spot_count_indexed","spot_count_ice","indexing_rate","indexing_lattice_count","indexing_unit_cell_length","indexing_unit_cell_angle","profile_radius","mosaicity","b_factor","error_pixels","saturated_pixels","image_collection_efficiency","receiver_delay","receiver_free_send_buf","strong_pixels","roi_sum","roi_mean","roi_max_count","roi_pixels","roi_weighted_x","roi_weighted_y","packets_received","max_pixel_value","resolution_estimate","pixel_sum","processing_time","beam_center_x","beam_center_y","integrated_reflections","image_scale_factor","image_scale_cc","compression_ratio","ice_ring_score","image_merged"]}},"roi":{"in":"query","name":"roi","required":false,"schema":{"type":"string","minLength":1},"description":"Name of ROI for which plot is requested"},"image_id":{"in":"query","name":"id","required":false,"schema":{"type":"integer","format":"int64","default":-1,"minimum":-2},"description":"Image ID in the image buffer. Special values: -1 - last image in the buffer, -2: last indexed image in the buffer"},"saturation":{"in":"query","name":"saturation","required":false,"schema":{"type":"number","format":"float","minimum":-32767,"maximum":32767},"description":"Saturation value to set contrast in the preview image; if not provided, then autocontrast procedure is used"},"background":{"in":"query","name":"background","required":false,"schema":{"type":"number","format":"float","default":0,"minimum":-32767,"maximum":32767},"description":"Background value to set contrast in the preview image"},"show_spots":{"in":"query","name":"show_spots","schema":{"type":"boolean","default":true},"description":"Show spot finding results on the image"},"show_predictions":{"in":"query","name":"show_predictions","schema":{"type":"boolean","Line truncated
const __redoc_options = {"skipBundle":true,"disableTelemetry":false};
hydrate(__redoc_definition, __redoc_options, document.getElementById('redoc'));
+3
View File
@@ -499,6 +499,9 @@ struct EndMessage {
std::vector<int32_t> spot_count_low_res;
std::vector<int32_t> spot_count_indexed;
std::vector<uint8_t> image_indexed;
// 1 where the image's observations reached the merged data, 0 where none did (MergeStatistics::image_merged);
// empty where no merge ran. Offline (rugnux) only.
std::vector<uint8_t> image_merged;
std::vector<int32_t> indexed_lattice_count;
std::vector<float> v_bkg_estimate;
// Per-image spindle_blind_fraction; NaN where the frame had no value. NaN is the CANNOT-SAY
+12
View File
@@ -88,6 +88,7 @@ void JFJochReceiverPlots::ClearVectors_i(size_t r) {
bkg_estimate.Clear(r);
spindle_blind_fraction.Clear(r);
ice_ring_score.Clear(r);
image_merged.Clear(r);
spot_count.Clear(r);
spot_count_low_res.Clear(r);
spot_count_indexed.Clear(r);
@@ -230,6 +231,11 @@ void JFJochReceiverPlots::Add(const DataMessage &msg, const AzimuthalIntegration
}
}
void JFJochReceiverPlots::SetImageMerged(const std::vector<uint8_t> &merged) {
for (size_t i = 0; i < merged.size(); ++i)
image_merged.AddElement(static_cast<uint32_t>(i), static_cast<float>(merged[i]));
}
void JFJochReceiverPlots::AddEmptyImage(const DataMessage &msg) {
image_collection_efficiency.AddElement(msg.number, msg.image_collection_efficiency);
}
@@ -301,6 +307,9 @@ MultiLinePlot JFJochReceiverPlots::GetPlots(const PlotRequest &request) {
case PlotType::IceRingScore:
ret = ice_ring_score.GetMeanPlot(nbins, start, incr, request.fill_value);
break;
case PlotType::ImageMerged:
ret = image_merged.GetMeanPlot(nbins, start, incr, request.fill_value);
break;
case PlotType::ResolutionEstimate:
ret = resolution_estimate.GetMeanPlot(nbins, start, incr, request.fill_value);
break;
@@ -608,6 +617,9 @@ void JFJochReceiverPlots::GetPlotRaw(std::vector<float> &v, PlotType type, const
case PlotType::IceRingScore:
v = ice_ring_score.ExportArray();
break;
case PlotType::ImageMerged:
v = image_merged.ExportArray();
break;
case PlotType::ResolutionEstimate:
v = resolution_estimate.ExportArray();
break;
+3
View File
@@ -73,6 +73,7 @@ class JFJochReceiverPlots {
StatusVector image_scale_factor;
StatusVector image_scale_cc;
StatusVector compression_ratio;
StatusVector image_merged; // known once the run's merge is made (SetImageMerged)
// StatusVector objects are fully thread-safe (protected by internal mutex)
// It is OK to have concurrent access to StatusVector
@@ -118,6 +119,8 @@ public:
void Clear();
void Add(const DataMessage& msg, const AzimuthalIntegrationProfile &profile);
// Per image, whether its observations reached the merge (MergeStatistics::image_merged).
void SetImageMerged(const std::vector<uint8_t> &merged);
void AddEmptyImage(const DataMessage& msg);
MultiLinePlot GetPlots(const PlotRequest& request);
+1 -1
View File
@@ -15,7 +15,7 @@ enum class PlotType {
ROISum, ROIMean, ROIMaxCount, ROIPixels, ROIWeightedX, ROIWeightedY, PacketsReceived, MaxValue,
ResolutionEstimate, ProfileRadius, Mosaicity, BFactor, PixelSum, StrongPixels,
RefinementBeamX, RefinementBeamY, ImageProcessingTime, IntegratedReflections,
ImageScaleFactor, ImageScaleCC, CompressionRatio, IndexingLatticeCount, IceRingScore
ImageScaleFactor, ImageScaleCC, CompressionRatio, IndexingLatticeCount, IceRingScore, ImageMerged
};
enum class PlotAzintUnit {
+1
View File
@@ -305,6 +305,7 @@ Every scalar of the image message is optional: a quantity that was not computed
| spot_count_low_res | Array(int32) | Per-image number of low-resolution spots | |
| spot_count_indexed | Array(int32) | Per-image number of spots fitting indexing solution | |
| image_indexed | Array(uint8) | Per-image indexing result; 0 = not indexed, nonzero = indexed | |
| image_merged | Array(uint8) | Per-image merge result; 1 = the image's observations reached the merged data, 0 = none did. Empty where no merge ran (rugnux) | |
| v_bkg_estimate | Array(float) | Per-image background estimate | |
| v_spindle_blind_fraction | Array(float) | Per-image spindle_blind_fraction; NaN where the frame had no value (which is "cannot say", not zero) | |
| ice_ring_score | Array(float) | Per-image strongest ice-ring intensity over the smooth radial background (1 = no ice) | |
+1
View File
@@ -391,6 +391,7 @@ in the data files only).
| `imageScaleMosaicity` | deg | scaling-model mosaicity |
| `sweepQuality` | | why this image's stretch of the sweep was flagged — see below |
| `frameDisposition` | | what became of this image's observations in the merged data — see below |
| `imageMerged` | | image's observations reached the merged data (0/1); written by rugnux beside the merge (stills: the crystal took part; rotation: the frame is not `rejected` in `frameDisposition`), absent where no merge ran (`--no-merge`) and in files written before it existed |
**Per-image lattices:** `latticeIndexed` `[n_images, 9]` (Å) — the real-space lattice (flattened
3×3); `latticeIndexedExtra` `[n_images, max_extra_lattices, 9]` (Å) — additional orientation
+2
View File
@@ -234,6 +234,8 @@ keyboard without them `Home` / `End` are `Fn`+`←` / `Fn`+`→` and `Page Up` /
| Dataset-info plot: `Shift` + hover | Load the hovered image |
| Spot / reflection list: double click | Zoom the diffraction image on that spot or prediction |
| Image list: double click | Load that image |
| Image list: click a column header | Sort by that column (Index and Merged sort Yes, No, N/A) |
| Image list: right click the Index or Merged header | Filter the rows by that column - tick Yes, No or N/A; a filtered header is marked `*`, and the two filters combine, so Index Yes with Merged No lists the images indexed but not merged. Merged is read from `/entry/MX/imageMerged` of a rugnux `_process.h5` and is N/A for a file without it |
## Opening data
+3 -1
View File
@@ -289,7 +289,9 @@ different space group, resolution limit, anomalous setting or outlier rejection.
`-o/-N/-s/-e/-S/-A/-z/--scaling-*` options as the full run, and (unlike the full pipeline) does
not run a space-group search: it merges in the space group and unit cell the file records, and `-S`
/ `-C` override them. A `_process.h5` written before the group was stored carries none, and merges
in P1 unless `-S` says otherwise.
in P1 unless `-S` says otherwise. Beside the reflection files and the report it writes
`<prefix>_plot.txt` for its own merge - per image, the scale and whether the image's observations are
in the merged data (the report's `IMAGES_MERGED` is the sum of that column).
A reference MTZ (`-z`) is accepted here on **stills** data, where it fixes the space group and cell,
reports CC<sub>ref</sub> and hands over its R-free flags; on rotation data the rotation scaler
+5 -1
View File
@@ -243,11 +243,13 @@ The keys most scripts read, by section; the rest of this page explains the ones
| `OSCILLATION_RANGE`, `STARTING_ANGLE`, `ROTATION_AXIS` | Rotation only: the per-image rotation the run integrated at (deg: the file's, scaled where the run adopted a goniometer rotation scale), the first image's angle and the axis |
| `DETECTOR_DISTANCE`, `BEAM_CENTRE`, `DETECTOR_TILT` | The geometry the run integrated at (mm; the point of normal incidence in pixels; the three PONI rotations in degrees) - on a rotation run the post-refined one, not the header's |
| `DIRECT_BEAM` | Where the direct beam meets the detector under that tilt (pixels) |
| `IMAGES_INDEXED`, `LATTICES_INDEXED`, `IMAGES_INTEGRATED` | Section 2: the images the per-image indexer indexed (the count behind `INDEXING_RATE`), the lattices it indexed on them - an image of two crystals counts two - and the images integrated. An image is integrated on its main lattice alone: an extra lattice is indexed and its spots assigned, but it is not integrated, so `IMAGES_INTEGRATED` is also the number of integrated lattices. As `INDEXING_RATE`, the first two are a stills measure and are written on rotation only with `--developer`; `--mode scale` writes `IMAGES_INDEXED` from the `imageIndexed` its input files record (absent where one of them does not), and `IMAGES_INTEGRATED` as the images with integrated reflections in them. `IMAGES_PROCESSED` ≥ `IMAGES_INDEXED`, `LATTICES_INDEXED` ≥ `IMAGES_INDEXED` |
| `LATTICE_FOUND`, `UNIT_CELL_CONSTANTS`, `CELL_ESD` | Section 2: whether a lattice was found, the cell (a b c α β γ, Å and degrees) and its e.s.d.s ([below](#cell-esds)) |
| `CELL_FORMS`, `OTHER_CELL_FORM_*` | Stills: whether the frames on the run's cell were one crystal form or two, and the other one, left out of the merge ([below](#two-crystal-forms-stills)) |
| `SPACE_GROUP_NAME`, `SPACE_GROUP_NUMBER` | The space group the files are written in ([below](#the-space-group-and-the-sohncke-answer-beside-it)) |
| `MOSAICITY_DEG` | Rotation only ([below](#mosaicity)) |
| `MERGE` | Section 3: `PERFORMED` or `NOT_PERFORMED` (`--no-merge`) |
| `IMAGES_MERGED` | Section 3: the images whose observations are in the merged data - on stills the crystals (patterns) that went into the merge, past the cell form, the scale and `--min-image-cc`; on rotation the frames not rejected (`FRAMES_MERGED` + `FRAMES_DOWNGRADED` where the sweep-quality diagnostic ran). The sum of the `merged` column of `<prefix>_plot.txt` and of `/entry/MX/imageMerged` in the `_process.h5`; written by `--mode scale` too |
| `INCLUDE_RESOLUTION_RANGE` | `dmax dmin` of the written reflections (Å), as XDS's keyword |
| `FITTED_RESOLUTION` | Where the fall-off of the standard CC1/2 (the `CC_HALF` of the shell table) crosses the `--resolution-cc-target` - the number to quote; absent where the curve is not a fall-off (a `RESOLUTION_FIT` warning says why) |
| `FITTED_RESOLUTION_INTERNAL` | Rotation only, internal: the crossing of the automatic cut's own fit, which weights each reflection by the scale of the frames it was measured on so that a weak stretch of the sweep does not drag it in. The written range (`INCLUDE_RESOLUTION_RANGE`) is cut from this one; on a sweep without a weak stretch it equals `FITTED_RESOLUTION` |
@@ -297,7 +299,9 @@ hundred lines.
deliberately low: a warning is a prompt to check something, not a verdict, and some of them fire on
data that turn out fine — the closed
type for machinery, the open sentence for a person.
- Below them, one line each, the facts a reader needs before reading further: space group, cell,
- Below them, one line each, the facts a reader needs before reading further: the images (`Images`:
processed, indexed with the lattices on them - stills only -, integrated and merged, the counts of the
keys above), space group, cell,
mosaicity, the beam centre, the powder and ice rings, overloads, overlapping spots, multiple lattices, resolution, completeness, signal, anomalous
signal, anisotropy, pseudo-symmetry, twinning, the indexing ambiguity of serial stills (`Indexing ambiguity`:
how the data resolved it, in the terms of the `AMBIGUITY_*` keys below, or `UNRESOLVED` where the hands are
+5 -3
View File
@@ -379,12 +379,14 @@ Absorption is corrected only by the empirical surface a rotation run fits and ke
| `sigma_M_deg` | Mosaicity as the rocking-curve **standard deviation** σ<sub>M</sub> in degrees — not a FWHM, and not XDS's `REFLECTING_RANGE`. See the note below. |
| `cc_to_merge` | Correlation of the image's own observations with the merged data. |
| `cc_n` | Observations that correlation was computed over; 0 where none was. |
| `merged` | 1 where the image's observations are in the merged data, 0 where they are not — an image that never indexed, one an earlier guard dropped, and one ΔCC<sub>1/2</sub> convicted all read 0. `nan` on a run that merged nothing, `--no-merge` included, where the question has no answer. |
| `disposition` | The same answer graded, to plot directly: 1 merged, 0.5 downgraded (merged, but over a degraded stretch of the sweep), 0 rejected. 0.5 is a marker, not a weight — the merge applies no factor of its own to a downgraded frame, which is carried at the weight its own scale and sigmas give it. `nan` for an image that never got a scale and so never reached the merge, and on a run that merged nothing. |
| `merged` | 1 where the image's observations are in the merged data, 0 where they are not — an image that never indexed, one an earlier guard dropped (on stills: another crystal form, a collapsed scale, `--min-image-cc`), and one ΔCC<sub>1/2</sub> convicted all read 0. An image of several lattices is integrated, and so merged, on its main lattice alone, so the column is 0 or 1 and its sum is the report's `IMAGES_MERGED`; the same vector is `/entry/MX/imageMerged` of the `_process.h5`. `nan` on a run that merged nothing, `--no-merge` included, where the question has no answer. |
| `disposition` | The same answer graded, to plot directly: 1 merged, 0.5 downgraded (merged, but over a degraded stretch of the sweep), 0 rejected. 0.5 is a marker, not a weight — the merge applies no factor of its own to a downgraded frame, which is carried at the weight its own scale and sigmas give it. `nan` for an image that never got a scale and so never reached the merge, and on a run that merged nothing. On stills, with no sweep to grade, 1 merged and 0 scaled but left out. |
| `indexed` | 1 where the per-image indexer indexed the image, 0 where it did not (the `imageIndexed` of the HDF5 file; its sum is the report's `IMAGES_INDEXED`). `--mode scale` takes it from the `imageIndexed` of its input files, `nan` for a file without it. |
A quantity nothing measured for an image is written `nan`, which gnuplot and numpy both skip; the
columns never shift and a row is never left out. A stills run writes the table too, with `nan` in
the columns a sweep would fill; `--no-merge` writes it with the scaling columns `nan`. A joint merge of
the columns a sweep would fill; `--no-merge` writes it with the scaling columns `nan`; `--mode scale`
writes it for its own merge, with the per-image analysis columns (`bkg`, `resolution_A`, `spots`) `nan`. A joint merge of
sweeps writes one row per frame of the sweeps it merged, sweep after sweep in input order:
each row's per-image analysis is its own sweep's, the scaling columns the joint
merge's; a sweep left out of the merge has no rows.
@@ -1791,6 +1791,8 @@ namespace {
GetCBORInt32Array(value, message.spot_count_indexed);
else if (key == "image_indexed")
GetCBORUInt8Array(value, message.image_indexed);
else if (key == "image_merged")
GetCBORUInt8Array(value, message.image_merged);
else if (key == "v_bkg_estimate")
GetCBORFloatArray(value, message.v_bkg_estimate);
else if (key == "v_spindle_blind_fraction")
@@ -987,6 +987,7 @@ void CBORStream2Serializer::SerializeSequenceEnd(const EndMessage& message) {
CBOR_ENC(mapEncoder, "spot_count_low_res", message.spot_count_low_res);
CBOR_ENC(mapEncoder, "spot_count_indexed", message.spot_count_indexed);
CBOR_ENC(mapEncoder, "image_indexed", message.image_indexed);
CBOR_ENC(mapEncoder, "image_merged", message.image_merged);
CBOR_ENC(mapEncoder, "v_bkg_estimate", message.v_bkg_estimate);
CBOR_ENC(mapEncoder, "v_spindle_blind_fraction", message.v_spindle_blind_fraction);
CBOR_ENC(mapEncoder, "ice_ring_score", message.ice_ring_score);
+4 -3
View File
@@ -3026,7 +3026,8 @@ export const plot_type = {
IMAGE_SCALE_FACTOR: 'image_scale_factor',
IMAGE_SCALE_CC: 'image_scale_cc',
COMPRESSION_RATIO: 'compression_ratio',
ICE_RING_SCORE: 'ice_ring_score'
ICE_RING_SCORE: 'ice_ring_score',
IMAGE_MERGED: 'image_merged'
} as const;
/**
@@ -4569,7 +4570,7 @@ export type getPreviewPlotData = {
/**
* Type of requested plot
*/
type: 'bkg_estimate' | 'spindle_blind_fraction' | 'azint' | 'azint_1d' | 'spot_count' | 'spot_count_low_res' | 'spot_count_indexed' | 'spot_count_ice' | 'indexing_rate' | 'indexing_lattice_count' | 'indexing_unit_cell_length' | 'indexing_unit_cell_angle' | 'profile_radius' | 'mosaicity' | 'b_factor' | 'error_pixels' | 'saturated_pixels' | 'image_collection_efficiency' | 'receiver_delay' | 'receiver_free_send_buf' | 'strong_pixels' | 'roi_sum' | 'roi_mean' | 'roi_max_count' | 'roi_pixels' | 'roi_weighted_x' | 'roi_weighted_y' | 'packets_received' | 'max_pixel_value' | 'resolution_estimate' | 'pixel_sum' | 'processing_time' | 'beam_center_x' | 'beam_center_y' | 'integrated_reflections' | 'image_scale_factor' | 'image_scale_cc' | 'compression_ratio' | 'ice_ring_score';
type: 'bkg_estimate' | 'spindle_blind_fraction' | 'azint' | 'azint_1d' | 'spot_count' | 'spot_count_low_res' | 'spot_count_indexed' | 'spot_count_ice' | 'indexing_rate' | 'indexing_lattice_count' | 'indexing_unit_cell_length' | 'indexing_unit_cell_angle' | 'profile_radius' | 'mosaicity' | 'b_factor' | 'error_pixels' | 'saturated_pixels' | 'image_collection_efficiency' | 'receiver_delay' | 'receiver_free_send_buf' | 'strong_pixels' | 'roi_sum' | 'roi_mean' | 'roi_max_count' | 'roi_pixels' | 'roi_weighted_x' | 'roi_weighted_y' | 'packets_received' | 'max_pixel_value' | 'resolution_estimate' | 'pixel_sum' | 'processing_time' | 'beam_center_x' | 'beam_center_y' | 'integrated_reflections' | 'image_scale_factor' | 'image_scale_cc' | 'compression_ratio' | 'ice_ring_score' | 'image_merged';
/**
* Fill value for elements that were missed during data collection
*
@@ -4620,7 +4621,7 @@ export type getPreviewPlotBinData = {
/**
* Type of requested plot
*/
type: 'bkg_estimate' | 'spindle_blind_fraction' | 'azint' | 'azint_1d' | 'spot_count' | 'spot_count_low_res' | 'spot_count_indexed' | 'spot_count_ice' | 'indexing_rate' | 'indexing_lattice_count' | 'indexing_unit_cell_length' | 'indexing_unit_cell_angle' | 'profile_radius' | 'mosaicity' | 'b_factor' | 'error_pixels' | 'saturated_pixels' | 'image_collection_efficiency' | 'receiver_delay' | 'receiver_free_send_buf' | 'strong_pixels' | 'roi_sum' | 'roi_mean' | 'roi_max_count' | 'roi_pixels' | 'roi_weighted_x' | 'roi_weighted_y' | 'packets_received' | 'max_pixel_value' | 'resolution_estimate' | 'pixel_sum' | 'processing_time' | 'beam_center_x' | 'beam_center_y' | 'integrated_reflections' | 'image_scale_factor' | 'image_scale_cc' | 'compression_ratio' | 'ice_ring_score';
type: 'bkg_estimate' | 'spindle_blind_fraction' | 'azint' | 'azint_1d' | 'spot_count' | 'spot_count_low_res' | 'spot_count_indexed' | 'spot_count_ice' | 'indexing_rate' | 'indexing_lattice_count' | 'indexing_unit_cell_length' | 'indexing_unit_cell_angle' | 'profile_radius' | 'mosaicity' | 'b_factor' | 'error_pixels' | 'saturated_pixels' | 'image_collection_efficiency' | 'receiver_delay' | 'receiver_free_send_buf' | 'strong_pixels' | 'roi_sum' | 'roi_mean' | 'roi_max_count' | 'roi_pixels' | 'roi_weighted_x' | 'roi_weighted_y' | 'packets_received' | 'max_pixel_value' | 'resolution_estimate' | 'pixel_sum' | 'processing_time' | 'beam_center_x' | 'beam_center_y' | 'integrated_reflections' | 'image_scale_factor' | 'image_scale_cc' | 'compression_ratio' | 'ice_ring_score' | 'image_merged';
/**
* Name of ROI for which plot is requested
*/
+6 -3
View File
@@ -1330,7 +1330,8 @@ export const zPlotType = z.enum([
'image_scale_factor',
'image_scale_cc',
'compression_ratio',
'ice_ring_score'
'ice_ring_score',
'image_merged'
]);
/**
@@ -1677,7 +1678,8 @@ export const zGetPreviewPlotQuery = z.object({
'image_scale_factor',
'image_scale_cc',
'compression_ratio',
'ice_ring_score'
'ice_ring_score',
'image_merged'
]),
fill: z.number().optional(),
experimental_coord: z.boolean().optional().default(false),
@@ -1733,7 +1735,8 @@ export const zGetPreviewPlotBinQuery = z.object({
'image_scale_factor',
'image_scale_cc',
'compression_ratio',
'ice_ring_score'
'ice_ring_score',
'image_merged'
]),
roi: z.string().min(1).optional()
});
@@ -70,6 +70,7 @@ function AxisTypeY(plot: plot_type) : string | ReactNode {
case plot_type.ICE_RING_SCORE:
return "Ratio";
case plot_type.SPINDLE_BLIND_FRACTION:
case plot_type.IMAGE_MERGED:
return "Fraction";
case plot_type.AZINT:
case plot_type.AZINT_1D:
@@ -76,6 +76,7 @@ function DataProcessingPlots({type: initialType, height}: MyProps) {
<MenuItem value={plot_type.BEAM_CENTER_X}>Beam center X (post-indexing geometry refinement)</MenuItem>
<MenuItem value={plot_type.BEAM_CENTER_Y}>Beam center Y (post-indexing geometry refinement)</MenuItem>
<MenuItem value={plot_type.INDEXING_LATTICE_COUNT}>Indexed lattice count</MenuItem>
<MenuItem value={plot_type.IMAGE_MERGED}>Merged (images in the merged data)</MenuItem>
<MenuItem value={plot_type.INTEGRATED_REFLECTIONS}>Integrated reflections</MenuItem>
<MenuItem value={plot_type.IMAGE_COLLECTION_EFFICIENCY}>Image collection efficiency</MenuItem>
<MenuItem value={plot_type.COMPRESSION_RATIO}>Compression ratio</MenuItem>
+2
View File
@@ -232,6 +232,7 @@ void MergeOnTheFly::AddImages(const std::vector<IntegrationOutcome> &outcomes) {
std::unique_lock ul(merged_mutex);
std::vector<size_t> rejected(outcomes.size(), 0);
images_merged.assign(outcomes.size(), 0);
ReduceImagesInOrder<Contribution>(
outcomes.size(), nthreads,
[&](size_t i, std::vector<Contribution> &out) {
@@ -240,6 +241,7 @@ void MergeOnTheFly::AddImages(const std::vector<IntegrationOutcome> &outcomes) {
},
[&](size_t i, const std::vector<Contribution> &out) {
reject_count += rejected[i];
images_merged[i] = !out.empty();
Accumulate(out, static_cast<int64_t>(i));
});
}
+7
View File
@@ -246,6 +246,10 @@ struct MergeStatistics {
// (MeasureSweepQuality), and what became of every frame of the sweep (MeasureBatchDeltaCCHalf).
SweepQuality sweep_quality;
// Per processed image: 1 where its observations reached the merge, 0 where none did - on stills the
// crystal took part, on rotation the frame is not rejected (FrameDisposition). Empty where no merge ran.
std::vector<uint8_t> image_merged;
// Diffraction anisotropy (AnalyzeAnisotropy): the anisotropy tensor, the diffraction limit along
// each of its principal directions, and whether either is established above this dataset's own
// systematic error. Report-only - nothing is corrected and no reflection is removed. Empty
@@ -423,6 +427,7 @@ class MergeOnTheFly {
// ln(I / median); the stills merge sets both to n b, the linear band.
double reject_up = 0.0, reject_down = 0.0;
size_t reject_count = 0;
std::vector<uint8_t> images_merged; // per outcome of AddImages: 1 where an observation of it reached the merge
// One accepted observation, reduced to what the accumulator adds for it.
struct Contribution {
@@ -483,6 +488,8 @@ public:
// AddImage for every outcome, the image id being its index: the same merge, bit for bit, with the
// per-observation work spread over the workers and the accumulation kept in image order.
void AddImages(const std::vector<IntegrationOutcome> &outcomes);
// Per outcome of the last AddImages: 1 where at least one of its observations went into the merge.
[[nodiscard]] const std::vector<uint8_t> &ImagesMerged() const { return images_merged; }
// Every test AddImages applies, each evaluated on its own, for every reflection of every outcome (in
// their order); call after AddImages. A diagnostic: the merge itself does not use it.
@@ -8194,6 +8194,14 @@ RotationScaleMerge::Result RotationScaleMerge::RunArm(bool scale_partials, bool
TimingMark merge_mark(*logger, "rsm merge and statistics");
auto r = MergeAndStats(n_groups, for_search, combined_on_gpu && scaled_fulls_on_gpu, full_stats);
merge_mark.Stop();
// A frame is in the merge as the disposition ledger counts it (CountSweepDisposition); where the ledger
// did not run, as the per-frame guards left it.
const auto &disposition = r.statistics.sweep_quality.frame_disposition;
r.statistics.image_merged.assign(n_frames, 0);
for (int f = 0; f < n_frames; ++f)
r.statistics.image_merged[f] = disposition.empty()
? frame_in_merge[f]
: disposition[f] != static_cast<uint8_t>(FrameDisposition::Rejected);
r.scaling_iterations_partials = partial_loop.iterations;
r.scaling_iterations_fulls = fulls_loop.iterations;
r.scaling_converged = partial_loop.converged && (fulls_loop.iterations == 0 || fulls_loop.converged);
+1
View File
@@ -599,6 +599,7 @@ HDF5MetadataSource::OpenResult HDF5MetadataSource::Open(const std::string &filen
dataset->spot_count_ice_control = master_file->ReadOptVector<float>("/entry/MX/peakCountIceRingControl");
dataset->indexing_result = master_file->ReadOptVector<float>("/entry/MX/imageIndexed");
dataset->merge_result = master_file->ReadOptVector<float>("/entry/MX/imageMerged");
dataset->bkg_estimate = master_file->ReadOptVector<float>("/entry/MX/bkgEstimate");
dataset->spindle_blind_fraction = master_file->ReadOptVector<float>("/entry/MX/spindleBlindFraction");
dataset->ice_ring_score = master_file->ReadOptVector<float>("/entry/MX/iceRingScore");
+1
View File
@@ -98,6 +98,7 @@ JFJochMultiReader::JFJochMultiReader(std::vector<std::unique_ptr<JFJochReader>>
dataset->spot_count_ice_rings.clear();
dataset->spot_count_ice_control.clear();
dataset->indexing_result.clear();
dataset->merge_result.clear();
dataset->indexing_lattice_count.clear();
dataset->bkg_estimate.clear();
dataset->pixel_sum.clear();
+1
View File
@@ -62,6 +62,7 @@ struct JFJochReaderDataset {
std::vector<float> spot_count_ice_control;
std::vector<float> indexing_result;
std::vector<float> merge_result; // /entry/MX/imageMerged: 1 merged, 0 not; empty in a file without it
std::vector<float> indexing_lattice_count;
std::vector<float> bkg_estimate;
// The XFEL pulse ID and event code of each image (/entry/xfel), where the file records them.
@@ -329,6 +329,8 @@ void JFJochReceiverRugnuxRotation::RugnuxThread() {
result.model_validation ? &*result.model_validation : nullptr,
result.refinement ? &*result.refinement : nullptr);
live_status.Frames(result.end_message.image_scale_factor, LiveFrameRanges(result.merge_statistics.sweep_quality));
plots.SetImageMerged(result.merge_statistics.image_merged);
image_buffer.ResultsChanged();
} else
live_status.Headline(result.space_group ? result.space_group->xhm() : "", std::nullopt, std::nullopt);
// For the broker's viewer routes; a copy of the map coefficients, as the end message takes the originals
@@ -426,6 +428,7 @@ void JFJochReceiverRugnuxRotation::AddToEndMessage(EndMessage &message) {
message.refined_poni_rot2 = rugnux_end.refined_poni_rot2;
message.refined_poni_rot3 = rugnux_end.refined_poni_rot3;
message.refined_rotation_axis = rugnux_end.refined_rotation_axis;
message.image_merged = rugnux_end.image_merged;
message.report = rugnux_report;
// For the writer to write beside the master; moved, the result is not read again
if (!rugnux_result->outputs)
+2
View File
@@ -134,6 +134,8 @@ void JFJochReceiverRugnuxStills::FinishCollection() {
std::exception_ptr error;
try {
stills->Finish();
if (const auto &merged = stills->GetMerged())
plots.SetImageMerged(merged->merge.statistics.image_merged);
} catch (const std::exception &) {
error = std::current_exception();
}
+2
View File
@@ -631,6 +631,8 @@ void StillsGPUAnalysis::FillEndMessage(EndMessage &message) const {
message.refined_beam_center_x = GetBeamX_pxl();
message.refined_beam_center_y = GetBeamY_pxl();
message.refined_detector_distance_mm = GetDetectorDistance_mm();
if (const auto &merge_output = GetMerged())
message.image_merged = merge_output->merge.statistics.image_merged;
// In the model's frame where a model was placed against them
if (const auto &model_step = GetModel()) {
DiffractionExperiment model_experiment(GetMergeExperiment());
+17 -21
View File
@@ -110,7 +110,7 @@ std::string RenderDetectorDiagnostic(const DiffractionExperiment &experiment,
}
void FillJointPlotRows(PerImagePlot &joint, size_t first_row, int dataset, const PerImagePlot &own) {
for (auto *v : {&joint.angle_deg, &joint.bkg_estimate, &joint.resolution_A, &joint.spot_count})
for (auto *v : {&joint.angle_deg, &joint.bkg_estimate, &joint.resolution_A, &joint.spot_count, &joint.indexed})
v->resize(joint.image_count, NAN);
for (size_t f = 0; f < own.image_count; f++) {
const size_t row = first_row + f;
@@ -120,6 +120,7 @@ void FillJointPlotRows(PerImagePlot &joint, size_t first_row, int dataset, const
joint.bkg_estimate[row] = Element(own.bkg_estimate, f);
joint.resolution_A[row] = Element(own.resolution_A, f);
joint.spot_count[row] = Element(own.spot_count, f);
joint.indexed[row] = Element(own.indexed, f);
}
}
@@ -141,30 +142,23 @@ std::string PerImagePlotText(const PerImagePlot &plot) {
"# cc_to_merge: correlation of the image's observations with the merged data\n"
"# cc_n: observations that correlation was taken over\n"
"# merged: 1 where the image's observations are in the merged data, 0 where not\n"
"# disposition: 1 merged, 0.5 downgraded, 0 rejected\n";
"# disposition: 1 merged, 0.5 downgraded, 0 rejected\n"
"# indexed: 1 where the per-image indexer indexed the image, 0 where not\n";
for (size_t k = 0; k < plot.dataset_files.size(); k++)
file += fmt::format("# dataset {}: {}\n", k + 1, plot.dataset_files[k]);
file += "#" + fmt::format("{:>7} {:>7} {:>7} {:>10} {:>12} {:>12} {:>8} {:>12} {:>13} {:>12} {:>8}"
" {:>7} {:>11}\n",
" {:>7} {:>11} {:>7}\n",
"ordinal", "dataset", "image", "angle_deg", "bkg", "resolution_A", "spots",
"scale_G", "sigma_M_deg", "cc_to_merge", "cc_n", "merged", "disposition");
"scale_G", "sigma_M_deg", "cc_to_merge", "cc_n", "merged", "disposition", "indexed");
// Whether "merged" has anything to answer at all. With no merge behind the run - --no-merge, or
// a sweep nothing indexed on - a column of zeros would read as a sweep every frame of which was
// thrown out, which is a different statement from "no merge was made".
const bool merge_ran = !plot.frame_disposition.empty()
|| std::any_of(plot.scale_g.begin(), plot.scale_g.end(),
[](float g) { return std::isfinite(g); });
// With no merge behind the run - --no-merge, or a sweep nothing indexed on - "merged" has nothing to
// answer: a column of zeros would read as a run every image of which was thrown out, which is a
// different statement from "no merge was made".
const bool merge_ran = !plot.merged.empty();
for (size_t i = 0; i < plot.image_count; i++) {
const float scale_g = Element(plot.scale_g, i);
// Did this image's observations reach the merged data? The sweep-quality diagnostic says so
// outright; where it did not run, an image with no fitted scale contributed nothing, which
// is the same statement by a weaker route.
const int merged = i < plot.frame_disposition.size()
? plot.frame_disposition[i]
!= static_cast<uint8_t>(FrameDisposition::Rejected)
: std::isfinite(scale_g);
const bool merged = merge_ran && i < plot.merged.size() && plot.merged[i];
// The same answer graded, to plot directly: 1 merged, 0.5 downgraded, 0 rejected. 0.5 marks
// "downgraded" and is not a weight - the merge applies no factor of its own to those frames,
// they are carried at whatever weight their scale and sigmas give them. A frame that never got
@@ -176,15 +170,17 @@ std::string PerImagePlotText(const PerImagePlot &plot) {
case FrameDisposition::Downgraded: disposition = 0.5f; break;
case FrameDisposition::Rejected: disposition = std::isfinite(scale_g) ? 0.0f : NAN; break;
}
} else if (merge_ran && std::isfinite(scale_g))
disposition = 1.0f;
} else if (merge_ran)
disposition = merged ? 1.0f : std::isfinite(scale_g) ? 0.0f : NAN;
const float indexed = Element(plot.indexed, i);
file += fmt::format("{:>8} {:>7} {:>7} {:>10.4f} {:>12.4g} {:>12.4g} {:>8.0f} {:>12.5g} {:>13.4f}"
" {:>12.4f} {:>8} {:>7} {:>11}\n",
" {:>12.4f} {:>8} {:>7} {:>11} {:>7}\n",
i, plot.dataset[i], plot.image[i], Element(plot.angle_deg, i), Element(plot.bkg_estimate, i), Element(plot.resolution_A, i),
Element(plot.spot_count, i), scale_g, Element(plot.mosaicity_deg, i),
Element(plot.cc_to_merge, i), Element(plot.cc_n, i),
merge_ran ? std::to_string(merged) : "nan", disposition);
merge_ran ? std::to_string(static_cast<int>(merged)) : "nan", disposition,
std::isfinite(indexed) ? std::to_string(static_cast<int>(indexed > 0.0f)) : "nan");
}
return file;
}
+4 -1
View File
@@ -32,7 +32,8 @@ std::string RenderDetectorDiagnostic(const DiffractionExperiment &experiment,
// dataset and image say which input (from 1, in the order given) and which of its own images a row is,
// so a run made of several files - stills masters read as one, a joint merge of sweeps - plots per file.
// frame_disposition holds FrameDisposition (Merge.h) and is empty when the sweep-quality diagnostic
// did not run.
// did not run; merged (MergeStatistics::image_merged) is empty when no merge ran, indexed when the run
// indexed no image itself (--mode scale).
struct PerImagePlot {
std::vector<std::string> dataset_files; // the file of dataset k + 1, for the header
std::vector<int> dataset;
@@ -46,6 +47,8 @@ struct PerImagePlot {
std::vector<float> cc_to_merge;
std::vector<int> cc_n;
std::vector<uint8_t> frame_disposition;
std::vector<uint8_t> merged;
std::vector<float> indexed;
size_t image_count = 0; // the rows
};
+25
View File
@@ -1330,6 +1330,14 @@ ReportDocument BuildReportDocument(const std::string &output_prefix,
// 0 while the whole sweep merges.
if (result.indexing_rate.has_value())
Add(s, KeyReal("INDEXING_RATE", *result.indexing_rate, "{:.4f}", rotation));
// The same as counts: the images the per-image indexer indexed and the lattices it indexed on them
// (a stills measure, as the rate), and the images integrated - each on its main lattice alone.
if (result.images_indexed)
Add(s, KeyInt("IMAGES_INDEXED", *result.images_indexed, rotation));
if (result.lattices_indexed)
Add(s, KeyInt("LATTICES_INDEXED", *result.lattices_indexed, rotation));
if (result.images_integrated)
Add(s, KeyInt("IMAGES_INTEGRATED", *result.images_integrated));
Add(s, KeyBool("LATTICE_FOUND", result.consensus_cell.has_value()));
if (!result.indexing_ambiguity_operators.empty())
Add(s, KeyText("INDEXING_AMBIGUITY_OPERATORS", result.indexing_ambiguity_operators));
@@ -1737,6 +1745,9 @@ ReportDocument BuildReportDocument(const std::string &output_prefix,
const double cc_target = experiment.GetScalingSettings().GetResolutionCCTarget();
Add(s, KeyEnum("MERGE", "PERFORMED", {"PERFORMED", "NOT_PERFORMED"}));
// The images whose observations are in the merged data - the merged column of <prefix>_plot.txt.
if (!ms.image_merged.empty())
Add(s, KeyInt("IMAGES_MERGED", std::ranges::count(ms.image_merged, uint8_t{1})));
Add(s, KeyText("INCLUDE_RESOLUTION_RANGE", fmt::format("{:.3f} {:.3f}", o.d_max, o.d_min)));
// FITTED_RESOLUTION is presented by this report as "the number to quote", so it must not
@@ -3819,6 +3830,20 @@ ReportDocument BuildReportDocument(const std::string &output_prefix,
const auto untested = [&](const char *label, std::vector<std::string> codes) {
doc.checks.push_back({label, std::move(codes), ReportCheck::Status::Undetermined, "not measured", {}});
};
{
// How many images made it through each stage, as counted in sections 1-3.
std::string images = fmt::format("{} processed", result.images_processed);
if (result.images_indexed && !rotation)
images += fmt::format(", {} indexed", *result.images_indexed);
if (result.lattices_indexed && !rotation)
images += fmt::format(" ({} lattices)", *result.lattices_indexed);
if (result.images_integrated)
images += fmt::format(", {} integrated", *result.images_integrated);
if (result.has_merge_statistics && !result.merge_statistics.image_merged.empty())
images += fmt::format(", {} merged",
std::ranges::count(result.merge_statistics.image_merged, uint8_t{1}));
row("Images", images);
}
if (result.space_group)
row("Space group", group);
if (result.consensus_cell)
+10 -1
View File
@@ -1478,8 +1478,12 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
plot.cc_to_merge = scaling.image_cc;
plot.cc_n = scaling.image_cc_n;
}
if (result.has_merge_statistics)
if (result.has_merge_statistics) {
plot.frame_disposition = result.merge_statistics.sweep_quality.frame_disposition;
plot.merged = result.merge_statistics.image_merged;
}
if (!joint)
plot.indexed = result.image_indexed;
// One input as far as this run knows: the caller renumbers a run of several (rugnux_cli.cpp).
// The goniometer is already shifted to the processed range, so ordinal i is at its angle i.
const auto goniometer = experiment_.GetGoniometer();
@@ -1503,6 +1507,11 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
end_msg.image_scale_factor = scaling.image_scale_g;
end_msg.image_scale_cc = scaling.image_cc;
end_msg.image_scale_mosaicity = scaling.mosaicity_deg;
if (result.has_merge_statistics) {
const auto &merged = result.merge_statistics.image_merged;
end_msg.image_merged.assign(merged.begin(), merged.begin() + std::min<int64_t>(end_msg.max_image_number,
merged.size()));
}
}
if (result.cell_esd) {
+5 -1
View File
@@ -595,7 +595,7 @@ struct RugnuxOutputs {
};
// One of several _process.h5 files --mode scale merged as one: its frames, those of them indexed, and those of
// them that went into the merge (MergedCrystals).
// them that went into the merge (MergeStatistics::image_merged).
struct ScaleInputCount {
std::string file;
int64_t images = 0, indexed = 0, merged = 0;
@@ -640,6 +640,10 @@ struct ProcessResult {
std::optional<float> indexing_rate;
// Per image of this pass, by ordinal (image start + ordinal * stride): whether it indexed (> 0).
std::vector<float> image_indexed;
// This pass's images counted (absent where the pass did not count them): those the per-image indexer
// indexed, the lattices it indexed on them (an image of two crystals counts two), and those integrated -
// on the main lattice alone: an extra lattice of an image is indexed, not integrated.
std::optional<int64_t> images_indexed, lattices_indexed, images_integrated;
// How far the merged data are expected to reach, predicted from spot finding alone (the median of
// the per-image estimates, see SpotUtils.h GetResolution). Independent of the merge, so it can be
// read against INCLUDE_RESOLUTION_RANGE - or on a run that never merges.
+12
View File
@@ -772,6 +772,18 @@ void Rugnux::ImagePass(PipelineLocals &p) {
result.mean_processing_time = plots.GetMeanProcessingTime();
result.indexing_rate = plots.GetIndexingRate();
plots.GetPlotRaw(result.image_indexed, PlotType::IndexingRate, "");
{
std::vector<float> lattices;
plots.GetPlotRaw(lattices, PlotType::IndexingLatticeCount, "");
result.images_indexed = std::ranges::count_if(result.image_indexed, [](float v) { return v > 0.0f; });
result.lattices_indexed = 0;
for (size_t i = 0; i < lattices.size() && i < result.image_indexed.size(); ++i)
if (result.image_indexed[i] > 0.0f)
*result.lattices_indexed += std::lround(lattices[i]);
if (indexer)
result.images_integrated = std::ranges::count_if(indexer->GetIntegrationOutcome(),
[](const IntegrationOutcome &o) { return !o.reflections.empty(); });
}
// Powder calibration: the whole run has now been accumulated - the summed (q x azimuth) profile in
// plots, or the pooled spot list - so fit the detector geometry to it. The caller reports it and
+1
View File
@@ -852,6 +852,7 @@ void StillsMergeScaled(const DiffractionExperiment &experiment, const std::vecto
phase("Computing statistics");
TimingMark stats_mark(logger, "stills statistics");
out.statistics = merge_engine.MergeStats(out.merged, outcomes, reference_data, out.d_min);
out.statistics.image_merged = merge_engine.ImagesMerged();
// The same kept reflections binned over the reference range (--report-resolution); the
// stills error model is fitted over the whole range already, so no ISa of its own.
if (const auto range = cut_ss.GetReportResolutionRange_A(); range && !for_search)
+1 -13
View File
@@ -282,6 +282,7 @@ ProcessResult StillsLiveResult(const DiffractionExperiment &experiment, const St
result.images_processed = static_cast<uint64_t>(images_analysed);
if (images_analysed > 0)
result.indexing_rate = static_cast<float>(images_indexed) / static_cast<float>(images_analysed);
result.images_indexed = images_indexed;
result.consensus_cell = experiment.GetUnitCell();
result.space_group = experiment.GetGemmiSpaceGroup();
result.powder_measured = false;
@@ -369,16 +370,3 @@ void AppendScaleInput(std::vector<IntegrationOutcome> &outcomes, std::vector<Int
}
}
size_t MergedCrystals(const DiffractionExperiment &experiment, const std::vector<IntegrationOutcome> &outcomes,
size_t first, size_t last) {
MergeOnTheFly merge(experiment);
merge.ReferenceCell(experiment.GetUnitCell());
merge.FilterByImageCC(experiment.GetScalingSettings().GetMinCCForImage() > 0.0);
size_t n = 0;
for (size_t i = first; i < last; ++i)
n += !merge.Mask(outcomes[i])
&& std::any_of(outcomes[i].reflections.begin(), outcomes[i].reflections.end(), [](const Reflection &r) {
return r.image_scale_corr > 0.0f && std::isfinite(r.image_scale_corr);
});
return n;
}
-4
View File
@@ -146,7 +146,3 @@ std::string ScaleInputsRefusal(const std::vector<ScaleInput> &inputs, bool cell_
// frames already there, so that frames of different files stay distinct.
void AppendScaleInput(std::vector<IntegrationOutcome> &outcomes, std::vector<IntegrationOutcome> &&more);
// Of the frames [first, last) as MergeStills left them, the crystals that went into the merge: on the cell
// merged in, scaled, and not left out by --min-image-cc.
size_t MergedCrystals(const DiffractionExperiment &experiment, const std::vector<IntegrationOutcome> &outcomes,
size_t first, size_t last);
+53 -2
View File
@@ -2489,11 +2489,21 @@ int RugnuxInputRun::PrepareRugnuxInput() {
}
}
auto reflections = hdf5_reader.ReadReflections(start_image, last_image);
// Per frame, whether the run that wrote its file indexed it (imageIndexed); NaN in a file without it.
std::vector<float> image_indexed;
bool image_indexed_known = true;
const auto append_indexed = [&](const JFJochReaderDataset &d, size_t first, size_t n) {
image_indexed_known = image_indexed_known && d.indexing_result.size() >= first + n;
for (size_t i = first; i < first + n; ++i)
image_indexed.push_back(i < d.indexing_result.size() ? d.indexing_result[i] : NAN);
};
append_indexed(*dataset, start_image, reflections.size());
std::vector<size_t> input_first_frame = {0}; // of each file, in `reflections`
for (const auto &r : more) {
input_first_frame.push_back(reflections.size());
AppendScaleInput(reflections, r->ReadReflections());
const auto &d = *r->GetDataset();
append_indexed(d, 0, reflections.size() - input_first_frame.back());
ice_ring_score.insert(ice_ring_score.end(), d.ice_ring_score.begin(), d.ice_ring_score.end());
spot_count_ice_rings.insert(spot_count_ice_rings.end(), d.spot_count_ice_rings.begin(),
d.spot_count_ice_rings.end());
@@ -2796,8 +2806,9 @@ int RugnuxInputRun::PrepareRugnuxInput() {
experiment.ImagesPerTrigger(static_cast<int64_t>(so.images));
for (size_t k = 0; k < input_counts.size(); ++k) {
auto &c = input_counts[k];
c.merged = static_cast<int64_t>(MergedCrystals(experiment, reflections, input_first_frame[k],
input_first_frame[k + 1]));
const auto &merged = so.merge.statistics.image_merged;
c.merged = std::count(merged.begin() + input_first_frame[k], merged.begin() + input_first_frame[k + 1],
uint8_t{1});
logger.Info("Input {}: {} of {} indexed crystals merged, {} left out, from {}", k + 1, c.merged,
c.indexed, c.indexed - c.merged, c.file);
}
@@ -3010,11 +3021,45 @@ int RugnuxInputRun::PrepareRugnuxInput() {
&scaled_fulls);
}
// The per-image table of this merge: which input and image each row is, the scaling columns and whether
// the image's observations are in the merged data, and whether the run that wrote the file indexed it.
// The rest of that run's per-image analysis is not re-read, so its columns are nan.
if (!output_prefix.empty()) {
PerImagePlot plot;
plot.image_count = reflections.size();
plot.dataset.resize(plot.image_count);
plot.image.resize(plot.image_count);
plot.angle_deg.assign(plot.image_count, NAN);
const auto goniometer = experiment.GetGoniometer();
for (size_t k = 0; k + 1 < input_first_frame.size(); ++k) {
plot.dataset_files.push_back(std::filesystem::absolute(input_files[k]).lexically_normal().string());
for (size_t i = input_first_frame[k]; i < input_first_frame[k + 1]; ++i) {
plot.dataset[i] = static_cast<int>(k) + 1;
plot.image[i] = static_cast<int64_t>(i - input_first_frame[k] + (k == 0 ? start_image : 0));
if (goniometer)
plot.angle_deg[i] = goniometer->GetAngle_deg(static_cast<float>(plot.image[i]))
+ goniometer->GetWedge_deg() / 2.0f;
}
}
const ScalingResult scaling(reflections);
plot.scale_g = scaling.image_scale_g;
plot.mosaicity_deg = scaling.mosaicity_deg;
plot.cc_to_merge = scaling.image_cc;
plot.cc_n = scaling.image_cc_n;
plot.frame_disposition = merged_statistics.sweep_quality.frame_disposition;
plot.merged = merged_statistics.image_merged;
plot.indexed = image_indexed;
WritePerImagePlot(output_prefix, plot);
}
// --mode scale re-merges stored reflections, so it determines a merging result and gets the
// same report. It has no ProcessResult of its own; fill the parts that exist here.
ProcessResult scale_result;
scale_result.images_processed = input_files.size() > 1 ? reflections.size() : reader.GetNumberOfImages();
scale_result.scale_inputs = input_counts;
scale_result.images_integrated = static_cast<int64_t>(n_images);
if (image_indexed_known)
scale_result.images_indexed = std::ranges::count_if(image_indexed, [](float v) { return v > 0.0f; });
scale_result.consensus_cell = experiment.GetUnitCell();
scale_result.space_group = experiment.GetGemmiSpaceGroup();
scale_result.used_beam_x_pxl = experiment.GetBeamX_pxl();
@@ -4048,6 +4093,12 @@ int RugnuxInputRun::RunPreparedInput() {
logger.Info("Total throughput:{:.2f} MB/s", result.throughput_MBs);
if (result.indexing_rate.has_value())
logger.Info("Indexing rate: {:.2f}%", result.indexing_rate.value() * 100.0);
if (result.images_indexed)
logger.Info("Images: {} processed, {} indexed ({} lattices), {} integrated{}", result.images_processed,
*result.images_indexed, result.lattices_indexed.value_or(0), result.images_integrated.value_or(0),
result.has_merge_statistics
? fmt::format(", {} merged", std::ranges::count(result.merge_statistics.image_merged, uint8_t{1}))
: std::string());
for (size_t k = 0; k < result.dataset_inputs.size(); k++) {
const DatasetInputCount &in = result.dataset_inputs[k];
logger.Info(" input {}: {:.2f}% of {} images ({})", k + 1,
+2
View File
@@ -93,6 +93,8 @@ TEST_CASE("JFJochBrokerHttp_BearerTokens", "[broker]") {
REQUIRE(res->body.find("protected_run") != std::string::npos);
REQUIRE(client.Get("/statistics/data_collection", other)->status == 200);
REQUIRE(client.Get("/preview/plot?type=bkg_estimate", good)->status == 200);
// Merged or not per image, which a viewer on the broker reads (empty until a live merge is made).
REQUIRE(client.Get("/preview/plot?type=image_merged", good)->status == 200);
REQUIRE(client.Get("/rugnux/status", good)->status == 200);
// No live run placed a model: nothing to serve, but the token is accepted.
REQUIRE(client.Get("/rugnux/model.cif", good)->status == 404);
+17
View File
@@ -593,6 +593,23 @@ TEST_CASE("CBORSerialize_End_SpaceGroup", "[CBOR]") {
CHECK(*out.space_group_number == static_cast<uint64_t>(sg->number));
}
TEST_CASE("CBORSerialize_End_ImageMerged", "[CBOR]") {
std::vector<uint8_t> buffer(1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
EndMessage message{};
message.max_image_number = 4;
message.image_indexed = {1, 1, 0, 1};
message.image_merged = {1, 0, 0, 1};
REQUIRE_NOTHROW(serializer.SerializeSequenceEnd(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->end_message);
CHECK(deserialized->end_message->image_indexed == message.image_indexed);
CHECK(deserialized->end_message->image_merged == message.image_merged);
}
TEST_CASE("CBORSerialize_End_RefinedGeometry", "[CBOR]") {
std::vector<uint8_t> buffer(1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
+22 -6
View File
@@ -126,6 +126,8 @@ TEST_CASE("DiagnosticPlot_Format") {
static_cast<uint8_t>(FrameDisposition::Downgraded),
static_cast<uint8_t>(FrameDisposition::Rejected),
static_cast<uint8_t>(FrameDisposition::Rejected)};
plot.merged = {1, 1, 0, 0};
plot.indexed = {1.0f, 1.0f, 0.0f, 1.0f};
WritePerImagePlot(prefix, plot);
const auto all = ReadLines(prefix + "_plot.txt");
@@ -141,7 +143,7 @@ TEST_CASE("DiagnosticPlot_Format") {
// Every column is described in the header.
const auto legend = Columns(lines[0]);
REQUIRE(legend.size() == 13);
REQUIRE(legend.size() == 14);
for (size_t c = 1; c < legend.size(); c++) {
bool described = false;
for (size_t l = 0; l + 1 < header; l++)
@@ -156,6 +158,7 @@ TEST_CASE("DiagnosticPlot_Format") {
CHECK(legend[10] == "cc_n");
CHECK(legend[11] == "merged");
CHECK(legend[12] == "disposition");
CHECK(legend[13] == "indexed");
for (size_t i = 1; i < lines.size(); i++) {
const auto row = Columns(lines[i]);
@@ -181,6 +184,10 @@ TEST_CASE("DiagnosticPlot_Format") {
CHECK(Columns(lines[3])[12] == "nan");
CHECK(Columns(lines[4])[12] == "0");
// Indexed by the per-image indexer, whatever became of the image in the merge.
CHECK(Columns(lines[3])[13] == "0");
CHECK(Columns(lines[4])[13] == "1");
// The count the correlation beside it was taken over.
CHECK(Columns(lines[1])[10] == "500");
@@ -191,24 +198,28 @@ TEST_CASE("DiagnosticPlot_Format") {
TEST_CASE("DiagnosticPlot_NoDisposition") {
const std::string prefix = "diagnostic_plot_nodisp_test";
// Without the sweep-quality diagnostic, a frame counts as merged when it was scaled at all.
// Stills, with no sweep-quality diagnostic: merged is the merge's own answer, and the disposition
// follows it - a crystal scaled but left out (here the third) is rejected, one never scaled is nan.
PerImagePlot plot;
OneDataset(plot, 3);
plot.scale_g = {1.0f, NAN, 0.8f};
plot.merged = {1, 0, 0};
WritePerImagePlot(prefix, plot);
const auto lines = Rows(ReadLines(prefix + "_plot.txt"));
REQUIRE(lines.size() == 3);
CHECK(Columns(lines[0])[11] == "1");
CHECK(Columns(lines[1])[11] == "0");
CHECK(Columns(lines[2])[11] == "1");
CHECK(Columns(lines[2])[11] == "0");
CHECK(Columns(lines[0])[12] == "1");
CHECK(Columns(lines[1])[12] == "nan");
CHECK(Columns(lines[2])[12] == "1");
CHECK(Columns(lines[2])[12] == "0");
// No goniometer: the angle column is present and unplottable, so the columns never shift.
CHECK(Columns(lines[0]).size() == 13);
// No goniometer: the angle column is present and unplottable, so the columns never shift; no
// per-image indexing (--mode scale) leaves indexed nan.
CHECK(Columns(lines[0]).size() == 14);
CHECK(Columns(lines[0])[3] == "nan");
CHECK(Columns(lines[0])[13] == "nan");
}
TEST_CASE("DiagnosticPlot_NoMerge") {
@@ -242,6 +253,7 @@ TEST_CASE("DiagnosticPlot_JointSweeps") {
OneDataset(b, 3);
b.angle_deg = {90.05f, 90.15f, 90.25f};
b.spot_count = {7.0f, 8.0f, 9.0f};
b.indexed = {1.0f, 0.0f, 1.0f};
// The joint merge's table: its rows are the sweeps' frames end to end, its scale the joint merge's.
PerImagePlot joint;
@@ -273,4 +285,8 @@ TEST_CASE("DiagnosticPlot_JointSweeps") {
CHECK(Columns(lines[2])[4] == "nan");
CHECK(std::stof(Columns(lines[3])[6]) == 8.0f);
CHECK(std::stof(Columns(lines[2])[7]) == 2.0f);
// Each sweep's own per-image indexing.
CHECK(Columns(lines[0])[13] == "nan");
CHECK(Columns(lines[3])[13] == "0");
CHECK(Columns(lines[4])[13] == "1");
}
+2
View File
@@ -1509,6 +1509,8 @@ TEST_CASE("JFJochReader_NiggliClass_VDS", "[HDF5][Full]") {
REQUIRE_NOTHROW(reader.ReadFile("test95_master.h5"));
auto dataset = reader.GetDataset();
CHECK(dataset->experiment.GetImageNum() == 2);
// Written without a merge, as every file before imageMerged existed: no merge status, no error.
CHECK(dataset->merge_result.empty());
std::shared_ptr<JFJochReaderImage> reader_image, reader_image_2;
+17 -1
View File
@@ -140,4 +140,20 @@ TEST_CASE("JFJochReceiverPlots_Grid") {
REQUIRE(p.GetSizeX() == 10.0 * 5);
REQUIRE(p.GetSizeY() == 20.0 * 3);
}
}
TEST_CASE("JFJochReceiverPlots_ImageMerged") {
DiffractionExperiment experiment;
PixelMask mask(experiment);
AzimuthalIntegrationMapping integration(experiment, mask);
JFJochReceiverPlots plots;
plots.Setup(experiment, integration);
// Nothing merged yet: no plot, rather than a run of zeros.
std::vector<float> v;
plots.GetPlotRaw(v, PlotType::ImageMerged, "");
CHECK(v.empty());
plots.SetImageMerged({1, 0, 1});
plots.GetPlotRaw(v, PlotType::ImageMerged, "");
CHECK(v == std::vector<float>{1.0f, 0.0f, 1.0f});
}
+8
View File
@@ -186,6 +186,14 @@ TEST_CASE("JFJochReceiverRugnux_Stills", "[JFJochReceiverRugnux]") {
CHECK(end->refined_detector_distance_mm.has_value());
REQUIRE(pusher.Start());
CHECK(pusher.Start()->beam_center_x == Catch::Approx(end->refined_beam_center_x.value()));
// Which frames the merge took, per image: in the end message for the writer, and as a plot for the viewer
REQUIRE(end->image_merged.size() == images);
const auto merged = std::ranges::count(end->image_merged, uint8_t{1});
CHECK(merged > 0);
std::vector<float> merged_plot;
service.GetPlotRaw(merged_plot, PlotType::ImageMerged, "");
REQUIRE(merged_plot.size() == images);
CHECK(std::ranges::count(merged_plot, 1.0f) == merged);
const auto live = service.GetRugnuxStatus();
CHECK(live.state == RugnuxLiveState::Done);
+39
View File
@@ -788,6 +788,45 @@ TEST_CASE("MergeOnTheFly_AddImagesIsTheSerialMerge") {
CHECK(std::isnan(serial_stats.overall.r_pim));
}
// Which images reached the merge: one with an observation in it does; one with no reflections, one never
// scaled and one off the cell merged in do not.
TEST_CASE("MergeOnTheFly_ImagesMerged") {
DiffractionExperiment x;
x.SetSpaceGroup(TestSpaceGroup());
const UnitCell cell{50.0f, 60.0f, 70.0f, 90.0f, 90.0f, 90.0f};
const UnitCell other_cell{55.0f, 60.0f, 70.0f, 90.0f, 90.0f, 90.0f};
const auto image = [](const UnitCell &c, float image_scale_corr) {
IntegrationOutcome o;
o.latt = CrystalLattice(c);
for (int h = 1; h <= 5; ++h) {
Reflection r{};
r.h = h;
r.k = 1;
r.l = 2;
r.d = 10.0f / static_cast<float>(h);
r.I = 100.0f;
r.sigma = 10.0f;
r.prescaling_corr = 1.0f;
r.partiality = 1.0f;
r.image_scale_corr = image_scale_corr;
o.reflections.push_back(r);
}
return o;
};
std::vector<IntegrationOutcome> outcomes;
outcomes.push_back(image(cell, 1.0f));
outcomes.emplace_back(); // nothing integrated
outcomes.push_back(image(cell, 0.0f)); // never scaled
outcomes.push_back(image(other_cell, 1.0f)); // off the cell merged in
outcomes.push_back(image(cell, 1.0f));
MergeOnTheFly merge(x);
merge.ReferenceCell(cell);
merge.AddImages(outcomes);
CHECK(merge.ImagesMerged() == std::vector<uint8_t>{1, 0, 0, 0, 1});
}
// With the error model fitted, the merge takes every observation, however small its partiality, but reports a
// reflection only if one of its observations reaches the existence partiality (0.2).
TEST_CASE("MergeOnTheFly_EveryObservationMergedReflectionReportedOnlyIfOneReachesTheExistencePartiality") {
+50
View File
@@ -329,6 +329,7 @@ TEST_CASE("SweepQuality_HDF5RoundTrip", "[HDF5][Full][Diagnostics]") {
end_message.sweep_quality_reasons = ReasonVocabulary();
end_message.frame_disposition = expected_disposition;
end_message.frame_disposition_codes = DispositionVocabulary();
end_message.image_merged = {1, 1, 0, 0, 1, 1};
FileWriter writer(start_message);
std::vector<int16_t> image(x.GetPixelsNum(), 42);
@@ -351,6 +352,8 @@ TEST_CASE("SweepQuality_HDF5RoundTrip", "[HDF5][Full][Diagnostics]") {
CHECK(dataset->sweep_quality_reasons == ReasonVocabulary());
CHECK(dataset->frame_disposition == expected_disposition);
CHECK(dataset->frame_disposition_codes == DispositionVocabulary());
// Merged or not, per image: the viewer's image list sorts on it.
CHECK(dataset->merge_result == std::vector<float>{1, 1, 0, 0, 1, 1});
}
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
@@ -1155,6 +1158,53 @@ TEST_CASE("ResultReport_RotationScale", "[Diagnostics]") {
CHECK(kept.find("\nGONIOMETER_ROTATION_SCALE_SUSPECT= FALSE\n") != std::string::npos);
}
// How many images made it through each stage, as counts beside the rate: indexed (with the lattices on them),
// integrated and merged - in the keys and in the summary line.
TEST_CASE("ResultReport_ImageCounts", "[Diagnostics]") {
DiffractionExperiment x(DetJF(1));
ProcessResult result;
result.images_processed = 10;
result.indexing_rate = 0.6f;
result.images_indexed = 6;
result.lattices_indexed = 8;
result.images_integrated = 6;
result.consensus_cell = UnitCell{.a = 50.0f, .b = 60.0f, .c = 70.0f, .alpha = 90.0f, .beta = 90.0f, .gamma = 90.0f};
result.has_merge_statistics = true;
result.merge_statistics.stills = true;
result.merge_statistics.image_merged = {1, 1, 0, 1, 0, 0, 0, 0, 0, 0};
const auto text = RenderResultReport("prefix", "in.h5", x, result);
CHECK(text.find("\nIMAGES_PROCESSED= 10\n") != std::string::npos);
CHECK(text.find("\nIMAGES_INDEXED= 6\n") != std::string::npos);
CHECK(text.find("\nLATTICES_INDEXED= 8\n") != std::string::npos);
CHECK(text.find("\nIMAGES_INTEGRATED= 6\n") != std::string::npos);
CHECK(text.find("\nIMAGES_MERGED= 3\n") != std::string::npos);
CHECK(text.find("10 processed, 6 indexed (8 lattices), 6 integrated, 3 merged") != std::string::npos);
// Rotation indexes the sweep as a whole: the per-image indexer's counts are for developers, as its rate.
x.Goniometer(GoniometerAxis("omega", 0.0f, 0.1f, Coord(1, 0, 0), {}));
x.ImportIndexingSettings(IndexingSettings().RotationIndexing(true));
const auto rotation = RenderResultReport("prefix", "in.h5", x, result);
CHECK(rotation.find("IMAGES_INDEXED=") == std::string::npos);
CHECK(rotation.find("LATTICES_INDEXED=") == std::string::npos);
CHECK(rotation.find("\nIMAGES_INTEGRATED= 6\n") != std::string::npos);
CHECK(rotation.find("\nIMAGES_MERGED= 3\n") != std::string::npos);
CHECK(rotation.find("10 processed, 6 integrated, 3 merged") != std::string::npos);
RunProvenance dev;
dev.developer = true;
CHECK(RenderResultReport("prefix", "in.h5", x, result, dev).find("\nIMAGES_INDEXED= 6\n") != std::string::npos);
// --mode scale reads the images indexed from its file but counts no lattices.
x.Goniometer(std::nullopt);
result.lattices_indexed.reset();
CHECK(RenderResultReport("prefix", "in.h5", x, result).find("10 processed, 6 indexed, 6 integrated, 3 merged")
!= std::string::npos);
// No merge: no count of merged images, rather than a zero.
result.has_merge_statistics = false;
CHECK(RenderResultReport("prefix", "in.h5", x, result).find("IMAGES_MERGED=") == std::string::npos);
}
TEST_CASE("LeftoverLattice_Misorientation", "[Diagnostics]") {
const CrystalLattice triclinic(52.0f, 61.0f, 73.0f, 81.0f, 76.0f, 69.0f);
const auto rotated = [](const CrystalLattice &l, float deg, const Coord &axis) {
+1
View File
@@ -271,6 +271,7 @@ std::shared_ptr<JFJochReaderDataset> JFJochHttpReader::UpdateDataset_i() {
dataset->spot_count_indexed = GetPlot_i("spot_count_indexed");
dataset->indexing_result = GetPlot_i("indexing_rate");
dataset->merge_result = GetPlot_i("image_merged");
dataset->indexing_lattice_count = GetPlot_i("indexing_lattice_count");
dataset->profile_radius = GetPlot_i("profile_radius");
dataset->mosaicity_deg = GetPlot_i("mosaicity");
+87 -16
View File
@@ -6,10 +6,36 @@
#include <QVBoxLayout>
#include <QHeaderView>
#include <QCloseEvent>
#include <QMenu>
namespace {
constexpr int ScaleSortRole = Qt::UserRole + 1;
constexpr double ScaleNotAvailableSortValue = -1.0e100;
// The Yes/No columns, filtered from the header's context menu.
constexpr int IndexColumn = 2;
constexpr int MergedColumn = 3;
const QStringList YesNoValues = {"Yes", "No", "N/A"};
}
void ImageListFilterModel::SetFilter(int column, const QSet<QString> &values) {
allowed[column] = values;
invalidateFilter();
}
void ImageListFilterModel::ClearFilter(int column) {
allowed.remove(column);
invalidateFilter();
}
bool ImageListFilterModel::Allows(int column, const QString &value) const {
return !allowed.contains(column) || allowed[column].contains(value);
}
bool ImageListFilterModel::filterAcceptsRow(int source_row, const QModelIndex &source_parent) const {
for (auto it = allowed.cbegin(); it != allowed.cend(); ++it)
if (!it.value().contains(sourceModel()->index(source_row, it.key(), source_parent).data().toString()))
return false;
return true;
}
JFJochViewerImageListWindow::JFJochViewerImageListWindow(QWidget *parent) : JFJochHelperWindow(parent) {
@@ -24,7 +50,7 @@ JFJochViewerImageListWindow::JFJochViewerImageListWindow(QWidget *parent) : JFJo
tableModel = new QStandardItemModel(this);
setupTableModel();
proxyModel = new QSortFilterProxyModel(this);
proxyModel = new ImageListFilterModel(this);
proxyModel->setSourceModel(tableModel);
proxyModel->setSortRole(ScaleSortRole);
@@ -47,23 +73,61 @@ JFJochViewerImageListWindow::JFJochViewerImageListWindow(QWidget *parent) : JFJo
resize(800, 600);
connect(tableView, &QTableView::doubleClicked, this, &JFJochViewerImageListWindow::onTableRowDoubleClicked);
// A click on a header sorts; its context menu filters the Yes/No columns.
tableView->horizontalHeader()->setContextMenuPolicy(Qt::CustomContextMenu);
connect(tableView->horizontalHeader(), &QHeaderView::customContextMenuRequested,
this, &JFJochViewerImageListWindow::onHeaderMenu);
}
void JFJochViewerImageListWindow::onHeaderMenu(const QPoint &pos) {
const int column = tableView->horizontalHeader()->logicalIndexAt(pos);
if (column != IndexColumn && column != MergedColumn)
return;
QMenu menu(this);
for (const QString &value : YesNoValues) {
QAction *action = menu.addAction(value);
action->setCheckable(true);
action->setChecked(proxyModel->Allows(column, value));
}
menu.addSeparator();
QAction *all = menu.addAction("Show all");
QAction *chosen = menu.exec(tableView->horizontalHeader()->mapToGlobal(pos));
if (!chosen)
return;
QSet<QString> values;
if (chosen != all)
for (QAction *action : menu.actions())
if (action->isCheckable() && action->isChecked())
values.insert(action->text());
if (chosen == all || values.size() == YesNoValues.size())
proxyModel->ClearFilter(column);
else
proxyModel->SetFilter(column, values);
const QString name = column == IndexColumn ? "Index" : "Merged";
tableModel->setHeaderData(column, Qt::Horizontal, proxyModel->IsFiltered(column) ? name + " *" : name);
}
void JFJochViewerImageListWindow::setupTableModel()
{
tableModel->setColumnCount(8);
tableModel->setColumnCount(9);
tableModel->setHeaderData(0, Qt::Horizontal, "#");
tableModel->setHeaderData(1, Qt::Horizontal, "Bkg");
tableModel->setHeaderData(2, Qt::Horizontal, "Index");
tableModel->setHeaderData(3, Qt::Horizontal, "Spots");
tableModel->setHeaderData(4, Qt::Horizontal, "Res. [A]");
tableModel->setHeaderData(5, Qt::Horizontal, "Max val");
tableModel->setHeaderData(6, Qt::Horizontal, "Scale factor");
tableModel->setHeaderData(7, Qt::Horizontal, "Scale CC");
tableModel->setHeaderData(3, Qt::Horizontal, "Merged");
tableModel->setHeaderData(4, Qt::Horizontal, "Spots");
tableModel->setHeaderData(5, Qt::Horizontal, "Res. [A]");
tableModel->setHeaderData(6, Qt::Horizontal, "Max val");
tableModel->setHeaderData(7, Qt::Horizontal, "Scale factor");
tableModel->setHeaderData(8, Qt::Horizontal, "Scale CC");
}
void JFJochViewerImageListWindow::addDataRow(int imageNumber, double backgroundEstimate,
const QString& indexingResult,
const QString& mergeResult,
int spotCount,
double resolutionEstimate,
int64_t max_value,
@@ -81,14 +145,16 @@ void JFJochViewerImageListWindow::addDataRow(int imageNumber, double backgroundE
bgItem->setData(backgroundEstimate, ScaleSortRole);
rowItems.append(bgItem);
QStandardItem *indexingItem = new QStandardItem(indexingResult);
if (indexingResult == "Yes")
indexingItem->setData(1, ScaleSortRole);
else if (indexingResult == "No")
indexingItem->setData(0, ScaleSortRole);
else
indexingItem->setData(-1, ScaleSortRole);
rowItems.append(indexingItem);
for (const QString &result : {indexingResult, mergeResult}) {
QStandardItem *item = new QStandardItem(result);
if (result == "Yes")
item->setData(1, ScaleSortRole);
else if (result == "No")
item->setData(0, ScaleSortRole);
else
item->setData(-1, ScaleSortRole);
rowItems.append(item);
}
QStandardItem *spotItem = new QStandardItem();
spotItem->setData(spotCount, Qt::DisplayRole);
@@ -136,7 +202,7 @@ void JFJochViewerImageListWindow::clearAllData() {
void JFJochViewerImageListWindow::datasetLoaded(std::shared_ptr<const JFJochReaderDataset> in_dataset) {
dataset = std::move(in_dataset);
// One row of eight QStandardItems per image, arriving on every dataset tick of a live
// One row of nine QStandardItems per image, arriving on every dataset tick of a live
// acquisition. Not worth doing for a window nobody has open.
if (!isVisible()) {
pending_fill = true;
@@ -181,6 +247,10 @@ void JFJochViewerImageListWindow::fillTable() {
indexing_result = "Yes";
}
QString merge_result = "N/A";
if (dataset->merge_result.size() > i)
merge_result = dataset->merge_result[i] == 0 ? "No" : "Yes";
int64_t max_value = 0;
if (dataset->max_value.size() > i)
max_value = dataset->max_value[i];
@@ -196,6 +266,7 @@ void JFJochViewerImageListWindow::fillTable() {
addDataRow(i + 1,
bkg_estimate,
indexing_result,
merge_result,
spot_count,
res_estimation,
max_value,
+19 -2
View File
@@ -7,21 +7,37 @@
#include <QTableView>
#include <QStandardItemModel>
#include <QSortFilterProxyModel>
#include <QMap>
#include <QSet>
#include "JFJochHelperWindow.h"
#include "../../reader/JFJochReader.h"
// The rows whose text in each filtered column is one of the values let through there, as a spreadsheet's
// column filter; a column with no entry is not filtered.
class ImageListFilterModel : public QSortFilterProxyModel {
QMap<int, QSet<QString>> allowed;
public:
using QSortFilterProxyModel::QSortFilterProxyModel;
void SetFilter(int column, const QSet<QString> &values);
void ClearFilter(int column);
bool IsFiltered(int column) const { return allowed.contains(column); }
bool Allows(int column, const QString &value) const;
protected:
bool filterAcceptsRow(int source_row, const QModelIndex &source_parent) const override;
};
class JFJochViewerImageListWindow : public JFJochHelperWindow {
Q_OBJECT
QTableView *tableView;
QStandardItemModel *tableModel;
QSortFilterProxyModel *proxyModel;
ImageListFilterModel *proxyModel;
void setupTableModel();
void addDataRow(int imageNumber, double backgroundEstimate,
const QString &indexingResult, int spotCount,
const QString &indexingResult, const QString &mergeResult, int spotCount,
double resolutionEstimate, int64_t max_value,
double image_scale_factor,
double image_scale_cc_percent);
@@ -51,6 +67,7 @@ public slots:
void imageLoaded(std::shared_ptr<const JFJochReaderImage> image) override;
private slots:
void onTableRowDoubleClicked(const QModelIndex &index);
void onHeaderMenu(const QPoint &pos);
};
+1
View File
@@ -1306,6 +1306,7 @@ void NXmx::EndResultVectors(const EndMessage &end) {
SaveVectorIfMissing(*hdf5_file, "/entry/MX/peakCountLowRes", end.spot_count_low_res);
SaveVectorIfMissing(*hdf5_file, "/entry/MX/peakCountIndexed", end.spot_count_indexed);
SaveVectorIfMissing(*hdf5_file, "/entry/MX/imageIndexed", end.image_indexed);
SaveVectorIfMissing(*hdf5_file, "/entry/MX/imageMerged", end.image_merged);
SaveVectorIfMissing(*hdf5_file, "/entry/MX/indexedLatticeCount", end.indexed_lattice_count);
SaveVectorIfMissing(*hdf5_file, "/entry/MX/bkgEstimate", end.v_bkg_estimate);
SaveVectorIfMissing(*hdf5_file, "/entry/MX/spindleBlindFraction", end.v_spindle_blind_fraction);