mx/Jungfraujoch
2
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 75 Wiki Activity
Files
7c10d62daba739ed6e06507fc795bc07df22b5fd
Jungfraujoch/tests/CBORTest.cpp
T
leonarski_fandClaude Opus 5 6140de7249 rugnux: the unmerged MTZ carries the sensor efficiency in its own column, not inside LP 
LP is a column other programs read and un-apply. It is documented as the
Lorentz-polarization factor, and until the sensor efficiency was folded into the
same product that is what it held. Measured on our own unmerged file, LP spanned a
factor of 1.1343 across the detector where pure L/P spans nothing of the sort - the
excess is the efficiency, 13% end to end at 13 keV and 34% at 18 keV.

Both reference implementations keep it out. Recomputing pure L/P from a stored XDS
file's own geometry over its 124k reflections leaves RLP flat to 0.1% from 8.6 to
33.7 degrees, where a folded-in efficiency would have shown a 7% trend - and XDS has
the sensor numbers in hand. DIALS fills LP from lorentz and polarization alone and
writes QE as a separate column, even a column of ones.

Split them: LP is L/P again, QE is the efficiency, and the intensity is unchanged,
so a reader that un-applies LP recovers what it expects and one that wants raw
counts divides by LP and multiplies by QE. Only the unmerged file moves - every
other column is bit-identical and the file grows by exactly one float per reflection
plus one header record. The merged files are byte for byte what they were.

The process file gains an optional qe dataset beside the existing one rather than
changing what that one means, so a file written before the efficiency existed still
loads, and reads back as a correction of exactly one - which is what it was. It is
stored rather than recomputed on read because the writer has no geometry to
recompute it from, and because recomputing would have written a radial trend into
every stored file that never had one.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EFEJG6WBQv8th4UJFNe53N
2026-09-05 14:29:45 +02:00

1473 lines
63 KiB
C++
Raw Blame History

// SPDX-FileCopyrightText: 2024 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include <catch2/catch_all.hpp>
#include "../frame_serialize/CBORStream2Serializer.h"
#include "../frame_serialize/CBORStream2Deserializer.h"
#include "../compression/JFJochCompressor.h"
#include "../frame_serialize/CborUtil.h"
#include "../common/DiffractionExperiment.h" // kImageMetadataHeadroom
TEST_CASE("CBORSerialize_Start", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
StartMessage message{
.detector_distance = 0.0005,
.beam_center_x = 456.6,
.beam_center_y = 124.3,
.direct_beam_x = 461.2,
.direct_beam_y = 129.9,
.number_of_images = 34567,
.image_size_x = 456,
.image_size_y = 457,
.bit_depth_image = 32,
.bit_depth_readout = 16,
.pixel_signed = true,
.countrate_correction_enabled = true,
.incident_energy = 12400,
.incident_wavelength = 0.988,
.beam_size_x = 8e-5,
.beam_size_y = 3e-5,
.frame_time = 0.0001,
.count_time = 0.000098,
.saturation_value = 65534,
.error_value = 65535,
.pixel_size_x = 0.000075,
.pixel_size_y = 0.000075,
.sensor_thickness = 0.0005,
.sensor_material = "Si",
.unit_cell = UnitCell{.a = 45, .b = 37, .c = 45, .alpha = 90, .beta = 108, . gamma = 120},
.space_group_number = 154,
.max_spot_count = 250,
.storage_cell_number = 16,
.storage_cell_delay_ns = 15345,
.pixel_mask_enabled = true,
.arm_date = "abc",
.sample_name = "test_sample",
.file_prefix = "test_sample/dir/file",
.images_per_file = 12345,
.channels = {"default", "sc2"},
.detector_description = "EIGER 16M",
.detector_serial_number = "123",
.run_name = "bla",
.run_number = 4567,
.gain_file_names = {"abc", "def", "/dsadasdsa/dadsadas/dsadsa/M056.bin"},
.detector_translation = {0.5f, 0.0f, 0.5f},
.source_type = "Synchrotron X-ray Source",
.source_name = "Swiss Light Source",
.instrument_name = "X06SA",
.summation = 567,
.az_int_bin_to_q = {0.1, 0.2, 0.3, 0.5},
.az_int_q_bin_count = 35,
.az_int_phi_bin_count = 120,
.total_flux = 123,
.attenuator_transmission = 0.345,
.write_master_file = true,
.user_data = R"({"pi":3.1415, "z":"string"})"_json,
.data_reduction_factor_serialmx = 0.75,
.experiment_group = "p10001",
.jfjoch_release = "1.4.98",
.socket_number = 3,
.writer_notification_zmq_addr = "tcp://1.2.3.4:5678",
.jungfrau_conversion_enabled = true,
.jungfrau_conversion_factor = 17.56f,
.geometry_transformation_enabled = false,
.overwrite = true,
.file_format = FileWriterFormat::NXmxVDS,
.ring_current_mA = 123,
.sample_temperature_K = 345,
.indexing_algorithm = IndexingAlgorithmEnum::FFT
};
REQUIRE_NOTHROW(serializer.SerializeSequenceStart(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::START);
REQUIRE(deserialized->start_message);
StartMessage &output_message = *deserialized->start_message;
CHECK(output_message.images_per_file == message.images_per_file);
CHECK(output_message.detector_distance == Catch::Approx(message.detector_distance));
CHECK(output_message.beam_center_x == Catch::Approx(message.beam_center_x));
CHECK(output_message.beam_center_y == Catch::Approx(message.beam_center_y));
// The PONI (beam_center_*) and the direct beam are different points on a tilted detector, so the
// round trip has to keep them apart, not merely keep two floats.
REQUIRE(output_message.direct_beam_x);
REQUIRE(output_message.direct_beam_y);
CHECK(output_message.direct_beam_x.value() == Catch::Approx(message.direct_beam_x.value()));
CHECK(output_message.direct_beam_y.value() == Catch::Approx(message.direct_beam_y.value()));
CHECK(output_message.number_of_images == message.number_of_images);
CHECK(output_message.image_size_x == message.image_size_x);
CHECK(output_message.image_size_y == message.image_size_y);
CHECK(output_message.bit_depth_image == message.bit_depth_image);
CHECK(output_message.bit_depth_readout == message.bit_depth_readout);
CHECK(output_message.incident_energy == Catch::Approx(message.incident_energy));
CHECK(output_message.incident_wavelength == Catch::Approx(message.incident_wavelength));
CHECK(output_message.frame_time == Catch::Approx(message.frame_time));
CHECK(output_message.count_time == Catch::Approx(message.count_time));
CHECK(output_message.saturation_value == message.saturation_value);
CHECK(output_message.error_value == message.error_value);
CHECK(output_message.pixel_size_x == Catch::Approx(message.pixel_size_x));
CHECK(output_message.pixel_size_y == Catch::Approx(message.pixel_size_y));
CHECK(output_message.sensor_thickness == Catch::Approx(message.sensor_thickness));
CHECK(output_message.sensor_material == message.sensor_material);
CHECK(output_message.pixel_mask_enabled == message.pixel_mask_enabled);
CHECK(output_message.space_group_number == message.space_group_number);
CHECK(output_message.arm_date == message.arm_date);
CHECK(output_message.storage_cell_number == message.storage_cell_number);
CHECK(output_message.storage_cell_delay_ns == message.storage_cell_delay_ns);
CHECK(output_message.pixel_signed == message.pixel_signed);
CHECK(output_message.sample_name == message.sample_name);
CHECK(output_message.file_prefix == message.file_prefix);
CHECK(output_message.max_spot_count == message.max_spot_count);
CHECK(output_message.channels == message.channels);
CHECK(output_message.detector_description == message.detector_description);
CHECK(output_message.detector_serial_number == message.detector_serial_number);
CHECK(output_message.run_name == message.run_name);
CHECK(output_message.run_number == message.run_number);
CHECK(output_message.source_type == message.source_type);
CHECK(output_message.source_name == message.source_name);
CHECK(output_message.instrument_name == message.instrument_name);
CHECK(output_message.az_int_q_bin_count == message.az_int_q_bin_count);
CHECK(output_message.az_int_phi_bin_count == message.az_int_phi_bin_count);
CHECK(output_message.summation == message.summation);
CHECK(output_message.az_int_bin_to_q == message.az_int_bin_to_q);
for (int i = 0; i < 3; i++)
CHECK(output_message.detector_translation[i] == message.detector_translation[i]);
CHECK(output_message.unit_cell);
CHECK(output_message.unit_cell->a == message.unit_cell->a);
CHECK(output_message.unit_cell->b == message.unit_cell->b);
CHECK(output_message.unit_cell->c == message.unit_cell->c);
CHECK(output_message.unit_cell->alpha == message.unit_cell->alpha);
CHECK(output_message.unit_cell->beta == message.unit_cell->beta);
CHECK(output_message.unit_cell->gamma == message.unit_cell->gamma);
REQUIRE(output_message.beam_size_x);
CHECK(output_message.beam_size_x.value() == message.beam_size_x.value());
REQUIRE(output_message.beam_size_y);
CHECK(output_message.beam_size_y.value() == message.beam_size_y.value());
REQUIRE(output_message.total_flux);
CHECK(output_message.total_flux.value() == message.total_flux.value());
REQUIRE(output_message.attenuator_transmission);
CHECK(output_message.attenuator_transmission.value() == message.attenuator_transmission.value());
CHECK(output_message.user_data == message.user_data);
REQUIRE(output_message.user_data.is_object());
CHECK(output_message.user_data.size() == 2);
CHECK(output_message.gain_file_names == message.gain_file_names);
CHECK(output_message.countrate_correction_enabled == message.countrate_correction_enabled);
CHECK(output_message.flatfield_enabled == message.flatfield_enabled);
CHECK(output_message.write_master_file == message.write_master_file);
CHECK(output_message.data_reduction_factor_serialmx == message.data_reduction_factor_serialmx);
CHECK(output_message.experiment_group == message.experiment_group);
CHECK(output_message.jfjoch_release == message.jfjoch_release);
CHECK(output_message.socket_number == message.socket_number);
CHECK(output_message.writer_notification_zmq_addr == message.writer_notification_zmq_addr);
CHECK(output_message.geometry_transformation_enabled == message.geometry_transformation_enabled);
CHECK(output_message.jungfrau_conversion_enabled == message.jungfrau_conversion_enabled);
CHECK(output_message.jungfrau_conversion_factor == message.jungfrau_conversion_factor);
CHECK(output_message.overwrite == message.overwrite);
CHECK(output_message.file_format == message.file_format);
CHECK(output_message.sample_temperature_K == message.sample_temperature_K);
CHECK(output_message.ring_current_mA == message.ring_current_mA);
CHECK(output_message.indexing_algorithm == message.indexing_algorithm);
}
TEST_CASE("CBORSerialize_Start_GoniometerAxis", "[CBOR]") {
GoniometerAxis axis("z", 115, 0.456, {1, 2, 3}, Coord{50, 20, 30});
axis.ScreeningWedge(0.01);
StartMessage message{
.goniometer = axis
};
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
REQUIRE_NOTHROW(serializer.SerializeSequenceStart(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::START);
REQUIRE(deserialized->start_message);
StartMessage &output_message = *deserialized->start_message;
REQUIRE(output_message.goniometer.has_value());
CHECK(output_message.goniometer->GetName() == axis.GetName());
CHECK(output_message.goniometer->GetStart_deg() == axis.GetStart_deg());
CHECK(output_message.goniometer->GetIncrement_deg() == axis.GetIncrement_deg());
CHECK(output_message.goniometer->GetAxis().x == Catch::Approx(axis.GetAxis().x));
CHECK(output_message.goniometer->GetAxis().y == Catch::Approx(axis.GetAxis().y));
CHECK(output_message.goniometer->GetAxis().z == Catch::Approx(axis.GetAxis().z));
REQUIRE(output_message.goniometer->GetHelicalStep());
CHECK(output_message.goniometer->GetHelicalStep() == axis.GetHelicalStep());
REQUIRE(output_message.goniometer->GetScreeningWedge());
CHECK(output_message.goniometer->GetScreeningWedge() == axis.GetScreeningWedge());
}
TEST_CASE("CBORSerialize_Start_GridScan", "[CBOR]") {
GridScanSettings grid(123, 0.1, 34.0, true, true);
StartMessage message{
.grid_scan = grid
};
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
REQUIRE_NOTHROW(serializer.SerializeSequenceStart(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::START);
REQUIRE(deserialized->start_message);
StartMessage &output_message = *deserialized->start_message;
REQUIRE(!output_message.goniometer.has_value());
REQUIRE(output_message.grid_scan.has_value());
CHECK(output_message.grid_scan->GetNFast() == grid.GetNFast());
CHECK(output_message.grid_scan->IsVerticalScan() == grid.IsVerticalScan());
CHECK(output_message.grid_scan->IsSnakeScan() == grid.IsSnakeScan());
CHECK(output_message.grid_scan->GetGridElemFast_um() == Catch::Approx(grid.GetGridElemFast_um()));
CHECK(output_message.grid_scan->GetGridElemSlow_um() == Catch::Approx(grid.GetGridElemSlow_um()));
}
// A grid scan and a goniometer axis travel together on the wire: the axis with increment 0 is how
// the angle the spindle stood at reaches the file.
TEST_CASE("CBORSerialize_Start_GridScanAtStationaryAxis", "[CBOR]") {
GridScanSettings grid(20, 5.0, 5.0, false, false);
GoniometerAxis axis("omega", 90.0f, 0.0f, Coord(-1, 0, 0), {});
StartMessage message{
.grid_scan = grid,
.goniometer = axis
};
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
REQUIRE_NOTHROW(serializer.SerializeSequenceStart(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->start_message);
StartMessage &output_message = *deserialized->start_message;
REQUIRE(output_message.grid_scan.has_value());
CHECK(output_message.grid_scan->GetNFast() == grid.GetNFast());
REQUIRE(output_message.goniometer.has_value());
CHECK(output_message.goniometer->GetName() == "omega");
CHECK(!output_message.goniometer->IsScanning());
CHECK(output_message.goniometer->GetStart_deg() == Catch::Approx(90.0f));
}
TEST_CASE("CBORSerialize_Start_ThresholdEnergy", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
StartMessage message{};
message.threshold_energy["thr1"] = 12500;
message.threshold_energy["thr2"] = 17000;
REQUIRE_NOTHROW(serializer.SerializeSequenceStart(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::START);
REQUIRE(deserialized->start_message);
StartMessage &output_message = *deserialized->start_message;
REQUIRE(output_message.threshold_energy.size() == 2);
REQUIRE(output_message.threshold_energy == message.threshold_energy);
}
TEST_CASE("CBORSerialize_Start_Fluorescence", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
StartMessage message{};
// prepare fluorescence spectrum
std::vector<float> energy{1.0f, 2.0f, 3.0f};
std::vector<float> data{10.0f, 20.0f, 15.0f};
message.fluorescence_spectrum = XrayFluorescenceSpectrum(energy, data);
REQUIRE_NOTHROW(serializer.SerializeSequenceStart(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::START);
REQUIRE(deserialized->start_message);
const StartMessage &out = *deserialized->start_message;
REQUIRE(!out.fluorescence_spectrum.empty());
CHECK(out.fluorescence_spectrum.GetEnergy_eV() == energy);
CHECK(out.fluorescence_spectrum.GetData() == data);
}
TEST_CASE("CBORSerialize_Start_Smargon", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
StartMessage message{};
SmargonPosition smargon;
smargon.phi_deg = 45.0f;
smargon.chi_deg = 30.0f;
smargon.phi_axis = {0, -1, 0};
smargon.chi_axis = {0, 0, 1};
message.smargon_position = smargon;
REQUIRE_NOTHROW(serializer.SerializeSequenceStart(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::START);
REQUIRE(deserialized->start_message);
const StartMessage &out = *deserialized->start_message;
REQUIRE(out.smargon_position.has_value());
CHECK(out.smargon_position->phi_deg == 45.0f);
CHECK(out.smargon_position->chi_deg == 30.0f);
CHECK(out.smargon_position->phi_axis == Coord{0, -1, 0});
CHECK(out.smargon_position->chi_axis == Coord{0, 0, 1});
}
TEST_CASE("CBORSerialize_ROI", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
StartMessage message{
.rois = {
ROIConfig{
.type = ROIConfig::ROIType::Circle,
.name = "roi1",
.circle = ROIConfigCircle{
.r = 3.0,
.x = 5.0,
.y = 4.0
}
},
ROIConfig{
.type = ROIConfig::ROIType::Box,
.name = "roi2",
.box = ROIConfigBox{
.xmin = 5,
.xmax = 12,
.ymin = 7,
.ymax = 3
}
},
ROIConfig{
.type = ROIConfig::ROIType::Azim,
.name = "roi3",
.azim = ROIConfigAzim{
.qmin = 4.0,
.qmax = 5.0,
.phi_min = 30.0f,
.phi_max = 90.0f
}
},
ROIConfig{
.type = ROIConfig::ROIType::Azim,
.name = "roi4",
.azim = ROIConfigAzim{
.qmin = 1.0,
.qmax = 2.0,
.phi_min = 0.0f,
.phi_max = 0.0f
}
}
}
};
REQUIRE_NOTHROW(serializer.SerializeSequenceStart(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::START);
REQUIRE(deserialized->start_message);
StartMessage &output_message = *deserialized->start_message;
REQUIRE(output_message.rois.size() == 4);
for (int i = 0; i < 4; i++) {
CHECK(output_message.rois[i].name == message.rois[i].name);
CHECK(output_message.rois[i].type == message.rois[i].type);
}
CHECK(output_message.rois[0].circle.x == message.rois[0].circle.x);
CHECK(output_message.rois[0].circle.y == message.rois[0].circle.y);
CHECK(output_message.rois[0].circle.r == message.rois[0].circle.r);
CHECK(output_message.rois[1].box.xmin == message.rois[1].box.xmin);
CHECK(output_message.rois[1].box.xmax == message.rois[1].box.xmax);
CHECK(output_message.rois[1].box.ymin == message.rois[1].box.ymin);
CHECK(output_message.rois[1].box.ymax == message.rois[1].box.ymax);
CHECK(output_message.rois[2].azim.qmin == message.rois[2].azim.qmin);
CHECK(output_message.rois[2].azim.qmax == message.rois[2].azim.qmax);
CHECK(output_message.rois[2].azim.phi_min == 30.0f);
CHECK(output_message.rois[2].azim.phi_max == 90.0f);
// Full ring: phi_min == phi_max round-trips (no sector emitted)
CHECK(output_message.rois[3].azim.phi_min == output_message.rois[3].azim.phi_max);
}
TEST_CASE("CBORSerialize_Start_EmptyString", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
StartMessage message{
.sample_name = "",
.file_prefix = ""
};
REQUIRE_NOTHROW(serializer.SerializeSequenceStart(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::START);
REQUIRE(deserialized->start_message);
StartMessage &output_message = *deserialized->start_message;
CHECK(output_message.file_prefix.empty());
CHECK(output_message.sample_name.empty());
CHECK(output_message.arm_date.empty());
}
TEST_CASE("CBORSerialize_Start_PixelMask", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
std::vector<uint32_t> mask_0(456 * 457, 15);
std::vector<uint32_t> mask_1(456 * 457, 12);
StartMessage message{
.image_size_x = 456,
.image_size_y = 457,
.pixel_mask = {{"default", mask_0}, {"mask_1", mask_1}}
};
REQUIRE_NOTHROW(serializer.SerializeSequenceStart(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::START);
REQUIRE(deserialized->start_message);
StartMessage &output_message = *deserialized->start_message;
REQUIRE(output_message.pixel_mask.size() == 2);
REQUIRE(output_message.pixel_mask.contains("default"));
REQUIRE(output_message.pixel_mask.contains("mask_1"));
REQUIRE(output_message.pixel_mask["default"].size() == 456*457);
REQUIRE(output_message.pixel_mask["mask_1"].size() == 456*457);
REQUIRE(output_message.pixel_mask["default"] == mask_0);
REQUIRE(output_message.pixel_mask["mask_1"] == mask_1);
}
TEST_CASE("CBORSerialize_Calibration", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
std::vector<float> calib1(256);
for (int i = 0; i < 256; i++) {
calib1[i] = i * 34.567;
}
CompressedImage image1(calib1, 16, 16);
image1.Channel("calib1");
REQUIRE_NOTHROW(serializer.SerializeCalibration(image1));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::CALIBRATION);
REQUIRE(deserialized->calibration);
CompressedImage &output_message = *deserialized->calibration;
CHECK(output_message.GetMode() == CompressedImageMode::Float32);
CHECK(output_message.GetWidth() == 16);
CHECK(output_message.GetHeight() == 16);
CHECK(output_message.GetChannel() == "calib1");
CHECK(memcmp(output_message.GetCompressed(), calib1.data(), 256 * sizeof(float)) == 0);
}
TEST_CASE("CBORSerialize_End", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
EndMessage message{
.max_image_number = 57789,
.images_collected_count = 50000,
.images_sent_to_write_count = 40000,
.max_receiver_delay = 3456,
.efficiency = 0.99,
.spindle_blind_fraction = 0.31f,
.spindle_lost_unique_fraction = 0.021f,
.end_date = "ccc",
.run_name = "bla5",
.run_number = 45676782,
.rotation_lattice_type = LatticeMessage{
.centering = 'R',
.niggli_class = 22,
.crystal_system = gemmi::CrystalSystem::Tetragonal,
},
.rotation_lattice = CrystalLattice(40, 50, 60, 90, 90, 90),
.v_spindle_blind_fraction = {0.25f, NAN, 1.0f}
};
REQUIRE_NOTHROW(serializer.SerializeSequenceEnd(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::END);
REQUIRE(deserialized->end_message);
EndMessage &output_message = *deserialized->end_message;
REQUIRE(output_message.max_receiver_delay == message.max_receiver_delay);
REQUIRE(output_message.max_image_number == message.max_image_number);
REQUIRE(output_message.images_collected_count == message.images_collected_count);
REQUIRE(output_message.images_sent_to_write_count == message.images_sent_to_write_count);
REQUIRE(output_message.efficiency);
REQUIRE(output_message.efficiency == Catch::Approx(message.efficiency.value()));
REQUIRE(output_message.end_date == message.end_date);
REQUIRE(output_message.run_number == message.run_number);
REQUIRE(output_message.run_name == message.run_name);
REQUIRE(output_message.az_int_result.empty());
REQUIRE(output_message.rotation_lattice_type.has_value());
CHECK(output_message.rotation_lattice_type->centering == 'R');
CHECK(output_message.rotation_lattice_type->crystal_system == gemmi::CrystalSystem::Tetragonal);
CHECK(output_message.rotation_lattice_type->niggli_class == 22);
REQUIRE(output_message.rotation_lattice.has_value());
CHECK(output_message.rotation_lattice->GetUnitCell().c == Catch::Approx(60.0));
REQUIRE(output_message.spindle_blind_fraction == message.spindle_blind_fraction);
REQUIRE(output_message.spindle_lost_unique_fraction == message.spindle_lost_unique_fraction);
// The per-image vector holds NaN where a frame had no value (CANNOT SAY); the hole must
// survive the round trip as a hole, not as a number.
REQUIRE(output_message.v_spindle_blind_fraction.size() == 3);
CHECK(output_message.v_spindle_blind_fraction[0] == Catch::Approx(0.25f));
CHECK(std::isnan(output_message.v_spindle_blind_fraction[1]));
CHECK(output_message.v_spindle_blind_fraction[2] == Catch::Approx(1.0f));
}
TEST_CASE("CBORSerialize_End_SpaceGroup", "[CBOR]") {
// The determined group travels as both spellings. Only the name keeps the setting, so a
// non-reference setting has to survive the round trip as itself and not as its number's group.
const gemmi::SpaceGroup *sg = gemmi::find_spacegroup_by_name("P 1 1 2");
REQUIRE(sg != nullptr);
REQUIRE(gemmi::find_spacegroup_by_number(sg->number)->xhm() != sg->xhm());
std::vector<uint8_t> buffer(1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
EndMessage message{};
message.max_image_number = 0;
message.space_group_name = sg->xhm();
message.space_group_number = static_cast<uint64_t>(sg->number);
REQUIRE_NOTHROW(serializer.SerializeSequenceEnd(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->end_message);
const EndMessage &out = *deserialized->end_message;
REQUIRE(out.space_group_name.has_value());
CHECK(*out.space_group_name == sg->xhm());
REQUIRE(out.space_group_number.has_value());
CHECK(*out.space_group_number == static_cast<uint64_t>(sg->number));
}
TEST_CASE("CBORSerialize_End_SpaceGroupNumberOnly", "[CBOR]") {
// A sender written before the name existed fills in only the number; the message still arrives
// and the missing name is absent rather than empty, so a reader can tell it apart.
std::vector<uint8_t> buffer(1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
EndMessage message{};
message.max_image_number = 0;
message.space_group_number = 96;
REQUIRE_NOTHROW(serializer.SerializeSequenceEnd(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->end_message);
const EndMessage &out = *deserialized->end_message;
CHECK_FALSE(out.space_group_name.has_value());
REQUIRE(out.space_group_number.has_value());
CHECK(*out.space_group_number == 96);
}
TEST_CASE("CBORSerialize_End_RadIntResult", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
EndMessage message{
.max_image_number = 57789,
.max_receiver_delay = 3456,
.efficiency = 0.99,
.end_date = "ccc",
.run_name = "bla5",
.run_number = 45676782
};
message.az_int_result["avg"] = {11.0, 12.0, 13.0};
message.az_int_result["file0"] = {56.0, 75.0, 34.0};
REQUIRE_NOTHROW(serializer.SerializeSequenceEnd(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::END);
EndMessage &output_message = *deserialized->end_message;
REQUIRE(output_message.az_int_result.size() == 2);
REQUIRE(output_message.az_int_result.contains("avg"));
REQUIRE(output_message.az_int_result.contains("file0"));
CHECK(message.az_int_result["avg"] == output_message.az_int_result["avg"]);
CHECK(message.az_int_result["file0"] == output_message.az_int_result["file0"]);
}
TEST_CASE("CBORSerialize_End_ADUHistogram", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
EndMessage message{
.max_image_number = 57789,
.max_receiver_delay = 3456,
.efficiency = 0.99,
.end_date = "ccc",
.run_name = "bla5",
.run_number = 45676782,
.adu_histogram_bin_width = 55
};
message.adu_histogram["avg"] = {11, 12, 13};
message.adu_histogram["file0"] = {56, 75, 34};
REQUIRE_NOTHROW(serializer.SerializeSequenceEnd(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::END);
REQUIRE(deserialized->end_message);
EndMessage &output_message = *deserialized->end_message;
REQUIRE(output_message.adu_histogram.size() == 2);
REQUIRE(output_message.adu_histogram.contains("avg"));
REQUIRE(output_message.adu_histogram.contains("file0"));
CHECK(message.adu_histogram["avg"] == output_message.adu_histogram["avg"]);
CHECK(message.adu_histogram["file0"] == output_message.adu_histogram["file0"]);
CHECK(message.adu_histogram_bin_width == output_message.adu_histogram_bin_width);
}
TEST_CASE("CBORSerialize_Image", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
std::vector<SpotToSave> spots;
std::vector<uint16_t> test(512);
for (int i = 0; i < test.size(); i++)
test[i] = (i * 253 + 56) % 256;
CompressedImage image(test, 256, 2);
DataMessage message{
.number = 456,
.image = image,
.image_collection_efficiency = 0.11,
.spots = spots,
.spot_count_ice_rings = 157,
.bkg_estimate = 12.345f,
.spindle_blind_fraction = 0.62f,
.indexing_result = true,
.indexing_unit_cell = UnitCell{.a = 123, .b = 145, .c = 67.5, .alpha = 90, .beta = 120, .gamma = 134},
.adu_histogram = {3, 4, 5, 8},
.timestamp = 1ul << 27 | 1ul << 35,
.exptime = 1000,
.run_name = "bla2",
.run_number = 4567678,
.saturated_pixel_count = 378,
.error_pixel_count = 123,
.strong_pixel_count = 1234,
.min_viable_pixel_value = 123,
.max_viable_pixel_value = 6789,
.user_data = R"({"pi":3.1415, "z":"string"})"_json,
.jf_info = UINT32_MAX,
.receiver_aq_dev_delay = 2323,
.storage_cell = 0xF,
.xfel_pulse_id = UINT64_MAX - 5678,
.xfel_event_code = UINT64_MAX - 123,
.original_number = 12789
};
REQUIRE_NOTHROW(serializer.SerializeImage(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::IMAGE);
REQUIRE(deserialized->data_message);
DataMessage &image_array = *deserialized->data_message;
REQUIRE(image_array.image.GetCompressionAlgorithm() == CompressionAlgorithm::NO_COMPRESSION);
REQUIRE(image_array.image.GetWidth() == 256);
REQUIRE(image_array.image.GetHeight() == 2);
REQUIRE(image_array.image.GetMode() == CompressedImageMode::Uint16);
REQUIRE(image_array.image.GetChannel() == "default");
REQUIRE(image_array.image.GetCompressedSize() == test.size() * sizeof(uint16_t));
REQUIRE(image_array.indexing_result == message.indexing_result);
REQUIRE(image_array.number == 456);
REQUIRE(image_array.run_number == message.run_number);
REQUIRE(image_array.run_name == message.run_name);
REQUIRE(memcmp(image_array.image.GetCompressed(), test.data(), test.size() * sizeof(uint16_t)) == 0);
REQUIRE(image_array.xfel_pulse_id == message.xfel_pulse_id);
REQUIRE(image_array.xfel_event_code == message.xfel_event_code);
REQUIRE(image_array.jf_info == message.jf_info);
REQUIRE(image_array.timestamp == message.timestamp);
REQUIRE(image_array.storage_cell == message.storage_cell);
REQUIRE(image_array.exptime == message.exptime);
REQUIRE(image_array.receiver_aq_dev_delay == image_array.receiver_aq_dev_delay);
REQUIRE(image_array.adu_histogram == message.adu_histogram);
REQUIRE(image_array.saturated_pixel_count == message.saturated_pixel_count);
REQUIRE(image_array.error_pixel_count == message.error_pixel_count);
REQUIRE(image_array.strong_pixel_count == message.strong_pixel_count);
REQUIRE(image_array.bkg_estimate == message.bkg_estimate);
REQUIRE(image_array.spindle_blind_fraction == message.spindle_blind_fraction);
REQUIRE(image_array.image_collection_efficiency == message.image_collection_efficiency);
REQUIRE(image_array.user_data == message.user_data);
REQUIRE(image_array.original_number == message.original_number);
REQUIRE(image_array.spot_count_ice_rings == message.spot_count_ice_rings);
REQUIRE(image_array.indexing_unit_cell.has_value());
REQUIRE(image_array.indexing_unit_cell.value().a == message.indexing_unit_cell.value().a);
REQUIRE(image_array.indexing_unit_cell.value().b == message.indexing_unit_cell.value().b);
REQUIRE(image_array.indexing_unit_cell.value().c == message.indexing_unit_cell.value().c);
REQUIRE(image_array.indexing_unit_cell.value().alpha == message.indexing_unit_cell.value().alpha);
REQUIRE(image_array.indexing_unit_cell.value().beta == message.indexing_unit_cell.value().beta);
REQUIRE(image_array.indexing_unit_cell.value().gamma == message.indexing_unit_cell.value().gamma);
REQUIRE(image_array.min_viable_pixel_value == message.min_viable_pixel_value);
REQUIRE(image_array.max_viable_pixel_value == message.max_viable_pixel_value);
}
TEST_CASE("CBORSerialize_Image_CrystalLattice", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
std::vector<SpotToSave> spots;
std::vector<uint16_t> test(512);
for (int i = 0; i < test.size(); i++)
test[i] = (i * 253 + 56) % 256;
CompressedImage image(test, 256, 2);
std::vector<float> latt = {1, 2, 3, 4, 5, 6, 7, 8, 9};
DataMessage message{
.image = image,
.indexing_lattice = CrystalLattice(latt)
};
REQUIRE_NOTHROW(serializer.SerializeImage(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::IMAGE);
REQUIRE(deserialized->data_message);
DataMessage &image_array = *deserialized->data_message;
REQUIRE(deserialized->data_message->indexing_lattice);
REQUIRE(deserialized->data_message->indexing_lattice->GetVector() == latt);
}
TEST_CASE("CBORSerialize_Image_2", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
std::vector<SpotToSave> spots;
std::vector<uint8_t> test(512 * 1024);
for (int i = 0; i < test.size(); i++)
test[i] = (i * 253 + 56) % 256;
CompressedImage image(test, 1024, 512);
DataMessage message{
.number = 480,
.image = image,
.spots = spots,
.indexing_result = true
};
REQUIRE_NOTHROW(serializer.SerializeImage(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::IMAGE);
REQUIRE(deserialized->data_message);
DataMessage &image_array = *deserialized->data_message;
REQUIRE(image_array.image.GetCompressionAlgorithm() == CompressionAlgorithm::NO_COMPRESSION);
REQUIRE(image_array.image.GetWidth() == 1024);
REQUIRE(image_array.image.GetHeight() == 512);
REQUIRE(image_array.image.GetMode() == CompressedImageMode::Uint8);
REQUIRE(image_array.image.GetChannel() == "default");
REQUIRE(image_array.image.GetCompressedSize() == test.size());
REQUIRE(image_array.indexing_result == message.indexing_result);
REQUIRE(image_array.number == 480);
REQUIRE(memcmp(image_array.image.GetCompressed(), test.data(), test.size()) == 0);
REQUIRE(!image_array.original_number);
}
TEST_CASE("CBORSerialize_Image_Float", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
std::vector<SpotToSave> spots;
std::vector<float> test(512 * 1024);
for (int i = 0; i < test.size(); i++)
test[i] = i * 0.1f;
CompressedImage image(test, 1024, 512);
DataMessage message{
.number = 480,
.image = image,
.spots = spots,
.indexing_result = false
};
REQUIRE_NOTHROW(serializer.SerializeImage(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::IMAGE);
REQUIRE(deserialized->data_message);
DataMessage &image_array = *deserialized->data_message;
REQUIRE(image_array.image.GetCompressionAlgorithm() == CompressionAlgorithm::NO_COMPRESSION);
REQUIRE(image_array.image.GetWidth() == 1024);
REQUIRE(image_array.image.GetHeight() == 512);
REQUIRE(image_array.image.GetMode() == CompressedImageMode::Float32);
REQUIRE(image_array.image.GetChannel() == "default");
REQUIRE(image_array.image.GetCompressedSize() == test.size() * sizeof(float));
REQUIRE(image_array.indexing_result == message.indexing_result);
REQUIRE(image_array.number == 480);
REQUIRE(memcmp(image_array.image.GetCompressed(), test.data(), test.size() * sizeof(float)) == 0);
}
TEST_CASE("CBORSerialize_Image_Rgb", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
std::vector<SpotToSave> spots;
std::vector<rgb> test(512 * 1024);
for (int i = 0; i < test.size(); i++) {
test[i].r = (i / 256) % 256;
test[i].g = i % 256;
test[i].b = (i / 771) % 256;
}
CompressedImage image(test, 1024, 512);
DataMessage message{
.number = 480,
.image = image,
.spots = spots,
.indexing_result = false
};
REQUIRE_NOTHROW(serializer.SerializeImage(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::IMAGE);
REQUIRE(deserialized->data_message);
DataMessage &image_array = *deserialized->data_message;
REQUIRE(image_array.image.GetCompressionAlgorithm() == CompressionAlgorithm::NO_COMPRESSION);
REQUIRE(image_array.image.GetWidth() == 1024);
REQUIRE(image_array.image.GetHeight() == 512);
REQUIRE(image_array.image.GetMode() == CompressedImageMode::RGB);
REQUIRE(image_array.image.GetChannel() == "default");
REQUIRE(image_array.image.GetCompressedSize() == test.size() * sizeof(rgb));
REQUIRE(image_array.indexing_result == message.indexing_result);
REQUIRE(image_array.number == 480);
REQUIRE(memcmp(image_array.image.GetCompressed(), test.data(), test.size() * sizeof(rgb)) == 0);
}
TEST_CASE("CBORSerialize_Image_Compressed", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
std::vector<SpotToSave> spots;
std::vector<uint8_t> test(512);
for (int i = 0; i < test.size(); i++)
test[i] = (i * 253 + 56) % 256;
CompressedImage image(test, 256, 2, CompressedImageMode::Int32, CompressionAlgorithm::BSHUF_LZ4);
DataMessage message{
.number = 456,
.image = image,
.spots = spots
};
REQUIRE_NOTHROW(serializer.SerializeImage(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::IMAGE);
REQUIRE(deserialized->data_message);
DataMessage &image_array = *deserialized->data_message;
REQUIRE(image_array.image.GetCompressionAlgorithm() == CompressionAlgorithm::BSHUF_LZ4);
REQUIRE(image_array.image.GetWidth() == 256);
REQUIRE(image_array.image.GetHeight() == 2);
REQUIRE(image_array.image.GetMode() == CompressedImageMode::Int32);
REQUIRE(image_array.image.GetChannel() == "default");
REQUIRE(image_array.image.GetCompressedSize() == test.size());
REQUIRE(image_array.number == 456);
REQUIRE(memcmp(image_array.image.GetCompressed(), test.data(), test.size()) == 0);
}
TEST_CASE("CBORSerialize_Image_Rad_Int_Profile", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
std::vector<uint16_t> test(512);
for (int i = 0; i < test.size(); i++)
test[i] = (i * 253 + 56) % 256;
CompressedImage image(test, 256, 2);
DataMessage message{
.number = 789,
.image = image,
.az_int_profile = {4.0, 5.0, 7.0, 12.0, 13.25, 0.125},
.az_int_profile_std = {1.0, 0.0, NAN, NAN, 8.0, 12.0},
.az_int_profile_count = {3,2,0,1,5,7}
};
REQUIRE_NOTHROW(serializer.SerializeImage(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::IMAGE);
REQUIRE(deserialized->data_message);
DataMessage &image_array = *deserialized->data_message;
REQUIRE(image_array.number == 789);
REQUIRE(image_array.image.GetCompressedSize() == test.size() * sizeof(uint16_t));
REQUIRE(memcmp(image_array.image.GetCompressed(), test.data(), test.size() * sizeof(uint16_t)) == 0);
REQUIRE(image_array.az_int_profile == message.az_int_profile);
REQUIRE(image_array.az_int_profile_std.size() == message.az_int_profile_std.size());
for (int i = 0; i < image_array.az_int_profile_std.size(); i++) {
if (std::isnan(message.az_int_profile_std[i]))
CHECK(std::isnan(image_array.az_int_profile_std[i]));
else
CHECK(message.az_int_profile_std[i] == Catch::Approx(image_array.az_int_profile_std[i]));
}
REQUIRE(image_array.az_int_profile_count == message.az_int_profile_count);
}
TEST_CASE("CBORSerialize_Start_AzintMap", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
std::vector<uint16_t> test(512);
for (int i = 0; i < test.size(); i++)
test[i] = (i * 253 + 56) % 256;
std::vector<uint16_t> azint_map(32 * 15, 57);
azint_map[0] = 123;
azint_map[32*14] = UINT16_MAX;
StartMessage msg{
.image_size_x = 32,
.image_size_y = 15,
.az_int_map = azint_map
};
REQUIRE_NOTHROW(serializer.SerializeSequenceStart(msg));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::START);
REQUIRE(deserialized->start_message);
CHECK(deserialized->start_message->az_int_map == azint_map);
}
TEST_CASE("CBORSerialize_Start_ROIMap", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
std::vector<uint16_t> roi_map(32 * 15, 0);
roi_map[0] = (1 << 0) | (1 << 3);
roi_map[32 * 14] = (1 << 2);
StartMessage msg{
.image_size_x = 32,
.image_size_y = 15,
.roi_map = roi_map
};
REQUIRE_NOTHROW(serializer.SerializeSequenceStart(msg));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::START);
REQUIRE(deserialized->start_message);
CHECK(deserialized->start_message->roi_map == roi_map);
}
TEST_CASE("CBORSerialize_Image_Spots", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
std::vector<SpotToSave> spots;
spots.push_back(SpotToSave{.x = 7, .y = 8, .intensity = 34, .indexed = false});
spots.push_back(SpotToSave{.x = 37, .y = 48, .intensity = 123, .indexed = true});
std::vector<uint16_t> test(512);
for (int i = 0; i < test.size(); i++)
test[i] = (i * 253 + 56) % 256;
CompressedImage image(test, 256, 2);
DataMessage message{
.number = 789,
.image = image,
.spots = spots
};
REQUIRE_NOTHROW(serializer.SerializeImage(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::IMAGE);
REQUIRE(deserialized->data_message);
DataMessage &image_array = *deserialized->data_message;
REQUIRE(image_array.number == 789);
REQUIRE(image_array.image.GetCompressedSize() == test.size() * sizeof(uint16_t));
REQUIRE(memcmp(image_array.image.GetCompressed(), test.data(), test.size() * sizeof(uint16_t)) == 0);
REQUIRE(image_array.spots.size() == 2);
REQUIRE(image_array.spots[0].intensity == 34);
REQUIRE(!image_array.spots[0].indexed);
REQUIRE(image_array.spots[1].x == 37);
REQUIRE(image_array.spots[1].y == 48);
REQUIRE(image_array.spots[1].intensity == 123);
REQUIRE(image_array.spots[1].indexed);
}
TEST_CASE("CBORSerialize_Image_Reflections") {
// Prepare a few reflections with distinct values
std::vector<Reflection> refs_in;
// Every float is given a distinct value: a field the encoder forgets reads back as 0 and fails
// here. var_bkg was added to Reflection and to the HDF5 writer but not to the CBOR map, so every
// broker-written file carried a zeroed background_variance until this test was widened to see it.
refs_in.push_back(Reflection{
.h = 1, .k = 0, .l = -1,
.image_number = 789.0f, .delta_phi_deg = 0.125f,
.predicted_x = 1024.5f, .predicted_y = 768.25f,
.observed_x = 1024.75f, .observed_y = 768.5f,
.d = 2.345f, .I = 1234.5f, .bkg = 3.25f, .var_bkg = 41.5f, .sigma = 37.125f,
.dist_ewald = 0.0025f, .prescaling_corr = 1.75f, .qe_corr = 0.9375f, .partiality = 0.875f, .zeta = 0.625f,
.image_scale_corr = 1.125f
});
refs_in.push_back(Reflection{
.h = -3, .k = 2, .l = 7,
.image_number = 790.0f, .delta_phi_deg = -0.25f,
.predicted_x = 5.0f, .predicted_y = 10.0f,
.observed_x = 5.5f, .observed_y = 10.5f,
.d = 4.0f, .I = 12.0f, .bkg = 0.5f, .var_bkg = 6.75f, .sigma = 4.25f,
.dist_ewald = -0.001f, .prescaling_corr = 2.5f, .qe_corr = 0.8125f, .partiality = 0.5f, .zeta = 0.25f,
.image_scale_corr = 0.875f
});
refs_in.push_back(Reflection{
.h = 0, .k = 0, .l = 1,
.image_number = 791.0f, .delta_phi_deg = 0.0f,
.predicted_x = 0.0f, .predicted_y = 0.0f,
.observed_x = 0.25f, .observed_y = 0.5f,
.d = 10.0f, .I = -1.5f, .bkg = 0.125f, .var_bkg = 0.875f, .sigma = 1.5f,
.dist_ewald = 0.0f, .prescaling_corr = 1.0f, .qe_corr = 1.0f, .partiality = 1.0f, .zeta = 1.0f,
.image_scale_corr = 1.0f
});
// Minimal DataMessage carrying reflections
std::vector<uint16_t> test(512);
CompressedImage image(test, 256, 2);
DataMessage msg_in{
.number = 789,
.image = image,
.reflections = refs_in
};
// Serialize
std::vector<uint8_t> buffer(1 << 20); // 1 MB buffer for safety
CBORStream2Serializer ser(buffer.data(), buffer.size());
ser.SerializeImage(msg_in);
// Deserialize
auto deserialized = CBORStream2Deserialize(buffer.data(), ser.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::IMAGE);
REQUIRE(deserialized->data_message);
DataMessage &msg_out = *deserialized->data_message;
// Check reflections
REQUIRE(msg_out.reflections.size() == refs_in.size());
for (size_t i = 0; i < refs_in.size(); ++i) {
const auto &a = refs_in[i];
const auto &b = msg_out.reflections[i];
CHECK(a.h == b.h);
CHECK(a.k == b.k);
CHECK(a.l == b.l);
CHECK(b.predicted_x == Catch::Approx(a.predicted_x).margin(1e-6f));
CHECK(b.predicted_y == Catch::Approx(a.predicted_y).margin(1e-6f));
CHECK(b.observed_x == Catch::Approx(a.observed_x).margin(1e-6f));
CHECK(b.observed_y == Catch::Approx(a.observed_y).margin(1e-6f));
CHECK(b.d == Catch::Approx(a.d).margin(1e-6f));
CHECK(b.image_number == Catch::Approx(a.image_number).margin(1e-6f));
CHECK(b.delta_phi_deg == Catch::Approx(a.delta_phi_deg).margin(1e-6f));
CHECK(b.I == Catch::Approx(a.I).margin(1e-6f));
CHECK(b.bkg == Catch::Approx(a.bkg).margin(1e-6f));
CHECK(b.var_bkg == Catch::Approx(a.var_bkg).margin(1e-6f));
CHECK(b.sigma == Catch::Approx(a.sigma).margin(1e-6f));
CHECK(b.dist_ewald == Catch::Approx(a.dist_ewald).margin(1e-6f));
CHECK(b.prescaling_corr == Catch::Approx(a.prescaling_corr).margin(1e-6f));
CHECK(b.qe_corr == Catch::Approx(a.qe_corr).margin(1e-6f));
CHECK(b.partiality == Catch::Approx(a.partiality).margin(1e-6f));
CHECK(b.zeta == Catch::Approx(a.zeta).margin(1e-6f));
CHECK(b.image_scale_corr == Catch::Approx(a.image_scale_corr).margin(1e-6f));
}
}
TEST_CASE("CBORSerialize_Image_MetadataHeadroom", "[CBOR]") {
// The receiver serializes the per-image CBOR metadata (spots, reflections,
// azimuthal profile, ...) into the same buffer slot as the compressed image, and
// DiffractionExperiment::GetImageBufferLocationSize() reserves
// kImageMetadataHeadroom on top of GetMaxCompressedSize() for it. A frame whose
// metadata exceeds that headroom is dropped by the receiver. This test guards that
// even the largest possible metadata still fits, so ordinary frames are never
// dropped: the kOnlineMaxReflections (10000, BraggPrediction.h) reflection cap, the
// API maximum of 2000 spots (jfjoch_api.yaml), and the 65534-bin azimuthal cap
// (AzimuthalIntegrationMapping, UINT16_MAX - 1), all serialized together. Take the
// headroom from the constant rather than restate it: written down twice, the two
// drifted apart and this test kept passing against the value the code had left.
constexpr size_t max_reflections = 10000;
constexpr size_t max_spots = 2000;
constexpr size_t max_az_bins = 65534;
constexpr size_t headroom = DiffractionExperiment::kImageMetadataHeadroom;
std::vector<Reflection> reflections(max_reflections);
for (size_t i = 0; i < reflections.size(); ++i)
reflections[i] = Reflection{
.h = static_cast<int32_t>(i % 200) - 100,
.k = static_cast<int32_t>(i % 200) - 100,
.l = static_cast<int32_t>(i % 200) - 100,
.image_number = 789.0f, .delta_phi_deg = 0.123f,
.predicted_x = 1234.5f, .predicted_y = 678.9f,
.observed_x = 1234.0f, .observed_y = 678.0f,
.d = 2.345f, .I = 1234.5f, .bkg = 12.3f, .sigma = 35.6f,
.dist_ewald = 0.001f, .prescaling_corr = 0.5f, .partiality = 0.9f,
.zeta = 0.3f, .image_scale_corr = 1.1f};
std::vector<SpotToSave> spots(max_spots);
for (size_t i = 0; i < spots.size(); ++i)
spots[i] = SpotToSave{
.x = 1234.5f, .y = 678.9f, .intensity = 1234.5f, .maxc = 5000,
.lattice = 0, .image = 789,
.h = static_cast<int64_t>(i % 200) - 100,
.k = static_cast<int64_t>(i % 200) - 100,
.l = static_cast<int64_t>(i % 200) - 100,
.dist_ewald_sphere = 0.01f, .indexed = true};
DataMessage message{
.number = 789,
.image = CompressedImage(nullptr, 0, 4000, 4000,
CompressedImageMode::Uint32, CompressionAlgorithm::BSHUF_LZ4),
.spots = spots,
.az_int_profile = std::vector<float>(max_az_bins, 1.0f),
.az_int_profile_std = std::vector<float>(max_az_bins, 1.0f),
.az_int_profile_count = std::vector<uint64_t>(max_az_bins, 1),
.reflections = reflections};
std::vector<uint8_t> buffer(16 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
REQUIRE_NOTHROW(serializer.SerializeImage(message));
// GetImageAppendOffset() is where the compressed image is appended, i.e. the size
// of everything-but-the-image; it must stay below the metadata headroom.
INFO("worst-case metadata size = " << serializer.GetImageAppendOffset() << " B");
CHECK(serializer.GetImageAppendOffset() < headroom);
}
TEST_CASE("CBORSerialize_Image_ROI", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
std::vector<uint16_t> test(512);
for (int i = 0; i < test.size(); i++)
test[i] = (i * 253 + 56) % 256;
CompressedImage image(test, 256, 2);
DataMessage message{
.number = 789,
.image = image,
};
message.roi["roi0"] = {
.sum = -876, .sum_square = 89998, .max_count = -1, .pixels = 4567, .x_weighted = 123, .y_weighted = 122
};
message.roi["roi1"] = {
.sum = 876, .sum_square = 998, .max_count = 12, .pixels = 234, .x_weighted = -11222233443123LL, .y_weighted = -5
};
REQUIRE_NOTHROW(serializer.SerializeImage(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::IMAGE);
REQUIRE(deserialized->data_message);
DataMessage &image_array = *deserialized->data_message;
REQUIRE(image_array.number == 789);
REQUIRE(image_array.roi.size() == 2);
REQUIRE(image_array.roi.contains("roi0"));
REQUIRE(image_array.roi.contains("roi1"));
CHECK(image_array.roi["roi0"].sum == message.roi["roi0"].sum);
CHECK(image_array.roi["roi0"].sum_square == message.roi["roi0"].sum_square);
CHECK(image_array.roi["roi0"].max_count == message.roi["roi0"].max_count);
CHECK(image_array.roi["roi0"].pixels == message.roi["roi0"].pixels);
CHECK(image_array.roi["roi0"].x_weighted == message.roi["roi0"].x_weighted);
CHECK(image_array.roi["roi0"].y_weighted == message.roi["roi0"].y_weighted);
CHECK(image_array.roi["roi1"].sum == message.roi["roi1"].sum);
CHECK(image_array.roi["roi1"].sum_square == message.roi["roi1"].sum_square);
CHECK(image_array.roi["roi1"].max_count == message.roi["roi1"].max_count);
CHECK(image_array.roi["roi1"].pixels == message.roi["roi1"].pixels);
CHECK(image_array.roi["roi1"].x_weighted == message.roi["roi1"].x_weighted);
CHECK(image_array.roi["roi1"].y_weighted == message.roi["roi1"].y_weighted);
}
TEST_CASE("CBORSerialize_Image_Append", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
std::vector<uint8_t> test(512 * 1024);
for (int i = 0; i < test.size(); i++)
test[i] = (i * 253 + 56) % 256;
CompressedImage image(nullptr, 0, 1024, 512, CompressedImageMode::Uint8, CompressionAlgorithm::BSHUF_LZ4);
DataMessage message{
.number = 480,
.image = image
};
REQUIRE_NOTHROW(serializer.SerializeImage(message));
memcpy(buffer.data() + serializer.GetImageAppendOffset(), test.data(), 512 * 1024);
//REQUIRE_THROWS(serializer.AppendImage(16*1024*1024));
REQUIRE_NOTHROW(serializer.AppendImage(512 * 1024));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::IMAGE);
REQUIRE(deserialized->data_message);
DataMessage &image_array = *deserialized->data_message;
REQUIRE(image_array.image.GetCompressionAlgorithm() == CompressionAlgorithm::BSHUF_LZ4);
REQUIRE(image_array.image.GetWidth() == 1024);
REQUIRE(image_array.image.GetHeight() == 512);
REQUIRE(image_array.image.GetMode() == CompressedImageMode::Uint8);
REQUIRE(image_array.image.GetChannel() == "default");
REQUIRE(image_array.image.GetCompressedSize() == test.size());
REQUIRE(image_array.indexing_result == message.indexing_result);
REQUIRE(image_array.number == 480);
REQUIRE(memcmp(image_array.image.GetCompressed(), test.data(), test.size()) == 0);
}
TEST_CASE("CBORSerialize_Metadata", "[CBOR]") {
MetadataMessage msgs;
CompressedImage image(nullptr, 0, 123, 145, CompressedImageMode::Uint8,
CompressionAlgorithm::BSHUF_LZ4);
msgs.images.push_back(DataMessage{.number = 172, .image = image, .bkg_estimate = 45});
msgs.images.push_back(DataMessage{.number = 173, .image = image, .bkg_estimate = 48});
//msgs.push_back(DataMessage{.number = 174, .image = CompressedImage{.GetWidth() = 123, .GetHeight() = 145}, .bkg_estimate = 48});
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
REQUIRE_NOTHROW(serializer.SerializeMetadata(msgs));
REQUIRE(serializer.GetBufferSize() > 0);
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::METADATA);
REQUIRE(deserialized->metadata);
REQUIRE(deserialized->metadata->images.size() == 2);
CHECK(deserialized->metadata->images.at(0).number == 172);
CHECK(deserialized->metadata->images.at(0).bkg_estimate == 45);
CHECK(!deserialized->metadata->images.at(0).spindle_blind_fraction.has_value());
CHECK(deserialized->metadata->images.at(0).image.GetWidth() == 0);
CHECK(deserialized->metadata->images.at(1).number == 173);
CHECK(deserialized->metadata->images.at(1).bkg_estimate == 48);
CHECK(deserialized->metadata->images.at(1).image.GetWidth() == 0);
}
TEST_CASE("CBORSerialize_Image_LatticeType", "[CBOR][Lattice]") {
std::vector<uint8_t> buffer(1 << 20);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
// Prepare an image with lattice_type set
std::vector<uint16_t> pixels(16, 0);
CompressedImage img(pixels, 4, 4);
DataMessage in_msg{
.number = 1,
.image = img
};
// Case 1: Tetragonal, body-centered -> expect same after round trip
in_msg.lattice_type = LatticeMessage{
.centering = 'I',
.niggli_class = 25,
.crystal_system = gemmi::CrystalSystem::Tetragonal
};
REQUIRE_NOTHROW(serializer.SerializeImage(in_msg));
auto out = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(out);
REQUIRE(out->msg_type == CBORImageType::IMAGE);
REQUIRE(out->data_message);
// LatticeMessage present
REQUIRE(out->data_message->lattice_type.has_value());
const auto lm = out->data_message->lattice_type.value();
CHECK(lm.centering == 'I');
CHECK(lm.niggli_class == 25);
CHECK(lm.crystal_system == gemmi::CrystalSystem::Tetragonal);
// Case 2: Absent lattice_type should remain absent
CBORStream2Serializer serializer2(buffer.data(), buffer.size());
DataMessage in_msg2{
.number = 2,
.image = img
};
REQUIRE_NOTHROW(serializer2.SerializeImage(in_msg2));
auto out2 = CBORStream2Deserialize(buffer.data(), serializer2.GetBufferSize());
REQUIRE(out2);
REQUIRE(out2->msg_type == CBORImageType::IMAGE);
REQUIRE(out2->data_message);
CHECK_FALSE(out2->data_message->lattice_type.has_value());
// Case 3: Various systems/centerings sanity round-trip
struct Case {
gemmi::CrystalSystem cs;
char ctr;
};
std::vector<Case> cases = {
{gemmi::CrystalSystem::Triclinic, 'P'},
{gemmi::CrystalSystem::Monoclinic, 'C'},
{gemmi::CrystalSystem::Orthorhombic, 'F'},
{gemmi::CrystalSystem::Orthorhombic, 'A'},
{gemmi::CrystalSystem::Tetragonal, 'P'},
{gemmi::CrystalSystem::Trigonal, 'R'},
{gemmi::CrystalSystem::Hexagonal, 'P'},
{gemmi::CrystalSystem::Cubic, 'I'},
};
int i = 0;
for (const auto &c: cases) {
CBORStream2Serializer ser(buffer.data(), buffer.size());
DataMessage m{
.number = 3,
.image = img
};
m.lattice_type = LatticeMessage{.centering = c.ctr, .niggli_class = i, .crystal_system = c.cs};
REQUIRE_NOTHROW(ser.SerializeImage(m));
auto d = CBORStream2Deserialize(buffer.data(), ser.GetBufferSize());
REQUIRE(d);
REQUIRE(d->msg_type == CBORImageType::IMAGE);
REQUIRE(d->data_message);
REQUIRE(d->data_message->lattice_type.has_value());
auto l = d->data_message->lattice_type.value();
CHECK(l.centering == c.ctr);
CHECK(l.niggli_class == i);
CHECK(l.crystal_system == c.cs);
i++;
}
}
// The chain is sent as an ARRAY because the mounting order has to survive, and a CBOR map's order
// carries no guarantee - RFC 8949 requires deterministic encoders to sort map keys. This asserts the
// order round trips, not merely the contents, and that a moving axis keeps one value per image
// while a stationary one keeps a single value.
TEST_CASE("CBORSerialize_End_Transformations", "[CBOR]") {
EndMessage message{};
message.max_image_number = 4;
message.transformations = {
DetectorTransformation("grid_scan_x", TransformationType::Translation, {1, 0, 0})
.Values({0.0f, 1e-4f, 2e-4f, 3e-4f}),
DetectorTransformation("omega", TransformationType::Rotation, {-1, 0, 0})
.DependsOn("grid_scan_x").Values({95.0f, 95.1f, 95.2f, 95.3f}),
DetectorTransformation("chi", TransformationType::Rotation, {0, 0, 1})
.DependsOn("omega").Value(12.5f),
DetectorTransformation("phi", TransformationType::Rotation, {1, 0, 0})
.DependsOn("chi").Value(-7.25f).Offset({1, 2, 3}),
};
std::vector<uint8_t> buffer(MESSAGE_SIZE_FOR_START_END);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
REQUIRE_NOTHROW(serializer.SerializeSequenceEnd(message));
buffer.resize(serializer.GetBufferSize());
auto output = CBORStream2Deserialize(buffer.data(), buffer.size());
REQUIRE(output->end_message.has_value());
const auto &chain = output->end_message->transformations;
REQUIRE(chain.size() == message.transformations.size());
for (size_t i = 0; i < chain.size(); i++) {
const auto &got = chain[i];
const auto &want = message.transformations[i];
CHECK(got.GetName() == want.GetName());
CHECK(got.IsRotation() == want.IsRotation());
CHECK(got.GetUnits() == want.GetUnits());
CHECK(got.GetDependsOn() == want.GetDependsOn());
CHECK(got.GetVector().x == Catch::Approx(want.GetVector().x));
CHECK(got.GetVector().y == Catch::Approx(want.GetVector().y));
CHECK(got.GetVector().z == Catch::Approx(want.GetVector().z));
CHECK(got.GetOffset().x == Catch::Approx(want.GetOffset().x));
CHECK(got.IsConstant() == want.IsConstant());
REQUIRE(got.GetValues().size() == want.GetValues().size());
for (size_t j = 0; j < got.GetValues().size(); j++)
CHECK(got.GetValues()[j] == Catch::Approx(want.GetValues()[j]).margin(1e-4));
}
// The base of the chain names no parent.
CHECK(chain[0].GetDependsOn().empty());
CHECK(!chain[1].IsConstant());
CHECK(chain[2].IsConstant());
}
Reference in New Issue View Git Blame Copy Permalink