mx/Jungfraujoch
2
1
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases 73 Wiki Activity
Files
55d9e2536f343452d9b294cb5a39b440fc790dce
Jungfraujoch/tests/CBORTest.cpp
T
leonarski_fandClaude Opus 5 26fc4b02b3 symmetry: carry the space group as the group, not as its number 
The adopted space group travelled the pipeline as a bare int and was rebuilt
downstream with find_spacegroup_by_number, which returns the reference setting.
So every setting a number cannot name was destroyed one line after it was
determined: P 1 1 2 came back as P 1 2 1, I 1 1 2 as C 1 2 1, R 3:R as R 3:H.

DatasetSettings now holds the gemmi::SpaceGroup itself, DiffractionExperiment
exposes it as GetGemmiSpaceGroup() / GetSpaceGroupOrP1(), and everything that
used to take an int - HKLKeyGenerator (its int constructor is gone, so the
compiler finds the callers), the merge, the R-free flags, French-Wilson, the
reindexing ambiguity, the completeness enumeration, the MTZ and mmCIF exports,
the model validation - takes the group. -S keeps the setting the symbol names
rather than reducing it to a number.

The end message carries both spellings and a reader prefers the name, since
only the name keeps the setting while the number is what a reader written
before the name understands. It carries them over CBOR too: the determined
group was never serialised at all, so a group rugnux chose reached the master
file only when the same process wrote it, and an online writer fell back to
whatever the user had supplied at the start. Both keys are optional additions,
so an older reader skips them and a newer one reads an older sender.

On disk the master's /entry/sample/space_group carries the extended
Hermann-Mauguin name and is what the reader takes the group from, so a setting
survives a _process.h5 and the --mode scale that re-reads it; the number stays
beside it and is the fallback for files written before. Every one of the 230
reference settings the old writer could produce reads back as itself, so older
files are unaffected.

Stage A and Stage B of the search still enumerate reference settings only, so
this determines no group differently today - it is what the enumeration needs
before it can be widened.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EFEJG6WBQv8th4UJFNe53N
2026-08-31 07:16:43 +02:00

1417 lines
60 KiB
C++
Raw Blame History

// SPDX-FileCopyrightText: 2024 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include <catch2/catch_all.hpp>
#include "../frame_serialize/CBORStream2Serializer.h"
#include "../frame_serialize/CBORStream2Deserializer.h"
#include "../compression/JFJochCompressor.h"
#include "../frame_serialize/CborUtil.h"
#include "../common/DiffractionExperiment.h" // kImageMetadataHeadroom
TEST_CASE("CBORSerialize_Start", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
StartMessage message{
.detector_distance = 0.0005,
.beam_center_x = 456.6,
.beam_center_y = 124.3,
.number_of_images = 34567,
.image_size_x = 456,
.image_size_y = 457,
.bit_depth_image = 32,
.bit_depth_readout = 16,
.pixel_signed = true,
.countrate_correction_enabled = true,
.incident_energy = 12400,
.incident_wavelength = 0.988,
.frame_time = 0.0001,
.count_time = 0.000098,
.saturation_value = 65534,
.error_value = 65535,
.pixel_size_x = 0.000075,
.pixel_size_y = 0.000075,
.sensor_thickness = 0.0005,
.sensor_material = "Si",
.unit_cell = UnitCell{.a = 45, .b = 37, .c = 45, .alpha = 90, .beta = 108, . gamma = 120},
.space_group_number = 154,
.max_spot_count = 250,
.storage_cell_number = 16,
.storage_cell_delay_ns = 15345,
.pixel_mask_enabled = true,
.arm_date = "abc",
.sample_name = "test_sample",
.file_prefix = "test_sample/dir/file",
.images_per_file = 12345,
.channels = {"default", "sc2"},
.detector_description = "EIGER 16M",
.detector_serial_number = "123",
.run_name = "bla",
.run_number = 4567,
.gain_file_names = {"abc", "def", "/dsadasdsa/dadsadas/dsadsa/M056.bin"},
.detector_translation = {0.5f, 0.0f, 0.5f},
.source_type = "Synchrotron X-ray Source",
.source_name = "Swiss Light Source",
.instrument_name = "X06SA",
.summation = 567,
.az_int_bin_to_q = {0.1, 0.2, 0.3, 0.5},
.az_int_q_bin_count = 35,
.az_int_phi_bin_count = 120,
.total_flux = 123,
.attenuator_transmission = 0.345,
.write_master_file = true,
.user_data = R"({"pi":3.1415, "z":"string"})"_json,
.data_reduction_factor_serialmx = 0.75,
.experiment_group = "p10001",
.jfjoch_release = "1.4.98",
.socket_number = 3,
.writer_notification_zmq_addr = "tcp://1.2.3.4:5678",
.jungfrau_conversion_enabled = true,
.jungfrau_conversion_factor = 17.56f,
.geometry_transformation_enabled = false,
.overwrite = true,
.file_format = FileWriterFormat::NXmxVDS,
.ring_current_mA = 123,
.sample_temperature_K = 345,
.indexing_algorithm = IndexingAlgorithmEnum::FFT
};
REQUIRE_NOTHROW(serializer.SerializeSequenceStart(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::START);
REQUIRE(deserialized->start_message);
StartMessage &output_message = *deserialized->start_message;
CHECK(output_message.images_per_file == message.images_per_file);
CHECK(output_message.detector_distance == Catch::Approx(message.detector_distance));
CHECK(output_message.beam_center_x == Catch::Approx(message.beam_center_x));
CHECK(output_message.beam_center_y == Catch::Approx(message.beam_center_y));
CHECK(output_message.number_of_images == message.number_of_images);
CHECK(output_message.image_size_x == message.image_size_x);
CHECK(output_message.image_size_y == message.image_size_y);
CHECK(output_message.bit_depth_image == message.bit_depth_image);
CHECK(output_message.bit_depth_readout == message.bit_depth_readout);
CHECK(output_message.incident_energy == Catch::Approx(message.incident_energy));
CHECK(output_message.incident_wavelength == Catch::Approx(message.incident_wavelength));
CHECK(output_message.frame_time == Catch::Approx(message.frame_time));
CHECK(output_message.count_time == Catch::Approx(message.count_time));
CHECK(output_message.saturation_value == message.saturation_value);
CHECK(output_message.error_value == message.error_value);
CHECK(output_message.pixel_size_x == Catch::Approx(message.pixel_size_x));
CHECK(output_message.pixel_size_y == Catch::Approx(message.pixel_size_y));
CHECK(output_message.sensor_thickness == Catch::Approx(message.sensor_thickness));
CHECK(output_message.sensor_material == message.sensor_material);
CHECK(output_message.pixel_mask_enabled == message.pixel_mask_enabled);
CHECK(output_message.space_group_number == message.space_group_number);
CHECK(output_message.arm_date == message.arm_date);
CHECK(output_message.storage_cell_number == message.storage_cell_number);
CHECK(output_message.storage_cell_delay_ns == message.storage_cell_delay_ns);
CHECK(output_message.pixel_signed == message.pixel_signed);
CHECK(output_message.sample_name == message.sample_name);
CHECK(output_message.file_prefix == message.file_prefix);
CHECK(output_message.max_spot_count == message.max_spot_count);
CHECK(output_message.channels == message.channels);
CHECK(output_message.detector_description == message.detector_description);
CHECK(output_message.detector_serial_number == message.detector_serial_number);
CHECK(output_message.run_name == message.run_name);
CHECK(output_message.run_number == message.run_number);
CHECK(output_message.source_type == message.source_type);
CHECK(output_message.source_name == message.source_name);
CHECK(output_message.instrument_name == message.instrument_name);
CHECK(output_message.az_int_q_bin_count == message.az_int_q_bin_count);
CHECK(output_message.az_int_phi_bin_count == message.az_int_phi_bin_count);
CHECK(output_message.summation == message.summation);
CHECK(output_message.az_int_bin_to_q == message.az_int_bin_to_q);
for (int i = 0; i < 3; i++)
CHECK(output_message.detector_translation[i] == message.detector_translation[i]);
CHECK(output_message.unit_cell);
CHECK(output_message.unit_cell->a == message.unit_cell->a);
CHECK(output_message.unit_cell->b == message.unit_cell->b);
CHECK(output_message.unit_cell->c == message.unit_cell->c);
CHECK(output_message.unit_cell->alpha == message.unit_cell->alpha);
CHECK(output_message.unit_cell->beta == message.unit_cell->beta);
CHECK(output_message.unit_cell->gamma == message.unit_cell->gamma);
REQUIRE(output_message.total_flux);
CHECK(output_message.total_flux.value() == message.total_flux.value());
REQUIRE(output_message.attenuator_transmission);
CHECK(output_message.attenuator_transmission.value() == message.attenuator_transmission.value());
CHECK(output_message.user_data == message.user_data);
REQUIRE(output_message.user_data.is_object());
CHECK(output_message.user_data.size() == 2);
CHECK(output_message.gain_file_names == message.gain_file_names);
CHECK(output_message.countrate_correction_enabled == message.countrate_correction_enabled);
CHECK(output_message.flatfield_enabled == message.flatfield_enabled);
CHECK(output_message.write_master_file == message.write_master_file);
CHECK(output_message.data_reduction_factor_serialmx == message.data_reduction_factor_serialmx);
CHECK(output_message.experiment_group == message.experiment_group);
CHECK(output_message.jfjoch_release == message.jfjoch_release);
CHECK(output_message.socket_number == message.socket_number);
CHECK(output_message.writer_notification_zmq_addr == message.writer_notification_zmq_addr);
CHECK(output_message.geometry_transformation_enabled == message.geometry_transformation_enabled);
CHECK(output_message.jungfrau_conversion_enabled == message.jungfrau_conversion_enabled);
CHECK(output_message.jungfrau_conversion_factor == message.jungfrau_conversion_factor);
CHECK(output_message.overwrite == message.overwrite);
CHECK(output_message.file_format == message.file_format);
CHECK(output_message.sample_temperature_K == message.sample_temperature_K);
CHECK(output_message.ring_current_mA == message.ring_current_mA);
CHECK(output_message.indexing_algorithm == message.indexing_algorithm);
}
TEST_CASE("CBORSerialize_Start_GoniometerAxis", "[CBOR]") {
GoniometerAxis axis("z", 115, 0.456, {1, 2, 3}, Coord{50, 20, 30});
axis.ScreeningWedge(0.01);
StartMessage message{
.goniometer = axis
};
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
REQUIRE_NOTHROW(serializer.SerializeSequenceStart(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::START);
REQUIRE(deserialized->start_message);
StartMessage &output_message = *deserialized->start_message;
REQUIRE(output_message.goniometer.has_value());
CHECK(output_message.goniometer->GetName() == axis.GetName());
CHECK(output_message.goniometer->GetStart_deg() == axis.GetStart_deg());
CHECK(output_message.goniometer->GetIncrement_deg() == axis.GetIncrement_deg());
CHECK(output_message.goniometer->GetAxis().x == Catch::Approx(axis.GetAxis().x));
CHECK(output_message.goniometer->GetAxis().y == Catch::Approx(axis.GetAxis().y));
CHECK(output_message.goniometer->GetAxis().z == Catch::Approx(axis.GetAxis().z));
REQUIRE(output_message.goniometer->GetHelicalStep());
CHECK(output_message.goniometer->GetHelicalStep() == axis.GetHelicalStep());
REQUIRE(output_message.goniometer->GetScreeningWedge());
CHECK(output_message.goniometer->GetScreeningWedge() == axis.GetScreeningWedge());
}
TEST_CASE("CBORSerialize_Start_GridScan", "[CBOR]") {
GridScanSettings grid(123, 0.1, 34.0, true, true);
StartMessage message{
.grid_scan = grid
};
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
REQUIRE_NOTHROW(serializer.SerializeSequenceStart(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::START);
REQUIRE(deserialized->start_message);
StartMessage &output_message = *deserialized->start_message;
REQUIRE(!output_message.goniometer.has_value());
REQUIRE(output_message.grid_scan.has_value());
CHECK(output_message.grid_scan->GetNFast() == grid.GetNFast());
CHECK(output_message.grid_scan->IsVerticalScan() == grid.IsVerticalScan());
CHECK(output_message.grid_scan->IsSnakeScan() == grid.IsSnakeScan());
CHECK(output_message.grid_scan->GetGridElemFast_um() == Catch::Approx(grid.GetGridElemFast_um()));
CHECK(output_message.grid_scan->GetGridElemSlow_um() == Catch::Approx(grid.GetGridElemSlow_um()));
}
TEST_CASE("CBORSerialize_Start_ThresholdEnergy", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
StartMessage message{};
message.threshold_energy["thr1"] = 12500;
message.threshold_energy["thr2"] = 17000;
REQUIRE_NOTHROW(serializer.SerializeSequenceStart(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::START);
REQUIRE(deserialized->start_message);
StartMessage &output_message = *deserialized->start_message;
REQUIRE(output_message.threshold_energy.size() == 2);
REQUIRE(output_message.threshold_energy == message.threshold_energy);
}
TEST_CASE("CBORSerialize_Start_Fluorescence", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
StartMessage message{};
// prepare fluorescence spectrum
std::vector<float> energy{1.0f, 2.0f, 3.0f};
std::vector<float> data{10.0f, 20.0f, 15.0f};
message.fluorescence_spectrum = XrayFluorescenceSpectrum(energy, data);
REQUIRE_NOTHROW(serializer.SerializeSequenceStart(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::START);
REQUIRE(deserialized->start_message);
const StartMessage &out = *deserialized->start_message;
REQUIRE(!out.fluorescence_spectrum.empty());
CHECK(out.fluorescence_spectrum.GetEnergy_eV() == energy);
CHECK(out.fluorescence_spectrum.GetData() == data);
}
TEST_CASE("CBORSerialize_Start_Smargon", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
StartMessage message{};
SmargonPosition smargon;
smargon.phi_deg = 45.0f;
smargon.chi_deg = 30.0f;
smargon.phi_axis = {0, -1, 0};
smargon.chi_axis = {0, 0, 1};
message.smargon_position = smargon;
REQUIRE_NOTHROW(serializer.SerializeSequenceStart(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::START);
REQUIRE(deserialized->start_message);
const StartMessage &out = *deserialized->start_message;
REQUIRE(out.smargon_position.has_value());
CHECK(out.smargon_position->phi_deg == 45.0f);
CHECK(out.smargon_position->chi_deg == 30.0f);
CHECK(out.smargon_position->phi_axis == Coord{0, -1, 0});
CHECK(out.smargon_position->chi_axis == Coord{0, 0, 1});
}
TEST_CASE("CBORSerialize_ROI", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
StartMessage message{
.rois = {
ROIConfig{
.type = ROIConfig::ROIType::Circle,
.name = "roi1",
.circle = ROIConfigCircle{
.r = 3.0,
.x = 5.0,
.y = 4.0
}
},
ROIConfig{
.type = ROIConfig::ROIType::Box,
.name = "roi2",
.box = ROIConfigBox{
.xmin = 5,
.xmax = 12,
.ymin = 7,
.ymax = 3
}
},
ROIConfig{
.type = ROIConfig::ROIType::Azim,
.name = "roi3",
.azim = ROIConfigAzim{
.qmin = 4.0,
.qmax = 5.0,
.phi_min = 30.0f,
.phi_max = 90.0f
}
},
ROIConfig{
.type = ROIConfig::ROIType::Azim,
.name = "roi4",
.azim = ROIConfigAzim{
.qmin = 1.0,
.qmax = 2.0,
.phi_min = 0.0f,
.phi_max = 0.0f
}
}
}
};
REQUIRE_NOTHROW(serializer.SerializeSequenceStart(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::START);
REQUIRE(deserialized->start_message);
StartMessage &output_message = *deserialized->start_message;
REQUIRE(output_message.rois.size() == 4);
for (int i = 0; i < 4; i++) {
CHECK(output_message.rois[i].name == message.rois[i].name);
CHECK(output_message.rois[i].type == message.rois[i].type);
}
CHECK(output_message.rois[0].circle.x == message.rois[0].circle.x);
CHECK(output_message.rois[0].circle.y == message.rois[0].circle.y);
CHECK(output_message.rois[0].circle.r == message.rois[0].circle.r);
CHECK(output_message.rois[1].box.xmin == message.rois[1].box.xmin);
CHECK(output_message.rois[1].box.xmax == message.rois[1].box.xmax);
CHECK(output_message.rois[1].box.ymin == message.rois[1].box.ymin);
CHECK(output_message.rois[1].box.ymax == message.rois[1].box.ymax);
CHECK(output_message.rois[2].azim.qmin == message.rois[2].azim.qmin);
CHECK(output_message.rois[2].azim.qmax == message.rois[2].azim.qmax);
CHECK(output_message.rois[2].azim.phi_min == 30.0f);
CHECK(output_message.rois[2].azim.phi_max == 90.0f);
// Full ring: phi_min == phi_max round-trips (no sector emitted)
CHECK(output_message.rois[3].azim.phi_min == output_message.rois[3].azim.phi_max);
}
TEST_CASE("CBORSerialize_Start_EmptyString", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
StartMessage message{
.sample_name = "",
.file_prefix = ""
};
REQUIRE_NOTHROW(serializer.SerializeSequenceStart(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::START);
REQUIRE(deserialized->start_message);
StartMessage &output_message = *deserialized->start_message;
CHECK(output_message.file_prefix.empty());
CHECK(output_message.sample_name.empty());
CHECK(output_message.arm_date.empty());
}
TEST_CASE("CBORSerialize_Start_PixelMask", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
std::vector<uint32_t> mask_0(456 * 457, 15);
std::vector<uint32_t> mask_1(456 * 457, 12);
StartMessage message{
.image_size_x = 456,
.image_size_y = 457,
.pixel_mask = {{"default", mask_0}, {"mask_1", mask_1}}
};
REQUIRE_NOTHROW(serializer.SerializeSequenceStart(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::START);
REQUIRE(deserialized->start_message);
StartMessage &output_message = *deserialized->start_message;
REQUIRE(output_message.pixel_mask.size() == 2);
REQUIRE(output_message.pixel_mask.contains("default"));
REQUIRE(output_message.pixel_mask.contains("mask_1"));
REQUIRE(output_message.pixel_mask["default"].size() == 456*457);
REQUIRE(output_message.pixel_mask["mask_1"].size() == 456*457);
REQUIRE(output_message.pixel_mask["default"] == mask_0);
REQUIRE(output_message.pixel_mask["mask_1"] == mask_1);
}
TEST_CASE("CBORSerialize_Calibration", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
std::vector<float> calib1(256);
for (int i = 0; i < 256; i++) {
calib1[i] = i * 34.567;
}
CompressedImage image1(calib1, 16, 16);
image1.Channel("calib1");
REQUIRE_NOTHROW(serializer.SerializeCalibration(image1));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::CALIBRATION);
REQUIRE(deserialized->calibration);
CompressedImage &output_message = *deserialized->calibration;
CHECK(output_message.GetMode() == CompressedImageMode::Float32);
CHECK(output_message.GetWidth() == 16);
CHECK(output_message.GetHeight() == 16);
CHECK(output_message.GetChannel() == "calib1");
CHECK(memcmp(output_message.GetCompressed(), calib1.data(), 256 * sizeof(float)) == 0);
}
TEST_CASE("CBORSerialize_End", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
EndMessage message{
.max_image_number = 57789,
.images_collected_count = 50000,
.images_sent_to_write_count = 40000,
.max_receiver_delay = 3456,
.efficiency = 0.99,
.end_date = "ccc",
.run_name = "bla5",
.run_number = 45676782,
.rotation_lattice_type = LatticeMessage{
.centering = 'R',
.niggli_class = 22,
.crystal_system = gemmi::CrystalSystem::Tetragonal,
},
.rotation_lattice = CrystalLattice(40, 50, 60, 90, 90, 90)
};
REQUIRE_NOTHROW(serializer.SerializeSequenceEnd(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::END);
REQUIRE(deserialized->end_message);
EndMessage &output_message = *deserialized->end_message;
REQUIRE(output_message.max_receiver_delay == message.max_receiver_delay);
REQUIRE(output_message.max_image_number == message.max_image_number);
REQUIRE(output_message.images_collected_count == message.images_collected_count);
REQUIRE(output_message.images_sent_to_write_count == message.images_sent_to_write_count);
REQUIRE(output_message.efficiency);
REQUIRE(output_message.efficiency == Catch::Approx(message.efficiency.value()));
REQUIRE(output_message.end_date == message.end_date);
REQUIRE(output_message.run_number == message.run_number);
REQUIRE(output_message.run_name == message.run_name);
REQUIRE(output_message.az_int_result.empty());
REQUIRE(output_message.rotation_lattice_type.has_value());
CHECK(output_message.rotation_lattice_type->centering == 'R');
CHECK(output_message.rotation_lattice_type->crystal_system == gemmi::CrystalSystem::Tetragonal);
CHECK(output_message.rotation_lattice_type->niggli_class == 22);
REQUIRE(output_message.rotation_lattice.has_value());
CHECK(output_message.rotation_lattice->GetUnitCell().c == Catch::Approx(60.0));
}
TEST_CASE("CBORSerialize_End_SpaceGroup", "[CBOR]") {
// The determined group travels as both spellings. Only the name keeps the setting, so a
// non-reference setting has to survive the round trip as itself and not as its number's group.
const gemmi::SpaceGroup *sg = gemmi::find_spacegroup_by_name("P 1 1 2");
REQUIRE(sg != nullptr);
REQUIRE(gemmi::find_spacegroup_by_number(sg->number)->xhm() != sg->xhm());
std::vector<uint8_t> buffer(1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
EndMessage message{};
message.max_image_number = 0;
message.space_group_name = sg->xhm();
message.space_group_number = static_cast<uint64_t>(sg->number);
REQUIRE_NOTHROW(serializer.SerializeSequenceEnd(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->end_message);
const EndMessage &out = *deserialized->end_message;
REQUIRE(out.space_group_name.has_value());
CHECK(*out.space_group_name == sg->xhm());
REQUIRE(out.space_group_number.has_value());
CHECK(*out.space_group_number == static_cast<uint64_t>(sg->number));
}
TEST_CASE("CBORSerialize_End_SpaceGroupNumberOnly", "[CBOR]") {
// A sender written before the name existed fills in only the number; the message still arrives
// and the missing name is absent rather than empty, so a reader can tell it apart.
std::vector<uint8_t> buffer(1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
EndMessage message{};
message.max_image_number = 0;
message.space_group_number = 96;
REQUIRE_NOTHROW(serializer.SerializeSequenceEnd(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->end_message);
const EndMessage &out = *deserialized->end_message;
CHECK_FALSE(out.space_group_name.has_value());
REQUIRE(out.space_group_number.has_value());
CHECK(*out.space_group_number == 96);
}
TEST_CASE("CBORSerialize_End_RadIntResult", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
EndMessage message{
.max_image_number = 57789,
.max_receiver_delay = 3456,
.efficiency = 0.99,
.end_date = "ccc",
.run_name = "bla5",
.run_number = 45676782
};
message.az_int_result["avg"] = {11.0, 12.0, 13.0};
message.az_int_result["file0"] = {56.0, 75.0, 34.0};
REQUIRE_NOTHROW(serializer.SerializeSequenceEnd(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::END);
EndMessage &output_message = *deserialized->end_message;
REQUIRE(output_message.az_int_result.size() == 2);
REQUIRE(output_message.az_int_result.contains("avg"));
REQUIRE(output_message.az_int_result.contains("file0"));
CHECK(message.az_int_result["avg"] == output_message.az_int_result["avg"]);
CHECK(message.az_int_result["file0"] == output_message.az_int_result["file0"]);
}
TEST_CASE("CBORSerialize_End_ADUHistogram", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
EndMessage message{
.max_image_number = 57789,
.max_receiver_delay = 3456,
.efficiency = 0.99,
.end_date = "ccc",
.run_name = "bla5",
.run_number = 45676782,
.adu_histogram_bin_width = 55
};
message.adu_histogram["avg"] = {11, 12, 13};
message.adu_histogram["file0"] = {56, 75, 34};
REQUIRE_NOTHROW(serializer.SerializeSequenceEnd(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::END);
REQUIRE(deserialized->end_message);
EndMessage &output_message = *deserialized->end_message;
REQUIRE(output_message.adu_histogram.size() == 2);
REQUIRE(output_message.adu_histogram.contains("avg"));
REQUIRE(output_message.adu_histogram.contains("file0"));
CHECK(message.adu_histogram["avg"] == output_message.adu_histogram["avg"]);
CHECK(message.adu_histogram["file0"] == output_message.adu_histogram["file0"]);
CHECK(message.adu_histogram_bin_width == output_message.adu_histogram_bin_width);
}
TEST_CASE("CBORSerialize_Image", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
std::vector<SpotToSave> spots;
std::vector<uint16_t> test(512);
for (int i = 0; i < test.size(); i++)
test[i] = (i * 253 + 56) % 256;
CompressedImage image(test, 256, 2);
DataMessage message{
.number = 456,
.image = image,
.image_collection_efficiency = 0.11,
.spots = spots,
.spot_count_ice_rings = 157,
.bkg_estimate = 12.345f,
.indexing_result = true,
.indexing_unit_cell = UnitCell{.a = 123, .b = 145, .c = 67.5, .alpha = 90, .beta = 120, .gamma = 134},
.adu_histogram = {3, 4, 5, 8},
.timestamp = 1ul << 27 | 1ul << 35,
.exptime = 1000,
.run_name = "bla2",
.run_number = 4567678,
.saturated_pixel_count = 378,
.error_pixel_count = 123,
.strong_pixel_count = 1234,
.min_viable_pixel_value = 123,
.max_viable_pixel_value = 6789,
.user_data = R"({"pi":3.1415, "z":"string"})"_json,
.jf_info = UINT32_MAX,
.receiver_aq_dev_delay = 2323,
.storage_cell = 0xF,
.xfel_pulse_id = UINT64_MAX - 5678,
.xfel_event_code = UINT64_MAX - 123,
.original_number = 12789
};
REQUIRE_NOTHROW(serializer.SerializeImage(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::IMAGE);
REQUIRE(deserialized->data_message);
DataMessage &image_array = *deserialized->data_message;
REQUIRE(image_array.image.GetCompressionAlgorithm() == CompressionAlgorithm::NO_COMPRESSION);
REQUIRE(image_array.image.GetWidth() == 256);
REQUIRE(image_array.image.GetHeight() == 2);
REQUIRE(image_array.image.GetMode() == CompressedImageMode::Uint16);
REQUIRE(image_array.image.GetChannel() == "default");
REQUIRE(image_array.image.GetCompressedSize() == test.size() * sizeof(uint16_t));
REQUIRE(image_array.indexing_result == message.indexing_result);
REQUIRE(image_array.number == 456);
REQUIRE(image_array.run_number == message.run_number);
REQUIRE(image_array.run_name == message.run_name);
REQUIRE(memcmp(image_array.image.GetCompressed(), test.data(), test.size() * sizeof(uint16_t)) == 0);
REQUIRE(image_array.xfel_pulse_id == message.xfel_pulse_id);
REQUIRE(image_array.xfel_event_code == message.xfel_event_code);
REQUIRE(image_array.jf_info == message.jf_info);
REQUIRE(image_array.timestamp == message.timestamp);
REQUIRE(image_array.storage_cell == message.storage_cell);
REQUIRE(image_array.exptime == message.exptime);
REQUIRE(image_array.receiver_aq_dev_delay == image_array.receiver_aq_dev_delay);
REQUIRE(image_array.adu_histogram == message.adu_histogram);
REQUIRE(image_array.saturated_pixel_count == message.saturated_pixel_count);
REQUIRE(image_array.error_pixel_count == message.error_pixel_count);
REQUIRE(image_array.strong_pixel_count == message.strong_pixel_count);
REQUIRE(image_array.bkg_estimate == message.bkg_estimate);
REQUIRE(image_array.image_collection_efficiency == message.image_collection_efficiency);
REQUIRE(image_array.user_data == message.user_data);
REQUIRE(image_array.original_number == message.original_number);
REQUIRE(image_array.spot_count_ice_rings == message.spot_count_ice_rings);
REQUIRE(image_array.indexing_unit_cell.has_value());
REQUIRE(image_array.indexing_unit_cell.value().a == message.indexing_unit_cell.value().a);
REQUIRE(image_array.indexing_unit_cell.value().b == message.indexing_unit_cell.value().b);
REQUIRE(image_array.indexing_unit_cell.value().c == message.indexing_unit_cell.value().c);
REQUIRE(image_array.indexing_unit_cell.value().alpha == message.indexing_unit_cell.value().alpha);
REQUIRE(image_array.indexing_unit_cell.value().beta == message.indexing_unit_cell.value().beta);
REQUIRE(image_array.indexing_unit_cell.value().gamma == message.indexing_unit_cell.value().gamma);
REQUIRE(image_array.min_viable_pixel_value == message.min_viable_pixel_value);
REQUIRE(image_array.max_viable_pixel_value == message.max_viable_pixel_value);
}
TEST_CASE("CBORSerialize_Image_CrystalLattice", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
std::vector<SpotToSave> spots;
std::vector<uint16_t> test(512);
for (int i = 0; i < test.size(); i++)
test[i] = (i * 253 + 56) % 256;
CompressedImage image(test, 256, 2);
std::vector<float> latt = {1, 2, 3, 4, 5, 6, 7, 8, 9};
DataMessage message{
.image = image,
.indexing_lattice = CrystalLattice(latt)
};
REQUIRE_NOTHROW(serializer.SerializeImage(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::IMAGE);
REQUIRE(deserialized->data_message);
DataMessage &image_array = *deserialized->data_message;
REQUIRE(deserialized->data_message->indexing_lattice);
REQUIRE(deserialized->data_message->indexing_lattice->GetVector() == latt);
}
TEST_CASE("CBORSerialize_Image_2", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
std::vector<SpotToSave> spots;
std::vector<uint8_t> test(512 * 1024);
for (int i = 0; i < test.size(); i++)
test[i] = (i * 253 + 56) % 256;
CompressedImage image(test, 1024, 512);
DataMessage message{
.number = 480,
.image = image,
.spots = spots,
.indexing_result = true
};
REQUIRE_NOTHROW(serializer.SerializeImage(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::IMAGE);
REQUIRE(deserialized->data_message);
DataMessage &image_array = *deserialized->data_message;
REQUIRE(image_array.image.GetCompressionAlgorithm() == CompressionAlgorithm::NO_COMPRESSION);
REQUIRE(image_array.image.GetWidth() == 1024);
REQUIRE(image_array.image.GetHeight() == 512);
REQUIRE(image_array.image.GetMode() == CompressedImageMode::Uint8);
REQUIRE(image_array.image.GetChannel() == "default");
REQUIRE(image_array.image.GetCompressedSize() == test.size());
REQUIRE(image_array.indexing_result == message.indexing_result);
REQUIRE(image_array.number == 480);
REQUIRE(memcmp(image_array.image.GetCompressed(), test.data(), test.size()) == 0);
REQUIRE(!image_array.original_number);
}
TEST_CASE("CBORSerialize_Image_Float", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
std::vector<SpotToSave> spots;
std::vector<float> test(512 * 1024);
for (int i = 0; i < test.size(); i++)
test[i] = i * 0.1f;
CompressedImage image(test, 1024, 512);
DataMessage message{
.number = 480,
.image = image,
.spots = spots,
.indexing_result = false
};
REQUIRE_NOTHROW(serializer.SerializeImage(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::IMAGE);
REQUIRE(deserialized->data_message);
DataMessage &image_array = *deserialized->data_message;
REQUIRE(image_array.image.GetCompressionAlgorithm() == CompressionAlgorithm::NO_COMPRESSION);
REQUIRE(image_array.image.GetWidth() == 1024);
REQUIRE(image_array.image.GetHeight() == 512);
REQUIRE(image_array.image.GetMode() == CompressedImageMode::Float32);
REQUIRE(image_array.image.GetChannel() == "default");
REQUIRE(image_array.image.GetCompressedSize() == test.size() * sizeof(float));
REQUIRE(image_array.indexing_result == message.indexing_result);
REQUIRE(image_array.number == 480);
REQUIRE(memcmp(image_array.image.GetCompressed(), test.data(), test.size() * sizeof(float)) == 0);
}
TEST_CASE("CBORSerialize_Image_Rgb", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
std::vector<SpotToSave> spots;
std::vector<rgb> test(512 * 1024);
for (int i = 0; i < test.size(); i++) {
test[i].r = (i / 256) % 256;
test[i].g = i % 256;
test[i].b = (i / 771) % 256;
}
CompressedImage image(test, 1024, 512);
DataMessage message{
.number = 480,
.image = image,
.spots = spots,
.indexing_result = false
};
REQUIRE_NOTHROW(serializer.SerializeImage(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::IMAGE);
REQUIRE(deserialized->data_message);
DataMessage &image_array = *deserialized->data_message;
REQUIRE(image_array.image.GetCompressionAlgorithm() == CompressionAlgorithm::NO_COMPRESSION);
REQUIRE(image_array.image.GetWidth() == 1024);
REQUIRE(image_array.image.GetHeight() == 512);
REQUIRE(image_array.image.GetMode() == CompressedImageMode::RGB);
REQUIRE(image_array.image.GetChannel() == "default");
REQUIRE(image_array.image.GetCompressedSize() == test.size() * sizeof(rgb));
REQUIRE(image_array.indexing_result == message.indexing_result);
REQUIRE(image_array.number == 480);
REQUIRE(memcmp(image_array.image.GetCompressed(), test.data(), test.size() * sizeof(rgb)) == 0);
}
TEST_CASE("CBORSerialize_Image_Compressed", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
std::vector<SpotToSave> spots;
std::vector<uint8_t> test(512);
for (int i = 0; i < test.size(); i++)
test[i] = (i * 253 + 56) % 256;
CompressedImage image(test, 256, 2, CompressedImageMode::Int32, CompressionAlgorithm::BSHUF_LZ4);
DataMessage message{
.number = 456,
.image = image,
.spots = spots
};
REQUIRE_NOTHROW(serializer.SerializeImage(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::IMAGE);
REQUIRE(deserialized->data_message);
DataMessage &image_array = *deserialized->data_message;
REQUIRE(image_array.image.GetCompressionAlgorithm() == CompressionAlgorithm::BSHUF_LZ4);
REQUIRE(image_array.image.GetWidth() == 256);
REQUIRE(image_array.image.GetHeight() == 2);
REQUIRE(image_array.image.GetMode() == CompressedImageMode::Int32);
REQUIRE(image_array.image.GetChannel() == "default");
REQUIRE(image_array.image.GetCompressedSize() == test.size());
REQUIRE(image_array.number == 456);
REQUIRE(memcmp(image_array.image.GetCompressed(), test.data(), test.size()) == 0);
}
TEST_CASE("CBORSerialize_Image_Rad_Int_Profile", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
std::vector<uint16_t> test(512);
for (int i = 0; i < test.size(); i++)
test[i] = (i * 253 + 56) % 256;
CompressedImage image(test, 256, 2);
DataMessage message{
.number = 789,
.image = image,
.az_int_profile = {4.0, 5.0, 7.0, 12.0, 13.25, 0.125},
.az_int_profile_std = {1.0, 0.0, NAN, NAN, 8.0, 12.0},
.az_int_profile_count = {3,2,0,1,5,7}
};
REQUIRE_NOTHROW(serializer.SerializeImage(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::IMAGE);
REQUIRE(deserialized->data_message);
DataMessage &image_array = *deserialized->data_message;
REQUIRE(image_array.number == 789);
REQUIRE(image_array.image.GetCompressedSize() == test.size() * sizeof(uint16_t));
REQUIRE(memcmp(image_array.image.GetCompressed(), test.data(), test.size() * sizeof(uint16_t)) == 0);
REQUIRE(image_array.az_int_profile == message.az_int_profile);
REQUIRE(image_array.az_int_profile_std.size() == message.az_int_profile_std.size());
for (int i = 0; i < image_array.az_int_profile_std.size(); i++) {
if (std::isnan(message.az_int_profile_std[i]))
CHECK(std::isnan(image_array.az_int_profile_std[i]));
else
CHECK(message.az_int_profile_std[i] == Catch::Approx(image_array.az_int_profile_std[i]));
}
REQUIRE(image_array.az_int_profile_count == message.az_int_profile_count);
}
TEST_CASE("CBORSerialize_Start_AzintMap", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
std::vector<uint16_t> test(512);
for (int i = 0; i < test.size(); i++)
test[i] = (i * 253 + 56) % 256;
std::vector<uint16_t> azint_map(32 * 15, 57);
azint_map[0] = 123;
azint_map[32*14] = UINT16_MAX;
StartMessage msg{
.image_size_x = 32,
.image_size_y = 15,
.az_int_map = azint_map
};
REQUIRE_NOTHROW(serializer.SerializeSequenceStart(msg));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::START);
REQUIRE(deserialized->start_message);
CHECK(deserialized->start_message->az_int_map == azint_map);
}
TEST_CASE("CBORSerialize_Start_ROIMap", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
std::vector<uint16_t> roi_map(32 * 15, 0);
roi_map[0] = (1 << 0) | (1 << 3);
roi_map[32 * 14] = (1 << 2);
StartMessage msg{
.image_size_x = 32,
.image_size_y = 15,
.roi_map = roi_map
};
REQUIRE_NOTHROW(serializer.SerializeSequenceStart(msg));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::START);
REQUIRE(deserialized->start_message);
CHECK(deserialized->start_message->roi_map == roi_map);
}
TEST_CASE("CBORSerialize_Image_Spots", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
std::vector<SpotToSave> spots;
spots.push_back(SpotToSave{.x = 7, .y = 8, .intensity = 34, .indexed = false});
spots.push_back(SpotToSave{.x = 37, .y = 48, .intensity = 123, .indexed = true});
std::vector<uint16_t> test(512);
for (int i = 0; i < test.size(); i++)
test[i] = (i * 253 + 56) % 256;
CompressedImage image(test, 256, 2);
DataMessage message{
.number = 789,
.image = image,
.spots = spots
};
REQUIRE_NOTHROW(serializer.SerializeImage(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::IMAGE);
REQUIRE(deserialized->data_message);
DataMessage &image_array = *deserialized->data_message;
REQUIRE(image_array.number == 789);
REQUIRE(image_array.image.GetCompressedSize() == test.size() * sizeof(uint16_t));
REQUIRE(memcmp(image_array.image.GetCompressed(), test.data(), test.size() * sizeof(uint16_t)) == 0);
REQUIRE(image_array.spots.size() == 2);
REQUIRE(image_array.spots[0].intensity == 34);
REQUIRE(!image_array.spots[0].indexed);
REQUIRE(image_array.spots[1].x == 37);
REQUIRE(image_array.spots[1].y == 48);
REQUIRE(image_array.spots[1].intensity == 123);
REQUIRE(image_array.spots[1].indexed);
}
TEST_CASE("CBORSerialize_Image_Reflections") {
// Prepare a few reflections with distinct values
std::vector<Reflection> refs_in;
// Every float is given a distinct value: a field the encoder forgets reads back as 0 and fails
// here. var_bkg was added to Reflection and to the HDF5 writer but not to the CBOR map, so every
// broker-written file carried a zeroed background_variance until this test was widened to see it.
refs_in.push_back(Reflection{
.h = 1, .k = 0, .l = -1,
.image_number = 789.0f, .delta_phi_deg = 0.125f,
.predicted_x = 1024.5f, .predicted_y = 768.25f,
.observed_x = 1024.75f, .observed_y = 768.5f,
.d = 2.345f, .I = 1234.5f, .bkg = 3.25f, .var_bkg = 41.5f, .sigma = 37.125f,
.dist_ewald = 0.0025f, .rlp = 1.75f, .partiality = 0.875f, .zeta = 0.625f,
.image_scale_corr = 1.125f
});
refs_in.push_back(Reflection{
.h = -3, .k = 2, .l = 7,
.image_number = 790.0f, .delta_phi_deg = -0.25f,
.predicted_x = 5.0f, .predicted_y = 10.0f,
.observed_x = 5.5f, .observed_y = 10.5f,
.d = 4.0f, .I = 12.0f, .bkg = 0.5f, .var_bkg = 6.75f, .sigma = 4.25f,
.dist_ewald = -0.001f, .rlp = 2.5f, .partiality = 0.5f, .zeta = 0.25f,
.image_scale_corr = 0.875f
});
refs_in.push_back(Reflection{
.h = 0, .k = 0, .l = 1,
.image_number = 791.0f, .delta_phi_deg = 0.0f,
.predicted_x = 0.0f, .predicted_y = 0.0f,
.observed_x = 0.25f, .observed_y = 0.5f,
.d = 10.0f, .I = -1.5f, .bkg = 0.125f, .var_bkg = 0.875f, .sigma = 1.5f,
.dist_ewald = 0.0f, .rlp = 1.0f, .partiality = 1.0f, .zeta = 1.0f,
.image_scale_corr = 1.0f
});
// Minimal DataMessage carrying reflections
std::vector<uint16_t> test(512);
CompressedImage image(test, 256, 2);
DataMessage msg_in{
.number = 789,
.image = image,
.reflections = refs_in
};
// Serialize
std::vector<uint8_t> buffer(1 << 20); // 1 MB buffer for safety
CBORStream2Serializer ser(buffer.data(), buffer.size());
ser.SerializeImage(msg_in);
// Deserialize
auto deserialized = CBORStream2Deserialize(buffer.data(), ser.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::IMAGE);
REQUIRE(deserialized->data_message);
DataMessage &msg_out = *deserialized->data_message;
// Check reflections
REQUIRE(msg_out.reflections.size() == refs_in.size());
for (size_t i = 0; i < refs_in.size(); ++i) {
const auto &a = refs_in[i];
const auto &b = msg_out.reflections[i];
CHECK(a.h == b.h);
CHECK(a.k == b.k);
CHECK(a.l == b.l);
CHECK(b.predicted_x == Catch::Approx(a.predicted_x).margin(1e-6f));
CHECK(b.predicted_y == Catch::Approx(a.predicted_y).margin(1e-6f));
CHECK(b.observed_x == Catch::Approx(a.observed_x).margin(1e-6f));
CHECK(b.observed_y == Catch::Approx(a.observed_y).margin(1e-6f));
CHECK(b.d == Catch::Approx(a.d).margin(1e-6f));
CHECK(b.image_number == Catch::Approx(a.image_number).margin(1e-6f));
CHECK(b.delta_phi_deg == Catch::Approx(a.delta_phi_deg).margin(1e-6f));
CHECK(b.I == Catch::Approx(a.I).margin(1e-6f));
CHECK(b.bkg == Catch::Approx(a.bkg).margin(1e-6f));
CHECK(b.var_bkg == Catch::Approx(a.var_bkg).margin(1e-6f));
CHECK(b.sigma == Catch::Approx(a.sigma).margin(1e-6f));
CHECK(b.dist_ewald == Catch::Approx(a.dist_ewald).margin(1e-6f));
CHECK(b.rlp == Catch::Approx(a.rlp).margin(1e-6f));
CHECK(b.partiality == Catch::Approx(a.partiality).margin(1e-6f));
CHECK(b.zeta == Catch::Approx(a.zeta).margin(1e-6f));
CHECK(b.image_scale_corr == Catch::Approx(a.image_scale_corr).margin(1e-6f));
}
}
TEST_CASE("CBORSerialize_Image_MetadataHeadroom", "[CBOR]") {
// The receiver serializes the per-image CBOR metadata (spots, reflections,
// azimuthal profile, ...) into the same buffer slot as the compressed image, and
// DiffractionExperiment::GetImageBufferLocationSize() reserves
// kImageMetadataHeadroom on top of GetMaxCompressedSize() for it. A frame whose
// metadata exceeds that headroom is dropped by the receiver. This test guards that
// even the largest possible metadata still fits, so ordinary frames are never
// dropped: the kOnlineMaxReflections (10000, BraggPrediction.h) reflection cap, the
// API maximum of 2000 spots (jfjoch_api.yaml), and the 65534-bin azimuthal cap
// (AzimuthalIntegrationMapping, UINT16_MAX - 1), all serialized together. Take the
// headroom from the constant rather than restate it: written down twice, the two
// drifted apart and this test kept passing against the value the code had left.
constexpr size_t max_reflections = 10000;
constexpr size_t max_spots = 2000;
constexpr size_t max_az_bins = 65534;
constexpr size_t headroom = DiffractionExperiment::kImageMetadataHeadroom;
std::vector<Reflection> reflections(max_reflections);
for (size_t i = 0; i < reflections.size(); ++i)
reflections[i] = Reflection{
.h = static_cast<int32_t>(i % 200) - 100,
.k = static_cast<int32_t>(i % 200) - 100,
.l = static_cast<int32_t>(i % 200) - 100,
.image_number = 789.0f, .delta_phi_deg = 0.123f,
.predicted_x = 1234.5f, .predicted_y = 678.9f,
.observed_x = 1234.0f, .observed_y = 678.0f,
.d = 2.345f, .I = 1234.5f, .bkg = 12.3f, .sigma = 35.6f,
.dist_ewald = 0.001f, .rlp = 0.5f, .partiality = 0.9f,
.zeta = 0.3f, .image_scale_corr = 1.1f};
std::vector<SpotToSave> spots(max_spots);
for (size_t i = 0; i < spots.size(); ++i)
spots[i] = SpotToSave{
.x = 1234.5f, .y = 678.9f, .intensity = 1234.5f, .maxc = 5000,
.lattice = 0, .image = 789,
.h = static_cast<int64_t>(i % 200) - 100,
.k = static_cast<int64_t>(i % 200) - 100,
.l = static_cast<int64_t>(i % 200) - 100,
.dist_ewald_sphere = 0.01f, .indexed = true};
DataMessage message{
.number = 789,
.image = CompressedImage(nullptr, 0, 4000, 4000,
CompressedImageMode::Uint32, CompressionAlgorithm::BSHUF_LZ4),
.spots = spots,
.az_int_profile = std::vector<float>(max_az_bins, 1.0f),
.az_int_profile_std = std::vector<float>(max_az_bins, 1.0f),
.az_int_profile_count = std::vector<uint64_t>(max_az_bins, 1),
.reflections = reflections};
std::vector<uint8_t> buffer(16 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
REQUIRE_NOTHROW(serializer.SerializeImage(message));
// GetImageAppendOffset() is where the compressed image is appended, i.e. the size
// of everything-but-the-image; it must stay below the metadata headroom.
INFO("worst-case metadata size = " << serializer.GetImageAppendOffset() << " B");
CHECK(serializer.GetImageAppendOffset() < headroom);
}
TEST_CASE("CBORSerialize_Image_ROI", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
std::vector<uint16_t> test(512);
for (int i = 0; i < test.size(); i++)
test[i] = (i * 253 + 56) % 256;
CompressedImage image(test, 256, 2);
DataMessage message{
.number = 789,
.image = image,
};
message.roi["roi0"] = {
.sum = -876, .sum_square = 89998, .max_count = -1, .pixels = 4567, .x_weighted = 123, .y_weighted = 122
};
message.roi["roi1"] = {
.sum = 876, .sum_square = 998, .max_count = 12, .pixels = 234, .x_weighted = -11222233443123LL, .y_weighted = -5
};
REQUIRE_NOTHROW(serializer.SerializeImage(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::IMAGE);
REQUIRE(deserialized->data_message);
DataMessage &image_array = *deserialized->data_message;
REQUIRE(image_array.number == 789);
REQUIRE(image_array.roi.size() == 2);
REQUIRE(image_array.roi.contains("roi0"));
REQUIRE(image_array.roi.contains("roi1"));
CHECK(image_array.roi["roi0"].sum == message.roi["roi0"].sum);
CHECK(image_array.roi["roi0"].sum_square == message.roi["roi0"].sum_square);
CHECK(image_array.roi["roi0"].max_count == message.roi["roi0"].max_count);
CHECK(image_array.roi["roi0"].pixels == message.roi["roi0"].pixels);
CHECK(image_array.roi["roi0"].x_weighted == message.roi["roi0"].x_weighted);
CHECK(image_array.roi["roi0"].y_weighted == message.roi["roi0"].y_weighted);
CHECK(image_array.roi["roi1"].sum == message.roi["roi1"].sum);
CHECK(image_array.roi["roi1"].sum_square == message.roi["roi1"].sum_square);
CHECK(image_array.roi["roi1"].max_count == message.roi["roi1"].max_count);
CHECK(image_array.roi["roi1"].pixels == message.roi["roi1"].pixels);
CHECK(image_array.roi["roi1"].x_weighted == message.roi["roi1"].x_weighted);
CHECK(image_array.roi["roi1"].y_weighted == message.roi["roi1"].y_weighted);
}
TEST_CASE("CBORSerialize_Image_Append", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
std::vector<uint8_t> test(512 * 1024);
for (int i = 0; i < test.size(); i++)
test[i] = (i * 253 + 56) % 256;
CompressedImage image(nullptr, 0, 1024, 512, CompressedImageMode::Uint8, CompressionAlgorithm::BSHUF_LZ4);
DataMessage message{
.number = 480,
.image = image
};
REQUIRE_NOTHROW(serializer.SerializeImage(message));
memcpy(buffer.data() + serializer.GetImageAppendOffset(), test.data(), 512 * 1024);
//REQUIRE_THROWS(serializer.AppendImage(16*1024*1024));
REQUIRE_NOTHROW(serializer.AppendImage(512 * 1024));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::IMAGE);
REQUIRE(deserialized->data_message);
DataMessage &image_array = *deserialized->data_message;
REQUIRE(image_array.image.GetCompressionAlgorithm() == CompressionAlgorithm::BSHUF_LZ4);
REQUIRE(image_array.image.GetWidth() == 1024);
REQUIRE(image_array.image.GetHeight() == 512);
REQUIRE(image_array.image.GetMode() == CompressedImageMode::Uint8);
REQUIRE(image_array.image.GetChannel() == "default");
REQUIRE(image_array.image.GetCompressedSize() == test.size());
REQUIRE(image_array.indexing_result == message.indexing_result);
REQUIRE(image_array.number == 480);
REQUIRE(memcmp(image_array.image.GetCompressed(), test.data(), test.size()) == 0);
}
TEST_CASE("CBORSerialize_Metadata", "[CBOR]") {
MetadataMessage msgs;
CompressedImage image(nullptr, 0, 123, 145, CompressedImageMode::Uint8,
CompressionAlgorithm::BSHUF_LZ4);
msgs.images.push_back(DataMessage{.number = 172, .image = image, .bkg_estimate = 45});
msgs.images.push_back(DataMessage{.number = 173, .image = image, .bkg_estimate = 48});
//msgs.push_back(DataMessage{.number = 174, .image = CompressedImage{.GetWidth() = 123, .GetHeight() = 145}, .bkg_estimate = 48});
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
REQUIRE_NOTHROW(serializer.SerializeMetadata(msgs));
REQUIRE(serializer.GetBufferSize() > 0);
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->msg_type == CBORImageType::METADATA);
REQUIRE(deserialized->metadata);
REQUIRE(deserialized->metadata->images.size() == 2);
CHECK(deserialized->metadata->images.at(0).number == 172);
CHECK(deserialized->metadata->images.at(0).bkg_estimate == 45);
CHECK(deserialized->metadata->images.at(0).image.GetWidth() == 0);
CHECK(deserialized->metadata->images.at(1).number == 173);
CHECK(deserialized->metadata->images.at(1).bkg_estimate == 48);
CHECK(deserialized->metadata->images.at(1).image.GetWidth() == 0);
}
TEST_CASE("CBORSerialize_Image_LatticeType", "[CBOR][Lattice]") {
std::vector<uint8_t> buffer(1 << 20);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
// Prepare an image with lattice_type set
std::vector<uint16_t> pixels(16, 0);
CompressedImage img(pixels, 4, 4);
DataMessage in_msg{
.number = 1,
.image = img
};
// Case 1: Tetragonal, body-centered -> expect same after round trip
in_msg.lattice_type = LatticeMessage{
.centering = 'I',
.niggli_class = 25,
.crystal_system = gemmi::CrystalSystem::Tetragonal
};
REQUIRE_NOTHROW(serializer.SerializeImage(in_msg));
auto out = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(out);
REQUIRE(out->msg_type == CBORImageType::IMAGE);
REQUIRE(out->data_message);
// LatticeMessage present
REQUIRE(out->data_message->lattice_type.has_value());
const auto lm = out->data_message->lattice_type.value();
CHECK(lm.centering == 'I');
CHECK(lm.niggli_class == 25);
CHECK(lm.crystal_system == gemmi::CrystalSystem::Tetragonal);
// Case 2: Absent lattice_type should remain absent
CBORStream2Serializer serializer2(buffer.data(), buffer.size());
DataMessage in_msg2{
.number = 2,
.image = img
};
REQUIRE_NOTHROW(serializer2.SerializeImage(in_msg2));
auto out2 = CBORStream2Deserialize(buffer.data(), serializer2.GetBufferSize());
REQUIRE(out2);
REQUIRE(out2->msg_type == CBORImageType::IMAGE);
REQUIRE(out2->data_message);
CHECK_FALSE(out2->data_message->lattice_type.has_value());
// Case 3: Various systems/centerings sanity round-trip
struct Case {
gemmi::CrystalSystem cs;
char ctr;
};
std::vector<Case> cases = {
{gemmi::CrystalSystem::Triclinic, 'P'},
{gemmi::CrystalSystem::Monoclinic, 'C'},
{gemmi::CrystalSystem::Orthorhombic, 'F'},
{gemmi::CrystalSystem::Orthorhombic, 'A'},
{gemmi::CrystalSystem::Tetragonal, 'P'},
{gemmi::CrystalSystem::Trigonal, 'R'},
{gemmi::CrystalSystem::Hexagonal, 'P'},
{gemmi::CrystalSystem::Cubic, 'I'},
};
int i = 0;
for (const auto &c: cases) {
CBORStream2Serializer ser(buffer.data(), buffer.size());
DataMessage m{
.number = 3,
.image = img
};
m.lattice_type = LatticeMessage{.centering = c.ctr, .niggli_class = i, .crystal_system = c.cs};
REQUIRE_NOTHROW(ser.SerializeImage(m));
auto d = CBORStream2Deserialize(buffer.data(), ser.GetBufferSize());
REQUIRE(d);
REQUIRE(d->msg_type == CBORImageType::IMAGE);
REQUIRE(d->data_message);
REQUIRE(d->data_message->lattice_type.has_value());
auto l = d->data_message->lattice_type.value();
CHECK(l.centering == c.ctr);
CHECK(l.niggli_class == i);
CHECK(l.crystal_system == c.cs);
i++;
}
}
// The chain is sent as an ARRAY because the mounting order has to survive, and a CBOR map's order
// carries no guarantee - RFC 8949 requires deterministic encoders to sort map keys. This asserts the
// order round trips, not merely the contents, and that a moving axis keeps one value per image
// while a stationary one keeps a single value.
TEST_CASE("CBORSerialize_End_Transformations", "[CBOR]") {
EndMessage message{};
message.max_image_number = 4;
message.transformations = {
DetectorTransformation("grid_scan_x", TransformationType::Translation, {1, 0, 0})
.Values({0.0f, 1e-4f, 2e-4f, 3e-4f}),
DetectorTransformation("omega", TransformationType::Rotation, {-1, 0, 0})
.DependsOn("grid_scan_x").Values({95.0f, 95.1f, 95.2f, 95.3f}),
DetectorTransformation("chi", TransformationType::Rotation, {0, 0, 1})
.DependsOn("omega").Value(12.5f),
DetectorTransformation("phi", TransformationType::Rotation, {1, 0, 0})
.DependsOn("chi").Value(-7.25f).Offset({1, 2, 3}),
};
std::vector<uint8_t> buffer(MESSAGE_SIZE_FOR_START_END);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
REQUIRE_NOTHROW(serializer.SerializeSequenceEnd(message));
buffer.resize(serializer.GetBufferSize());
auto output = CBORStream2Deserialize(buffer.data(), buffer.size());
REQUIRE(output->end_message.has_value());
const auto &chain = output->end_message->transformations;
REQUIRE(chain.size() == message.transformations.size());
for (size_t i = 0; i < chain.size(); i++) {
const auto &got = chain[i];
const auto &want = message.transformations[i];
CHECK(got.GetName() == want.GetName());
CHECK(got.IsRotation() == want.IsRotation());
CHECK(got.GetUnits() == want.GetUnits());
CHECK(got.GetDependsOn() == want.GetDependsOn());
CHECK(got.GetVector().x == Catch::Approx(want.GetVector().x));
CHECK(got.GetVector().y == Catch::Approx(want.GetVector().y));
CHECK(got.GetVector().z == Catch::Approx(want.GetVector().z));
CHECK(got.GetOffset().x == Catch::Approx(want.GetOffset().x));
CHECK(got.IsConstant() == want.IsConstant());
REQUIRE(got.GetValues().size() == want.GetValues().size());
for (size_t j = 0; j < got.GetValues().size(); j++)
CHECK(got.GetValues()[j] == Catch::Approx(want.GetValues()[j]).margin(1e-4));
}
// The base of the chain names no parent.
CHECK(chain[0].GetDependsOn().empty());
CHECK(!chain[1].IsConstant());
CHECK(chain[2].IsConstant());
}
Reference in New Issue View Git Blame Copy Permalink