// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute // SPDX-License-Identifier: GPL-3.0-only #include #include #include #include "../compression/JFJochCompressor.h" #include "../frame_serialize/CBORStream2Deserializer.h" #include "../image_analysis/structure_refinement/RugnuxModel.h" #include "../image_puller/TestImagePuller.h" #include "../image_puller/ZMQImagePuller.h" #include "../image_pusher/ImagePusher.h" #include "../image_pusher/ZMQStream2Pusher.h" #include "../receiver/JFJochReceiverDarkMask.h" #include "../receiver/JFJochReceiverLite.h" #include "../receiver/JFJochReceiverRugnuxRotation.h" #include "../receiver/JFJochReceiverRugnuxStills.h" #include "../receiver/JFJochReceiverService.h" #include "../writer/HDF5Objects.h" #include "../writer/StreamWriter.h" namespace { // What the writer would be given, in order. class RecordingPusher : public ImagePusher { mutable std::mutex m; std::vector events; std::optional start; std::optional end; std::vector images; public: void StartDataCollection(StartMessage &message) override { std::unique_lock ul(m); events.emplace_back("start"); start = message; } bool EndDataCollection(const EndMessage &message) override { std::unique_lock ul(m); events.emplace_back("end"); end = message; return true; } bool SendImage(const uint8_t *, size_t, int64_t number) override { std::unique_lock ul(m); if (events.empty() || events.back() != "image") events.emplace_back("image"); images.push_back(number); return true; } bool SendCalibration(const CompressedImage &) override { std::unique_lock ul(m); events.emplace_back("calibration"); return true; } std::string PrintSetup() const override { return "recording"; } ImagePusherType GetType() const override { return ImagePusherType::Test; } std::vector Events() const { std::unique_lock ul(m); return events; } std::vector Images() const { std::unique_lock ul(m); return images; } std::optional Start() const { std::unique_lock ul(m); return start; } std::optional End() const { std::unique_lock ul(m); return end; } }; DiffractionExperiment StillsExperiment(int64_t images) { DiffractionExperiment x(DetDECTRIS(2068, 2164, "Test", {})); // As JFJochReceiverLite's test: the frames are signed int16, and the stream's start message says so. x.ImagesPerTrigger(images).NumTriggers(1).ImagesPerFile(2).FilePrefix("rugnux_stills").JungfrauConvPhotonCnt(false) .PixelSigned(true).DetectorDistance_mm(75).BeamY_pxl(1136).BeamX_pxl(1090).IncidentEnergy_keV(12.4) .SetUnitCell(UnitCell{.a = 36.9, .b = 78.95, .c = 78.95, .alpha = 90, .beta = 90, .gamma = 90}); return x; } // The frames of the test data, as a stream delivers them: compressed, one DataMessage each, no more. class Stream { std::vector> frames; size_t width, height; public: const std::shared_ptr puller = std::make_shared(); // With same_frame >= 0, every frame is that frame of the test data. Stream(const DiffractionExperiment &x, int64_t nframes, bool blank = false, int64_t same_frame = -1) : width(x.GetXPixelsNum()), height(x.GetYPixelsNum()) { HDF5ReadOnlyFile data("../../tests/test_data/compression_benchmark.h5"); HDF5DataSet dataset(data, "/entry/data/data"); HDF5DataSpace file_space(dataset); std::vector image(file_space.GetDimensions()[1] * file_space.GetDimensions()[2]); JFJochBitShuffleCompressor compressor(CompressionAlgorithm::BSHUF_LZ4); for (int64_t i = 0; i < nframes; i++) { const int64_t frame = (same_frame >= 0) ? same_frame : i; const std::vector start = {static_cast(frame % file_space.GetDimensions()[0]), 0, 0}; const std::vector size = {1, file_space.GetDimensions()[1], file_space.GetDimensions()[2]}; if (blank) std::fill(image.begin(), image.end(), 0); else dataset.ReadVector(image, start, size); frames.push_back(compressor.Compress(image)); } StartMessage start_msg; x.FillMessage(start_msg); puller->Put(ImagePullerOutput{.cbor = std::make_shared(start_msg)}); } void Frames(int64_t first, int64_t count) { for (int64_t i = first; i < first + count; i++) { DataMessage message; message.number = i; message.image = CompressedImage(frames[i].data(), frames[i].size(), width, height, CompressedImageMode::Int16, CompressionAlgorithm::BSHUF_LZ4); puller->Put(ImagePullerOutput{.cbor = std::make_shared(message)}); } } const std::vector &Frame(int64_t i) const { return frames[i]; } void End() { puller->Put(ImagePullerOutput{.cbor = std::make_shared(EndMessage{})}); } }; void WaitForStatus(JFJochReceiverService &service, const std::function &what) { const auto give_up = std::chrono::steady_clock::now() + std::chrono::minutes(5); while (!what(service.GetRugnuxStatus())) { REQUIRE(std::chrono::steady_clock::now() < give_up); std::this_thread::sleep_for(std::chrono::milliseconds(10)); } } bool HasNote(const RugnuxLiveSnapshot &s, const std::string &text) { for (const auto &step: s.steps) for (const auto ¬e: step.notes) if (note.find(text) != std::string::npos) return true; return false; } } TEST_CASE("JFJochReceiverRugnux_Stills", "[JFJochReceiverRugnux]") { Logger logger(Catch::getResultCapture().getCurrentTestName()); RegisterHDF5Filter(); const int64_t images = 12; DiffractionExperiment x = StillsExperiment(images); PixelMask mask(x); Stream stream(x, images); stream.Frames(0, images); stream.End(); RecordingPusher pusher; AcquisitionDeviceGroup group; JFJochReceiverService service(group, logger, pusher); service.NumThreads(4); service.Indexing(x.GetIndexingSettings()); service.RugnuxLive(true, nullptr, 4, 60); // Integration in live stills is rugnux's own, whatever the broker's spot-finding settings say. SpotFindingSettings spot_finding; spot_finding.quick_integration = false; service.SetSpotFindingSettings(spot_finding); service.Start(x, mask, nullptr, stream.puller); const auto out = service.Stop(); CHECK(out.efficiency == 1.0f); CHECK(!out.status.cancelled); CHECK(out.status.images_collected == images); CHECK(out.status.images_sent == images); REQUIRE(out.status.indexing_rate); CHECK(out.status.indexing_rate.value() > 0.5f); CHECK(out.scan_result.images.size() == images); // The writer sees the start message before the frames, the end message last. CHECK(pusher.Events() == std::vector{"start", "image", "end"}); auto sent = pusher.Images(); std::sort(sent.begin(), sent.end()); REQUIRE(sent.size() == images); CHECK(sent.front() == 0); CHECK(sent.back() == images - 1); // The merged data travel in the end message, with the geometry the frames were analysed at. const auto end = pusher.End(); REQUIRE(end); CHECK(end->merged.has_value()); CHECK(end->unit_cell.has_value()); CHECK(end->refined_beam_center_x.has_value()); CHECK(end->refined_detector_distance_mm.has_value()); REQUIRE(pusher.Start()); CHECK(pusher.Start()->beam_center_x == Catch::Approx(end->refined_beam_center_x.value())); // Which frames the merge took, per image: in the end message for the writer, and as a plot for the viewer REQUIRE(end->image_merged.size() == images); const auto merged = std::ranges::count(end->image_merged, uint8_t{1}); CHECK(merged > 0); std::vector merged_plot; service.GetPlotRaw(merged_plot, PlotType::ImageMerged, ""); REQUIRE(merged_plot.size() == images); CHECK(std::ranges::count(merged_plot, 1.0f) == merged); const auto live = service.GetRugnuxStatus(); CHECK(live.state == RugnuxLiveState::Done); CHECK(live.cut.empty()); // Twelve strong frames: enough for the geometry fit CHECK(HasNote(live, "Geometry refined from")); CHECK(HasNote(live, "Merge of 6 of 12 images (provisional)")); CHECK(!HasNote(live, "Merge of 9 of 12 images (provisional)")); CHECK(HasNote(live, "Merge of the collection")); // The report's checklist of the merge: completeness looked at, powder rings not (no pre-scan here) const auto triage_line = [&](const std::string &label) { return std::ranges::find(live.triage, label, &RugnuxLiveTriage::label); }; REQUIRE(triage_line("Completeness") != live.triage.end()); CHECK(triage_line("Completeness")->severity != RugnuxLiveTriageSeverity::NotMeasured); REQUIRE(triage_line("Powder") != live.triage.end()); CHECK(triage_line("Powder")->severity == RugnuxLiveTriageSeverity::NotMeasured); // The shell table of the merge of the collection, without the rotation-only R_meas CHECK(live.shells_merge == "the collection"); REQUIRE(!live.shells.empty()); for (const auto &sh : live.shells) { CHECK(sh.n_unique > 0); CHECK(!sh.r_meas); } REQUIRE(live.stills); CHECK(live.stills->prepass == RugnuxLivePrePass::Done); CHECK(live.stills->images_analysed == images); CHECK(live.stills->images_indexed > images / 2); REQUIRE(live.last_image_to_written_s); CHECK(!live.space_group.empty()); REQUIRE(!service.GetProgress().has_value()); } TEST_CASE("JFJochReceiverRugnux_Stills_ImagesVisibleDuringPrePass", "[JFJochReceiverRugnux]") { // A frame held for the pre-pass can be seen through the image routes as it came, with a start message on the stated // geometry; once analysed, the copy with the analysis is the only one there. The writer gets each image once. Logger logger(Catch::getResultCapture().getCurrentTestName()); RegisterHDF5Filter(); const int64_t images = 8; const int64_t held = 3; DiffractionExperiment x = StillsExperiment(images); PixelMask mask(x); Stream stream(x, images); stream.Frames(0, held); RecordingPusher pusher; AcquisitionDeviceGroup group; JFJochReceiverService service(group, logger, pusher); service.NumThreads(4); service.Indexing(x.GetIndexingSettings()); service.RugnuxLive(true, nullptr, 4, 60); service.Start(x, mask, nullptr, stream.puller); WaitForStatus(service, [&](const RugnuxLiveSnapshot &s) { return s.stills && s.stills->images_held == held; }); std::vector cbor; service.GetStartMessageFromBuffer(cbor); REQUIRE(!cbor.empty()); auto msg = CBORStream2Deserialize(cbor); REQUIRE(msg->start_message); CHECK(msg->start_message->beam_center_x == Catch::Approx(x.GetBeamX_pxl())); for (int64_t i = 0; i < held; i++) { REQUIRE(service.GetImageFromBuffer(cbor, i)); msg = CBORStream2Deserialize(cbor); REQUIRE(msg->data_message); CHECK(msg->data_message->number == i); CHECK(!msg->data_message->spot_count.has_value()); const auto &frame = stream.Frame(i); REQUIRE(msg->data_message->image.GetCompressedSize() == frame.size()); CHECK(memcmp(msg->data_message->image.GetCompressed(), frame.data(), frame.size()) == 0); } REQUIRE(service.GetImageFromBuffer(cbor, ImageBuffer::MaxImage)); CHECK(CBORStream2Deserialize(cbor)->data_message->number == held - 1); // They wait in a slot of the image buffer each CHECK(service.GetImageBufferStatus().sending_slots == held); CHECK(service.GetImageBufferStatus().images_in_the_buffer == std::vector{0, 1, 2}); // Nothing goes to the writer before the pre-pass CHECK(pusher.Events().empty()); stream.Frames(held, images - held); stream.End(); const auto out = service.Stop(); CHECK(out.status.images_sent == images); CHECK(pusher.Events() == std::vector{"start", "image", "end"}); auto sent = pusher.Images(); std::sort(sent.begin(), sent.end()); std::vector all(images); std::iota(all.begin(), all.end(), 0); CHECK(sent == all); // The analysed copies only, with the start message the frames were analysed with CHECK(service.GetImageBufferStatus().images_in_the_buffer == all); for (int64_t i = 0; i < images; i++) { REQUIRE(service.GetImageFromBuffer(cbor, i)); msg = CBORStream2Deserialize(cbor); REQUIRE(msg->data_message); CHECK(msg->data_message->spot_count.has_value()); } service.GetStartMessageFromBuffer(cbor); REQUIRE(pusher.Start()); CHECK(CBORStream2Deserialize(cbor)->start_message->arm_date == pusher.Start()->arm_date); } TEST_CASE("JFJochReceiverRugnux_Stills_CancelDuringPrePass", "[JFJochReceiverRugnux]") { Logger logger(Catch::getResultCapture().getCurrentTestName()); RegisterHDF5Filter(); const int64_t images = 8; DiffractionExperiment x = StillsExperiment(images); PixelMask mask(x); Stream stream(x, images); RecordingPusher pusher; AcquisitionDeviceGroup group; JFJochReceiverService service(group, logger, pusher); service.NumThreads(4); service.Indexing(x.GetIndexingSettings()); service.RugnuxLive(true, nullptr, 4, 60); int64_t given = 0; SECTION("while it holds frames") { given = 3; stream.Frames(0, given); service.Start(x, mask, nullptr, stream.puller); WaitForStatus(service, [&](const RugnuxLiveSnapshot &s) { return s.stills && s.stills->images_held == given; }); } SECTION("while it runs") { given = images; stream.Frames(0, given); service.Start(x, mask, nullptr, stream.puller); WaitForStatus(service, [&](const RugnuxLiveSnapshot &s) { return s.stills && s.stills->prepass == RugnuxLivePrePass::Running; }); } service.Cancel(false); const auto out = service.Stop(); // The frames that came are still analysed and written, and the collection ends properly. CHECK(out.status.cancelled); CHECK(out.status.images_sent == given); CHECK(pusher.Events() == std::vector{"start", "image", "end"}); const auto live = service.GetRugnuxStatus(); CHECK(live.state == RugnuxLiveState::Done); REQUIRE(live.stills); CHECK(live.stills->images_analysed == given); REQUIRE(!service.GetProgress().has_value()); } TEST_CASE("JFJochReceiverRugnux_Stills_CancelDuringRun", "[JFJochReceiverRugnux]") { Logger logger(Catch::getResultCapture().getCurrentTestName()); RegisterHDF5Filter(); const int64_t images = 40; DiffractionExperiment x = StillsExperiment(images); PixelMask mask(x); Stream stream(x, images); // Four slots and a pre-pass of two frames, the two the frames wait in: the analysis starts with most of the run to // come. The frames are given one by one until it does. const size_t buffer_MiB = 4 * x.GetImageBufferLocationSize() / (1024 * 1024) + 1; RecordingPusher pusher; AcquisitionDeviceGroup group; JFJochReceiverService service(group, logger, pusher, buffer_MiB); service.NumThreads(2); service.Indexing(x.GetIndexingSettings()); // A model only so that there is a model step for the cancel to cut. service.RugnuxLive(true, std::make_shared(), 4, 60, false, false, 1, 2); service.Start(x, mask, nullptr, stream.puller); int64_t given = 0; const auto holding = [&] { const auto s = service.GetRugnuxStatus(); return !s.stills || s.stills->prepass == RugnuxLivePrePass::Holding; }; while (holding() && given < images - 1) { stream.Frames(given++, 1); std::this_thread::sleep_for(std::chrono::milliseconds(100)); } REQUIRE(given < images - 1); WaitForStatus(service, [](const RugnuxLiveSnapshot &s) { return s.stills && s.stills->prepass == RugnuxLivePrePass::Done && s.stills->images_analysed > 0; }); service.Cancel(false); const auto out = service.Stop(); CHECK(out.status.cancelled); CHECK(out.status.images_collected == given); CHECK(out.status.images_sent == given); CHECK(pusher.Events() == std::vector{"start", "image", "end"}); const auto live = service.GetRugnuxStatus(); CHECK(live.state == RugnuxLiveState::Done); CHECK(live.cut == "cancel"); REQUIRE(!service.GetProgress().has_value()); } TEST_CASE("JFJochReceiverRugnux_Stills_ShortRun", "[JFJochReceiverRugnux]") { Logger logger(Catch::getResultCapture().getCurrentTestName()); RegisterHDF5Filter(); DiffractionExperiment x = StillsExperiment(8); PixelMask mask(x); Stream stream(x, 3); stream.Frames(0, 3); stream.End(); RecordingPusher pusher; AcquisitionDeviceGroup group; JFJochReceiverService service(group, logger, pusher); service.NumThreads(4); service.Indexing(x.GetIndexingSettings()); service.RugnuxLive(true, nullptr, 4, 60); service.Start(x, mask, nullptr, stream.puller); const auto out = service.Stop(); // The pre-pass runs on what came. CHECK(out.efficiency == Catch::Approx(3.0 / 8.0)); CHECK(!out.status.cancelled); CHECK(out.status.images_sent == 3); CHECK(pusher.Events() == std::vector{"start", "image", "end"}); const auto live = service.GetRugnuxStatus(); CHECK(live.state == RugnuxLiveState::Done); REQUIRE(live.stills); CHECK(live.stills->images_analysed == 3); } TEST_CASE("JFJochReceiverRugnux_Stills_FramesBeyondTheCollection", "[JFJochReceiverRugnux]") { Logger logger(Catch::getResultCapture().getCurrentTestName()); RegisterHDF5Filter(); DiffractionExperiment x = StillsExperiment(3); PixelMask mask(x); Stream stream(x, 5); stream.Frames(0, 5); stream.End(); RecordingPusher pusher; AcquisitionDeviceGroup group; JFJochReceiverService service(group, logger, pusher); service.NumThreads(4); service.Indexing(x.GetIndexingSettings()); service.RugnuxLive(true, nullptr, 4, 60); service.Start(x, mask, nullptr, stream.puller); const auto out = service.Stop(); CHECK(out.status.images_collected == 3); CHECK(out.status.images_sent == 3); } TEST_CASE("JFJochReceiverRugnux_Stills_NoFrames", "[JFJochReceiverRugnux]") { Logger logger(Catch::getResultCapture().getCurrentTestName()); RegisterHDF5Filter(); DiffractionExperiment x = StillsExperiment(8); PixelMask mask(x); Stream stream(x, 0); stream.End(); RecordingPusher pusher; AcquisitionDeviceGroup group; JFJochReceiverService service(group, logger, pusher); service.NumThreads(4); service.Indexing(x.GetIndexingSettings()); service.RugnuxLive(true, nullptr, 4, 60); service.Start(x, mask, nullptr, stream.puller); const auto out = service.Stop(); CHECK(out.status.images_sent == 0); CHECK(pusher.Events() == std::vector{"start", "end"}); CHECK(service.GetRugnuxStatus().state == RugnuxLiveState::Done); } TEST_CASE("JFJochReceiverRugnux_Stills_NoIndexableFrames", "[JFJochReceiverRugnux]") { Logger logger(Catch::getResultCapture().getCurrentTestName()); RegisterHDF5Filter(); const int64_t images = 6; DiffractionExperiment x = StillsExperiment(images); PixelMask mask(x); Stream stream(x, images, true); stream.Frames(0, images); stream.End(); RecordingPusher pusher; AcquisitionDeviceGroup group; JFJochReceiverService service(group, logger, pusher); service.NumThreads(4); service.Indexing(x.GetIndexingSettings()); service.RugnuxLive(true, nullptr, 4, 60); service.Start(x, mask, nullptr, stream.puller); const auto out = service.Stop(); // Written, with no merge, at the geometry in the header. CHECK(!out.status.cancelled); CHECK(out.status.images_sent == images); CHECK(pusher.Events() == std::vector{"start", "image", "end"}); REQUIRE(pusher.End()); CHECK(!pusher.End()->merged.has_value()); CHECK(pusher.End()->refined_beam_center_x == Catch::Approx(x.GetBeamX_pxl())); const auto live = service.GetRugnuxStatus(); CHECK(live.state == RugnuxLiveState::Done); CHECK(HasNote(live, "header geometry used")); REQUIRE(live.stills); CHECK(live.stills->images_analysed == images); CHECK(live.stills->images_indexed == 0); } TEST_CASE("JFJochReceiverRugnux_Stills_WriterDown", "[JFJochReceiverRugnux]") { // The start message goes out after the pre-pass: where it fails, the collection ends with the error. class NoWriter : public RecordingPusher { public: bool down = true; void StartDataCollection(StartMessage &message) override { if (down) throw JFJochException(JFJochExceptionCategory::ZeroMQ, "no writer"); RecordingPusher::StartDataCollection(message); } }; Logger logger(Catch::getResultCapture().getCurrentTestName()); RegisterHDF5Filter(); const int64_t images = 4; DiffractionExperiment x = StillsExperiment(images); PixelMask mask(x); Stream stream(x, images); stream.Frames(0, images); stream.End(); NoWriter pusher; AcquisitionDeviceGroup group; JFJochReceiverService service(group, logger, pusher); service.NumThreads(4); service.Indexing(x.GetIndexingSettings()); service.RugnuxLive(true, nullptr, 4, 60); service.Start(x, mask, nullptr, stream.puller); REQUIRE_THROWS_AS(service.Stop(), JFJochException); CHECK(pusher.Events().empty()); CHECK(service.GetRugnuxStatus().state == RugnuxLiveState::Failed); // And the next one runs. pusher.down = false; Stream next(x, images); next.Frames(0, images); next.End(); service.Start(x, mask, nullptr, next.puller); CHECK(service.Stop().status.images_sent == images); CHECK(pusher.Events() == std::vector{"start", "image", "end"}); } TEST_CASE("JFJochReceiverRugnux_Stills_ZMQWriter", "[JFJochReceiverRugnux]") { // As the removed GPU-stills test of the FPGA receiver: five copies of one frame, through ZeroMQ to the writer. Logger logger(Catch::getResultCapture().getCurrentTestName()); RegisterHDF5Filter(); const int64_t images = 5; DiffractionExperiment x = StillsExperiment(images); x.FilePrefix("rugnux_stills_zmq").SetFileWriterFormat(FileWriterFormat::NXmxVDS).OverwriteExistingFiles(true); PixelMask mask(x); Stream stream(x, images, false, 4); stream.Frames(0, images); stream.End(); ZMQStream2Pusher pusher({"ipc://*"}); pusher.WriterNotificationSocket("ipc://*"); ZMQImagePuller puller(pusher.GetAddress()[0]); StreamWriter writer(logger, puller); auto writer_future = std::async(std::launch::async, &StreamWriter::Run, &writer); AcquisitionDeviceGroup group; JFJochReceiverService service(group, logger, pusher); service.NumThreads(4); service.Indexing(x.GetIndexingSettings()); service.RugnuxLive(true, nullptr, 4, 60); REQUIRE(!service.GetProgress().has_value()); SpotFindingSettings settings = DiffractionExperiment::DefaultDataProcessingSettings(); settings.signal_to_noise_threshold = 2.5; settings.photon_count_threshold = 5; settings.min_pix_per_spot = 1; settings.max_pix_per_spot = 200; settings.high_resolution_limit = 2.0; settings.low_resolution_limit = 50.0; service.SetSpotFindingSettings(settings); service.Start(x, mask, nullptr, stream.puller); const auto out = service.Stop(); CHECK(out.efficiency == 1.0f); REQUIRE(out.status.indexing_rate); CHECK(out.status.indexing_rate.value() == 1.0f); CHECK(out.status.images_sent == images); CHECK(!out.status.cancelled); // Done once everything is written, with the latency from the last image to then. const auto live = service.GetRugnuxStatus(); CHECK(live.state == RugnuxLiveState::Done); REQUIRE(live.last_image_to_written_s); REQUIRE(live.last_image_s); CHECK(live.elapsed_s >= live.last_image_s.value() + live.last_image_to_written_s.value() - 1e-6); REQUIRE(!service.GetProgress().has_value()); REQUIRE_NOTHROW(writer_future.get()); } TEST_CASE("JFJochReceiverRugnux_Stills_ZMQWriterDown", "[JFJochReceiverRugnux]") { // With no writer listening the collection ends with the error, and the next one runs. Logger logger(Catch::getResultCapture().getCurrentTestName()); RegisterHDF5Filter(); const int64_t images = 5; DiffractionExperiment x = StillsExperiment(images); x.FilePrefix("rugnux_stills_zmq_writer_down").SetFileWriterFormat(FileWriterFormat::NXmxVDS) .OverwriteExistingFiles(true); PixelMask mask(x); ZMQStream2Pusher pusher({"ipc://*"}); pusher.WriterNotificationSocket("ipc://*"); AcquisitionDeviceGroup group; JFJochReceiverService service(group, logger, pusher); service.NumThreads(4); service.Indexing(x.GetIndexingSettings()); service.RugnuxLive(true, nullptr, 4, 60); Stream stream(x, images, false, 4); stream.Frames(0, images); stream.End(); service.Start(x, mask, nullptr, stream.puller); REQUIRE_THROWS_AS(service.Stop(), JFJochException); REQUIRE(!service.GetProgress().has_value()); ZMQImagePuller puller(pusher.GetAddress()[0]); StreamWriter writer(logger, puller); auto writer_future = std::async(std::launch::async, &StreamWriter::Run, &writer); Stream next(x, images, false, 4); next.Frames(0, images); next.End(); service.Start(x, mask, nullptr, next.puller); const auto out = service.Stop(); CHECK(out.status.images_sent == images); CHECK(!out.status.cancelled); REQUIRE_NOTHROW(writer_future.get()); } TEST_CASE("JFJochReceiverRugnux_Stills_PrePassDoesNotFit", "[JFJochReceiverRugnux]") { // The pre-pass's images wait one to a slot in half of the slots: seven of them in a twelve-slot image buffer do // not fit, and the collection is refused at the start, with nothing sent Logger logger(Catch::getResultCapture().getCurrentTestName()); RegisterHDF5Filter(); const int64_t images = 200; DiffractionExperiment x = StillsExperiment(images); PixelMask mask(x); Stream stream(x, images); const size_t buffer_MiB = 12 * x.GetImageBufferLocationSize() / (1024 * 1024) + 1; RecordingPusher pusher; AcquisitionDeviceGroup group; JFJochReceiverService service(group, logger, pusher, buffer_MiB); service.NumThreads(2); service.Indexing(x.GetIndexingSettings()); service.RugnuxLive(true, nullptr, 4, 60, false, false, 1, 7); try { service.Start(x, mask, nullptr, stream.puller); FAIL("the collection was not refused"); } catch (const JFJochException &e) { CHECK(e.Category() == JFJochExceptionCategory::InputParameterInvalid); CHECK_THAT(e.what(), Catch::Matchers::ContainsSubstring("has 12 slots (6 for the pre-pass)")); CHECK_THAT(e.what(), Catch::Matchers::ContainsSubstring("lower stills_prepass_images to 6")); } CHECK(pusher.Events().empty()); } TEST_CASE("JFJochReceiverRugnux_Stills_BufferFullDuringPrePass", "[JFJochReceiverRugnux]") { // Many more frames than half of a twelve-slot image buffer holds, all there before the pre-pass ends. A frame past // the half waits for the pre-pass and the analysis to make room: every frame is written. Logger logger(Catch::getResultCapture().getCurrentTestName()); RegisterHDF5Filter(); const int64_t images = 200; DiffractionExperiment x = StillsExperiment(images); PixelMask mask(x); Stream stream(x, images); const size_t buffer_MiB = 12 * x.GetImageBufferLocationSize() / (1024 * 1024) + 1; RecordingPusher pusher; AcquisitionDeviceGroup group; JFJochReceiverService service(group, logger, pusher, buffer_MiB); service.NumThreads(2); service.Indexing(x.GetIndexingSettings()); service.RugnuxLive(true, nullptr, 4, 60, false, false, 1, 6); stream.Frames(0, images); stream.End(); service.Start(x, mask, nullptr, stream.puller); const auto out = service.Stop(); // No deadlock, and the end message goes out. CHECK(!out.status.cancelled); CHECK(pusher.Events() == std::vector{"start", "image", "end"}); CHECK(out.status.images_collected == images); CHECK(out.status.images_sent == images); CHECK(!out.writer_queue_full_warning); CHECK(out.scan_result.images.size() == out.status.images_sent); REQUIRE(pusher.End()); CHECK(pusher.End()->images_collected_count == images); CHECK(pusher.End()->images_sent_to_write_count == out.status.images_sent); CHECK(pusher.Images().size() == out.status.images_sent); const auto live = service.GetRugnuxStatus(); CHECK(live.state == RugnuxLiveState::Done); REQUIRE(live.stills); CHECK(live.stills->images_needed == 6); CHECK(live.stills->images_analysed == out.status.images_sent); } TEST_CASE("JFJochReceiverRugnux_Stills_PrePassImages", "[JFJochReceiverRugnux]") { // The pre-pass looks at the first stills_prepass_images frames only; every frame is analysed Logger logger(Catch::getResultCapture().getCurrentTestName()); RegisterHDF5Filter(); const int64_t images = 12; DiffractionExperiment x = StillsExperiment(images); PixelMask mask(x); Stream stream(x, images); stream.Frames(0, images); stream.End(); RecordingPusher pusher; AcquisitionDeviceGroup group; JFJochReceiverService service(group, logger, pusher); service.NumThreads(4); service.Indexing(x.GetIndexingSettings()); service.RugnuxLive(true, nullptr, 4, 60, false, false, 1, 5); service.Start(x, mask, nullptr, stream.puller); const auto out = service.Stop(); CHECK(out.status.images_sent == images); // Five frames, fewer than the 6 strong ones the geometry fit needs: the header geometry, exactly as given REQUIRE(pusher.End()); CHECK(pusher.End()->refined_beam_center_x.value() == x.GetBeamX_pxl()); CHECK(pusher.End()->refined_beam_center_y.value() == x.GetBeamY_pxl()); CHECK(pusher.End()->refined_detector_distance_mm.value() == x.GetDetectorDistance_mm()); const auto live = service.GetRugnuxStatus(); CHECK(live.state == RugnuxLiveState::Done); CHECK(HasNote(live, "Fewer than 6 strong frames in the first 5: header geometry used")); REQUIRE(live.stills); CHECK(live.stills->images_needed == 5); CHECK(live.stills->images_analysed == images); } TEST_CASE("JFJochReceiverRugnux_Rotation", "[JFJochReceiverRugnux]") { // A scan stays with rugnux's own run: the start message goes out at once, and there are no stills results. Logger logger(Catch::getResultCapture().getCurrentTestName()); RegisterHDF5Filter(); const int64_t images = 5; DiffractionExperiment x = StillsExperiment(images); x.Goniometer(GoniometerAxis("omega", 0.0f, 0.1f, {1, 0, 0}, {})); PixelMask mask(x); Stream stream(x, images); stream.Frames(0, images); stream.End(); RecordingPusher pusher; AcquisitionDeviceGroup group; JFJochReceiverService service(group, logger, pusher); service.NumThreads(4); service.Indexing(x.GetIndexingSettings()); service.RugnuxLive(true, nullptr, 4, 60); service.Start(x, mask, nullptr, stream.puller); const auto out = service.Stop(); CHECK(out.status.images_sent == images); CHECK(pusher.Events() == std::vector{"start", "image", "end"}); const auto live = service.GetRugnuxStatus(); CHECK(!live.stills); CHECK(live.images_received == images); CHECK(live.state != RugnuxLiveState::Idle); CHECK(live.state != RugnuxLiveState::Running); // The report's checklist comes with its verdict CHECK(live.verdict.empty() == live.triage.empty()); // And the report itself goes to the writer, for _report.txt const auto end = pusher.End(); REQUIRE(end); CHECK(end->report.has_value() == !live.verdict.empty()); if (end->report) CHECK(end->report->find("\nVERDICT= " + live.verdict + "\n") != std::string::npos); } TEST_CASE("JFJochReceiverRugnux_Rotation_ForceStill", "[JFJochReceiverRugnux]") { // With force_still a scan goes the stills way - pre-pass, analysis, stills merge - and the writer still gets its // goniometer. Logger logger(Catch::getResultCapture().getCurrentTestName()); RegisterHDF5Filter(); const int64_t images = 12; DiffractionExperiment x = StillsExperiment(images); x.Goniometer(GoniometerAxis("omega", 0.0f, 0.1f, {1, 0, 0}, {})); PixelMask mask(x); Stream stream(x, images); stream.Frames(0, images); stream.End(); RecordingPusher pusher; AcquisitionDeviceGroup group; JFJochReceiverService service(group, logger, pusher); service.NumThreads(4); service.Indexing(x.GetIndexingSettings()); service.RugnuxLive(true, nullptr, 4, 60, false, true); service.Start(x, mask, nullptr, stream.puller); const auto out = service.Stop(); CHECK(out.status.images_sent == images); CHECK(pusher.Events() == std::vector{"start", "image", "end"}); REQUIRE(pusher.Start()); REQUIRE(pusher.Start()->goniometer); CHECK(pusher.Start()->goniometer->GetIncrement_deg() == Catch::Approx(0.1f)); const auto end = pusher.End(); REQUIRE(end); CHECK(end->merged.has_value()); const auto live = service.GetRugnuxStatus(); CHECK(live.state == RugnuxLiveState::Done); REQUIRE(live.stills); CHECK(live.stills->images_analysed == images); CHECK(HasNote(live, "Merge of the collection")); } TEST_CASE("JFJochReceiverRugnux_ClearStatus", "[JFJochReceiverRugnux]") { // Clearing the result puts the live status back as before any collection; refused while a collection runs. Logger logger(Catch::getResultCapture().getCurrentTestName()); RegisterHDF5Filter(); const int64_t images = 8; DiffractionExperiment x = StillsExperiment(images); PixelMask mask(x); Stream stream(x, images); stream.Frames(0, 3); RecordingPusher pusher; AcquisitionDeviceGroup group; JFJochReceiverService service(group, logger, pusher); service.NumThreads(4); service.Indexing(x.GetIndexingSettings()); service.RugnuxLive(true, nullptr, 4, 60); service.Start(x, mask, nullptr, stream.puller); WaitForStatus(service, [](const RugnuxLiveSnapshot &s) { return s.stills && s.stills->images_held == 3; }); CHECK_THROWS(service.ClearRugnuxStatus()); CHECK(service.GetRugnuxStatus().state == RugnuxLiveState::Running); stream.Frames(3, images - 3); stream.End(); service.Stop(); REQUIRE(service.GetRugnuxStatus().state == RugnuxLiveState::Done); REQUIRE_NOTHROW(service.ClearRugnuxStatus()); const auto live = service.GetRugnuxStatus(); CHECK(live.state == RugnuxLiveState::Idle); CHECK(live.images_received == 0); CHECK(live.steps.empty()); CHECK(live.shells.empty()); CHECK(!live.stills); CHECK(!live.model); CHECK(!live.structure); } TEST_CASE("JFJochReceiverRugnux_Rotation_DoesNotFit", "[JFJochReceiverRugnux]") { // A scan the image buffer cannot hold is analysed image by image, and the live status says why rugnux did not run. Logger logger(Catch::getResultCapture().getCurrentTestName()); RegisterHDF5Filter(); const int64_t images = 5; DiffractionExperiment x = StillsExperiment(images); x.Goniometer(GoniometerAxis("omega", 0.0f, 0.1f, {1, 0, 0}, {})); PixelMask mask(x); Stream stream(x, images); stream.Frames(0, images); stream.End(); // Three slots const size_t buffer_MiB = (3 * x.GetImageBufferLocationSize()) / (1024 * 1024) + 1; REQUIRE(buffer_MiB * 1024 * 1024 / x.GetImageBufferLocationSize() < images); RecordingPusher pusher; AcquisitionDeviceGroup group; JFJochReceiverService service(group, logger, pusher, buffer_MiB); service.NumThreads(4); service.Indexing(x.GetIndexingSettings()); service.RugnuxLive(true, nullptr, 4, 60); service.Start(x, mask, nullptr, stream.puller); const auto out = service.Stop(); CHECK(out.status.images_collected == images); CHECK(pusher.Events() == std::vector{"start", "image", "end"}); CHECK(out.analysis_warning.find("image buffer") != std::string::npos); // The Lite receiver took the collection: every image has its spot finding, and the per-image plots are there REQUIRE(out.scan_result.images.size() == images); for (const auto &i: out.scan_result.images) CHECK(i.spot_count.has_value()); const auto spot_plot = service.GetDataProcessingPlot(PlotRequest{.type = PlotType::SpotCount, .binning = 1}); REQUIRE(spot_plot.GetPlots().size() == 1); CHECK(spot_plot.GetPlots()[0].x.size() == images); const auto live = service.GetRugnuxStatus(); CHECK(live.state == RugnuxLiveState::Skipped); CHECK(live.message.find("image buffer") != std::string::npos); } TEST_CASE("JFJochReceiverRugnux_Rotation_Cancel", "[JFJochReceiverRugnux]") { Logger logger(Catch::getResultCapture().getCurrentTestName()); RegisterHDF5Filter(); const int64_t images = 5; DiffractionExperiment x = StillsExperiment(images); x.Goniometer(GoniometerAxis("omega", 0.0f, 0.1f, {1, 0, 0}, {})); PixelMask mask(x); RecordingPusher pusher; AcquisitionDeviceGroup group; JFJochReceiverService service(group, logger, pusher); service.NumThreads(4); service.Indexing(x.GetIndexingSettings()); service.RugnuxLive(true, nullptr, 4, 60); SECTION("Before the start message") { service.Start(x, mask, nullptr, std::make_shared()); service.Cancel(false); const auto out = service.Stop(); CHECK(out.status.cancelled); CHECK(pusher.Events().empty()); } SECTION("Detector finished, no start message") { service.Start(x, mask, nullptr, std::make_shared()); service.Cancel(true); const auto out = service.Stop(); CHECK(out.status.cancelled); CHECK(pusher.Events().empty()); } SECTION("During the collection") { Stream stream(x, images); stream.Frames(0, 2); service.Start(x, mask, nullptr, stream.puller); WaitForStatus(service, [](const RugnuxLiveSnapshot &s) { return s.images_received == 2; }); service.Cancel(false); const auto out = service.Stop(); CHECK(out.status.cancelled); CHECK(out.status.images_sent == 2); CHECK(pusher.Events() == std::vector{"start", "image", "end"}); const auto live = service.GetRugnuxStatus(); CHECK(live.cut == "cancel"); CHECK(live.state != RugnuxLiveState::Running); // A cancelled collection, not a failed analysis, and counted in the images that came CHECK(live.verdict == "CANCELLED"); CHECK(live.verdict_text.find("after 2 images") != std::string::npos); for (const auto &step: live.steps) { const size_t pos = step.title.rfind(", "); if (pos != std::string::npos && step.title.ends_with(" images")) CHECK(std::stoll(step.title.substr(pos + 2)) <= 2); } } SECTION("A silent cancel lets the collection finish") { Stream stream(x, images); stream.Frames(0, 2); service.Start(x, mask, nullptr, stream.puller); WaitForStatus(service, [](const RugnuxLiveSnapshot &s) { return s.images_received == 2; }); service.Cancel(true); stream.Frames(2, images - 2); stream.End(); const auto out = service.Stop(); CHECK(!out.status.cancelled); CHECK(out.status.images_sent == images); CHECK(pusher.Events() == std::vector{"start", "image", "end"}); CHECK(service.GetRugnuxStatus().cut.empty()); } } // A leaf joins its threads in its own destructor, with no one having asked for the end of the collection. TEST_CASE("JFJochReceiverRugnux_DestroyedMidRun", "[JFJochReceiverRugnux]") { Logger logger(Catch::getResultCapture().getCurrentTestName()); RegisterHDF5Filter(); const int64_t images = 5; DiffractionExperiment x = StillsExperiment(images); PixelMask mask(x); Stream stream(x, images); stream.Frames(0, 2); RecordingPusher pusher; JFJochReceiverCurrentStatus status; JFJochReceiverPlots plots; RugnuxLiveStatus live; ImageBuffer buffer(512 * 1024 * 1024); SECTION("JFJochReceiverLite") { JFJochReceiverLite receiver(x, mask, *stream.puller, pusher, logger, 4, DiffractionExperiment::DefaultDataProcessingSettings(), status, plots, buffer); std::this_thread::sleep_for(std::chrono::milliseconds(200)); } SECTION("JFJochReceiverDarkMask") { DiffractionExperiment dark = x; dark.Mode(DetectorMode::DarkMask); JFJochReceiverDarkMask receiver(dark, mask, *stream.puller, pusher, logger, 4, DiffractionExperiment::DefaultDataProcessingSettings(), status, plots, buffer); std::this_thread::sleep_for(std::chrono::milliseconds(200)); } SECTION("JFJochReceiverRugnuxRotation") { DiffractionExperiment rotation = x; rotation.Goniometer(GoniometerAxis("omega", 0.0f, 0.1f, {1, 0, 0}, {})); JFJochReceiverRugnuxRotation receiver(rotation, mask, *stream.puller, pusher, logger, 4, 4, nullptr, 60, false, live, DiffractionExperiment::DefaultDataProcessingSettings(), status, plots, buffer); std::this_thread::sleep_for(std::chrono::milliseconds(200)); } SECTION("JFJochReceiverRugnuxStills") { JFJochReceiverRugnuxStills receiver(x, mask, *stream.puller, pusher, logger, 4, nullptr, 60, false, live, DiffractionExperiment::DefaultDataProcessingSettings(), status, plots, buffer); std::this_thread::sleep_for(std::chrono::milliseconds(200)); } SUCCEED("the destructor joined the threads"); } TEST_CASE("JFJochReceiverRugnux_DarkMaskStaysOut", "[JFJochReceiverRugnux]") { // The mask sequence of a later POST /pedestal, with rugnux on: the state machine's experiment still has the // last /start's goniometer, or none. Logger logger(Catch::getResultCapture().getCurrentTestName()); RegisterHDF5Filter(); DiffractionExperiment x(DetDECTRIS(2068, 2164, "Test", {})); SECTION("after a serial collection") {} SECTION("after a rotation collection") { x.Goniometer(GoniometerAxis("omega", 0.0f, 0.1f, {1, 0, 0}, {})); } DarkMaskSettings mask_settings; mask_settings.NumberOfFrames(10).MaxCounts(1).MaxFramesWithCounts(5); x.Mode(DetectorMode::DarkMask).ImportDarkMaskSettings(mask_settings); PixelMask mask(x); auto puller = std::make_shared(); StartMessage start_msg; x.FillMessage(start_msg); puller->Put(ImagePullerOutput{.cbor = std::make_shared(start_msg)}); std::vector image(x.GetPixelsNum(), 0); image[2] = 2; for (int i = 0; i < x.GetFrameNum(); i++) { DataMessage message; message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum()); message.number = i; puller->Put(ImagePullerOutput{.cbor = std::make_shared(message)}); } puller->Put(ImagePullerOutput{.cbor = std::make_shared(EndMessage{})}); RecordingPusher pusher; AcquisitionDeviceGroup group; JFJochReceiverService service(group, logger, pusher); service.NumThreads(4); service.RugnuxLive(true, nullptr, 4, 60); service.Start(x, mask, nullptr, puller); const auto out = service.Stop(); REQUIRE(out.dark_mask_result.size() == x.GetPixelsNum()); CHECK(out.dark_mask_result[2] != 0); CHECK(out.dark_mask_result[0] == 0); CHECK(out.status.images_sent == 0); CHECK(!service.GetRugnuxStatus().stills); }