v1.0.0-rc.178 (#88)

* Rugnux: Processes serial crystallography data from XFELs - SACLA (MPCCD), PAL-XFEL and European XFEL (CXI), LCLS (XTC) and SwissFEL Alvra (JUNGFRAU with a CrystFEL module geometry) - with several files or Jungfraujoch master files merged in one run (`rugnux *_master.h5`).
* Rugnux: Stills processing takes a known unit cell and space group as a starting point, finds the beam centre (and, with a cell, the detector distance) from the images, detects the saturation level in other facilities' files and refines the placement of individual detector panels.
* Rugnux: The stills report gives R_split and CC1/2 from frame half-sets, comparable with CrystFEL.
* jfjoch_viewer: Opens the same XFEL file formats.
* Validation: A stills battery checks Rugnux against deposited structures from SACLA, PAL-XFEL, LCLS, European XFEL and SwissFEL.
* Rugnux: More reliable convergence of rotation scaling, and live molecular replacement starts as soon as the model is uploaded.
* jfjoch_broker: Initialize clears the previous dataset's bearer tokens, image buffer and live Rugnux status, and a slow CUDA JIT cache is reported at start.

Reviewed-on: #88
Co-authored-by: Filip Leonarski <filip.leonarski@psi.ch>

tests/JFJochReaderTest.cpp

@@ -10,10 +10,12 @@
 #include "../reader/JFJochCBFReader.h"
 #include "../reader/JFJochMarCCDReader.h"
 #include "../reader/JFJochSMVReader.h"
+#include "../reader/JFJochXTCReader.h"
 #include "../reader/MiniCBF.h"
 #include "../compression/JFJochCompressor.h"
 
 #include <cstring>
+#include <filesystem>
 #include <fstream>
 #include <future>
 #include <iomanip>
@@ -4832,3 +4834,100 @@ TEST_CASE("JFJochReader_DECTRISMultichannel", "[HDF5][Full]") {
     // No leftover HDF5 objects
     REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
 }
+
+// LCLS-I XTC: datagrams of a 40-byte header and a tree of Xtcs; a camera image is a Camera::FrameV1
+// leaf. The fixture writes a Configure carrying a Rayonix configuration, and L1Accepts with the
+// electron beam's photon energy. The first event carries a second, smaller camera beside the
+// diffraction detector, which the reader must pass over; one event's image is damaged, and the stream
+// ends inside a last event, as a partly downloaded file does.
+namespace {
+    void PutWords(std::string &s, std::initializer_list<uint32_t> words) {
+        for (uint32_t w: words)
+            s.append(reinterpret_cast<const char *>(&w), 4);
+    }
+
+    std::string XtcLeaf(uint32_t type, const std::string &payload, uint32_t source = 0, uint32_t damage = 0) {
+        std::string s;
+        PutWords(s, {damage, 0, source, type, static_cast<uint32_t>(20 + payload.size())});
+        return s + payload;
+    }
+
+    std::string XtcFrame(uint32_t width, uint32_t height, uint16_t base, uint32_t source, uint32_t damage = 0) {
+        std::string p;
+        PutWords(p, {width, height, 16, 3});   // an offset of 3 counts, taken off when read
+        for (uint32_t i = 0; i < width * height; i++) {
+            const uint16_t v = base + i % 100;
+            p.append(reinterpret_cast<const char *>(&v), 2);
+        }
+        return XtcLeaf(2 | (1u << 16), p, source, damage);
+    }
+
+    std::string XtcEBeam(double photon_energy_eV) {
+        std::string p(4 + 20 * 8, '\0');
+        std::memcpy(p.data() + 4 + 17 * 8, &photon_energy_eV, 8);
+        return XtcLeaf(15 | (7u << 16), p);
+    }
+
+    std::string XtcRayonixConfig(const std::string &model, uint8_t binning) {
+        std::string p(60, '\0');
+        p[0] = static_cast<char>(binning);
+        p[1] = static_cast<char>(binning);
+        p.replace(20, model.size(), model);
+        return XtcLeaf(73 | (2u << 16), p);
+    }
+
+    std::string XtcDatagram(uint32_t transition, const std::string &payload) {
+        std::string s;
+        PutWords(s, {0, 0, transition << 24, 0, 0});
+        PutWords(s, {0, 0, 0, 1, static_cast<uint32_t>(20 + payload.size())});
+        return s + payload;
+    }
+}
+
+TEST_CASE("JFJochXTCReader", "[portable]") {
+    // 1920 wide at 4x4 binning: a whole MX340-HS readout, so 340 mm / 1920 pixels.
+    const uint32_t nx = 1920, ny = 4;
+    std::filesystem::remove_all("xtc_test");
+    std::filesystem::create_directories("xtc_test");
+    {
+        std::ofstream f("xtc_test/run-s00-c00.xtc", std::ios::binary);
+        f << XtcDatagram(4, XtcLeaf(1, XtcRayonixConfig("MX340-HS:125", 4)));
+        f << XtcDatagram(12, XtcLeaf(1, XtcLeaf(13, std::string(32, '\0')) + XtcFrame(4, 4, 7, 2) + XtcEBeam(9500))
+                             + XtcLeaf(1, XtcFrame(nx, ny, 100, 1)));
+        f << XtcDatagram(12, XtcLeaf(1, XtcFrame(nx, ny, 300, 1, 0x4000) + XtcEBeam(9520)));   // damaged
+        f << XtcDatagram(12, XtcLeaf(1, XtcFrame(nx, ny, 200, 1) + XtcEBeam(9510)));
+        const std::string cut = XtcDatagram(12, XtcLeaf(1, XtcFrame(nx, ny, 400, 1)));
+        f << cut.substr(0, cut.size() / 2);
+    }
+
+    REQUIRE(JFJochXTCReader::CanRead("xtc_test/run-s00-c00.xtc"));
+    CHECK_FALSE(JFJochXTCReader::CanRead("xtc_test"));   // a directory is not read
+
+    {
+        // The XTC's own beam records and Rayonix configuration.
+        JFJochXTCReader reader;
+        REQUIRE_NOTHROW(reader.ReadFiles({"xtc_test/run-s00-c00.xtc"}));
+        CHECK(reader.GetNumberOfImages() == 2);
+        CHECK(reader.HasWavelength());
+        CHECK(reader.HasPixelSize());
+        const auto x = reader.GetDataset()->experiment;
+        CHECK(x.GetXPixelsNum() == nx);
+        CHECK(x.GetPixelSize_mm() == Catch::Approx(340.0 / 1920.0));
+        // The damaged image's beam record still counts: three records, 9500, 9520 and 9510 eV.
+        CHECK(x.GetIncidentEnergy_keV() == Catch::Approx(9.510).epsilon(1e-5));
+        CHECK(x.GetBeamX_pxl() == Catch::Approx(nx / 2.0));
+        for (int i = 0; i < 2; i++) {
+            auto raw = reader.GetRawImage(i);
+            REQUIRE(raw);
+            const auto *pixels = reinterpret_cast<const int32_t *>(raw->image_buffer.data());
+            CHECK(pixels[0] == 100 * (i + 1) - 3);
+            CHECK(pixels[99] == 100 * (i + 1) + 99 - 3);
+        }
+        reader.ApplyGain(2.0f);
+        CHECK(reader.CountsInPhotons());
+        auto raw = reader.GetRawImage(0);
+        CHECK(reinterpret_cast<const int32_t *>(raw->image_buffer.data())[1] == 49);   // (100 + 1 - 3) / 2
+    }
+
+    std::filesystem::remove_all("xtc_test");
+}