1.0.0-rc.174 (#84)

* Rugnux: Performance improvements on GPU and CPU (more of the pre-scan and of scaling on the GPU, faster CPU spot finding and crystal refinement), with unchanged results.
* Rugnux: More robust processing - patches of persistently hot pixels are masked, an inconsistent merge triggers a retry at the measured beam centre, and builds targeting different CPU levels give the same results.
* Rugnux: Improved scaling and merging - reflections with an overloaded pixel are dropped, as in XDS, sparse rotation sweeps are scaled more reliably, and French-Wilson amplitudes use an anisotropic Wilson prior.
* Rugnux: Improved space-group determination - glide planes in groups without a centre of symmetry, screw axes from short or weak axial rows kept when a higher group is adopted, and more reliable decisions on twinned and pseudo-symmetric crystals.
* Rugnux: Improved small-molecule processing - spots that grow wider than the integration disk and split spots are integrated over their measured footprint, sparse lattices are integrated on every frame, and the `.hkl` file holds unmerged scaled reflections (SHELX HKLF 4).
* Rugnux: Reads Rigaku d*TREK SMV images (Saturn CCD), including detector 2theta and encoded pixel overflows; home-source (rotating-anode) datasets were added to the validation battery.
* jfjoch_viewer: Fixed processing failing at the end with "Wrong JPEG library version" on Linux; the merge window shows the space group with proper subscripts and a checklist of crystal pathologies.

Reviewed-on: #84
Co-authored-by: Filip Leonarski <filip.leonarski@psi.ch>

reader/SMV.cpp

@@ -10,6 +10,7 @@
 #include <fstream>
 #include <map>
 #include <optional>
+#include <sstream>
 #include <tuple>
 
 #include "../common/JFJochException.h"
@@ -86,6 +87,115 @@ double NumAny(const std::map<std::string, std::string> &kv,
     return dflt;
 }
 
+// A value holding several numbers, as d*TREK writes its vectors and circle lists.
+std::vector<double> Nums(const std::map<std::string, std::string> &kv, const std::string &key) {
+    std::vector<double> out;
+    const auto it = kv.find(key);
+    if (it == kv.end()) return out;
+    std::istringstream in(it->second);
+    double v;
+    while (in >> v) out.push_back(v);
+    return out;
+}
+
+std::vector<std::string> Words(const std::map<std::string, std::string> &kv, const std::string &key) {
+    std::vector<std::string> out;
+    const auto it = kv.find(key);
+    if (it == kv.end()) return out;
+    std::istringstream in(it->second);
+    std::string w;
+    while (in >> w) out.push_back(w);
+    return out;
+}
+
+// Following Pflugrath (1999) Acta Cryst. D55, 1718-1725 (d*TREK) and dxtbx, Parkhurst et al. (2014)
+// J. Appl. Cryst. 47, 1459-1465.
+// d*TREK, as Rigaku's CrystalClear writes it for the Saturn and R-AXIS families. Read the way dxtbx's
+// FormatSMVRigakuSaturn reads it: the image directions are <det>DETECTOR_VECTORS combined by
+// <det>SPATIAL_DISTORTION_VECTORS, the point of normal incidence (pixels) and the pixel size (mm) come
+// from <det>SPATIAL_DISTORTION_INFO, and the detector circles from <det>GONIO_*, where the distance is
+// a translation along the normal applied before the rotations - so the beam position stays the PONI
+// and the distance the normal distance whatever the arm does.
+//
+// The distortion vectors are not optional. dxtbx's variant for headers without DTREK_DATE_TIME
+// leaves them out, and on a Saturn 944+ sweep that puts the spindle 90 degrees off: DIALS indexes
+// 12% of the spots with it and 98% with the vectors applied, at the deposited cell.
+void ParseDtrek(const std::map<std::string, std::string> &kv, const std::string &path, Header &h) {
+    const auto detector_names = Words(kv, "DETECTOR_NAMES");
+    const std::string det = detector_names.empty() ? "" : detector_names[0];
+    const auto info = Nums(kv, det + "SPATIAL_DISTORTION_INFO");
+    const auto vectors = Nums(kv, det + "DETECTOR_VECTORS");
+    const auto names = Words(kv, det + "GONIO_NAMES");
+    const auto units = Words(kv, det + "GONIO_UNITS");
+    const auto values = Nums(kv, det + "GONIO_VALUES");
+    const auto circle_vectors = Nums(kv, det + "GONIO_VECTORS");
+    const auto rotation = Nums(kv, "ROTATION");      // start, end, increment, exposure, ...
+    const auto spindle = Nums(kv, "ROTATION_VECTOR");
+    if (info.size() < 4 || vectors.size() < 6 || rotation.size() < 4 || spindle.size() < 3
+        || names.size() != values.size() || units.size() != values.size()
+        || circle_vectors.size() != 3 * values.size())
+        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
+                              path + ": d*TREK header does not describe the detector and the rotation");
+
+    const auto type = kv.find("DATA_TYPE");
+    if (type != kv.end() && type->second.find("short") == std::string::npos)
+        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
+                              path + ": only 16-bit SMV images are supported (Data_type=" + type->second + ")");
+
+    h.beam_x_px = info[0];
+    h.beam_y_px = info[1];
+    h.pixel_x_m = info[2] * 1e-3;
+    h.pixel_y_m = info[3] * 1e-3;
+    // Each image direction as a combination of the two detector vectors: fast = d0 f + d1 s,
+    // slow = d2 f + d3 s. Absent, the detector vectors are the image directions.
+    auto d = Nums(kv, det + "SPATIAL_DISTORTION_VECTORS");
+    if (d.size() < 4)
+        d = {1, 0, 0, 1};
+    std::array<double, 3> fast{}, slow{};
+    for (int i = 0; i < 3; i++) {
+        fast[i] = d[0] * vectors[i] + d[1] * vectors[3 + i];
+        slow[i] = d[2] * vectors[i] + d[3] * vectors[3 + i];
+    }
+    h.fast_direction = fast;
+    h.slow_direction = slow;
+
+    h.distance_m = 0;
+    for (size_t i = 0; i < values.size(); i++) {
+        const std::array<double, 3> axis{circle_vectors[3 * i], circle_vectors[3 * i + 1],
+                                         circle_vectors[3 * i + 2]};
+        if (units[i] == "deg")
+            h.detector_circles.push_back({axis, values[i]});
+        else if (names[i] == "Distance")
+            h.distance_m = values[i] * 1e-3;
+        else if (values[i] != 0.0)
+            throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
+                                  path + ": d*TREK detector translation " + names[i] + " is not supported");
+    }
+
+    h.start_angle_deg = rotation[0];
+    h.angle_increment_deg = rotation[2];
+    h.exposure_s = rotation[3];
+    h.spindle_axis = std::array<double, 3>{spindle[0], spindle[1], spindle[2]};
+    const auto axis_name = Words(kv, "ROTATION_AXIS_NAME");
+    if (!axis_name.empty()) {
+        h.axis_name = axis_name[0];
+        std::transform(h.axis_name.begin(), h.axis_name.end(), h.axis_name.begin(),
+                       [](unsigned char c) { return std::tolower(c); });
+    }
+
+    // SOURCE_WAVELENGTH leads with HOW MANY wavelengths follow ("1.000000 1.541780"), so its first
+    // number is a count, not a wavelength. SCAN_WAVELENGTH is the one the scan used.
+    const auto scan_wavelength = Nums(kv, "SCAN_WAVELENGTH");
+    const auto source_wavelength = Nums(kv, "SOURCE_WAVELENGTH");
+    h.wavelength_A = !scan_wavelength.empty() ? scan_wavelength[0]
+                   : (source_wavelength.size() >= 2 ? source_wavelength[1] : 0.0);
+
+    h.overflow_ratio = static_cast<int64_t>(Num(kv, "RAXIS_COMPRESSION_RATIO"));
+    const auto id = kv.find(det + "DETECTOR_IDENTIFICATION");
+    if (id != kv.end())
+        h.detector = id->second;
+}
+
 std::optional<std::map<std::string, std::string>> HeaderBlock(const std::string &path) {
     return ParseBlock(ReadPrefix(path, PROBE_BYTES));
 }
@@ -123,7 +233,7 @@ Header Parse(const std::map<std::string, std::string> &kv, const std::string &pa
     h.beam_x_px = h.pixel_x_m > 0 ? bx_mm * 1e-3 / h.pixel_x_m : 0.0;
     h.beam_y_px = h.pixel_y_m > 0 ? by_mm * 1e-3 / h.pixel_y_m : 0.0;
 
-    h.wavelength_A = NumAny(kv, {"WAVELENGTH", "SOURCE_WAVELENGTH"});
+    h.wavelength_A = NumAny(kv, {"WAVELENGTH"});
     h.angle_increment_deg = NumAny(kv, {"OSC_RANGE", "OSCILLATION_RANGE"});
     // OSC_START is the angle of THIS image; PHI is where the circle stands, which is the same
     // thing on the single-axis goniometers that write this format, and is the only value some
@@ -142,6 +252,8 @@ Header Parse(const std::map<std::string, std::string> &kv, const std::string &pa
         std::transform(a.begin(), a.end(), a.begin(), [](unsigned char c) { return std::tolower(c); });
         h.axis_name = a;
     }
+    if (kv.count("DETECTOR_NAMES"))
+        ParseDtrek(kv, path, h);
     return h;
 }
 
@@ -276,6 +388,11 @@ Header ReadInto(const std::string &path, int32_t *out, size_t capacity, std::vec
         for (size_t i = 0; i < npixel; i++)
             out[i] = static_cast<int32_t>(static_cast<uint16_t>((src[2 * i] << 8) | src[2 * i + 1]));
     }
+    if (h.overflow_ratio > 0) {
+        for (size_t i = 0; i < npixel; i++)
+            if (out[i] > 32767)
+                out[i] = static_cast<int32_t>((out[i] - 32768) * h.overflow_ratio);
+    }
     return h;
 }