v1.0.0-rc.179 (#90)

* Rugnux: Stills processing needs only the unit cell and space group beyond what the file records - detector distance, beam centre, tilt, panel placement, saturation level, gain and polarisation plane are found from the images - and is up to twice as fast.
* Rugnux: Stills can be indexed without a GPU, with a CPU implementation of the fast-feedback indexer (macOS, computers without a GPU).
* Rugnux: Rotation and stills data are integrated in the detector's own units with one gain model, improving data from CCD detectors.
* Web frontend: The Rugnux live page shows the placed model with its electron density and anomalous difference map in a Mol* viewer.

Reviewed-on: #90
Co-authored-by: Filip Leonarski <filip.leonarski@psi.ch>

reader/CrystFELGeometry.cpp

@@ -3,6 +3,9 @@
 
 #include "CrystFELGeometry.h"
 
+#include <algorithm>
+#include <array>
+#include <filesystem>
 #include <fstream>
 #include <map>
 #include <regex>
@@ -98,8 +101,8 @@ bool Bookkeeping(const std::string &key) {
 // Keywords that change where a pixel lands or what it is worth, which this reader does not do.
 const std::set<std::string> REFUSED = {"detector_shift_x", "detector_shift_y"};
 
-// Sets one panel property; false if the key is not one. Only where the panel is: its pixel range, axes,
-// corner, pixel size, depth offset and camera length.
+// Sets one panel property; false if the key is not one. Where the panel is - its pixel range, axes,
+// corner, pixel size, depth offset and camera length - and which of its pixels are left out.
 bool SetPanel(const std::string &filename, Panel &p, const std::string &key, const std::string &value) {
     if (key == "min_fs") p.min_fs = std::stoll(value);
     else if (key == "max_fs") p.max_fs = std::stoll(value);
@@ -112,11 +115,58 @@ bool SetPanel(const std::string &filename, Panel &p, const std::string &key, con
     else if (key == "res") p.res = std::stod(value);
     else if (key == "clen") p.clen = ParseValue(filename, key, value, LENGTH_UNITS, 1e-3);
     else if (key == "coffset") p.coffset = std::stod(value);
+    else if (key == "mask_edge_pixels") p.mask_edge_pixels = std::stoll(value);
+    else if (key == "no_index") p.no_index = value == "1" || value == "true";
     else
         return false;
     return true;
 }
 
+// The up to eight masks a panel has (maskN_data, maskN_file, maskN_goodbits, maskN_badbits, N = 0..7;
+// mask, mask_file, mask_good and mask_bad are mask 0's). Sets the one the key names; false if the key
+// is not a mask key.
+using Masks = std::array<xfel::Mask, 8>;
+bool SetMask(Masks &masks, const std::string &key, const std::string &value) {
+    static const std::map<std::string, std::string> OLD = {
+        {"mask", "data"}, {"mask_file", "file"}, {"mask_good", "goodbits"}, {"mask_bad", "badbits"}};
+    static const std::regex CURRENT(R"(mask([0-7])_(data|file|goodbits|badbits))");
+    size_t n = 0;
+    std::string field;
+    std::smatch match;
+    if (OLD.contains(key))
+        field = OLD.at(key);
+    else if (std::regex_match(key, match, CURRENT)) {
+        n = std::stoul(match[1].str());
+        field = match[2].str();
+    } else
+        return false;
+    if (field == "data") masks[n].data = value;
+    else if (field == "file") masks[n].file = value;
+    else if (field == "goodbits") masks[n].good = static_cast<uint32_t>(std::stoul(value, nullptr, 0));
+    else masks[n].bad = static_cast<uint32_t>(std::stoul(value, nullptr, 0));
+    return true;
+}
+
+// Sets one property of a bad region: a box in the laboratory plane (min_x .. max_y, in pixels) or a
+// pixel range in the stored frame (min_fs .. max_ss, inclusive) on one panel - or, in the old syntax,
+// with no panel, on the whole frame. False if the key is not one.
+bool SetBad(xfel::BadRegion &b, const std::string &key, const std::string &value) {
+    if (key == "min_x") b.min_x = std::stod(value);
+    else if (key == "max_x") b.max_x = std::stod(value);
+    else if (key == "min_y") b.min_y = std::stod(value);
+    else if (key == "max_y") b.max_y = std::stod(value);
+    else if (key == "min_fs") b.min_fs = std::stoll(value);
+    else if (key == "max_fs") b.max_fs = std::stoll(value);
+    else if (key == "min_ss") b.min_ss = std::stoll(value);
+    else if (key == "max_ss") b.max_ss = std::stoll(value);
+    else if (key == "panel") b.panel = value;
+    else
+        return false;
+    if (key.ends_with("_x") || key.ends_with("_y"))
+        b.in_lab = true;
+    return true;
+}
+
 }  // namespace
 
 xfel::Layout Read(const std::string &filename, std::vector<std::string> &ignored) {
@@ -128,7 +178,9 @@ xfel::Layout Read(const std::string &filename, std::vector<std::string> &ignored
     // Global panel keys hold for every panel that appears after them, as in the old syntax; a file in
     // the current syntax states them first, so this reads both the same.
     Panel defaults;
-    std::map<std::string, size_t> panel_index;
+    Masks default_masks;
+    std::vector<Masks> panel_masks;    // each panel's, beside geom.panels
+    std::map<std::string, size_t> panel_index, bad_index;
     std::map<std::string, std::string> dims;
 
     std::string line;
@@ -147,7 +199,8 @@ xfel::Layout Read(const std::string &filename, std::vector<std::string> &ignored
         if (slash == std::string::npos) {
             if (key == "data") geom.data = value;
             else if (key.starts_with("dim")) dims[key] = value;
-            else if (!SetPanel(filename, defaults, key, value) && !Bookkeeping(key))
+            else if (!SetPanel(filename, defaults, key, value) && !SetMask(default_masks, key, value)
+                     && !Bookkeeping(key))
                 ignored.push_back(key);
             continue;
         }
@@ -156,7 +209,12 @@ xfel::Layout Read(const std::string &filename, std::vector<std::string> &ignored
         const std::string prop = key.substr(slash + 1);
 
         if (name.starts_with("bad")) {
-            ignored.push_back(key);
+            if (!bad_index.contains(name)) {
+                bad_index[name] = geom.bad.size();
+                geom.bad.emplace_back();
+            }
+            if (!SetBad(geom.bad[bad_index[name]], prop, value))
+                ignored.push_back(key);
             continue;
         }
 
@@ -173,8 +231,11 @@ xfel::Layout Read(const std::string &filename, std::vector<std::string> &ignored
             panel_index[name] = geom.panels.size();
             geom.panels.push_back(defaults);
             geom.panels.back().name = name;
+            panel_masks.push_back(default_masks);
         }
-        if (!SetPanel(filename, geom.panels[panel_index[name]], prop, value) && !Bookkeeping(prop))
+        const size_t n = panel_index[name];
+        if (!SetPanel(filename, geom.panels[n], prop, value) && !SetMask(panel_masks[n], prop, value)
+            && !Bookkeeping(prop))
             ignored.push_back(key);
     }
 
@@ -195,6 +256,33 @@ xfel::Layout Read(const std::string &filename, std::vector<std::string> &ignored
             throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
                                   filename + ": panel " + p.name + " has no pixel size (res)");
     }
+
+    // Each mask once for the whole frame where every panel has it alike (the usual case: stated once,
+    // before the panels), otherwise once for each panel that has it, on that panel's pixels only.
+    for (size_t n = 0; n < default_masks.size(); n++) {
+        const auto &first = panel_masks[0][n];
+        const bool alike = std::all_of(panel_masks.begin(), panel_masks.end(),
+                                       [&](const Masks &m) { return m[n] == first; });
+        if (alike) {
+            if (!first.data.empty())
+                geom.masks.push_back(first);
+            continue;
+        }
+        for (size_t i = 0; i < geom.panels.size(); i++) {
+            if (panel_masks[i][n].data.empty())
+                continue;
+            geom.masks.push_back(panel_masks[i][n]);
+            geom.masks.back().panel = geom.panels[i].name;
+        }
+    }
+    // A mask file is named as it is, or relative to the working directory, as CrystFEL reads it; a
+    // relative one not found there is looked for beside the geometry file.
+    for (auto &m: geom.masks) {
+        const std::filesystem::path file(m.file);
+        const auto beside = std::filesystem::path(filename).parent_path() / file;
+        if (!m.file.empty() && file.is_relative() && !std::filesystem::exists(file) && std::filesystem::exists(beside))
+            m.file = beside.string();
+    }
     return geom;
 }