JUNGFRAU modules for Jungfraujoch data: written into new files, rebuilt from the mask of old ones

Stills module placement needs the detector's modules, which came only from NXdetector_module
groups, CrystFEL panels or --module-geometry; a Jungfraujoch master carries none (or one
whole-image module), so placement never ran on our own JUNGFRAU 4M/8M/9M data.

- A JUNGFRAU experiment that assembles its image now puts the modules it places into the start
  message, so the master file holds module_0..N (placed as imaged) and the reader gives them back.
  The broker's own processing is unchanged.
- A JUNGFRAU file without module groups (decided from the detector description) gets its modules
  from the pixel mask: every region clear of the module-gap bit must be one whole 1030 x 514
  module either way round, else none are taken. DECTRIS files, whose masks use the same bit, are
  not touched.
- Rotation runs keep processing the image as one panel where the modules sit as imaged (no
  module step there); merged data bit-identical on a JUNGFRAU 4M rotation set.

On the 8M and 9M serial sets 16/18 modules are found; the held-out test favours placement on all
three (capped rms 1.651 -> 1.615, 1.367 -> 1.350, 1.498 -> 1.365 px) but HDF5 input keeps the
existing measure-and-report rule, so only the per-panel spot finding / integration changes:
CCref last shell 0.273 -> 0.272, 0.421 -> 0.426, 0.443 -> 0.444 (noise level).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HVeDQqN9qRD57vnShzTC37

common/DetectorModules.cpp

@@ -4,6 +4,7 @@
 #include "DetectorModules.h"
 #include "DetectorGeometry.h"
 #include "Definitions.h"
+#include "PixelMask.h"
 
 #include <algorithm>
 #include <cmath>
@@ -193,3 +194,39 @@ DetectorModules ModulesAsImaged(const DetectorGeometry &geom) {
     }
     return DetectorModules(out);
 }
+
+DetectorModules ModulesFromGapMask(const std::vector<uint32_t> &mask, int64_t width, int64_t height) {
+    if (width <= 0 || height <= 0 || mask.size() != static_cast<size_t>(width * height))
+        return {};
+    const auto clear = [&](int64_t x, int64_t y) {
+        return x >= 0 && y >= 0 && x < width && y < height
+               && (mask[y * width + x] & (1u << PixelMask::ModuleGapPixelBit)) == 0;
+    };
+    std::vector<uint8_t> taken(mask.size(), 0);
+    std::vector<DetectorModule> out;
+    for (int64_t y = 0; y < height; y++)
+        for (int64_t x = 0; x < width; x++) {
+            if (!clear(x, y) || taken[y * width + x])
+                continue;
+            // The first clear pixel of a region not yet taken is a module's corner: the module runs to the
+            // gap along the row and down the column, and has to be whole and bounded by gap all round.
+            int64_t w = 0, h = 0;
+            while (clear(x + w, y)) w++;
+            while (clear(x, y + h)) h++;
+            if (!((w == CONVERTED_MODULE_COLS && h == CONVERTED_MODULE_LINES)
+                  || (w == CONVERTED_MODULE_LINES && h == CONVERTED_MODULE_COLS)))
+                return {};
+            for (int64_t yy = y - 1; yy <= y + h; yy++)
+                for (int64_t xx = x - 1; xx <= x + w; xx++) {
+                    const bool inside = xx >= x && xx < x + w && yy >= y && yy < y + h;
+                    if (clear(xx, yy) != inside)
+                        return {};
+                    if (inside)
+                        taken[yy * width + xx] = 1;
+                }
+            out.push_back(DetectorModule::AsImaged(x, y, w, h));
+        }
+    if (out.size() < 2)
+        return {};
+    return DetectorModules(out);
+}

common/DetectorModules.h

@@ -121,3 +121,9 @@ public:
 // CONVERTED_MODULE_COLS x CONVERTED_MODULE_LINES turned the way the geometry turns it, placed as imaged.
 // Empty for a detector of one module.
 DetectorModules ModulesAsImaged(const DetectorGeometry &geom);
+
+// The modules of a JUNGFRAU image where only its pixel mask says where they are: every region of pixels
+// clear of the module-gap bit (PixelMask::ModuleGapPixelBit) has to be one whole module,
+// CONVERTED_MODULE_COLS x CONVERTED_MODULE_LINES either way round, and is taken as placed as imaged.
+// Empty where any region is not, or where there are fewer than two. The mask is row-major, width x height.
+DetectorModules ModulesFromGapMask(const std::vector<uint32_t> &mask, int64_t width, int64_t height);

common/DiffractionExperiment.cpp

@@ -653,6 +653,10 @@ SpotFindingSettings DiffractionExperiment::DefaultDataProcessingSettings() {
 void DiffractionExperiment::FillMessage(StartMessage &message) const {
     message.images_per_file = GetImagesPerFile();
     message.detector_modules = GetModules().Get();
+    // A JUNGFRAU image assembled here is made of modules this experiment places itself: they go into
+    // the file one by one, so that what reads it back knows them.
+    if (message.detector_modules.empty() && GetDetectorType() == DetectorType::JUNGFRAU && IsGeometryTransformed())
+        message.detector_modules = ModulesAsImaged(detector.GetGeometry()).Get();
     message.beam_center_x = GetBeamX_pxl();
     message.beam_center_y = GetBeamY_pxl();
     const auto direct_beam = GetDiffractionGeometry().GetDirectBeam_pxl();

reader/HDF5MetadataSource.cpp

@@ -1,6 +1,8 @@
 // SPDX-FileCopyrightText: 2025 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
 // SPDX-License-Identifier: GPL-3.0-only
 
+#include <algorithm>
+#include <cctype>
 #include <cmath>
 #include <cstring>
 #include <set>
@@ -1148,6 +1150,14 @@ HDF5MetadataSource::OpenResult HDF5MetadataSource::Open(const std::string &filen
             }
             if (mask_tmp.empty())
                 mask_tmp = std::vector<uint32_t>(image_size_x * image_size_y);
+            // A JUNGFRAU file that does not state its modules one by one still marks the gaps between
+            // them in its mask: each region clear of the gap bit is one module, placed as imaged. Only for
+            // JUNGFRAU - a DECTRIS mask uses the same bit for its gaps, and its modules are not read here.
+            std::string detector_upper = detector_name;
+            std::transform(detector_upper.begin(), detector_upper.end(), detector_upper.begin(),
+                           [](unsigned char c) { return static_cast<char>(std::toupper(c)); });
+            if (dataset->experiment.GetModules().empty() && detector_upper.find("JUNGFRAU") != std::string::npos)
+                dataset->experiment.Modules(ModulesFromGapMask(mask_tmp, image_size_x, image_size_y));
             dataset->pixel_mask = std::make_shared<const PixelMask>(mask_tmp);
         }
 

rugnux/Rugnux.cpp

@@ -387,6 +387,12 @@ Rugnux::Rugnux(JFJochReader &reader, DiffractionExperiment experiment,
     // and DIALS decide on. The anomalous signal is not lost: the merge always keeps the Bijvoet split,
     // and I(+)/I(-), SigAno and CCanom come from it in either mode. Stills keep -A as it was - their
     // merge has no separate split.
+    // Modules where the image already has them (a JUNGFRAU file's own, or rebuilt from its mask) tell
+    // the stills module placement what may move; a rotation run has no such step and processes the
+    // image as one panel, as it always has.
+    if (experiment_.IsRotationIndexing() && !experiment_.GetModules().Moved())
+        experiment_.Modules({});
+
     if (experiment_.IsRotationIndexing() && !experiment_.GetScalingSettings().GetMergeFriedel()) {
         ScalingSettings s = experiment_.GetScalingSettings();
         experiment_.ImportScalingSettings(s.MergeFriedel(true));

tests/DetectorModulesTest.cpp

@@ -9,6 +9,7 @@
 #include "../common/DetectorModules.h"
 #include "../common/DiffractionExperiment.h"
 #include "../common/JFJochMath.h"
+#include "../common/PixelMask.h"
 #include "../reader/NXmxModules.h"
 #include "../writer/NXmxModules.h"
 
@@ -110,3 +111,30 @@ TEST_CASE("DetectorModules_NXmxRoundTrip", "[DetectorModules]") {
         CHECK(b.sy == Catch::Approx(a.sy).margin(1e-6));
     }
 }
+
+// A JUNGFRAU file that marks only the gaps between its modules in the mask: the modules come back from
+// the mask, one per gap-bounded region; a region that is not one whole module gives none at all.
+TEST_CASE("DetectorModules_FromGapMask", "[DetectorModules]") {
+    const DetectorGeometryModular geom(4, 2, 8, 36, false);
+    const DetectorModules expected = ModulesAsImaged(geom);
+    const int64_t width = geom.GetWidth(true), height = geom.GetHeight(true);
+    std::vector<uint32_t> mask(width * height, 1u << PixelMask::ModuleGapPixelBit);
+    expected.ForEachPixel(width, height, [&](int64_t, size_t i, double, double) { mask[i] = 0; });
+    mask[5] |= 1u << PixelMask::ErrorPixelBit;   // other bits do not matter
+
+    const DetectorModules modules = ModulesFromGapMask(mask, width, height);
+    REQUIRE(modules.size() == expected.size());
+    for (size_t i = 0; i < modules.size(); i++) {
+        CHECK(modules[i].x0 == expected[i].x0);
+        CHECK(modules[i].y0 == expected[i].y0);
+        CHECK(modules[i].width == expected[i].width);
+        CHECK(modules[i].height == expected[i].height);
+    }
+
+    // A gap pixel inside a module: no longer whole modules.
+    auto holed = mask;
+    holed[100 * width + 100] |= 1u << PixelMask::ModuleGapPixelBit;
+    CHECK(ModulesFromGapMask(holed, width, height).empty());
+    // No gaps at all: one region the size of the image, not a module.
+    CHECK(ModulesFromGapMask(std::vector<uint32_t>(width * height, 0), width, height).empty());
+}

tests/JFJochReaderTest.cpp

@@ -5168,3 +5168,54 @@ TEST_CASE("JFJochReader_PanelId", "[HDF5][PanelId]") {
 
     REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
 }
+
+// A JUNGFRAU image assembled by the experiment is written with its modules one by one (module_0, ...),
+// and the reader gives them back where the assembly put them; a DECTRIS detector is written without.
+TEST_CASE("JFJochReader_JungfrauModules", "[HDF5][Full]") {
+    RegisterHDF5Filter();
+    for (const bool jungfrau: {true, false}) {
+        DiffractionExperiment x(jungfrau ? DetJF(4) : DetDECTRIS(2068, 1064, "Detector", {}));
+        x.FilePrefix("test_modules").ImagesPerTrigger(1).OverwriteExistingFiles(true);
+        x.BeamX_pxl(100).BeamY_pxl(200).DetectorDistance_mm(150)
+                .IncidentEnergy_keV(WVL_1A_IN_KEV).PixelSigned(false).BitDepthImage(16);
+        if (jungfrau)
+            x.FrameTime(std::chrono::microseconds(500), std::chrono::microseconds(10));
+        const DetectorModules expected = jungfrau ? ModulesAsImaged(x.GetDetectorSetup().GetGeometry())
+                                                  : DetectorModules();
+        CHECK(expected.size() == (jungfrau ? 4 : 0));
+        ScanResultGenerator generator(x);
+        std::vector<uint16_t> image(x.GetPixelsNum(), 0);
+        {
+            StartMessage start_message;
+            x.FillMessage(start_message);
+            FileWriter file_set(start_message);
+            DataMessage message{};
+            message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
+            message.number = 0;
+            REQUIRE_NOTHROW(file_set.WriteHDF5(message));
+            REQUIRE_NOTHROW(generator.Add(message));
+            EndMessage end_message;
+            end_message.max_image_number = 1;
+            generator.FillEndMessage(end_message);
+            file_set.WriteHDF5(end_message);
+            file_set.Finalize();
+        }
+        {
+            JFJochHDF5Reader reader;
+            reader.ReadFile("test_modules_master.h5");
+            const DetectorModules &back = reader.GetDataset()->experiment.GetModules();
+            REQUIRE(back.size() == expected.size());
+            for (size_t i = 0; i < back.size(); i++) {
+                CHECK(back[i].x0 == expected[i].x0);
+                CHECK(back[i].y0 == expected[i].y0);
+                CHECK(back[i].width == expected[i].width);
+                CHECK(back[i].height == expected[i].height);
+                CHECK(back[i].ox == Catch::Approx(expected[i].ox).margin(1e-3));
+                CHECK(back[i].oy == Catch::Approx(expected[i].oy).margin(1e-3));
+            }
+            CHECK_FALSE(back.Moved());
+        }
+        remove("test_modules_master.h5");
+    }
+    REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
+}