VERSION: 1.0.0-rc.131

docs/CHANGELOG.md

@@ -4,8 +4,7 @@
 This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.124.
 
 * jfjoch_broker: Fix bug in saving JUNGFRAU calibration (pedestal/pedestalRMS)
-* jfjoch_viewer: Fix calibration (pedestal) images being open flipped
-* jfjoch_process: Add space group detection (EXPERIMENTAL)
+* jfjoch_process: Add experimental space group detection
 
 ### 1.0.0-rc.130
 This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.124.

image_analysis/scale_merge/SearchSpaceGroup.cpp

@@ -1,10 +1,6 @@
-// SPDX-FileCopyrightText: 2025 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
-// SPDX-License-Identifier: GPL-3.0-only
-
 #include "SearchSpaceGroup.h"
 
 #include <algorithm>
-#include <array>
 #include <cmath>
 #include <cctype>
 #include <iomanip>
@@ -55,7 +51,7 @@ namespace {
 
     MergedHKLKey CanonicalizeMergedHKL(int h, int k, int l, bool merge_friedel) {
         MergedHKLKey key{h, k, l, true};
-        if (!merge_friedel)
+        if (merge_friedel)
             return key;
 
         const std::tuple<int, int, int> pos{h, k, l};
@@ -181,15 +177,6 @@ namespace {
         return out;
     }
 
-    struct ResolutionBinAccumulator {
-        double d_min_A = std::numeric_limits<double>::infinity();
-        double d_max_A = 0.0;
-        double absent_sum = 0.0;
-        int absent_count = 0;
-        double allowed_sum = 0.0;
-        int allowed_count = 0;
-    };
-
     SpaceGroupAbsenceScore ScoreSystematicAbsences(
         const gemmi::GroupOps& gops,
         const std::vector<MergedReflection>& merged,
@@ -200,144 +187,27 @@ namespace {
         if (!opt.test_systematic_absences)
             return out;
 
-        int n_bins = opt.absence_resolution_bins;
-        if (n_bins <= 0)
-            n_bins = 1;
-
-        double min_inv_d2 = std::numeric_limits<double>::infinity();
-        double max_inv_d2 = -std::numeric_limits<double>::infinity();
+        double sum_i_over_sigma = 0.0;
 
         for (const auto& r : merged) {
             if (!ReflectionPassesFilters(r, opt))
                 continue;
 
-            const double inv_d2 = 1.0 / (r.d * r.d);
-            min_inv_d2 = std::min(min_inv_d2, inv_d2);
-            max_inv_d2 = std::max(max_inv_d2, inv_d2);
-        }
-
-        if (!std::isfinite(min_inv_d2) || !std::isfinite(max_inv_d2))
-            return out;
-
-        if (max_inv_d2 < min_inv_d2)
-            std::swap(max_inv_d2, min_inv_d2);
-
-        std::vector<ResolutionBinAccumulator> bins(static_cast<size_t>(n_bins));
-
-        auto bin_index = [&](double d) {
-            if (n_bins == 1 || max_inv_d2 <= min_inv_d2)
-                return 0;
-
-            const double inv_d2 = 1.0 / (d * d);
-            const double t = (inv_d2 - min_inv_d2) / (max_inv_d2 - min_inv_d2);
-            int idx = static_cast<int>(t * n_bins);
-            if (idx < 0)
-                idx = 0;
-            if (idx >= n_bins)
-                idx = n_bins - 1;
-            return idx;
-        };
-
-        for (const auto& r : merged) {
-            if (!ReflectionPassesFilters(r, opt))
-                continue;
-
-            const int idx = bin_index(r.d);
-            auto& bin = bins[static_cast<size_t>(idx)];
-            bin.d_min_A = std::min(bin.d_min_A, r.d);
-            bin.d_max_A = std::max(bin.d_max_A, r.d);
-
-            const double i_over_sigma = std::max(0.0, r.I / r.sigma);
             const gemmi::Op::Miller hkl{{r.h, r.k, r.l}};
+            if (!gops.is_systematically_absent(hkl))
+                continue;
 
-            if (gops.is_systematically_absent(hkl)) {
-                bin.absent_sum += i_over_sigma;
-                bin.absent_count += 1;
-                out.absent_reflections += 1;
-            } else {
-                bin.allowed_sum += i_over_sigma;
-                bin.allowed_count += 1;
-                out.allowed_reflections += 1;
-            }
+            out.violating_reflections += 1;
+            sum_i_over_sigma += std::max(0.0, r.I / r.sigma);
         }
 
-        double global_absent_sum = 0.0;
-        double global_allowed_sum = 0.0;
-        double weighted_ratio_sum = 0.0;
-        int weighted_ratio_weight = 0;
-        double worst_ratio = 0.0;
-        bool any_decision_bin_failed = false;
-
-        out.bins.reserve(bins.size());
-
-        for (const auto& bin : bins) {
-            SpaceGroupAbsenceBinScore bin_score;
-            if (std::isfinite(bin.d_min_A))
-                bin_score.d_min_A = bin.d_min_A;
-            if (bin.d_max_A > 0.0)
-                bin_score.d_max_A = bin.d_max_A;
-
-            bin_score.absent_reflections = bin.absent_count;
-            bin_score.allowed_reflections = bin.allowed_count;
-
-            if (bin.absent_count > 0)
-                bin_score.mean_absent_i_over_sigma = bin.absent_sum / static_cast<double>(bin.absent_count);
-            if (bin.allowed_count > 0)
-                bin_score.mean_allowed_i_over_sigma = bin.allowed_sum / static_cast<double>(bin.allowed_count);
-
-            global_absent_sum += bin.absent_sum;
-            global_allowed_sum += bin.allowed_sum;
-
-            if (bin.absent_count >= opt.min_absent_reflections_per_bin &&
-                bin.allowed_count >= opt.min_allowed_reflections_per_bin &&
-                bin_score.mean_allowed_i_over_sigma > 0.0) {
-
-                bin_score.used_for_decision = true;
-                bin_score.absent_to_allowed_ratio =
-                    bin_score.mean_absent_i_over_sigma / bin_score.mean_allowed_i_over_sigma;
-                bin_score.accepted =
-                    bin_score.absent_to_allowed_ratio <= opt.max_absent_to_allowed_i_over_sigma_ratio_in_any_bin;
-
-                out.compared_bins += 1;
-                if (bin_score.accepted)
-                    out.accepted_bins += 1;
-                else
-                    any_decision_bin_failed = true;
-
-                weighted_ratio_sum += bin_score.absent_to_allowed_ratio * bin.absent_count;
-                weighted_ratio_weight += bin.absent_count;
-                worst_ratio = std::max(worst_ratio, bin_score.absent_to_allowed_ratio);
-            }
-
-            out.bins.push_back(bin_score);
-        }
-
-        if (out.absent_reflections > 0)
-            out.mean_absent_i_over_sigma = global_absent_sum / static_cast<double>(out.absent_reflections);
-        if (out.allowed_reflections > 0)
-            out.mean_allowed_i_over_sigma = global_allowed_sum / static_cast<double>(out.allowed_reflections);
-
-        if (weighted_ratio_weight > 0) {
-            out.weighted_absent_to_allowed_ratio = weighted_ratio_sum / static_cast<double>(weighted_ratio_weight);
-            out.worst_absent_to_allowed_ratio = worst_ratio;
-            out.accepted =
-                !any_decision_bin_failed &&
-                out.weighted_absent_to_allowed_ratio <= opt.max_absent_to_allowed_i_over_sigma_ratio;
-        } else if (out.absent_reflections == 0) {
-            out.weighted_absent_to_allowed_ratio = 0.0;
-            out.worst_absent_to_allowed_ratio = 0.0;
-            out.accepted = true;
-        } else if (out.mean_allowed_i_over_sigma > 0.0) {
-            const double global_ratio = out.mean_absent_i_over_sigma / out.mean_allowed_i_over_sigma;
-            out.weighted_absent_to_allowed_ratio = global_ratio;
-            out.worst_absent_to_allowed_ratio = global_ratio;
-            out.accepted = global_ratio <= opt.max_absent_to_allowed_i_over_sigma_ratio_in_any_bin;
-        } else {
-            out.weighted_absent_to_allowed_ratio = std::numeric_limits<double>::infinity();
-            out.worst_absent_to_allowed_ratio = std::numeric_limits<double>::infinity();
-            out.accepted = false;
-        }
+        if (out.violating_reflections > 0)
+            out.mean_i_over_sigma = sum_i_over_sigma / out.violating_reflections;
+        else
+            out.mean_i_over_sigma = 0.0;
 
+        out.accepted = (out.violating_reflections <= opt.max_absent_violations) &&
+                       (out.mean_i_over_sigma <= opt.max_mean_absent_i_over_sigma);
         return out;
     }
 
@@ -453,24 +323,17 @@ SearchSpaceGroupResult SearchSpaceGroup(
                   if (a.accepted != b.accepted)
                       return a.accepted > b.accepted;
 
-                  if (a.absence_score.weighted_absent_to_allowed_ratio !=
-                      b.absence_score.weighted_absent_to_allowed_ratio)
-                      return a.absence_score.weighted_absent_to_allowed_ratio <
-                             b.absence_score.weighted_absent_to_allowed_ratio;
+                  if (a.absence_score.mean_i_over_sigma != b.absence_score.mean_i_over_sigma)
+                      return a.absence_score.mean_i_over_sigma < b.absence_score.mean_i_over_sigma;
 
-                  if (a.absence_score.worst_absent_to_allowed_ratio !=
-                      b.absence_score.worst_absent_to_allowed_ratio)
-                      return a.absence_score.worst_absent_to_allowed_ratio <
-                             b.absence_score.worst_absent_to_allowed_ratio;
+                  if (a.absence_score.violating_reflections != b.absence_score.violating_reflections)
+                      return a.absence_score.violating_reflections < b.absence_score.violating_reflections;
 
                   const int order_a = a.space_group.operations().derive_symmorphic().order();
                   const int order_b = b.space_group.operations().derive_symmorphic().order();
                   if (order_a != order_b)
                       return order_a > order_b;
 
-                  if (a.absence_score.absent_reflections != b.absence_score.absent_reflections)
-                      return a.absence_score.absent_reflections > b.absence_score.absent_reflections;
-
                   if (a.accepted_operators != b.accepted_operators)
                       return a.accepted_operators > b.accepted_operators;
                   if (a.min_cc != b.min_cc)
@@ -478,7 +341,7 @@ SearchSpaceGroupResult SearchSpaceGroup(
                   if (a.mean_cc != b.mean_cc)
                       return a.mean_cc > b.mean_cc;
 
-                  return a.space_group.number > b.space_group.number;
+                  return a.space_group.number < b.space_group.number;
               });
 
     for (const auto& cand : result.candidates) {
@@ -513,11 +376,7 @@ std::string SearchSpaceGroupResultToText(const SearchSpaceGroupResult& result,
        << " "
        << std::setw(8) << "Nabs"
        << " "
-       << std::setw(8) << "Nallow"
-       << " "
-       << std::setw(8) << "Abs/All"
-       << " "
-       << std::setw(8) << "worst"
+       << std::setw(10) << "<I/sig>abs"
        << "\n";
 
     os << "  "
@@ -537,11 +396,7 @@ std::string SearchSpaceGroupResultToText(const SearchSpaceGroupResult& result,
        << " "
        << std::setw(8) << "--------"
        << " "
-       << std::setw(8) << "--------"
-       << " "
-       << std::setw(8) << "--------"
-       << " "
-       << std::setw(8) << "--------"
+       << std::setw(10) << "----------"
        << "\n";
 
     const size_t n = std::min(max_candidates_to_print, result.candidates.size());
@@ -562,15 +417,9 @@ std::string SearchSpaceGroupResultToText(const SearchSpaceGroupResult& result,
            << " "
            << std::setw(5) << (c.absence_score.accepted ? "yes" : "no")
            << " "
-           << std::setw(8) << c.absence_score.absent_reflections
+           << std::setw(8) << c.absence_score.violating_reflections
            << " "
-           << std::setw(8) << c.absence_score.allowed_reflections
-           << " "
-           << std::setw(8) << std::fixed << std::setprecision(3)
-           << c.absence_score.weighted_absent_to_allowed_ratio
-           << " "
-           << std::setw(8) << std::fixed << std::setprecision(3)
-           << c.absence_score.worst_absent_to_allowed_ratio
+           << std::setw(10) << std::fixed << std::setprecision(2) << c.absence_score.mean_i_over_sigma
            << "\n";
     }
 

image_analysis/scale_merge/SearchSpaceGroup.h

@@ -1,6 +1,3 @@
-// SPDX-FileCopyrightText: 2025 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
-// SPDX-License-Identifier: GPL-3.0-only
-
 #pragma once
 
 #include <optional>
@@ -17,29 +14,10 @@ struct SpaceGroupOperatorScore {
     bool accepted = false;
 };
 
-struct SpaceGroupAbsenceBinScore {
-    double d_min_A = 0.0;
-    double d_max_A = 0.0;
-    int absent_reflections = 0;
-    int allowed_reflections = 0;
-    double mean_absent_i_over_sigma = 0.0;
-    double mean_allowed_i_over_sigma = 0.0;
-    double absent_to_allowed_ratio = 0.0;
-    bool used_for_decision = false;
-    bool accepted = true;
-};
-
 struct SpaceGroupAbsenceScore {
-    int absent_reflections = 0;
-    int allowed_reflections = 0;
-    int compared_bins = 0;
-    int accepted_bins = 0;
-    double mean_absent_i_over_sigma = 0.0;
-    double mean_allowed_i_over_sigma = 0.0;
-    double weighted_absent_to_allowed_ratio = 0.0;
-    double worst_absent_to_allowed_ratio = 0.0;
+    int violating_reflections = 0;
+    double mean_i_over_sigma = 0.0;
     bool accepted = true;
-    std::vector<SpaceGroupAbsenceBinScore> bins;
 };
 
 struct SpaceGroupCandidateScore {
@@ -81,16 +59,11 @@ struct SearchSpaceGroupOptions {
     // Test systematic absences from centering / screws / glides.
     bool test_systematic_absences = true;
 
-    // Number of resolution bins used for absence-vs-allowed comparison.
-    int absence_resolution_bins = 10;
+    // Candidate is rejected if forbidden reflections are too strong on average.
+    double max_mean_absent_i_over_sigma = 1.0;
 
-    // Minimum counts in a bin before it contributes to the decision.
-    int min_absent_reflections_per_bin = 5;
-    int min_allowed_reflections_per_bin = 10;
-
-    // Acceptance thresholds for absent/allowed <I/sig> ratios.
-    double max_absent_to_allowed_i_over_sigma_ratio = 0.20;
-    double max_absent_to_allowed_i_over_sigma_ratio_in_any_bin = 0.50;
+    // Hard cap on number of observed forbidden reflections.
+    int max_absent_violations = 0;
 };
 
 struct SearchSpaceGroupResult {

reader/JFJochHDF5Reader.cpp

@@ -800,7 +800,7 @@ CompressedImage JFJochHDF5Reader::ReadCalibration(std::vector<uint8_t> &tmp, con
     dataset.ReadVectorToU8(tmp, start, {dim[0], dim[1]});
     algorithm = CompressionAlgorithm::NO_COMPRESSION;
 
-    return {tmp, dim[1], dim[0],
+    return {tmp, dim[0], dim[1],
             CalcImageMode(datatype.GetElemSize(), datatype.IsFloat(), datatype.IsSigned()),
             algorithm};
 }

tests/HDF5WritingTest.cpp

@@ -799,103 +799,6 @@ TEST_CASE("HDF5Writer_NoMasterFile", "[HDF5][Full]") {
     REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
 }
 
-TEST_CASE("HDF5Writer_Calibration", "[HDF5][Full]") {
-    DiffractionExperiment x(DetJF(2));
-
-    std::vector<int16_t> calib_1(x.GetModulesNum() * RAW_MODULE_SIZE);
-    std::vector<float> calib_2(x.GetModulesNum() * RAW_MODULE_SIZE);
-    for (int i = 0; i < x.GetModulesNum(); i++) {
-        calib_1[i] = i * 3 - 1024;
-        calib_2[i] = static_cast<float>(i) / 16.0 + 123.25f;
-    }
-
-    JFJochBitShuffleCompressor compressor(CompressionAlgorithm::BSHUF_LZ4);
-    std::vector<uint8_t> calib_3 = compressor.Compress(calib_1);
-
-    x.ImagesPerTrigger(7).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("calib");
-    x.OverwriteExistingFiles(true);
-    {
-        RegisterHDF5Filter();
-
-        StartMessage start_message;
-        x.FillMessage(start_message);
-
-        EndMessage end_message;
-        end_message.max_image_number = x.GetImageNum() - 2;
-
-        FileWriter writer(start_message);
-
-        CompressedImage image_1(calib_1, RAW_MODULE_COLS, x.GetModulesNum() * RAW_MODULE_LINES);
-        CompressedImage image_2(calib_2, RAW_MODULE_COLS, x.GetModulesNum() * RAW_MODULE_LINES);
-        CompressedImage image_3(calib_3, RAW_MODULE_COLS, x.GetModulesNum() * RAW_MODULE_LINES,
-            CompressedImageMode::Int16, CompressionAlgorithm::BSHUF_LZ4);
-        image_1.Channel("calib1");
-        image_2.Channel("calib2");
-        image_3.Channel("calib3");
-
-        writer.WriteHDF5(image_1);
-        writer.WriteHDF5(image_2);
-        writer.WriteHDF5(image_3);
-
-        writer.WriteHDF5(end_message);
-        writer.Finalize();
-    }
-    REQUIRE(std::filesystem::exists("calib_master.h5"));
-    {
-        HDF5ReadOnlyFile file("calib_master.h5");
-        {
-            std::unique_ptr<HDF5DataSet> dataset;
-            REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/instrument/detector/calibration/calib1"));
-            HDF5DataSpace file_space(*dataset);
-            REQUIRE(file_space.GetNumOfDimensions() == 2);
-            HDF5DataType type(*dataset);
-            REQUIRE(type.GetElemSize() == 2);
-            REQUIRE(type.IsSigned());
-            REQUIRE(type.IsInteger());
-
-            REQUIRE(file_space.GetDimensions()[0] == RAW_MODULE_COLS);
-            REQUIRE(file_space.GetDimensions()[1] == RAW_MODULE_LINES * x.GetModulesNum());
-            std::vector<int16_t> output(file_space.GetDimensions()[0] * file_space.GetDimensions()[1]);
-            dataset->ReadVector(output, {0,0}, file_space.GetDimensions());
-            CHECK(memcmp(output.data(), calib_1.data(), output.size() * type.GetElemSize()) == 0);
-        }
-        {
-            std::unique_ptr<HDF5DataSet> dataset;
-            REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/instrument/detector/calibration/calib2"));
-            HDF5DataSpace file_space(*dataset);
-            REQUIRE(file_space.GetNumOfDimensions() == 2);
-            HDF5DataType type(*dataset);
-            REQUIRE(type.GetElemSize() == 4);
-            REQUIRE(type.IsSigned());
-            REQUIRE(!type.IsInteger());
-
-            REQUIRE(file_space.GetDimensions()[0] == RAW_MODULE_COLS);
-            REQUIRE(file_space.GetDimensions()[1] == RAW_MODULE_LINES * x.GetModulesNum());
-            std::vector<float> output(file_space.GetDimensions()[0] * file_space.GetDimensions()[1]);
-            dataset->ReadVector(output, {0,0}, file_space.GetDimensions());
-            CHECK(memcmp(output.data(), calib_2.data(), output.size() * type.GetElemSize()) == 0);
-        }
-        {
-            std::unique_ptr<HDF5DataSet> dataset;
-            REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/instrument/detector/calibration/calib3"));
-            HDF5DataSpace file_space(*dataset);
-            REQUIRE(file_space.GetNumOfDimensions() == 2);
-            HDF5DataType type(*dataset);
-            REQUIRE(type.GetElemSize() == 2);
-            REQUIRE(type.IsSigned());
-            REQUIRE(type.IsInteger());
-
-            REQUIRE(file_space.GetDimensions()[0] == RAW_MODULE_COLS);
-            REQUIRE(file_space.GetDimensions()[1] == RAW_MODULE_LINES * x.GetModulesNum());
-            std::vector<int16_t> output(file_space.GetDimensions()[0] * file_space.GetDimensions()[1]);
-            dataset->ReadVector(output, {0,0}, file_space.GetDimensions());
-            CHECK(memcmp(output.data(), calib_1.data(), output.size() * type.GetElemSize()) == 0);
-        }
-    }
-    // No leftover HDF5 objects
-    REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
-}
-
 
 TEST_CASE("HDF5Writer_Link_zero_images", "[HDF5][Full]") {
     DiffractionExperiment x(DetJF(1));

tests/JFJochReaderTest.cpp

@@ -6,7 +6,6 @@
 #include "../common/DiffractionExperiment.h"
 #include "../writer/FileWriter.h"
 #include "../reader/JFJochHDF5Reader.h"
-#include "../compression/JFJochCompressor.h"
 
 TEST_CASE("HDF5DataType_Sign","[HDF5]") {
     HDF5DataType type_u8((uint8_t)0), type_fl(0.0f), type_i32((int32_t) 0), type_u32((uint32_t) 0);
@@ -86,9 +85,6 @@ TEST_CASE("JFJochReader_MasterFile_Calibration", "[HDF5][Full]") {
     std::vector<uint16_t> calib_1(200*300, 10);
     std::vector<int32_t> calib_2(100*400, 55);
     std::vector<float> calib_f(100*400, 1234.56f);
-    JFJochBitShuffleCompressor compressor(CompressionAlgorithm::BSHUF_LZ4);
-    auto calib_1_compressed = compressor.Compress(calib_1);
-
     {
         StartMessage start_message;
         x.FillMessage(start_message);
@@ -96,21 +92,14 @@ TEST_CASE("JFJochReader_MasterFile_Calibration", "[HDF5][Full]") {
         CompressedImage calibration_01(calib_1, 200, 300);
         CompressedImage calibration_02(calib_2, 100, 400);
         CompressedImage calibration_f(calib_f, 100, 400);
-        CompressedImage calibration_01_lz4(
-            calib_1_compressed.data(), calib_1_compressed.size(),
-            200, 300, CompressedImageMode::Uint16, CompressionAlgorithm::BSHUF_LZ4
-        );
-
         calibration_01.Channel("c1");
         calibration_02.Channel("c2");
         calibration_f.Channel("cf");
-        calibration_01_lz4.Channel("c1_lz4");
 
         EndMessage end_message;
         end_message.max_image_number = 0;
         std::unique_ptr<NXmx> master = std::make_unique<NXmx>(start_message);
         master->WriteCalibration(calibration_01);
-        master->WriteCalibration(calibration_01_lz4);
         master->WriteCalibration(calibration_02);
         master->WriteCalibration(calibration_f);
         master->Finalize(end_message);
@@ -120,11 +109,10 @@ TEST_CASE("JFJochReader_MasterFile_Calibration", "[HDF5][Full]") {
         JFJochHDF5Reader reader;
         REQUIRE_NOTHROW(reader.ReadFile("test_reader_calibration_master.h5"));
         auto dataset = reader.GetDataset();
-        REQUIRE(dataset->calibration_data.size() == 4);
+        REQUIRE(dataset->calibration_data.size() == 3);
         CHECK(dataset->calibration_data[0] == "c1");
-        CHECK(dataset->calibration_data[1] == "c1_lz4");
-        CHECK(dataset->calibration_data[2] == "c2");
-        CHECK(dataset->calibration_data[3] == "cf");
+        CHECK(dataset->calibration_data[1] == "c2");
+        CHECK(dataset->calibration_data[2] == "cf");
 
         std::vector<uint8_t> buffer;
         std::vector<uint8_t> buff_2;
@@ -132,29 +120,22 @@ TEST_CASE("JFJochReader_MasterFile_Calibration", "[HDF5][Full]") {
         CompressedImage test;
         REQUIRE_NOTHROW(test = reader.ReadCalibration(buffer, "c1"));
         CHECK(test.GetByteDepth() == 2);
-        CHECK(test.GetHeight() == 300);
-        CHECK(test.GetWidth() == 200);
-        CHECK(test.GetMode() == CompressedImageMode::Uint16);
-        CHECK(reinterpret_cast<const uint16_t *>(test.GetUncompressedPtr(buff_2))[76] == 10);
-
-        REQUIRE_NOTHROW(test = reader.ReadCalibration(buffer, "c1_lz4"));
-        CHECK(test.GetByteDepth() == 2);
-        CHECK(test.GetHeight() == 300);
-        CHECK(test.GetWidth() == 200);
+        CHECK(test.GetHeight() == 200);
+        CHECK(test.GetWidth() == 300);
         CHECK(test.GetMode() == CompressedImageMode::Uint16);
         CHECK(reinterpret_cast<const uint16_t *>(test.GetUncompressedPtr(buff_2))[76] == 10);
 
         REQUIRE_NOTHROW(test = reader.ReadCalibration(buffer, "c2"));
         CHECK(test.GetByteDepth() == 4);
-        CHECK(test.GetHeight() == 400);
-        CHECK(test.GetWidth() == 100);
+        CHECK(test.GetHeight() == 100);
+        CHECK(test.GetWidth() == 400);
         CHECK(test.GetMode() == CompressedImageMode::Int32);
         CHECK(reinterpret_cast<const int32_t *>(test.GetUncompressedPtr(buff_2))[76] == 55);
 
         REQUIRE_NOTHROW(test = reader.ReadCalibration(buffer, "cf"));
         CHECK(test.GetByteDepth() == 4);
-        CHECK(test.GetHeight() == 400);
-        CHECK(test.GetWidth() == 100);
+        CHECK(test.GetHeight() == 100);
+        CHECK(test.GetWidth() == 400);
         CHECK(test.GetMode() == CompressedImageMode::Float32);
         CHECK(reinterpret_cast<const float *>(test.GetUncompressedPtr(buff_2))[76] == Catch::Approx(1234.56f));
     }

tests/JFJochReceiverProcessingTest.cpp

@@ -1630,62 +1630,3 @@ TEST_CASE("JFJochIntegrationTest_TCP_lysozyme_spot_and_index", "[JFJochReceiver]
     REQUIRE(ack.has_value());
     CHECK(ack == experiment.GetImageNum());
 }
-
-TEST_CASE("JFJochIntegrationTest_TCP_calibration", "[JFJochReceiver]") {
-    Logger logger(Catch::getResultCapture().getCurrentTestName());
-
-    RegisterHDF5Filter();
-
-    const uint16_t nthreads = 4;
-
-    DiffractionExperiment experiment(DetJF4M());
-    experiment.ImagesPerTrigger(5).NumTriggers(1).UseInternalPacketGenerator(true).ImagesPerFile(2).SaveCalibration(true)
-            .FilePrefix("calib_integration_test").JungfrauConvPhotonCnt(true).SetFileWriterFormat(FileWriterFormat::NXmxVDS).OverwriteExistingFiles(true)
-            .DetectorDistance_mm(75).BeamY_pxl(1136).BeamX_pxl(1090).IncidentEnergy_keV(12.4)
-            .SetUnitCell(UnitCell{.a = 36.9, .b = 78.95, .c = 78.95, .alpha =90, .beta = 90, .gamma = 90});
-    experiment.SampleTemperature_K(123.0).RingCurrent_mA(115);
-
-    PixelMask pixel_mask(experiment);
-    JFCalibration calibration(experiment);
-
-    for (int i = 0; i < experiment.GetModulesNum(); i++) {
-        std::vector<int16_t> pedestal_g0(RAW_MODULE_SIZE, i);
-        std::vector<int16_t> pedestal_g1(RAW_MODULE_SIZE);
-        std::vector<int16_t> pedestal_g2(RAW_MODULE_SIZE);
-        for (int j = 0; j < RAW_MODULE_SIZE; j++) {
-            pedestal_g1[j] = j % RAW_MODULE_COLS;
-            pedestal_g2[j] = i * (RAW_MODULE_LINES) + j / RAW_MODULE_COLS; // line number
-        }
-        calibration.Pedestal(i,0).LoadPedestal(pedestal_g0);
-        calibration.Pedestal(i,1).LoadPedestal(pedestal_g1);
-        calibration.Pedestal(i,2).LoadPedestal(pedestal_g2);
-    }
-
-    // Setup acquisition device
-    AcquisitionDeviceGroup aq_devices;
-    std::unique_ptr<HLSSimulatedDevice> test = std::make_unique<HLSSimulatedDevice>(0, 64);
-
-    aq_devices.Add(std::move(test));
-
-    TCPStreamPusher pusher("tcp://127.0.0.1:*", 1);
-
-    TCPImagePuller puller(pusher.GetAddress()[0]);
-    StreamWriter writer(logger, puller);
-    auto writer_future = std::async(std::launch::async, &StreamWriter::Run, &writer);
-
-    JFJochReceiverService service(aq_devices, logger, pusher);
-    service.NumThreads(nthreads);
-    service.Indexing(experiment.GetIndexingSettings());
-
-    service.Start(experiment, pixel_mask, &calibration);
-    auto receiver_out = service.Stop();
-
-    CHECK(receiver_out.efficiency == 1.0);
-    CHECK(receiver_out.status.images_sent == experiment.GetImageNum());
-    CHECK(!receiver_out.status.cancelled);
-
-    // No progress value at the end of measurement
-    REQUIRE(!service.GetProgress().has_value());
-
-    REQUIRE_NOTHROW(writer_future.get());
-}

tests/SearchSpaceGroupTest.cpp

@@ -69,10 +69,9 @@ namespace {
                     if (h == 0 && k == 0 && l == 0)
                         continue;
 
-                    bool absent = false;
                     gemmi::Op::Miller hkl{{h, k, l}};
                     if (gops.is_systematically_absent(hkl))
-                        absent = true;
+                        continue;
 
                     const auto [asu, sign_plus] = rasu.to_asu_sign(hkl, gops);
                     if (!sign_plus)
@@ -87,7 +86,7 @@ namespace {
                         .h = h,
                         .k = k,
                         .l = l,
-                        .I = absent ? 0.0 : SyntheticIntensityFromAsu(asu),
+                        .I = SyntheticIntensityFromAsu(asu),
                         .sigma = 1.0,
                         .d = CalcSyntheticD(h, k, l)
                     });
@@ -111,13 +110,10 @@ TEST_CASE("SearchSpaceGroup detects synthetic space groups") {
         {"P 3 2 1", "P321"},
         {"P 4 2 2", "P422"},
         {"P 4 3 2", "P432"},
-        {"P 43 21 2", "P43212"},
         {"P 6 2 2", "P622"},
         {"C 1 2 1", "C2"},
         {"C 2 2 2", "C222"},
         {"I 4 3 2", "I432"},
-        {"I 21 21 21", "I212121"},
-        {"I 2 1 3", "I213"},
     };
 
     for (const auto& tc : cases) {
@@ -134,11 +130,8 @@ TEST_CASE("SearchSpaceGroup detects synthetic space groups") {
             opt.min_pairs_per_operator = 20;
             opt.min_total_compared = 80;
             opt.test_systematic_absences = true;
-            opt.absence_resolution_bins = 10;
-            opt.min_absent_reflections_per_bin = 5;
-            opt.min_allowed_reflections_per_bin = 10;
-            opt.max_absent_to_allowed_i_over_sigma_ratio = 0.05;
-            opt.max_absent_to_allowed_i_over_sigma_ratio_in_any_bin = 0.10;
+            opt.max_mean_absent_i_over_sigma = 0.1;
+            opt.max_absent_violations = 0;
 
             const auto result = SearchSpaceGroup(merged, opt);
 

tools/jfjoch_process.cpp

@@ -556,6 +556,8 @@ int main(int argc, char **argv) {
             sg_opts.min_pairs_per_operator = 20;
             sg_opts.min_total_compared = 100;
             sg_opts.test_systematic_absences = true;
+            sg_opts.max_mean_absent_i_over_sigma = 1.5;
+            sg_opts.max_absent_violations = 3;
 
             const auto sg_search = SearchSpaceGroup(scale_result->merged, sg_opts);