Improvements before MAX IV test

2024-04-25 20:11:58 +02:00
parent 2c8e1fd83d
commit ea70b27e85
80 changed files with 1835 additions and 1781 deletions
--- a/acquisition_device/AcquisitionDevice.cpp
+++ b/acquisition_device/AcquisitionDevice.cpp
@@ -191,7 +191,7 @@ void AcquisitionDevice::InitializeROIMap(const DiffractionExperiment& experiment
    auto offset = experiment.GetFirstModuleOfDataStream(data_stream);
    size_t modules = experiment.GetModulesNum(data_stream);
    for (int m = 0; m < modules; m++) {
-        experiment.ExportROIMask(tmp.data(), offset + m);
+        experiment.ExportROIMap(tmp.data(), offset + m);
        InitializeROIMap(tmp.data(), m);
    }
 }
--- a/acquisition_device/FPGAAcquisitionDevice.cpp
+++ b/acquisition_device/FPGAAcquisitionDevice.cpp
@@ -179,39 +179,22 @@ void FPGAAcquisitionDevice::InitializeCalibration(const DiffractionExperiment &e
    }

    for (int s = 0; s < storage_cells; s++) {
-        auto mask = calib.CalculateMask(experiment, s);
        for (int m = 0; m < modules; m++) {
-
            auto pedestal_g1 = calib.Pedestal(offset + m, 1, s).GetPedestal();
-            for (int i = 0; i < RAW_MODULE_SIZE; i++) {
-                if (experiment.GetApplyPixelMaskInFPGA() && (mask[(offset + m) * RAW_MODULE_SIZE + i] != 0))
-                    buffer_device[0]->pixels[i] = 16384;
-                else
-                    ((uint16_t *) buffer_device[0]->pixels)[i] = pedestal_g1[i];
-            }
+            memcpy(buffer_device[0]->pixels, pedestal_g1, RAW_MODULE_SIZE * sizeof(uint16_t));
            buffer_device[0]->module_statistics.module_number = m + modules * s;
            buffer_device[0]->module_statistics.load_calibration_destination = LOAD_CALIBRATION_DEST_PEDESTAL_G1;
            LoadCalibration(0);

            if (!experiment.IsFixedGainG1()) {
                auto pedestal_g0 = calib.Pedestal(offset + m, 0, s).GetPedestal();
-                for (int i = 0; i < RAW_MODULE_SIZE; i++) {
-                    if (experiment.GetApplyPixelMaskInFPGA() && (mask[(offset + m) * RAW_MODULE_SIZE + i] != 0))
-                        buffer_device[0]->pixels[i] = 16384;
-                    else
-                        ((uint16_t *) buffer_device[0]->pixels)[i] = pedestal_g0[i];
-                }
+                memcpy(buffer_device[0]->pixels, pedestal_g0, RAW_MODULE_SIZE * sizeof(uint16_t));
                buffer_device[0]->module_statistics.module_number = m + modules * s;
                buffer_device[0]->module_statistics.load_calibration_destination = LOAD_CALIBRATION_DEST_PEDESTAL_G0;
                LoadCalibration(0);

                auto pedestal_g2 = calib.Pedestal(offset + m, 2, s).GetPedestal();
-                for (int i = 0; i < RAW_MODULE_SIZE; i++) {
-                    if (experiment.GetApplyPixelMaskInFPGA() && (mask[(offset + m) * RAW_MODULE_SIZE + i] != 0))
-                        buffer_device[0]->pixels[i] = 16384;
-                    else
-                        ((uint16_t *) buffer_device[0]->pixels)[i] = pedestal_g2[i];
-                }
+                memcpy(buffer_device[0]->pixels, pedestal_g2, RAW_MODULE_SIZE * sizeof(uint16_t));
                buffer_device[0]->module_statistics.module_number = m + modules * s;
                buffer_device[0]->module_statistics.load_calibration_destination = LOAD_CALIBRATION_DEST_PEDESTAL_G2;
                LoadCalibration(0);
--- a/broker/JFJochBrokerHttp.cpp
+++ b/broker/JFJochBrokerHttp.cpp
@@ -48,6 +48,7 @@ inline org::openapitools::server::model::Measurement_statistics Convert(const Me
    ret.setImagesExpected(input.images_expected);
    ret.setImagesCollected(input.images_collected);
    ret.setImagesSent(input.images_sent);
+    ret.setImagesDiscardedLossyCompression(input.images_skipped);
    ret.setMaxImageNumberSent(input.max_image_number_sent);
    if (input.collection_efficiency)
        ret.setCollectionEfficiency(input.collection_efficiency.value());
@@ -167,7 +168,7 @@ inline org::openapitools::server::model::Calibration_statistics_inner Convert(co
    org::openapitools::server::model::Calibration_statistics_inner output;

    output.setModuleNumber(input.module_number);
-    output.setMaskedPixels(input.masked_pixels);
+    output.setMaskedPixels(input.bad_pixels);
    output.setStorageCellNumber(input.storage_cell_number);
    output.setGainG0Mean(input.gain_g0_mean);
    output.setGainG1Mean(input.gain_g1_mean);
--- a/broker/JFJochStateMachine.cpp
+++ b/broker/JFJochStateMachine.cpp
@@ -406,6 +406,7 @@ void JFJochStateMachine::SetFullMeasurementOutput(const JFJochServicesOutput &ou
    tmp.collection_efficiency = output.receiver_output.efficiency;
    tmp.images_collected = output.receiver_output.status.images_collected;
    tmp.images_sent = output.receiver_output.status.images_sent;
+    tmp.images_skipped = output.receiver_output.status.images_skipped;
    tmp.cancelled = output.receiver_output.status.cancelled;
    tmp.max_image_number_sent = output.receiver_output.status.max_image_number_sent;
    tmp.max_receive_delay = output.receiver_output.status.max_receive_delay;
--- a/broker/JFJochStateMachine.h
+++ b/broker/JFJochStateMachine.h
@@ -13,7 +13,7 @@
 #include "../common/Logger.h"

 #include "JFJochServices.h"
-#include "../common/ROIMask.h"
+#include "../common/ROIMap.h"

 enum class JFJochState {Inactive, Idle, Measuring, Error, Busy, Pedestal};

@@ -40,6 +40,7 @@ struct MeasurementStatistics {
    int64_t images_expected;
    int64_t images_collected;
    int64_t images_sent;
+    int64_t images_skipped;
    int64_t max_image_number_sent;
    std::optional<float> collection_efficiency;
    std::optional<float> compression_ratio;
--- a/broker/gen/model/Measurement_statistics.cpp
+++ b/broker/gen/model/Measurement_statistics.cpp
@@ -29,6 +29,8 @@ Measurement_statistics::Measurement_statistics()
    m_Images_collectedIsSet = false;
    m_Images_sent = 0L;
    m_Images_sentIsSet = false;
+    m_Images_discarded_lossy_compression = 0L;
+    m_Images_discarded_lossy_compressionIsSet = false;
    m_Max_image_number_sent = 0L;
    m_Max_image_number_sentIsSet = false;
    m_Collection_efficiency = 0.0f;
@@ -75,7 +77,7 @@ bool Measurement_statistics::validate(std::stringstream& msg, const std::string&
    bool success = true;
    const std::string _pathPrefix = pathPrefix.empty() ? "Measurement_statistics" : pathPrefix;

-                             
+                                 
    if (collectionEfficiencyIsSet())
    {
        const float& value = m_Collection_efficiency;
@@ -130,6 +132,9 @@ bool Measurement_statistics::operator==(const Measurement_statistics& rhs) const
    ((!imagesSentIsSet() && !rhs.imagesSentIsSet()) || (imagesSentIsSet() && rhs.imagesSentIsSet() && getImagesSent() == rhs.getImagesSent())) &&
    
    
+    ((!imagesDiscardedLossyCompressionIsSet() && !rhs.imagesDiscardedLossyCompressionIsSet()) || (imagesDiscardedLossyCompressionIsSet() && rhs.imagesDiscardedLossyCompressionIsSet() && getImagesDiscardedLossyCompression() == rhs.getImagesDiscardedLossyCompression())) &&
+    
+    
    ((!maxImageNumberSentIsSet() && !rhs.maxImageNumberSentIsSet()) || (maxImageNumberSentIsSet() && rhs.maxImageNumberSentIsSet() && getMaxImageNumberSent() == rhs.getMaxImageNumberSent())) &&
    
    
@@ -184,6 +189,8 @@ void to_json(nlohmann::json& j, const Measurement_statistics& o)
        j["images_collected"] = o.m_Images_collected;
    if(o.imagesSentIsSet())
        j["images_sent"] = o.m_Images_sent;
+    if(o.imagesDiscardedLossyCompressionIsSet())
+        j["images_discarded_lossy_compression"] = o.m_Images_discarded_lossy_compression;
    if(o.maxImageNumberSentIsSet())
        j["max_image_number_sent"] = o.m_Max_image_number_sent;
    if(o.collectionEfficiencyIsSet())
@@ -233,6 +240,11 @@ void from_json(const nlohmann::json& j, Measurement_statistics& o)
        j.at("images_sent").get_to(o.m_Images_sent);
        o.m_Images_sentIsSet = true;
    } 
+    if(j.find("images_discarded_lossy_compression") != j.end())
+    {
+        j.at("images_discarded_lossy_compression").get_to(o.m_Images_discarded_lossy_compression);
+        o.m_Images_discarded_lossy_compressionIsSet = true;
+    } 
    if(j.find("max_image_number_sent") != j.end())
    {
        j.at("max_image_number_sent").get_to(o.m_Max_image_number_sent);
@@ -364,6 +376,23 @@ void Measurement_statistics::unsetImages_sent()
 {
    m_Images_sentIsSet = false;
 }
+int64_t Measurement_statistics::getImagesDiscardedLossyCompression() const
+{
+    return m_Images_discarded_lossy_compression;
+}
+void Measurement_statistics::setImagesDiscardedLossyCompression(int64_t const value)
+{
+    m_Images_discarded_lossy_compression = value;
+    m_Images_discarded_lossy_compressionIsSet = true;
+}
+bool Measurement_statistics::imagesDiscardedLossyCompressionIsSet() const
+{
+    return m_Images_discarded_lossy_compressionIsSet;
+}
+void Measurement_statistics::unsetImages_discarded_lossy_compression()
+{
+    m_Images_discarded_lossy_compressionIsSet = false;
+}
 int64_t Measurement_statistics::getMaxImageNumberSent() const
 {
    return m_Max_image_number_sent;
--- a/broker/gen/model/Measurement_statistics.h
+++ b/broker/gen/model/Measurement_statistics.h
@@ -73,20 +73,27 @@ public:
    bool imagesExpectedIsSet() const;
    void unsetImages_expected();
    /// <summary>
-    /// 
+    /// Images collected by the receiver. This number will be lower than images expected if there were issues with data collection performance. 
    /// </summary>
    int64_t getImagesCollected() const;
    void setImagesCollected(int64_t const value);
    bool imagesCollectedIsSet() const;
    void unsetImages_collected();
    /// <summary>
-    /// 
+    /// Images sent to the writer.  The value does not include images discarded by lossy compression filter and images not forwarded due to full ZeroMQ queue. 
    /// </summary>
    int64_t getImagesSent() const;
    void setImagesSent(int64_t const value);
    bool imagesSentIsSet() const;
    void unsetImages_sent();
    /// <summary>
+    /// Images discarded by the lossy compression filter
+    /// </summary>
+    int64_t getImagesDiscardedLossyCompression() const;
+    void setImagesDiscardedLossyCompression(int64_t const value);
+    bool imagesDiscardedLossyCompressionIsSet() const;
+    void unsetImages_discarded_lossy_compression();
+    /// <summary>
    /// 
    /// </summary>
    int64_t getMaxImageNumberSent() const;
@@ -182,6 +189,8 @@ protected:
    bool m_Images_collectedIsSet;
    int64_t m_Images_sent;
    bool m_Images_sentIsSet;
+    int64_t m_Images_discarded_lossy_compression;
+    bool m_Images_discarded_lossy_compressionIsSet;
    int64_t m_Max_image_number_sent;
    bool m_Max_image_number_sentIsSet;
    float m_Collection_efficiency;
--- a/broker/jfjoch_api.yaml
+++ b/broker/jfjoch_api.yaml
@@ -457,9 +457,18 @@ components:
        images_collected:
          type: integer
          format: int64
+          description: |
+            Images collected by the receiver. This number will be lower than images expected if there were issues with data collection performance.
        images_sent:
          type: integer
          format: int64
+          description: |
+            Images sent to the writer. 
+            The value does not include images discarded by lossy compression filter and images not forwarded due to full ZeroMQ queue.
+        images_discarded_lossy_compression:
+          type: integer
+          format: int64
+          description: Images discarded by the lossy compression filter
        max_image_number_sent:
          type: integer
          format: int64
--- a/broker/redoc-static.html
+++ b/broker/redoc-static.html
--- a/common/CMakeLists.txt
+++ b/common/CMakeLists.txt
@@ -45,13 +45,14 @@ ADD_LIBRARY( JFJochCommon STATIC
        ../fpga/include/jfjoch_fpga.h
        ZMQWrappers.cpp ZMQWrappers.h
        DatasetSettings.cpp DatasetSettings.h
-        ROIMask.cpp ROIMask.h
+        ROIMap.cpp ROIMap.h
        ROIElement.cpp ROIElement.h
        ROICircle.cpp ROICircle.h
        ROIBox.cpp ROIBox.h
        SendBuffer.cpp SendBuffer.h
        SendBufferControl.cpp SendBufferControl.h
        time_utc.h
+        PixelMask.cpp PixelMask.h
 )

 TARGET_LINK_LIBRARIES(JFJochCommon Compression JFCalibration "$<BUILD_INTERFACE:libzmq-static>" -lrt)
--- a/common/DiffractionExperiment.cpp
+++ b/common/DiffractionExperiment.cpp
@@ -72,7 +72,7 @@ DiffractionExperiment::DiffractionExperiment(const DetectorSetup& det_setup)
 // setter functions
 DiffractionExperiment &DiffractionExperiment::Detector(const DetectorSetup &input) {
    detector = input;
-    roi_mask = ROIMask(input);
+    roi_mask = ROIMap(input);
    return *this;
 }

@@ -702,7 +702,7 @@ void DiffractionExperiment::FillMessage(StartMessage &message) const {
    message.pixel_signed = IsPixelSigned();
    message.sample_name = GetSampleName();
    message.max_spot_count = GetMaxSpotCount();
-    message.pixel_mask_enabled = GetApplyPixelMaskInFPGA();
+    message.pixel_mask_enabled = false;
    message.detector_description = GetDetectorDescription();
    message.space_group_number = GetSpaceGroupNumber();
    message.unit_cell = GetUnitCell();
@@ -739,18 +739,6 @@ void DiffractionExperiment::FillMessage(StartMessage &message) const {
        message.roi_names.emplace_back(x);
 }

-DiffractionExperiment &DiffractionExperiment::ApplyPixelMaskInFPGA(bool input) {
-    debug_pixel_mask = !input;
-    return *this;
-}
-
-bool DiffractionExperiment::GetApplyPixelMaskInFPGA() const {
-    if (GetDetectorMode() == DetectorMode::Conversion)
-        return !debug_pixel_mask;
-    else
-        return false;
-}
-
 float DiffractionExperiment::GetPixelSize_mm() const {
    return detector.GetPixelSize_mm();
 }
@@ -1141,20 +1129,20 @@ DatasetSettings DiffractionExperiment::GetDatasetSettings() const {
    return dataset;
 }

-ROIMask &DiffractionExperiment::ROI() {
+ROIMap &DiffractionExperiment::ROI() {
    return roi_mask;
 }

-const ROIMask &DiffractionExperiment::ROI() const {
+const ROIMap &DiffractionExperiment::ROI() const {
    return roi_mask;
 }

-void DiffractionExperiment::ExportROIMask(uint16_t *dest, size_t module_number) const {
+void DiffractionExperiment::ExportROIMap(uint16_t *v, size_t module_number) const {
    if (GetDetectorMode() == DetectorMode::Conversion)
-        ConvertedToRawGeometry(*this, module_number, dest, roi_mask.GetMask().data());
+        ConvertedToRawGeometry(*this, module_number, v, roi_mask.GetROIMap().data());
    else {
        for (int i = 0; i < RAW_MODULE_SIZE; i++)
-            dest[i] = UINT16_MAX;
+            v[i] = UINT16_MAX;
    }
 }

@@ -1178,4 +1166,9 @@ int64_t DiffractionExperiment::GetSendBufferLocationSize() const {

 float DiffractionExperiment::GetDataReductionFactorSerialMX() const {
    return dataset.GetDataReductionFactorSerialMX();
-}
+}
+
+DiffractionExperiment &DiffractionExperiment::DataReductionFactorSerialMX(float input) {
+    dataset.DataReductionFactorSerialMX(input);
+    return *this;
+}
--- a/common/DiffractionExperiment.h
+++ b/common/DiffractionExperiment.h
@@ -16,7 +16,7 @@
 #include "DetectorSetup.h"
 #include "../image_analysis/SpotFindingSettings.h"
 #include "DatasetSettings.h"
-#include "ROIMask.h"
+#include "ROIMap.h"

 enum class DetectorMode {
    Conversion, Raw, PedestalG0, PedestalG1, PedestalG2
@@ -97,7 +97,7 @@ class DiffractionExperiment {

    // Dataset settings
    DatasetSettings dataset;
-    ROIMask roi_mask;
+    ROIMap roi_mask;

    int64_t max_spot_count;
 public:
@@ -138,7 +138,6 @@ public:
    DiffractionExperiment& SourceNameShort(std::string input);
    DiffractionExperiment& InstrumentName(std::string input);
    DiffractionExperiment& InstrumentNameShort(std::string input);
-    DiffractionExperiment& ApplyPixelMaskInFPGA(bool input);
    DiffractionExperiment& DefaultOmegaAxis(const Coord &c);

    DiffractionExperiment& UsingGainHG0(bool input);
@@ -173,6 +172,7 @@ public:
    DiffractionExperiment& FPGAOutputMode(FPGAPixelOutput input);
    DiffractionExperiment& MaxSpotCount(int64_t input);
    DiffractionExperiment& ImagesPerFile(int64_t input);
+    DiffractionExperiment& DataReductionFactorSerialMX(float input);

    DiffractionExperiment& ImportDatasetSettings(const DatasetSettings& input);
    DatasetSettings GetDatasetSettings() const;
@@ -255,7 +255,6 @@ public:

    int64_t GetMaxSpotCount() const;

-    bool GetApplyPixelMaskInFPGA() const;
    float GetPixelSize_mm() const;
    std::string GetSourceName() const;
    std::string GetSourceNameShort() const;
@@ -320,9 +319,9 @@ public:
    CompressionAlgorithm GetCompressionAlgorithm() const;
    int64_t GetNumTriggers() const;

-    ROIMask& ROI();
-    const ROIMask& ROI() const;
-    void ExportROIMask(uint16_t *v, size_t module_number) const;
+    ROIMap& ROI();
+    const ROIMap& ROI() const;
+    void ExportROIMap(uint16_t *v, size_t module_number) const;
    int64_t GetImagesPerFile() const;

    float GetDataReductionFactorSerialMX() const;
--- a/common/DiffractionSpot.cpp
+++ b/common/DiffractionSpot.cpp
@@ -39,15 +39,15 @@ int64_t DiffractionSpot::PixelCount() const {
 }

 Coord DiffractionSpot::LabCoord(const DiffractionExperiment &experiment) const {
-    return experiment.LabCoord(x  / (float)photons, y / (float)photons);
+    return experiment.LabCoord(x  / (float)photons + 0.5f, y / (float)photons + 0.5f);
 }

 Coord DiffractionSpot::ReciprocalCoord(const DiffractionExperiment &experiment) const {
-    return DetectorToRecip(experiment, x  / (float)photons, y / (float)photons);
+    return DetectorToRecip(experiment, x  / (float)photons + 0.5f, y / (float)photons + 0.5f);
 }

 double DiffractionSpot::GetResolution(const DiffractionExperiment &experiment) const {
-    return PxlToRes(experiment, x  / (float)photons, y / (float)photons);
+    return PxlToRes(experiment, x  / (float)photons + 0.5f, y / (float)photons + 0.5f);
 }

 DiffractionSpot::operator SpotToSave() const {
--- a/common/PixelMask.cpp
+++ b/common/PixelMask.cpp
@@ -0,0 +1,80 @@
+// Copyright (2019-2024) Paul Scherrer Institute
+
+#include "PixelMask.h"
+#include "RawToConvertedGeometry.h"
+#include "JFJochException.h"
+
+PixelMask::PixelMask(const DetectorSetup &detector)
+        : nmodules(detector.GetModulesNum()),
+          mask(detector.GetModulesNum() * RAW_MODULE_SIZE, 0) {
+}
+
+PixelMask::PixelMask(const DiffractionExperiment& experiment)
+        : PixelMask(experiment.GetDetectorSetup()) {}
+
+void PixelMask::LoadDetectorBadPixelMask(const std::vector<uint32_t> &input_mask, uint8_t bit) {
+    if (input_mask.size() != mask.size())
+        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
+                              "Input match doesn't fit the detector ");
+
+    for (int i = 0; i < mask.size(); i++) {
+        if (input_mask[i] != 0)
+            mask[i] |=  (1<<bit);
+        else
+            mask[i] &= ~(1<<bit);
+    }
+}
+
+std::vector<uint32_t> PixelMask::GetMask(const DiffractionExperiment &experiment) {
+    std::vector<uint32_t> mask_out = mask;
+
+    // apply edge mask
+    for (int64_t module = 0; module < nmodules; module++) {
+        for (int64_t line = 0; line < RAW_MODULE_LINES; line++) {
+            for (int64_t col = 0; col < RAW_MODULE_COLS; col++) {
+                int64_t pixel = module * RAW_MODULE_SIZE + line * RAW_MODULE_COLS + col;
+
+                if (experiment.GetMaskModuleEdges()) {
+                    if ((line == 0)
+                        || (line == RAW_MODULE_LINES - 1)
+                        || (col == 0)
+                        || (col == RAW_MODULE_COLS - 1))
+                        mask_out[pixel] |= (1 << 30);
+                }
+
+                if (experiment.GetMaskChipEdges()) {
+                    if ((col == 255) || (col == 256)
+                        || (col == 511) || (col == 512)
+                        || (col == 767) || (col == 768)
+                        || (line == 255) || (line== 256))
+                        mask_out[pixel] |= (1 << 31);
+                }
+            }
+        }
+    }
+
+    if (experiment.GetDetectorMode() == DetectorMode::Conversion) {
+        std::vector<uint32_t> tmp(experiment.GetPixelsNum(), 1); // nonfuctional areas (i.e. gaps) are filled with 1
+        RawToConvertedGeometry<uint32_t, uint32_t>(experiment, tmp.data(), mask_out.data());
+        return tmp;
+    } else
+        return mask_out;
+}
+
+void PixelMask::LoadUserMask(const DiffractionExperiment &experiment,
+                             const std::vector<uint32_t> &in_mask,
+                             uint8_t bit) {
+    if (experiment.GetModulesNum() != nmodules)
+        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
+                              "Mismatch in module size");
+
+    if (in_mask.size() == nmodules * RAW_MODULE_SIZE) {
+        LoadDetectorBadPixelMask(in_mask, bit);
+    } else if (in_mask.size() == experiment.GetPixelsNum()) {
+        std::vector<uint32_t> raw_mask(nmodules * RAW_MODULE_SIZE);
+        ConvertedToRawGeometry(experiment, raw_mask.data(), in_mask.data());
+        LoadDetectorBadPixelMask(in_mask, bit);
+    } else
+        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
+                              "Size of input user mask invalid");
+}
--- a/common/PixelMask.h
+++ b/common/PixelMask.h
@@ -0,0 +1,21 @@
+// Copyright (2019-2024) Paul Scherrer Institute
+
+#ifndef JUNGFRAUJOCH_PIXELMASK_H
+#define JUNGFRAUJOCH_PIXELMASK_H
+
+#include "DetectorSetup.h"
+#include "DiffractionExperiment.h"
+
+class PixelMask {
+    size_t nmodules;
+    std::vector<uint32_t> mask;
+public:
+    PixelMask(const DetectorSetup& detector);
+    PixelMask(const DiffractionExperiment& experiment);
+    void LoadUserMask(const DiffractionExperiment& experiment, const std::vector<uint32_t>& mask, uint8_t bit);
+    void LoadDetectorBadPixelMask(const std::vector<uint32_t>& mask, uint8_t bit);
+    std::vector<uint32_t> GetMask(const DiffractionExperiment& experiment);
+};
+
+
+#endif //JUNGFRAUJOCH_PIXELMASK_H
--- a/common/ROIMask.cpp
+++ b/common/ROIMask.cpp
@@ -1,43 +1,43 @@
 // Copyright (2019-2024) Paul Scherrer Institute

-#include "ROIMask.h"
+#include "ROIMap.h"
 #include "JFJochException.h"

-ROIMask::ROIMask(int64_t in_xpixel, int64_t in_ypixel)
-        : mask(in_xpixel * in_ypixel, UINT16_MAX),
+ROIMap::ROIMap(int64_t in_xpixel, int64_t in_ypixel)
+        : map(in_xpixel * in_ypixel, UINT16_MAX),
          xpixel(in_xpixel), ypixel(in_ypixel) {
 }

-ROIMask::ROIMask(const DetectorSetup &setup)
-        : ROIMask(setup.GetGeometry().GetWidth(), setup.GetGeometry().GetHeight()) {}
+ROIMap::ROIMap(const DetectorSetup &setup)
+        : ROIMap(setup.GetGeometry().GetWidth(), setup.GetGeometry().GetHeight()) {}

-void ROIMask::UpdateMask() {
+void ROIMap::UpdateMask() {
    roi_name_map.clear();

-    for (auto &i: mask)
+    for (auto &i: map)
        i = UINT16_MAX;

    for (int i = 0; i < boxes.size(); i++) {
-            boxes[i].MarkROI(mask, i, xpixel, ypixel);
+            boxes[i].MarkROI(map, i, xpixel, ypixel);
            roi_name_map[boxes[i].GetName()] = i;
    }

    for (int i = 0; i < circles.size(); i++) {
        size_t id = boxes.size() + i;
-        circles[i].MarkROI(mask, id, xpixel, ypixel);
+        circles[i].MarkROI(map, id, xpixel, ypixel);
        roi_name_map[circles[i].GetName()] = id;
    }
 }

-const std::vector<uint16_t>& ROIMask::GetMask() const {
-    return mask;
+const std::vector<uint16_t>& ROIMap::GetROIMap() const {
+    return map;
 }

-const std::map<std::string, uint16_t> &ROIMask::GetROINameMap() const {
+const std::map<std::string, uint16_t> &ROIMap::GetROINameMap() const {
    return roi_name_map;
 }

-void ROIMask::SetROIBox(const std::vector<ROIBox> &input) {
+void ROIMap::SetROIBox(const std::vector<ROIBox> &input) {
    if (boxes.size() > box_array_size)
        throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, "Limit of box ROIs exceeded");
    for (const auto &i: input) {
@@ -51,17 +51,17 @@ void ROIMask::SetROIBox(const std::vector<ROIBox> &input) {

 }

-void ROIMask::SetROICircle(const std::vector<ROICircle> &input) {
+void ROIMap::SetROICircle(const std::vector<ROICircle> &input) {
    if (circles.size() > circle_array_size)
        throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, "Limit of circular ROIs exceeded");
    circles = input;
    UpdateMask();
 }

-const std::vector<ROIBox> &ROIMask::GetROIBox() const {
+const std::vector<ROIBox> &ROIMap::GetROIBox() const {
    return boxes;
 }

-const std::vector<ROICircle> &ROIMask::GetROICircle() const {
+const std::vector<ROICircle> &ROIMap::GetROICircle() const {
    return circles;
 }
--- a/common/ROIMask.h
+++ b/common/ROIMask.h
@@ -1,7 +1,7 @@
 // Copyright (2019-2024) Paul Scherrer Institute

-#ifndef JUNGFRAUJOCH_ROIMASK_H
-#define JUNGFRAUJOCH_ROIMASK_H
+#ifndef JUNGFRAUJOCH_ROIMAP_H
+#define JUNGFRAUJOCH_ROIMAP_H

 #include <map>
 #include <vector>
@@ -12,10 +12,10 @@
 #include "ROIBox.h"
 #include "ROICircle.h"

-class ROIMask {
+class ROIMap {
    std::vector<ROIBox> boxes;
    std::vector<ROICircle> circles;
-    std::vector<uint16_t> mask;
+    std::vector<uint16_t> map;
    std::map<std::string, uint16_t> roi_name_map;

    int64_t xpixel;
@@ -25,14 +25,14 @@ class ROIMask {
    static constexpr const size_t circle_array_size = 32;
    void UpdateMask();
 public:
-    explicit ROIMask(const DetectorSetup& setup);
-    ROIMask(int64_t xpixel, int64_t ypixel);
+    explicit ROIMap(const DetectorSetup& setup);
+    ROIMap(int64_t xpixel, int64_t ypixel);
    void SetROIBox(const std::vector<ROIBox> &input);
    void SetROICircle(const std::vector<ROICircle> &input);
    [[nodiscard]] const std::vector<ROIBox>& GetROIBox() const;
    [[nodiscard]] const std::vector<ROICircle>& GetROICircle() const;
-    [[nodiscard]] const std::vector<uint16_t>& GetMask() const;
+    [[nodiscard]] const std::vector<uint16_t>& GetROIMap() const;
    [[nodiscard]] const std::map<std::string, uint16_t>& GetROINameMap() const;
 };

-#endif //JUNGFRAUJOCH_ROIMASK_H
+#endif //JUNGFRAUJOCH_ROIMAP_H
--- a/common/ZMQWrappers.cpp
+++ b/common/ZMQWrappers.cpp
@@ -194,6 +194,7 @@ std::string ZMQSocket::GetEndpointName() {
 }

 bool ZMQSocket::Receive(ZMQMessage &msg, bool blocking) {
+    std::unique_lock<std::mutex> ul(m);
    return (zmq_msg_recv(msg.GetMsg(), socket, blocking ? 0 : ZMQ_DONTWAIT) >= 0);
 }

--- a/export_images/PreviewImage.cpp
+++ b/export_images/PreviewImage.cpp
@@ -16,7 +16,7 @@ PreviewImage::PreviewImage(const DiffractionExperiment &experiment) :
        pixel_depth_bytes(experiment.GetPixelDepth()),
        pixel_is_signed(experiment.IsPixelSigned()),
        uncompressed_image(experiment.GetPixelsNum() * experiment.GetPixelDepth()),
-        roi_mask(experiment.ROI().GetMask()) {}
+        roi_mask(experiment.ROI().GetROIMap()) {}

 void PreviewImage::UpdateImage(const void *in_uncompressed_image,
                               const std::vector<SpotToSave> &in_spots) {
--- a/fpga/hls/spot_finder_mask.cpp
+++ b/fpga/hls/spot_finder_mask.cpp
@@ -71,6 +71,7 @@ void spot_finder_mask(STREAM_768 &data_in,
                    || (((col == 7) || (col == 15) || (col == 23)) && (j == 31))
                    || (((col == 8) || (col == 16) || (col == 24)) && (j ==  0))
                    || (pixel_val[j] == INT24_MIN)
+                    || (pixel_val[j] == INT24_MAX)
                    || ((d[j] != 0) && (d[j] < min_d_value))
                    || ((d[j] != 0) && (d[j] > max_d_value)))
                    mask_val[j] = 0;
--- a/frame_serialize/CBORStream2Deserializer.cpp
+++ b/frame_serialize/CBORStream2Deserializer.cpp
--- a/frame_serialize/CBORStream2Deserializer.h
+++ b/frame_serialize/CBORStream2Deserializer.h
@@ -13,52 +13,14 @@
 #include "JFJochMessages.h"
 #include <mutex>

-class CBORStream2Deserializer {
-public:
-    enum class Type {START, END, IMAGE, CALIBRATION, NONE};
-private:
-    mutable std::mutex m;
-
-    Type msg_type = Type::NONE;
-
-    DataMessage data_message;
-
-    StartMessage start_message;
-    EndMessage end_message;
-    CompressedImage calibration;
-
-    void DecodeType(CborValue &value);
-
-    void GetCBORSpots(CborValue &value);
-    void ProcessGoniometerMap(CborValue &value);
-    void ProcessGoniometerOmega(CborValue &value);
-    void ProcessAxis(CborValue &value, float v[3]);
-
-    void ProcessChannels(CborValue &value);
-    void ProcessImageData(CborValue &value, CompressedImage& image);
-    void ProcessPixelMaskElement(CborValue &value);
-    void ProcessRadIntResultElement(CborValue &value);
-    void ProcessADUHistogramElement(CborValue &value);
-    void ProcessROIElement(CborValue &value);
-    void ProcessROIElementMap(CborValue &value);
-    void ProcessUnitCellElement(CborValue &value);
-    void ProcessStartUserData(CborValue &value);
-
-    bool ProcessCalibration(CborValue &value);
-    bool ProcessImageMessageElement(CborValue &value);
-    bool ProcessStartMessageElement(CborValue &value);
-    bool ProcessEndMessageElement(CborValue &value);
-public:
-    void Process(const std::vector<uint8_t>& buffer);
-    void Process(const uint8_t *msg, size_t msg_size);
-    [[nodiscard]] Type GetType() const;
-    [[nodiscard]] EndMessage GetEndMessage() const;
-    [[nodiscard]] StartMessage GetStartMessage() const;
-
-    // WARNING!!! pointer to data is valid only as long as input buffer is valid
-    [[nodiscard]] DataMessage GetDataMessage() const;
-    [[nodiscard]] CompressedImage GetCalibrationImage() const;
+struct CBORStream2DeserializerOutput {
+    CBORImageType msg_type = CBORImageType::NONE;
+    std::optional<DataMessage> data_message;
+    std::optional<StartMessage> start_message;
+    std::optional<EndMessage> end_message;
+    std::optional<CompressedImage> calibration;
 };

+std::shared_ptr<CBORStream2DeserializerOutput> CBORStream2Deserialize(const uint8_t *msg, size_t msg_size);

 #endif //JUNGFRAUJOCH_CBORSTREAM2DESERIALIZER_H
--- a/frame_serialize/CBORStream2Serializer.cpp
+++ b/frame_serialize/CBORStream2Serializer.cpp
@@ -441,6 +441,8 @@ void CBORStream2Serializer::SerializeImage(const DataMessage& message) {
    CBOR_ENC(mapEncoder, "series_id", message.series_id);
    CBOR_ENC(mapEncoder, "image_id", message.number);

+    CBOR_ENC(mapEncoder, "original_image_id", message.original_number);
+
    CBOR_ENC_RATIONAL(mapEncoder, "real_time", message.exptime, message.exptime_base);
    CBOR_ENC_RATIONAL(mapEncoder, "start_time", message.timestamp, message.timestamp_base);
    CBOR_ENC_RATIONAL(mapEncoder, "end_time", message.timestamp + message.exptime, message.timestamp_base);
@@ -465,6 +467,9 @@ void CBORStream2Serializer::SerializeImage(const DataMessage& message) {
    CBOR_ENC(mapEncoder, "roi_integrals", message.roi);
    CBOR_ENC(mapEncoder, "user_data", message.user_data);
    CBOR_ENC(mapEncoder, "data", message.image);
+    CBOR_ENC(mapEncoder, "corr_beam_x_pxl", message.corr_beam_x_pxl);
+    CBOR_ENC(mapEncoder, "corr_beam_y_pxl", message.corr_beam_y_pxl);
+    CBOR_ENC(mapEncoder, "corr_det_dist_mm", message.corr_det_dist_mm);
    cborErr(cbor_encoder_close_container(&encoder, &mapEncoder));

    curr_size = cbor_encoder_get_buffer_size(&encoder, buffer);
--- a/frame_serialize/JFJochMessages.h
+++ b/frame_serialize/JFJochMessages.h
@@ -17,6 +17,8 @@
 constexpr const uint64_t user_data_release = 4;
 constexpr const uint64_t user_data_magic_number = 0x52320000UL | user_data_release;

+enum class CBORImageType {START, END, IMAGE, CALIBRATION, NONE};
+
 struct CompressedImage {
    const uint8_t *data;
    size_t size; // Including compression
@@ -48,8 +50,9 @@ struct DataMessage {
    float bkg_estimate;
    uint64_t indexing_result; // 0 - not tried, 1 - tried and failed, 2 - tried and success
    std::vector<float> indexing_lattice;
-    std::optional<float> indexing_drift_x_pxl;
-    std::optional<float> indexing_drift_y_pxl;
+    std::optional<float> corr_beam_x_pxl;
+    std::optional<float> corr_beam_y_pxl;
+    std::optional<float> corr_det_dist_mm;

    UnitCell indexing_unit_cell;

@@ -81,6 +84,8 @@ struct DataMessage {
    std::optional<uint64_t> xfel_event_code;

    std::map<std::string, ROIMessage> roi;
+
+    std::optional<int64_t> original_number;
 };

 struct GoniometerAxis {
--- a/frame_serialize/TestImagePusher.cpp
+++ b/frame_serialize/TestImagePusher.cpp
@@ -42,12 +42,13 @@ bool TestImagePusher::SendImage(const uint8_t *image_data, size_t image_size, in

    frame_counter++;
    if (image_number == image_id) {
-        CBORStream2Deserializer deserializer;
-        deserializer.Process(image_data, image_size);
-
-        auto image_array = deserializer.GetDataMessage();
-        receiver_generated_image.resize(image_array.image.size);
-        memcpy(receiver_generated_image.data(), image_array.image.data, image_array.image.size);
+        auto deserialized = CBORStream2Deserialize(image_data, image_size);
+        if (deserialized->data_message) {
+            receiver_generated_image.resize(deserialized->data_message->image.size);
+            memcpy(receiver_generated_image.data(),
+                   deserialized->data_message->image.data,
+                   deserialized->data_message->image.size);
+        }
    }
    return true;
 }
@@ -58,12 +59,13 @@ void TestImagePusher::SendImage(const uint8_t *image_data, size_t image_size, in

    frame_counter++;
    if (image_number == image_id) {
-        CBORStream2Deserializer deserializer;
-        deserializer.Process(image_data, image_size);
-
-        auto image_array = deserializer.GetDataMessage();
-        receiver_generated_image.resize(image_array.image.size);
-        memcpy(receiver_generated_image.data(), image_array.image.data, image_array.image.size);
+        auto deserialized = CBORStream2Deserialize(image_data, image_size);
+        if (deserialized->data_message) {
+            receiver_generated_image.resize(deserialized->data_message->image.size);
+            memcpy(receiver_generated_image.data(),
+                   deserialized->data_message->image.data,
+                   deserialized->data_message->image.size);
+        }
    }
    z->release();
 }
--- a/frontend_ui/src/components/MeasurementStatistics.tsx
+++ b/frontend_ui/src/components/MeasurementStatistics.tsx
@@ -19,6 +19,7 @@ class MeasurementStatistics extends Component<MyProps, MyState> {
            images_expected: 0,
            images_collected: 0,
            images_sent: 0,
+            images_discarded_lossy_compression: 0,
            collection_efficiency : undefined,
            compression_ratio: undefined,
            max_receiver_delay: undefined,
@@ -72,6 +73,10 @@ class MeasurementStatistics extends Component<MyProps, MyState> {
                                    <TableCell component="th" scope="row"> Images sent to writer: </TableCell>
                                    <TableCell align="right">{this.state.s.images_sent}</TableCell>
                                </TableRow>
+                                <TableRow>
+                                    <TableCell component="th" scope="row"> Images discarded by lossy compression: </TableCell>
+                                    <TableCell align="right">{this.state.s.images_discarded_lossy_compression}</TableCell>
+                                </TableRow>
                                <TableRow>
                                    <TableCell component="th" scope="row"> Compression ratio: </TableCell>
                                    <TableCell align="right">{(this.state.s.compression_ratio !== undefined)
--- a/frontend_ui/src/openapi/models/measurement_statistics.ts
+++ b/frontend_ui/src/openapi/models/measurement_statistics.ts
@@ -6,8 +6,21 @@
 export type measurement_statistics = {
    file_prefix?: string;
    images_expected?: number;
+    /**
+     * Images collected by the receiver. This number will be lower than images expected if there were issues with data collection performance.
+     *
+     */
    images_collected?: number;
+    /**
+     * Images sent to the writer.
+     * The value does not include images discarded by lossy compression filter and images not forwarded due to full ZeroMQ queue.
+     *
+     */
    images_sent?: number;
+    /**
+     * Images discarded by the lossy compression filter
+     */
+    images_discarded_lossy_compression?: number;
    max_image_number_sent?: number;
    collection_efficiency?: number;
    compression_ratio?: number;
--- a/image_analysis/CMakeLists.txt
+++ b/image_analysis/CMakeLists.txt
@@ -9,35 +9,17 @@ ADD_LIBRARY(JFJochImageAnalysis STATIC
        CPUSpotFinder.cpp
        CPUSpotFinder.h
        MXAnalyzer.cpp
-        MXAnalyzer.h)
+        MXAnalyzer.h
+        Regression.h)

 TARGET_LINK_LIBRARIES(JFJochImageAnalysis JFJochCommon)

 TARGET_INCLUDE_DIRECTORIES(JFJochImageAnalysis PUBLIC fast-feedback-indexer/eigen)

 IF (CMAKE_CUDA_COMPILER)
-    TARGET_SOURCES(JFJochImageAnalysis PRIVATE
-            fast-feedback-indexer/indexer/src/indexer.cpp
-            fast-feedback-indexer/indexer/src/indexer_gpu.cu
-            fast-feedback-indexer/indexer/src/log.cpp)
-
-    TARGET_SOURCES(JFJochImageAnalysis PUBLIC
-            fast-feedback-indexer/indexer/src/ffbidx/indexer.h
-            fast-feedback-indexer/indexer/src/ffbidx/indexer_gpu.h
-            fast-feedback-indexer/indexer/src/ffbidx/refine.h
-            fast-feedback-indexer/indexer/src/ffbidx/log.h
-            fast-feedback-indexer/indexer/src/ffbidx/exception.h)
-
-    ADD_CUSTOM_TARGET(version_txt
-            ${CMAKE_COMMAND} -D SRC=${CMAKE_CURRENT_SOURCE_DIR}/fast-feedback-indexer/indexer/src/version.h.in
-            -D DST=${CMAKE_CURRENT_SOURCE_DIR}/fast-feedback-indexer/indexer/src/ffbidx/version.h
-            -P ${CMAKE_CURRENT_SOURCE_DIR}/fast-feedback-indexer/indexer/src/GenerateVersionH.cmake
-    )
-    ADD_DEPENDENCIES(JFJochImageAnalysis version_txt)
-    TARGET_COMPILE_DEFINITIONS(JFJochImageAnalysis PUBLIC -DVECTOR_CANDIDATE_REFINEMENT=1)
-    TARGET_INCLUDE_DIRECTORIES(JFJochImageAnalysis PUBLIC fast-feedback-indexer/indexer/src/)
-
-    TARGET_LINK_LIBRARIES(JFJochImageAnalysis ${CUDART_LIBRARY} ${CMAKE_DL_LIBS} rt)
+    INCLUDE_DIRECTORIES(fast-feedback-indexer)
+    TARGET_LINK_LIBRARIES(JFJochImageAnalysis fast_indexer_static)
+    TARGET_INCLUDE_DIRECTORIES(JFJochImageAnalysis PUBLIC fast-feedback-indexer/indexer/src)
 ELSE()
    MESSAGE(WARNING "CUDA is strongly recommended for image analysis." )
 ENDIF()
--- a/image_analysis/CPUSpotFinder.cpp
+++ b/image_analysis/CPUSpotFinder.cpp
@@ -56,6 +56,9 @@ void FindSpots(DeviceOutput &output,
                strong_pixel = false;
            }

+            if (image[coord] > 32760)
+                strong_pixel = false;
+
            if (settings.signal_to_noise_threshold > 0) {
                int64_t in_minus_mean = image[coord] * valid_count - sum;
                if ((in_minus_mean * in_minus_mean <= threshold)
--- a/image_analysis/IndexerWrapper.cpp
+++ b/image_analysis/IndexerWrapper.cpp
@@ -1,7 +1,6 @@
 // Copyright (2019-2024) Paul Scherrer Institute

 #include "IndexerWrapper.h"
-#define MAX_SPOT_COUNT_INDEXING 50

 void IndexerWrapper::Setup(const UnitCell &cell) {
 #ifdef JFJOCH_USE_CUDA
@@ -13,12 +12,10 @@ std::vector<IndexingResult> IndexerWrapper::Run(const std::vector<Coord> &coord,
 #ifdef JFJOCH_USE_CUDA
    std::vector<IndexingResult> ret;

-    if (coord.size() <= min_spots)
+    if (coord.size() <= viable_cell_min_spots)
        return ret;

-    assert(coord.size() < MAX_SPOT_COUNT);
-
-    size_t nspots = std::min<size_t>(MAX_SPOT_COUNT_INDEXING, coord.size());
+    assert(coord.size() <= MAX_SPOT_COUNT);

    for (int i = 0; i < coord.size(); i++) {
        indexer.spotX(i) = coord[i].x;
@@ -27,12 +24,12 @@ std::vector<IndexingResult> IndexerWrapper::Run(const std::vector<Coord> &coord,
    }

    // Index
-    indexer.index(1, nspots);
+    indexer.index(1, coord.size());

    fast_feedback::refine::indexer_ifssr<float>::refine(indexer.spotM().topRows(coord.size()),
                                                        indexer.oCellM(),
                                                        indexer.oScoreV(),
-                                                        fast_feedback::refine::config_ifssr<float>{});
+                                                        cifssr);

    // Find cell most similar to the current one
    for (unsigned i=0u; i<cpers.max_output_cells; i++)
@@ -41,7 +38,8 @@ std::vector<IndexingResult> IndexerWrapper::Run(const std::vector<Coord> &coord,
    // Get best cell
    auto id = fast_feedback::refine::best_cell(indexer.oScoreV());

-    bool indexed =  fast_feedback::refine::is_viable_cell(indexer.oCell(id), indexer.Spots(), indexing_threshold, 9);
+    bool indexed =  fast_feedback::refine::is_viable_cell(indexer.oCell(id), indexer.Spots(),
+                                                          indexing_threshold, viable_cell_min_spots);

    // Check if result is viable
    if (indexed) {
--- a/image_analysis/IndexerWrapper.h
+++ b/image_analysis/IndexerWrapper.h
@@ -22,14 +22,18 @@ class IndexerWrapper {
 #ifdef JFJOCH_USE_CUDA
    fast_feedback::config_runtime<float> crt{};
    fast_feedback::config_persistent<float> cpers{
-            .max_output_cells = 16,
+            .max_output_cells = 32,
            .max_input_cells = 1,
-            .max_spots = MAX_SPOT_COUNT
+            .max_spots = MAX_SPOT_COUNT,
+            .num_candidate_vectors = 32,
+            .redundant_computations = true,
    };    // default persistent config
+    fast_feedback::refine::config_ifssr<float> cifssr{
+        .min_spots = 6u
+    };
    fast_feedback::refine::indexer<float> indexer{cpers, crt};
 #endif
-    constexpr const static uint64_t min_spots = 9;
-    constexpr const static float threshold = 0.05f;
+    constexpr const static uint32_t viable_cell_min_spots = 9;
 public:
    void Setup(const UnitCell &cell);
    std::vector<IndexingResult> Run(const std::vector<Coord> &coord, float indexing_threshold);
--- a/image_analysis/MXAnalyzer.cpp
+++ b/image_analysis/MXAnalyzer.cpp
@@ -3,6 +3,7 @@
 #include "MXAnalyzer.h"
 #include "CPUSpotFinder.h"
 #include "../common/DiffractionGeometry.h"
+#include "Regression.h"

 double stddev(const std::vector<float> &v) {
    if (v.size() <= 1)
@@ -45,6 +46,9 @@ void MXAnalyzer::ReadFromFPGA(const DeviceOutput *output, const SpotFindingSetti
 void MXAnalyzer::ReadFromCPU(const int16_t *image, const SpotFindingSettings &settings, size_t module_number) {
    if (!find_spots)
        return;
+    if (experiment.GetPixelDepth() == 4)
+        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
+                              "CPU spot finder simulation doesn't support 32-bit images");
    std::vector<float> d_map(RAW_MODULE_SIZE);

    DeviceOutput output{};
@@ -82,25 +86,46 @@ bool MXAnalyzer::Process(DataMessage &message, const SpotFindingSettings& settin

        if (!indexer_result.empty()) {
            message.indexing_result = 1;
-            for (int i = 0; i < recip.size(); i++) {
-                auto predicted_pos = RecipToDector(experiment, indexer_result[0].predicted_spots[i]);
-                float x_diff = predicted_pos.first - spots_out[i].RawCoord().x;
-                float y_diff = predicted_pos.second - spots_out[i].RawCoord().y;
-                x_drift += x_diff;
-                y_drift += y_diff;
-
-                message.spots[i].indexed = (x_diff * x_diff + y_diff * y_diff < spot_distance_threshold);
-            }
+            CalculateBeamCenterAndDistance(spots_out,
+                                           indexer_result[0],
+                                           message);

            indexer_result[0].l.Save(message.indexing_lattice);
            message.indexing_unit_cell = indexer_result[0].l.GetUnitCell();

-            if (!recip.empty()) {
-                message.indexing_drift_x_pxl = x_drift / recip.size();
-                message.indexing_drift_y_pxl = y_drift / recip.size();
-            }
        }
    }
    spots.clear();
    return indexed;
 }
+
+void MXAnalyzer::CalculateBeamCenterAndDistance(const std::vector<DiffractionSpot> &spots,
+                                                const IndexingResult& result,
+                                                DataMessage &message) {
+    if (spots.empty())
+        return;
+    assert(result.predicted_spots.size() == spots.size());
+
+    std::vector<float> spot_x_position(spots.size());
+    std::vector<float> spot_y_position(spots.size());
+    std::vector<float> spot_x_Sd1_over_Sd3(spots.size());
+    std::vector<float> spot_y_Sd2_over_Sd3(spots.size());
+
+    for (int i = 0; i < spots.size(); i++) {
+        spot_x_position[i] = spots[i].RawCoord().x * experiment.GetPixelSize_mm();
+        spot_y_position[i] = spots[i].RawCoord().y * experiment.GetPixelSize_mm();
+
+        Coord S = experiment.GetScatteringVector() + result.predicted_spots[i];
+        if (S.z != 0.0f) {
+            spot_x_Sd1_over_Sd3[i] = S.x / S.z;
+            spot_y_Sd2_over_Sd3[i] = S.y / S.z;
+        }
+    }
+
+    auto reg_x = regression(spot_x_Sd1_over_Sd3, spot_x_position);
+    auto reg_y = regression(spot_y_Sd2_over_Sd3, spot_y_position);
+
+    message.corr_beam_x_pxl = experiment.GetBeamX_pxl() - reg_x.intercept / experiment.GetPixelSize_mm();
+    message.corr_beam_y_pxl = experiment.GetBeamY_pxl() - reg_y.intercept / experiment.GetPixelSize_mm();
+    message.corr_det_dist_mm = experiment.GetDetectorDistance_mm() - (reg_x.slope + reg_y.slope) / 2.0;
+}
--- a/image_analysis/MXAnalyzer.h
+++ b/image_analysis/MXAnalyzer.h
@@ -27,6 +27,10 @@ public:

    bool Process(DataMessage &message,
                 const SpotFindingSettings& settings);
+
+    void CalculateBeamCenterAndDistance(const std::vector<DiffractionSpot> &spots,
+                                        const IndexingResult& result,
+                                        DataMessage &message);
 };


--- a/image_analysis/Regression.h
+++ b/image_analysis/Regression.h
@@ -0,0 +1,43 @@
+// Copyright (2019-2024) Paul Scherrer Institute
+
+// Copyright (2019-2024) Paul Scherrer Institute
+
+#ifndef JUNGFRAUJOCH_REGRESSION_H
+#define JUNGFRAUJOCH_REGRESSION_H
+
+#include <vector>
+#include <utility>
+#include "../common/JFJochException.h"
+
+struct RegressionResult {
+    double intercept;
+    double slope;
+};
+
+inline RegressionResult regression(std::vector<float> &x, std::vector<float> &y) {
+    if (x.size() != y.size())
+        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
+                              "Regression: mismatch between input parameter size");
+
+    double x_sum = 0.0;
+    double y_sum = 0.0;
+    double xy_sum = 0.0;
+    double x2_sum = 0.0;
+
+    for (int i = 0; i < x.size(); i++) {
+        x_sum += x[i];
+        y_sum += y[i];
+        x2_sum += x[i] * x[i];
+        xy_sum += x[i] * y[i];
+    }
+
+    double denominator = x.size() * x2_sum - x_sum * x_sum;
+    double intercept = (y_sum * x2_sum - x_sum * xy_sum) / denominator;
+    double slope = (x.size() * xy_sum - x_sum * y_sum) / denominator;
+    return {
+        .intercept = intercept,
+            .slope = slope
+    };
+};
+
+#endif //JUNGFRAUJOCH_REGRESSION_H
--- a/image_analysis/SpotFindingSettings.h
+++ b/image_analysis/SpotFindingSettings.h
@@ -15,7 +15,7 @@ struct SpotFindingSettings {
    float high_resolution_limit = 2.5;
    float low_resolution_limit = 50.0;

-    float indexing_tolerance = 0.05;
+    float indexing_tolerance = 0.1;
 };

 #endif //JUNGFRAUJOCH_SPOTFINDINGSETTINGS_H
--- a/image_analysis/StrongPixelSet.cpp
+++ b/image_analysis/StrongPixelSet.cpp
@@ -176,7 +176,7 @@ void StrongPixelSet::ReadFPGAOutput(const DiffractionExperiment & experiment,
                for (int j = 0; j < 32; j++) {
                    if (bitset.test(j)) {
                        size_t col = (npixel | j) % RAW_MODULE_COLS;
-                        AddStrongPixel(col, line, output.pixels[npixel]);
+                        AddStrongPixel(col, line, output.pixels[npixel | j]);
                    }
                }
            }
@@ -192,7 +192,7 @@ void StrongPixelSet::ReadFPGAOutput(const DiffractionExperiment & experiment,
                for (int j = 0; j < 32; j++) {
                    if (bitset.test(j)) {
                        size_t col = (npixel | j) % RAW_MODULE_COLS;
-                        AddStrongPixel(col, line, ((int32_t *) output.pixels)[npixel]);
+                        AddStrongPixel(col, line, ((int32_t *) output.pixels)[npixel | j]);
                    }
                }
            }
--- a/jungfrau/JFCalibration.cpp
+++ b/jungfrau/JFCalibration.cpp
@@ -1,4 +1,4 @@
-// Copyright (2019-2023) Paul Scherrer Institute
+// Copyright (2019-2024) Paul Scherrer Institute

 #include "JFCalibration.h"

@@ -8,8 +8,7 @@ JFCalibration::JFCalibration(size_t in_nmodules, size_t in_nstorage_cells) :
        nmodules(in_nmodules),
        nstorage_cells(in_nstorage_cells),
        pedestal(in_nmodules * in_nstorage_cells * 3),
-        gain_calibration(in_nmodules),
-        mask(in_nmodules * RAW_MODULE_SIZE) {
+        gain_calibration(in_nmodules) {
    if (in_nmodules * in_nstorage_cells == 0)
        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
                              "Size of JFCalibration cannot be 0");
@@ -26,14 +25,6 @@ size_t JFCalibration::GetStorageCellNum() const {
    return nstorage_cells;
 }

-uint32_t &JFCalibration::Mask(size_t id) {
-    return mask.at(id);
-}
-
-const uint32_t &JFCalibration::Mask(size_t id) const {
-    return mask.at(id);
-}
-
 JFModulePedestal &JFCalibration::Pedestal(size_t module_number, size_t gain_level, size_t storage_cell) {
    if (gain_level >= 3)
        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
@@ -60,10 +51,7 @@ const JFModulePedestal &JFCalibration::Pedestal(size_t module_number, size_t gai
    return pedestal.at((storage_cell * nmodules + module_number) * 3 + gain_level);
 }

-std::vector<uint32_t> JFCalibration::CalculateMask(const DiffractionExperiment &experiment, size_t storage_cell) const {
-    if (experiment.GetModulesNum() != nmodules)
-        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Mismatch in module size");
-
+std::vector<uint32_t> JFCalibration::CalculateMask(size_t storage_cell) const {
    std::vector<uint32_t> tmp(RAW_MODULE_SIZE * nmodules, 0);

    for (int64_t module = 0; module < nmodules; module++) {
@@ -75,23 +63,6 @@ std::vector<uint32_t> JFCalibration::CalculateMask(const DiffractionExperiment &
            for (int64_t col = 0; col < RAW_MODULE_COLS; col++) {
                int64_t pixel = module * RAW_MODULE_SIZE + line * RAW_MODULE_COLS + col;

-                if (experiment.GetMaskModuleEdges()) {
-                    if ((line == 0)
-                        || (line == RAW_MODULE_LINES - 1)
-                        || (col == 0)
-                        || (col == RAW_MODULE_COLS - 1))
-                        tmp[pixel] |= (1 << 30);
-                }
-
-                if (experiment.GetMaskChipEdges()) {
-                    if ((col == 255) || (col == 256)
-                        || (col == 511) || (col == 512)
-                        || (col == 767) || (col == 768)
-                        || (line == 255) || (line== 256))
-                        tmp[pixel] |= (1 << 31);
-                }
-
-                tmp[pixel] |= mask[pixel];
                tmp[pixel] |= mask_g0[line * RAW_MODULE_COLS + col];
                tmp[pixel] |= mask_g1[line * RAW_MODULE_COLS + col];
                tmp[pixel] |= mask_g2[line * RAW_MODULE_COLS + col];
@@ -102,34 +73,14 @@ std::vector<uint32_t> JFCalibration::CalculateMask(const DiffractionExperiment &
    return tmp;
 }

-std::vector<uint32_t> JFCalibration::CalculateNexusMask(const DiffractionExperiment &experiment, size_t storage_cell) const {
-    auto mask_raw = CalculateMask(experiment, storage_cell);
-
-    if (experiment.GetDetectorMode() == DetectorMode::Conversion) {
-        std::vector<uint32_t> tmp(experiment.GetPixelsNum(), 1);
-        RawToConvertedGeometry<uint32_t, uint32_t>(experiment, tmp.data(), mask_raw.data());
-        return tmp;
-    } else
-        return mask_raw;
-}
-
-std::vector<uint8_t> JFCalibration::CalculateOneByteMask(const DiffractionExperiment &experiment, size_t storage_cell) const {
-    auto nexus_mask = CalculateNexusMask(experiment, storage_cell);
-
-    std::vector<uint8_t> tmp(experiment.GetPixelsNum());
-    for (int i = 0; i < experiment.GetPixelsNum(); i++)
-        tmp[i] = (nexus_mask[i] == 0);
-    return tmp;
-}
-
-int64_t JFCalibration::CountMaskedPixels(size_t module_number, size_t storage_cell) const {
+int64_t JFCalibration::CountBadPixels(size_t module_number, size_t storage_cell) const {
    int64_t ret = 0;
    auto mask_g0 = Pedestal(module_number, 0, storage_cell).GetPedestalMask();
    auto mask_g1 = Pedestal(module_number, 1, storage_cell).GetPedestalMask();
    auto mask_g2 = Pedestal(module_number, 2, storage_cell).GetPedestalMask();

    for (int i = 0; i < RAW_MODULE_SIZE; i++) {
-        if ((mask[RAW_MODULE_SIZE * module_number+i] != 0) || (mask_g0[i] != 0) || (mask_g1[i] != 0) || (mask_g2[i] != 0))
+        if ((mask_g0[i] != 0) || (mask_g1[i] != 0) || (mask_g2[i] != 0))
            ret++;
    }
    return ret;
@@ -145,7 +96,7 @@ JFCalibrationModuleStatistics JFCalibration::GetModuleStatistics(size_t module_n
    ret.gain_g0_mean = GainCalibration(module_number).GetG0Mean();
    ret.gain_g1_mean = GainCalibration(module_number).GetG1Mean();
    ret.gain_g2_mean = GainCalibration(module_number).GetG2Mean();
-    ret.masked_pixels = (CountMaskedPixels(module_number, storage_cell));
+    ret.bad_pixels = (CountBadPixels(module_number, storage_cell));
    return ret;
 }

--- a/jungfrau/JFCalibration.h
+++ b/jungfrau/JFCalibration.h
@@ -1,4 +1,4 @@
-// Copyright (2019-2023) Paul Scherrer Institute
+// Copyright (2019-2024) Paul Scherrer Institute

 #ifndef JUNGFRAUJOCH_JFCALIBRATION_H
 #define JUNGFRAUJOCH_JFCALIBRATION_H
@@ -9,11 +9,12 @@
 #include "../common/DiffractionExperiment.h"
 #include "../common/RawToConvertedGeometry.h"
 #include "../common/JFJochException.h"
+#include "../common/PixelMask.h"

 struct JFCalibrationModuleStatistics {
    int64_t module_number;
    int64_t storage_cell_number;
-    uint64_t masked_pixels;
+    uint64_t bad_pixels;

    float pedestal_g0_mean;
    float pedestal_g1_mean;
@@ -30,8 +31,6 @@ class JFCalibration {

    std::vector<JFModulePedestal> pedestal;
    std::vector<JFModuleGainCalibration> gain_calibration;
-
-    std::vector<uint32_t> mask; // pedestal independent mask
 public:
    explicit JFCalibration(size_t nmodules, size_t nstorage_cells = 1);
    explicit JFCalibration(const DiffractionExperiment& experiment);
@@ -39,54 +38,19 @@ public:
    [[nodiscard]] size_t GetModulesNum() const;
    [[nodiscard]] size_t GetStorageCellNum() const;

-    [[nodiscard]] uint32_t &Mask(size_t id);
-    [[nodiscard]] const uint32_t &Mask(size_t id) const;
-
    [[nodiscard]] JFModuleGainCalibration& GainCalibration(size_t module_num);
    [[nodiscard]] const JFModuleGainCalibration& GainCalibration(size_t module_num) const;

    [[nodiscard]] JFModulePedestal& Pedestal(size_t module_num, size_t gain_level, size_t storage_cell = 0);
    [[nodiscard]] const JFModulePedestal& Pedestal(size_t module_num, size_t gain_level, size_t storage_cell = 0) const;

-    [[nodiscard]] std::vector<uint32_t> CalculateMask(const DiffractionExperiment& experiment, size_t storage_cell = 0) const;
-    [[nodiscard]] std::vector<uint8_t> CalculateOneByteMask(const DiffractionExperiment& experiment, size_t storage_cell = 0) const;
-    [[nodiscard]] std::vector<uint32_t> CalculateNexusMask(const DiffractionExperiment& experiment, size_t storage_cell = 0) const;
-    [[nodiscard]] int64_t CountMaskedPixels(size_t module_number, size_t storage_cell = 0) const;
+    [[nodiscard]] std::vector<uint32_t> CalculateMask(size_t storage_cell = 0) const;
+    [[nodiscard]] int64_t CountBadPixels(size_t module_number, size_t storage_cell = 0) const;
    [[nodiscard]] JFCalibrationModuleStatistics GetModuleStatistics(size_t module_number, size_t storage_cell) const;
    [[nodiscard]] std::vector<JFCalibrationModuleStatistics> GetModuleStatistics(size_t storage_cell) const;
    [[nodiscard]] std::vector<JFCalibrationModuleStatistics> GetModuleStatistics() const;

    [[nodiscard]] std::vector<uint16_t> GetPedestal(size_t gain_level, size_t storage_cell = 0) const;
-
-    template <class T> void LoadMask(const DiffractionExperiment &experiment, const std::vector<T> &conv_mask,
-                                     int32_t bit_val = -1) {
-        if (experiment.GetModulesNum() != nmodules)
-            throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
-                                  "Mismatch in module size");
-
-        if (experiment.GetDetectorMode() != DetectorMode::Conversion)
-            throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
-                                  "The operation makes sense only in conversion mode");
-
-        if (conv_mask.size() != experiment.GetPixelsNum())
-            throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
-                                  "Mismatch in size of input mask");
-
-        std::vector<T> raw_mask(nmodules * RAW_MODULE_SIZE);
-        ConvertedToRawGeometry(experiment, raw_mask.data(), conv_mask.data());
-
-        if (bit_val >= 0) {
-            for (int i = 0; i < nmodules * RAW_MODULE_SIZE; i++) {
-                if (raw_mask[i] != 0)
-                    mask[i] |= 1 << bit_val;
-                else
-                    mask[i] &= ~(1 << bit_val);
-            }
-        } else {
-            for (int i = 0; i < nmodules * RAW_MODULE_SIZE; i++)
-                mask[i] = raw_mask[i];
-        }
-    }
 };


--- a/receiver/JFJochReceiver.cpp
+++ b/receiver/JFJochReceiver.cpp
@@ -9,6 +9,7 @@
 #include "ImageMetadata.h"
 #include "../resonet/NeuralNetResPredictor.h"
 #include "../common/time_utc.h"
+#include "../common/PixelMask.h"

 JFJochReceiver::JFJochReceiver(const DiffractionExperiment& in_experiment,
                               const JFCalibration *in_calibration,
@@ -100,26 +101,29 @@ void JFJochReceiver::SendStartMessage() {
    message.az_int_bin_to_q = az_int_mapping.GetBinToQ();
    message.write_master_file = true;

+    PixelMask pixel_mask(experiment);
+
+    if (calibration)
+        pixel_mask.LoadDetectorBadPixelMask(calibration->CalculateMask(), 1);
+
    std::vector<uint32_t> nexus_mask;
-    if (calibration != nullptr) {
-        nexus_mask = calibration->CalculateNexusMask(experiment, 0);
+    nexus_mask = pixel_mask.GetMask(experiment);

-        size_t xpixel = experiment.GetXPixelsNum();
-        size_t ypixel = experiment.GetYPixelsNum();
+    size_t xpixel = experiment.GetXPixelsNum();
+    size_t ypixel = experiment.GetYPixelsNum();

-        CompressedImage image{
-                .data = (uint8_t *) nexus_mask.data(),
-                .size = nexus_mask.size() * sizeof(nexus_mask[0]),
-                .xpixel = xpixel,
-                .ypixel = ypixel,
-                .pixel_depth_bytes = sizeof(nexus_mask[0]),
-                .pixel_is_signed = false,
-                .pixel_is_float = false,
-                .algorithm = CompressionAlgorithm::NO_COMPRESSION,
-                .channel = "default"
-        };
-        message.AddPixelMask(image);
-    }
+    CompressedImage image{
+            .data = (uint8_t *) nexus_mask.data(),
+            .size = nexus_mask.size() * sizeof(nexus_mask[0]),
+            .xpixel = xpixel,
+            .ypixel = ypixel,
+            .pixel_depth_bytes = sizeof(nexus_mask[0]),
+            .pixel_is_signed = false,
+            .pixel_is_float = false,
+            .algorithm = CompressionAlgorithm::NO_COMPRESSION,
+            .channel = "default"
+    };
+    message.AddPixelMask(image);

    image_pusher.StartDataCollection(message);
 }
@@ -261,19 +265,7 @@ void JFJochReceiver::FrameTransformationThread() {
        Cancel(e);
    }

-    std::unique_ptr<NeuralNetResPredictor> neural_net_predictor;
-
-#ifdef JFJOCH_USE_TORCH
-    if (!experiment.GetNeuralNetModelPath().empty()) {
-        try {
-            neural_net_predictor = std::make_unique<NeuralNetResPredictor>(experiment.GetNeuralNetModelPath());
-        } catch (const JFJochException &e) {
-            logger.Warning("Torch error {}", e.what());
-            neural_net_predictor = nullptr;
-        }
-    }
-#endif
-
+    NeuralNetResPredictor neural_net_predictor(experiment.GetNeuralNetModelPath());
    FrameTransformation transformation(experiment);

    MXAnalyzer analyzer(experiment);
@@ -298,6 +290,7 @@ void JFJochReceiver::FrameTransformationThread() {

            DataMessage message{};
            message.number = image_number;
+            message.original_number = image_number;
            message.user_data = experiment.GetImageAppendix();
            message.series_id = experiment.GetSeriesID();
            message.series_unique_id = experiment.GetSeriesIDString();
@@ -345,15 +338,8 @@ void JFJochReceiver::FrameTransformationThread() {
            message.receiver_free_send_buf = send_buf_ctrl.GetAvailBufLocations();
            message.az_int_profile = az_int_profile_image.GetResult();
            message.bkg_estimate = az_int_profile_image.GetMeanValueOfBins(az_int_min_bin, az_int_max_bin);
+            message.resolution_estimation = neural_net_predictor.Inference(experiment, transformation.GetImage());

-            if (neural_net_predictor) {
-                try {
-                    message.resolution_estimation = neural_net_predictor->Inference(experiment, transformation.GetImage());
-                } catch (const std::exception &e) {
-                    logger.ErrorException(e);
-                    message.resolution_estimation.reset();
-                }
-            }
            plots.Add(message, image_number % experiment.GetTimePointNumber(), az_int_profile_image);

            if (preview_counter.GeneratePreview())
@@ -362,29 +348,36 @@ void JFJochReceiver::FrameTransformationThread() {
            if (indexed && preview_counter_indexed.GeneratePreview())
                    preview_image_indexed.UpdateImage(transformation.GetImage(), message.spots);

-            if (push_images_to_writer && serialmx_filter.ApplyFilter(message)) {
-                auto loc = send_buf_ctrl.GetBufLocation();
-                if (loc != nullptr) {
-                    auto writer_buffer = (uint8_t *)loc->get_ptr();
+            images_collected++;

-                    CBORStream2Serializer serializer(writer_buffer, experiment.GetSendBufferLocationSize());
-                    PrepareCBORImage(message, experiment, nullptr, 0);
-                    serializer.SerializeImage(message);
-                    if (experiment.GetSendBufferLocationSize() - serializer.GetImageAppendOffset()
-                        < experiment.GetMaxCompressedSize())
-                        throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds,
-                                              "Not enough memory to save image");
-                    size_t image_size = transformation.CompressImage(writer_buffer + serializer.GetImageAppendOffset());
-                    serializer.AppendImage(image_size);
-                    compressed_size += image_size;
-                    image_pusher.SendImage(writer_buffer, serializer.GetBufferSize(), image_number, loc);
+            if (!push_images_to_writer)
+                continue;

-                    images_sent++; // Handle case when image not sent properly
-                    UpdateMaxImage(image_number);
-                }
+            if (!serialmx_filter.ApplyFilter(message)) {
+                images_skipped++;
+                continue;
            }

-            images_collected++;
+            auto loc = send_buf_ctrl.GetBufLocation();
+            if (loc == nullptr)
+                continue;
+
+            auto writer_buffer = (uint8_t *)loc->get_ptr();
+
+            CBORStream2Serializer serializer(writer_buffer, experiment.GetSendBufferLocationSize());
+            PrepareCBORImage(message, experiment, nullptr, 0);
+            serializer.SerializeImage(message);
+            if (experiment.GetSendBufferLocationSize() - serializer.GetImageAppendOffset()
+                < experiment.GetMaxCompressedSize())
+                throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, "Not enough memory to save image");
+            size_t image_size = transformation.CompressImage(writer_buffer + serializer.GetImageAppendOffset());
+            serializer.AppendImage(image_size);
+            compressed_size += image_size;
+            image_pusher.SendImage(writer_buffer, serializer.GetBufferSize(), image_number, loc);
+
+            images_sent++; // Handle case when image not sent properly
+            UpdateMaxImage(image_number);
+
            logger.Debug("Frame transformation thread - done sending image {}", image_number);
        } catch (const JFJochException &e) {
            logger.ErrorException(e);
@@ -563,7 +556,7 @@ JFJochReceiverStatus JFJochReceiver::GetStatus() const {
    ret.beam_center_drift_pxl = plots.GetBeamCenterDriftPxl();

    if ((experiment.GetImageNum() > 0) && (compressed_size > 0)) {
-        ret.compressed_ratio = static_cast<double> (images_sent
+        ret.compressed_ratio = static_cast<double> ((images_sent + images_skipped)
                                                * experiment.GetPixelDepth()
                                                * experiment.GetModulesNum()
                                                * RAW_MODULE_SIZE)
@@ -576,6 +569,7 @@ JFJochReceiverStatus JFJochReceiver::GetStatus() const {
    ret.max_image_number_sent = max_image_number_sent;
    ret.images_collected = images_collected;
    ret.images_sent = images_sent;
+    ret.images_skipped = images_skipped;
    ret.cancelled = cancelled;

    return ret;
--- a/receiver/JFJochReceiver.h
+++ b/receiver/JFJochReceiver.h
@@ -44,6 +44,7 @@ struct JFJochReceiverStatus {
    uint64_t max_image_number_sent;
    uint64_t images_collected;
    uint64_t images_sent;
+    uint64_t images_skipped;
    bool cancelled;
 };

@@ -92,6 +93,7 @@ class JFJochReceiver {
    std::atomic<size_t> compressed_size{0};
    std::atomic<size_t> images_sent{0};
    std::atomic<size_t> images_collected{0};
+    std::atomic<size_t> images_skipped{0};

    const int64_t frame_transformation_nthreads;

--- a/receiver/JFJochReceiverPlots.cpp
+++ b/receiver/JFJochReceiverPlots.cpp
@@ -36,10 +36,12 @@ void JFJochReceiverPlots::Add(const DataMessage &msg, uint64_t file_number, cons
    indexing_solution.AddElement(msg.number, msg.indexing_result);
    indexing_solution_per_file.Add(file_number, msg.indexing_result);

-    if (msg.indexing_drift_x_pxl)
-        indexing_drift_x_pxl.AddElement(msg.number, msg.indexing_drift_x_pxl.value());
-    if (msg.indexing_drift_y_pxl)
-        indexing_drift_y_pxl.AddElement(msg.number, msg.indexing_drift_y_pxl.value());
+    if (msg.corr_beam_x_pxl)
+        corr_beam_x_pxl.AddElement(msg.number, msg.corr_beam_x_pxl.value());
+    if (msg.corr_beam_y_pxl)
+        corr_beam_y_pxl.AddElement(msg.number, msg.corr_beam_y_pxl.value());
+    if (msg.corr_det_dist_mm)
+        corr_det_dist_mm.AddElement(msg.number, msg.corr_det_dist_mm.value());

    if (msg.resolution_estimation)
        resolution_estimation.Add(msg.resolution_estimation.value());
@@ -85,12 +87,15 @@ MultiLinePlot JFJochReceiverPlots::GetPlots(const PlotRequest &request) {
        case PlotType::IndexingRatePerFile:
            return indexing_solution_per_file.GetPlot();
        case PlotType::IndexingDrift:
-            tmp = indexing_drift_x_pxl.GetMeanPlot(nbins);
+            tmp = corr_beam_x_pxl.GetMeanPlot(nbins);
            tmp.at(0).title = "X beam drift (pxl)";
            ret.emplace_back(tmp.at(0));
-            tmp = indexing_drift_y_pxl.GetMeanPlot(nbins);
+            tmp = corr_beam_y_pxl.GetMeanPlot(nbins);
            tmp.at(0).title = "Y beam drift (pxl)";
            ret.emplace_back(tmp.at(0));
+            tmp = corr_det_dist_mm.GetMeanPlot(nbins);
+            tmp.at(0).title = "Det. distance drift (mm)";
+            ret.emplace_back(tmp.at(0));
            return ret;
        case PlotType::ErrorPixels:
            tmp = saturated_pixels.GetMeanPlot(nbins);
@@ -168,8 +173,8 @@ std::optional<float> JFJochReceiverPlots::GetBkgEstimate() const {
 }

 std::optional<std::pair<float, float>> JFJochReceiverPlots::GetBeamCenterDriftPxl() const {
-    auto diffx = indexing_drift_x_pxl.Mean();
-    auto diffy = indexing_drift_y_pxl.Mean();
+    auto diffx = corr_beam_x_pxl.Mean();
+    auto diffy = corr_beam_y_pxl.Mean();
    if (std::isfinite(diffx) && std::isfinite(diffy))
        return std::make_pair(diffx, diffy);
    else
@@ -194,5 +199,3 @@ std::vector<float> JFJochReceiverPlots::GetAzIntProfilePerFile(uint64_t file_num
    else
        throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, "JFJochReceiverPlots::GetAzIntProfilePerFile out of bounds");
 }
-
-
--- a/receiver/JFJochReceiverPlots.h
+++ b/receiver/JFJochReceiverPlots.h
@@ -27,8 +27,9 @@ class JFJochReceiverPlots {
    StatusVector<float> image_collection_efficiency;
    StatusVector<float> unit_cell[6];
    SetAverage<uint64_t> indexing_solution_per_file;
-    StatusVector<float> indexing_drift_x_pxl;
-    StatusVector<float> indexing_drift_y_pxl;
+    StatusVector<float> corr_beam_x_pxl;
+    StatusVector<float> corr_beam_y_pxl;
+    StatusVector<float> corr_det_dist_mm;

    std::map<std::string, std::unique_ptr<StatusVector<int64_t>>> roi_sum;
    std::map<std::string, std::unique_ptr<StatusVector<int64_t>>> roi_max_count;
--- a/receiver/LossyFilter.cpp
+++ b/receiver/LossyFilter.cpp
@@ -16,11 +16,15 @@ bool LossyFilter::RollDice() {
 }

 bool LossyFilter::ApplyFilter(DataMessage &message) {
-    if ((p == 1.0) || message.indexing_result)
+    if (p == 1.0)
        return true;
-    else if (p == 0.0)
-        return message.indexing_result;
-    return RollDice();
+    else {
+        if (message.indexing_result || ((p > 0.0) && RollDice())) {
+            message.number = image_number++;
+            return true;
+        } else
+            return false;
+    }
 }

 LossyFilter::LossyFilter(const DiffractionExperiment &x)
--- a/receiver/LossyFilter.h
+++ b/receiver/LossyFilter.h
@@ -9,6 +9,7 @@

 #include "../frame_serialize/JFJochMessages.h"
 #include "../common/DiffractionExperiment.h"
+#include <atomic>

 class LossyFilter {
    std::mutex random_m;
@@ -17,6 +18,8 @@ class LossyFilter {
    std::mt19937 mt{rdev()};
    std::uniform_real_distribution<float> distr{0.0, 1.0};

+    std::atomic<int64_t> image_number{0};
+
    float p;
    bool RollDice();
 public:
--- a/resonet/NeuralNetResPredictor.cpp
+++ b/resonet/NeuralNetResPredictor.cpp
@@ -6,14 +6,19 @@
 #include "../common/JFJochException.h"

 NeuralNetResPredictor::NeuralNetResPredictor(const std::string& model_path)
-        : model_input(512*512)
+        : model_input(512*512),
+          enable(!model_path.empty())
 #ifdef JFJOCH_USE_TORCH
-          , device(torch::kCUDA)
+        , device(torch::kCUDA)
 #endif
-        {
+{
 #ifdef JFJOCH_USE_TORCH
-    module = torch::jit::load(model_path);
-    module.to(device);
+    if (enable) {
+        module = torch::jit::load(model_path);
+        module.to(device);
+    }
+#else
+    enable = false;
 #endif
 }

@@ -65,13 +70,15 @@ size_t NeuralNetResPredictor::GetMaxPoolFactor(const DiffractionExperiment& expe
    return pool_factor;
 }

-float NeuralNetResPredictor::Inference(const DiffractionExperiment& experiment, const void *image) {
+std::optional<float> NeuralNetResPredictor::Inference(const DiffractionExperiment& experiment, const void *image) {
+    if (!enable)
+        return {};
+#ifdef JFJOCH_USE_TORCH
    if (experiment.GetPixelDepth() == 2)
        Prepare(experiment, (int16_t *) image);
    else
        Prepare(experiment, (int32_t *) image);

-#ifdef JFJOCH_USE_TORCH
    auto options = torch::TensorOptions().dtype(at::kFloat);
    auto model_input_tensor = torch::from_blob(model_input.data(), {1,1,512,512}, options).to(device);

@@ -83,10 +90,10 @@ float NeuralNetResPredictor::Inference(const DiffractionExperiment& experiment,
    float two_theta = atanf(((2.0f * experiment.GetPixelSize_mm() / experiment.GetDetectorDistance_mm()) * tensor_output));
    float stheta = sinf(two_theta * 0.5f);
    float resolution = experiment.GetWavelength_A() / (2.0f * stheta);
-#else
-    float resolution = 50.0;
-#endif
    return resolution;
+#else
+    return {};
+#endif
 }

 const std::vector<float> &NeuralNetResPredictor::GetModelInput() const {
--- a/resonet/NeuralNetResPredictor.h
+++ b/resonet/NeuralNetResPredictor.h
@@ -16,6 +16,7 @@

 class NeuralNetResPredictor {
    std::vector<float> model_input;
+    bool enable;
 #ifdef JFJOCH_USE_TORCH
    torch::Device device;
    torch::jit::script::Module module;
@@ -26,7 +27,7 @@ public:
    explicit NeuralNetResPredictor(const std::string& model_path);
    void Prepare(const DiffractionExperiment& experiment, const int16_t* image);
    void Prepare(const DiffractionExperiment& experiment, const int32_t* image);
-    float Inference(const DiffractionExperiment& experiment, const void* image);
+    std::optional<float> Inference(const DiffractionExperiment& experiment, const void* image);
    size_t GetMaxPoolFactor(const DiffractionExperiment& experiment) const;
    const std::vector<float> &GetModelInput() const;
 };
--- a/resonet/resonet_test.cpp
+++ b/resonet/resonet_test.cpp
@@ -32,15 +32,20 @@ int main(int argc, char **argv) {

    float x = 0;
    size_t niterations = 25;
+    std::optional<float> result;

    auto start_time = std::chrono::system_clock::now();
-    for (int i = 0; i < niterations; i++)
-        x += predictor.Inference(experiment, image_conv.data());
+    for (int i = 0; i < niterations; i++) {
+        result =   predictor.Inference(experiment, image_conv.data());
+        if (result)
+            x += result.value();
+    }
    auto end_time = std::chrono::system_clock::now();
    auto elapsed = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time);
    int64_t frequency_Hz = (niterations * 1e6) / (double) (elapsed.count());

    std::cout << ((double) elapsed.count()) / (1e3 * niterations) << " ms" << std::endl;
    std::cout << frequency_Hz << " Hz" << std::endl;
-    std::cout << "Resolution " << predictor.Inference(experiment, image_conv.data()) << std::endl;
+    if (result)
+        std::cout << "Resolution " << result.value() << std::endl;
 }
--- a/tests/CBORTest.cpp
+++ b/tests/CBORTest.cpp
@@ -69,12 +69,12 @@ TEST_CASE("CBORSerialize_Start", "[CBOR]") {

    REQUIRE_NOTHROW(serializer.SerializeSequenceStart(message));

-    CBORStream2Deserializer deserializer;
-    REQUIRE_NOTHROW(deserializer.Process(buffer.data(), serializer.GetBufferSize()));
-    REQUIRE(deserializer.GetType() == CBORStream2Deserializer::Type::START);
+    auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
+    REQUIRE(deserialized);
+    REQUIRE(deserialized->msg_type == CBORImageType::START);
+    REQUIRE(deserialized->start_message);
+    StartMessage &output_message = *deserialized->start_message;

-    StartMessage output_message;
-    REQUIRE_NOTHROW(output_message = deserializer.GetStartMessage());
    CHECK(output_message.images_per_file == message.images_per_file);
    CHECK(output_message.detector_distance == Approx(message.detector_distance));
    CHECK(output_message.beam_center_x == Approx(message.beam_center_x));
@@ -166,12 +166,12 @@ TEST_CASE("CBORSerialize_Start_PixelMask", "[CBOR]") {
    message.AddPixelMask(image_mask);
    REQUIRE_NOTHROW(serializer.SerializeSequenceStart(message));

-    CBORStream2Deserializer deserializer;
-    REQUIRE_NOTHROW(deserializer.Process(buffer.data(), serializer.GetBufferSize()));
-    REQUIRE(deserializer.GetType() == CBORStream2Deserializer::Type::START);
+    auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
+    REQUIRE(deserialized);
+    REQUIRE(deserialized->msg_type == CBORImageType::START);
+    REQUIRE(deserialized->start_message);
+    StartMessage &output_message = *deserialized->start_message;

-    StartMessage output_message;
-    REQUIRE_NOTHROW(output_message = deserializer.GetStartMessage());
    REQUIRE(output_message.pixel_mask.size() == 1);
    CHECK(!output_message.pixel_mask[0].pixel_is_float);
    CHECK(output_message.pixel_mask[0].xpixel == 456);
@@ -205,12 +205,12 @@ TEST_CASE("CBORSerialize_Calibration", "[CBOR]") {

    REQUIRE_NOTHROW(serializer.SerializeCalibration(image1));

-    CBORStream2Deserializer deserializer;
-    REQUIRE_NOTHROW(deserializer.Process(buffer.data(), serializer.GetBufferSize()));
-    REQUIRE(deserializer.GetType() == CBORStream2Deserializer::Type::CALIBRATION);
+    auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
+    REQUIRE(deserialized);
+    REQUIRE(deserialized->msg_type == CBORImageType::CALIBRATION);
+    REQUIRE(deserialized->calibration);
+    CompressedImage &output_message = *deserialized->calibration;

-    CompressedImage output_message;
-    REQUIRE_NOTHROW(output_message = deserializer.GetCalibrationImage());
    CHECK(output_message.pixel_is_float);
    CHECK(output_message.xpixel == 16);
    CHECK(output_message.ypixel == 16);
@@ -234,13 +234,13 @@ TEST_CASE("CBORSerialize_End", "[CBOR]") {
    };

    REQUIRE_NOTHROW(serializer.SerializeSequenceEnd(message));
-    
-    CBORStream2Deserializer deserializer;
-    REQUIRE_NOTHROW(deserializer.Process(buffer.data(), serializer.GetBufferSize()));
-    REQUIRE(deserializer.GetType() == CBORStream2Deserializer::Type::END);

-    EndMessage output_message{};
-    REQUIRE_NOTHROW(output_message = deserializer.GetEndMessage());
+    auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
+    REQUIRE(deserialized);
+    REQUIRE(deserialized->msg_type == CBORImageType::END);
+    REQUIRE(deserialized->end_message);
+    EndMessage &output_message = *deserialized->end_message;
+
    REQUIRE(output_message.max_receiver_delay == message.max_receiver_delay);
    REQUIRE(output_message.max_image_number == message.max_image_number);
    REQUIRE(output_message.images_collected_count == message.images_collected_count);
@@ -255,7 +255,7 @@ TEST_CASE("CBORSerialize_End", "[CBOR]") {

 TEST_CASE("CBORSerialize_End_RadIntResult", "[CBOR]") {
    std::vector<uint8_t> buffer(8*1024*1024);
-CBORStream2Serializer serializer(buffer.data(), buffer.size());
+    CBORStream2Serializer serializer(buffer.data(), buffer.size());

    EndMessage message {
            .max_image_number = 57789,
@@ -271,12 +271,11 @@ CBORStream2Serializer serializer(buffer.data(), buffer.size());

    REQUIRE_NOTHROW(serializer.SerializeSequenceEnd(message));

-    CBORStream2Deserializer deserializer;
-    REQUIRE_NOTHROW(deserializer.Process(buffer.data(), serializer.GetBufferSize()));
-    REQUIRE(deserializer.GetType() == CBORStream2Deserializer::Type::END);
+    auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
+    REQUIRE(deserialized);
+    REQUIRE(deserialized->msg_type == CBORImageType::END);
+    EndMessage &output_message = *deserialized->end_message;

-    EndMessage output_message{};
-    REQUIRE_NOTHROW(output_message = deserializer.GetEndMessage());
    REQUIRE(output_message.az_int_result.size() == 2);
    REQUIRE(output_message.az_int_result.contains("avg"));
    REQUIRE(output_message.az_int_result.contains("file0"));
@@ -303,12 +302,12 @@ TEST_CASE("CBORSerialize_End_ADUHistogram", "[CBOR]") {

    REQUIRE_NOTHROW(serializer.SerializeSequenceEnd(message));

-    CBORStream2Deserializer deserializer;
-    REQUIRE_NOTHROW(deserializer.Process(buffer.data(), serializer.GetBufferSize()));
-    REQUIRE(deserializer.GetType() == CBORStream2Deserializer::Type::END);
+    auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
+    REQUIRE(deserialized);
+    REQUIRE(deserialized->msg_type == CBORImageType::END);
+    REQUIRE(deserialized->end_message);
+    EndMessage &output_message = *deserialized->end_message;

-    EndMessage output_message{};
-    REQUIRE_NOTHROW(output_message = deserializer.GetEndMessage());
    REQUIRE(output_message.adu_histogram.size() == 2);
    REQUIRE(output_message.adu_histogram.contains("avg"));
    REQUIRE(output_message.adu_histogram.contains("file0"));
@@ -317,7 +316,6 @@ TEST_CASE("CBORSerialize_End_ADUHistogram", "[CBOR]") {
    CHECK(message.adu_histogram_bin_width == output_message.adu_histogram_bin_width);
 }

-
 TEST_CASE("CBORSerialize_Image", "[CBOR]") {
    std::vector<uint8_t> buffer(8*1024*1024);
    CBORStream2Serializer serializer(buffer.data(), buffer.size());
@@ -347,6 +345,9 @@ TEST_CASE("CBORSerialize_Image", "[CBOR]") {
            .spots = spots,
            .bkg_estimate = 12.345f,
            .indexing_result = 1,
+            .corr_beam_x_pxl = -123.4,
+            .corr_beam_y_pxl = -125.4,
+            .corr_det_dist_mm = 12345.0,
            .adu_histogram = {3, 4, 5, 8},
            .timestamp = 1ul<<27 | 1ul <<35,
            .exptime = 1000,
@@ -361,15 +362,17 @@ TEST_CASE("CBORSerialize_Image", "[CBOR]") {
            .storage_cell = 0xF,
            .xfel_pulse_id = UINT64_MAX - 5678,
            .xfel_event_code = UINT64_MAX - 123,
+            .original_number = 12789
    };

    REQUIRE_NOTHROW(serializer.SerializeImage(message));

-    CBORStream2Deserializer deserializer;
-    REQUIRE_NOTHROW(deserializer.Process(buffer.data(), serializer.GetBufferSize()));
-    REQUIRE(deserializer.GetType() == CBORStream2Deserializer::Type::IMAGE);
+    auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
+    REQUIRE(deserialized);
+    REQUIRE(deserialized->msg_type == CBORImageType::IMAGE);
+    REQUIRE(deserialized->data_message);
+    DataMessage &image_array = *deserialized->data_message;

-    auto image_array = deserializer.GetDataMessage();
    REQUIRE(image_array.image.algorithm == CompressionAlgorithm::NO_COMPRESSION);
    REQUIRE(image_array.image.xpixel == 256);
    REQUIRE(image_array.image.ypixel == 2);
@@ -398,11 +401,15 @@ TEST_CASE("CBORSerialize_Image", "[CBOR]") {
    REQUIRE(image_array.bkg_estimate == message.bkg_estimate);
    REQUIRE(image_array.image_collection_efficiency == message.image_collection_efficiency);
    REQUIRE(image_array.user_data == message.user_data);
+    REQUIRE(image_array.original_number == message.original_number);
+    REQUIRE(image_array.corr_beam_x_pxl == message.corr_beam_x_pxl);
+    REQUIRE(image_array.corr_beam_y_pxl == message.corr_beam_y_pxl);
+    REQUIRE(image_array.corr_det_dist_mm == message.corr_det_dist_mm);
 }

 TEST_CASE("CBORSerialize_Image_2", "[CBOR]") {
    std::vector<uint8_t> buffer(8*1024*1024);
-CBORStream2Serializer serializer(buffer.data(), buffer.size());
+    CBORStream2Serializer serializer(buffer.data(), buffer.size());

    std::vector<SpotToSave> spots;

@@ -431,11 +438,12 @@ CBORStream2Serializer serializer(buffer.data(), buffer.size());

    REQUIRE_NOTHROW(serializer.SerializeImage(message));

-    CBORStream2Deserializer deserializer;
-    REQUIRE_NOTHROW(deserializer.Process(buffer.data(), serializer.GetBufferSize()));
-    REQUIRE(deserializer.GetType() == CBORStream2Deserializer::Type::IMAGE);
+    auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
+    REQUIRE(deserialized);
+    REQUIRE(deserialized->msg_type == CBORImageType::IMAGE);
+    REQUIRE(deserialized->data_message);
+    DataMessage &image_array = *deserialized->data_message;

-    auto image_array = deserializer.GetDataMessage();
    REQUIRE(image_array.image.algorithm == CompressionAlgorithm::NO_COMPRESSION);
    REQUIRE(image_array.image.xpixel == 1024);
    REQUIRE(image_array.image.ypixel == 512);
@@ -447,6 +455,10 @@ CBORStream2Serializer serializer(buffer.data(), buffer.size());
    REQUIRE(image_array.indexing_result == message.indexing_result);
    REQUIRE(image_array.number == 480);
    REQUIRE(memcmp(image_array.image.data, test.data(), test.size()) == 0);
+    REQUIRE(!image_array.original_number);
+    REQUIRE(!image_array.corr_det_dist_mm);
+    REQUIRE(!image_array.corr_beam_x_pxl);
+    REQUIRE(!image_array.corr_beam_y_pxl);
 }

 TEST_CASE("CBORSerialize_Image_Float", "[CBOR]") {
@@ -480,11 +492,12 @@ TEST_CASE("CBORSerialize_Image_Float", "[CBOR]") {

    REQUIRE_NOTHROW(serializer.SerializeImage(message));

-    CBORStream2Deserializer deserializer;
-    REQUIRE_NOTHROW(deserializer.Process(buffer.data(), serializer.GetBufferSize()));
-    REQUIRE(deserializer.GetType() == CBORStream2Deserializer::Type::IMAGE);
+    auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
+    REQUIRE(deserialized);
+    REQUIRE(deserialized->msg_type == CBORImageType::IMAGE);
+    REQUIRE(deserialized->data_message);
+    DataMessage &image_array = *deserialized->data_message;

-    auto image_array = deserializer.GetDataMessage();
    REQUIRE(image_array.image.algorithm == CompressionAlgorithm::NO_COMPRESSION);
    REQUIRE(image_array.image.xpixel == 1024);
    REQUIRE(image_array.image.ypixel == 512);
@@ -500,7 +513,7 @@ TEST_CASE("CBORSerialize_Image_Float", "[CBOR]") {

 TEST_CASE("CBORSerialize_Image_Compressed", "[CBOR]") {
    std::vector<uint8_t> buffer(8*1024*1024);
-CBORStream2Serializer serializer(buffer.data(), buffer.size());
+    CBORStream2Serializer serializer(buffer.data(), buffer.size());

    std::vector<SpotToSave> spots;

@@ -528,11 +541,12 @@ CBORStream2Serializer serializer(buffer.data(), buffer.size());

    REQUIRE_NOTHROW(serializer.SerializeImage(message));

-    CBORStream2Deserializer deserializer;
-    REQUIRE_NOTHROW(deserializer.Process(buffer.data(), serializer.GetBufferSize()));
-    REQUIRE(deserializer.GetType() == CBORStream2Deserializer::Type::IMAGE);
+    auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
+    REQUIRE(deserialized);
+    REQUIRE(deserialized->msg_type == CBORImageType::IMAGE);
+    REQUIRE(deserialized->data_message);
+    DataMessage &image_array = *deserialized->data_message;

-    auto image_array = deserializer.GetDataMessage();
    REQUIRE(image_array.image.algorithm == CompressionAlgorithm::BSHUF_LZ4);
    REQUIRE(image_array.image.xpixel == 256);
    REQUIRE(image_array.image.ypixel == 2);
@@ -571,11 +585,12 @@ TEST_CASE("CBORSerialize_Image_Rad_Int_Profile", "[CBOR]") {

    REQUIRE_NOTHROW(serializer.SerializeImage(message));

-    CBORStream2Deserializer deserializer;
-    REQUIRE_NOTHROW(deserializer.Process(buffer.data(), serializer.GetBufferSize()));
-    REQUIRE(deserializer.GetType() == CBORStream2Deserializer::Type::IMAGE);
+    auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
+    REQUIRE(deserialized);
+    REQUIRE(deserialized->msg_type == CBORImageType::IMAGE);
+    REQUIRE(deserialized->data_message);
+    DataMessage &image_array = *deserialized->data_message;

-    auto image_array = deserializer.GetDataMessage();
    REQUIRE(image_array.number == 789);

    REQUIRE(image_array.image.size == test.size());
@@ -613,11 +628,12 @@ TEST_CASE("CBORSerialize_Image_Spots", "[CBOR]") {

    REQUIRE_NOTHROW(serializer.SerializeImage(message));

-    CBORStream2Deserializer deserializer;
-    REQUIRE_NOTHROW(deserializer.Process(buffer.data(), serializer.GetBufferSize()));
-    REQUIRE(deserializer.GetType() == CBORStream2Deserializer::Type::IMAGE);
+    auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
+    REQUIRE(deserialized);
+    REQUIRE(deserialized->msg_type == CBORImageType::IMAGE);
+    REQUIRE(deserialized->data_message);
+    DataMessage &image_array = *deserialized->data_message;

-    auto image_array = deserializer.GetDataMessage();
    REQUIRE(image_array.number == 789);

    REQUIRE(image_array.image.size == test.size());
@@ -662,11 +678,12 @@ TEST_CASE("CBORSerialize_Image_ROI", "[CBOR]") {

    REQUIRE_NOTHROW(serializer.SerializeImage(message));

-    CBORStream2Deserializer deserializer;
-    REQUIRE_NOTHROW(deserializer.Process(buffer.data(), serializer.GetBufferSize()));
-    REQUIRE(deserializer.GetType() == CBORStream2Deserializer::Type::IMAGE);
+    auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
+    REQUIRE(deserialized);
+    REQUIRE(deserialized->msg_type == CBORImageType::IMAGE);
+    REQUIRE(deserialized->data_message);
+    DataMessage &image_array = *deserialized->data_message;

-    auto image_array = deserializer.GetDataMessage();
    REQUIRE(image_array.number == 789);

    REQUIRE(image_array.roi.size() == 2);
@@ -712,12 +729,13 @@ TEST_CASE("CBORSerialize_Image_Append", "[CBOR]") {
    memcpy(buffer.data() + serializer.GetImageAppendOffset(), test.data(), 512 * 1024);
    //REQUIRE_THROWS(serializer.AppendImage(16*1024*1024));
    REQUIRE_NOTHROW(serializer.AppendImage(512 * 1024));
-    CBORStream2Deserializer deserializer;

-    REQUIRE_NOTHROW(deserializer.Process(buffer.data(), serializer.GetBufferSize()));
-    REQUIRE(deserializer.GetType() == CBORStream2Deserializer::Type::IMAGE);
+    auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
+    REQUIRE(deserialized);
+    REQUIRE(deserialized->msg_type == CBORImageType::IMAGE);
+    REQUIRE(deserialized->data_message);
+    DataMessage &image_array = *deserialized->data_message;

-    auto image_array = deserializer.GetDataMessage();
    REQUIRE(image_array.image.algorithm == CompressionAlgorithm::NO_COMPRESSION);
    REQUIRE(image_array.image.xpixel == 1024);
    REQUIRE(image_array.image.ypixel == 512);
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -33,9 +33,11 @@ ADD_EXECUTABLE(CatchTest
        NeuralNetResPredictorTest.cpp
        HistogramTest.cpp
        FPGAEigerReorderTest.cpp
-        ROIMaskTest.cpp
+        ROIMapTest.cpp
        LossyFilterTest.cpp
        SendBufferTest.cpp
+        PixelMaskTest.cpp
+        RegressionTest.cpp
 )

 target_link_libraries(CatchTest JFJochBroker JFJochReceiver JFJochWriter JFJochImageAnalysis JFJochCommon JFJochHLSSimulation JFJochImageExport JFJochResonet)
--- a/tests/DiffractionExperimentTest.cpp
+++ b/tests/DiffractionExperimentTest.cpp
@@ -871,26 +871,26 @@ TEST_CASE("DiffractioExperiment_ExportROIMask", "[DiffractionExperiment]") {
        ROIBox("roi1", 0 , 1, 2162, 2163)
    });

-    CHECK(x.ROI().GetMask()[800] == 0);
-    CHECK(x.ROI().GetMask()[801] == UINT16_MAX);
-    CHECK(x.ROI().GetMask()[2162 * x.GetXPixelsNum() + 1] == 1);
+    CHECK(x.ROI().GetROIMap()[800] == 0);
+    CHECK(x.ROI().GetROIMap()[801] == UINT16_MAX);
+    CHECK(x.ROI().GetROIMap()[2162 * x.GetXPixelsNum() + 1] == 1);

    std::vector<uint16_t> tmp(RAW_MODULE_SIZE, 899);

-    x.ExportROIMask(tmp.data(), 0);
+    x.ExportROIMap(tmp.data(), 0);
    CHECK(tmp[0] == 1);
    CHECK(tmp[1] == 1);
    CHECK(tmp[2] == UINT16_MAX);
    CHECK(tmp[RAW_MODULE_SIZE-1] == UINT16_MAX);

-    x.ExportROIMask(tmp.data(), 6);
+    x.ExportROIMap(tmp.data(), 6);
    CHECK(tmp[0] == UINT16_MAX);
    CHECK(tmp[511 * RAW_MODULE_COLS + 800-6] == 0);
    CHECK(tmp[509 * RAW_MODULE_COLS + 800-6] == 0);
    CHECK(tmp[507 * RAW_MODULE_COLS + 800-6] == 0);
    CHECK(tmp[506 * RAW_MODULE_COLS + 800-6] == UINT16_MAX);

-    x.ExportROIMask(tmp.data(), 3);
+    x.ExportROIMap(tmp.data(), 3);
    size_t diff = 0;
    for (int i = 0; i < RAW_MODULE_SIZE; i++) {
        if (tmp[i] != UINT16_MAX)
--- a/tests/FPGAIntegrationTest.cpp
+++ b/tests/FPGAIntegrationTest.cpp
@@ -912,77 +912,6 @@ TEST_CASE("HLS_C_Simulation_internal_packet_generator_convert_full_range_adu_his
    REQUIRE(err == 0);
 }

-TEST_CASE("HLS_C_Simulation_internal_packet_generator_apply_pixel_mask", "[FPGA][Full]") {
-    double energy = 6.0;
-
-    const uint16_t nmodules = 1;
-    DiffractionExperiment x((DetectorGeometry(nmodules)));
-
-    JFModulePedestal pedestal_g0, pedestal_g1, pedestal_g2;
-    std::vector<double> gain(3 * RAW_MODULE_SIZE);
-
-    std::vector<uint16_t> test_frame(RAW_MODULE_SIZE);
-
-    for (int i = 0; i < RAW_MODULE_SIZE; i++) {
-        test_frame[i] = 5000 | ((i > 128*1024) ? 16384:0) | ((i > 256*1024) ? 32768 : 0);
-    }
-
-    for (int i = 0; i < RAW_MODULE_SIZE; i++) {
-        pedestal_g0.GetPedestal()[i] = 3000;
-        pedestal_g1.GetPedestal()[i] = 14000;
-        pedestal_g2.GetPedestal()[i] = 15000;
-    }
-    pedestal_g0.GetPedestalMask()[0] = 1;
-    pedestal_g1.GetPedestalMask()[1] = 1;
-    pedestal_g2.GetPedestalMask()[2] = 1;
-
-    pedestal_g0.GetPedestalMask()[128*1024+1] = 1;
-    pedestal_g1.GetPedestalMask()[128*1024+2] = 1;
-    pedestal_g2.GetPedestalMask()[128*1024+3] = 1;
-
-    pedestal_g0.GetPedestalMask()[256*1024+1] = 1;
-    pedestal_g1.GetPedestalMask()[256*1024+2] = 1;
-    pedestal_g2.GetPedestalMask()[256*1024+3] = 1;
-
-    x.Mode(DetectorMode::Conversion);
-    x.PedestalG0Frames(0).NumTriggers(1).ImagesPerTrigger(1).UseInternalPacketGenerator(true).PhotonEnergy_keV(energy)
-            .ApplyPixelMaskInFPGA(true);
-    REQUIRE(x.GetPhotonEnergy_keV() == Approx(energy));
-
-    JFCalibration c(x);
-    c.Pedestal(0,0) = pedestal_g0;
-    c.Pedestal(0,1) = pedestal_g1;
-    c.Pedestal(0,2) = pedestal_g2;
-    c.Mask(3)          = UINT32_MAX;
-    c.Mask(128*1024+3) = UINT32_MAX;
-    c.Mask(256*1024+3) = UINT32_MAX;
-
-    HLSSimulatedDevice test(0, 64);
-
-    for (int m = 0; m < x.GetModulesNum(); m++)
-        test.SetInternalGeneratorFrame(test_frame.data(), m);
-
-    REQUIRE_NOTHROW(test.InitializeCalibration(x, c));
-    REQUIRE_NOTHROW(test.StartAction(x));
-    REQUIRE_NOTHROW(test.WaitForActionComplete());
-
-    REQUIRE_NOTHROW(test.OutputStream().read());
-    REQUIRE(test.OutputStream().size() == 0);
-
-    REQUIRE(test.GetBytesReceived() == 128 * JUNGFRAU_PACKET_SIZE_BYTES);
-
-    size_t err = 0;
-    auto frame = test.GetDeviceOutput(0,0)->pixels;
-    auto mask = c.CalculateMask(x);
-    for (int i = 0; i < RAW_MODULE_SIZE; i++) {
-        if (mask[i] && (frame[i] != INT16_MIN))
-            err++;
-        if (!mask[i] && (frame[i] == INT16_MIN))
-            err++;
-    }
-    REQUIRE(err == 0);
-}
-
 TEST_CASE("HLS_C_Simulation_check_2_trigger_convert", "[FPGA][Full]") {
    std::vector<float> pedestal_g0(RAW_MODULE_SIZE), pedestal_g1(RAW_MODULE_SIZE), pedestal_g2(RAW_MODULE_SIZE);

--- a/tests/JFCalibrationTest.cpp
+++ b/tests/JFCalibrationTest.cpp
@@ -56,14 +56,6 @@ TEST_CASE("JFCalibration_PedestalAndMaskStatistics","[JFCalibration]") {
            calibration.Pedestal(module, 2, 1).GetPedestal()[i] = 3000 * (module+1);
        }
    }
-    calibration.Mask(1024*511) = 4;
-
-    calibration.Mask(RAW_MODULE_SIZE + 1024*511+5) = 8;
-    calibration.Mask(RAW_MODULE_SIZE + 1024*511+6) = 8;
-    calibration.Mask(RAW_MODULE_SIZE + 1024*511+7) = 8;
-
-    calibration.Mask(3*RAW_MODULE_SIZE + 1024*511+5) = 8;
-    calibration.Mask(3*RAW_MODULE_SIZE + 1024*511+6) = 8;

    calibration.Pedestal(3, 0, 1).GetPedestalMask()[0] = 5;
    calibration.Pedestal(3, 1, 1).GetPedestalMask()[1] = 5;
@@ -84,10 +76,10 @@ TEST_CASE("JFCalibration_PedestalAndMaskStatistics","[JFCalibration]") {
    REQUIRE(s1[2].pedestal_g1_mean == Approx(6000));
    REQUIRE(s1[2].pedestal_g2_mean == Approx(9000));

-    REQUIRE(s1[0].masked_pixels == 1);
-    REQUIRE(s1[1].masked_pixels == 3);
-    REQUIRE(s1[2].masked_pixels == 0);
-    REQUIRE(s1[3].masked_pixels == 5);
+    REQUIRE(s1[0].bad_pixels == 0);
+    REQUIRE(s1[1].bad_pixels == 0);
+    REQUIRE(s1[2].bad_pixels == 0);
+    REQUIRE(s1[3].bad_pixels == 3);

    auto s0 = calibration.GetModuleStatistics(0);

@@ -95,10 +87,10 @@ TEST_CASE("JFCalibration_PedestalAndMaskStatistics","[JFCalibration]") {
    REQUIRE(s0[0].pedestal_g1_mean == 0);
    REQUIRE(s0[0].pedestal_g2_mean == 0);

-    REQUIRE(s0[0].masked_pixels == 1);
-    REQUIRE(s0[1].masked_pixels == 3);
-    REQUIRE(s0[2].masked_pixels == 0);
-    REQUIRE(s0[3].masked_pixels == 2);
+    REQUIRE(s0[0].bad_pixels == 0);
+    REQUIRE(s0[1].bad_pixels == 0);
+    REQUIRE(s0[2].bad_pixels == 0);
+    REQUIRE(s0[3].bad_pixels == 0);
 }

 TEST_CASE("JFCalibration_Statistics_All","[JFCalibration]") {
@@ -111,14 +103,6 @@ TEST_CASE("JFCalibration_Statistics_All","[JFCalibration]") {
            calibration.Pedestal(module, 2, 1).GetPedestal()[i] = 3000 * (module+1);
        }
    }
-    calibration.Mask(1024*511) = 4;
-
-    calibration.Mask(RAW_MODULE_SIZE + 1024*511+5) = 8;
-    calibration.Mask(RAW_MODULE_SIZE + 1024*511+6) = 8;
-    calibration.Mask(RAW_MODULE_SIZE + 1024*511+7) = 8;
-
-    calibration.Mask(3*RAW_MODULE_SIZE + 1024*511+5) = 8;
-    calibration.Mask(3*RAW_MODULE_SIZE + 1024*511+6) = 8;

    calibration.Pedestal(3, 0, 1).GetPedestalMask()[0] = 5;
    calibration.Pedestal(3, 1, 1).GetPedestalMask()[1] = 5;
@@ -138,10 +122,10 @@ TEST_CASE("JFCalibration_Statistics_All","[JFCalibration]") {
    REQUIRE(s0[0].pedestal_g1_mean == 0);
    REQUIRE(s0[0].pedestal_g2_mean == 0);

-    REQUIRE(s0[0].masked_pixels == 1);
-    REQUIRE(s0[1].masked_pixels == 3);
-    REQUIRE(s0[2].masked_pixels == 0);
-    REQUIRE(s0[3].masked_pixels == 2);
+    REQUIRE(s0[0].bad_pixels == 0);
+    REQUIRE(s0[1].bad_pixels == 0);
+    REQUIRE(s0[2].bad_pixels == 0);
+    REQUIRE(s0[3].bad_pixels == 0);

    REQUIRE(s0[4].pedestal_g0_mean == Approx(1000));
    REQUIRE(s0[4].pedestal_g1_mean == Approx(2000));
@@ -155,180 +139,10 @@ TEST_CASE("JFCalibration_Statistics_All","[JFCalibration]") {
    REQUIRE(s0[6].pedestal_g1_mean == Approx(6000));
    REQUIRE(s0[6].pedestal_g2_mean == Approx(9000));

-    REQUIRE(s0[4].masked_pixels == 1);
-    REQUIRE(s0[5].masked_pixels == 3);
-    REQUIRE(s0[6].masked_pixels == 0);
-    REQUIRE(s0[7].masked_pixels == 5);
-}
-
-TEST_CASE("JFCalibration_MaskModuleEdges","[JFCalibration]") {
-    DiffractionExperiment experiment(DetectorGeometry(4, 1));
-    experiment.MaskModuleEdges(true).Mode(DetectorMode::Raw);
-
-    JFCalibration calibration(experiment);
-
-    auto mask_out = calibration.CalculateNexusMask(experiment);
-
-    CHECK(mask_out[0] == (1u<<30));
-    CHECK(mask_out[1022] == (1u<<30));
-    CHECK(mask_out[1024*5] == (1u<<30));
-    CHECK(mask_out[1024*5+1023] == (1u<<30));
-    CHECK(mask_out[1024*512-1] == (1u<<30));
-    CHECK(mask_out[1024*512] == (1u<<30));
-    CHECK(mask_out[1024*512*3+123] == (1u<<30));
-    CHECK(mask_out[1024*512*3+1024] == (1u<<30));
-    CHECK(mask_out[1024*800+256] == 0);
-}
-
-TEST_CASE("JFCalibration_MaskChipEdges","[JFCalibration]") {
-    DiffractionExperiment experiment(DetectorGeometry(4, 1));
-    experiment.MaskChipEdges(true).Mode(DetectorMode::Raw);
-
-    JFCalibration calibration(experiment);
-
-    auto mask_out = calibration.CalculateNexusMask(experiment);
-
-    CHECK(mask_out[255] == (1u<<31));
-    CHECK(mask_out[1024*800+256] == (1u<<31));
-
-    CHECK(mask_out[1024*233+511] == (1u<<31));
-    CHECK(mask_out[1024*512+512] == (1u<<31));
-    CHECK(mask_out[1024*1000+767] == (1u<<31));
-    CHECK(mask_out[1024*(512*3+12)+768] == (1u<<31));
-    CHECK(mask_out[1024*(512+255)+345] == (1u<<31));
-    CHECK(mask_out[1024*(1024+256)+876] == (1u<<31));
-
-    CHECK(mask_out[1022] == 0);
-    CHECK(mask_out[1024*5+1023] == 0);
-    CHECK(mask_out[1024*18] == 0);
-}
-
-TEST_CASE("JFCalibration_CalculateNexusMask","[JFCalibration]") {
-    DiffractionExperiment experiment(DetectorGeometry(4, 1, 8, 36, false));
-    experiment.MaskModuleEdges(false).MaskChipEdges(false);
-    JFCalibration calibration(4, 2);
-
-    calibration.Mask(0) = 50;
-    calibration.Mask(1) = 30;
-    calibration.Pedestal(0, 0, 0).GetPedestalMask()[2] = 4;
-    calibration.Pedestal(0, 0, 1).GetPedestalMask()[3] = 4;
-    calibration.Pedestal(0, 1, 0).GetPedestalMask()[4] = 8;
-    calibration.Pedestal(0, 2, 0).GetPedestalMask()[5] = 16;
-
-    auto mask_v = calibration.CalculateNexusMask(experiment);
-
-    REQUIRE(mask_v.size() == experiment.GetPixelsNum() );
-    REQUIRE(mask_v[0] == 50);
-    REQUIRE(mask_v[1] == 30);
-    REQUIRE(mask_v[2] == 4);
-    REQUIRE(mask_v[3] == 0);
-    REQUIRE(mask_v[4] == 8);
-    REQUIRE(mask_v[5] == 16);
-    REQUIRE(mask_v[6] == 0);
-    REQUIRE(mask_v[1030 * 700 + 300] == 0);
-
-    mask_v = calibration.CalculateNexusMask(experiment, 1);
-    REQUIRE(mask_v[0] == 50);
-    REQUIRE(mask_v[1] == 30);
-    REQUIRE(mask_v[2] == 0);
-    REQUIRE(mask_v[3] == 4);
-    REQUIRE(mask_v[4] == 0);
-    REQUIRE(mask_v[5] == 0);
-    REQUIRE(mask_v[6] == 0);
-    REQUIRE(mask_v[1030 * 700 + 300] == 0);
-}
-
-TEST_CASE("JFCalibration_CalculateOneByteMask","[JFCalibration]") {
-    DiffractionExperiment experiment(DetectorGeometry(4, 1, 8, 36, false));
-    experiment.MaskModuleEdges(false).MaskChipEdges(false);
-    JFCalibration calibration(experiment);
-    calibration.Mask(0) = 50;
-    calibration.Mask(1) = 30;
-    calibration.Mask(RAW_MODULE_SIZE * 4 - 1) = 10;
-
-    auto mask_v = calibration.CalculateOneByteMask(experiment);
-
-    REQUIRE(mask_v.size() == experiment.GetPixelsNum() );
-    REQUIRE(mask_v[0] == 0); // Explicitly masked
-    REQUIRE(mask_v[1] == 0); // Explicitly masked
-    REQUIRE(mask_v[5] == 1);
-    REQUIRE(mask_v[(514*2+36*2+1)*(experiment.GetXPixelsNum()) + 876] == 1);
-    REQUIRE(mask_v[experiment.GetPixelsNum() - 256] == 1);
-    REQUIRE(mask_v[experiment.GetPixelsNum() - 2] == 1);
-    REQUIRE(mask_v[experiment.GetPixelsNum() - 1] == 0); // Explicitly masked
-    REQUIRE(mask_v[518*(experiment.GetXPixelsNum())] == 0); // Y Gap
-}
-
-TEST_CASE("JFCalibration_LoadMask_Bitwise","[JFCalibration]") {
-    DiffractionExperiment experiment(DetectorGeometry(8, 2, 8, 36, true));
-
-    experiment.PhotonEnergy_keV(WVL_1A_IN_KEV);
-
-    std::vector<uint32_t> new_mask(experiment.GetPixelsNum());
-    new_mask[experiment.GetXPixelsNum() * 323 + 567] = 1;
-    new_mask[experiment.GetXPixelsNum() * 2000 + 1500] = 2;
-
-    JFCalibration calibration(experiment);
-
-    calibration.LoadMask(experiment, new_mask, 4);
-
-    auto output = calibration.CalculateNexusMask(experiment);
-    REQUIRE(output[experiment.GetXPixelsNum() * 323 + 567] == 1<<4);
-    REQUIRE(output[experiment.GetXPixelsNum() * 2000 + 1500] == 1<<4);
-
-    std::vector<uint32_t> second_mask(experiment.GetPixelsNum());
-    second_mask[experiment.GetXPixelsNum() * 323 + 568] = 1;
-    calibration.LoadMask(experiment, second_mask, 4);
-
-    output = calibration.CalculateNexusMask(experiment);
-    REQUIRE(output[experiment.GetXPixelsNum() * 323 + 567] == 0);
-    REQUIRE(output[experiment.GetXPixelsNum() * 323 + 568] == 1 << 4);
-    REQUIRE(output[experiment.GetXPixelsNum() * 2000 + 1500] == 0);
-
-    calibration.LoadMask(experiment, new_mask, 3);
-
-    output = calibration.CalculateNexusMask(experiment);
-    REQUIRE(output[experiment.GetXPixelsNum() * 323 + 567] == 1 << 3);
-    REQUIRE(output[experiment.GetXPixelsNum() * 323 + 568] == 1 << 4);
-    REQUIRE(output[experiment.GetXPixelsNum() * 2000 + 1500] == 1 << 3);
-}
-
-TEST_CASE("JFCalibration_LoadMask","[JFCalibration]") {
-    DiffractionExperiment experiment(DetectorGeometry(8, 2, 8, 36, true));
-
-    experiment.PhotonEnergy_keV(WVL_1A_IN_KEV);
-
-    std::vector<uint32_t> new_mask(experiment.GetPixelsNum());
-    new_mask[experiment.GetXPixelsNum() * 323 + 567] = 1;
-    new_mask[experiment.GetXPixelsNum() * 2000 + 1500] = 2;
-
-    JFCalibration calibration(experiment);
-
-    calibration.LoadMask(experiment, new_mask);
-
-    auto output = calibration.CalculateNexusMask(experiment);
-    REQUIRE(output[experiment.GetXPixelsNum() * 323 + 567] == 1);
-    REQUIRE(output[experiment.GetXPixelsNum() * 2000 + 1500] == 2);
-}
-
-TEST_CASE("JFCalibration_MaskDetectorGaps","[JFCalibration]") {
-    DiffractionExperiment experiment(DetectorGeometry(8, 2, 8, 36, true));
-
-    experiment.PhotonEnergy_keV(WVL_1A_IN_KEV);
-    JFCalibration calibration(experiment);
-
-    auto mask_export = calibration.CalculateNexusMask(experiment);
-
-    REQUIRE(mask_export[0] == 0);
-    REQUIRE(mask_export[1029] == 0);
-    REQUIRE(mask_export[1030] == 1);
-    REQUIRE(mask_export[1031] == 1);
-    REQUIRE(mask_export[1037] == 1);
-    REQUIRE(mask_export[1038] == 0);
-
-    REQUIRE(mask_export[(1030*2+8)*514] == 1);
-    REQUIRE(mask_export[(1030*2+8)*514 + 566] == 1);
-    REQUIRE(mask_export[(1030*2+8)*(514*3+36*2+12) + 566] == 1);
+    REQUIRE(s0[4].bad_pixels == 0);
+    REQUIRE(s0[5].bad_pixels == 0);
+    REQUIRE(s0[6].bad_pixels == 0);
+    REQUIRE(s0[7].bad_pixels == 3);
 }

 TEST_CASE("JFCalibration_GetPedestal","[JFCalibration]") {
--- a/tests/LossyFilterTest.cpp
+++ b/tests/LossyFilterTest.cpp
@@ -12,8 +12,11 @@ TEST_CASE("LossyFilterDisabled","[LossyFilter]") {
    };

    REQUIRE(filter.ApplyFilter(message));
+    REQUIRE(message.number == 123);
    REQUIRE(filter.ApplyFilter(message));
+    REQUIRE(message.number == 123);
    REQUIRE(filter.ApplyFilter(message));
+    REQUIRE(message.number == 123);
 }

 TEST_CASE("LossyFilterEnable","[LossyFilter]") {
@@ -30,9 +33,28 @@ TEST_CASE("LossyFilterEnable","[LossyFilter]") {
    };

    REQUIRE(filter.ApplyFilter(message_1));
+    REQUIRE(message_1.number == 0);
    REQUIRE(!filter.ApplyFilter(message_2));
    REQUIRE(filter.ApplyFilter(message_1));
+    REQUIRE(message_1.number == 1);
    REQUIRE(filter.ApplyFilter(message_1));
+    REQUIRE(message_1.number == 2);
+}
+
+TEST_CASE("LossyFilterEnable_IncrNumber","[LossyFilter]") {
+    LossyFilter filter(0.999);
+
+    DataMessage message_1{
+            .number = 123,
+            .indexing_result = 1
+    };
+
+    REQUIRE(filter.ApplyFilter(message_1));
+    REQUIRE(message_1.number == 0);
+    REQUIRE(filter.ApplyFilter(message_1));
+    REQUIRE(message_1.number == 1);
+    REQUIRE(filter.ApplyFilter(message_1));
+    REQUIRE(message_1.number == 2);
 }

 TEST_CASE("LossyFilterEnable_Prob","[LossyFilter]") {
--- a/tests/NeuralNetResPredictorTest.cpp
+++ b/tests/NeuralNetResPredictorTest.cpp
@@ -45,6 +45,7 @@ TEST_CASE("NeuralNetResPredictor_Inference", "[LinearAlgebra][Coord]") {
    dataset.ReadVector(image_conv, start, file_size);

    auto res = predictor.Inference(experiment, image_conv.data());
-    REQUIRE(res < 1.5);
-    REQUIRE(res > 1.4);
+    REQUIRE(res);
+    REQUIRE(res.value() < 1.5);
+    REQUIRE(res.value() > 1.4);
 }
--- a/tests/PixelMaskTest.cpp
+++ b/tests/PixelMaskTest.cpp
@@ -0,0 +1,95 @@
+// Copyright (2019-2023) Paul Scherrer Institute
+
+#include "catch2/catch.hpp"
+#include "../common/PixelMask.h"
+
+TEST_CASE("PixelMask_MaskModuleEdges","[PixelMask]") {
+    DiffractionExperiment experiment(DetectorGeometry(4, 1));
+    experiment.MaskModuleEdges(true).Mode(DetectorMode::Raw);
+
+    PixelMask mask(experiment);
+
+    auto mask_out = mask.GetMask(experiment);
+
+    REQUIRE(mask_out.size() == experiment.GetModulesNum() * RAW_MODULE_SIZE);
+
+    CHECK(mask_out[0] == (1u<<30));
+    CHECK(mask_out[1022] == (1u<<30));
+    CHECK(mask_out[1024*5] == (1u<<30));
+    CHECK(mask_out[1024*5+1023] == (1u<<30));
+    CHECK(mask_out[1024*512-1] == (1u<<30));
+    CHECK(mask_out[1024*512] == (1u<<30));
+    CHECK(mask_out[1024*512*3+123] == (1u<<30));
+    CHECK(mask_out[1024*512*3+1024] == (1u<<30));
+    CHECK(mask_out[1024*800+256] == 0);
+}
+
+TEST_CASE("PixelMask_MaskChipEdges","[PixelMask]") {
+    DiffractionExperiment experiment(DetectorGeometry(4, 1));
+    experiment.MaskChipEdges(true).Mode(DetectorMode::Raw);
+
+    PixelMask mask(experiment);
+    auto mask_out = mask.GetMask(experiment);
+    REQUIRE(mask_out.size() == experiment.GetModulesNum() * RAW_MODULE_SIZE);
+
+    CHECK(mask_out[255] == (1u<<31));
+    CHECK(mask_out[1024*800+256] == (1u<<31));
+
+    CHECK(mask_out[1024*233+511] == (1u<<31));
+    CHECK(mask_out[1024*512+512] == (1u<<31));
+    CHECK(mask_out[1024*1000+767] == (1u<<31));
+    CHECK(mask_out[1024*(512*3+12)+768] == (1u<<31));
+    CHECK(mask_out[1024*(512+255)+345] == (1u<<31));
+    CHECK(mask_out[1024*(1024+256)+876] == (1u<<31));
+
+    CHECK(mask_out[1022] == 0);
+    CHECK(mask_out[1024*5+1023] == 0);
+    CHECK(mask_out[1024*18] == 0);
+}
+
+TEST_CASE("PixelMask_CalculateNexusMask","[PixelMask]") {
+    DiffractionExperiment experiment(DetectorGeometry(4, 1, 8, 36, false));
+    experiment.MaskModuleEdges(false).MaskChipEdges(false);
+
+    PixelMask mask(experiment);
+
+    std::vector<uint32_t> v(4 * RAW_MODULE_SIZE, 0);
+    v[2] = 1;
+    v[4] = 1;
+    v[5] = 1;
+
+    REQUIRE_NOTHROW(mask.LoadDetectorBadPixelMask(v, 2));
+
+    auto mask_v = mask.GetMask(experiment);
+
+    REQUIRE(mask_v.size() == experiment.GetPixelsNum() );
+    REQUIRE(mask_v[0] == 0);
+    REQUIRE(mask_v[1] == 0);
+    REQUIRE(mask_v[2] == 4);
+    REQUIRE(mask_v[3] == 0);
+    REQUIRE(mask_v[4] == 4);
+    REQUIRE(mask_v[5] == 4);
+    REQUIRE(mask_v[6] == 0);
+    REQUIRE(mask_v[1030 * 700 + 300] == 0);
+}
+
+TEST_CASE("PixelMask_MaskDetectorGaps","[PixelMask]") {
+    DiffractionExperiment experiment(DetectorGeometry(8, 2, 8, 36, true));
+
+    experiment.PhotonEnergy_keV(WVL_1A_IN_KEV);
+
+    PixelMask mask(experiment);
+    auto mask_export = mask.GetMask(experiment);
+    REQUIRE(mask_export.size() == experiment.GetPixelsNum());
+
+    REQUIRE(mask_export[0] == 0);
+    REQUIRE(mask_export[1029] == 0);
+    REQUIRE(mask_export[1030] == 1);
+    REQUIRE(mask_export[1031] == 1);
+    REQUIRE(mask_export[1037] == 1);
+    REQUIRE(mask_export[1038] == 0);
+
+    REQUIRE(mask_export[(1030*2+8)*514] == 1);
+    REQUIRE(mask_export[(1030*2+8)*514 + 566] == 1);
+    REQUIRE(mask_export[(1030*2+8)*(514*3+36*2+12) + 566] == 1);
+}
--- a/tests/ROIMaskTest.cpp
+++ b/tests/ROIMaskTest.cpp
@@ -1,10 +1,10 @@
 // Copyright (2019-2024) Paul Scherrer Institute

 #include <catch2/catch.hpp>
-#include "../common/ROIMask.h"
+#include "../common/ROIMap.h"
 #include "../common/DiffractionExperiment.h"

-TEST_CASE("ROICircle", "[ROIMask]") {
+TEST_CASE("ROICircle", "[ROIMap]") {
    DiffractionExperiment x(DetectorGeometry(1));

    std::unique_ptr<ROICircle> circle;
@@ -28,7 +28,7 @@ TEST_CASE("ROICircle", "[ROIMask]") {
    REQUIRE (mask[218 * x.GetXPixelsNum() + 202] == UINT16_MAX);
 }

-TEST_CASE("ROIBox", "[ROIMask]") {
+TEST_CASE("ROIBox", "[ROIMap]") {
    DiffractionExperiment x(DetectorGeometry(1));

    std::unique_ptr<ROIBox> rectangle;
@@ -60,10 +60,10 @@ TEST_CASE("ROIBox", "[ROIMask]") {
    REQUIRE (mask[223 * x.GetXPixelsNum() + 200] == UINT16_MAX);
 }

-TEST_CASE("ROIMask") {
+TEST_CASE("ROIMap") {
    DiffractionExperiment x(DetectorGeometry(1));

-    ROIMask mask(x.GetDetectorSetup());
+    ROIMap mask(x.GetDetectorSetup());

    mask.SetROICircle({ROICircle("roi7", 300, 400, 5)});

@@ -77,23 +77,23 @@ TEST_CASE("ROIMask") {

    // Check arrays are proper size
    REQUIRE(mask.GetROINameMap().size() == 2);
-    REQUIRE(mask.GetMask().size() == x.GetPixelsNum());
+    REQUIRE(mask.GetROIMap().size() == x.GetPixelsNum());

    auto it1 = mask.GetROINameMap().find("roi12");
    REQUIRE(it1 != mask.GetROINameMap().end());
    REQUIRE(it1->second == 0);
-    REQUIRE(mask.GetMask()[260 * x.GetXPixelsNum()+100] == 0);
+    REQUIRE(mask.GetROIMap()[260 * x.GetXPixelsNum() + 100] == 0);

    auto it2 = mask.GetROINameMap().find("roi7");
    REQUIRE(it2 != mask.GetROINameMap().end());
    REQUIRE(it2->second == 1);
-    REQUIRE(mask.GetMask()[405 * x.GetXPixelsNum()+300] == 1);
+    REQUIRE(mask.GetROIMap()[405 * x.GetXPixelsNum() + 300] == 1);
 }

-TEST_CASE("ROIMask_Empty") {
+TEST_CASE("ROIMap_Empty") {
    DiffractionExperiment x(DetectorGeometry(1));
    size_t err = 0;
-    for (const auto &i: x.ROI().GetMask()) {
+    for (const auto &i: x.ROI().GetROIMap()) {
        if (i != UINT16_MAX)
            err++;
    }
--- a/tests/RegressionTest.cpp
+++ b/tests/RegressionTest.cpp
@@ -0,0 +1,19 @@
+// Copyright (2019-2024) Paul Scherrer Institute
+
+// Copyright (2019-2024) Paul Scherrer Institute
+
+#include <catch2/catch.hpp>
+
+#include "../image_analysis/Regression.h"
+
+TEST_CASE("Regression") {
+    std::vector<float> x(10);
+    std::vector<float> y(10);
+    for (int i = 0; i < x.size(); i++) {
+        x[i] = i;
+        y[i] = 7 * i + 5;
+    }
+    auto reg = regression(x, y);
+    REQUIRE(reg.intercept == Approx(5.0));
+    REQUIRE(reg.slope == Approx(7.0));
+}
--- a/tests/ZMQImagePusherTest.cpp
+++ b/tests/ZMQImagePusherTest.cpp
@@ -14,40 +14,39 @@ void test_puller(ZMQImagePuller *puller,
                 std::vector<size_t> &diff_content,
                 std::vector<size_t> &nimages) {

-    puller->WaitForImage();
-    if (puller->GetFrameType() != CBORStream2Deserializer::Type::START) {
+    auto img = puller->WaitForImage();
+    if (!img.cbor || !img.cbor->start_message) {
        diff_content[writer_id]++;
        return;
    }
-    puller->WaitForImage();
-    while (puller->GetFrameType() != CBORStream2Deserializer::Type::END) {
-        if (puller->GetFrameType() == CBORStream2Deserializer::Type::IMAGE) {
-            auto image = puller->GetDataMessage();
-            if ((nwriter > 1) && ((image.number / 16) % nwriter != writer_id))
+
+    if ((!img.cbor->start_message->write_master_file) || (img.cbor->start_message->write_master_file.value() != (writer_id == 0)))
+        diff_content[writer_id]++;
+    img = puller->WaitForImage();
+    while (img.cbor && !img.cbor->end_message) {
+        if (img.cbor->data_message) {
+            if ((nwriter > 1) && ((img.cbor->data_message->number / 16) % nwriter != writer_id))
                diff_split[writer_id]++;


-            if (image.image.size != x.GetPixelsNum() * sizeof(uint16_t))
+            if (img.cbor->data_message->image.size != x.GetPixelsNum() * sizeof(uint16_t))
                diff_size[writer_id]++;
-            else if (memcmp(image.image.data, image1.data() + image.number * x.GetPixelsNum(),
+            else if (memcmp(img.cbor->data_message->image.data, image1.data() + img.cbor->data_message->number * x.GetPixelsNum(),
                            x.GetPixelsNum() * sizeof(uint16_t)) != 0)
                diff_content[writer_id]++;
-            if (image.image.xpixel != RAW_MODULE_COLS)
+            if (img.cbor->data_message->image.xpixel != RAW_MODULE_COLS)
                diff_content[writer_id]++;
-            if (image.image.ypixel != RAW_MODULE_LINES)
+            if (img.cbor->data_message->image.ypixel != RAW_MODULE_LINES)
                diff_content[writer_id]++;
-            if (image.image.pixel_depth_bytes != 2)
+            if (img.cbor->data_message->image.pixel_depth_bytes != 2)
                diff_content[writer_id]++;
-            if (image.image.algorithm != CompressionAlgorithm::NO_COMPRESSION)
+            if (img.cbor->data_message->image.algorithm != CompressionAlgorithm::NO_COMPRESSION)
                diff_content[writer_id]++;
            nimages[writer_id]++;
        }
-        puller->WaitForImage();
+        img = puller->WaitForImage();
    }

-    auto start = puller->GetStartMessage();
-    if ((!start.write_master_file) || (start.write_master_file.value() != (writer_id == 0)))
-        diff_content[writer_id]++;
 }

 TEST_CASE("ZMQImageCommTest_1Writer","[ZeroMQ]") {
--- a/tools/HDF5DatasetWriteTest.cpp
+++ b/tools/HDF5DatasetWriteTest.cpp
@@ -6,7 +6,7 @@
 #include "../writer/HDF5Writer.h"
 #include "../receiver/FrameTransformation.h"
 #include "../common/RawToConvertedGeometry.h"
-#include "../jungfrau/JFCalibration.h"
+#include "../common/PixelMask.h"
 #include "../writer/HDF5NXmx.h"

 int main(int argc, char **argv) {
@@ -100,9 +100,9 @@ int main(int argc, char **argv) {

    StartMessage start_message;
    x.FillMessage(start_message);
-    JFCalibration calib(x);

-    auto pixel_mask = calib.CalculateNexusMask(x, 0);
+    PixelMask calib(x);
+    auto pixel_mask = calib.GetMask(x);
    {
        size_t xpixel = x.GetXPixelsNum();
        size_t ypixel = x.GetYPixelsNum();
--- a/tools/jfjoch_spot_finding_test.cpp
+++ b/tools/jfjoch_spot_finding_test.cpp
@@ -12,6 +12,7 @@
 #include "../receiver/FrameTransformation.h"
 #include "../image_analysis/MXAnalyzer.h"
 #include "../export_images/PreviewImage.h"
+#include "../receiver/LossyFilter.h"

 Logger logger("jfjoch_spot_finding_test");

@@ -32,6 +33,7 @@ void print_usage() {
    logger.Info("-i<int>       image number");
    logger.Info("-I            enable indexing");
    logger.Info("-t<float>     indexing tolerance (0.0-1.0)");
+    logger.Info("-l<float>     lossy filter probability (0.0-1.0)");
    logger.Info("");
 }

@@ -55,7 +57,8 @@ void GetGeometry(DiffractionExperiment &x, HDF5Object &master_file) {
    } catch (...) {}
 }

-int parse_options(DiffractionExperiment& experiment, SpotFindingSettings& settings,
+int parse_options(DiffractionExperiment& experiment,
+                  SpotFindingSettings& settings,
                  int argc, char **argv) {

    int c;
@@ -72,12 +75,13 @@ int parse_options(DiffractionExperiment& experiment, SpotFindingSettings& settin
            {"indexing",    no_argument, 0, 'I'},
            {"indexing_tolerance",    required_argument, 0, 't'},
            {"nimages",    required_argument, 0, 'n'},
+            {"lossy",    required_argument, 0, 'l'},
            {0, 0, 0, 0}
    };

    int option_index = 0;
    int opt;
-    while ((opt = getopt_long(argc, argv, "c:s:o:p:d:D:j?hS:Ii:t:",long_options, &option_index)) != -1 ) {
+    while ((opt = getopt_long(argc, argv, "c:s:o:p:d:D:j?hS:Ii:t:l:",long_options, &option_index)) != -1 ) {
        switch (opt) {
            case 'j':
                write_jpeg = true;
@@ -112,6 +116,9 @@ int parse_options(DiffractionExperiment& experiment, SpotFindingSettings& settin
            case 't':
                settings.indexing_tolerance = atof(optarg);
                break;
+            case 'l':
+                experiment.DataReductionFactorSerialMX(atof(optarg));
+                break;
            case '?':
            case 'h':
                print_usage();
@@ -178,6 +185,11 @@ int main(int argc, char **argv) {
    logger.Info("Detector distance {} mm", x.GetDetectorDistance_mm());
    logger.Info("Energy {} keV", x.GetPhotonEnergyForConversion_keV());
    logger.Info("Output file prefix {}", x.GetFilePrefix());
+    if (x.GetDataReductionFactorSerialMX() != 1.0)
+        logger.Info("MX reduction factor {:0.2f}", x.GetDataReductionFactorSerialMX());
+    if (settings.indexing)
+        logger.Info("Indexing tolerance {:0.4f}", settings.indexing_tolerance);
+
    logger.Info("Spot finding parameters SNR {:.2f} count {} minpix {} d {:.2f} - {:.2f}",
                settings.signal_to_noise_threshold,
                settings.photon_count_threshold,
@@ -200,6 +212,8 @@ int main(int argc, char **argv) {
    PreviewImage preview(x);
    uint64_t indexed_images = 0;

+    LossyFilter filter(x);
+
    for (int i = 0; i < nimages; i++) {
        std::vector<hsize_t> start = {(hsize_t) i,0,0};
        std::vector<hsize_t> size = {1, (hsize_t) x.GetYPixelsNum(), (hsize_t) x. GetXPixelsNum()};
@@ -222,6 +236,7 @@ int main(int argc, char **argv) {
        DataMessage message{};
        message.image = image;
        message.number = i;
+        message.original_number = i;

        analyzer.Process(message, settings);
        if (message.indexing_result)
@@ -240,7 +255,12 @@ int main(int argc, char **argv) {
            std::ofstream f(fname, std::ios::binary);
            f.write(preview_image.data(), preview_image.size());
        }
-        fileset->Write(message);
+        if (filter.ApplyFilter(message))
+            fileset->Write(message);
+
+        if ((i > 0) && (i % 1000 == 0))
+            logger.Info("Images {:10d}   indexing rate: {:0.1f}%", i,
+                        static_cast<float>(indexed_images) / static_cast<float>(i+1) * 100.0f);
    }
    fileset.reset();
    if (settings.indexing)
--- a/writer/HDF5DataFile.cpp
+++ b/writer/HDF5DataFile.cpp
@@ -9,6 +9,7 @@
 #include "HDF5DataFilePluginXFEL.h"
 #include "HDF5DataFilePluginJUNGFRAU.h"
 #include "HDF5DataFilePluginResEstimation.h"
+#include "../include/spdlog/fmt/fmt.h"

 HDF5DataFile::HDF5DataFile(const std::string &in_filename,
                           const std::vector<float>& in_rad_int_bin_to_q,
@@ -21,6 +22,8 @@ HDF5DataFile::HDF5DataFile(const std::string &in_filename,
    filename = in_filename;
    image_low = in_image_low;

+    tmp_filename = fmt::format("{}.tmp{:8x}", filename, std::chrono::system_clock::now().time_since_epoch().count());
+
    plugins.emplace_back(std::make_unique<HDF5DataFilePluginJUNGFRAU>());
    plugins.emplace_back(std::make_unique<HDF5DataFilePluginAzInt>(in_rad_int_bin_to_q));
    plugins.emplace_back(std::make_unique<HDF5DataFilePluginXFEL>());
@@ -46,8 +49,7 @@ HDF5DataFile::~HDF5DataFile() {
            data_set.reset();
        }
        data_file.reset();
-        std::string old_filename = filename + ".tmp";
-        std::rename(old_filename.c_str(), filename.c_str());
+        std::rename(tmp_filename.c_str(), filename.c_str());
    }
 }

@@ -67,8 +69,8 @@ void HDF5DataFile::CreateFile(const DataMessage& msg) {
            dcpl.SetFillValue32(INT32_MIN);
    }

-    data_file = std::make_unique<HDF5File>(filename + ".tmp");
-    chmod(filename.c_str(), S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); // default permissions
+    data_file = std::make_unique<HDF5File>(tmp_filename);
+    chmod(tmp_filename.c_str(), S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); // default permissions

    HDF5Group(*data_file, "/entry").NXClass("NXentry");
    HDF5Group(*data_file, "/entry/data").NXClass("NXdata");
@@ -80,7 +82,6 @@ void HDF5DataFile::CreateFile(const DataMessage& msg) {
 }

 void HDF5DataFile::Write(const DataMessage &msg, uint64_t image_number) {
-    std::lock_guard<std::mutex> lock(hdf5_mutex);
    bool new_file = false;

    if (!data_file) {
@@ -100,11 +101,11 @@ void HDF5DataFile::Write(const DataMessage &msg, uint64_t image_number) {
    data_set->WriteDirectChunk(msg.image.data, msg.image.size, {image_number, 0, 0});

    for (auto &p: plugins)
-        p->Write(msg);
+        p->Write(msg, image_number);

    timestamp[image_number] = msg.timestamp;
    exptime[image_number] = msg.exptime;
-    number[image_number] = msg.number;
+    number[image_number] = (msg.original_number) ? msg.original_number.value() : msg.number;
 }

 HDF5DataFileStatistics HDF5DataFile::GetStatistics() const {
--- a/writer/HDF5DataFile.h
+++ b/writer/HDF5DataFile.h
@@ -22,6 +22,7 @@ struct HDF5DataFileStatistics {

 class HDF5DataFile {
    std::string filename;
+    std::string tmp_filename;

    std::unique_ptr<HDF5File> data_file = nullptr;
    std::unique_ptr<HDF5DataSet> data_set = nullptr;
--- a/writer/HDF5DataFilePlugin.h
+++ b/writer/HDF5DataFilePlugin.h
@@ -11,7 +11,7 @@ protected:
    size_t max_image_number = 0;
 public:
    virtual void OpenFile(HDF5File &data_file, const DataMessage& msg) = 0;
-    virtual void Write(const DataMessage& msg) = 0;
+    virtual void Write(const DataMessage& msg, uint64_t image_number) = 0;
    virtual void WriteFinal(HDF5File &data_file) = 0;
    virtual ~HDF5DataFilePlugin() = default;
 };
--- a/writer/HDF5DataFilePluginAzInt.cpp
+++ b/writer/HDF5DataFilePluginAzInt.cpp
@@ -2,38 +2,37 @@

 #include "HDF5DataFilePluginAzInt.h"

+#define RESERVE_IMAGES 1000
+
 HDF5DataFilePluginAzInt::HDF5DataFilePluginAzInt(const std::vector<float> &in_rad_int_bin_to_q) :
-rad_int_bin_to_q(in_rad_int_bin_to_q) {}
+az_int_bin_to_q(in_rad_int_bin_to_q) {}

 void HDF5DataFilePluginAzInt::OpenFile(HDF5File &data_file, const DataMessage &msg) {
-    if (rad_int_bin_to_q.empty())
+    if (az_int_bin_to_q.empty())
        return;

    HDF5Group(data_file, "/entry/azint").NXClass("NXcollection");
+    data_file.SaveVector("/entry/azint/binToQ", az_int_bin_to_q);

-    data_file.SaveVector("/entry/azint/binToQ", rad_int_bin_to_q);
-
-    HDF5DataSpace data_space_rad_int({1, rad_int_bin_to_q.size()}, {H5S_UNLIMITED, rad_int_bin_to_q.size()});
-    HDF5Dcpl dcpl_rad_int;
-    dcpl_rad_int.SetChunking({1, rad_int_bin_to_q.size()});
-
-    data_set_az_int = std::make_unique<HDF5DataSet>(data_file, "/entry/azint/result", HDF5DataType(0.0f),
-                                                    data_space_rad_int, dcpl_rad_int);
+    az_int.reserve(RESERVE_IMAGES * az_int_bin_to_q.size());
 }

-void HDF5DataFilePluginAzInt::Write(const DataMessage &msg) {
-    if (rad_int_bin_to_q.empty())
+void HDF5DataFilePluginAzInt::Write(const DataMessage &msg, uint64_t image_number) {
+    if (az_int_bin_to_q.empty())
        return;

-    size_t image_number = msg.number;
-
    if (image_number >= max_image_number) {
        max_image_number = image_number;
-        data_set_az_int->SetExtent({max_image_number + 1, rad_int_bin_to_q.size()});
+        az_int.resize((max_image_number + 1) * az_int_bin_to_q.size());
    }

-    if (!msg.az_int_profile.empty() && (msg.az_int_profile.size() == rad_int_bin_to_q.size()))
-        data_set_az_int->WriteVec(msg.az_int_profile, {image_number, 0}, {1, rad_int_bin_to_q.size()});
+    if (!msg.az_int_profile.empty() && (msg.az_int_profile.size() == az_int_bin_to_q.size())) {
+        for (int i = 0; i < az_int_bin_to_q.size(); i++)
+            az_int[image_number * az_int_bin_to_q.size() + i] = msg.az_int_profile[i];
+    }
 }

-void HDF5DataFilePluginAzInt::WriteFinal(HDF5File &data_file) {}
+void HDF5DataFilePluginAzInt::WriteFinal(HDF5File &data_file) {
+    if (!az_int.empty())
+        data_file.SaveVector("/entry/azint/result", az_int, {(hsize_t) (max_image_number + 1), az_int_bin_to_q.size()});
+}
--- a/writer/HDF5DataFilePluginAzInt.h
+++ b/writer/HDF5DataFilePluginAzInt.h
@@ -6,13 +6,12 @@
 #include "HDF5DataFilePlugin.h"

 class HDF5DataFilePluginAzInt : public HDF5DataFilePlugin {
-    std::unique_ptr<HDF5DataSet> data_set_az_int = nullptr;
-    std::vector<float> rad_int_bin_to_q;
-
+    std::vector<float> az_int_bin_to_q;
+    std::vector<float> az_int;
 public:
    explicit HDF5DataFilePluginAzInt(const std::vector<float> &rad_int_bin_to_q);
    void OpenFile(HDF5File &data_file, const DataMessage& msg) override;
-    void Write(const DataMessage& msg) override;
+    void Write(const DataMessage& msg, uint64_t image_number) override;
    void WriteFinal(HDF5File &data_file) override;
 };

--- a/writer/HDF5DataFilePluginJUNGFRAU.cpp
+++ b/writer/HDF5DataFilePluginJUNGFRAU.cpp
@@ -4,14 +4,12 @@

 void HDF5DataFilePluginJUNGFRAU::OpenFile(HDF5File &in_data_file, const DataMessage &msg) {}

-void HDF5DataFilePluginJUNGFRAU::Write(const DataMessage &msg) {
+void HDF5DataFilePluginJUNGFRAU::Write(const DataMessage &msg, uint64_t image_number) {
    if (!msg.jf_info.has_value()
        || !msg.receiver_aq_dev_delay.has_value()
        || !msg.storage_cell.has_value())
        return;

-    size_t image_number = msg.number;
-
    if (image_number >= jf_info.size()) {
        jf_info.resize(image_number + 1);
        storage_cell.resize(image_number + 1);
--- a/writer/HDF5DataFilePluginJUNGFRAU.h
+++ b/writer/HDF5DataFilePluginJUNGFRAU.h
@@ -11,7 +11,7 @@ class HDF5DataFilePluginJUNGFRAU : public HDF5DataFilePlugin {
    std::vector<uint64_t> receiver_aq_dev_delay;
 public:
    void OpenFile(HDF5File &data_file, const DataMessage& msg) override;
-    void Write(const DataMessage& msg) override;
+    void Write(const DataMessage& msg, uint64_t image_number) override;
    void WriteFinal(HDF5File &data_file) override;
 };

--- a/writer/HDF5DataFilePluginMX.cpp
+++ b/writer/HDF5DataFilePluginMX.cpp
@@ -1,6 +1,7 @@
 // Copyright (2019-2024) Paul Scherrer Institute

 #include "HDF5DataFilePluginMX.h"
+#define RESERVE_IMAGES 1000

 HDF5DataFilePluginMX::HDF5DataFilePluginMX(size_t in_max_spots) : max_spots(in_max_spots) {}

@@ -8,55 +9,47 @@ void HDF5DataFilePluginMX::OpenFile(HDF5File &data_file, const DataMessage &msg)
    if (max_spots == 0)
        return;

-    HDF5Group(data_file, "/entry/MX").NXClass("NXcollection");
+    spot_x.reserve(max_spots * RESERVE_IMAGES);
+    spot_y.reserve(max_spots * RESERVE_IMAGES);
+    spot_int.reserve(max_spots * RESERVE_IMAGES);
+    npeaks.reserve(RESERVE_IMAGES);
+    strong_pixel_count.reserve(RESERVE_IMAGES);
+    indexed.reserve(RESERVE_IMAGES);
+    indexed_lattice.reserve(9 * RESERVE_IMAGES);

-    HDF5DataSpace data_space_spots({1, max_spots}, {H5S_UNLIMITED, max_spots});
-    HDF5Dcpl dcpl_spots;
-    dcpl_spots.SetChunking({1, max_spots});
-    data_set_spot_x = std::make_unique<HDF5DataSet>(data_file, "/entry/MX/peakXPosRaw", HDF5DataType(0.0f),
-                                                    data_space_spots, dcpl_spots);
-    data_set_spot_y = std::make_unique<HDF5DataSet>(data_file, "/entry/MX/peakYPosRaw", HDF5DataType(0.0f),
-                                                    data_space_spots, dcpl_spots);
-    data_set_spot_int = std::make_unique<HDF5DataSet>(data_file, "/entry/MX/peakTotalIntensity", HDF5DataType(0.0f),
-                                                      data_space_spots, dcpl_spots);
-    data_set_spot_indexed = std::make_unique<HDF5DataSet>(data_file, "/entry/MX/peakIndexed", HDF5DataType(1, false),
-                                                          data_space_spots, dcpl_spots);
+    indexing_drift_beam_center_x_pxl.reserve(RESERVE_IMAGES);
+    indexing_drift_beam_center_y_pxl.reserve(RESERVE_IMAGES);
+    indexing_drift_detector_distance_mm.reserve(RESERVE_IMAGES);
 }

-void HDF5DataFilePluginMX::Write(const DataMessage &msg) {
+void HDF5DataFilePluginMX::Write(const DataMessage &msg, uint64_t image_number) {
    if (max_spots == 0)
        return;

-    size_t image_number = msg.number;
-
    if (image_number >= max_image_number) {
        max_image_number = image_number;
-        data_set_spot_x->SetExtent({max_image_number+1, max_spots});
-        data_set_spot_y->SetExtent({max_image_number+1, max_spots});
-        data_set_spot_int->SetExtent({max_image_number+1, max_spots});
-        data_set_spot_indexed->SetExtent({max_image_number+1, max_spots});
+
+        spot_x.resize(max_spots * (max_image_number + 1));
+        spot_y.resize(max_spots * (max_image_number + 1));
+        spot_int.resize(max_spots * (max_image_number + 1));

        npeaks.resize(max_image_number + 1);
        strong_pixel_count.resize(max_image_number + 1);
        indexed.resize(max_image_number + 1);
        indexed_lattice.resize((max_image_number + 1) * 9);
-    }

-    std::vector<float> spot_x(max_spots), spot_y(max_spots), spot_intensity(max_spots);
-    std::vector<uint8_t> spot_indexed(max_spots);
+        indexing_drift_beam_center_x_pxl.resize(max_image_number + 1);
+        indexing_drift_beam_center_y_pxl.resize(max_image_number + 1);
+        indexing_drift_detector_distance_mm.resize(max_image_number + 1);
+    }

    uint32_t spot_cnt = std::min(msg.spots.size(), max_spots);

    for (int i = 0; i < spot_cnt; i++) {
-        spot_x[i] = msg.spots[i].x;
-        spot_y[i] =  msg.spots[i].y;
-        spot_intensity[i] = msg.spots[i].intensity;
-        spot_indexed[i] = msg.spots[i].indexed;
+        spot_x[max_spots * image_number + i] = msg.spots[i].x;
+        spot_y[max_spots * image_number + i] =  msg.spots[i].y;
+        spot_int[max_spots * image_number + i] = msg.spots[i].intensity;
    }
-    data_set_spot_x->WriteVec(spot_x, {image_number, 0}, {1, max_spots});
-    data_set_spot_y->WriteVec(spot_y, {image_number, 0}, {1, max_spots});
-    data_set_spot_int->WriteVec(spot_intensity, {image_number, 0}, {1, max_spots});
-    data_set_spot_indexed->WriteVec(spot_indexed, {image_number, 0}, {1, max_spots});

    npeaks[image_number] = spot_cnt;
    strong_pixel_count[image_number] = msg.strong_pixel_count;
@@ -65,15 +58,33 @@ void HDF5DataFilePluginMX::Write(const DataMessage &msg) {
        for (int i = 0; i < 9; i++)
            indexed_lattice[image_number * 9 + i] = msg.indexing_lattice[i];
    }
+    if (msg.corr_beam_x_pxl) {
+        indexing_drift_beam_center_x_pxl[image_number] = msg.corr_beam_x_pxl.value();
+        indexing_drift_beam_center_y_pxl[image_number] = msg.corr_beam_y_pxl.value();
+        indexing_drift_detector_distance_mm[image_number] = msg.corr_det_dist_mm.value();
+    }
 }

 void HDF5DataFilePluginMX::WriteFinal(HDF5File &data_file) {
-    if (!npeaks.empty())
+    HDF5Group(data_file, "/entry/MX").NXClass("NXcollection");
+
+    if (!spot_x.empty()) {
+        data_file.SaveVector("/entry/MX/peakXPosRaw", spot_x, {(hsize_t) (max_image_number + 1), max_spots});
+        data_file.SaveVector("/entry/MX/peakYPosRaw", spot_y, {(hsize_t) (max_image_number + 1), max_spots});
+        data_file.SaveVector("/entry/MX/peakTotalIntensity", spot_int, {(hsize_t) (max_image_number + 1), max_spots});
+
        data_file.SaveVector("/entry/MX/nPeaks", npeaks);
-    if (!strong_pixel_count.empty())
        data_file.SaveVector("/entry/MX/strongPixels", strong_pixel_count);
+    }
+
    if (!indexed.empty())
        data_file.SaveVector("/entry/MX/imageIndexed", indexed);
    if (!indexed_lattice.empty())
        data_file.SaveVector("/entry/MX/latticeIndexed", indexed_lattice, {(hsize_t) (max_image_number + 1), 9});
+
+    if (!indexing_drift_beam_center_x_pxl.empty()) {
+        data_file.SaveVector("/entry/MX/indexingDriftBeamCenterX", indexing_drift_beam_center_x_pxl)->Units("pxl");
+        data_file.SaveVector("/entry/MX/indexingDriftBeamCenterY", indexing_drift_beam_center_y_pxl)->Units("pxl");
+        data_file.SaveVector("/entry/MX/indexingDriftDetectorDistance", indexing_drift_detector_distance_mm)->Units("mm");
+    }
 }
--- a/writer/HDF5DataFilePluginMX.h
+++ b/writer/HDF5DataFilePluginMX.h
@@ -9,20 +9,23 @@ class HDF5DataFilePluginMX : public HDF5DataFilePlugin {
    size_t max_spots;

    // spots
-    std::unique_ptr<HDF5DataSet> data_set_spot_x = nullptr;
-    std::unique_ptr<HDF5DataSet> data_set_spot_y = nullptr;
-    std::unique_ptr<HDF5DataSet> data_set_spot_int = nullptr;
-    std::unique_ptr<HDF5DataSet> data_set_spot_indexed = nullptr;
+    std::vector<float> spot_x;
+    std::vector<float> spot_y;
+    std::vector<float> spot_int;
+
    std::vector<uint32_t> npeaks;
    std::vector<uint32_t> strong_pixel_count;

    // indexing
    std::vector<uint8_t> indexed;
    std::vector<float> indexed_lattice;
+    std::vector<float> indexing_drift_beam_center_x_pxl;
+    std::vector<float> indexing_drift_beam_center_y_pxl;
+    std::vector<float> indexing_drift_detector_distance_mm;
 public:
    explicit HDF5DataFilePluginMX(size_t max_spots);
    void OpenFile(HDF5File &data_file, const DataMessage& msg) override;
-    void Write(const DataMessage& msg) override;
+    void Write(const DataMessage& msg, uint64_t image_number) override;
    void WriteFinal(HDF5File &data_file) override;
 };

--- a/writer/HDF5DataFilePluginResEstimation.cpp
+++ b/writer/HDF5DataFilePluginResEstimation.cpp
@@ -4,14 +4,14 @@

 void HDF5DataFilePluginResEstimation::OpenFile(HDF5File &data_file, const DataMessage &msg) {}

-void HDF5DataFilePluginResEstimation::Write(const DataMessage &msg) {
+void HDF5DataFilePluginResEstimation::Write(const DataMessage &msg, uint64_t image_number) {
    if (!msg.resolution_estimation)
        return;

-    if (msg.number >= res_estimation.size())
-        res_estimation.resize(msg.number + 1);
+    if (image_number >= res_estimation.size())
+        res_estimation.resize(image_number + 1);

-    res_estimation[msg.number] = msg.resolution_estimation.value();
+    res_estimation[image_number] = msg.resolution_estimation.value();
 }

 void HDF5DataFilePluginResEstimation::WriteFinal(HDF5File &data_file) {
--- a/writer/HDF5DataFilePluginResEstimation.h
+++ b/writer/HDF5DataFilePluginResEstimation.h
@@ -9,7 +9,7 @@ class HDF5DataFilePluginResEstimation : public HDF5DataFilePlugin {
    std::vector<float> res_estimation;
 public:
    void OpenFile(HDF5File &data_file, const DataMessage &msg) override;
-    void Write(const DataMessage &msg) override;
+    void Write(const DataMessage &msg, uint64_t image_number) override;
    void WriteFinal(HDF5File &data_file) override;
 };

--- a/writer/HDF5DataFilePluginXFEL.cpp
+++ b/writer/HDF5DataFilePluginXFEL.cpp
@@ -4,13 +4,11 @@

 void HDF5DataFilePluginXFEL::OpenFile(HDF5File &data_file, const DataMessage &msg) {}

-void HDF5DataFilePluginXFEL::Write(const DataMessage &msg) {
+void HDF5DataFilePluginXFEL::Write(const DataMessage &msg, uint64_t image_number) {
    if (!msg.xfel_pulse_id.has_value()
        || !msg.xfel_event_code.has_value())
        return;

-    size_t image_number = msg.number;
-
    if (image_number >= pulseid.size()) {
        max_image_number = image_number;
        pulseid.resize(image_number + 1);
--- a/writer/HDF5DataFilePluginXFEL.h
+++ b/writer/HDF5DataFilePluginXFEL.h
@@ -10,7 +10,7 @@ class HDF5DataFilePluginXFEL : public HDF5DataFilePlugin {
    std::vector<uint32_t> event_code;
 public:
    void OpenFile(HDF5File &data_file, const DataMessage& msg) override;
-    void Write(const DataMessage& msg) override;
+    void Write(const DataMessage& msg, uint64_t image_number) override;
    void WriteFinal(HDF5File &data_file) override;
 };

--- a/writer/HDF5NXmx.cpp
+++ b/writer/HDF5NXmx.cpp
@@ -2,6 +2,7 @@

 #include <cmath>
 #include <sys/stat.h>
+
 #include "HDF5NXmx.h"

 #include "../common/GitInfo.h"
@@ -13,10 +14,12 @@ NXmx::NXmx(const StartMessage &start)
        : start_message(start),
          filename(start.file_prefix + "_master.h5") {

+    tmp_filename = fmt::format("{}.tmp_{:x}", filename, std::chrono::system_clock::now().time_since_epoch().count());
+
    MakeDirectory(filename);

-    hdf5_file = std::make_unique<HDF5File>(filename + ".tmp");
-    chmod(filename.c_str(), S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); // default permissions
+    hdf5_file = std::make_unique<HDF5File>(tmp_filename);
+    chmod(tmp_filename.c_str(), S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); // default permissions

    hdf5_file->Attr("file_name", filename);

@@ -33,8 +36,7 @@ NXmx::NXmx(const StartMessage &start)
 }

 NXmx::~NXmx() {
-    std::string old_filename = filename + ".tmp";
-    std::rename(old_filename.c_str(), filename.c_str());
+    std::rename(tmp_filename.c_str(), filename.c_str());
 }

 std::string HDF5Metadata::DataFileName(const std::string &prefix, int64_t file_number) {
--- a/writer/HDF5NXmx.h
+++ b/writer/HDF5NXmx.h
@@ -15,6 +15,7 @@ class NXmx {
    std::unique_ptr<HDF5File> hdf5_file;
    const StartMessage start_message;
    const std::string filename;
+    std::string tmp_filename;

    void LinkToData(const StartMessage &start, const EndMessage &end);

--- a/writer/HDF5Writer.cpp
+++ b/writer/HDF5Writer.cpp
@@ -10,6 +10,7 @@ max_spot_count(request.max_spot_count),
 az_int_bin_to_q(request.az_int_bin_to_q) {}

 void HDF5Writer::Write(const DataMessage& message) {
+    std::lock_guard<std::mutex> lock(hdf5_mutex);
    if (message.image.size == 0)
        return;

@@ -41,6 +42,7 @@ void HDF5Writer::Write(const DataMessage& message) {
 }

 std::vector<HDF5DataFileStatistics> HDF5Writer::Finalize() {
+    std::lock_guard<std::mutex> lock(hdf5_mutex);
    for (auto &f: files) {
        if (f) {
            auto tmp = f->GetStatistics();
--- a/writer/StreamWriter.cpp
+++ b/writer/StreamWriter.cpp
@@ -13,9 +13,10 @@ StreamWriter::StreamWriter(ZMQContext &context, Logger &in_logger, const std::st
 }

 void StreamWriter::CollectImages(std::vector<HDF5DataFileStatistics> &v) {
-    bool run = true;
-    while (run && (image_puller.GetFrameType() != CBORStream2Deserializer::Type::START)) {
-        if (image_puller.GetFrameType() == CBORStream2Deserializer::Type::IMAGE)
+    bool run = WaitForImage();
+
+    while (run && !image_puller_output.cbor->start_message) {
+        if (image_puller_output.cbor->msg_type == CBORImageType::IMAGE)
            logger.Warning("Missing meaningful image while waiting for START");
        run = WaitForImage();
    }
@@ -23,58 +24,55 @@ void StreamWriter::CollectImages(std::vector<HDF5DataFileStatistics> &v) {
    if (!run)
        return;

-    StartMessage start_message = image_puller.GetStartMessage();
-    logger.Info("Starting writing for dataset {} of {} images", start_message.file_prefix, start_message.number_of_images);
+    logger.Info("Starting writing for dataset {} of {} images",
+                image_puller_output.cbor->start_message->file_prefix,
+                image_puller_output.cbor->start_message->number_of_images);
    state = StreamWriterState::Started;
    uint64_t max_image_number = 0;

    processed_images = 0;
    processed_image_size = 0;
-    file_prefix = start_message.file_prefix;
+    file_prefix = image_puller_output.cbor->start_message->file_prefix;

-    CheckPath(start_message.file_prefix);
-    MakeDirectory(start_message.file_prefix);
-    HDF5Writer writer(start_message);
+    CheckPath(image_puller_output.cbor->start_message->file_prefix);
+    MakeDirectory(image_puller_output.cbor->start_message->file_prefix);
+    HDF5Writer writer(*image_puller_output.cbor->start_message);
    std::unique_ptr<NXmx> master_file;
-    if (!start_message.write_master_file || start_message.write_master_file.value())
-        master_file = std::make_unique<NXmx>(start_message);
+    if (!image_puller_output.cbor->start_message->write_master_file || image_puller_output.cbor->start_message->write_master_file.value())
+        master_file = std::make_unique<NXmx>(*image_puller_output.cbor->start_message);

    bool first_image = true;
    run = WaitForImage();
-    while (run && (image_puller.GetFrameType() == CBORStream2Deserializer::Type::CALIBRATION)) {
+    while (run && image_puller_output.cbor->calibration) {
        if (master_file)
-            master_file->WriteCalibration(image_puller.GetCalibrationMessage());
+            master_file->WriteCalibration(*image_puller_output.cbor->calibration);
        run = WaitForImage();
    }

-    while (run && (image_puller.GetFrameType() == CBORStream2Deserializer::Type::IMAGE)) {
+    while (run && image_puller_output.cbor->data_message) {
        if (first_image) {
            state = StreamWriterState::Receiving;
            start_time = std::chrono::system_clock::now();
            first_image = false;
        }
-        auto image_array = image_puller.GetDataMessage();
-        writer.Write(image_array);
-        if (max_image_number < image_array.number + 1)
-            max_image_number = image_array.number + 1;

-        if (start_message.pixel_bit_depth == 0)
-            start_message.pixel_bit_depth = image_array.image.pixel_depth_bytes * 8;
+        writer.Write(*image_puller_output.cbor->data_message);
+        if (max_image_number < image_puller_output.cbor->data_message->number + 1)
+            max_image_number = image_puller_output.cbor->data_message->number + 1;

        processed_images++;
-        processed_image_size += image_array.image.size;
+        processed_image_size += image_puller_output.cbor->data_message->image.size;
        run = WaitForImage();
    }

-    if (image_puller.GetFrameType() == CBORStream2Deserializer::Type::END) {
-        EndMessage end_message = image_puller.GetEndMessage();
+    if (run && image_puller_output.cbor->end_message) {
        end_time = std::chrono::system_clock::now();

-        if ((end_message.max_image_number == 0) && (max_image_number > 0))
-            end_message.max_image_number = max_image_number;
+        if ((image_puller_output.cbor->end_message->max_image_number == 0) && (max_image_number > 0))
+            image_puller_output.cbor->end_message->max_image_number = max_image_number;

        if (master_file)
-            master_file->Finalize(end_message);
+            master_file->Finalize(*image_puller_output.cbor->end_message);
        master_file.reset();
        state = StreamWriterState::Idle;
    }
@@ -105,7 +103,8 @@ StreamWriterOutput StreamWriter::Run() {

 bool StreamWriter::WaitForImage() {
    try {
-        return image_puller.WaitForImage();
+        image_puller_output = image_puller.WaitForImage();
+        return (image_puller_output.cbor != nullptr);
    } catch (const JFJochException &e) {
        logger.ErrorException(e);
        return false;
--- a/writer/StreamWriter.h
+++ b/writer/StreamWriter.h
@@ -26,6 +26,8 @@ struct StreamWriterOutput {
 class StreamWriter {
    StreamWriterState state = StreamWriterState::Idle;

+    ZMQImagePullerOutput image_puller_output;
+
    std::atomic<uint64_t> processed_images;
    std::atomic<uint64_t> processed_image_size;
    std::chrono::time_point<std::chrono::system_clock> start_time;
--- a/writer/ZMQImagePuller.cpp
+++ b/writer/ZMQImagePuller.cpp
@@ -1,6 +1,5 @@
 // Copyright (2019-2023) Paul Scherrer Institute

-#include <cmath>
 #include "ZMQImagePuller.h"

 ZMQImagePuller::ZMQImagePuller(ZMQContext &context, const std::string &repub_address) :
@@ -16,93 +15,103 @@ ZMQImagePuller::ZMQImagePuller(ZMQContext &context, const std::string &repub_add
    }
 }

+ZMQImagePuller::~ZMQImagePuller() {
+    Disconnect();
+}
+
 void ZMQImagePuller::Connect(const std::string &in_address) {
    Disconnect();

+    disconnect = 0;
    abort = 0;
    addr = in_address;
    socket.Connect(in_address);
+
+    puller_thread = std::thread(&ZMQImagePuller::PullerThread, this);
+    cbor_thread = std::thread(&ZMQImagePuller::CBORThread, this);
+    if (repub_socket)
+        repub_thread = std::thread(&ZMQImagePuller::RepubThread, this);
 }

 void ZMQImagePuller::Disconnect() {
-    if (!addr.empty())
+    disconnect = 1;
+
+    if (puller_thread.joinable())
+        puller_thread.join();
+    if (cbor_thread.joinable())
+        cbor_thread.join();
+    if (repub_thread.joinable())
+        repub_thread.join();
+
+    if (!addr.empty()) {
+
        socket.Disconnect(addr);
+    }
    addr = "";
+
 }

 void ZMQImagePuller::Abort() {
    abort = 1;
 }

-bool ZMQImagePuller::WaitForImage() {
-    bool received;
-
-    do
-        received = socket.Receive(msg, true);
-    while (!received && !abort);
-
-    if (received) {
-        deserializer.Process(msg.data(), msg.size());
-        switch (deserializer.GetType()) {
-            case CBORStream2Deserializer::Type::START:
-                start_message = std::make_unique<StartMessage>(deserializer.GetStartMessage());
-                end_message.reset();
-                break;
-            case CBORStream2Deserializer::Type::END:
-                end_message = std::make_unique<EndMessage>(deserializer.GetEndMessage());
-                break;
-            case CBORStream2Deserializer::Type::IMAGE:
-                deserialized_image_message = std::make_unique<DataMessage>(deserializer.GetDataMessage());
-                break;
-            case CBORStream2Deserializer::Type::CALIBRATION:
-                calibration_message = std::make_unique<CompressedImage>(deserializer.GetCalibrationImage());
-                break;
-            case CBORStream2Deserializer::Type::NONE:
-                break;
+void ZMQImagePuller::PullerThread() {
+    while (true) {
+        ZMQImagePullerOutput ret;
+        ret.msg = std::make_shared<ZMQMessage>();
+        bool received = false;
+        while (!received) {
+            if (disconnect) {
+                cbor_fifo.Put(ZMQImagePullerOutput{});
+                return;
+            }
+            try {
+                received = socket.Receive(*ret.msg, true);
+            } catch (const JFJochException &e) {
+                logger.ErrorException(e);
+            }
        }
+        cbor_fifo.Put(ret);
+    }
+}

-        if (repub_socket) {
-            // Republishing is non-blocking for images
-            // and blocking (with 100ms timeout) for START/END
-            repub_socket->Send(msg.data(), msg.size(), deserializer.GetType() != CBORStream2Deserializer::Type::IMAGE);
+void ZMQImagePuller::CBORThread() {
+    auto ret = cbor_fifo.GetBlocking();
+
+    while (ret.msg) {
+        try {
+            ret.cbor = CBORStream2Deserialize(ret.msg->data(), ret.msg->size());
+            outside_fifo.Put(ret);
+            if (repub_socket)
+                repub_fifo.Put(ret);
+        } catch (const JFJochException &e) {
+            logger.ErrorException(e);
        }
-
-        return true;
-    } else
-        return false; // This is all kinds of error
+        ret = cbor_fifo.GetBlocking();
+    }
+    if (repub_socket)
+        repub_fifo.Put(ret);
+    outside_fifo.Put(ret);
 }

-const DataMessage &ZMQImagePuller::GetDataMessage() const {
-    if (deserialized_image_message)
-        return *deserialized_image_message;
-    else
-        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Image message not received so far");
+void ZMQImagePuller::RepubThread() {
+    auto ret = repub_fifo.GetBlocking();
+
+    while (ret.msg) {
+        // Republishing is non-blocking for images
+        // and blocking (with 100ms timeout) for START/END
+        try {
+            repub_socket->Send(ret.msg->data(), ret.msg->size(), ret.cbor->msg_type != CBORImageType::IMAGE);
+        } catch (const JFJochException &e) {
+            logger.ErrorException(e);
+        }
+        ret = repub_fifo.GetBlocking();
+    }
 }

-CBORStream2Deserializer::Type ZMQImagePuller::GetFrameType() const {
-    if (abort)
-        return CBORStream2Deserializer::Type::NONE;
-    else
-        return deserializer.GetType();
-}
-
-StartMessage ZMQImagePuller::GetStartMessage() const {
-    if (start_message)
-        return *start_message;
-    else
-        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Start message not received so far");
-}
-
-EndMessage ZMQImagePuller::GetEndMessage() const {
-    if (end_message)
-        return *end_message;
-    else
-        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Start message not received so far");
-}
-
-CompressedImage ZMQImagePuller::GetCalibrationMessage() const {
-    if (calibration_message)
-        return *calibration_message;
-    else
-        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Calibration message not received so far");
+ZMQImagePullerOutput ZMQImagePuller::WaitForImage() {
+    ZMQImagePullerOutput ret{};
+    while (!outside_fifo.Get(ret) && !abort)
+        std::this_thread::sleep_for(std::chrono::microseconds(100));
+    return ret;
 }
--- a/writer/ZMQImagePuller.h
+++ b/writer/ZMQImagePuller.h
@@ -10,38 +10,45 @@
 #include "../common/SpotToSave.h"
 #include "../frame_serialize/CBORStream2Deserializer.h"

+struct ZMQImagePullerOutput {
+    std::shared_ptr<ZMQMessage> msg;
+    std::shared_ptr<CBORStream2DeserializerOutput> cbor;
+};
+
 class ZMQImagePuller {

-    CBORStream2Deserializer deserializer;
-
-    constexpr const static uint32_t ReceiverWaterMark = 500;
+    constexpr const static uint32_t ReceiverWaterMark = 100;
    // ZeroMQ receive timeout allows to check for abort value from time to time
    constexpr const static auto ReceiveTimeout = std::chrono::milliseconds(100);

    ZMQSocket socket;
    std::string addr;
-    volatile int abort = 0;

-    std::unique_ptr<StartMessage> start_message;
-    std::unique_ptr<EndMessage> end_message;
-    std::unique_ptr<DataMessage> deserialized_image_message;
-    std::unique_ptr<CompressedImage> calibration_message;
+    volatile int abort = 0;
+    volatile int disconnect = 0;

    std::unique_ptr<ZMQSocket> repub_socket;

-    ZMQMessage msg;
+    ThreadSafeFIFO<ZMQImagePullerOutput> cbor_fifo{200};
+    ThreadSafeFIFO<ZMQImagePullerOutput> repub_fifo{200};
+    ThreadSafeFIFO<ZMQImagePullerOutput> outside_fifo{200};
+
+    std::thread puller_thread;
+    std::thread cbor_thread;
+    std::thread repub_thread;
+
+    void PullerThread();
+    void CBORThread();
+    void RepubThread();
+    Logger logger{"ZMQImagePuller"};
 public:
-    ZMQImagePuller(ZMQContext &context, const std::string &repub_address = "");
+    explicit ZMQImagePuller(ZMQContext &context, const std::string &repub_address = "");
+    ~ZMQImagePuller();
    void Connect(const std::string &in_address);
    void Disconnect();
    void Abort();

-    [[nodiscard]] bool WaitForImage();
-    const DataMessage &GetDataMessage() const;
-    [[nodiscard]] CBORStream2Deserializer::Type GetFrameType() const;
-    [[nodiscard]] StartMessage GetStartMessage() const;
-    [[nodiscard]] EndMessage GetEndMessage() const;
-    [[nodiscard]] CompressedImage GetCalibrationMessage() const;
+    [[nodiscard]] ZMQImagePullerOutput WaitForImage();
 };

 #endif //JUNGFRAUJOCH_ZMQIMAGEPULLER_H