JFJOchReader: Add ReadAllSpots option to handle reading just spot finding result

reader/CMakeLists.txt

@@ -8,6 +8,7 @@ ADD_LIBRARY(JFJochReader STATIC
         JFJochReaderDataset.h
         JFJochHttpReader.cpp
         JFJochHttpReader.h
+        JFJochReaderSpots.h
 )
 
 TARGET_LINK_LIBRARIES(JFJochReader JFJochImageAnalysis JFJochAPI JFJochCommon JFJochZMQ JFJochLogger

reader/JFJochHDF5Reader.cpp

@@ -1249,9 +1249,12 @@ std::vector<IntegrationOutcome> JFJochHDF5Reader::ReadReflections(size_t start_i
     return ret;
 }
 
-std::vector<SpotToSave> JFJochHDF5Reader::ReadSpots(size_t image) const {
+std::vector<SpotToSave> JFJochHDF5Reader::ReadSpots(int64_t image) const {
     std::unique_lock ul(hdf5_mutex);
 
+    if (image < 0)
+        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
+            "image number must be non-negative");
     if (image >= number_of_images)
         throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
                               "image must be less than number_of_images");

reader/JFJochHDF5Reader.h

@@ -4,6 +4,7 @@
 #pragma once
 
 #include "JFJochReader.h"
+#include "JFJochReaderSpots.h"
 #include "../writer/HDF5Objects.h"
 #include "../image_analysis/IntegrationOutcome.h"
 
@@ -56,7 +57,7 @@ public:
 
     std::vector<IntegrationOutcome> ReadReflections(size_t start_image = 0, std::optional<size_t> end_image = {}) const;
 
-    std::vector<SpotToSave> ReadSpots(size_t image) const;
+    std::vector<SpotToSave> ReadSpots(int64_t image) const override;
 
     CompressedImage ReadCalibration(std::vector<uint8_t> &tmp, const std::string &name) const;
 

reader/JFJochHttpReader.cpp

@@ -374,3 +374,31 @@ void JFJochHttpReader::UploadUserMask(const std::vector<uint32_t>& mask) {
         throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
                               "Server rejected user mask upload");
 }
+
+std::vector<SpotToSave> JFJochHttpReader::ReadSpots(int64_t image) const {
+    std::unique_lock ul(http_mutex);
+
+    if (image < 0)
+        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
+            "Image number must be non-negative");
+
+    if (addr.empty())
+        return {};
+
+    httplib::Client cli_cmd(addr);
+    auto res = cli_cmd.Get("/image_buffer/image.cbor?id=" + std::to_string(image));
+
+    if (!res || res->status != httplib::StatusCode::OK_200 || res->body.empty())
+        return {};
+
+    try {
+        auto msg = CBORStream2Deserialize(res->body);
+
+        if (msg->msg_type != CBORImageType::IMAGE)
+            return {};
+
+        return msg->data_message->spots;
+    } catch (std::exception &e) {
+        return {};
+    }
+}

reader/JFJochHttpReader.h

@@ -36,6 +36,7 @@ public:
     BrokerStatus GetBrokerStatus() const;
 
     std::shared_ptr<JFJochReaderRawImage> GetRawImage(int64_t image_number) override;
+    std::vector<SpotToSave> ReadSpots(int64_t image) const override;
 };
 
 

reader/JFJochReader.cpp

@@ -98,3 +98,30 @@ void JFJochReader::UpdateUserMask(const std::vector<uint32_t> &mask) {
     new_dataset->pixel_mask.LoadUserMask(dataset->experiment, mask);
     dataset = new_dataset;
 }
+
+std::shared_ptr<JFJochReaderSpots> JFJochReader::ReadAllSpots(int64_t start_image, int64_t end_image,
+    int64_t stride) const {
+
+    if (start_image < 0)
+        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
+            "Start image must be non-negative");
+
+    if (start_image > end_image)
+        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
+            "Start image number is greater than end image number");
+
+    if (stride == 0)
+        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
+            "Stride cannot be zero");
+
+    size_t nelems = (end_image - start_image) / stride + 1;
+
+    auto ret = std::make_shared<JFJochReaderSpots>();
+    ret->start_image = static_cast<int64_t>(start_image);
+    ret->stride = static_cast<int64_t>(stride);
+    ret->spots.reserve(nelems);
+
+    for (int i = 0; i < nelems; i++)
+        ret->spots.emplace_back(ReadSpots(start_image + i * stride));
+    return ret;
+}

reader/JFJochReader.h

@@ -14,6 +14,7 @@
 #include "../common/DiffractionExperiment.h"
 #include "JFJochReaderDataset.h"
 #include "JFJochReaderImage.h"
+#include "JFJochReaderSpots.h"
 
 class JFJochReader {
     mutable std::mutex m;
@@ -49,6 +50,9 @@ public:
     void UpdateUserMask(const std::vector<uint32_t>& mask);
 
     virtual std::shared_ptr<JFJochReaderRawImage> GetRawImage(int64_t image_number) = 0;
+
+    virtual std::vector<SpotToSave> ReadSpots(int64_t image) const = 0;
+    std::shared_ptr<JFJochReaderSpots> ReadAllSpots(int64_t start_image, int64_t end_image, int64_t stride = 1) const;
 };
 
 #endif //JFJOCHIMAGEREADER_H

reader/JFJochReaderSpots.h

@@ -0,0 +1,12 @@
+// SPDX-FileCopyrightText: 2025 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
+// SPDX-License-Identifier: GPL-3.0-only
+
+#pragma once
+
+#include "../common/SpotToSave.h"
+
+struct JFJochReaderSpots {
+    int64_t start_image;
+    int64_t stride;
+    std::vector<std::vector<SpotToSave>> spots;
+};

tests/JFJochReaderTest.cpp

@@ -2352,6 +2352,82 @@ TEST_CASE("JFJochReader_ReadSpots_VDS", "[HDF5][Full]") {
     REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
 }
 
+TEST_CASE("JFJochReader_ReadAllSpots_VDS", "[HDF5][Full]") {
+    DiffractionExperiment x(DetJF(1));
+
+    x.FilePrefix("read_spots_vds")
+     .ImagesPerTrigger(20)
+     .ImagesPerFile(3)
+     .OverwriteExistingFiles(true)
+     .BitDepthImage(16)
+     .PixelSigned(true)
+     .SetFileWriterFormat(FileWriterFormat::NXmxVDS)
+     .IndexingAlgorithm(IndexingAlgorithmEnum::FFT)
+     .Compression(CompressionAlgorithm::NO_COMPRESSION);
+
+    std::vector<int16_t> image(x.GetPixelsNum(), 0);
+
+    RegisterHDF5Filter();
+    {
+        StartMessage start_message;
+        x.FillMessage(start_message);
+        FileWriter file_set(start_message);
+
+        for (int i = 0; i < x.GetImageNum(); i++) {
+            DataMessage message{};
+            message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
+            message.number = i;
+            message.spots = MakeTestSpots(i);
+            message.spot_count = 72 + i;
+            message.spot_count_ice_rings = 45 + 2 * i;
+            message.spot_count_low_res = 12 + 3 * i;
+            message.spot_count_indexed = 15 + 4 * i;
+            REQUIRE_NOTHROW(file_set.WriteHDF5(message));
+        }
+
+        EndMessage end_message;
+        end_message.max_image_number = x.GetImageNum();
+        file_set.WriteHDF5(end_message);
+        file_set.Finalize();
+    }
+
+    {
+        JFJochHDF5Reader reader;
+        REQUIRE_NOTHROW(reader.ReadFile("read_spots_vds_master.h5"));
+
+        std::shared_ptr<JFJochReaderSpots> ret;
+        REQUIRE_NOTHROW(ret = reader.ReadAllSpots(1, 15, 2));
+
+        // 1,3,5,7,9,11,13,15
+        REQUIRE(ret);
+        REQUIRE(ret->start_image == 1);
+        REQUIRE(ret->stride == 2);
+        REQUIRE(ret->spots.size() == 8);
+
+        for (int i = 0; i < ret->spots.size(); i++) {
+            REQUIRE(ret->spots[i].size() == 2);
+            CheckSpotFields(ret->spots[i][0], ret->spots[i][1], 2 * i + 1);
+        }
+    }
+
+    {
+        JFJochHDF5Reader reader;
+        REQUIRE_NOTHROW(reader.ReadFile("read_spots_vds_master.h5"));
+        REQUIRE_THROWS(reader.ReadAllSpots(-5,0));
+        REQUIRE_THROWS(reader.ReadAllSpots(5,0));
+    }
+
+    remove("read_spots_vds_master.h5");
+    remove("read_spots_vds_data_000001.h5");
+    remove("read_spots_vds_data_000002.h5");
+    remove("read_spots_vds_data_000003.h5");
+    remove("read_spots_vds_data_000004.h5");
+
+    REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
+}
+
+
+
 TEST_CASE("JFJochReader_ReadSpots_Integrated", "[HDF5][Full]") {
     DiffractionExperiment x(DetJF(1));