FileWriter: Fix

common/JFJochMessages.h

@@ -251,6 +251,7 @@ struct StartMessage {
     std::optional<float> attenuator_transmission;
 
     std::optional<bool> write_master_file;
+    std::optional<bool> write_images;
 
     nlohmann::json user_data;
 

docs/CBOR.md

@@ -96,6 +96,7 @@ There are minor differences at the moment:
 |                                  |                      | type "azim": qmin, qmax (numbers)                                                                                                 |                           |
 | - gain_file_names                | Array(string)        | Names of JUNGFRAU gain files used for the current detector                                                                        |                           |
 | - write_master_file              | bool                 | With multiple sockets, it selects which socket will provide master file                                                           |                           |
+| - write_images                   | bool                 | Write images in the HDF5 file (if false, will only write metadata)                                                                |                           |
 | - data_reduction_factor_serialmx | uint64               | Data reduction factor for serial MX                                                                                               |                           |
 | - experiment_group               | string               | ID of instrument user, e.g., p-group (SLS/SwissFEL) or proposal number                                                            |                           |
 | - jfjoch_release                 | string               | Jungfraujoch release number                                                                                                       |                           |

frame_serialize/CBORStream2Deserializer.cpp

@@ -1012,6 +1012,8 @@ namespace {
                 ProcessROIConfig(message, j["roi"]);
             if (j.contains("gain_file_names"))
                 message.gain_file_names = j["gain_file_names"];
+            if (j.contains("write_images"))
+                message.write_images = j["write_images"];
             if (j.contains("write_master_file"))
                 message.write_master_file = j["write_master_file"];
             if (j.contains("data_reduction_factor_serialmx"))

frame_serialize/CBORStream2Serializer.cpp

@@ -493,6 +493,8 @@ inline void CBOR_ENC_START_USER_DATA(CborEncoder& encoder, const char* key,
     j["gain_file_names"] = message.gain_file_names;
     if (message.write_master_file)
         j["write_master_file"] = message.write_master_file.value();
+    if (message.write_images)
+        j["write_images"] = message.write_images.value();
     if (message.data_reduction_factor_serialmx)
         j["data_reduction_factor_serialmx"] = message.data_reduction_factor_serialmx.value();
     j["experiment_group"] = message.experiment_group;

tools/jfjoch_process.cpp

@@ -37,21 +37,30 @@ void print_usage(Logger &logger) {
     logger.Info("   -s<num>    Start image number (default: 0)");
     logger.Info("   -e<num>    End image number (default: all)");
     logger.Info("   -v         Verbose output");
-    logger.Info("   -R[num]    Rotation indexing (optional: min angular range deg)");
-    logger.Info("   -F         Use FFT indexing algorithm (shortcut for -XFFT)");
-    logger.Info("   -X<txt>    Indexing algorithm (FFBIDX|FFT|FFTW|Auto|None)");
-    logger.Info("   -x         No least-square beam center refinement");
+    logger.Info("   -W         Include images in the written HDF5 file (otherwise only analysis results are saved)");
+    logger.Info("   -U         Unmerged intensities are written to a text file");
+    logger.Info("");
+
+    logger.Info("  Spot finding");
+    logger.Info("   -T<num>    Noise sigma level for spot finding (default: 3.0)");
+    logger.Info("   -t<num>    Photon count threshold for spot finding (default: 10)");
     logger.Info("   -d<num>    High resolution limit for spot finding (default: 1.5)");
+    logger.Info("   -c<num>    Max spot count (default: 250)");
+    logger.Info("");
+    logger.Info("  Indexing");
+    logger.Info("   -R[num]    Rotation indexing (optional: min angular range deg)");
+    logger.Info("   -X<txt>    Indexing algorithm (FFBIDX|FFT|FFTW|Auto|None)");
+    logger.Info("   -F         Use FFT indexing algorithm (shortcut for -XFFT)");
+    logger.Info("   -S<num>    Space group number - used for both indexing and scaling");
+    logger.Info("   -C<cell>   Fix reference unit cell: -C\"a,b,c,alpha,beta,gamma\" (comma-separated, no spaces; quotes optional)");
+    logger.Info("   -x         No least-square beam center refinement");
+
+    logger.Info("");
+    logger.Info("  Scaling and merging");
     logger.Info("   -D<num>    High resolution limit for scaling/merging (default: 0.0; no limit)");
-    logger.Info("   -S<num>    Space group number");
     logger.Info("   -M         Scale and merge (refine mosaicity) and write scaled.hkl + image.dat");
     logger.Info("   -P<txt>    Partiality refinement fixed|rot|unity (default: fixed)");
     logger.Info("   -A         Anomalous mode (don't merge Friedel pairs)");
-    logger.Info("   -C<cell>   Fix reference unit cell: -C\"a,b,c,alpha,beta,gamma\" (comma-separated, no spaces; quotes optional)");
-    logger.Info("   -c<num>    Max spot count (default: 250)");
-    logger.Info("   -W         HDF5 file with analysis results is written");
-    logger.Info("   -T<num>    Noise sigma level for spot finding (default: 3.0)");
-    logger.Info("   -t<num>    Photon count threshold for spot finding (default: 10)");
 }
 
 void trim_in_place(std::string& t) {
@@ -364,10 +373,12 @@ int main(int argc, char **argv) {
 
     start_message.pixel_mask["default"] = pixel_mask.GetMask(experiment);
     start_message.max_spot_count = experiment.GetMaxSpotCount();
+    start_message.write_images = write_output;
+    start_message.file_format = FileWriterFormat::NXmxIntegrated;
 
     std::unique_ptr<FileWriter> writer;
     try {
-        if (write_output)
+        if (!output_prefix.empty())
             writer = std::make_unique<FileWriter>(start_message);
     } catch (const std::exception &e) {
         logger.Error("Failed to initialize file writer: {}", e.what());
@@ -536,7 +547,6 @@ int main(int argc, char **argv) {
         logger.Info("Rotation Indexing found lattice");
     }
 
-    // --- Optional: run scaling (mosaicity refinement) on accumulated reflections ---
     // --- Optional: run scaling (mosaicity refinement) on accumulated reflections ---
     if (run_scaling) {
         logger.Info("Running scaling (mosaicity refinement) ...");

writer/FileWriter.cpp

@@ -71,26 +71,34 @@ void FileWriter::WriteHDF5(const DataMessage& msg) {
     if (msg.number < 0)
         throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, "No support for negative images");
 
-    const uint64_t file_number = (start_message.images_per_file == 0) ? 0 : msg.number / start_message.images_per_file;
-    const uint64_t image_number = (start_message.images_per_file == 0) ? msg.number : msg.number % start_message.images_per_file;
-
-    if (closed_files.contains(file_number))
-        return;
-
-    if (files.size() <= file_number)
-        files.resize(file_number + 1);
-
-    if (!files[file_number]) {
-        files[file_number] = std::make_unique<HDF5DataFile>(start_message, file_number);
-        if (format == FileWriterFormat::NXmxIntegrated && master_file)
-            files[file_number]->CreateFile(msg, master_file->GetFile());
-    }
-    files[file_number]->Write(msg, image_number);
-
-    if (files[file_number]->GetNumImages() == start_message.images_per_file) {
-        CloseFile(file_number);
+    if (format == FileWriterFormat::NXmxIntegrated && master_file) {
+        if (files.empty() )
+            files.resize(1);
+        if (!files[0]) {
+            files[0] = std::make_unique<HDF5DataFile>(start_message, 0, HDF5Metadata::MasterFileName(start_message));
+            files[0]->CreateFile(msg, master_file->GetFile());
+        }
+        files[0]->Write(msg, msg.number);
     } else {
-        CloseOldFiles(static_cast<uint64_t>(msg.number));
+        const uint64_t file_number = (start_message.images_per_file == 0) ? 0 : msg.number / start_message.images_per_file;
+        const uint64_t image_number = (start_message.images_per_file == 0) ? msg.number : msg.number % start_message.images_per_file;
+
+        if (closed_files.contains(file_number))
+            return;
+
+        if (files.size() <= file_number)
+            files.resize(file_number + 1);
+
+        if (!files[file_number])
+            files[file_number] = std::make_unique<HDF5DataFile>(start_message, file_number,
+                                                                HDF5Metadata::DataFileName(start_message, file_number));
+        files[file_number]->Write(msg, image_number);
+
+        if (files[file_number]->GetNumImages() == start_message.images_per_file) {
+            CloseFile(file_number);
+        } else {
+            CloseOldFiles(static_cast<uint64_t>(msg.number));
+        }
     }
 }
 
@@ -230,6 +238,15 @@ void FileWriter::WriteHDF5(const CompressedImage &msg) {
 void FileWriter::WriteHDF5(const EndMessage &msg) {
     if (master_file) {
         std::lock_guard<std::mutex> lock(hdf5_mutex);
+
+        if (format == FileWriterFormat::NXmxIntegrated) {
+            try {
+                CloseFile(0);
+            } catch (...) {
+                throw;
+            }
+        }
+
         master_file->Finalize(msg);
     }
 }

writer/HDF5DataFile.cpp

@@ -20,9 +20,10 @@
 #include "HDF5NXmx.h"
 #include "../common/time_utc.h"
 
-HDF5DataFile::HDF5DataFile(const StartMessage &msg, uint64_t in_file_number) {
-    file_number = in_file_number;
-
+HDF5DataFile::HDF5DataFile(const StartMessage &msg, uint64_t file_number, const std::string &filename) :
+filename(filename),
+file_number(file_number),
+write_images(msg.write_images.value_or(true)) {
     if (msg.overwrite.has_value())
         overwrite = msg.overwrite.value();
 
@@ -30,9 +31,14 @@ HDF5DataFile::HDF5DataFile(const StartMessage &msg, uint64_t in_file_number) {
     ypixel = 0;
     max_image_number = 0;
     nimages = 0;
-    filename = HDF5Metadata::DataFileName(msg, file_number);
-    image_low = file_number * msg.images_per_file;
-    images_per_file = msg.images_per_file;
+
+    if (msg.file_format == FileWriterFormat::NXmxIntegrated) {
+        image_low = 0;
+        images_per_file = msg.number_of_images;
+    } else {
+        image_low = file_number * msg.images_per_file;
+        images_per_file = msg.images_per_file;
+    }
 
     timestamp.reserve(images_per_file);
     exptime.reserve(images_per_file);
@@ -110,7 +116,6 @@ HDF5DataFile::~HDF5DataFile() {
     if (data_file) {
         try {
             data_set.reset();
-            data_set_image_number.reset();
             data_file.reset();
             if (manage_file) {
                 std::error_code ec;
@@ -124,42 +129,44 @@ HDF5DataFile::~HDF5DataFile() {
     }
 }
 
-void HDF5DataFile::CreateFile(const DataMessage& msg, std::shared_ptr<HDF5File> in_data_file, bool integrated) {
-    HDF5Dcpl dcpl;
-
-    HDF5DataType data_type(msg.image.GetMode());
-
-    xpixel = msg.image.GetWidth();
-    ypixel = msg.image.GetHeight();
-
-    dcpl.SetCompression(msg.image.GetCompressionAlgorithm(), JFJochBitShuffleCompressor::DefaultBlockSize);
-    dcpl.SetChunking( {1, ypixel, xpixel});
-
-    H5Pset_fill_time(dcpl.GetID(), H5D_FILL_TIME_NEVER);
-    H5Pset_alloc_time(dcpl.GetID(), H5D_ALLOC_TIME_INCR);
-
-    switch (msg.image.GetMode()) {
-        case CompressedImageMode::Int8:
-            dcpl.SetFillValue8(INT8_MIN);
-            break;
-        case CompressedImageMode::Int16:
-            dcpl.SetFillValue16(INT16_MIN);
-            break;
-        case CompressedImageMode::Int32:
-            dcpl.SetFillValue32(INT32_MIN);
-            break;
-        default:
-            break;
-    }
-
+void HDF5DataFile::CreateFile(const DataMessage& msg, std::shared_ptr<HDF5File> in_data_file) {
     data_file = in_data_file;
 
     HDF5Group(*data_file, "/entry").NXClass("NXentry");
-    HDF5Group(*data_file, "/entry/data").NXClass("NXdata");
+    if (write_images) {
+        HDF5Dcpl dcpl;
+
+        HDF5DataType data_type(msg.image.GetMode());
+
+        xpixel = msg.image.GetWidth();
+        ypixel = msg.image.GetHeight();
+
+        dcpl.SetCompression(msg.image.GetCompressionAlgorithm(), JFJochBitShuffleCompressor::DefaultBlockSize);
+        dcpl.SetChunking( {1, ypixel, xpixel});
+
+        H5Pset_fill_time(dcpl.GetID(), H5D_FILL_TIME_NEVER);
+        H5Pset_alloc_time(dcpl.GetID(), H5D_ALLOC_TIME_INCR);
+
+        switch (msg.image.GetMode()) {
+            case CompressedImageMode::Int8:
+                dcpl.SetFillValue8(INT8_MIN);
+                break;
+            case CompressedImageMode::Int16:
+                dcpl.SetFillValue16(INT16_MIN);
+                break;
+            case CompressedImageMode::Int32:
+                dcpl.SetFillValue32(INT32_MIN);
+                break;
+            default:
+                break;
+        }
+
+        HDF5Group(*data_file, "/entry/data").NXClass("NXdata");
+        HDF5DataSpace data_space({1, ypixel, xpixel}, {H5S_UNLIMITED, ypixel, xpixel});
+        data_set = std::make_unique<HDF5DataSet>(*data_file, "/entry/data/data", data_type, data_space, dcpl);
+        data_set->SetExtent({images_per_file, ypixel, xpixel});
+    }
 
-    HDF5DataSpace data_space({1, ypixel, xpixel}, {H5S_UNLIMITED, ypixel, xpixel});
-    data_set = std::make_unique<HDF5DataSet>(*data_file, "/entry/data/data", data_type, data_space, dcpl);
-    data_set->SetExtent({images_per_file, ypixel, xpixel});
     for (auto &p: plugins)
         p->OpenFile(*data_file, msg, images_per_file);
 }
@@ -186,7 +193,8 @@ void HDF5DataFile::Write(const DataMessage &msg, uint64_t image_number) {
     }
 
     nimages++;
-    data_set->WriteDirectChunk(msg.image.GetCompressed(), msg.image.GetCompressedSize(), {image_number, 0, 0});
+    if (data_set)
+        data_set->WriteDirectChunk(msg.image.GetCompressed(), msg.image.GetCompressedSize(), {image_number, 0, 0});
 
     for (auto &p: plugins)
         p->Write(msg, image_number);

writer/HDF5DataFile.h

@@ -22,12 +22,14 @@ struct HDF5DataFileStatistics {
 };
 
 class HDF5DataFile {
-    std::string filename;
+    const std::string filename;
+    const uint64_t file_number;
+    const bool write_images;
+
     std::string tmp_filename;
 
     std::shared_ptr<HDF5File> data_file = nullptr;
     std::unique_ptr<HDF5DataSet> data_set = nullptr;
-    std::unique_ptr<HDF5DataSet> data_set_image_number = nullptr;
     std::vector<std::unique_ptr<HDF5DataFilePlugin>> plugins;
     size_t images_per_file;
     size_t xpixel;
@@ -46,17 +48,16 @@ class HDF5DataFile {
     bool closed = false;
 
     bool overwrite = false;
-    int64_t file_number;
     bool new_file = true;
     bool manage_file = false;
 public:
-    HDF5DataFile(const StartMessage &msg, uint64_t file_number);
+    HDF5DataFile(const StartMessage &msg, uint64_t file_number, const std::string &filename);
     ~HDF5DataFile();
     std::optional<HDF5DataFileStatistics> Close();
     void Write(const DataMessage& msg, uint64_t image_number);
     size_t GetNumImages() const;
 
-    void CreateFile(const DataMessage& msg, std::shared_ptr<HDF5File> data_file, bool integrated = false);
+    void CreateFile(const DataMessage& msg, std::shared_ptr<HDF5File> data_file);
 };
 
 #endif //HDF5DATAFILE_H

writer/HDF5NXmx.cpp

@@ -11,15 +11,13 @@
 #include "../common/time_utc.h"
 #include "gemmi/symmetry.hpp"
 
-namespace {
-    std::string GenFilename(const StartMessage &start) {
-        return fmt::format("{:s}_master.h5", start.file_prefix);
-    }
+std::string HDF5Metadata::MasterFileName(const StartMessage &start) {
+    return fmt::format("{:s}_master.h5", start.file_prefix);
 }
 
 NXmx::NXmx(const StartMessage &start)
         : start_message(start),
-          filename(GenFilename(start)) {
+          filename(HDF5Metadata::MasterFileName(start)) {
     uint64_t tmp_suffix;
     try {
         if (!start.arm_date.empty())

writer/HDF5NXmx.h

@@ -9,6 +9,7 @@
 #include "HDF5Objects.h"
 
 namespace HDF5Metadata {
+    std::string MasterFileName(const StartMessage &msg);
     std::string DataFileName(const StartMessage &msg, int64_t file_number);
 }