v1.0.0-rc.146 (#56)

This is an UNSTABLE release. The release has significant modifications for data processing - in case of troubles go back to 1.0.0-rc.144.

jfjoch_process: Generate a dedicated file (_process.h5), which can be used as a replacement for the _master.h5 file for a reanalyzed dataset.
jfjoch_process: Improve the performance of scaling and merging, implement on the fly scaling.
jfjoch_writer: All final data analysis results are repopulated in the _master.h5 file.
jfjoch_scale: Dedicated tool for rescaling/merging existing data.
jfjoch_viewer: Fix bugs where pixel labels where displayed on a wrong pixel.

WARNING! Scaling and merging are experimental at the moment, and may not provide reasonable results for the time being.

Reviewed-on: #56

writer/HDF5DataFile.cpp

@@ -20,9 +20,10 @@
 #include "HDF5NXmx.h"
 #include "../common/time_utc.h"
 
-HDF5DataFile::HDF5DataFile(const StartMessage &msg, uint64_t in_file_number) {
-    file_number = in_file_number;
-
+HDF5DataFile::HDF5DataFile(const StartMessage &msg, uint64_t file_number, const std::string &filename) :
+filename(filename),
+file_number(file_number),
+write_images(msg.write_images.value_or(true)) {
     if (msg.overwrite.has_value())
         overwrite = msg.overwrite.value();
 
@@ -30,9 +31,14 @@ HDF5DataFile::HDF5DataFile(const StartMessage &msg, uint64_t in_file_number) {
     ypixel = 0;
     max_image_number = 0;
     nimages = 0;
-    filename = HDF5Metadata::DataFileName(msg, file_number);
-    image_low = file_number * msg.images_per_file;
-    images_per_file = msg.images_per_file;
+
+    if (msg.file_format == FileWriterFormat::NXmxIntegrated) {
+        image_low = 0;
+        images_per_file = msg.number_of_images;
+    } else {
+        image_low = file_number * msg.images_per_file;
+        images_per_file = msg.images_per_file;
+    }
 
     timestamp.reserve(images_per_file);
     exptime.reserve(images_per_file);
@@ -110,7 +116,6 @@ HDF5DataFile::~HDF5DataFile() {
     if (data_file) {
         try {
             data_set.reset();
-            data_set_image_number.reset();
             data_file.reset();
             if (manage_file) {
                 std::error_code ec;
@@ -124,42 +129,44 @@ HDF5DataFile::~HDF5DataFile() {
     }
 }
 
-void HDF5DataFile::CreateFile(const DataMessage& msg, std::shared_ptr<HDF5File> in_data_file, bool integrated) {
-    HDF5Dcpl dcpl;
-
-    HDF5DataType data_type(msg.image.GetMode());
-
-    xpixel = msg.image.GetWidth();
-    ypixel = msg.image.GetHeight();
-
-    dcpl.SetCompression(msg.image.GetCompressionAlgorithm(), JFJochBitShuffleCompressor::DefaultBlockSize);
-    dcpl.SetChunking( {1, ypixel, xpixel});
-
-    H5Pset_fill_time(dcpl.GetID(), H5D_FILL_TIME_NEVER);
-    H5Pset_alloc_time(dcpl.GetID(), H5D_ALLOC_TIME_INCR);
-
-    switch (msg.image.GetMode()) {
-        case CompressedImageMode::Int8:
-            dcpl.SetFillValue8(INT8_MIN);
-            break;
-        case CompressedImageMode::Int16:
-            dcpl.SetFillValue16(INT16_MIN);
-            break;
-        case CompressedImageMode::Int32:
-            dcpl.SetFillValue32(INT32_MIN);
-            break;
-        default:
-            break;
-    }
-
+void HDF5DataFile::CreateFile(const DataMessage& msg, std::shared_ptr<HDF5File> in_data_file) {
     data_file = in_data_file;
 
     HDF5Group(*data_file, "/entry").NXClass("NXentry");
-    HDF5Group(*data_file, "/entry/data").NXClass("NXdata");
+    if (write_images) {
+        HDF5Dcpl dcpl;
+
+        HDF5DataType data_type(msg.image.GetMode());
+
+        xpixel = msg.image.GetWidth();
+        ypixel = msg.image.GetHeight();
+
+        dcpl.SetCompression(msg.image.GetCompressionAlgorithm(), JFJochBitShuffleCompressor::DefaultBlockSize);
+        dcpl.SetChunking( {1, ypixel, xpixel});
+
+        H5Pset_fill_time(dcpl.GetID(), H5D_FILL_TIME_NEVER);
+        H5Pset_alloc_time(dcpl.GetID(), H5D_ALLOC_TIME_INCR);
+
+        switch (msg.image.GetMode()) {
+            case CompressedImageMode::Int8:
+                dcpl.SetFillValue8(INT8_MIN);
+                break;
+            case CompressedImageMode::Int16:
+                dcpl.SetFillValue16(INT16_MIN);
+                break;
+            case CompressedImageMode::Int32:
+                dcpl.SetFillValue32(INT32_MIN);
+                break;
+            default:
+                break;
+        }
+
+        HDF5Group(*data_file, "/entry/data").NXClass("NXdata");
+        HDF5DataSpace data_space({1, ypixel, xpixel}, {H5S_UNLIMITED, ypixel, xpixel});
+        data_set = std::make_unique<HDF5DataSet>(*data_file, "/entry/data/data", data_type, data_space, dcpl);
+        data_set->SetExtent({images_per_file, ypixel, xpixel});
+    }
 
-    HDF5DataSpace data_space({1, ypixel, xpixel}, {H5S_UNLIMITED, ypixel, xpixel});
-    data_set = std::make_unique<HDF5DataSet>(*data_file, "/entry/data/data", data_type, data_space, dcpl);
-    data_set->SetExtent({images_per_file, ypixel, xpixel});
     for (auto &p: plugins)
         p->OpenFile(*data_file, msg, images_per_file);
 }
@@ -186,7 +193,8 @@ void HDF5DataFile::Write(const DataMessage &msg, uint64_t image_number) {
     }
 
     nimages++;
-    data_set->WriteDirectChunk(msg.image.GetCompressed(), msg.image.GetCompressedSize(), {image_number, 0, 0});
+    if (data_set)
+        data_set->WriteDirectChunk(msg.image.GetCompressed(), msg.image.GetCompressedSize(), {image_number, 0, 0});
 
     for (auto &p: plugins)
         p->Write(msg, image_number);