v1.0.0-rc.129 (#36)

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.124.

* jfjoch_broker: Significant improvements in TCP image socket, as a viable alternative for ZeroMQ sockets (only a single port on broker side, dynamically change number of writers, acknowledgments for written files)
* jfjoch_broker: Delta phi is calculated also for still data in Bragg prediction
* jfjoch_broker: Image pusher statistics are accessible via the REST interface
* jfjoch_writer: Supports TCP image socket and for these auto-forking option

Reviewed-on: #36
Co-authored-by: Filip Leonarski <filip.leonarski@psi.ch>
Co-committed-by: Filip Leonarski <filip.leonarski@psi.ch>

writer/HDF5DataFile.cpp

@@ -31,6 +31,12 @@ HDF5DataFile::HDF5DataFile(const StartMessage &msg, uint64_t in_file_number) {
     nimages = 0;
     filename = HDF5Metadata::DataFileName(msg, file_number);
     image_low = file_number * msg.images_per_file;
+    images_per_file = msg.images_per_file;
+
+    timestamp.reserve(images_per_file);
+    exptime.reserve(images_per_file);
+    number.reserve(images_per_file);
+
     uint64_t tmp_suffix;
     try {
         if (!msg.arm_date.empty())
@@ -63,6 +69,7 @@ std::optional<HDF5DataFileStatistics> HDF5DataFile::Close() {
         p->WriteFinal(*data_file);
 
     if (data_set) {
+        data_set->SetExtent({max_image_number + 1, ypixel, xpixel});
         data_set
                 ->Attr("image_nr_low", (int32_t) (image_low + 1))
                 .Attr("image_nr_high", (int32_t) (image_low + 1 + max_image_number));
@@ -98,8 +105,6 @@ HDF5DataFile::~HDF5DataFile() {
 void HDF5DataFile::CreateFile(const DataMessage& msg) {
     HDF5Dcpl dcpl;
 
-    bool pixel_signed;
-
     HDF5DataType data_type(msg.image.GetMode());
 
     xpixel = msg.image.GetWidth();
@@ -107,6 +112,10 @@ void HDF5DataFile::CreateFile(const DataMessage& msg) {
 
     dcpl.SetCompression(msg.image.GetCompressionAlgorithm(), JFJochBitShuffleCompressor::DefaultBlockSize);
     dcpl.SetChunking( {1, ypixel, xpixel});
+
+    H5Pset_fill_time(dcpl.GetID(), H5D_FILL_TIME_NEVER);
+    H5Pset_alloc_time(dcpl.GetID(), H5D_ALLOC_TIME_INCR);
+
     switch (msg.image.GetMode()) {
         case CompressedImageMode::Int8:
             dcpl.SetFillValue8(INT8_MIN);
@@ -128,14 +137,18 @@ void HDF5DataFile::CreateFile(const DataMessage& msg) {
 
     HDF5DataSpace data_space({1, ypixel, xpixel}, {H5S_UNLIMITED, ypixel, xpixel});
     data_set = std::make_unique<HDF5DataSet>(*data_file, "/entry/data/data", data_type, data_space, dcpl);
+    data_set->SetExtent({images_per_file, ypixel, xpixel});
     for (auto &p: plugins)
-        p->OpenFile(*data_file, msg);
+        p->OpenFile(*data_file, msg, images_per_file);
 }
 
 void HDF5DataFile::Write(const DataMessage &msg, uint64_t image_number) {
     if (closed)
-        return;
-
+        throw JFJochException(JFJochExceptionCategory::FileWriteError,
+            "Trying to write to already closed file");
+    if (image_number >= images_per_file)
+        throw JFJochException(JFJochExceptionCategory::FileWriteError,
+            "Image number out of bounds");
     bool new_file = false;
 
     if (!data_file) {
@@ -145,7 +158,6 @@ void HDF5DataFile::Write(const DataMessage &msg, uint64_t image_number) {
 
     if (new_file || (static_cast<int64_t>(image_number) > max_image_number)) {
         max_image_number = image_number;
-        data_set->SetExtent({max_image_number+1, ypixel, xpixel});
         timestamp.resize(max_image_number + 1);
         exptime.resize(max_image_number + 1);
         number.resize(max_image_number + 1);