Merge pull request #419 from slsdetectorgroup/jsonmaster

Jsonmaster

RELEASE.txt

@@ -57,6 +57,7 @@ This document describes the differences between v7.0.0 and v6.x.x
 - ctb: can set names for all the dacs
 - fpga/kernel programming, checks if drive is a special file and not a normal file
 - gotthard 25 um image reconstructed in gui and virtual hdf5 (firmware updated for slave to reverse channels)
+- master binary file in json format now
 - fixed bug introduced in 6.0.0: hdf5 files created 1 file per frame after the initial file which had maxframesperfile
 
 2. Resolved Issues

slsReceiverSoftware/CMakeLists.txt

@@ -6,7 +6,6 @@ set(SOURCES
     src/Receiver.cpp
     src/File.cpp
     src/BinaryDataFile.cpp
-    src/BinaryMasterFile.cpp
     src/ThreadObject.cpp
     src/Listener.cpp
     src/DataProcessor.cpp
@@ -14,6 +13,7 @@ set(SOURCES
     src/Fifo.cpp
     src/Arping.cpp
     src/MasterAttributes.cpp
+    src/MasterFileUtility.cpp
 )
 
 set(PUBLICHEADERS
@@ -28,8 +28,6 @@ if (SLS_USE_HDF5)
 	    )
 	    list (APPEND SOURCES 
 	        src/HDF5DataFile.cpp 
-	        src/HDF5MasterFile.cpp 
-	        src/HDF5VirtualFile.cpp 
 	    )
 endif (SLS_USE_HDF5)
 

slsReceiverSoftware/src/BinaryMasterFile.cpp

@@ -1,60 +0,0 @@
-// SPDX-License-Identifier: LGPL-3.0-or-other
-// Copyright (C) 2021 Contributors to the SLS Detector Package
-#include "BinaryMasterFile.h"
-#include "MasterAttributes.h"
-
-BinaryMasterFile::BinaryMasterFile() : File(BINARY) {}
-
-BinaryMasterFile::~BinaryMasterFile() { CloseFile(); }
-
-void BinaryMasterFile::CloseFile() {
-    if (fd_) {
-        fclose(fd_);
-    }
-    fd_ = nullptr;
-}
-
-void BinaryMasterFile::CreateMasterFile(const std::string filePath,
-                                        const std::string fileNamePrefix,
-                                        const uint64_t fileIndex,
-                                        const bool overWriteEnable,
-                                        const bool silentMode,
-                                        MasterAttributes *attr) {
-    // create file name
-    std::ostringstream os;
-    os << filePath << "/" << fileNamePrefix << "_master"
-       << "_" << fileIndex << ".raw";
-    fileName_ = os.str();
-
-    // create file
-    if (!overWriteEnable) {
-        if (nullptr == (fd_ = fopen((const char *)fileName_.c_str(), "wx"))) {
-            fd_ = nullptr;
-            throw sls::RuntimeError("Could not create binary master file " +
-                                    fileName_);
-        }
-    } else if (nullptr == (fd_ = fopen((const char *)fileName_.c_str(), "w"))) {
-        fd_ = nullptr;
-        throw sls::RuntimeError(
-            "Could not create/overwrite binary master file " + fileName_);
-    }
-    if (!silentMode) {
-        LOG(logINFO) << "Master File: " << fileName_;
-    }
-    attr->WriteMasterBinaryAttributes(fd_);
-    CloseFile();
-}
-
-void BinaryMasterFile::UpdateMasterFile(MasterAttributes *attr,
-                                        bool silentMode) {
-    if (nullptr == (fd_ = fopen((const char *)fileName_.c_str(), "a"))) {
-        fd_ = nullptr;
-        throw sls::RuntimeError("Could not append binary master file " +
-                                fileName_);
-    }
-    attr->WriteFinalBinaryAttributes(fd_);
-    CloseFile();
-    if (!silentMode) {
-        LOG(logINFO) << "Updated Master File";
-    }
-}

slsReceiverSoftware/src/BinaryMasterFile.h

@@ -1,25 +0,0 @@
-// SPDX-License-Identifier: LGPL-3.0-or-other
-// Copyright (C) 2021 Contributors to the SLS Detector Package
-#pragma once
-
-#include "File.h"
-#include "MasterAttributes.h"
-
-class BinaryMasterFile : private virtual slsDetectorDefs, public File {
-
-  public:
-    BinaryMasterFile();
-    ~BinaryMasterFile();
-
-    void CloseFile() override;
-    void CreateMasterFile(const std::string filePath,
-                          const std::string fileNamePrefix,
-                          const uint64_t fileIndex, const bool overWriteEnable,
-                          const bool silentMode,
-                          MasterAttributes *attr) override;
-    void UpdateMasterFile(MasterAttributes *attr, bool silentMode) override;
-
-  private:
-    FILE *fd_{nullptr};
-    std::string fileName_;
-};

slsReceiverSoftware/src/DataProcessor.cpp

@@ -9,14 +9,12 @@
 
 #include "DataProcessor.h"
 #include "BinaryDataFile.h"
-#include "BinaryMasterFile.h"
 #include "Fifo.h"
 #include "GeneralData.h"
 #include "MasterAttributes.h"
+#include "MasterFileUtility.h"
 #ifdef HDF5C
 #include "HDF5DataFile.h"
-#include "HDF5MasterFile.h"
-#include "HDF5VirtualFile.h"
 #endif
 #include "DataStreamer.h"
 #include "sls/container_utils.h"
@@ -82,12 +80,6 @@ void DataProcessor::SetGeneralData(GeneralData *generalData) {
 void DataProcessor::CloseFiles() {
     if (dataFile_)
         dataFile_->CloseFile();
-    if (masterFile_)
-        masterFile_->CloseFile();
-#ifdef HDF5C
-    if (virtualFile_)
-        virtualFile_->CloseFile();
-#endif
 }
 
 void DataProcessor::DeleteFiles() {
@@ -96,39 +88,20 @@ void DataProcessor::DeleteFiles() {
         delete dataFile_;
         dataFile_ = nullptr;
     }
-    if (masterFile_) {
-        delete masterFile_;
-        masterFile_ = nullptr;
-    }
-#ifdef HDF5C
-    if (virtualFile_) {
-        delete virtualFile_;
-        virtualFile_ = nullptr;
-    }
-#endif
 }
 void DataProcessor::SetupFileWriter(const bool filewriteEnable,
-                                    const bool masterFilewriteEnable,
                                     const fileFormat fileFormatType,
-                                    const int modulePos, std::mutex *hdf5Lib) {
+                                    std::mutex *hdf5LibMutex) {
     DeleteFiles();
     if (filewriteEnable) {
         switch (fileFormatType) {
 #ifdef HDF5C
         case HDF5:
-            dataFile_ = new HDF5DataFile(index, hdf5Lib);
+            dataFile_ = new HDF5DataFile(index, hdf5LibMutex);
-            if (modulePos == 0 && index == 0) {
-                if (masterFilewriteEnable) {
-                    masterFile_ = new HDF5MasterFile(hdf5Lib);
-                }
-            }
             break;
 #endif
         case BINARY:
             dataFile_ = new BinaryDataFile(index);
-            if (modulePos == 0 && index == 0 && masterFilewriteEnable) {
-                masterFile_ = new BinaryMasterFile();
-            }
             break;
         default:
             throw sls::RuntimeError(
@@ -138,23 +111,17 @@ void DataProcessor::SetupFileWriter(const bool filewriteEnable,
 }
 
 void DataProcessor::CreateFirstFiles(
-    MasterAttributes *attr, const std::string filePath,
+    const std::string &filePath, const std::string &fileNamePrefix,
-    const std::string fileNamePrefix, const uint64_t fileIndex,
+    const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode,
-    const bool overWriteEnable, const bool silentMode, const int modulePos,
+    const int modulePos, const int numUnitsPerReadout,
-    const int numUnitsPerReadout, const uint32_t udpPortNumber,
+    const uint32_t udpPortNumber, const uint32_t maxFramesPerFile,
-    const uint32_t maxFramesPerFile, const uint64_t numImages,
+    const uint64_t numImages, const uint32_t dynamicRange,
-    const uint32_t dynamicRange, const bool detectorDataStream) {
+    const bool detectorDataStream) {
     if (dataFile_ == nullptr) {
         throw sls::RuntimeError("file object not contstructed");
     }
     CloseFiles();
 
-    // master file write enabled
-    if (masterFile_) {
-        masterFile_->CreateMasterFile(filePath, fileNamePrefix, fileIndex,
-                                      overWriteEnable, silentMode, attr);
-    }
-
     // deactivated (half module/ single port), dont write file
     if ((!*activated_) || (!detectorDataStream)) {
         return;
@@ -189,21 +156,17 @@ uint32_t DataProcessor::GetFilesInAcquisition() const {
     return dataFile_->GetFilesInAcquisition();
 }
 
-void DataProcessor::CreateVirtualFile(
+std::array<std::string, 2> DataProcessor::CreateVirtualFile(
-    const std::string filePath, const std::string fileNamePrefix,
+    const std::string &filePath, const std::string &fileNamePrefix,
     const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode,
     const int modulePos, const int numUnitsPerReadout,
     const uint32_t maxFramesPerFile, const uint64_t numImages,
-    const uint32_t dynamicRange, const int numModX, const int numModY,
+    const int numModX, const int numModY, const uint32_t dynamicRange,
-    std::mutex *hdf5Lib) {
+    std::mutex *hdf5LibMutex) {
 
-    if (virtualFile_) {
-        delete virtualFile_;
-    }
     bool gotthard25um =
         ((detectorType_ == GOTTHARD || detectorType_ == GOTTHARD2) &&
          (numModX * numModY) == 2);
-    virtualFile_ = new HDF5VirtualFile(hdf5Lib, gotthard25um);
 
     // maxframesperfile = 0 for infinite files
     uint32_t framesPerFile =
@@ -213,63 +176,55 @@ void DataProcessor::CreateVirtualFile(
     // files (they exist anyway) assumption2: virtual file max frame index
     // is from R0 P0 (difference from others when missing frames or for a
     // stop acquisition)
-    virtualFile_->CreateVirtualFile(
+    return masterFileUtility::CreateVirtualHDF5File(
         filePath, fileNamePrefix, fileIndex, overWriteEnable, silentMode,
         modulePos, numUnitsPerReadout, framesPerFile, numImages,
         generalData_->nPixelsX, generalData_->nPixelsY, dynamicRange,
         numFramesCaught_, numModX, numModY, dataFile_->GetPDataType(),
-        dataFile_->GetParameterNames(), dataFile_->GetParameterDataTypes());
+        dataFile_->GetParameterNames(), dataFile_->GetParameterDataTypes(),
+        hdf5LibMutex, gotthard25um);
 }
 
-void DataProcessor::LinkDataInMasterFile(const bool silentMode) {
+void DataProcessor::LinkFileInMaster(const std::string &masterFileName,
-    std::string fname, datasetName;
+                                     const std::string &virtualFileName,
-    if (virtualFile_) {
+                                     const std::string &virtualDatasetName,
-        auto res = virtualFile_->GetFileAndDatasetName();
+                                     const bool silentMode,
-        fname = res[0];
+                                     std::mutex *hdf5LibMutex) {
-        datasetName = res[1];
+    std::string fname{virtualFileName}, datasetName{virtualDatasetName};
-    } else {
+    // if no virtual file, link data file
+    if (virtualFileName.empty()) {
         auto res = dataFile_->GetFileAndDatasetName();
         fname = res[0];
         datasetName = res[1];
     }
-    // link in master
+    masterFileUtility::LinkHDF5FileInMaster(masterFileName, fname, datasetName,
-    masterFile_->LinkDataFile(fname, datasetName,
+                                            dataFile_->GetParameterNames(),
-                              dataFile_->GetParameterNames(), silentMode);
+                                            silentMode, hdf5LibMutex);
 }
 #endif
 
-void DataProcessor::UpdateMasterFile(bool silentMode) {
+std::string DataProcessor::CreateMasterFile(
-    if (masterFile_) {
+    const std::string &filePath, const std::string &fileNamePrefix,
-        // final attributes
+    const uint64_t fileIndex, const bool overWriteEnable, bool silentMode,
-        std::unique_ptr<MasterAttributes> masterAttributes;
+    const fileFormat fileFormatType, MasterAttributes *attr,
-        switch (detectorType_) {
+    std::mutex *hdf5LibMutex) {
-        case GOTTHARD:
+
-            masterAttributes = sls::make_unique<GotthardMasterAttributes>();
+    attr->framesInFile = numFramesCaught_;
-            break;
+
-        case JUNGFRAU:
+    std::unique_ptr<File> masterFile{nullptr};
-            masterAttributes = sls::make_unique<JungfrauMasterAttributes>();
+    switch (fileFormatType) {
-            break;
+#ifdef HDF5C
-        case EIGER:
+    case HDF5:
-            masterAttributes = sls::make_unique<EigerMasterAttributes>();
+        return masterFileUtility::CreateMasterHDF5File(
-            break;
+            filePath, fileNamePrefix, fileIndex, overWriteEnable, silentMode,
-        case MYTHEN3:
+            attr, hdf5LibMutex);
-            masterAttributes = sls::make_unique<Mythen3MasterAttributes>();
+#endif
-            break;
+    case BINARY:
-        case GOTTHARD2:
+        return masterFileUtility::CreateMasterBinaryFile(
-            masterAttributes = sls::make_unique<Gotthard2MasterAttributes>();
+            filePath, fileNamePrefix, fileIndex, overWriteEnable, silentMode,
-            break;
+            attr);
-        case MOENCH:
+    default:
-            masterAttributes = sls::make_unique<MoenchMasterAttributes>();
+        throw sls::RuntimeError("Unknown file format (compile with hdf5 flags");
-            break;
-        case CHIPTESTBOARD:
-            masterAttributes = sls::make_unique<CtbMasterAttributes>();
-            break;
-        default:
-            throw sls::RuntimeError(
-                "Unknown detector type to set up master file attributes");
-        }
-        masterAttributes->framesInFile = numFramesCaught_;
-        masterFile_->UpdateMasterFile(masterAttributes.get(), silentMode);
     }
 }
 

slsReceiverSoftware/src/DataProcessor.h

@@ -45,12 +45,11 @@ class DataProcessor : private virtual slsDetectorDefs, public ThreadObject {
     void CloseFiles();
     void DeleteFiles();
     void SetupFileWriter(const bool filewriteEnable,
-                         const bool masterFilewriteEnable,
+                         const fileFormat fileFormatType,
-                         const fileFormat fileFormatType, const int modulePos,
+                         std::mutex *hdf5LibMutex);
-                         std::mutex *hdf5Lib);
 
-    void CreateFirstFiles(MasterAttributes *attr, const std::string filePath,
+    void CreateFirstFiles(const std::string &filePath,
-                          const std::string fileNamePrefix,
+                          const std::string &fileNamePrefix,
                           const uint64_t fileIndex, const bool overWriteEnable,
                           const bool silentMode, const int modulePos,
                           const int numUnitsPerReadout,
@@ -60,18 +59,26 @@ class DataProcessor : private virtual slsDetectorDefs, public ThreadObject {
                           const bool detectorDataStream);
 #ifdef HDF5C
     uint32_t GetFilesInAcquisition() const;
-    void CreateVirtualFile(const std::string filePath,
+    std::array<std::string, 2> CreateVirtualFile(
-                           const std::string fileNamePrefix,
+        const std::string &filePath, const std::string &fileNamePrefix,
-                           const uint64_t fileIndex, const bool overWriteEnable,
+        const uint64_t fileIndex, const bool overWriteEnable,
-                           const bool silentMode, const int modulePos,
+        const bool silentMode, const int modulePos,
-                           const int numUnitsPerReadout,
+        const int numUnitsPerReadout, const uint32_t maxFramesPerFile,
-                           const uint32_t maxFramesPerFile,
+        const uint64_t numImages, const int numModX, const int numModY,
-                           const uint64_t numImages,
+        const uint32_t dynamicRange, std::mutex *hdf5LibMutex);
-                           const uint32_t dynamicRange, const int numModX,
+    void LinkFileInMaster(const std::string &masterFileName,
-                           const int numModY, std::mutex *hdf5Lib);
+                          const std::string &virtualFileName,
-    void LinkDataInMasterFile(const bool silentMode);
+                          const std::string &virtualDatasetName,
+                          const bool silentMode, std::mutex *hdf5LibMutex);
 #endif
-    void UpdateMasterFile(bool silentMode);
+
+    std::string CreateMasterFile(const std::string &filePath,
+                                 const std::string &fileNamePrefix,
+                                 const uint64_t fileIndex,
+                                 const bool overWriteEnable, bool silentMode,
+                                 const fileFormat fileFormatType,
+                                 MasterAttributes *attr,
+                                 std::mutex *hdf5LibMutex);
     /**
      * Call back for raw data
      * args to raw data ready callback are
@@ -179,10 +186,6 @@ class DataProcessor : private virtual slsDetectorDefs, public ThreadObject {
     bool firstStreamerFrame_{false};
 
     File *dataFile_{nullptr};
-    File *masterFile_{nullptr};
-#ifdef HDF5C
-    File *virtualFile_{nullptr};
-#endif
 
     // call back
     /**

slsReceiverSoftware/src/File.h

@@ -60,20 +60,6 @@ class File : private virtual slsDetectorDefs {
         return std::vector<DataType>{};
     };
 
-    virtual void CreateVirtualFile(
-        const std::string filePath, const std::string fileNamePrefix,
-        const uint64_t fileIndex, const bool overWriteEnable,
-        const bool silentMode, const int modulePos,
-        const int numUnitsPerReadout, const uint32_t maxFramesPerFile,
-        const uint64_t numImages, const uint32_t nPixelsX,
-        const uint32_t nPixelsY, const uint32_t dynamicRange,
-        const uint64_t numImagesCaught, const int numModX, const int numModY,
-        const DataType dataType, const std::vector<std::string> parameterNames,
-        const std::vector<DataType> parameterDataTypes) {
-        LOG(logERROR) << "This is a generic function CreateVirtualFile that "
-                         "should be overloaded by a derived class";
-    }
-
     virtual void CreateFirstHDF5DataFile(
         const std::string filePath, const std::string fileNamePrefix,
         const uint64_t fileIndex, const bool overWriteEnable,
@@ -85,13 +71,6 @@ class File : private virtual slsDetectorDefs {
         LOG(logERROR) << "This is a generic function CreateFirstDataFile that "
                          "should be overloaded by a derived class";
     };
-
-    virtual void LinkDataFile(std::string dataFilename, std::string dataSetname,
-                              const std::vector<std::string> parameterNames,
-                              const bool silentMode) {
-        LOG(logERROR) << "This is a generic function LinkDataFile that "
-                         "should be overloaded by a derived class";
-    };
 #endif
     virtual void CreateFirstBinaryDataFile(
         const std::string filePath, const std::string fileNamePrefix,
@@ -103,27 +82,9 @@ class File : private virtual slsDetectorDefs {
                          "should be overloaded by a derived class";
     };
 
-    virtual void CreateMasterFile(const std::string filePath,
-                                  const std::string fileNamePrefix,
-                                  const uint64_t fileIndex,
-                                  const bool overWriteEnable,
-                                  const bool silentMode,
-                                  MasterAttributes *attr) {
-        LOG(logERROR) << "This is a generic function CreateMasterFile that "
-                         "should be overloaded by a derived class";
-    };
-
     virtual void WriteToFile(char *buffer, const int buffersize,
                              const uint64_t currentFrameNumber,
-                             const uint32_t numPacketsCaught) {
+                             const uint32_t numPacketsCaught) = 0;
-        LOG(logERROR) << "This is a generic function WriteToFile that "
-                         "should be overloaded by a derived class";
-    };
-
-    virtual void UpdateMasterFile(MasterAttributes *attr, bool silentMode) {
-        LOG(logERROR) << "This is a generic function UpdateMasterFile that "
-                         "should be overloaded by a derived class";
-    };
 
   protected:
     slsDetectorDefs::fileFormat format_;

slsReceiverSoftware/src/HDF5MasterFile.cpp

@@ -1,162 +0,0 @@
-// SPDX-License-Identifier: LGPL-3.0-or-other
-// Copyright (C) 2021 Contributors to the SLS Detector Package
-#include "HDF5MasterFile.h"
-#include "MasterAttributes.h"
-
-HDF5MasterFile::HDF5MasterFile(std::mutex *hdf5Lib)
-    : File(HDF5), hdf5Lib_(hdf5Lib) {}
-
-HDF5MasterFile::~HDF5MasterFile() { CloseFile(); }
-
-void HDF5MasterFile::CloseFile() {
-    std::lock_guard<std::mutex> lock(*hdf5Lib_);
-    try {
-        Exception::dontPrint(); // to handle errors
-        if (fd_) {
-            fd_->close();
-            delete fd_;
-            fd_ = nullptr;
-        }
-    } catch (const Exception &error) {
-        LOG(logERROR) << "Could not close master HDF5 handles";
-        error.printErrorStack();
-    }
-}
-
-void HDF5MasterFile::LinkDataFile(std::string dataFilename,
-                                  std::string dataSetname,
-                                  const std::vector<std::string> parameterNames,
-                                  const bool silentMode) {
-
-    std::lock_guard<std::mutex> lock(*hdf5Lib_);
-    try {
-        Exception::dontPrint(); // to handle errors
-
-        FileAccPropList flist;
-        flist.setFcloseDegree(H5F_CLOSE_STRONG);
-
-        // open master file
-        H5File masterfd(fileName_.c_str(), H5F_ACC_RDWR,
-                        FileCreatPropList::DEFAULT, flist);
-
-        // open data file
-        H5File fd(dataFilename.c_str(), H5F_ACC_RDONLY,
-                  FileCreatPropList::DEFAULT, flist);
-
-        // create link for data dataset
-        DataSet dset = fd.openDataSet(dataSetname.c_str());
-        std::string linkname = std::string("/entry/data/") + dataSetname;
-        if (H5Lcreate_external(dataFilename.c_str(), dataSetname.c_str(),
-                               masterfd.getLocId(), linkname.c_str(),
-                               H5P_DEFAULT, H5P_DEFAULT) < 0) {
-            throw sls::RuntimeError(
-                "Could not create link to data dataset in master");
-        }
-
-        // create link for parameter datasets
-        for (unsigned int i = 0; i < parameterNames.size(); ++i) {
-            DataSet pDset = fd.openDataSet(parameterNames[i].c_str());
-            linkname = std::string("/entry/data/") + parameterNames[i];
-            if (H5Lcreate_external(dataFilename.c_str(),
-                                   parameterNames[i].c_str(),
-                                   masterfd.getLocId(), linkname.c_str(),
-                                   H5P_DEFAULT, H5P_DEFAULT) < 0) {
-                throw sls::RuntimeError(
-                    "Could not create link to parameter dataset in master");
-            }
-        }
-        fd.close();
-        masterfd.close();
-    } catch (const Exception &error) {
-        error.printErrorStack();
-        CloseFile();
-        throw sls::RuntimeError("Could not link in master hdf5 file");
-    }
-    if (!silentMode) {
-        LOG(logINFO) << "Linked in Master File: " << dataFilename;
-    }
-}
-
-void HDF5MasterFile::CreateMasterFile(const std::string filePath,
-                                      const std::string fileNamePrefix,
-                                      const uint64_t fileIndex,
-                                      const bool overWriteEnable,
-                                      const bool silentMode,
-                                      MasterAttributes *attr) {
-
-    std::ostringstream os;
-    os << filePath << "/" << fileNamePrefix << "_master"
-       << "_" << fileIndex << ".h5";
-    fileName_ = os.str();
-
-    std::lock_guard<std::mutex> lock(*hdf5Lib_);
-
-    try {
-        Exception::dontPrint(); // to handle errors
-
-        FileAccPropList flist;
-        flist.setFcloseDegree(H5F_CLOSE_STRONG);
-        fd_ = nullptr;
-        if (!(overWriteEnable))
-            fd_ = new H5File(fileName_.c_str(), H5F_ACC_EXCL,
-                             FileCreatPropList::DEFAULT, flist);
-        else
-            fd_ = new H5File(fileName_.c_str(), H5F_ACC_TRUNC,
-                             FileCreatPropList::DEFAULT, flist);
-
-        // attributes - version
-        double dValue = HDF5_WRITER_VERSION;
-        DataSpace dataspace_attr = DataSpace(H5S_SCALAR);
-        Attribute attribute = fd_->createAttribute(
-            "version", PredType::NATIVE_DOUBLE, dataspace_attr);
-        attribute.write(PredType::NATIVE_DOUBLE, &dValue);
-
-        // Create a group in the file
-        Group group1(fd_->createGroup("entry"));
-        Group group2(group1.createGroup("data"));
-        Group group3(group1.createGroup("instrument"));
-        Group group4(group3.createGroup("beam"));
-        Group group5(group3.createGroup("detector"));
-        Group group6(group1.createGroup("sample"));
-
-        // TODO find a way to get complete group link
-        attrGroupName_ = "/entry/instrument/detector";
-
-        attr->WriteMasterHDF5Attributes(fd_, &group5);
-        fd_->close();
-
-    } catch (const Exception &error) {
-        error.printErrorStack();
-        CloseFile();
-        throw sls::RuntimeError(
-            "Could not create/overwrite master HDF5 handles");
-    }
-    if (!silentMode) {
-        LOG(logINFO) << "Master File: " << fileName_;
-    }
-}
-
-void HDF5MasterFile::UpdateMasterFile(MasterAttributes *attr, bool silentMode) {
-    std::lock_guard<std::mutex> lock(*hdf5Lib_);
-
-    try {
-        Exception::dontPrint(); // to handle errors
-        FileAccPropList flist;
-        flist.setFcloseDegree(H5F_CLOSE_STRONG);
-        fd_ = new H5File(fileName_.c_str(), H5F_ACC_RDWR,
-                         FileCreatPropList::DEFAULT, flist);
-
-        Group group = fd_->openGroup(attrGroupName_.c_str());
-        attr->WriteFinalHDF5Attributes(fd_, &group);
-        fd_->close();
-
-    } catch (const Exception &error) {
-        error.printErrorStack();
-        CloseFile();
-        throw sls::RuntimeError(
-            "Could not create/overwrite master HDF5 handles");
-    }
-    if (!silentMode) {
-        LOG(logINFO) << "Updated Master File";
-    }
-}

slsReceiverSoftware/src/HDF5MasterFile.h

@@ -1,31 +0,0 @@
-// SPDX-License-Identifier: LGPL-3.0-or-other
-// Copyright (C) 2021 Contributors to the SLS Detector Package
-#pragma once
-
-#include "File.h"
-
-#include <mutex>
-
-class HDF5MasterFile : private virtual slsDetectorDefs, public File {
-
-  public:
-    HDF5MasterFile(std::mutex *hdf5Lib);
-    ~HDF5MasterFile();
-
-    void CloseFile() override;
-    void LinkDataFile(std::string dataFilename, std::string dataSetname,
-                      const std::vector<std::string> parameterNames,
-                      const bool silentMode) override;
-    void CreateMasterFile(const std::string filePath,
-                          const std::string fileNamePrefix,
-                          const uint64_t fileIndex, const bool overWriteEnable,
-                          const bool silentMode,
-                          MasterAttributes *attr) override;
-    void UpdateMasterFile(MasterAttributes *attr, bool silentMode) override;
-
-  private:
-    std::mutex *hdf5Lib_;
-    H5File *fd_{nullptr};
-    std::string fileName_;
-    std::string attrGroupName_;
-};

slsReceiverSoftware/src/HDF5VirtualFile.cpp

@@ -1,211 +0,0 @@
-// SPDX-License-Identifier: LGPL-3.0-or-other
-// Copyright (C) 2021 Contributors to the SLS Detector Package
-#include "HDF5VirtualFile.h"
-#include "receiver_defs.h"
-
-#include <iomanip>
-
-HDF5VirtualFile::HDF5VirtualFile(std::mutex *hdf5Lib, bool g25)
-    : File(HDF5), hdf5Lib_(hdf5Lib), gotthard25um(g25) {}
-
-HDF5VirtualFile::~HDF5VirtualFile() { CloseFile(); }
-
-std::array<std::string, 2> HDF5VirtualFile::GetFileAndDatasetName() const {
-    return std::array<std::string, 2>{fileName_, dataSetName_};
-}
-
-void HDF5VirtualFile::CloseFile() {
-    std::lock_guard<std::mutex> lock(*hdf5Lib_);
-    try {
-        Exception::dontPrint(); // to handle errors
-        if (fd_) {
-            fd_->close();
-            delete fd_;
-            fd_ = nullptr;
-        }
-    } catch (const Exception &error) {
-        LOG(logERROR) << "Could not close virtual HDF5 handles of index";
-        error.printErrorStack();
-    }
-}
-
-void HDF5VirtualFile::CreateVirtualFile(
-    const std::string filePath, const std::string fileNamePrefix,
-    const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode,
-    const int modulePos, const int numUnitsPerReadout,
-    const uint32_t maxFramesPerFile, const uint64_t numImages,
-    const uint32_t nPixelsX, const uint32_t nPixelsY,
-    const uint32_t dynamicRange, const uint64_t numImagesCaught,
-    const int numModX, const int numModY, const DataType dataType,
-    const std::vector<std::string> parameterNames,
-    const std::vector<DataType> parameterDataTypes) {
-    // virtual file name
-    std::ostringstream osfn;
-    osfn << filePath << "/" << fileNamePrefix << "_virtual"
-         << "_" << fileIndex << ".h5";
-    fileName_ = osfn.str();
-
-    unsigned int paraSize = parameterNames.size();
-    uint64_t numModZ = numModX;
-    uint32_t nDimy = nPixelsY;
-    uint32_t nDimz = ((dynamicRange == 4) ? (nPixelsX / 2) : nPixelsX);
-
-    std::lock_guard<std::mutex> lock(*hdf5Lib_);
-
-    try {
-        Exception::dontPrint(); // to handle errors
-
-        // file
-        FileAccPropList fapl;
-        fapl.setFcloseDegree(H5F_CLOSE_STRONG);
-        fd_ = nullptr;
-        if (!overWriteEnable)
-            fd_ = new H5File(fileName_.c_str(), H5F_ACC_EXCL,
-                             FileCreatPropList::DEFAULT, fapl);
-        else
-            fd_ = new H5File(fileName_.c_str(), H5F_ACC_TRUNC,
-                             FileCreatPropList::DEFAULT, fapl);
-
-        // attributes - version
-        double dValue = HDF5_WRITER_VERSION;
-        DataSpace dataspace_attr = DataSpace(H5S_SCALAR);
-        Attribute attribute = fd_->createAttribute(
-            "version", PredType::NATIVE_DOUBLE, dataspace_attr);
-        attribute.write(PredType::NATIVE_DOUBLE, &dValue);
-
-        // virtual dataspace
-        hsize_t vdsDims[3] = {numImagesCaught, numModY * nDimy,
-                              numModZ * nDimz};
-        DataSpace vdsDataSpace(3, vdsDims, nullptr);
-        hsize_t vdsDimsPara[2] = {numImagesCaught,
-                                  (unsigned int)numModY * numModZ};
-        DataSpace vdsDataSpacePara(2, vdsDimsPara, nullptr);
-
-        // property list (fill value and datatype)
-        int fill_value = -1;
-        DSetCreatPropList plist;
-        plist.setFillValue(dataType, &fill_value);
-
-        // property list for parameters (datatype)
-        std::vector<DSetCreatPropList> plistPara(paraSize);
-
-        // hyperslab (files)
-        int numFiles = numImagesCaught / maxFramesPerFile;
-        if (numImagesCaught % maxFramesPerFile)
-            ++numFiles;
-        uint64_t framesSaved = 0;
-        for (int iFile = 0; iFile < numFiles; ++iFile) {
-
-            uint64_t nDimx =
-                ((numImagesCaught - framesSaved) > maxFramesPerFile)
-                    ? maxFramesPerFile
-                    : (numImagesCaught - framesSaved);
-
-            hsize_t startLocation[3] = {framesSaved, 0, 0};
-            hsize_t strideBetweenBlocks[3] = {1, 1, 1};
-            hsize_t numBlocks[3] = {nDimx, nDimy, nDimz};
-            hsize_t blockSize[3] = {1, 1, 1};
-
-            hsize_t startLocationPara[2] = {framesSaved, 0};
-            hsize_t strideBetweenBlocksPara[3] = {1, 1};
-            hsize_t numBlocksPara[2] = {1, 1};
-            hsize_t blockSizePara[3] = {nDimx, 1};
-
-            // interleaving for g2
-            if (gotthard25um) {
-                strideBetweenBlocks[2] = 2;
-            }
-
-            for (unsigned int iReadout = 0; iReadout < numModY * numModZ;
-                 ++iReadout) {
-
-                // interleaving for g2 (startLocation is 0 and 1)
-                if (gotthard25um) {
-                    startLocation[2] = iReadout;
-                }
-
-                vdsDataSpace.selectHyperslab(H5S_SELECT_SET, numBlocks,
-                                             startLocation, strideBetweenBlocks,
-                                             blockSize);
-
-                vdsDataSpacePara.selectHyperslab(
-                    H5S_SELECT_SET, numBlocksPara, startLocationPara,
-                    strideBetweenBlocksPara, blockSizePara);
-
-                // source file name
-                std::ostringstream os;
-                os << filePath << "/" << fileNamePrefix << "_d"
-                   << (modulePos * numUnitsPerReadout + iReadout) << "_f"
-                   << iFile << '_' << fileIndex << ".h5";
-                std::string srcFileName = os.str();
-                LOG(logDEBUG1) << srcFileName;
-
-                // find relative path
-                std::string relative_srcFileName = srcFileName;
-                {
-                    size_t p = srcFileName.rfind('/', srcFileName.length());
-                    if (p != std::string::npos)
-                        relative_srcFileName = (srcFileName.substr(
-                            p + 1, srcFileName.length() - p));
-                }
-
-                // source dataset name
-                std::ostringstream osfn;
-                osfn << "/data";
-                if (numImages > 1)
-                    osfn << "_f" << std::setfill('0') << std::setw(12) << iFile;
-                std::string srcDatasetName = osfn.str();
-
-                // source dataspace
-                hsize_t srcDims[3] = {nDimx, nDimy, nDimz};
-                hsize_t srcDimsMax[3] = {H5S_UNLIMITED, nDimy, nDimz};
-                DataSpace srcDataSpace(3, srcDims, srcDimsMax);
-                hsize_t srcDimsPara[1] = {nDimx};
-                hsize_t srcDimsMaxPara[1] = {H5S_UNLIMITED};
-                DataSpace srcDataSpacePara(1, srcDimsPara, srcDimsMaxPara);
-
-                // mapping of property list
-                plist.setVirtual(vdsDataSpace, relative_srcFileName.c_str(),
-                                 srcDatasetName.c_str(), srcDataSpace);
-                for (unsigned int p = 0; p < paraSize; ++p) {
-                    plistPara[p].setVirtual(
-                        vdsDataSpacePara, relative_srcFileName.c_str(),
-                        parameterNames[p].c_str(), srcDataSpacePara);
-                }
-
-                // H5Sclose(srcDataspace);
-                // H5Sclose(srcDataspace_para);
-
-                if (!gotthard25um) {
-                    startLocation[2] += nDimz;
-                    if (startLocation[2] >= (numModZ * nDimz)) {
-                        startLocation[2] = 0;
-                        startLocation[1] += nDimy;
-                    }
-                }
-                startLocationPara[1]++;
-            }
-            framesSaved += nDimx;
-        }
-        // datasets
-        dataSetName_ = "data";
-        DataSet vdsDataSet(fd_->createDataSet(dataSetName_.c_str(), dataType,
-                                              vdsDataSpace, plist));
-
-        for (unsigned int p = 0; p < paraSize; ++p) {
-            DataSet vdsDataSetPara(fd_->createDataSet(
-                parameterNames[p].c_str(), parameterDataTypes[p],
-                vdsDataSpacePara, plistPara[p]));
-        }
-
-        fd_->close();
-    } catch (const Exception &error) {
-        error.printErrorStack();
-        CloseFile();
-        throw sls::RuntimeError(
-            "Could not create/overwrite virtual HDF5 handles");
-    }
-    if (!silentMode) {
-        LOG(logINFO) << "Virtual File: " << fileName_;
-    }
-}

slsReceiverSoftware/src/HDF5VirtualFile.h

@@ -1,34 +0,0 @@
-// SPDX-License-Identifier: LGPL-3.0-or-other
-// Copyright (C) 2021 Contributors to the SLS Detector Package
-#pragma once
-
-#include "File.h"
-
-#include <mutex>
-
-class HDF5VirtualFile : private virtual slsDetectorDefs, public File {
-
-  public:
-    HDF5VirtualFile(std::mutex *hdf5Lib, bool g25);
-    ~HDF5VirtualFile();
-
-    std::array<std::string, 2> GetFileAndDatasetName() const override;
-    void CloseFile() override;
-    void CreateVirtualFile(
-        const std::string filePath, const std::string fileNamePrefix,
-        const uint64_t fileIndex, const bool overWriteEnable,
-        const bool silentMode, const int modulePos,
-        const int numUnitsPerReadout, const uint32_t maxFramesPerFile,
-        const uint64_t numImages, const uint32_t nPixelsX,
-        const uint32_t nPixelsY, const uint32_t dynamicRange,
-        const uint64_t numImagesCaught, const int numModX, const int numModY,
-        const DataType dataType, const std::vector<std::string> parameterNames,
-        const std::vector<DataType> parameterDataTypes) override;
-
-  private:
-    std::mutex *hdf5Lib_;
-    H5File *fd_{nullptr};
-    std::string fileName_;
-    std::string dataSetName_;
-    bool gotthard25um;
-};

slsReceiverSoftware/src/Implementation.cpp

@@ -240,9 +240,6 @@ void Implementation::setModulePositionId(const int id) {
     xy portGeometry = GetPortGeometry();
     streamingPort = DEFAULT_ZMQ_RX_PORTNO + modulePos * portGeometry.x;
 
-    for (const auto &it : dataProcessor)
-        it->SetupFileWriter(fileWriteEnable, masterFileWriteEnable,
-                            fileFormatType, modulePos, &hdf5Lib);
     assert(numModules.y != 0);
     for (unsigned int i = 0; i < listener.size(); ++i) {
         uint16_t row = 0, col = 0;
@@ -372,8 +369,7 @@ void Implementation::setFileFormat(const fileFormat f) {
             throw sls::RuntimeError("Unknown file format");
         }
         for (const auto &it : dataProcessor)
-            it->SetupFileWriter(fileWriteEnable, masterFileWriteEnable,
+            it->SetupFileWriter(fileWriteEnable, fileFormatType, &hdf5LibMutex);
-                                fileFormatType, modulePos, &hdf5Lib);
     }
 
     LOG(logINFO) << "File Format: " << sls::ToString(fileFormatType);
@@ -409,8 +405,7 @@ void Implementation::setFileWriteEnable(const bool b) {
     if (fileWriteEnable != b) {
         fileWriteEnable = b;
         for (const auto &it : dataProcessor)
-            it->SetupFileWriter(fileWriteEnable, masterFileWriteEnable,
+            it->SetupFileWriter(fileWriteEnable, fileFormatType, &hdf5LibMutex);
-                                fileFormatType, modulePos, &hdf5Lib);
     }
     LOG(logINFO) << "File Write Enable: "
                  << (fileWriteEnable ? "enabled" : "disabled");
@@ -423,9 +418,6 @@ bool Implementation::getMasterFileWriteEnable() const {
 void Implementation::setMasterFileWriteEnable(const bool b) {
     if (masterFileWriteEnable != b) {
         masterFileWriteEnable = b;
-        for (const auto &it : dataProcessor)
-            it->SetupFileWriter(fileWriteEnable, masterFileWriteEnable,
-                                fileFormatType, modulePos, &hdf5Lib);
     }
     LOG(logINFO) << "Master File Write Enable: "
                  << (masterFileWriteEnable ? "enabled" : "disabled");
@@ -584,29 +576,9 @@ void Implementation::stopReceiver() {
         std::this_thread::sleep_for(std::chrono::milliseconds(5));
     }
 
-#ifdef HDF5C
+    if (fileWriteEnable && modulePos == 0) {
-    if (fileWriteEnable && fileFormatType == HDF5) {
+        // master and virtual file (hdf5)
-        if (modulePos == 0) {
+        StartMasterWriter();
-            // more than 1 file, create virtual file
-            if (dataProcessor[0]->GetFilesInAcquisition() > 1 ||
-                (numModules.x * numModules.y) > 1) {
-                dataProcessor[0]->CreateVirtualFile(
-                    filePath, fileName, fileIndex, overwriteEnable, silentMode,
-                    modulePos, numUDPInterfaces, framesPerFile,
-                    numberOfTotalFrames, dynamicRange, numModules.x,
-                    numModules.y, &hdf5Lib);
-            }
-            // link file in master
-            dataProcessor[0]->LinkDataInMasterFile(silentMode);
-        }
-    }
-#endif
-    if (fileWriteEnable && masterFileWriteEnable && modulePos == 0) {
-        try {
-            dataProcessor[0]->UpdateMasterFile(silentMode);
-        } catch (...) {
-            ; // ignore it and just print it
-        }
     }
 
     // wait for the processes (dataStreamer) to be done
@@ -753,100 +725,12 @@ void Implementation::CreateUDPSockets() {
 }
 
 void Implementation::SetupWriter() {
-    // master file
-    std::unique_ptr<MasterAttributes> masterAttributes;
-    if (masterFileWriteEnable && modulePos == 0) {
-        switch (detType) {
-        case GOTTHARD:
-            masterAttributes = sls::make_unique<GotthardMasterAttributes>();
-            break;
-        case JUNGFRAU:
-            masterAttributes = sls::make_unique<JungfrauMasterAttributes>();
-            break;
-        case EIGER:
-            masterAttributes = sls::make_unique<EigerMasterAttributes>();
-            break;
-        case MYTHEN3:
-            masterAttributes = sls::make_unique<Mythen3MasterAttributes>();
-            break;
-        case GOTTHARD2:
-            masterAttributes = sls::make_unique<Gotthard2MasterAttributes>();
-            break;
-        case MOENCH:
-            masterAttributes = sls::make_unique<MoenchMasterAttributes>();
-            break;
-        case CHIPTESTBOARD:
-            masterAttributes = sls::make_unique<CtbMasterAttributes>();
-            break;
-        default:
-            throw sls::RuntimeError(
-                "Unknown detector type to set up master file attributes");
-        }
-        masterAttributes->detType = detType;
-        masterAttributes->timingMode = timingMode;
-        xy nm{numModules.x, numModules.y};
-        if (quadEnable) {
-            nm.x = 1;
-            nm.y = 2;
-        }
-        masterAttributes->geometry = xy(nm.x, nm.y);
-        masterAttributes->imageSize = generalData->imageSize;
-        masterAttributes->nPixels =
-            xy(generalData->nPixelsX, generalData->nPixelsY);
-        masterAttributes->maxFramesPerFile = framesPerFile;
-        masterAttributes->frameDiscardMode = frameDiscardMode;
-        masterAttributes->framePadding = framePadding;
-        masterAttributes->scanParams = scanParams;
-        masterAttributes->totalFrames = numberOfTotalFrames;
-        masterAttributes->exptime = acquisitionTime;
-        masterAttributes->period = acquisitionPeriod;
-        masterAttributes->burstMode = burstMode;
-        masterAttributes->numUDPInterfaces = numUDPInterfaces;
-        masterAttributes->dynamicRange = dynamicRange;
-        masterAttributes->tenGiga = tengigaEnable;
-        masterAttributes->thresholdEnergyeV = thresholdEnergyeV;
-        masterAttributes->thresholdAllEnergyeV = thresholdAllEnergyeV;
-        masterAttributes->subExptime = subExpTime;
-        masterAttributes->subPeriod = subPeriod;
-        masterAttributes->quad = quadEnable;
-        masterAttributes->readNRows = readNRows;
-        masterAttributes->ratecorr = rateCorrections;
-        masterAttributes->adcmask =
-            tengigaEnable ? adcEnableMaskTenGiga : adcEnableMaskOneGiga;
-        masterAttributes->analog =
-            (readoutType == ANALOG_ONLY || readoutType == ANALOG_AND_DIGITAL)
-                ? 1
-                : 0;
-        masterAttributes->analogSamples = numberOfAnalogSamples;
-        masterAttributes->digital =
-            (readoutType == DIGITAL_ONLY || readoutType == ANALOG_AND_DIGITAL)
-                ? 1
-                : 0;
-        masterAttributes->digitalSamples = numberOfDigitalSamples;
-        masterAttributes->dbitoffset = ctbDbitOffset;
-        masterAttributes->dbitlist = 0;
-        for (auto &i : ctbDbitList) {
-            masterAttributes->dbitlist |= (1 << i);
-        }
-        masterAttributes->roi = roi;
-        masterAttributes->counterMask = counterMask;
-        masterAttributes->exptime1 = acquisitionTime1;
-        masterAttributes->exptime2 = acquisitionTime2;
-        masterAttributes->exptime3 = acquisitionTime3;
-        masterAttributes->gateDelay1 = gateDelay1;
-        masterAttributes->gateDelay2 = gateDelay2;
-        masterAttributes->gateDelay3 = gateDelay3;
-        masterAttributes->gates = numberOfGates;
-        masterAttributes->additionalJsonHeader = additionalJsonHeader;
-    }
-
     try {
         for (unsigned int i = 0; i < dataProcessor.size(); ++i) {
             dataProcessor[i]->CreateFirstFiles(
-                masterAttributes.get(), filePath, fileName, fileIndex,
+                filePath, fileName, fileIndex, overwriteEnable, silentMode,
-                overwriteEnable, silentMode, modulePos, numUDPInterfaces,
+                modulePos, numUDPInterfaces, udpPortNum[i], framesPerFile,
-                udpPortNum[i], framesPerFile, numberOfTotalFrames, dynamicRange,
+                numberOfTotalFrames, dynamicRange, detectorDataStream[i]);
-                detectorDataStream[i]);
         }
     } catch (const sls::RuntimeError &e) {
         shutDownUDPSockets();
@@ -856,6 +740,100 @@ void Implementation::SetupWriter() {
     }
 }
 
+void Implementation::StartMasterWriter() {
+    try {
+        std::string masterFileName;
+        // master file
+        if (masterFileWriteEnable) {
+            MasterAttributes masterAttributes;
+            masterAttributes.detType = detType;
+            masterAttributes.timingMode = timingMode;
+            xy nm{numModules.x, numModules.y};
+            if (quadEnable) {
+                nm.x = 1;
+                nm.y = 2;
+            }
+            masterAttributes.geometry = xy(nm.x, nm.y);
+            masterAttributes.imageSize = generalData->imageSize;
+            masterAttributes.nPixels =
+                xy(generalData->nPixelsX, generalData->nPixelsY);
+            masterAttributes.maxFramesPerFile = framesPerFile;
+            masterAttributes.frameDiscardMode = frameDiscardMode;
+            masterAttributes.framePadding = framePadding;
+            masterAttributes.scanParams = scanParams;
+            masterAttributes.totalFrames = numberOfTotalFrames;
+            masterAttributes.exptime = acquisitionTime;
+            masterAttributes.period = acquisitionPeriod;
+            masterAttributes.burstMode = burstMode;
+            masterAttributes.numUDPInterfaces = numUDPInterfaces;
+            masterAttributes.dynamicRange = dynamicRange;
+            masterAttributes.tenGiga = tengigaEnable;
+            masterAttributes.thresholdEnergyeV = thresholdEnergyeV;
+            masterAttributes.thresholdAllEnergyeV = thresholdAllEnergyeV;
+            masterAttributes.subExptime = subExpTime;
+            masterAttributes.subPeriod = subPeriod;
+            masterAttributes.quad = quadEnable;
+            masterAttributes.readNRows = readNRows;
+            masterAttributes.ratecorr = rateCorrections;
+            masterAttributes.adcmask =
+                tengigaEnable ? adcEnableMaskTenGiga : adcEnableMaskOneGiga;
+            masterAttributes.analog = (readoutType == ANALOG_ONLY ||
+                                       readoutType == ANALOG_AND_DIGITAL)
+                                          ? 1
+                                          : 0;
+            masterAttributes.analogSamples = numberOfAnalogSamples;
+            masterAttributes.digital = (readoutType == DIGITAL_ONLY ||
+                                        readoutType == ANALOG_AND_DIGITAL)
+                                           ? 1
+                                           : 0;
+            masterAttributes.digitalSamples = numberOfDigitalSamples;
+            masterAttributes.dbitoffset = ctbDbitOffset;
+            masterAttributes.dbitlist = 0;
+            for (auto &i : ctbDbitList) {
+                masterAttributes.dbitlist |= (1 << i);
+            }
+            masterAttributes.roi = roi;
+            masterAttributes.counterMask = counterMask;
+            masterAttributes.exptimeArray[0] = acquisitionTime1;
+            masterAttributes.exptimeArray[1] = acquisitionTime2;
+            masterAttributes.exptimeArray[2] = acquisitionTime3;
+            masterAttributes.gateDelayArray[0] = gateDelay1;
+            masterAttributes.gateDelayArray[1] = gateDelay2;
+            masterAttributes.gateDelayArray[2] = gateDelay3;
+            masterAttributes.gates = numberOfGates;
+            masterAttributes.additionalJsonHeader = additionalJsonHeader;
+
+            // create master file
+            masterFileName = dataProcessor[0]->CreateMasterFile(
+                filePath, fileName, fileIndex, overwriteEnable, silentMode,
+                fileFormatType, &masterAttributes, &hdf5LibMutex);
+        }
+#ifdef HDF5C
+        if (fileFormatType == HDF5) {
+            std::array<std::string, 2> virtualFileAndDatasetNames;
+            // create virtual hdf5 file (if multiple files)
+            if (dataProcessor[0]->GetFilesInAcquisition() > 1 ||
+                (numModules.x * numModules.y) > 1) {
+                virtualFileAndDatasetNames =
+                    dataProcessor[0]->CreateVirtualFile(
+                        filePath, fileName, fileIndex, overwriteEnable,
+                        silentMode, modulePos, numUDPInterfaces, framesPerFile,
+                        numberOfTotalFrames, numModules.x, numModules.y,
+                        dynamicRange, &hdf5LibMutex);
+            }
+            // link file in master
+            if (masterFileWriteEnable) {
+                dataProcessor[0]->LinkFileInMaster(
+                    masterFileName, virtualFileAndDatasetNames[0],
+                    virtualFileAndDatasetNames[1], silentMode, &hdf5LibMutex);
+            }
+        }
+#endif
+    } catch (...) {
+        ; // ignore it and just print it
+    }
+}
+
 void Implementation::StartRunning() {
 
     // set running mask and post semaphore to start the inner loop in execution

slsReceiverSoftware/src/Implementation.h

@@ -274,6 +274,7 @@ class Implementation : private virtual slsDetectorDefs {
     void ResetParametersforNewAcquisition();
     void CreateUDPSockets();
     void SetupWriter();
+    void StartMasterWriter();
     void StartRunning();
 
     /**************************************************
@@ -386,5 +387,6 @@ class Implementation : private virtual slsDetectorDefs {
     std::vector<std::unique_ptr<Fifo>> fifo;
     Arping arping;
 
-    std::mutex hdf5Lib;
+    // mutex shared across all hdf5 virtual, master and data files
+    std::mutex hdf5LibMutex;
 };

slsReceiverSoftware/src/MasterAttributes.h

@@ -7,6 +7,9 @@
 #include "sls/logger.h"
 #include "sls/sls_detector_defs.h"
 
+#include <rapidjson/stringbuffer.h>
+#include <rapidjson/prettywriter.h>
+
 #ifdef HDF5C
 #include "H5Cpp.h"
 #ifndef H5_NO_NAMESPACE
@@ -53,94 +56,92 @@ class MasterAttributes {
     uint64_t dbitlist{0};
     slsDetectorDefs::ROI roi{};
     uint32_t counterMask{0};
-    ns exptime1{0};
+    std::array<ns, 3> exptimeArray{};
-    ns exptime2{0};
+    std::array<ns, 3> gateDelayArray{};
-    ns exptime3{0};
-    ns gateDelay1{0};
-    ns gateDelay2{0};
-    ns gateDelay3{0};
     uint32_t gates;
     std::map<std::string, std::string> additionalJsonHeader;
-
-    // Final Attributes (after acquisition)
     uint64_t framesInFile{0};
 
     MasterAttributes() = default;
-    virtual ~MasterAttributes() = default;
+    ~MasterAttributes() = default;
-    virtual void WriteMasterBinaryAttributes(FILE *fd);
+
-    std::string GetBinaryMasterAttributes();
+    void GetBinaryAttributes(rapidjson::PrettyWriter<rapidjson::StringBuffer> *w);
-    void WriteBinaryAttributes(FILE *fd, std::string message);
-    void WriteFinalBinaryAttributes(FILE *fd);
 #ifdef HDF5C
-    virtual void WriteMasterHDF5Attributes(H5File *fd, Group *group);
     void WriteHDF5Attributes(H5File *fd, Group *group);
+#endif
+
+    void
+    GetCommonBinaryAttributes(rapidjson::PrettyWriter<rapidjson::StringBuffer> *w);
+    void
+    GetFinalBinaryAttributes(rapidjson::PrettyWriter<rapidjson::StringBuffer> *w);
+#ifdef HDF5C
+    void WriteCommonHDF5Attributes(H5File *fd, Group *group);
     void WriteFinalHDF5Attributes(H5File *fd, Group *group);
     void WriteHDF5Exptime(H5File *fd, Group *group);
     void WriteHDF5Period(H5File *fd, Group *group);
     void WriteHDF5DynamicRange(H5File *fd, Group *group);
     void WriteHDF5TenGiga(H5File *fd, Group *group);
+    void WriteHDF5ROI(H5File *fd, Group *group);
+    void WriteHDF5NumUDPInterfaces(H5File *fd, Group *group);
+    void WriteHDF5ReadNRows(H5File *fd, Group *group);
+    void WriteHDF5ThresholdEnergy(H5File *fd, Group *group);
+    void WriteHDF5ThresholdEnergies(H5File *fd, Group *group);
+    void WriteHDF5SubExpTime(H5File *fd, Group *group);
+    void WriteHDF5SubPeriod(H5File *fd, Group *group);
+    void WriteHDF5SubQuad(H5File *fd, Group *group);
+    void WriteHDF5RateCorrections(H5File *fd, Group *group);
+    void WriteHDF5CounterMask(H5File *fd, Group *group);
+    void WriteHDF5ExptimeArray(H5File *fd, Group *group);
+    void WriteHDF5GateDelayArray(H5File *fd, Group *group);
+    void WriteHDF5Gates(H5File *fd, Group *group);
+    void WriteHDF5BurstMode(H5File *fd, Group *group);
+    void WriteHDF5AdcMask(H5File *fd, Group *group);
+    void WriteHDF5AnalogFlag(H5File *fd, Group *group);
+    void WriteHDF5AnalogSamples(H5File *fd, Group *group);
+    void WriteHDF5DigitalFlag(H5File *fd, Group *group);
+    void WriteHDF5DigitalSamples(H5File *fd, Group *group);
+    void WriteHDF5DbitOffset(H5File *fd, Group *group);
+    void WriteHDF5DbitList(H5File *fd, Group *group);
 #endif
-};
 
-class GotthardMasterAttributes : public MasterAttributes {
+    void
-  public:
+    GetGotthardBinaryAttributes(rapidjson::PrettyWriter<rapidjson::StringBuffer> *w);
-    GotthardMasterAttributes() = default;
-    void WriteMasterBinaryAttributes(FILE *fd) override;
 #ifdef HDF5C
-    void WriteMasterHDF5Attributes(H5File *fd, Group *group) override;
+    void WriteGotthardHDF5Attributes(H5File *fd, Group *group);
 #endif
-};
 
-class JungfrauMasterAttributes : public MasterAttributes {
+    void
-  public:
+    GetJungfrauBinaryAttributes(rapidjson::PrettyWriter<rapidjson::StringBuffer> *w);
-    JungfrauMasterAttributes() = default;
-    void WriteMasterBinaryAttributes(FILE *fd) override;
 #ifdef HDF5C
-    void WriteMasterHDF5Attributes(H5File *fd, Group *group) override;
+    void WriteJungfrauHDF5Attributes(H5File *fd, Group *group);
 #endif
-};
 
-class EigerMasterAttributes : public MasterAttributes {
+    void
-  public:
+    GetEigerBinaryAttributes(rapidjson::PrettyWriter<rapidjson::StringBuffer> *w);
-    EigerMasterAttributes() = default;
-    void WriteMasterBinaryAttributes(FILE *fd) override;
 #ifdef HDF5C
-    void WriteMasterHDF5Attributes(H5File *fd, Group *group) override;
+    void WriteEigerHDF5Attributes(H5File *fd, Group *group);
 #endif
-};
 
-class Mythen3MasterAttributes : public MasterAttributes {
+    void
-  public:
+    GetMythen3BinaryAttributes(rapidjson::PrettyWriter<rapidjson::StringBuffer> *w);
-    Mythen3MasterAttributes() = default;
-    void WriteMasterBinaryAttributes(FILE *fd) override;
 #ifdef HDF5C
-    void WriteMasterHDF5Attributes(H5File *fd, Group *group) override;
+    void WriteMythen3HDF5Attributes(H5File *fd, Group *group);
 #endif
-};
 
-class Gotthard2MasterAttributes : public MasterAttributes {
+    void
-  public:
+    GetGotthard2BinaryAttributes(rapidjson::PrettyWriter<rapidjson::StringBuffer> *w);
-    Gotthard2MasterAttributes() = default;
-    void WriteMasterBinaryAttributes(FILE *fd) override;
 #ifdef HDF5C
-    void WriteMasterHDF5Attributes(H5File *fd, Group *group) override;
+    void WriteGotthard2HDF5Attributes(H5File *fd, Group *group);
 #endif
-};
 
-class MoenchMasterAttributes : public MasterAttributes {
+    void
-  public:
+    GetMoenchBinaryAttributes(rapidjson::PrettyWriter<rapidjson::StringBuffer> *w);
-    MoenchMasterAttributes() = default;
-    void WriteMasterBinaryAttributes(FILE *fd) override;
 #ifdef HDF5C
-    void WriteMasterHDF5Attributes(H5File *fd, Group *group) override;
+    void WriteMoenchHDF5Attributes(H5File *fd, Group *group);
 #endif
-};
 
-class CtbMasterAttributes : public MasterAttributes {
+    void GetCtbBinaryAttributes(rapidjson::PrettyWriter<rapidjson::StringBuffer> *w);
-  public:
-    CtbMasterAttributes() = default;
-    void WriteMasterBinaryAttributes(FILE *fd) override;
 #ifdef HDF5C
-    void WriteMasterHDF5Attributes(H5File *fd, Group *group) override;
+    void WriteCtbHDF5Attributes(H5File *fd, Group *group);
 #endif
 };

slsReceiverSoftware/src/MasterFileUtility.cpp

@@ -0,0 +1,351 @@
+// SPDX-License-Identifier: LGPL-3.0-or-other
+// Copyright (C) 2021 Contributors to the SLS Detector Package
+#include "MasterFileUtility.h"
+#include "sls/container_utils.h"
+
+#include <iomanip>
+
+namespace masterFileUtility {
+
+std::string CreateMasterBinaryFile(const std::string &filePath,
+                                   const std::string &fileNamePrefix,
+                                   const uint64_t fileIndex,
+                                   const bool overWriteEnable,
+                                   const bool silentMode,
+                                   MasterAttributes *attr) {
+    std::ostringstream os;
+    os << filePath << "/" << fileNamePrefix << "_master"
+       << "_" << fileIndex << ".json";
+    std::string fileName = os.str();
+
+    std::string mode = "w";
+    if (!overWriteEnable)
+        mode = "wx";
+    FILE *fd = fopen(fileName.c_str(), mode.c_str());
+    if(!fd) {
+        throw sls::RuntimeError("Could not create/overwrite binary master file " +
+                                    fileName);
+    }
+
+    rapidjson::StringBuffer s;
+    rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(s);
+    attr->GetBinaryAttributes(&writer);
+    if (fwrite(s.GetString(), 1, strlen(s.GetString()), fd) !=
+        strlen(s.GetString())) {
+        throw sls::RuntimeError(
+            "Master binary file incorrect number of bytes written to file");
+    }
+    if (fd) {
+        fclose(fd);
+    }
+    if (!silentMode) {
+        LOG(logINFO) << "Master File: " << fileName;
+    }
+    return fileName;
+}
+
+#ifdef HDF5C
+void LinkHDF5FileInMaster(const std::string &masterFileName,
+                          const std::string &dataFilename,
+                          const std::string &dataSetname,
+                          const std::vector<std::string> parameterNames,
+                          const bool silentMode, std::mutex *hdf5LibMutex) {
+
+    std::lock_guard<std::mutex> lock(*hdf5LibMutex);
+    std::unique_ptr<H5File> fd{nullptr};
+    try {
+        Exception::dontPrint(); // to handle errors
+
+        FileAccPropList flist;
+        flist.setFcloseDegree(H5F_CLOSE_STRONG);
+
+        // open master file
+        H5File masterfd(masterFileName.c_str(), H5F_ACC_RDWR,
+                        FileCreatPropList::DEFAULT, flist);
+
+        // open data file
+        fd = sls::make_unique<H5File>(dataFilename.c_str(), H5F_ACC_RDONLY,
+                                      FileCreatPropList::DEFAULT, flist);
+
+        // create link for data dataset
+        DataSet dset = fd->openDataSet(dataSetname.c_str());
+        std::string linkname = std::string("/entry/data/") + dataSetname;
+        if (H5Lcreate_external(dataFilename.c_str(), dataSetname.c_str(),
+                               masterfd.getLocId(), linkname.c_str(),
+                               H5P_DEFAULT, H5P_DEFAULT) < 0) {
+            throw sls::RuntimeError(
+                "Could not create link to data dataset in master");
+        }
+
+        // create link for parameter datasets
+        for (unsigned int i = 0; i < parameterNames.size(); ++i) {
+            DataSet pDset = fd->openDataSet(parameterNames[i].c_str());
+            linkname = std::string("/entry/data/") + parameterNames[i];
+            if (H5Lcreate_external(dataFilename.c_str(),
+                                   parameterNames[i].c_str(),
+                                   masterfd.getLocId(), linkname.c_str(),
+                                   H5P_DEFAULT, H5P_DEFAULT) < 0) {
+                throw sls::RuntimeError(
+                    "Could not create link to parameter dataset in master");
+            }
+        }
+        fd->close();
+        masterfd.close();
+    } catch (const Exception &error) {
+        error.printErrorStack();
+        if (fd != nullptr)
+            fd->close();
+        throw sls::RuntimeError("Could not link in master hdf5 file");
+    }
+    if (!silentMode) {
+        LOG(logINFO) << "Linked in Master File: " << dataFilename;
+    }
+}
+
+std::string CreateMasterHDF5File(const std::string &filePath,
+                                 const std::string &fileNamePrefix,
+                                 const uint64_t fileIndex,
+                                 const bool overWriteEnable,
+                                 const bool silentMode, MasterAttributes *attr,
+                                 std::mutex *hdf5LibMutex) {
+
+    std::ostringstream os;
+    os << filePath << "/" << fileNamePrefix << "_master"
+       << "_" << fileIndex << ".h5";
+    std::string fileName = os.str();
+
+    std::lock_guard<std::mutex> lock(*hdf5LibMutex);
+
+    std::unique_ptr<H5File> fd{nullptr};
+    try {
+        Exception::dontPrint(); // to handle errors
+
+        FileAccPropList flist;
+        flist.setFcloseDegree(H5F_CLOSE_STRONG);
+
+        unsigned int createFlags = H5F_ACC_EXCL;
+        if (overWriteEnable) {
+            createFlags = H5F_ACC_TRUNC;
+        }
+        fd = sls::make_unique<H5File>(fileName.c_str(), createFlags,
+                                      FileCreatPropList::DEFAULT, flist);
+
+        // attributes - version
+        double dValue = HDF5_WRITER_VERSION;
+        DataSpace dataspace_attr = DataSpace(H5S_SCALAR);
+        Attribute attribute = fd->createAttribute(
+            "version", PredType::NATIVE_DOUBLE, dataspace_attr);
+        attribute.write(PredType::NATIVE_DOUBLE, &dValue);
+
+        // Create a group in the file
+        Group group1(fd->createGroup("entry"));
+        Group group2(group1.createGroup("data"));
+        Group group3(group1.createGroup("instrument"));
+        Group group4(group3.createGroup("beam"));
+        Group group5(group3.createGroup("detector"));
+        Group group6(group1.createGroup("sample"));
+
+        attr->WriteHDF5Attributes(fd.get(), &group5);
+        fd->close();
+    } catch (const Exception &error) {
+        error.printErrorStack();
+        if (fd != nullptr)
+            fd->close();
+        throw sls::RuntimeError(
+            "Could not create/overwrite master HDF5 handles");
+    }
+    if (!silentMode) {
+        LOG(logINFO) << "Master File: " << fileName;
+    }
+    return fileName;
+}
+
+std::array<std::string, 2> CreateVirtualHDF5File(
+    const std::string &filePath, const std::string &fileNamePrefix,
+    const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode,
+    const int modulePos, const int numUnitsPerReadout,
+    const uint32_t maxFramesPerFile, const uint64_t numImages,
+    const uint32_t nPixelsX, const uint32_t nPixelsY,
+    const uint32_t dynamicRange, const uint64_t numImagesCaught,
+    const int numModX, const int numModY, const DataType dataType,
+    const std::vector<std::string> parameterNames,
+    const std::vector<DataType> parameterDataTypes, std::mutex *hdf5LibMutex,
+    bool gotthard25um) {
+
+    // virtual file name
+    std::ostringstream osfn;
+    osfn << filePath << "/" << fileNamePrefix << "_virtual"
+         << "_" << fileIndex << ".h5";
+    std::string fileName = osfn.str();
+
+    std::string dataSetName = "data";
+
+    unsigned int paraSize = parameterNames.size();
+    uint64_t numModZ = numModX;
+    uint32_t nDimy = nPixelsY;
+    uint32_t nDimz = ((dynamicRange == 4) ? (nPixelsX / 2) : nPixelsX);
+
+    std::lock_guard<std::mutex> lock(*hdf5LibMutex);
+
+    std::unique_ptr<H5File> fd{nullptr};
+    try {
+        Exception::dontPrint(); // to handle errors
+
+        // file
+        FileAccPropList fapl;
+        fapl.setFcloseDegree(H5F_CLOSE_STRONG);
+        if (!overWriteEnable)
+            fd = sls::make_unique<H5File>(fileName.c_str(), H5F_ACC_EXCL,
+                                          FileCreatPropList::DEFAULT, fapl);
+        else
+            fd = sls::make_unique<H5File>(fileName.c_str(), H5F_ACC_TRUNC,
+                                          FileCreatPropList::DEFAULT, fapl);
+
+        // attributes - version
+        double dValue = HDF5_WRITER_VERSION;
+        DataSpace dataspace_attr = DataSpace(H5S_SCALAR);
+        Attribute attribute = fd->createAttribute(
+            "version", PredType::NATIVE_DOUBLE, dataspace_attr);
+        attribute.write(PredType::NATIVE_DOUBLE, &dValue);
+
+        // virtual dataspace
+        hsize_t vdsDims[3] = {numImagesCaught, numModY * nDimy,
+                              numModZ * nDimz};
+        DataSpace vdsDataSpace(3, vdsDims, nullptr);
+        hsize_t vdsDimsPara[2] = {numImagesCaught,
+                                  (unsigned int)numModY * numModZ};
+        DataSpace vdsDataSpacePara(2, vdsDimsPara, nullptr);
+
+        // property list (fill value and datatype)
+        int fill_value = -1;
+        DSetCreatPropList plist;
+        plist.setFillValue(dataType, &fill_value);
+
+        // property list for parameters (datatype)
+        std::vector<DSetCreatPropList> plistPara(paraSize);
+
+        // hyperslab (files)
+        int numFiles = numImagesCaught / maxFramesPerFile;
+        if (numImagesCaught % maxFramesPerFile)
+            ++numFiles;
+        uint64_t framesSaved = 0;
+        for (int iFile = 0; iFile < numFiles; ++iFile) {
+
+            uint64_t nDimx =
+                ((numImagesCaught - framesSaved) > maxFramesPerFile)
+                    ? maxFramesPerFile
+                    : (numImagesCaught - framesSaved);
+
+            hsize_t startLocation[3] = {framesSaved, 0, 0};
+            hsize_t strideBetweenBlocks[3] = {1, 1, 1};
+            hsize_t numBlocks[3] = {nDimx, nDimy, nDimz};
+            hsize_t blockSize[3] = {1, 1, 1};
+
+            hsize_t startLocationPara[2] = {framesSaved, 0};
+            hsize_t strideBetweenBlocksPara[3] = {1, 1};
+            hsize_t numBlocksPara[2] = {1, 1};
+            hsize_t blockSizePara[3] = {nDimx, 1};
+
+            // interleaving for g2
+            if (gotthard25um) {
+                strideBetweenBlocks[2] = 2;
+            }
+
+            for (unsigned int iReadout = 0; iReadout < numModY * numModZ;
+                 ++iReadout) {
+
+                // interleaving for g2 (startLocation is 0 and 1)
+                if (gotthard25um) {
+                    startLocation[2] = iReadout;
+                }
+
+                vdsDataSpace.selectHyperslab(H5S_SELECT_SET, numBlocks,
+                                             startLocation, strideBetweenBlocks,
+                                             blockSize);
+
+                vdsDataSpacePara.selectHyperslab(
+                    H5S_SELECT_SET, numBlocksPara, startLocationPara,
+                    strideBetweenBlocksPara, blockSizePara);
+
+                // source file name
+                std::ostringstream os;
+                os << filePath << "/" << fileNamePrefix << "_d"
+                   << (modulePos * numUnitsPerReadout + iReadout) << "_f"
+                   << iFile << '_' << fileIndex << ".h5";
+                std::string srcFileName = os.str();
+                LOG(logDEBUG1) << srcFileName;
+
+                // find relative path
+                std::string relative_srcFileName = srcFileName;
+                {
+                    size_t p = srcFileName.rfind('/', srcFileName.length());
+                    if (p != std::string::npos)
+                        relative_srcFileName = (srcFileName.substr(
+                            p + 1, srcFileName.length() - p));
+                }
+
+                // source dataset name
+                std::ostringstream osfn;
+                osfn << "/data";
+                if (numImages > 1)
+                    osfn << "_f" << std::setfill('0') << std::setw(12) << iFile;
+                std::string srcDatasetName = osfn.str();
+
+                // source dataspace
+                hsize_t srcDims[3] = {nDimx, nDimy, nDimz};
+                hsize_t srcDimsMax[3] = {H5S_UNLIMITED, nDimy, nDimz};
+                DataSpace srcDataSpace(3, srcDims, srcDimsMax);
+                hsize_t srcDimsPara[1] = {nDimx};
+                hsize_t srcDimsMaxPara[1] = {H5S_UNLIMITED};
+                DataSpace srcDataSpacePara(1, srcDimsPara, srcDimsMaxPara);
+
+                // mapping of property list
+                plist.setVirtual(vdsDataSpace, relative_srcFileName.c_str(),
+                                 srcDatasetName.c_str(), srcDataSpace);
+                for (unsigned int p = 0; p < paraSize; ++p) {
+                    plistPara[p].setVirtual(
+                        vdsDataSpacePara, relative_srcFileName.c_str(),
+                        parameterNames[p].c_str(), srcDataSpacePara);
+                }
+
+                // H5Sclose(srcDataspace);
+                // H5Sclose(srcDataspace_para);
+
+                if (!gotthard25um) {
+                    startLocation[2] += nDimz;
+                    if (startLocation[2] >= (numModZ * nDimz)) {
+                        startLocation[2] = 0;
+                        startLocation[1] += nDimy;
+                    }
+                }
+                startLocationPara[1]++;
+            }
+            framesSaved += nDimx;
+        }
+        // datasets
+        DataSet vdsDataSet(fd->createDataSet(dataSetName.c_str(), dataType,
+                                             vdsDataSpace, plist));
+
+        for (unsigned int p = 0; p < paraSize; ++p) {
+            DataSet vdsDataSetPara(fd->createDataSet(
+                parameterNames[p].c_str(), parameterDataTypes[p],
+                vdsDataSpacePara, plistPara[p]));
+        }
+
+        fd->close();
+    } catch (const Exception &error) {
+        error.printErrorStack();
+        if (fd) {
+            fd->close();
+        }
+        throw sls::RuntimeError(
+            "Could not create/overwrite virtual HDF5 handles");
+    }
+    if (!silentMode) {
+        LOG(logINFO) << "Virtual File: " << fileName;
+    }
+    return std::array<std::string, 2>{fileName, dataSetName};
+}
+#endif
+
+} // namespace masterFileUtility

slsReceiverSoftware/src/MasterFileUtility.h

@@ -0,0 +1,50 @@
+// SPDX-License-Identifier: LGPL-3.0-or-other
+// Copyright (C) 2021 Contributors to the SLS Detector Package
+#pragma once
+
+#include "MasterAttributes.h"
+
+#ifdef HDF5C
+#include "H5Cpp.h"
+#include <mutex>
+#ifndef H5_NO_NAMESPACE
+using namespace H5;
+#endif
+#endif
+
+namespace masterFileUtility {
+
+std::string CreateMasterBinaryFile(const std::string &filePath,
+                                   const std::string &fileNamePrefix,
+                                   const uint64_t fileIndex,
+                                   const bool overWriteEnable,
+                                   const bool silentMode,
+                                   MasterAttributes *attr);
+
+#ifdef HDF5C
+void LinkHDF5FileInMaster(const std::string &masterFileName,
+                          const std::string &dataFilename,
+                          const std::string &dataSetname,
+                          const std::vector<std::string> parameterNames,
+                          const bool silentMode, std::mutex *hdf5LibMutex);
+
+std::string CreateMasterHDF5File(const std::string &filePath,
+                                 const std::string &fileNamePrefix,
+                                 const uint64_t fileIndex,
+                                 const bool overWriteEnable,
+                                 const bool silentMode, MasterAttributes *attr,
+                                 std::mutex *hdf5LibMutex);
+
+std::array<std::string, 2> CreateVirtualHDF5File(
+    const std::string &filePath, const std::string &fileNamePrefix,
+    const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode,
+    const int modulePos, const int numUnitsPerReadout,
+    const uint32_t maxFramesPerFile, const uint64_t numImages,
+    const uint32_t nPixelsX, const uint32_t nPixelsY,
+    const uint32_t dynamicRange, const uint64_t numImagesCaught,
+    const int numModX, const int numModY, const DataType dataType,
+    const std::vector<std::string> parameterNames,
+    const std::vector<DataType> parameterDataTypes, std::mutex *hdf5LibMutex,
+    bool gotthard25um);
+#endif
+} // namespace masterFileUtility

slsReceiverSoftware/src/receiver_defs.h

@@ -18,7 +18,7 @@
 
 // versions
 #define HDF5_WRITER_VERSION   (6.4) // 1 decimal places
-#define BINARY_WRITER_VERSION (6.4) // 1 decimal places
+#define BINARY_WRITER_VERSION (7.0) // 1 decimal places
 
 #define MAX_FRAMES_PER_FILE           20000
 #define SHORT_MAX_FRAMES_PER_FILE     100000