Merge pull request #427 from slsdetectorgroup/m3mastertrigger

m3:bug fix: slaves shoudl always have trigger

RELEASE.txt

@@ -57,6 +57,8 @@ This document describes the differences between v7.0.0 and v6.x.x
 - ctb: can set names for all the dacs
 - fpga/kernel programming, checks if drive is a special file and not a normal file
 - gotthard 25 um image reconstructed in gui and virtual hdf5 (firmware updated for slave to reverse channels)
+- master binary file in json format now
+- fixed bug introduced in 6.0.0: hdf5 files created 1 file per frame after the initial file which had maxframesperfile
 
 2. Resolved Issues
 ==================

slsDetectorServers/mythen3DetectorServer/slsDetectorFunctionList.c

@@ -1603,7 +1603,7 @@ void setTiming(enum timingMode arg) {
 
     int master = 0;
     isMaster(&master);
-    if (master && arg == AUTO_TIMING)
+    if (!master && arg == AUTO_TIMING)
         arg = TRIGGER_EXPOSURE;
 
     uint32_t addr = CONFIG_REG;

slsDetectorSoftware/src/Detector.cpp

@@ -26,8 +26,8 @@ void freeSharedMemory(int detectorIndex, int moduleIndex) {
     // single module
     if (moduleIndex >= 0) {
         SharedMemory<sharedModule> moduleShm(detectorIndex, moduleIndex);
-        if (moduleShm.IsExisting()) {
-            moduleShm.RemoveSharedMemory();
+        if (moduleShm.exists()) {
+            moduleShm.removeSharedMemory();
         }
         return;
     }
@@ -36,21 +36,21 @@ void freeSharedMemory(int detectorIndex, int moduleIndex) {
     SharedMemory<sharedDetector> detectorShm(detectorIndex, -1);
     int numDetectors = 0;
 
-    if (detectorShm.IsExisting()) {
-        detectorShm.OpenSharedMemory();
+    if (detectorShm.exists()) {
+        detectorShm.openSharedMemory();
         numDetectors = detectorShm()->numberOfModules;
-        detectorShm.RemoveSharedMemory();
+        detectorShm.removeSharedMemory();
     }
 
     for (int i = 0; i < numDetectors; ++i) {
         SharedMemory<sharedModule> moduleShm(detectorIndex, i);
-        moduleShm.RemoveSharedMemory();
+        moduleShm.removeSharedMemory();
     }
 
     // Ctb configuration
     SharedMemory<CtbConfig> ctbShm(detectorIndex, -1, CtbConfig::shm_tag());
-    if (ctbShm.IsExisting())
-        ctbShm.RemoveSharedMemory();
+    if (ctbShm.exists())
+        ctbShm.removeSharedMemory();
 }
 
 using defs = slsDetectorDefs;

slsDetectorSoftware/src/DetectorImpl.cpp

@@ -46,8 +46,8 @@ void DetectorImpl::setupDetector(bool verify, bool update) {
         updateUserdetails();
     }
 
-    if (ctb_shm.IsExisting())
-        ctb_shm.OpenSharedMemory();
+    if (ctb_shm.exists())
+        ctb_shm.openSharedMemory();
 }
 
 void DetectorImpl::setAcquiringFlag(bool flag) { shm()->acquiringFlag = flag; }
@@ -58,8 +58,8 @@ void DetectorImpl::freeSharedMemory(int detectorIndex, int detPos) {
     // single
     if (detPos >= 0) {
         SharedMemory<sharedModule> moduleShm(detectorIndex, detPos);
-        if (moduleShm.IsExisting()) {
-            moduleShm.RemoveSharedMemory();
+        if (moduleShm.exists()) {
+            moduleShm.removeSharedMemory();
         }
         return;
     }
@@ -68,20 +68,20 @@ void DetectorImpl::freeSharedMemory(int detectorIndex, int detPos) {
     SharedMemory<sharedDetector> detectorShm(detectorIndex, -1);
     int numModules = 0;
 
-    if (detectorShm.IsExisting()) {
-        detectorShm.OpenSharedMemory();
+    if (detectorShm.exists()) {
+        detectorShm.openSharedMemory();
         numModules = detectorShm()->numberOfModules;
-        detectorShm.RemoveSharedMemory();
+        detectorShm.removeSharedMemory();
     }
 
     for (int i = 0; i < numModules; ++i) {
         SharedMemory<sharedModule> moduleShm(detectorIndex, i);
-        moduleShm.RemoveSharedMemory();
+        moduleShm.removeSharedMemory();
     }
 
     SharedMemory<CtbConfig> ctbShm(detectorIndex, -1, CtbConfig::shm_tag());
-    if (ctbShm.IsExisting())
-        ctbShm.RemoveSharedMemory();
+    if (ctbShm.exists())
+        ctbShm.removeSharedMemory();
 }
 
 void DetectorImpl::freeSharedMemory() {
@@ -92,11 +92,11 @@ void DetectorImpl::freeSharedMemory() {
     modules.clear();
 
     // clear detector shm
-    shm.RemoveSharedMemory();
+    shm.removeSharedMemory();
     client_downstream = false;
 
-    if (ctb_shm.IsExisting())
-        ctb_shm.RemoveSharedMemory();
+    if (ctb_shm.exists())
+        ctb_shm.removeSharedMemory();
 }
 
 std::string DetectorImpl::getUserDetails() {
@@ -140,11 +140,11 @@ void DetectorImpl::setInitialChecks(const bool value) {
 }
 
 void DetectorImpl::initSharedMemory(bool verify) {
-    if (!shm.IsExisting()) {
-        shm.CreateSharedMemory();
+    if (!shm.exists()) {
+        shm.createSharedMemory();
         initializeDetectorStructure();
     } else {
-        shm.OpenSharedMemory();
+        shm.openSharedMemory();
         if (verify && shm()->shmversion != DETECTOR_SHMVERSION) {
             LOG(logERROR) << "Detector shared memory (" << detectorIndex
                           << ") version mismatch "
@@ -155,6 +155,8 @@ void DetectorImpl::initSharedMemory(bool verify) {
             throw SharedMemoryError("Shared memory version mismatch!");
         }
     }
+
+    // std::cout <<
 }
 
 void DetectorImpl::initializeDetectorStructure() {
@@ -261,10 +263,10 @@ void DetectorImpl::setHostname(const std::vector<std::string> &name) {
     // if needed, CTB dac names are only on detector level
 
     if (shm()->detType == defs::CHIPTESTBOARD) {
-        if (ctb_shm.IsExisting())
-            ctb_shm.OpenSharedMemory();
+        if (ctb_shm.exists())
+            ctb_shm.openSharedMemory();
         else
-            ctb_shm.CreateSharedMemory();
+            ctb_shm.createSharedMemory();
     }
 }
 

slsDetectorSoftware/src/Module.cpp

@@ -33,11 +33,11 @@ Module::Module(detectorType type, int det_id, int module_index, bool verify)
     : moduleIndex(module_index), shm(det_id, module_index) {
 
     // ensure shared memory was not created before
-    if (shm.IsExisting()) {
+    if (shm.exists()) {
         LOG(logWARNING) << "This shared memory should have been "
                            "deleted before! "
-                        << shm.GetName() << ". Freeing it again";
-        shm.RemoveSharedMemory();
+                        << shm.getName() << ". Freeing it again";
+        shm.removeSharedMemory();
     }
 
     initSharedMemory(type, det_id, verify);
@@ -55,8 +55,8 @@ Module::Module(int det_id, int module_index, bool verify)
 Module::~Module() = default;
 
 void Module::freeSharedMemory() {
-    if (shm.IsExisting()) {
-        shm.RemoveSharedMemory();
+    if (shm.exists()) {
+        shm.removeSharedMemory();
     }
 }
 
@@ -3171,20 +3171,20 @@ Ret Module::sendToReceiver(int fnum, const Arg &args) {
 
 slsDetectorDefs::detectorType Module::getDetectorTypeFromShm(int det_id,
                                                              bool verify) {
-    if (!shm.IsExisting()) {
-        throw SharedMemoryError("Shared memory " + shm.GetName() +
+    if (!shm.exists()) {
+        throw SharedMemoryError("Shared memory " + shm.getName() +
                                 "does not exist.\n Corrupted Multi Shared "
                                 "memory. Please free shared memory.");
     }
 
-    shm.OpenSharedMemory();
+    shm.openSharedMemory();
     if (verify && shm()->shmversion != MODULE_SHMVERSION) {
         std::ostringstream ss;
         ss << "Single shared memory (" << det_id << "-" << moduleIndex
            << ":)version mismatch (expected 0x" << std::hex << MODULE_SHMVERSION
            << " but got 0x" << shm()->shmversion << ")" << std::dec
            << ". Clear Shared memory to continue.";
-        shm.UnmapSharedMemory();
+        shm.unmapSharedMemory();
         throw SharedMemoryError(ss.str());
     }
     return shm()->detType;
@@ -3192,11 +3192,11 @@ slsDetectorDefs::detectorType Module::getDetectorTypeFromShm(int det_id,
 
 void Module::initSharedMemory(detectorType type, int det_id, bool verify) {
     shm = SharedMemory<sharedModule>(det_id, moduleIndex);
-    if (!shm.IsExisting()) {
-        shm.CreateSharedMemory();
+    if (!shm.exists()) {
+        shm.createSharedMemory();
         initializeModuleStructure(type);
     } else {
-        shm.OpenSharedMemory();
+        shm.openSharedMemory();
         if (verify && shm()->shmversion != MODULE_SHMVERSION) {
             std::ostringstream ss;
             ss << "Single shared memory (" << det_id << "-" << moduleIndex

slsDetectorSoftware/src/SharedMemory.h

@@ -13,7 +13,8 @@
 #include "sls/logger.h"
 #include "sls/sls_detector_exceptions.h"
 
-#include "stdlib.h"
+// #include "stdlib.h"
+#include <cstdlib>
 #include <cerrno>  // errno
 #include <cstring> // strerror
 #include <fcntl.h> // O_CREAT, O_TRUNC..
@@ -33,268 +34,178 @@
 namespace sls {
 
 template <typename T> class SharedMemory {
+    static constexpr int NAME_MAX_LENGTH = 255;
+    std::string name;
+    T *shared_struct{};
 
   public:
-    //moduleid of -1 creates a detector only shared memory
-    SharedMemory(int detectorId, int moduleIndex, const std::string& tag = "") {
-        name = ConstructSharedMemoryName(detectorId, moduleIndex, tag);
+    // moduleid of -1 creates a detector only shared memory
+    SharedMemory(int detectorId, int moduleIndex, const std::string &tag = "") {
+        name = constructSharedMemoryName(detectorId, moduleIndex, tag);
     }
 
-    /**
-     * Delete the copy constructor and copy assignment since we don't want two
-     * objects managing the same resource
-     */
+    // Disable copy, since we refer to a unique location
     SharedMemory(const SharedMemory &) = delete;
     SharedMemory &operator=(const SharedMemory &other) = delete;
 
-    // Move constructor
     SharedMemory(SharedMemory &&other)
-        : name(other.name), fd(other.fd), shmSize(other.shmSize),
-          shared_struct(other.shared_struct) {
-
-        other.fd = -1;
+        : name(other.name), shared_struct(other.shared_struct) {
         other.shared_struct = nullptr;
-        other.shmSize = 0;
     }
 
-    // Move assignment
     SharedMemory &operator=(SharedMemory &&other) {
         name = other.name;
-        if (fd) {
-            close(fd);
-        }
-        fd = other.fd;
-        other.fd = -1;
-
-        if (shared_struct != nullptr) {
-            UnmapSharedMemory();
-        }
+        if (shared_struct != nullptr)
+            unmapSharedMemory();
         shared_struct = other.shared_struct;
         other.shared_struct = nullptr;
-
-        shmSize = other.shmSize;
-        other.shmSize = 0;
         return *this;
     }
 
     ~SharedMemory() {
-        if (fd >= 0)
-            close(fd);
-
-        if (shared_struct) {
-            UnmapSharedMemory();
-        }
+        if (shared_struct)
+            unmapSharedMemory();
     }
 
-    /**
-     * Verify if it exists
-     * @return true if exists, else false
-     */
-    bool IsExisting() {
-        bool ret = true;
+    T *operator()() { return shared_struct; }
+    const T *operator()() const { return shared_struct; }
+    std::string getName() const { return name; }
+
+    bool exists() {
         int tempfd = shm_open(name.c_str(), O_RDWR, 0);
         if ((tempfd < 0) && (errno == ENOENT)) {
-            ret = false;
+            return false;
         }
         close(tempfd);
-        return ret;
+        return true;
     }
 
-    std::string GetName() const { return name; }
-    size_t size() const { return shmSize; }
-
-    /**
-     * Create Shared memory and call MapSharedMemory to map it to an address
-     * throws a SharedMemoryError exception on failure to create, ftruncate or
-     * map
-     */
-    void CreateSharedMemory() {
-        fd = shm_open(name.c_str(), O_CREAT | O_TRUNC | O_EXCL | O_RDWR,
+    void createSharedMemory() {
+        int fd = shm_open(name.c_str(), O_CREAT | O_TRUNC | O_EXCL | O_RDWR,
                           S_IRUSR | S_IWUSR);
         if (fd < 0) {
             std::string msg =
                 "Create shared memory " + name + " failed: " + strerror(errno);
-            LOG(logERROR) << msg;
             throw SharedMemoryError(msg);
         }
 
         if (ftruncate(fd, sizeof(T)) < 0) {
             std::string msg = "Create shared memory " + name +
                               " failed at ftruncate: " + strerror(errno);
-            LOG(logERROR) << msg;
             close(fd);
-            RemoveSharedMemory();
+            removeSharedMemory();
             throw SharedMemoryError(msg);
         }
-        shared_struct = MapSharedMemory();
+        shared_struct = mapSharedMemory(fd);
         new (shared_struct) T{};
         LOG(logINFO) << "Shared memory created " << name;
     }
 
-    /**
-     * Open existing Shared memory and call MapSharedMemory to map it to an
-     * address throws a SharedMemoryError exception on failure to open or map
-     */
-    void OpenSharedMemory() {
-        fd = shm_open(name.c_str(), O_RDWR, 0);
+    void openSharedMemory() {
+        int fd = shm_open(name.c_str(), O_RDWR, 0);
         if (fd < 0) {
             std::string msg = "Open existing shared memory " + name +
                               " failed: " + strerror(errno);
-            LOG(logERROR) << msg;
             throw SharedMemoryError(msg);
         }
-
-        shared_struct = MapSharedMemory();
+        checkSize(fd);
+        shared_struct = mapSharedMemory(fd);
     }
 
-    /**
-     * Unmap shared memory from an address
-     * throws a SharedMemoryError exception on failure
-     */
-    void UnmapSharedMemory() {
+    void unmapSharedMemory() {
         if (shared_struct != nullptr) {
-            if (munmap(shared_struct, shmSize) < 0) {
+            if (munmap(shared_struct, sizeof(T)) < 0) {
                 std::string msg = "Unmapping shared memory " + name +
                                   " failed: " + strerror(errno);
-                LOG(logERROR) << msg;
-                close(fd);
                 throw SharedMemoryError(msg);
             }
             shared_struct = nullptr;
         }
     }
 
-    /**
-     * Remove existing Shared memory
-     */
-    void RemoveSharedMemory() {
-        UnmapSharedMemory();
+    void removeSharedMemory() {
+        unmapSharedMemory();
         if (shm_unlink(name.c_str()) < 0) {
             // silent exit if shm did not exist anyway
             if (errno == ENOENT)
                 return;
             std::string msg =
                 "Free Shared Memory " + name + " Failed: " + strerror(errno);
-            LOG(logERROR) << msg;
             throw SharedMemoryError(msg);
         }
         LOG(logINFO) << "Shared memory deleted " << name;
     }
 
-    /**
-     * Maximum length of name as from man pages
-     */
-    static const int NAME_MAX_LENGTH = 255;
-
-    /**
-     *Using the call operator to access the pointer
-     */
-    T *operator()() { return shared_struct; }
-
-    /**
-     *Using the call operator to access the pointer, const overload
-     */
-    const T *operator()() const { return shared_struct; }
-
   private:
-    /**
-     * Create Shared memory name
-     * throws exception if name created is longer than required 255(manpages)
-     * @param detectorId  detector id
-     * @param moduleIndex module id, -1 if a detector shared memory
-     * @returns shared memory name
-     */
-    std::string ConstructSharedMemoryName(int detectorId, int moduleIndex, const std::string& tag) {
+    std::string constructSharedMemoryName(int detectorId, int moduleIndex,
+                                          const std::string &tag) {
 
-        // using environment path
-        std::string sEnvPath;
+        // using environment variable
+        std::string slsdetname;
         char *envpath = getenv(SHM_ENV_NAME);
         if (envpath != nullptr) {
-            sEnvPath.assign(envpath);
-            sEnvPath.insert(0, "_");
+            slsdetname = envpath;
+            slsdetname.insert(0, "_");
         }
 
         std::stringstream ss;
-        if (moduleIndex < 0){
-            ss << SHM_DETECTOR_PREFIX << detectorId << sEnvPath;
+        if (moduleIndex < 0) {
+            ss << SHM_DETECTOR_PREFIX << detectorId << slsdetname;
             if (!tag.empty())
                 ss << "_" << tag;
+        } else {
+            ss << SHM_DETECTOR_PREFIX << detectorId << SHM_MODULE_PREFIX
+               << moduleIndex << slsdetname;
         }
 
-        else
-            ss << SHM_DETECTOR_PREFIX << detectorId << SHM_MODULE_PREFIX
-               << moduleIndex << sEnvPath;
-
-        std::string temp = ss.str();
-        if (temp.length() > NAME_MAX_LENGTH) {
+        std::string shm_name = ss.str();
+        if (shm_name.length() > NAME_MAX_LENGTH) {
             std::string msg =
-                "Shared memory initialization failed. " + temp + " has " +
-                std::to_string(temp.length()) + " characters. \n" +
+                "Shared memory initialization failed. " + shm_name + " has " +
+                std::to_string(shm_name.length()) + " characters. \n" +
                 "Maximum is " + std::to_string(NAME_MAX_LENGTH) +
                 ". Change the environment variable " + SHM_ENV_NAME;
-            LOG(logERROR) << msg;
             throw SharedMemoryError(msg);
         }
-        return temp;
+        return shm_name;
     }
 
-    /**
-     * Map shared memory to an address
-     * throws a SharedMemoryException exception on failure
-     */
-
-    T *MapSharedMemory() {
+    // from the Linux manual:
+    // After the mmap() call has returned, the file descriptor, fd, can
+    // be closed immediately without invalidating the mapping.
+    T *mapSharedMemory(int fd) {
         void *addr =
             mmap(nullptr, sizeof(T), PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+        close(fd);
         if (addr == MAP_FAILED) {
             std::string msg =
                 "Mapping shared memory " + name + " failed: " + strerror(errno);
-            LOG(logERROR) << msg;
-            close(fd);
             throw SharedMemoryError(msg);
         }
-        shmSize = sizeof(T);
-        close(fd);
-        return (T *)addr;
+        return static_cast<T *>(addr);
     }
 
-    /**
-     * Verify if existing shared memory size matches expected size
-     * @param expectedSize expected size of shared memory, replaced with smaller
-     * size if size does not match
-     * @return 0 for success, 1 for fail
-     */
-    int VerifySizeMatch(size_t expectedSize) {
+    void checkSize(int fd) {
         struct stat sb;
-        // could not fstat
         if (fstat(fd, &sb) < 0) {
             std::string msg = "Could not verify existing shared memory " +
                               name + " size match " +
                               "(could not fstat): " + strerror(errno);
-            LOG(logERROR) << msg;
             close(fd);
             throw SharedMemoryError(msg);
         }
 
-        // size does not match
-        auto sz = static_cast<size_t>(sb.st_size);
-        if (sz != expectedSize) {
-            std::string msg = "Existing shared memory " + name +
-                              " size does not match" + "Expected " +
-                              std::to_string(expectedSize) + ", found " +
-                              std::to_string(sz);
-            LOG(logERROR) << msg;
+        auto actual_size = static_cast<size_t>(sb.st_size);
+        auto expected_size = sizeof(T);
+        if (actual_size != expected_size) {
+            std::string msg =
+                "Existing shared memory " + name + " size does not match. " +
+                "Expected " + std::to_string(expected_size) + ", found " +
+                std::to_string(actual_size) +
+                ". Detector software mismatch? Try freeing shared memory.";
             throw SharedMemoryError(msg);
-            return 1;
         }
-        return 0;
     }
-
-
-    std::string name;
-    int fd{-1};
-    size_t shmSize{0};
-    T *shared_struct{nullptr};
 };
 
 } // namespace sls

slsDetectorSoftware/tests/test-Module.cpp

@@ -32,8 +32,8 @@ TEST_CASE("Is shm fixed pattern shm compatible") {
 
     // Set shm version to 0
     sls::SharedMemory<sls::sharedModule> shm(0, 0);
-    REQUIRE(shm.IsExisting() == true);
-    shm.OpenSharedMemory();
+    REQUIRE(shm.exists() == true);
+    shm.openSharedMemory();
     shm()->shmversion = 0;
 
     // Should fail since version is set to 0

slsDetectorSoftware/tests/test-SharedMemory.cpp

@@ -20,8 +20,8 @@ constexpr int shm_id = 10;
 TEST_CASE("Create SharedMemory read and write", "[detector]") {
 
     SharedMemory<Data> shm(shm_id, -1);
-    shm.CreateSharedMemory();
-    CHECK(shm.GetName() == std::string("/slsDetectorPackage_detector_") +
+    shm.createSharedMemory();
+    CHECK(shm.getName() == std::string("/slsDetectorPackage_detector_") +
                                std::to_string(shm_id));
 
     shm()->x = 3;
@@ -32,25 +32,25 @@ TEST_CASE("Create SharedMemory read and write", "[detector]") {
     CHECK(shm()->y == 5.7);
     CHECK(std::string(shm()->mess) == "Some string");
 
-    shm.UnmapSharedMemory();
-    shm.RemoveSharedMemory();
+    shm.unmapSharedMemory();
+    shm.removeSharedMemory();
 
-    CHECK(shm.IsExisting() == false);
+    CHECK(shm.exists() == false);
 }
 
 TEST_CASE("Open existing SharedMemory and read", "[detector]") {
 
     {
         SharedMemory<double> shm(shm_id, -1);
-        shm.CreateSharedMemory();
+        shm.createSharedMemory();
         *shm() = 5.3;
     }
 
     SharedMemory<double> shm2(shm_id, -1);
-    shm2.OpenSharedMemory();
+    shm2.openSharedMemory();
     CHECK(*shm2() == 5.3);
 
-    shm2.RemoveSharedMemory();
+    shm2.removeSharedMemory();
 }
 
 TEST_CASE("Creating a second shared memory with the same name throws",
@@ -59,24 +59,24 @@ TEST_CASE("Creating a second shared memory with the same name throws",
     SharedMemory<double> shm0(shm_id, -1);
     SharedMemory<double> shm1(shm_id, -1);
 
-    shm0.CreateSharedMemory();
-    CHECK_THROWS(shm1.CreateSharedMemory());
-    shm0.RemoveSharedMemory();
+    shm0.createSharedMemory();
+    CHECK_THROWS(shm1.createSharedMemory());
+    shm0.removeSharedMemory();
 }
 
 TEST_CASE("Open two shared memories to the same place", "[detector]") {
 
     // Create the first shared memory
     SharedMemory<Data> shm(shm_id, -1);
-    shm.CreateSharedMemory();
+    shm.createSharedMemory();
     shm()->x = 5;
     CHECK(shm()->x == 5);
 
     // Open the second shared memory with the same name
     SharedMemory<Data> shm2(shm_id, -1);
-    shm2.OpenSharedMemory();
+    shm2.openSharedMemory();
     CHECK(shm2()->x == 5);
-    CHECK(shm.GetName() == shm2.GetName());
+    CHECK(shm.getName() == shm2.getName());
 
     // Check that they still point to the same place
     shm2()->x = 7;
@@ -84,31 +84,28 @@ TEST_CASE("Open two shared memories to the same place", "[detector]") {
 
     // Remove only needs to be done once since they refer
     // to the same memory
-    shm2.RemoveSharedMemory();
-    CHECK(shm.IsExisting() == false);
-    CHECK(shm2.IsExisting() == false);
+    shm2.removeSharedMemory();
+    CHECK(shm.exists() == false);
+    CHECK(shm2.exists() == false);
 }
 
 TEST_CASE("Move SharedMemory", "[detector]") {
 
     SharedMemory<Data> shm(shm_id, -1);
-    CHECK(shm.GetName() == std::string("/slsDetectorPackage_detector_") +
+    CHECK(shm.getName() == std::string("/slsDetectorPackage_detector_") +
                                std::to_string(shm_id));
-    shm.CreateSharedMemory();
+    shm.createSharedMemory();
     shm()->x = 9;
 
-    CHECK(shm.size() == sizeof(Data));
 
     SharedMemory<Data> shm2(shm_id + 1, -1);
     shm2 = std::move(shm); // shm is now a moved from object!
 
     CHECK(shm2()->x == 9);
     CHECK(shm() == nullptr);
-    CHECK(shm.size() == 0);
-
-    CHECK(shm2.GetName() == std::string("/slsDetectorPackage_detector_") +
+    CHECK(shm2.getName() == std::string("/slsDetectorPackage_detector_") +
                                 std::to_string(shm_id));
-    shm2.RemoveSharedMemory();
+    shm2.removeSharedMemory();
 }
 
 TEST_CASE("Create several shared memories", "[detector]") {
@@ -117,27 +114,27 @@ TEST_CASE("Create several shared memories", "[detector]") {
     v.reserve(N);
     for (int i = 0; i != N; ++i) {
         v.emplace_back(shm_id + i, -1);
-        CHECK(v[i].IsExisting() == false);
-        v[i].CreateSharedMemory();
+        CHECK(v[i].exists() == false);
+        v[i].createSharedMemory();
         *v[i]() = i;
         CHECK(*v[i]() == i);
     }
 
     for (int i = 0; i != N; ++i) {
         CHECK(*v[i]() == i);
-        CHECK(v[i].GetName() == std::string("/slsDetectorPackage_detector_") +
+        CHECK(v[i].getName() == std::string("/slsDetectorPackage_detector_") +
                                     std::to_string(i + shm_id));
     }
 
     for (int i = 0; i != N; ++i) {
-        v[i].RemoveSharedMemory();
-        CHECK(v[i].IsExisting() == false);
+        v[i].removeSharedMemory();
+        CHECK(v[i].exists() == false);
     }
 }
 
 TEST_CASE("Create create a shared memory with a tag"){
     SharedMemory<int> shm(0, -1, "ctbdacs");
-    REQUIRE(shm.GetName() == "/slsDetectorPackage_detector_0_ctbdacs");
+    REQUIRE(shm.getName() == "/slsDetectorPackage_detector_0_ctbdacs");
 
 }
 
@@ -152,7 +149,7 @@ TEST_CASE("Create create a shared memory with a tag when SLSDETNAME is set"){
     setenv(SHM_ENV_NAME, "myprefix", 1);
 
     SharedMemory<int> shm(0, -1, "ctbdacs");
-    REQUIRE(shm.GetName() == "/slsDetectorPackage_detector_0_myprefix_ctbdacs");
+    REQUIRE(shm.getName() == "/slsDetectorPackage_detector_0_myprefix_ctbdacs");
 
     // Clean up after us
     if (old_slsdetname.empty())
@@ -162,3 +159,17 @@ TEST_CASE("Create create a shared memory with a tag when SLSDETNAME is set"){
 
 }
 
+TEST_CASE("map int64 to int32 throws"){
+    SharedMemory<int32_t> shm(shm_id, -1);
+    shm.createSharedMemory();
+    *shm() = 7;
+
+    SharedMemory<int64_t> shm2(shm_id, -1);
+    REQUIRE_THROWS(shm2.openSharedMemory());
+
+    shm.removeSharedMemory();
+    
+
+
+
+}

slsReceiverSoftware/CMakeLists.txt

@@ -6,7 +6,6 @@ set(SOURCES
     src/Receiver.cpp
     src/File.cpp
     src/BinaryDataFile.cpp
-    src/BinaryMasterFile.cpp
     src/ThreadObject.cpp
     src/Listener.cpp
     src/DataProcessor.cpp
@@ -14,6 +13,7 @@ set(SOURCES
     src/Fifo.cpp
     src/Arping.cpp
     src/MasterAttributes.cpp
+    src/MasterFileUtility.cpp
 )
 
 set(PUBLICHEADERS
@@ -28,8 +28,6 @@ if (SLS_USE_HDF5)
 	    )
 	    list (APPEND SOURCES 
 	        src/HDF5DataFile.cpp 
-	        src/HDF5MasterFile.cpp 
-	        src/HDF5VirtualFile.cpp 
 	    )
 endif (SLS_USE_HDF5)
 

slsReceiverSoftware/src/BinaryDataFile.cpp

@@ -72,7 +72,7 @@ void BinaryDataFile::WriteToFile(char *buffer, const int buffersize,
         ++subFileIndex_;
         CreateFile();
     }
-    numFramesInFile_++;
+    ++numFramesInFile_;
 
     // write to file
     int ret = 0;

slsReceiverSoftware/src/BinaryMasterFile.cpp

@@ -1,60 +0,0 @@
-// SPDX-License-Identifier: LGPL-3.0-or-other
-// Copyright (C) 2021 Contributors to the SLS Detector Package
-#include "BinaryMasterFile.h"
-#include "MasterAttributes.h"
-
-BinaryMasterFile::BinaryMasterFile() : File(BINARY) {}
-
-BinaryMasterFile::~BinaryMasterFile() { CloseFile(); }
-
-void BinaryMasterFile::CloseFile() {
-    if (fd_) {
-        fclose(fd_);
-    }
-    fd_ = nullptr;
-}
-
-void BinaryMasterFile::CreateMasterFile(const std::string filePath,
-                                        const std::string fileNamePrefix,
-                                        const uint64_t fileIndex,
-                                        const bool overWriteEnable,
-                                        const bool silentMode,
-                                        MasterAttributes *attr) {
-    // create file name
-    std::ostringstream os;
-    os << filePath << "/" << fileNamePrefix << "_master"
-       << "_" << fileIndex << ".raw";
-    fileName_ = os.str();
-
-    // create file
-    if (!overWriteEnable) {
-        if (nullptr == (fd_ = fopen((const char *)fileName_.c_str(), "wx"))) {
-            fd_ = nullptr;
-            throw sls::RuntimeError("Could not create binary master file " +
-                                    fileName_);
-        }
-    } else if (nullptr == (fd_ = fopen((const char *)fileName_.c_str(), "w"))) {
-        fd_ = nullptr;
-        throw sls::RuntimeError(
-            "Could not create/overwrite binary master file " + fileName_);
-    }
-    if (!silentMode) {
-        LOG(logINFO) << "Master File: " << fileName_;
-    }
-    attr->WriteMasterBinaryAttributes(fd_);
-    CloseFile();
-}
-
-void BinaryMasterFile::UpdateMasterFile(MasterAttributes *attr,
-                                        bool silentMode) {
-    if (nullptr == (fd_ = fopen((const char *)fileName_.c_str(), "a"))) {
-        fd_ = nullptr;
-        throw sls::RuntimeError("Could not append binary master file " +
-                                fileName_);
-    }
-    attr->WriteFinalBinaryAttributes(fd_);
-    CloseFile();
-    if (!silentMode) {
-        LOG(logINFO) << "Updated Master File";
-    }
-}

slsReceiverSoftware/src/BinaryMasterFile.h

@@ -1,25 +0,0 @@
-// SPDX-License-Identifier: LGPL-3.0-or-other
-// Copyright (C) 2021 Contributors to the SLS Detector Package
-#pragma once
-
-#include "File.h"
-#include "MasterAttributes.h"
-
-class BinaryMasterFile : private virtual slsDetectorDefs, public File {
-
-  public:
-    BinaryMasterFile();
-    ~BinaryMasterFile();
-
-    void CloseFile() override;
-    void CreateMasterFile(const std::string filePath,
-                          const std::string fileNamePrefix,
-                          const uint64_t fileIndex, const bool overWriteEnable,
-                          const bool silentMode,
-                          MasterAttributes *attr) override;
-    void UpdateMasterFile(MasterAttributes *attr, bool silentMode) override;
-
-  private:
-    FILE *fd_{nullptr};
-    std::string fileName_;
-};

slsReceiverSoftware/src/DataProcessor.cpp

@@ -9,14 +9,12 @@
 
 #include "DataProcessor.h"
 #include "BinaryDataFile.h"
-#include "BinaryMasterFile.h"
 #include "Fifo.h"
 #include "GeneralData.h"
 #include "MasterAttributes.h"
+#include "MasterFileUtility.h"
 #ifdef HDF5C
 #include "HDF5DataFile.h"
-#include "HDF5MasterFile.h"
-#include "HDF5VirtualFile.h"
 #endif
 #include "DataStreamer.h"
 #include "sls/container_utils.h"
@@ -82,12 +80,6 @@ void DataProcessor::SetGeneralData(GeneralData *generalData) {
 void DataProcessor::CloseFiles() {
     if (dataFile_)
         dataFile_->CloseFile();
-    if (masterFile_)
-        masterFile_->CloseFile();
-#ifdef HDF5C
-    if (virtualFile_)
-        virtualFile_->CloseFile();
-#endif
 }
 
 void DataProcessor::DeleteFiles() {
@@ -96,39 +88,20 @@ void DataProcessor::DeleteFiles() {
         delete dataFile_;
         dataFile_ = nullptr;
     }
-    if (masterFile_) {
-        delete masterFile_;
-        masterFile_ = nullptr;
-    }
-#ifdef HDF5C
-    if (virtualFile_) {
-        delete virtualFile_;
-        virtualFile_ = nullptr;
-    }
-#endif
 }
 void DataProcessor::SetupFileWriter(const bool filewriteEnable,
-                                    const bool masterFilewriteEnable,
                                     const fileFormat fileFormatType,
-                                    const int modulePos, std::mutex *hdf5Lib) {
+                                    std::mutex *hdf5LibMutex) {
     DeleteFiles();
     if (filewriteEnable) {
         switch (fileFormatType) {
 #ifdef HDF5C
         case HDF5:
-            dataFile_ = new HDF5DataFile(index, hdf5Lib);
-            if (modulePos == 0 && index == 0) {
-                if (masterFilewriteEnable) {
-                    masterFile_ = new HDF5MasterFile(hdf5Lib);
-                }
-            }
+            dataFile_ = new HDF5DataFile(index, hdf5LibMutex);
             break;
 #endif
         case BINARY:
             dataFile_ = new BinaryDataFile(index);
-            if (modulePos == 0 && index == 0 && masterFilewriteEnable) {
-                masterFile_ = new BinaryMasterFile();
-            }
             break;
         default:
             throw sls::RuntimeError(
@@ -138,23 +111,17 @@ void DataProcessor::SetupFileWriter(const bool filewriteEnable,
 }
 
 void DataProcessor::CreateFirstFiles(
-    MasterAttributes *attr, const std::string filePath,
-    const std::string fileNamePrefix, const uint64_t fileIndex,
-    const bool overWriteEnable, const bool silentMode, const int modulePos,
-    const int numUnitsPerReadout, const uint32_t udpPortNumber,
-    const uint32_t maxFramesPerFile, const uint64_t numImages,
-    const uint32_t dynamicRange, const bool detectorDataStream) {
+    const std::string &filePath, const std::string &fileNamePrefix,
+    const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode,
+    const int modulePos, const int numUnitsPerReadout,
+    const uint32_t udpPortNumber, const uint32_t maxFramesPerFile,
+    const uint64_t numImages, const uint32_t dynamicRange,
+    const bool detectorDataStream) {
     if (dataFile_ == nullptr) {
         throw sls::RuntimeError("file object not contstructed");
     }
     CloseFiles();
 
-    // master file write enabled
-    if (masterFile_) {
-        masterFile_->CreateMasterFile(filePath, fileNamePrefix, fileIndex,
-                                      overWriteEnable, silentMode, attr);
-    }
-
     // deactivated (half module/ single port), dont write file
     if ((!*activated_) || (!detectorDataStream)) {
         return;
@@ -189,21 +156,17 @@ uint32_t DataProcessor::GetFilesInAcquisition() const {
     return dataFile_->GetFilesInAcquisition();
 }
 
-void DataProcessor::CreateVirtualFile(
-    const std::string filePath, const std::string fileNamePrefix,
+std::array<std::string, 2> DataProcessor::CreateVirtualFile(
+    const std::string &filePath, const std::string &fileNamePrefix,
     const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode,
     const int modulePos, const int numUnitsPerReadout,
     const uint32_t maxFramesPerFile, const uint64_t numImages,
-    const uint32_t dynamicRange, const int numModX, const int numModY,
-    std::mutex *hdf5Lib) {
+    const int numModX, const int numModY, const uint32_t dynamicRange,
+    std::mutex *hdf5LibMutex) {
 
-    if (virtualFile_) {
-        delete virtualFile_;
-    }
     bool gotthard25um =
         ((detectorType_ == GOTTHARD || detectorType_ == GOTTHARD2) &&
          (numModX * numModY) == 2);
-    virtualFile_ = new HDF5VirtualFile(hdf5Lib, gotthard25um);
 
     // maxframesperfile = 0 for infinite files
     uint32_t framesPerFile =
@@ -213,63 +176,55 @@ void DataProcessor::CreateVirtualFile(
     // files (they exist anyway) assumption2: virtual file max frame index
     // is from R0 P0 (difference from others when missing frames or for a
     // stop acquisition)
-    virtualFile_->CreateVirtualFile(
+    return masterFileUtility::CreateVirtualHDF5File(
         filePath, fileNamePrefix, fileIndex, overWriteEnable, silentMode,
         modulePos, numUnitsPerReadout, framesPerFile, numImages,
         generalData_->nPixelsX, generalData_->nPixelsY, dynamicRange,
         numFramesCaught_, numModX, numModY, dataFile_->GetPDataType(),
-        dataFile_->GetParameterNames(), dataFile_->GetParameterDataTypes());
+        dataFile_->GetParameterNames(), dataFile_->GetParameterDataTypes(),
+        hdf5LibMutex, gotthard25um);
 }
 
-void DataProcessor::LinkDataInMasterFile(const bool silentMode) {
-    std::string fname, datasetName;
-    if (virtualFile_) {
-        auto res = virtualFile_->GetFileAndDatasetName();
-        fname = res[0];
-        datasetName = res[1];
-    } else {
+void DataProcessor::LinkFileInMaster(const std::string &masterFileName,
+                                     const std::string &virtualFileName,
+                                     const std::string &virtualDatasetName,
+                                     const bool silentMode,
+                                     std::mutex *hdf5LibMutex) {
+    std::string fname{virtualFileName}, datasetName{virtualDatasetName};
+    // if no virtual file, link data file
+    if (virtualFileName.empty()) {
         auto res = dataFile_->GetFileAndDatasetName();
         fname = res[0];
         datasetName = res[1];
     }
-    // link in master
-    masterFile_->LinkDataFile(fname, datasetName,
-                              dataFile_->GetParameterNames(), silentMode);
+    masterFileUtility::LinkHDF5FileInMaster(masterFileName, fname, datasetName,
+                                            dataFile_->GetParameterNames(),
+                                            silentMode, hdf5LibMutex);
 }
 #endif
 
-void DataProcessor::UpdateMasterFile(bool silentMode) {
-    if (masterFile_) {
-        // final attributes
-        std::unique_ptr<MasterAttributes> masterAttributes;
-        switch (detectorType_) {
-        case GOTTHARD:
-            masterAttributes = sls::make_unique<GotthardMasterAttributes>();
-            break;
-        case JUNGFRAU:
-            masterAttributes = sls::make_unique<JungfrauMasterAttributes>();
-            break;
-        case EIGER:
-            masterAttributes = sls::make_unique<EigerMasterAttributes>();
-            break;
-        case MYTHEN3:
-            masterAttributes = sls::make_unique<Mythen3MasterAttributes>();
-            break;
-        case GOTTHARD2:
-            masterAttributes = sls::make_unique<Gotthard2MasterAttributes>();
-            break;
-        case MOENCH:
-            masterAttributes = sls::make_unique<MoenchMasterAttributes>();
-            break;
-        case CHIPTESTBOARD:
-            masterAttributes = sls::make_unique<CtbMasterAttributes>();
-            break;
+std::string DataProcessor::CreateMasterFile(
+    const std::string &filePath, const std::string &fileNamePrefix,
+    const uint64_t fileIndex, const bool overWriteEnable, bool silentMode,
+    const fileFormat fileFormatType, MasterAttributes *attr,
+    std::mutex *hdf5LibMutex) {
+
+    attr->framesInFile = numFramesCaught_;
+
+    std::unique_ptr<File> masterFile{nullptr};
+    switch (fileFormatType) {
+#ifdef HDF5C
+    case HDF5:
+        return masterFileUtility::CreateMasterHDF5File(
+            filePath, fileNamePrefix, fileIndex, overWriteEnable, silentMode,
+            attr, hdf5LibMutex);
+#endif
+    case BINARY:
+        return masterFileUtility::CreateMasterBinaryFile(
+            filePath, fileNamePrefix, fileIndex, overWriteEnable, silentMode,
+            attr);
     default:
-            throw sls::RuntimeError(
-                "Unknown detector type to set up master file attributes");
-        }
-        masterAttributes->framesInFile = numFramesCaught_;
-        masterFile_->UpdateMasterFile(masterAttributes.get(), silentMode);
+        throw sls::RuntimeError("Unknown file format (compile with hdf5 flags");
     }
 }
 

slsReceiverSoftware/src/DataProcessor.h

@@ -45,12 +45,11 @@ class DataProcessor : private virtual slsDetectorDefs, public ThreadObject {
     void CloseFiles();
     void DeleteFiles();
     void SetupFileWriter(const bool filewriteEnable,
-                         const bool masterFilewriteEnable,
-                         const fileFormat fileFormatType, const int modulePos,
-                         std::mutex *hdf5Lib);
+                         const fileFormat fileFormatType,
+                         std::mutex *hdf5LibMutex);
 
-    void CreateFirstFiles(MasterAttributes *attr, const std::string filePath,
-                          const std::string fileNamePrefix,
+    void CreateFirstFiles(const std::string &filePath,
+                          const std::string &fileNamePrefix,
                           const uint64_t fileIndex, const bool overWriteEnable,
                           const bool silentMode, const int modulePos,
                           const int numUnitsPerReadout,
@@ -60,18 +59,26 @@ class DataProcessor : private virtual slsDetectorDefs, public ThreadObject {
                           const bool detectorDataStream);
 #ifdef HDF5C
     uint32_t GetFilesInAcquisition() const;
-    void CreateVirtualFile(const std::string filePath,
-                           const std::string fileNamePrefix,
+    std::array<std::string, 2> CreateVirtualFile(
+        const std::string &filePath, const std::string &fileNamePrefix,
         const uint64_t fileIndex, const bool overWriteEnable,
         const bool silentMode, const int modulePos,
-                           const int numUnitsPerReadout,
-                           const uint32_t maxFramesPerFile,
-                           const uint64_t numImages,
-                           const uint32_t dynamicRange, const int numModX,
-                           const int numModY, std::mutex *hdf5Lib);
-    void LinkDataInMasterFile(const bool silentMode);
+        const int numUnitsPerReadout, const uint32_t maxFramesPerFile,
+        const uint64_t numImages, const int numModX, const int numModY,
+        const uint32_t dynamicRange, std::mutex *hdf5LibMutex);
+    void LinkFileInMaster(const std::string &masterFileName,
+                          const std::string &virtualFileName,
+                          const std::string &virtualDatasetName,
+                          const bool silentMode, std::mutex *hdf5LibMutex);
 #endif
-    void UpdateMasterFile(bool silentMode);
+
+    std::string CreateMasterFile(const std::string &filePath,
+                                 const std::string &fileNamePrefix,
+                                 const uint64_t fileIndex,
+                                 const bool overWriteEnable, bool silentMode,
+                                 const fileFormat fileFormatType,
+                                 MasterAttributes *attr,
+                                 std::mutex *hdf5LibMutex);
     /**
      * Call back for raw data
      * args to raw data ready callback are
@@ -179,10 +186,6 @@ class DataProcessor : private virtual slsDetectorDefs, public ThreadObject {
     bool firstStreamerFrame_{false};
 
     File *dataFile_{nullptr};
-    File *masterFile_{nullptr};
-#ifdef HDF5C
-    File *virtualFile_{nullptr};
-#endif
 
     // call back
     /**

slsReceiverSoftware/src/File.h

@@ -60,20 +60,6 @@ class File : private virtual slsDetectorDefs {
         return std::vector<DataType>{};
     };
 
-    virtual void CreateVirtualFile(
-        const std::string filePath, const std::string fileNamePrefix,
-        const uint64_t fileIndex, const bool overWriteEnable,
-        const bool silentMode, const int modulePos,
-        const int numUnitsPerReadout, const uint32_t maxFramesPerFile,
-        const uint64_t numImages, const uint32_t nPixelsX,
-        const uint32_t nPixelsY, const uint32_t dynamicRange,
-        const uint64_t numImagesCaught, const int numModX, const int numModY,
-        const DataType dataType, const std::vector<std::string> parameterNames,
-        const std::vector<DataType> parameterDataTypes) {
-        LOG(logERROR) << "This is a generic function CreateVirtualFile that "
-                         "should be overloaded by a derived class";
-    }
-
     virtual void CreateFirstHDF5DataFile(
         const std::string filePath, const std::string fileNamePrefix,
         const uint64_t fileIndex, const bool overWriteEnable,
@@ -85,13 +71,6 @@ class File : private virtual slsDetectorDefs {
         LOG(logERROR) << "This is a generic function CreateFirstDataFile that "
                          "should be overloaded by a derived class";
     };
-
-    virtual void LinkDataFile(std::string dataFilename, std::string dataSetname,
-                              const std::vector<std::string> parameterNames,
-                              const bool silentMode) {
-        LOG(logERROR) << "This is a generic function LinkDataFile that "
-                         "should be overloaded by a derived class";
-    };
 #endif
     virtual void CreateFirstBinaryDataFile(
         const std::string filePath, const std::string fileNamePrefix,
@@ -103,27 +82,9 @@ class File : private virtual slsDetectorDefs {
                          "should be overloaded by a derived class";
     };
 
-    virtual void CreateMasterFile(const std::string filePath,
-                                  const std::string fileNamePrefix,
-                                  const uint64_t fileIndex,
-                                  const bool overWriteEnable,
-                                  const bool silentMode,
-                                  MasterAttributes *attr) {
-        LOG(logERROR) << "This is a generic function CreateMasterFile that "
-                         "should be overloaded by a derived class";
-    };
-
     virtual void WriteToFile(char *buffer, const int buffersize,
                              const uint64_t currentFrameNumber,
-                             const uint32_t numPacketsCaught) {
-        LOG(logERROR) << "This is a generic function WriteToFile that "
-                         "should be overloaded by a derived class";
-    };
-
-    virtual void UpdateMasterFile(MasterAttributes *attr, bool silentMode) {
-        LOG(logERROR) << "This is a generic function UpdateMasterFile that "
-                         "should be overloaded by a derived class";
-    };
+                             const uint32_t numPacketsCaught) = 0;
 
   protected:
     slsDetectorDefs::fileFormat format_;

slsReceiverSoftware/src/HDF5DataFile.cpp

@@ -128,7 +128,7 @@ void HDF5DataFile::CreateFirstHDF5DataFile(
 }
 
 void HDF5DataFile::CreateFile() {
-
+    numFramesInFile_ = 0;
     numFilesInAcquisition_++;
 
     std::ostringstream os;
@@ -237,7 +237,7 @@ void HDF5DataFile::WriteToFile(char *buffer, const int buffersize,
         ++subFileIndex_;
         CreateFile();
     }
-    numFramesInFile_++;
+    ++numFramesInFile_;
 
     // extend dataset (when receiver start followed by many status starts
     // (jungfrau)))

slsReceiverSoftware/src/HDF5MasterFile.cpp

@@ -1,162 +0,0 @@
-// SPDX-License-Identifier: LGPL-3.0-or-other
-// Copyright (C) 2021 Contributors to the SLS Detector Package
-#include "HDF5MasterFile.h"
-#include "MasterAttributes.h"
-
-HDF5MasterFile::HDF5MasterFile(std::mutex *hdf5Lib)
-    : File(HDF5), hdf5Lib_(hdf5Lib) {}
-
-HDF5MasterFile::~HDF5MasterFile() { CloseFile(); }
-
-void HDF5MasterFile::CloseFile() {
-    std::lock_guard<std::mutex> lock(*hdf5Lib_);
-    try {
-        Exception::dontPrint(); // to handle errors
-        if (fd_) {
-            fd_->close();
-            delete fd_;
-            fd_ = nullptr;
-        }
-    } catch (const Exception &error) {
-        LOG(logERROR) << "Could not close master HDF5 handles";
-        error.printErrorStack();
-    }
-}
-
-void HDF5MasterFile::LinkDataFile(std::string dataFilename,
-                                  std::string dataSetname,
-                                  const std::vector<std::string> parameterNames,
-                                  const bool silentMode) {
-
-    std::lock_guard<std::mutex> lock(*hdf5Lib_);
-    try {
-        Exception::dontPrint(); // to handle errors
-
-        FileAccPropList flist;
-        flist.setFcloseDegree(H5F_CLOSE_STRONG);
-
-        // open master file
-        H5File masterfd(fileName_.c_str(), H5F_ACC_RDWR,
-                        FileCreatPropList::DEFAULT, flist);
-
-        // open data file
-        H5File fd(dataFilename.c_str(), H5F_ACC_RDONLY,
-                  FileCreatPropList::DEFAULT, flist);
-
-        // create link for data dataset
-        DataSet dset = fd.openDataSet(dataSetname.c_str());
-        std::string linkname = std::string("/entry/data/") + dataSetname;
-        if (H5Lcreate_external(dataFilename.c_str(), dataSetname.c_str(),
-                               masterfd.getLocId(), linkname.c_str(),
-                               H5P_DEFAULT, H5P_DEFAULT) < 0) {
-            throw sls::RuntimeError(
-                "Could not create link to data dataset in master");
-        }
-
-        // create link for parameter datasets
-        for (unsigned int i = 0; i < parameterNames.size(); ++i) {
-            DataSet pDset = fd.openDataSet(parameterNames[i].c_str());
-            linkname = std::string("/entry/data/") + parameterNames[i];
-            if (H5Lcreate_external(dataFilename.c_str(),
-                                   parameterNames[i].c_str(),
-                                   masterfd.getLocId(), linkname.c_str(),
-                                   H5P_DEFAULT, H5P_DEFAULT) < 0) {
-                throw sls::RuntimeError(
-                    "Could not create link to parameter dataset in master");
-            }
-        }
-        fd.close();
-        masterfd.close();
-    } catch (const Exception &error) {
-        error.printErrorStack();
-        CloseFile();
-        throw sls::RuntimeError("Could not link in master hdf5 file");
-    }
-    if (!silentMode) {
-        LOG(logINFO) << "Linked in Master File: " << dataFilename;
-    }
-}
-
-void HDF5MasterFile::CreateMasterFile(const std::string filePath,
-                                      const std::string fileNamePrefix,
-                                      const uint64_t fileIndex,
-                                      const bool overWriteEnable,
-                                      const bool silentMode,
-                                      MasterAttributes *attr) {
-
-    std::ostringstream os;
-    os << filePath << "/" << fileNamePrefix << "_master"
-       << "_" << fileIndex << ".h5";
-    fileName_ = os.str();
-
-    std::lock_guard<std::mutex> lock(*hdf5Lib_);
-
-    try {
-        Exception::dontPrint(); // to handle errors
-
-        FileAccPropList flist;
-        flist.setFcloseDegree(H5F_CLOSE_STRONG);
-        fd_ = nullptr;
-        if (!(overWriteEnable))
-            fd_ = new H5File(fileName_.c_str(), H5F_ACC_EXCL,
-                             FileCreatPropList::DEFAULT, flist);
-        else
-            fd_ = new H5File(fileName_.c_str(), H5F_ACC_TRUNC,
-                             FileCreatPropList::DEFAULT, flist);
-
-        // attributes - version
-        double dValue = HDF5_WRITER_VERSION;
-        DataSpace dataspace_attr = DataSpace(H5S_SCALAR);
-        Attribute attribute = fd_->createAttribute(
-            "version", PredType::NATIVE_DOUBLE, dataspace_attr);
-        attribute.write(PredType::NATIVE_DOUBLE, &dValue);
-
-        // Create a group in the file
-        Group group1(fd_->createGroup("entry"));
-        Group group2(group1.createGroup("data"));
-        Group group3(group1.createGroup("instrument"));
-        Group group4(group3.createGroup("beam"));
-        Group group5(group3.createGroup("detector"));
-        Group group6(group1.createGroup("sample"));
-
-        // TODO find a way to get complete group link
-        attrGroupName_ = "/entry/instrument/detector";
-
-        attr->WriteMasterHDF5Attributes(fd_, &group5);
-        fd_->close();
-
-    } catch (const Exception &error) {
-        error.printErrorStack();
-        CloseFile();
-        throw sls::RuntimeError(
-            "Could not create/overwrite master HDF5 handles");
-    }
-    if (!silentMode) {
-        LOG(logINFO) << "Master File: " << fileName_;
-    }
-}
-
-void HDF5MasterFile::UpdateMasterFile(MasterAttributes *attr, bool silentMode) {
-    std::lock_guard<std::mutex> lock(*hdf5Lib_);
-
-    try {
-        Exception::dontPrint(); // to handle errors
-        FileAccPropList flist;
-        flist.setFcloseDegree(H5F_CLOSE_STRONG);
-        fd_ = new H5File(fileName_.c_str(), H5F_ACC_RDWR,
-                         FileCreatPropList::DEFAULT, flist);
-
-        Group group = fd_->openGroup(attrGroupName_.c_str());
-        attr->WriteFinalHDF5Attributes(fd_, &group);
-        fd_->close();
-
-    } catch (const Exception &error) {
-        error.printErrorStack();
-        CloseFile();
-        throw sls::RuntimeError(
-            "Could not create/overwrite master HDF5 handles");
-    }
-    if (!silentMode) {
-        LOG(logINFO) << "Updated Master File";
-    }
-}

slsReceiverSoftware/src/HDF5MasterFile.h

@@ -1,31 +0,0 @@
-// SPDX-License-Identifier: LGPL-3.0-or-other
-// Copyright (C) 2021 Contributors to the SLS Detector Package
-#pragma once
-
-#include "File.h"
-
-#include <mutex>
-
-class HDF5MasterFile : private virtual slsDetectorDefs, public File {
-
-  public:
-    HDF5MasterFile(std::mutex *hdf5Lib);
-    ~HDF5MasterFile();
-
-    void CloseFile() override;
-    void LinkDataFile(std::string dataFilename, std::string dataSetname,
-                      const std::vector<std::string> parameterNames,
-                      const bool silentMode) override;
-    void CreateMasterFile(const std::string filePath,
-                          const std::string fileNamePrefix,
-                          const uint64_t fileIndex, const bool overWriteEnable,
-                          const bool silentMode,
-                          MasterAttributes *attr) override;
-    void UpdateMasterFile(MasterAttributes *attr, bool silentMode) override;
-
-  private:
-    std::mutex *hdf5Lib_;
-    H5File *fd_{nullptr};
-    std::string fileName_;
-    std::string attrGroupName_;
-};

slsReceiverSoftware/src/HDF5VirtualFile.cpp

@@ -1,211 +0,0 @@
-// SPDX-License-Identifier: LGPL-3.0-or-other
-// Copyright (C) 2021 Contributors to the SLS Detector Package
-#include "HDF5VirtualFile.h"
-#include "receiver_defs.h"
-
-#include <iomanip>
-
-HDF5VirtualFile::HDF5VirtualFile(std::mutex *hdf5Lib, bool g25)
-    : File(HDF5), hdf5Lib_(hdf5Lib), gotthard25um(g25) {}
-
-HDF5VirtualFile::~HDF5VirtualFile() { CloseFile(); }
-
-std::array<std::string, 2> HDF5VirtualFile::GetFileAndDatasetName() const {
-    return std::array<std::string, 2>{fileName_, dataSetName_};
-}
-
-void HDF5VirtualFile::CloseFile() {
-    std::lock_guard<std::mutex> lock(*hdf5Lib_);
-    try {
-        Exception::dontPrint(); // to handle errors
-        if (fd_) {
-            fd_->close();
-            delete fd_;
-            fd_ = nullptr;
-        }
-    } catch (const Exception &error) {
-        LOG(logERROR) << "Could not close virtual HDF5 handles of index";
-        error.printErrorStack();
-    }
-}
-
-void HDF5VirtualFile::CreateVirtualFile(
-    const std::string filePath, const std::string fileNamePrefix,
-    const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode,
-    const int modulePos, const int numUnitsPerReadout,
-    const uint32_t maxFramesPerFile, const uint64_t numImages,
-    const uint32_t nPixelsX, const uint32_t nPixelsY,
-    const uint32_t dynamicRange, const uint64_t numImagesCaught,
-    const int numModX, const int numModY, const DataType dataType,
-    const std::vector<std::string> parameterNames,
-    const std::vector<DataType> parameterDataTypes) {
-    // virtual file name
-    std::ostringstream osfn;
-    osfn << filePath << "/" << fileNamePrefix << "_virtual"
-         << "_" << fileIndex << ".h5";
-    fileName_ = osfn.str();
-
-    unsigned int paraSize = parameterNames.size();
-    uint64_t numModZ = numModX;
-    uint32_t nDimy = nPixelsY;
-    uint32_t nDimz = ((dynamicRange == 4) ? (nPixelsX / 2) : nPixelsX);
-
-    std::lock_guard<std::mutex> lock(*hdf5Lib_);
-
-    try {
-        Exception::dontPrint(); // to handle errors
-
-        // file
-        FileAccPropList fapl;
-        fapl.setFcloseDegree(H5F_CLOSE_STRONG);
-        fd_ = nullptr;
-        if (!overWriteEnable)
-            fd_ = new H5File(fileName_.c_str(), H5F_ACC_EXCL,
-                             FileCreatPropList::DEFAULT, fapl);
-        else
-            fd_ = new H5File(fileName_.c_str(), H5F_ACC_TRUNC,
-                             FileCreatPropList::DEFAULT, fapl);
-
-        // attributes - version
-        double dValue = HDF5_WRITER_VERSION;
-        DataSpace dataspace_attr = DataSpace(H5S_SCALAR);
-        Attribute attribute = fd_->createAttribute(
-            "version", PredType::NATIVE_DOUBLE, dataspace_attr);
-        attribute.write(PredType::NATIVE_DOUBLE, &dValue);
-
-        // virtual dataspace
-        hsize_t vdsDims[3] = {numImagesCaught, numModY * nDimy,
-                              numModZ * nDimz};
-        DataSpace vdsDataSpace(3, vdsDims, nullptr);
-        hsize_t vdsDimsPara[2] = {numImagesCaught,
-                                  (unsigned int)numModY * numModZ};
-        DataSpace vdsDataSpacePara(2, vdsDimsPara, nullptr);
-
-        // property list (fill value and datatype)
-        int fill_value = -1;
-        DSetCreatPropList plist;
-        plist.setFillValue(dataType, &fill_value);
-
-        // property list for parameters (datatype)
-        std::vector<DSetCreatPropList> plistPara(paraSize);
-
-        // hyperslab (files)
-        int numFiles = numImagesCaught / maxFramesPerFile;
-        if (numImagesCaught % maxFramesPerFile)
-            ++numFiles;
-        uint64_t framesSaved = 0;
-        for (int iFile = 0; iFile < numFiles; ++iFile) {
-
-            uint64_t nDimx =
-                ((numImagesCaught - framesSaved) > maxFramesPerFile)
-                    ? maxFramesPerFile
-                    : (numImagesCaught - framesSaved);
-
-            hsize_t startLocation[3] = {framesSaved, 0, 0};
-            hsize_t strideBetweenBlocks[3] = {1, 1, 1};
-            hsize_t numBlocks[3] = {nDimx, nDimy, nDimz};
-            hsize_t blockSize[3] = {1, 1, 1};
-
-            hsize_t startLocationPara[2] = {framesSaved, 0};
-            hsize_t strideBetweenBlocksPara[3] = {1, 1};
-            hsize_t numBlocksPara[2] = {1, 1};
-            hsize_t blockSizePara[3] = {nDimx, 1};
-
-            // interleaving for g2
-            if (gotthard25um) {
-                strideBetweenBlocks[2] = 2;
-            }
-
-            for (unsigned int iReadout = 0; iReadout < numModY * numModZ;
-                 ++iReadout) {
-
-                // interleaving for g2 (startLocation is 0 and 1)
-                if (gotthard25um) {
-                    startLocation[2] = iReadout;
-                }
-
-                vdsDataSpace.selectHyperslab(H5S_SELECT_SET, numBlocks,
-                                             startLocation, strideBetweenBlocks,
-                                             blockSize);
-
-                vdsDataSpacePara.selectHyperslab(
-                    H5S_SELECT_SET, numBlocksPara, startLocationPara,
-                    strideBetweenBlocksPara, blockSizePara);
-
-                // source file name
-                std::ostringstream os;
-                os << filePath << "/" << fileNamePrefix << "_d"
-                   << (modulePos * numUnitsPerReadout + iReadout) << "_f"
-                   << iFile << '_' << fileIndex << ".h5";
-                std::string srcFileName = os.str();
-                LOG(logDEBUG1) << srcFileName;
-
-                // find relative path
-                std::string relative_srcFileName = srcFileName;
-                {
-                    size_t p = srcFileName.rfind('/', srcFileName.length());
-                    if (p != std::string::npos)
-                        relative_srcFileName = (srcFileName.substr(
-                            p + 1, srcFileName.length() - p));
-                }
-
-                // source dataset name
-                std::ostringstream osfn;
-                osfn << "/data";
-                if (numImages > 1)
-                    osfn << "_f" << std::setfill('0') << std::setw(12) << iFile;
-                std::string srcDatasetName = osfn.str();
-
-                // source dataspace
-                hsize_t srcDims[3] = {nDimx, nDimy, nDimz};
-                hsize_t srcDimsMax[3] = {H5S_UNLIMITED, nDimy, nDimz};
-                DataSpace srcDataSpace(3, srcDims, srcDimsMax);
-                hsize_t srcDimsPara[1] = {nDimx};
-                hsize_t srcDimsMaxPara[1] = {H5S_UNLIMITED};
-                DataSpace srcDataSpacePara(1, srcDimsPara, srcDimsMaxPara);
-
-                // mapping of property list
-                plist.setVirtual(vdsDataSpace, relative_srcFileName.c_str(),
-                                 srcDatasetName.c_str(), srcDataSpace);
-                for (unsigned int p = 0; p < paraSize; ++p) {
-                    plistPara[p].setVirtual(
-                        vdsDataSpacePara, relative_srcFileName.c_str(),
-                        parameterNames[p].c_str(), srcDataSpacePara);
-                }
-
-                // H5Sclose(srcDataspace);
-                // H5Sclose(srcDataspace_para);
-
-                if (!gotthard25um) {
-                    startLocation[2] += nDimz;
-                    if (startLocation[2] >= (numModZ * nDimz)) {
-                        startLocation[2] = 0;
-                        startLocation[1] += nDimy;
-                    }
-                }
-                startLocationPara[1]++;
-            }
-            framesSaved += nDimx;
-        }
-        // datasets
-        dataSetName_ = "data";
-        DataSet vdsDataSet(fd_->createDataSet(dataSetName_.c_str(), dataType,
-                                              vdsDataSpace, plist));
-
-        for (unsigned int p = 0; p < paraSize; ++p) {
-            DataSet vdsDataSetPara(fd_->createDataSet(
-                parameterNames[p].c_str(), parameterDataTypes[p],
-                vdsDataSpacePara, plistPara[p]));
-        }
-
-        fd_->close();
-    } catch (const Exception &error) {
-        error.printErrorStack();
-        CloseFile();
-        throw sls::RuntimeError(
-            "Could not create/overwrite virtual HDF5 handles");
-    }
-    if (!silentMode) {
-        LOG(logINFO) << "Virtual File: " << fileName_;
-    }
-}

slsReceiverSoftware/src/HDF5VirtualFile.h

@@ -1,34 +0,0 @@
-// SPDX-License-Identifier: LGPL-3.0-or-other
-// Copyright (C) 2021 Contributors to the SLS Detector Package
-#pragma once
-
-#include "File.h"
-
-#include <mutex>
-
-class HDF5VirtualFile : private virtual slsDetectorDefs, public File {
-
-  public:
-    HDF5VirtualFile(std::mutex *hdf5Lib, bool g25);
-    ~HDF5VirtualFile();
-
-    std::array<std::string, 2> GetFileAndDatasetName() const override;
-    void CloseFile() override;
-    void CreateVirtualFile(
-        const std::string filePath, const std::string fileNamePrefix,
-        const uint64_t fileIndex, const bool overWriteEnable,
-        const bool silentMode, const int modulePos,
-        const int numUnitsPerReadout, const uint32_t maxFramesPerFile,
-        const uint64_t numImages, const uint32_t nPixelsX,
-        const uint32_t nPixelsY, const uint32_t dynamicRange,
-        const uint64_t numImagesCaught, const int numModX, const int numModY,
-        const DataType dataType, const std::vector<std::string> parameterNames,
-        const std::vector<DataType> parameterDataTypes) override;
-
-  private:
-    std::mutex *hdf5Lib_;
-    H5File *fd_{nullptr};
-    std::string fileName_;
-    std::string dataSetName_;
-    bool gotthard25um;
-};

slsReceiverSoftware/src/Implementation.cpp

@@ -240,9 +240,6 @@ void Implementation::setModulePositionId(const int id) {
     xy portGeometry = GetPortGeometry();
     streamingPort = DEFAULT_ZMQ_RX_PORTNO + modulePos * portGeometry.x;
 
-    for (const auto &it : dataProcessor)
-        it->SetupFileWriter(fileWriteEnable, masterFileWriteEnable,
-                            fileFormatType, modulePos, &hdf5Lib);
     assert(numModules.y != 0);
     for (unsigned int i = 0; i < listener.size(); ++i) {
         uint16_t row = 0, col = 0;
@@ -372,8 +369,7 @@ void Implementation::setFileFormat(const fileFormat f) {
             throw sls::RuntimeError("Unknown file format");
         }
         for (const auto &it : dataProcessor)
-            it->SetupFileWriter(fileWriteEnable, masterFileWriteEnable,
-                                fileFormatType, modulePos, &hdf5Lib);
+            it->SetupFileWriter(fileWriteEnable, fileFormatType, &hdf5LibMutex);
     }
 
     LOG(logINFO) << "File Format: " << sls::ToString(fileFormatType);
@@ -409,8 +405,7 @@ void Implementation::setFileWriteEnable(const bool b) {
     if (fileWriteEnable != b) {
         fileWriteEnable = b;
         for (const auto &it : dataProcessor)
-            it->SetupFileWriter(fileWriteEnable, masterFileWriteEnable,
-                                fileFormatType, modulePos, &hdf5Lib);
+            it->SetupFileWriter(fileWriteEnable, fileFormatType, &hdf5LibMutex);
     }
     LOG(logINFO) << "File Write Enable: "
                  << (fileWriteEnable ? "enabled" : "disabled");
@@ -423,9 +418,6 @@ bool Implementation::getMasterFileWriteEnable() const {
 void Implementation::setMasterFileWriteEnable(const bool b) {
     if (masterFileWriteEnable != b) {
         masterFileWriteEnable = b;
-        for (const auto &it : dataProcessor)
-            it->SetupFileWriter(fileWriteEnable, masterFileWriteEnable,
-                                fileFormatType, modulePos, &hdf5Lib);
     }
     LOG(logINFO) << "Master File Write Enable: "
                  << (masterFileWriteEnable ? "enabled" : "disabled");
@@ -584,29 +576,9 @@ void Implementation::stopReceiver() {
         std::this_thread::sleep_for(std::chrono::milliseconds(5));
     }
 
-#ifdef HDF5C
-    if (fileWriteEnable && fileFormatType == HDF5) {
-        if (modulePos == 0) {
-            // more than 1 file, create virtual file
-            if (dataProcessor[0]->GetFilesInAcquisition() > 1 ||
-                (numModules.x * numModules.y) > 1) {
-                dataProcessor[0]->CreateVirtualFile(
-                    filePath, fileName, fileIndex, overwriteEnable, silentMode,
-                    modulePos, numUDPInterfaces, framesPerFile,
-                    numberOfTotalFrames, dynamicRange, numModules.x,
-                    numModules.y, &hdf5Lib);
-            }
-            // link file in master
-            dataProcessor[0]->LinkDataInMasterFile(silentMode);
-        }
-    }
-#endif
-    if (fileWriteEnable && masterFileWriteEnable && modulePos == 0) {
-        try {
-            dataProcessor[0]->UpdateMasterFile(silentMode);
-        } catch (...) {
-            ; // ignore it and just print it
-        }
+    if (fileWriteEnable && modulePos == 0) {
+        // master and virtual file (hdf5)
+        StartMasterWriter();
     }
 
     // wait for the processes (dataStreamer) to be done
@@ -753,100 +725,12 @@ void Implementation::CreateUDPSockets() {
 }
 
 void Implementation::SetupWriter() {
-    // master file
-    std::unique_ptr<MasterAttributes> masterAttributes;
-    if (masterFileWriteEnable && modulePos == 0) {
-        switch (detType) {
-        case GOTTHARD:
-            masterAttributes = sls::make_unique<GotthardMasterAttributes>();
-            break;
-        case JUNGFRAU:
-            masterAttributes = sls::make_unique<JungfrauMasterAttributes>();
-            break;
-        case EIGER:
-            masterAttributes = sls::make_unique<EigerMasterAttributes>();
-            break;
-        case MYTHEN3:
-            masterAttributes = sls::make_unique<Mythen3MasterAttributes>();
-            break;
-        case GOTTHARD2:
-            masterAttributes = sls::make_unique<Gotthard2MasterAttributes>();
-            break;
-        case MOENCH:
-            masterAttributes = sls::make_unique<MoenchMasterAttributes>();
-            break;
-        case CHIPTESTBOARD:
-            masterAttributes = sls::make_unique<CtbMasterAttributes>();
-            break;
-        default:
-            throw sls::RuntimeError(
-                "Unknown detector type to set up master file attributes");
-        }
-        masterAttributes->detType = detType;
-        masterAttributes->timingMode = timingMode;
-        xy nm{numModules.x, numModules.y};
-        if (quadEnable) {
-            nm.x = 1;
-            nm.y = 2;
-        }
-        masterAttributes->geometry = xy(nm.x, nm.y);
-        masterAttributes->imageSize = generalData->imageSize;
-        masterAttributes->nPixels =
-            xy(generalData->nPixelsX, generalData->nPixelsY);
-        masterAttributes->maxFramesPerFile = framesPerFile;
-        masterAttributes->frameDiscardMode = frameDiscardMode;
-        masterAttributes->framePadding = framePadding;
-        masterAttributes->scanParams = scanParams;
-        masterAttributes->totalFrames = numberOfTotalFrames;
-        masterAttributes->exptime = acquisitionTime;
-        masterAttributes->period = acquisitionPeriod;
-        masterAttributes->burstMode = burstMode;
-        masterAttributes->numUDPInterfaces = numUDPInterfaces;
-        masterAttributes->dynamicRange = dynamicRange;
-        masterAttributes->tenGiga = tengigaEnable;
-        masterAttributes->thresholdEnergyeV = thresholdEnergyeV;
-        masterAttributes->thresholdAllEnergyeV = thresholdAllEnergyeV;
-        masterAttributes->subExptime = subExpTime;
-        masterAttributes->subPeriod = subPeriod;
-        masterAttributes->quad = quadEnable;
-        masterAttributes->readNRows = readNRows;
-        masterAttributes->ratecorr = rateCorrections;
-        masterAttributes->adcmask =
-            tengigaEnable ? adcEnableMaskTenGiga : adcEnableMaskOneGiga;
-        masterAttributes->analog =
-            (readoutType == ANALOG_ONLY || readoutType == ANALOG_AND_DIGITAL)
-                ? 1
-                : 0;
-        masterAttributes->analogSamples = numberOfAnalogSamples;
-        masterAttributes->digital =
-            (readoutType == DIGITAL_ONLY || readoutType == ANALOG_AND_DIGITAL)
-                ? 1
-                : 0;
-        masterAttributes->digitalSamples = numberOfDigitalSamples;
-        masterAttributes->dbitoffset = ctbDbitOffset;
-        masterAttributes->dbitlist = 0;
-        for (auto &i : ctbDbitList) {
-            masterAttributes->dbitlist |= (1 << i);
-        }
-        masterAttributes->roi = roi;
-        masterAttributes->counterMask = counterMask;
-        masterAttributes->exptime1 = acquisitionTime1;
-        masterAttributes->exptime2 = acquisitionTime2;
-        masterAttributes->exptime3 = acquisitionTime3;
-        masterAttributes->gateDelay1 = gateDelay1;
-        masterAttributes->gateDelay2 = gateDelay2;
-        masterAttributes->gateDelay3 = gateDelay3;
-        masterAttributes->gates = numberOfGates;
-        masterAttributes->additionalJsonHeader = additionalJsonHeader;
-    }
-
     try {
         for (unsigned int i = 0; i < dataProcessor.size(); ++i) {
             dataProcessor[i]->CreateFirstFiles(
-                masterAttributes.get(), filePath, fileName, fileIndex,
-                overwriteEnable, silentMode, modulePos, numUDPInterfaces,
-                udpPortNum[i], framesPerFile, numberOfTotalFrames, dynamicRange,
-                detectorDataStream[i]);
+                filePath, fileName, fileIndex, overwriteEnable, silentMode,
+                modulePos, numUDPInterfaces, udpPortNum[i], framesPerFile,
+                numberOfTotalFrames, dynamicRange, detectorDataStream[i]);
         }
     } catch (const sls::RuntimeError &e) {
         shutDownUDPSockets();
@@ -856,6 +740,100 @@ void Implementation::SetupWriter() {
     }
 }
 
+void Implementation::StartMasterWriter() {
+    try {
+        std::string masterFileName;
+        // master file
+        if (masterFileWriteEnable) {
+            MasterAttributes masterAttributes;
+            masterAttributes.detType = detType;
+            masterAttributes.timingMode = timingMode;
+            xy nm{numModules.x, numModules.y};
+            if (quadEnable) {
+                nm.x = 1;
+                nm.y = 2;
+            }
+            masterAttributes.geometry = xy(nm.x, nm.y);
+            masterAttributes.imageSize = generalData->imageSize;
+            masterAttributes.nPixels =
+                xy(generalData->nPixelsX, generalData->nPixelsY);
+            masterAttributes.maxFramesPerFile = framesPerFile;
+            masterAttributes.frameDiscardMode = frameDiscardMode;
+            masterAttributes.framePadding = framePadding;
+            masterAttributes.scanParams = scanParams;
+            masterAttributes.totalFrames = numberOfTotalFrames;
+            masterAttributes.exptime = acquisitionTime;
+            masterAttributes.period = acquisitionPeriod;
+            masterAttributes.burstMode = burstMode;
+            masterAttributes.numUDPInterfaces = numUDPInterfaces;
+            masterAttributes.dynamicRange = dynamicRange;
+            masterAttributes.tenGiga = tengigaEnable;
+            masterAttributes.thresholdEnergyeV = thresholdEnergyeV;
+            masterAttributes.thresholdAllEnergyeV = thresholdAllEnergyeV;
+            masterAttributes.subExptime = subExpTime;
+            masterAttributes.subPeriod = subPeriod;
+            masterAttributes.quad = quadEnable;
+            masterAttributes.readNRows = readNRows;
+            masterAttributes.ratecorr = rateCorrections;
+            masterAttributes.adcmask =
+                tengigaEnable ? adcEnableMaskTenGiga : adcEnableMaskOneGiga;
+            masterAttributes.analog = (readoutType == ANALOG_ONLY ||
+                                       readoutType == ANALOG_AND_DIGITAL)
+                                          ? 1
+                                          : 0;
+            masterAttributes.analogSamples = numberOfAnalogSamples;
+            masterAttributes.digital = (readoutType == DIGITAL_ONLY ||
+                                        readoutType == ANALOG_AND_DIGITAL)
+                                           ? 1
+                                           : 0;
+            masterAttributes.digitalSamples = numberOfDigitalSamples;
+            masterAttributes.dbitoffset = ctbDbitOffset;
+            masterAttributes.dbitlist = 0;
+            for (auto &i : ctbDbitList) {
+                masterAttributes.dbitlist |= (1 << i);
+            }
+            masterAttributes.roi = roi;
+            masterAttributes.counterMask = counterMask;
+            masterAttributes.exptimeArray[0] = acquisitionTime1;
+            masterAttributes.exptimeArray[1] = acquisitionTime2;
+            masterAttributes.exptimeArray[2] = acquisitionTime3;
+            masterAttributes.gateDelayArray[0] = gateDelay1;
+            masterAttributes.gateDelayArray[1] = gateDelay2;
+            masterAttributes.gateDelayArray[2] = gateDelay3;
+            masterAttributes.gates = numberOfGates;
+            masterAttributes.additionalJsonHeader = additionalJsonHeader;
+
+            // create master file
+            masterFileName = dataProcessor[0]->CreateMasterFile(
+                filePath, fileName, fileIndex, overwriteEnable, silentMode,
+                fileFormatType, &masterAttributes, &hdf5LibMutex);
+        }
+#ifdef HDF5C
+        if (fileFormatType == HDF5) {
+            std::array<std::string, 2> virtualFileAndDatasetNames;
+            // create virtual hdf5 file (if multiple files)
+            if (dataProcessor[0]->GetFilesInAcquisition() > 1 ||
+                (numModules.x * numModules.y) > 1) {
+                virtualFileAndDatasetNames =
+                    dataProcessor[0]->CreateVirtualFile(
+                        filePath, fileName, fileIndex, overwriteEnable,
+                        silentMode, modulePos, numUDPInterfaces, framesPerFile,
+                        numberOfTotalFrames, numModules.x, numModules.y,
+                        dynamicRange, &hdf5LibMutex);
+            }
+            // link file in master
+            if (masterFileWriteEnable) {
+                dataProcessor[0]->LinkFileInMaster(
+                    masterFileName, virtualFileAndDatasetNames[0],
+                    virtualFileAndDatasetNames[1], silentMode, &hdf5LibMutex);
+            }
+        }
+#endif
+    } catch (...) {
+        ; // ignore it and just print it
+    }
+}
+
 void Implementation::StartRunning() {
 
     // set running mask and post semaphore to start the inner loop in execution

slsReceiverSoftware/src/Implementation.h

@@ -274,6 +274,7 @@ class Implementation : private virtual slsDetectorDefs {
     void ResetParametersforNewAcquisition();
     void CreateUDPSockets();
     void SetupWriter();
+    void StartMasterWriter();
     void StartRunning();
 
     /**************************************************
@@ -386,5 +387,6 @@ class Implementation : private virtual slsDetectorDefs {
     std::vector<std::unique_ptr<Fifo>> fifo;
     Arping arping;
 
-    std::mutex hdf5Lib;
+    // mutex shared across all hdf5 virtual, master and data files
+    std::mutex hdf5LibMutex;
 };

slsReceiverSoftware/src/MasterAttributes.h

@@ -7,6 +7,9 @@
 #include "sls/logger.h"
 #include "sls/sls_detector_defs.h"
 
+#include <rapidjson/stringbuffer.h>
+#include <rapidjson/prettywriter.h>
+
 #ifdef HDF5C
 #include "H5Cpp.h"
 #ifndef H5_NO_NAMESPACE
@@ -53,94 +56,92 @@ class MasterAttributes {
     uint64_t dbitlist{0};
     slsDetectorDefs::ROI roi{};
     uint32_t counterMask{0};
-    ns exptime1{0};
-    ns exptime2{0};
-    ns exptime3{0};
-    ns gateDelay1{0};
-    ns gateDelay2{0};
-    ns gateDelay3{0};
+    std::array<ns, 3> exptimeArray{};
+    std::array<ns, 3> gateDelayArray{};
     uint32_t gates;
     std::map<std::string, std::string> additionalJsonHeader;
-
-    // Final Attributes (after acquisition)
     uint64_t framesInFile{0};
 
     MasterAttributes() = default;
-    virtual ~MasterAttributes() = default;
-    virtual void WriteMasterBinaryAttributes(FILE *fd);
-    std::string GetBinaryMasterAttributes();
-    void WriteBinaryAttributes(FILE *fd, std::string message);
-    void WriteFinalBinaryAttributes(FILE *fd);
+    ~MasterAttributes() = default;
+
+    void GetBinaryAttributes(rapidjson::PrettyWriter<rapidjson::StringBuffer> *w);
 #ifdef HDF5C
-    virtual void WriteMasterHDF5Attributes(H5File *fd, Group *group);
     void WriteHDF5Attributes(H5File *fd, Group *group);
+#endif
+
+    void
+    GetCommonBinaryAttributes(rapidjson::PrettyWriter<rapidjson::StringBuffer> *w);
+    void
+    GetFinalBinaryAttributes(rapidjson::PrettyWriter<rapidjson::StringBuffer> *w);
+#ifdef HDF5C
+    void WriteCommonHDF5Attributes(H5File *fd, Group *group);
     void WriteFinalHDF5Attributes(H5File *fd, Group *group);
     void WriteHDF5Exptime(H5File *fd, Group *group);
     void WriteHDF5Period(H5File *fd, Group *group);
     void WriteHDF5DynamicRange(H5File *fd, Group *group);
     void WriteHDF5TenGiga(H5File *fd, Group *group);
+    void WriteHDF5ROI(H5File *fd, Group *group);
+    void WriteHDF5NumUDPInterfaces(H5File *fd, Group *group);
+    void WriteHDF5ReadNRows(H5File *fd, Group *group);
+    void WriteHDF5ThresholdEnergy(H5File *fd, Group *group);
+    void WriteHDF5ThresholdEnergies(H5File *fd, Group *group);
+    void WriteHDF5SubExpTime(H5File *fd, Group *group);
+    void WriteHDF5SubPeriod(H5File *fd, Group *group);
+    void WriteHDF5SubQuad(H5File *fd, Group *group);
+    void WriteHDF5RateCorrections(H5File *fd, Group *group);
+    void WriteHDF5CounterMask(H5File *fd, Group *group);
+    void WriteHDF5ExptimeArray(H5File *fd, Group *group);
+    void WriteHDF5GateDelayArray(H5File *fd, Group *group);
+    void WriteHDF5Gates(H5File *fd, Group *group);
+    void WriteHDF5BurstMode(H5File *fd, Group *group);
+    void WriteHDF5AdcMask(H5File *fd, Group *group);
+    void WriteHDF5AnalogFlag(H5File *fd, Group *group);
+    void WriteHDF5AnalogSamples(H5File *fd, Group *group);
+    void WriteHDF5DigitalFlag(H5File *fd, Group *group);
+    void WriteHDF5DigitalSamples(H5File *fd, Group *group);
+    void WriteHDF5DbitOffset(H5File *fd, Group *group);
+    void WriteHDF5DbitList(H5File *fd, Group *group);
 #endif
-};
 
-class GotthardMasterAttributes : public MasterAttributes {
-  public:
-    GotthardMasterAttributes() = default;
-    void WriteMasterBinaryAttributes(FILE *fd) override;
+    void
+    GetGotthardBinaryAttributes(rapidjson::PrettyWriter<rapidjson::StringBuffer> *w);
 #ifdef HDF5C
-    void WriteMasterHDF5Attributes(H5File *fd, Group *group) override;
+    void WriteGotthardHDF5Attributes(H5File *fd, Group *group);
 #endif
-};
 
-class JungfrauMasterAttributes : public MasterAttributes {
-  public:
-    JungfrauMasterAttributes() = default;
-    void WriteMasterBinaryAttributes(FILE *fd) override;
+    void
+    GetJungfrauBinaryAttributes(rapidjson::PrettyWriter<rapidjson::StringBuffer> *w);
 #ifdef HDF5C
-    void WriteMasterHDF5Attributes(H5File *fd, Group *group) override;
+    void WriteJungfrauHDF5Attributes(H5File *fd, Group *group);
 #endif
-};
 
-class EigerMasterAttributes : public MasterAttributes {
-  public:
-    EigerMasterAttributes() = default;
-    void WriteMasterBinaryAttributes(FILE *fd) override;
+    void
+    GetEigerBinaryAttributes(rapidjson::PrettyWriter<rapidjson::StringBuffer> *w);
 #ifdef HDF5C
-    void WriteMasterHDF5Attributes(H5File *fd, Group *group) override;
+    void WriteEigerHDF5Attributes(H5File *fd, Group *group);
 #endif
-};
 
-class Mythen3MasterAttributes : public MasterAttributes {
-  public:
-    Mythen3MasterAttributes() = default;
-    void WriteMasterBinaryAttributes(FILE *fd) override;
+    void
+    GetMythen3BinaryAttributes(rapidjson::PrettyWriter<rapidjson::StringBuffer> *w);
 #ifdef HDF5C
-    void WriteMasterHDF5Attributes(H5File *fd, Group *group) override;
+    void WriteMythen3HDF5Attributes(H5File *fd, Group *group);
 #endif
-};
 
-class Gotthard2MasterAttributes : public MasterAttributes {
-  public:
-    Gotthard2MasterAttributes() = default;
-    void WriteMasterBinaryAttributes(FILE *fd) override;
+    void
+    GetGotthard2BinaryAttributes(rapidjson::PrettyWriter<rapidjson::StringBuffer> *w);
 #ifdef HDF5C
-    void WriteMasterHDF5Attributes(H5File *fd, Group *group) override;
+    void WriteGotthard2HDF5Attributes(H5File *fd, Group *group);
 #endif
-};
 
-class MoenchMasterAttributes : public MasterAttributes {
-  public:
-    MoenchMasterAttributes() = default;
-    void WriteMasterBinaryAttributes(FILE *fd) override;
+    void
+    GetMoenchBinaryAttributes(rapidjson::PrettyWriter<rapidjson::StringBuffer> *w);
 #ifdef HDF5C
-    void WriteMasterHDF5Attributes(H5File *fd, Group *group) override;
+    void WriteMoenchHDF5Attributes(H5File *fd, Group *group);
 #endif
-};
 
-class CtbMasterAttributes : public MasterAttributes {
-  public:
-    CtbMasterAttributes() = default;
-    void WriteMasterBinaryAttributes(FILE *fd) override;
+    void GetCtbBinaryAttributes(rapidjson::PrettyWriter<rapidjson::StringBuffer> *w);
 #ifdef HDF5C
-    void WriteMasterHDF5Attributes(H5File *fd, Group *group) override;
+    void WriteCtbHDF5Attributes(H5File *fd, Group *group);
 #endif
 };

slsReceiverSoftware/src/MasterFileUtility.cpp

@@ -0,0 +1,351 @@
+// SPDX-License-Identifier: LGPL-3.0-or-other
+// Copyright (C) 2021 Contributors to the SLS Detector Package
+#include "MasterFileUtility.h"
+#include "sls/container_utils.h"
+
+#include <iomanip>
+
+namespace masterFileUtility {
+
+std::string CreateMasterBinaryFile(const std::string &filePath,
+                                   const std::string &fileNamePrefix,
+                                   const uint64_t fileIndex,
+                                   const bool overWriteEnable,
+                                   const bool silentMode,
+                                   MasterAttributes *attr) {
+    std::ostringstream os;
+    os << filePath << "/" << fileNamePrefix << "_master"
+       << "_" << fileIndex << ".json";
+    std::string fileName = os.str();
+
+    std::string mode = "w";
+    if (!overWriteEnable)
+        mode = "wx";
+    FILE *fd = fopen(fileName.c_str(), mode.c_str());
+    if(!fd) {
+        throw sls::RuntimeError("Could not create/overwrite binary master file " +
+                                    fileName);
+    }
+
+    rapidjson::StringBuffer s;
+    rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(s);
+    attr->GetBinaryAttributes(&writer);
+    if (fwrite(s.GetString(), 1, strlen(s.GetString()), fd) !=
+        strlen(s.GetString())) {
+        throw sls::RuntimeError(
+            "Master binary file incorrect number of bytes written to file");
+    }
+    if (fd) {
+        fclose(fd);
+    }
+    if (!silentMode) {
+        LOG(logINFO) << "Master File: " << fileName;
+    }
+    return fileName;
+}
+
+#ifdef HDF5C
+void LinkHDF5FileInMaster(const std::string &masterFileName,
+                          const std::string &dataFilename,
+                          const std::string &dataSetname,
+                          const std::vector<std::string> parameterNames,
+                          const bool silentMode, std::mutex *hdf5LibMutex) {
+
+    std::lock_guard<std::mutex> lock(*hdf5LibMutex);
+    std::unique_ptr<H5File> fd{nullptr};
+    try {
+        Exception::dontPrint(); // to handle errors
+
+        FileAccPropList flist;
+        flist.setFcloseDegree(H5F_CLOSE_STRONG);
+
+        // open master file
+        H5File masterfd(masterFileName.c_str(), H5F_ACC_RDWR,
+                        FileCreatPropList::DEFAULT, flist);
+
+        // open data file
+        fd = sls::make_unique<H5File>(dataFilename.c_str(), H5F_ACC_RDONLY,
+                                      FileCreatPropList::DEFAULT, flist);
+
+        // create link for data dataset
+        DataSet dset = fd->openDataSet(dataSetname.c_str());
+        std::string linkname = std::string("/entry/data/") + dataSetname;
+        if (H5Lcreate_external(dataFilename.c_str(), dataSetname.c_str(),
+                               masterfd.getLocId(), linkname.c_str(),
+                               H5P_DEFAULT, H5P_DEFAULT) < 0) {
+            throw sls::RuntimeError(
+                "Could not create link to data dataset in master");
+        }
+
+        // create link for parameter datasets
+        for (unsigned int i = 0; i < parameterNames.size(); ++i) {
+            DataSet pDset = fd->openDataSet(parameterNames[i].c_str());
+            linkname = std::string("/entry/data/") + parameterNames[i];
+            if (H5Lcreate_external(dataFilename.c_str(),
+                                   parameterNames[i].c_str(),
+                                   masterfd.getLocId(), linkname.c_str(),
+                                   H5P_DEFAULT, H5P_DEFAULT) < 0) {
+                throw sls::RuntimeError(
+                    "Could not create link to parameter dataset in master");
+            }
+        }
+        fd->close();
+        masterfd.close();
+    } catch (const Exception &error) {
+        error.printErrorStack();
+        if (fd != nullptr)
+            fd->close();
+        throw sls::RuntimeError("Could not link in master hdf5 file");
+    }
+    if (!silentMode) {
+        LOG(logINFO) << "Linked in Master File: " << dataFilename;
+    }
+}
+
+std::string CreateMasterHDF5File(const std::string &filePath,
+                                 const std::string &fileNamePrefix,
+                                 const uint64_t fileIndex,
+                                 const bool overWriteEnable,
+                                 const bool silentMode, MasterAttributes *attr,
+                                 std::mutex *hdf5LibMutex) {
+
+    std::ostringstream os;
+    os << filePath << "/" << fileNamePrefix << "_master"
+       << "_" << fileIndex << ".h5";
+    std::string fileName = os.str();
+
+    std::lock_guard<std::mutex> lock(*hdf5LibMutex);
+
+    std::unique_ptr<H5File> fd{nullptr};
+    try {
+        Exception::dontPrint(); // to handle errors
+
+        FileAccPropList flist;
+        flist.setFcloseDegree(H5F_CLOSE_STRONG);
+
+        unsigned int createFlags = H5F_ACC_EXCL;
+        if (overWriteEnable) {
+            createFlags = H5F_ACC_TRUNC;
+        }
+        fd = sls::make_unique<H5File>(fileName.c_str(), createFlags,
+                                      FileCreatPropList::DEFAULT, flist);
+
+        // attributes - version
+        double dValue = HDF5_WRITER_VERSION;
+        DataSpace dataspace_attr = DataSpace(H5S_SCALAR);
+        Attribute attribute = fd->createAttribute(
+            "version", PredType::NATIVE_DOUBLE, dataspace_attr);
+        attribute.write(PredType::NATIVE_DOUBLE, &dValue);
+
+        // Create a group in the file
+        Group group1(fd->createGroup("entry"));
+        Group group2(group1.createGroup("data"));
+        Group group3(group1.createGroup("instrument"));
+        Group group4(group3.createGroup("beam"));
+        Group group5(group3.createGroup("detector"));
+        Group group6(group1.createGroup("sample"));
+
+        attr->WriteHDF5Attributes(fd.get(), &group5);
+        fd->close();
+    } catch (const Exception &error) {
+        error.printErrorStack();
+        if (fd != nullptr)
+            fd->close();
+        throw sls::RuntimeError(
+            "Could not create/overwrite master HDF5 handles");
+    }
+    if (!silentMode) {
+        LOG(logINFO) << "Master File: " << fileName;
+    }
+    return fileName;
+}
+
+std::array<std::string, 2> CreateVirtualHDF5File(
+    const std::string &filePath, const std::string &fileNamePrefix,
+    const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode,
+    const int modulePos, const int numUnitsPerReadout,
+    const uint32_t maxFramesPerFile, const uint64_t numImages,
+    const uint32_t nPixelsX, const uint32_t nPixelsY,
+    const uint32_t dynamicRange, const uint64_t numImagesCaught,
+    const int numModX, const int numModY, const DataType dataType,
+    const std::vector<std::string> parameterNames,
+    const std::vector<DataType> parameterDataTypes, std::mutex *hdf5LibMutex,
+    bool gotthard25um) {
+
+    // virtual file name
+    std::ostringstream osfn;
+    osfn << filePath << "/" << fileNamePrefix << "_virtual"
+         << "_" << fileIndex << ".h5";
+    std::string fileName = osfn.str();
+
+    std::string dataSetName = "data";
+
+    unsigned int paraSize = parameterNames.size();
+    uint64_t numModZ = numModX;
+    uint32_t nDimy = nPixelsY;
+    uint32_t nDimz = ((dynamicRange == 4) ? (nPixelsX / 2) : nPixelsX);
+
+    std::lock_guard<std::mutex> lock(*hdf5LibMutex);
+
+    std::unique_ptr<H5File> fd{nullptr};
+    try {
+        Exception::dontPrint(); // to handle errors
+
+        // file
+        FileAccPropList fapl;
+        fapl.setFcloseDegree(H5F_CLOSE_STRONG);
+        if (!overWriteEnable)
+            fd = sls::make_unique<H5File>(fileName.c_str(), H5F_ACC_EXCL,
+                                          FileCreatPropList::DEFAULT, fapl);
+        else
+            fd = sls::make_unique<H5File>(fileName.c_str(), H5F_ACC_TRUNC,
+                                          FileCreatPropList::DEFAULT, fapl);
+
+        // attributes - version
+        double dValue = HDF5_WRITER_VERSION;
+        DataSpace dataspace_attr = DataSpace(H5S_SCALAR);
+        Attribute attribute = fd->createAttribute(
+            "version", PredType::NATIVE_DOUBLE, dataspace_attr);
+        attribute.write(PredType::NATIVE_DOUBLE, &dValue);
+
+        // virtual dataspace
+        hsize_t vdsDims[3] = {numImagesCaught, numModY * nDimy,
+                              numModZ * nDimz};
+        DataSpace vdsDataSpace(3, vdsDims, nullptr);
+        hsize_t vdsDimsPara[2] = {numImagesCaught,
+                                  (unsigned int)numModY * numModZ};
+        DataSpace vdsDataSpacePara(2, vdsDimsPara, nullptr);
+
+        // property list (fill value and datatype)
+        int fill_value = -1;
+        DSetCreatPropList plist;
+        plist.setFillValue(dataType, &fill_value);
+
+        // property list for parameters (datatype)
+        std::vector<DSetCreatPropList> plistPara(paraSize);
+
+        // hyperslab (files)
+        int numFiles = numImagesCaught / maxFramesPerFile;
+        if (numImagesCaught % maxFramesPerFile)
+            ++numFiles;
+        uint64_t framesSaved = 0;
+        for (int iFile = 0; iFile < numFiles; ++iFile) {
+
+            uint64_t nDimx =
+                ((numImagesCaught - framesSaved) > maxFramesPerFile)
+                    ? maxFramesPerFile
+                    : (numImagesCaught - framesSaved);
+
+            hsize_t startLocation[3] = {framesSaved, 0, 0};
+            hsize_t strideBetweenBlocks[3] = {1, 1, 1};
+            hsize_t numBlocks[3] = {nDimx, nDimy, nDimz};
+            hsize_t blockSize[3] = {1, 1, 1};
+
+            hsize_t startLocationPara[2] = {framesSaved, 0};
+            hsize_t strideBetweenBlocksPara[3] = {1, 1};
+            hsize_t numBlocksPara[2] = {1, 1};
+            hsize_t blockSizePara[3] = {nDimx, 1};
+
+            // interleaving for g2
+            if (gotthard25um) {
+                strideBetweenBlocks[2] = 2;
+            }
+
+            for (unsigned int iReadout = 0; iReadout < numModY * numModZ;
+                 ++iReadout) {
+
+                // interleaving for g2 (startLocation is 0 and 1)
+                if (gotthard25um) {
+                    startLocation[2] = iReadout;
+                }
+
+                vdsDataSpace.selectHyperslab(H5S_SELECT_SET, numBlocks,
+                                             startLocation, strideBetweenBlocks,
+                                             blockSize);
+
+                vdsDataSpacePara.selectHyperslab(
+                    H5S_SELECT_SET, numBlocksPara, startLocationPara,
+                    strideBetweenBlocksPara, blockSizePara);
+
+                // source file name
+                std::ostringstream os;
+                os << filePath << "/" << fileNamePrefix << "_d"
+                   << (modulePos * numUnitsPerReadout + iReadout) << "_f"
+                   << iFile << '_' << fileIndex << ".h5";
+                std::string srcFileName = os.str();
+                LOG(logDEBUG1) << srcFileName;
+
+                // find relative path
+                std::string relative_srcFileName = srcFileName;
+                {
+                    size_t p = srcFileName.rfind('/', srcFileName.length());
+                    if (p != std::string::npos)
+                        relative_srcFileName = (srcFileName.substr(
+                            p + 1, srcFileName.length() - p));
+                }
+
+                // source dataset name
+                std::ostringstream osfn;
+                osfn << "/data";
+                if (numImages > 1)
+                    osfn << "_f" << std::setfill('0') << std::setw(12) << iFile;
+                std::string srcDatasetName = osfn.str();
+
+                // source dataspace
+                hsize_t srcDims[3] = {nDimx, nDimy, nDimz};
+                hsize_t srcDimsMax[3] = {H5S_UNLIMITED, nDimy, nDimz};
+                DataSpace srcDataSpace(3, srcDims, srcDimsMax);
+                hsize_t srcDimsPara[1] = {nDimx};
+                hsize_t srcDimsMaxPara[1] = {H5S_UNLIMITED};
+                DataSpace srcDataSpacePara(1, srcDimsPara, srcDimsMaxPara);
+
+                // mapping of property list
+                plist.setVirtual(vdsDataSpace, relative_srcFileName.c_str(),
+                                 srcDatasetName.c_str(), srcDataSpace);
+                for (unsigned int p = 0; p < paraSize; ++p) {
+                    plistPara[p].setVirtual(
+                        vdsDataSpacePara, relative_srcFileName.c_str(),
+                        parameterNames[p].c_str(), srcDataSpacePara);
+                }
+
+                // H5Sclose(srcDataspace);
+                // H5Sclose(srcDataspace_para);
+
+                if (!gotthard25um) {
+                    startLocation[2] += nDimz;
+                    if (startLocation[2] >= (numModZ * nDimz)) {
+                        startLocation[2] = 0;
+                        startLocation[1] += nDimy;
+                    }
+                }
+                startLocationPara[1]++;
+            }
+            framesSaved += nDimx;
+        }
+        // datasets
+        DataSet vdsDataSet(fd->createDataSet(dataSetName.c_str(), dataType,
+                                             vdsDataSpace, plist));
+
+        for (unsigned int p = 0; p < paraSize; ++p) {
+            DataSet vdsDataSetPara(fd->createDataSet(
+                parameterNames[p].c_str(), parameterDataTypes[p],
+                vdsDataSpacePara, plistPara[p]));
+        }
+
+        fd->close();
+    } catch (const Exception &error) {
+        error.printErrorStack();
+        if (fd) {
+            fd->close();
+        }
+        throw sls::RuntimeError(
+            "Could not create/overwrite virtual HDF5 handles");
+    }
+    if (!silentMode) {
+        LOG(logINFO) << "Virtual File: " << fileName;
+    }
+    return std::array<std::string, 2>{fileName, dataSetName};
+}
+#endif
+
+} // namespace masterFileUtility

slsReceiverSoftware/src/MasterFileUtility.h

@@ -0,0 +1,50 @@
+// SPDX-License-Identifier: LGPL-3.0-or-other
+// Copyright (C) 2021 Contributors to the SLS Detector Package
+#pragma once
+
+#include "MasterAttributes.h"
+
+#ifdef HDF5C
+#include "H5Cpp.h"
+#include <mutex>
+#ifndef H5_NO_NAMESPACE
+using namespace H5;
+#endif
+#endif
+
+namespace masterFileUtility {
+
+std::string CreateMasterBinaryFile(const std::string &filePath,
+                                   const std::string &fileNamePrefix,
+                                   const uint64_t fileIndex,
+                                   const bool overWriteEnable,
+                                   const bool silentMode,
+                                   MasterAttributes *attr);
+
+#ifdef HDF5C
+void LinkHDF5FileInMaster(const std::string &masterFileName,
+                          const std::string &dataFilename,
+                          const std::string &dataSetname,
+                          const std::vector<std::string> parameterNames,
+                          const bool silentMode, std::mutex *hdf5LibMutex);
+
+std::string CreateMasterHDF5File(const std::string &filePath,
+                                 const std::string &fileNamePrefix,
+                                 const uint64_t fileIndex,
+                                 const bool overWriteEnable,
+                                 const bool silentMode, MasterAttributes *attr,
+                                 std::mutex *hdf5LibMutex);
+
+std::array<std::string, 2> CreateVirtualHDF5File(
+    const std::string &filePath, const std::string &fileNamePrefix,
+    const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode,
+    const int modulePos, const int numUnitsPerReadout,
+    const uint32_t maxFramesPerFile, const uint64_t numImages,
+    const uint32_t nPixelsX, const uint32_t nPixelsY,
+    const uint32_t dynamicRange, const uint64_t numImagesCaught,
+    const int numModX, const int numModY, const DataType dataType,
+    const std::vector<std::string> parameterNames,
+    const std::vector<DataType> parameterDataTypes, std::mutex *hdf5LibMutex,
+    bool gotthard25um);
+#endif
+} // namespace masterFileUtility

slsReceiverSoftware/src/receiver_defs.h

@@ -18,7 +18,7 @@
 
 // versions
 #define HDF5_WRITER_VERSION   (6.4) // 1 decimal places
-#define BINARY_WRITER_VERSION (6.4) // 1 decimal places
+#define BINARY_WRITER_VERSION (7.0) // 1 decimal places
 
 #define MAX_FRAMES_PER_FILE           20000
 #define SHORT_MAX_FRAMES_PER_FILE     100000

slsSupportLib/include/sls/versionAPI.h

@@ -11,5 +11,5 @@
 #define APIGOTTHARD 0x220328
 #define APIGOTTHARD2 0x220328
 #define APIJUNGFRAU 0x220328
-#define APIMYTHEN3 0x220328
 #define APIMOENCH 0x220328
+#define APIMYTHEN3 0x220404