wip

slsReceiverSoftware/src/DataProcessor.cpp

@@ -129,7 +129,6 @@ void DataProcessor::SetupFileWriter(const bool filewriteEnable,
                 if (masterFilewriteEnable) {
                     masterFile_ = new HDF5MasterFile(hdf5Lib_);
                 }
-                virtualFile_ = new HDF5VirtualFile(hdf5Lib_);
             }
             break;
 #endif
@@ -185,33 +184,30 @@ void DataProcessor::CreateFirstFiles(
 }
 
 #ifdef HDF5C
+uint32_t DataProcessor::GetFilesInAcquisition() const {
+    if (dataFile_ == nullptr) {
+        throw sls::RuntimeError("No data file object created to get number of "
+                                "files in acquiistion");
+    }
+    return dataFile_->GetFilesInAcquisition();
+}
+
 void DataProcessor::CreateVirtualFile(
     const std::string filePath, const std::string fileNamePrefix,
     const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode,
     const int modulePos, const int numUnitsPerReadout,
     const uint32_t maxFramesPerFile, const uint64_t numImages,
     const uint32_t dynamicRange, const int numModX, const int numModY) {
-    if (virtualFile_ == nullptr || dataFile_ == nullptr) {
-        throw sls::RuntimeError("No virtual or data file object created");
-    }
 
-    std::string fname, datasetName;
+    if (virtualFile_) {
-    // only 1 file (subfile and number of files in acquisition) (to link in
+        delete virtualFile_;
-    // master)
-    if (dataFile_->GetFilesInAcquisition() == 1 && (numModX * numModY) == 1) {
-        auto res = dataFile_->GetFileAndDatasetName();
-        fname = res[0];
-        datasetName = res[1];
     }
-
+    virtualFile_ = new HDF5VirtualFile(hdf5Lib_);
-    // create virtual file and link that in master
-    else {
 
     uint64_t numImagesProcessed = GetProcessedIndex() + 1;
     // maxframesperfile = 0 for infinite files
     uint32_t framesPerFile =
-            ((maxFramesPerFile == 0) ? numImagesProcessed + 1
+        ((maxFramesPerFile == 0) ? numImagesProcessed + 1 : maxFramesPerFile);
-                                     : maxFramesPerFile);
 
     // TODO: assumption 1: create virtual file even if no data in other
     // files (they exist anyway) assumption2: virtual file max frame index
@@ -223,12 +219,22 @@ void DataProcessor::CreateVirtualFile(
         generalData_->nPixelsX, generalData_->nPixelsY, dynamicRange,
         numImagesProcessed, numModX, numModY, dataFile_->GetPDataType(),
         dataFile_->GetParameterNames(), dataFile_->GetParameterDataTypes());
+}
 
+void DataProcessor::LinkDataInMasterFile(const bool silentMode) {
+    std::string fname, datasetName;
+    if (virtualFile_) {
         auto res = virtualFile_->GetFileAndDatasetName();
         fname = res[0];
         datasetName = res[1];
+    } else {
+        auto res = dataFile_->GetFileAndDatasetName();
+        fname = res[0];
+        datasetName = res[1];
     }
     // link in master
+    masterFile_->LinkDataFile(fname, datasetName,
+                              dataFile_->GetParameterNames(), silentMode);
 }
 #endif
 

slsReceiverSoftware/src/DataProcessor.h

@@ -62,6 +62,7 @@ class DataProcessor : private virtual slsDetectorDefs, public ThreadObject {
                           const uint64_t numImages,
                           const uint32_t dynamicRange);
 #ifdef HDF5C
+    uint32_t GetFilesInAcquisition() const;
     void CreateVirtualFile(const std::string filePath,
                            const std::string fileNamePrefix,
                            const uint64_t fileIndex, const bool overWriteEnable,
@@ -71,6 +72,7 @@ class DataProcessor : private virtual slsDetectorDefs, public ThreadObject {
                            const uint64_t numImages,
                            const uint32_t dynamicRange, const int numModX,
                            const int numModY);
+    void LinkDataInMasterFile(const bool silentMode);
 #endif
     /**
      * Call back for raw data

slsReceiverSoftware/src/File.h

@@ -83,6 +83,13 @@ class File : private virtual slsDetectorDefs {
         LOG(logERROR) << "This is a generic function CreateFirstDataFile that "
                          "should be overloaded by a derived class";
     };
+
+    virtual void LinkDataFile(std::string dataFilename, std::string dataSetname,
+                              const std::vector<std::string> parameterNames,
+                              const bool silentMode) {
+        LOG(logERROR) << "This is a generic function LinkDataFile that "
+                         "should be overloaded by a derived class";
+    };
 #endif
     virtual void CreateFirstBinaryDataFile(
         const std::string filePath, const std::string fileNamePrefix,

slsReceiverSoftware/src/HDF5MasterFile.cpp

@@ -21,6 +21,60 @@ void HDF5MasterFile::CloseFile() {
     }
 }
 
+void HDF5MasterFile::LinkDataFile(std::string dataFilename,
+                                  std::string dataSetname,
+                                  const std::vector<std::string> parameterNames,
+                                  const bool silentMode) {
+
+    std::lock_guard<std::mutex> lock(*hdf5Lib_);
+    try {
+        Exception::dontPrint(); // to handle errors
+
+        FileAccPropList flist;
+        flist.setFcloseDegree(H5F_CLOSE_STRONG);
+
+        // open master file
+        H5File masterfd(fileName_.c_str(), H5F_ACC_RDWR,
+                        FileCreatPropList::DEFAULT, flist);
+
+        // open data file
+        H5File fd(dataFilename.c_str(), H5F_ACC_RDONLY,
+                  FileCreatPropList::DEFAULT, flist);
+
+        // create link for data dataset
+        DataSet dset = fd.openDataSet(dataSetname.c_str());
+        std::string linkname = std::string("/entry/data/") + dataSetname;
+        if (H5Lcreate_external(dataFilename.c_str(), dataSetname.c_str(),
+                               masterfd.getLocId(), linkname.c_str(),
+                               H5P_DEFAULT, H5P_DEFAULT) < 0) {
+            throw sls::RuntimeError(
+                "Could not create link to data dataset in master");
+        }
+
+        // create link for parameter datasets
+        for (unsigned int i = 0; i < parameterNames.size(); ++i) {
+            DataSet pDset = fd.openDataSet(parameterNames[i].c_str());
+            linkname = std::string("/entry/data/") + parameterNames[i];
+            if (H5Lcreate_external(dataFilename.c_str(),
+                                   parameterNames[i].c_str(),
+                                   masterfd.getLocId(), linkname.c_str(),
+                                   H5P_DEFAULT, H5P_DEFAULT) < 0) {
+                throw sls::RuntimeError(
+                    "Could not create link to parameter dataset in master");
+            }
+        }
+        fd.close();
+        masterfd.close();
+    } catch (const Exception &error) {
+        error.printErrorStack();
+        CloseFile();
+        throw sls::RuntimeError("Could not link in master hdf5 file");
+    }
+    if (!silentMode) {
+        LOG(logINFO) << "Linked in Master File: " << dataFilename;
+    }
+}
+
 void HDF5MasterFile::CreateMasterFile(const std::string filePath,
                                       const std::string fileNamePrefix,
                                       const uint64_t fileIndex,
@@ -31,7 +85,7 @@ void HDF5MasterFile::CreateMasterFile(const std::string filePath,
     std::ostringstream os;
     os << filePath << "/" << fileNamePrefix << "_master"
        << "_" << fileIndex << ".h5";
-    std::string fileName = os.str();
+    fileName_ = os.str();
 
     std::lock_guard<std::mutex> lock(*hdf5Lib_);
 
@@ -42,10 +96,10 @@ void HDF5MasterFile::CreateMasterFile(const std::string filePath,
         flist.setFcloseDegree(H5F_CLOSE_STRONG);
         fd_ = nullptr;
         if (!(overWriteEnable))
-            fd_ = new H5File(fileName.c_str(), H5F_ACC_EXCL,
+            fd_ = new H5File(fileName_.c_str(), H5F_ACC_EXCL,
                              FileCreatPropList::DEFAULT, flist);
         else
-            fd_ = new H5File(fileName.c_str(), H5F_ACC_TRUNC,
+            fd_ = new H5File(fileName_.c_str(), H5F_ACC_TRUNC,
                              FileCreatPropList::DEFAULT, flist);
 
         // attributes - version
@@ -73,6 +127,6 @@ void HDF5MasterFile::CreateMasterFile(const std::string filePath,
             "Could not create/overwrite master HDF5 handles");
     }
     if (!silentMode) {
-        LOG(logINFO) << "Master File: " << fileName;
+        LOG(logINFO) << "Master File: " << fileName_;
     }
 }

slsReceiverSoftware/src/HDF5MasterFile.h

@@ -11,6 +11,9 @@ class HDF5MasterFile : private virtual slsDetectorDefs, public File {
     ~HDF5MasterFile();
 
     void CloseFile() override;
+    void LinkDataFile(std::string dataFilename, std::string dataSetname,
+                      const std::vector<std::string> parameterNames,
+                      const bool silentMode) override;
     void CreateMasterFile(const std::string filePath,
                           const std::string fileNamePrefix,
                           const uint64_t fileIndex, const bool overWriteEnable,
@@ -20,4 +23,5 @@ class HDF5MasterFile : private virtual slsDetectorDefs, public File {
   private:
     std::mutex *hdf5Lib_;
     H5File *fd_{nullptr};
+    std::string fileName_;
 };

slsReceiverSoftware/src/HDF5VirtualFile.cpp

@@ -97,6 +97,7 @@ void HDF5VirtualFile::CreateVirtualFile(
         if (numImagesCaught % maxFramesPerFile)
             ++numMajorHyperslab;
         uint64_t framesSaved = 0;
+        // loop through files
         for (int hyperSlab = 0; hyperSlab < numMajorHyperslab; ++hyperSlab) {
 
             uint64_t nDimx =
@@ -107,7 +108,7 @@ void HDF5VirtualFile::CreateVirtualFile(
             hsize_t count[3] = {nDimx, nDimy, nDimz};
             hsize_t startPara[2] = {framesSaved, 0};
             hsize_t countPara[2] = {nDimx, 1};
-
+            // loop through readouts
             for (unsigned int i = 0; i < numModY * numModZ; ++i) {
 
                 // setect data hyperslabs
@@ -150,7 +151,7 @@ void HDF5VirtualFile::CreateVirtualFile(
                 // source parameter dataspace
                 hsize_t srcDimsPara[1] = {nDimx};
                 hsize_t srcDimsMaxPara[1] = {H5S_UNLIMITED};
-                DataSpace srcDataSpacePara(3, srcDimsPara, srcDimsMaxPara);
+                DataSpace srcDataSpacePara(1, srcDimsPara, srcDimsMaxPara);
 
                 // mapping of data property list
                 plist.setVirtual(vdsDataSpace, relative_srcFileName.c_str(),

slsReceiverSoftware/src/Implementation.cpp

@@ -557,11 +557,19 @@ void Implementation::stopReceiver() {
 
 #ifdef HDF5C
     if (fileWriteEnable && fileFormatType == HDF5) {
+        if (modulePos == 0) {
+            // more than 1 file, create virtual file
+            if (dataProcessor[0]->GetFilesInAcquisition() > 1 ||
+                (numMods[X] * numMods[Y]) > 1) {
                 dataProcessor[0]->CreateVirtualFile(
                     filePath, fileName, fileIndex, overwriteEnable, silentMode,
                     modulePos, numThreads, framesPerFile, numberOfTotalFrames,
                     dynamicRange, numMods[X], numMods[Y]);
             }
+            // link file in master
+            dataProcessor[0]->LinkDataInMasterFile(silentMode);
+        }
+    }
 #endif
 
     // wait for the processes (dataStreamer) to be done