From b9aa0f46e4d0bfb0749760edd254a1bb378f12de Mon Sep 17 00:00:00 2001 From: Dhanya Thattil Date: Tue, 29 Mar 2022 13:02:57 +0200 Subject: [PATCH] wip, hdf5 refactored --- slsReceiverSoftware/CMakeLists.txt | 3 +- slsReceiverSoftware/src/DataProcessor.cpp | 58 ++---- slsReceiverSoftware/src/DataProcessor.h | 25 ++- slsReceiverSoftware/src/File.h | 21 -- slsReceiverSoftware/src/HDF5MasterFile.cpp | 114 ----------- slsReceiverSoftware/src/HDF5MasterFile.h | 22 --- .../{HDF5VirtualFile.cpp => HDF5Utility.cpp} | 183 ++++++++++++++---- slsReceiverSoftware/src/HDF5Utility.h | 43 ++++ slsReceiverSoftware/src/HDF5VirtualFile.h | 34 ---- slsReceiverSoftware/src/Implementation.cpp | 17 +- slsReceiverSoftware/src/MasterAttributes.cpp | 1 + 11 files changed, 229 insertions(+), 292 deletions(-) delete mode 100644 slsReceiverSoftware/src/HDF5MasterFile.cpp delete mode 100644 slsReceiverSoftware/src/HDF5MasterFile.h rename slsReceiverSoftware/src/{HDF5VirtualFile.cpp => HDF5Utility.cpp} (50%) create mode 100644 slsReceiverSoftware/src/HDF5Utility.h delete mode 100644 slsReceiverSoftware/src/HDF5VirtualFile.h diff --git a/slsReceiverSoftware/CMakeLists.txt b/slsReceiverSoftware/CMakeLists.txt index ef1bfb56d..8589e0314 100755 --- a/slsReceiverSoftware/CMakeLists.txt +++ b/slsReceiverSoftware/CMakeLists.txt @@ -28,8 +28,7 @@ if (SLS_USE_HDF5) ) list (APPEND SOURCES src/HDF5DataFile.cpp - src/HDF5MasterFile.cpp - src/HDF5VirtualFile.cpp + src/HDF5Utility.cpp ) endif (SLS_USE_HDF5) diff --git a/slsReceiverSoftware/src/DataProcessor.cpp b/slsReceiverSoftware/src/DataProcessor.cpp index ed6788981..927213509 100644 --- a/slsReceiverSoftware/src/DataProcessor.cpp +++ b/slsReceiverSoftware/src/DataProcessor.cpp @@ -15,8 +15,7 @@ #include "MasterAttributes.h" #ifdef HDF5C #include "HDF5DataFile.h" -#include "HDF5MasterFile.h" -#include "HDF5VirtualFile.h" +#include "HDF5Utility.h" #endif #include "DataStreamer.h" #include "sls/container_utils.h" @@ -82,10 +81,6 @@ void DataProcessor::SetGeneralData(GeneralData *generalData) { void DataProcessor::CloseFiles() { if (dataFile_) dataFile_->CloseFile(); -#ifdef HDF5C - if (virtualFile_) - virtualFile_->CloseFile(); -#endif } void DataProcessor::DeleteFiles() { @@ -94,12 +89,6 @@ void DataProcessor::DeleteFiles() { delete dataFile_; dataFile_ = nullptr; } -#ifdef HDF5C - if (virtualFile_) { - delete virtualFile_; - virtualFile_ = nullptr; - } -#endif } void DataProcessor::SetupFileWriter(const bool filewriteEnable, const fileFormat fileFormatType, @@ -168,21 +157,17 @@ uint32_t DataProcessor::GetFilesInAcquisition() const { return dataFile_->GetFilesInAcquisition(); } -void DataProcessor::CreateVirtualFile( - const std::string filePath, const std::string fileNamePrefix, +std::array DataProcessor::CreateVirtualFile( + const std::string &filePath, const std::string &fileNamePrefix, const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode, const int modulePos, const int numUnitsPerReadout, const uint32_t maxFramesPerFile, const uint64_t numImages, - const uint32_t dynamicRange, const int numModX, const int numModY, + const int numModX, const int numModY, const uint32_t dynamicRange, std::mutex *hdf5LibMutex) { - if (virtualFile_) { - delete virtualFile_; - } bool gotthard25um = ((detectorType_ == GOTTHARD || detectorType_ == GOTTHARD2) && (numModX * numModY) == 2); - virtualFile_ = new HDF5VirtualFile(hdf5LibMutex, gotthard25um); // maxframesperfile = 0 for infinite files uint32_t framesPerFile = @@ -192,31 +177,30 @@ void DataProcessor::CreateVirtualFile( // files (they exist anyway) assumption2: virtual file max frame index // is from R0 P0 (difference from others when missing frames or for a // stop acquisition) - virtualFile_->CreateVirtualFile( + return hdf5Utility::CreateVirtualFile( filePath, fileNamePrefix, fileIndex, overWriteEnable, silentMode, modulePos, numUnitsPerReadout, framesPerFile, numImages, generalData_->nPixelsX, generalData_->nPixelsY, dynamicRange, numFramesCaught_, numModX, numModY, dataFile_->GetPDataType(), - dataFile_->GetParameterNames(), dataFile_->GetParameterDataTypes()); + dataFile_->GetParameterNames(), dataFile_->GetParameterDataTypes(), + hdf5LibMutex, gotthard25um); } -void DataProcessor::LinkDataInMasterFile(const std::string &masterFileName, - const bool silentMode, - std::mutex *hdf5LibMutex) { - std::string fname, datasetName; - if (virtualFile_) { - auto res = virtualFile_->GetFileAndDatasetName(); - fname = res[0]; - datasetName = res[1]; - } else { +void DataProcessor::LinkFileInMaster(const std::string &masterFileName, + const std::string &virtualFileName, + const std::string &virtualDatasetName, + const bool silentMode, + std::mutex *hdf5LibMutex) { + std::string fname{virtualFileName}, datasetName{virtualDatasetName}; + // if no virtual file, link data file + if (virtualFileName.empty()) { auto res = dataFile_->GetFileAndDatasetName(); fname = res[0]; datasetName = res[1]; } - // link in master - HDF5MasterFile::LinkDataFile(masterFileName, fname, datasetName, - dataFile_->GetParameterNames(), silentMode, - &hdf5LibMutex); + hdf5Utility::LinkFileInMaster(masterFileName, fname, datasetName, + dataFile_->GetParameterNames(), silentMode, + hdf5LibMutex); } #endif @@ -232,9 +216,9 @@ std::string DataProcessor::CreateMasterFile( switch (fileFormatType) { #ifdef HDF5C case HDF5: - return HDF5MasterFile::CreateMasterFile(filePath, fileNamePrefix, - fileIndex, overWriteEnable, - silentMode, attr, hdf5LibMutex); + return hdf5Utility::CreateMasterFile(filePath, fileNamePrefix, + fileIndex, overWriteEnable, + silentMode, attr, hdf5LibMutex); #endif case BINARY: return BinaryMasterFile::CreateMasterFile(filePath, fileNamePrefix, diff --git a/slsReceiverSoftware/src/DataProcessor.h b/slsReceiverSoftware/src/DataProcessor.h index 2c8feed3d..adab81aac 100644 --- a/slsReceiverSoftware/src/DataProcessor.h +++ b/slsReceiverSoftware/src/DataProcessor.h @@ -59,17 +59,17 @@ class DataProcessor : private virtual slsDetectorDefs, public ThreadObject { const bool detectorDataStream); #ifdef HDF5C uint32_t GetFilesInAcquisition() const; - void CreateVirtualFile(const std::string &filePath, - const std::string &fileNamePrefix, - const uint64_t fileIndex, const bool overWriteEnable, - const bool silentMode, const int modulePos, - const int numUnitsPerReadout, - const uint32_t maxFramesPerFile, - const uint64_t numImages, - const uint32_t dynamicRange, const int numModX, - const int numModY, std::mutex *hdf5LibMutex); - void LinkDataInMasterFile(const std::string &masterFileName, - const bool silentMode, std::mutex *hdf5LibMutex); + std::array CreateVirtualFile( + const std::string &filePath, const std::string &fileNamePrefix, + const uint64_t fileIndex, const bool overWriteEnable, + const bool silentMode, const int modulePos, + const int numUnitsPerReadout, const uint32_t maxFramesPerFile, + const uint64_t numImages, const int numModX, const int numModY, + const uint32_t dynamicRange, std::mutex *hdf5LibMutex); + void LinkFileInMaster(const std::string &masterFileName, + const std::string &virtualFileName, + const std::string &virtualDatasetName, + const bool silentMode, std::mutex *hdf5LibMutex); #endif std::string CreateMasterFile(const std::string &filePath, @@ -186,9 +186,6 @@ class DataProcessor : private virtual slsDetectorDefs, public ThreadObject { bool firstStreamerFrame_{false}; File *dataFile_{nullptr}; -#ifdef HDF5C - File *virtualFile_{nullptr}; -#endif // call back /** diff --git a/slsReceiverSoftware/src/File.h b/slsReceiverSoftware/src/File.h index 8edb28052..57282c46a 100644 --- a/slsReceiverSoftware/src/File.h +++ b/slsReceiverSoftware/src/File.h @@ -60,20 +60,6 @@ class File : private virtual slsDetectorDefs { return std::vector{}; }; - virtual void CreateVirtualFile( - const std::string filePath, const std::string fileNamePrefix, - const uint64_t fileIndex, const bool overWriteEnable, - const bool silentMode, const int modulePos, - const int numUnitsPerReadout, const uint32_t maxFramesPerFile, - const uint64_t numImages, const uint32_t nPixelsX, - const uint32_t nPixelsY, const uint32_t dynamicRange, - const uint64_t numImagesCaught, const int numModX, const int numModY, - const DataType dataType, const std::vector parameterNames, - const std::vector parameterDataTypes) { - LOG(logERROR) << "This is a generic function CreateVirtualFile that " - "should be overloaded by a derived class"; - } - virtual void CreateFirstHDF5DataFile( const std::string filePath, const std::string fileNamePrefix, const uint64_t fileIndex, const bool overWriteEnable, @@ -85,13 +71,6 @@ class File : private virtual slsDetectorDefs { LOG(logERROR) << "This is a generic function CreateFirstDataFile that " "should be overloaded by a derived class"; }; - - virtual void LinkDataFile(std::string dataFilename, std::string dataSetname, - const std::vector parameterNames, - const bool silentMode) { - LOG(logERROR) << "This is a generic function LinkDataFile that " - "should be overloaded by a derived class"; - }; #endif virtual void CreateFirstBinaryDataFile( const std::string filePath, const std::string fileNamePrefix, diff --git a/slsReceiverSoftware/src/HDF5MasterFile.cpp b/slsReceiverSoftware/src/HDF5MasterFile.cpp deleted file mode 100644 index 35f15826e..000000000 --- a/slsReceiverSoftware/src/HDF5MasterFile.cpp +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-or-other -// Copyright (C) 2021 Contributors to the SLS Detector Package -#include "HDF5MasterFile.h" -#include "MasterAttributes.h" - -void HDF5MasterFile::LinkDataFile(const std::string &masterFileName, - const std::string &dataFilename, - const std::string &dataSetname, - const std::vector parameterNames, - const bool silentMode, - std::mutex *hdf5LibMutex) { - - std::lock_guard lock(*hdf5Lib_); - try { - Exception::dontPrint(); // to handle errors - - FileAccPropList flist; - flist.setFcloseDegree(H5F_CLOSE_STRONG); - - // open master file - H5File masterfd(masterFileName.c_str(), H5F_ACC_RDWR, - FileCreatPropList::DEFAULT, flist); - - // open data file - H5File fd(dataFilename.c_str(), H5F_ACC_RDONLY, - FileCreatPropList::DEFAULT, flist); - - // create link for data dataset - DataSet dset = fd.openDataSet(dataSetname.c_str()); - std::string linkname = std::string("/entry/data/") + dataSetname; - if (H5Lcreate_external(dataFilename.c_str(), dataSetname.c_str(), - masterfd.getLocId(), linkname.c_str(), - H5P_DEFAULT, H5P_DEFAULT) < 0) { - throw sls::RuntimeError( - "Could not create link to data dataset in master"); - } - - // create link for parameter datasets - for (unsigned int i = 0; i < parameterNames.size(); ++i) { - DataSet pDset = fd.openDataSet(parameterNames[i].c_str()); - linkname = std::string("/entry/data/") + parameterNames[i]; - if (H5Lcreate_external(dataFilename.c_str(), - parameterNames[i].c_str(), - masterfd.getLocId(), linkname.c_str(), - H5P_DEFAULT, H5P_DEFAULT) < 0) { - throw sls::RuntimeError( - "Could not create link to parameter dataset in master"); - } - } - fd.close(); - masterfd.close(); - } catch (const Exception &error) { - error.printErrorStack(); - fd.close(); - throw sls::RuntimeError("Could not link in master hdf5 file"); - } - if (!silentMode) { - LOG(logINFO) << "Linked in Master File: " << dataFilename; - } -} - -std::string HDF5MasterFile::CreateMasterFile( - const std::string filePath, const std::string fileNamePrefix, - const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode, - MasterAttributes *attr, std::mutex *hdf5LibMutex) { - - std::ostringstream os; - os << filePath << "/" << fileNamePrefix << "_master" - << "_" << fileIndex << ".h5"; - std::string fileName = os.str(); - - std::lock_guard lock(*hdf5Lib_); - - try { - Exception::dontPrint(); // to handle errors - - FileAccPropList flist; - flist.setFcloseDegree(H5F_CLOSE_STRONG); - - unsigned int createFlags = H5F_ACC_EXCL; - if (overWriteEnable) { - createFlags = H5F_ACC_TRUNC; - } - H5File fd(fileName.c_str(), createFlags, FileCreatPropList::DEFAULT, - flist); - - // attributes - version - double dValue = HDF5_WRITER_VERSION; - DataSpace dataspace_attr = DataSpace(H5S_SCALAR); - Attribute attribute = fd.createAttribute( - "version", PredType::NATIVE_DOUBLE, dataspace_attr); - attribute.write(PredType::NATIVE_DOUBLE, &dValue); - - // Create a group in the file - Group group1(fd.createGroup("entry")); - Group group2(group1.createGroup("data")); - Group group3(group1.createGroup("instrument")); - Group group4(group3.createGroup("beam")); - Group group5(group3.createGroup("detector")); - Group group6(group1.createGroup("sample")); - - attr->WriteHDF5Attributes(&fd, &group5); - fd.close(); - } catch (const Exception &error) { - error.printErrorStack(); - fd.close(); - throw sls::RuntimeError( - "Could not create/overwrite master HDF5 handles"); - } - if (!silentMode) { - LOG(logINFO) << "Master File: " << fileName; - } - return fileName; -} diff --git a/slsReceiverSoftware/src/HDF5MasterFile.h b/slsReceiverSoftware/src/HDF5MasterFile.h deleted file mode 100644 index 4a9c3ae7c..000000000 --- a/slsReceiverSoftware/src/HDF5MasterFile.h +++ /dev/null @@ -1,22 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-or-other -// Copyright (C) 2021 Contributors to the SLS Detector Package -#pragma once - -#include "MasterAttributes.h" - -#include - -class HDF5MasterFile : private virtual slsDetectorDefs { - public: - void LinkDataFile(const std::string &masterFileName, - const std::string &dataFilename, - const std::string &dataSetname, - const std::vector parameterNames, - const bool silentMode, std::mutex *hdf5LibMutex); - std::string CreateMasterFile(const std::string &filePath, - const std::string &fileNamePrefix, - const uint64_t fileIndex, - const bool overWriteEnable, - const bool silentMode, MasterAttributes *attr, - std::mutex *hdf5LibMutex); -}; \ No newline at end of file diff --git a/slsReceiverSoftware/src/HDF5VirtualFile.cpp b/slsReceiverSoftware/src/HDF5Utility.cpp similarity index 50% rename from slsReceiverSoftware/src/HDF5VirtualFile.cpp rename to slsReceiverSoftware/src/HDF5Utility.cpp index fe1f0d4e3..21891552b 100644 --- a/slsReceiverSoftware/src/HDF5VirtualFile.cpp +++ b/slsReceiverSoftware/src/HDF5Utility.cpp @@ -1,75 +1,172 @@ // SPDX-License-Identifier: LGPL-3.0-or-other // Copyright (C) 2021 Contributors to the SLS Detector Package -#include "HDF5VirtualFile.h" -#include "receiver_defs.h" +#include "HDF5Utility.h" +#include "sls/container_utils.h" #include -HDF5VirtualFile::HDF5VirtualFile(std::mutex *hdf5LibMutex, bool g25) - : File(HDF5), hdf5Lib_(hdf5LibMutex), gotthard25um(g25) {} +namespace hdf5Utility { -HDF5VirtualFile::~HDF5VirtualFile() { CloseFile(); } +void LinkFileInMaster(const std::string &masterFileName, + const std::string &dataFilename, + const std::string &dataSetname, + const std::vector parameterNames, + const bool silentMode, std::mutex *hdf5LibMutex) { -std::array HDF5VirtualFile::GetFileAndDatasetName() const { - return std::array{fileName_, dataSetName_}; -} - -void HDF5VirtualFile::CloseFile() { - std::lock_guard lock(*hdf5Lib_); + std::lock_guard lock(*hdf5LibMutex); + std::unique_ptr fd{nullptr}; try { Exception::dontPrint(); // to handle errors - if (fd_) { - fd_->close(); - delete fd_; - fd_ = nullptr; + + FileAccPropList flist; + flist.setFcloseDegree(H5F_CLOSE_STRONG); + + // open master file + H5File masterfd(masterFileName.c_str(), H5F_ACC_RDWR, + FileCreatPropList::DEFAULT, flist); + + // open data file + fd = sls::make_unique(dataFilename.c_str(), H5F_ACC_RDONLY, + FileCreatPropList::DEFAULT, flist); + + // create link for data dataset + DataSet dset = fd->openDataSet(dataSetname.c_str()); + std::string linkname = std::string("/entry/data/") + dataSetname; + if (H5Lcreate_external(dataFilename.c_str(), dataSetname.c_str(), + masterfd.getLocId(), linkname.c_str(), + H5P_DEFAULT, H5P_DEFAULT) < 0) { + throw sls::RuntimeError( + "Could not create link to data dataset in master"); } + + // create link for parameter datasets + for (unsigned int i = 0; i < parameterNames.size(); ++i) { + DataSet pDset = fd->openDataSet(parameterNames[i].c_str()); + linkname = std::string("/entry/data/") + parameterNames[i]; + if (H5Lcreate_external(dataFilename.c_str(), + parameterNames[i].c_str(), + masterfd.getLocId(), linkname.c_str(), + H5P_DEFAULT, H5P_DEFAULT) < 0) { + throw sls::RuntimeError( + "Could not create link to parameter dataset in master"); + } + } + fd->close(); + masterfd.close(); } catch (const Exception &error) { - LOG(logERROR) << "Could not close virtual HDF5 handles of index"; error.printErrorStack(); + if (fd != nullptr) + fd->close(); + throw sls::RuntimeError("Could not link in master hdf5 file"); + } + if (!silentMode) { + LOG(logINFO) << "Linked in Master File: " << dataFilename; } } -void HDF5VirtualFile::CreateVirtualFile( - const &std::string filePath, const &std::string fileNamePrefix, - const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode, - const int modulePos, const int numUnitsPerReadout, - const uint32_t maxFramesPerFile, const uint64_t numImages, - const uint32_t nPixelsX, const uint32_t nPixelsY, - const uint32_t dynamicRange, const uint64_t numImagesCaught, - const int numModX, const int numModY, const DataType dataType, - const std::vector parameterNames, - const std::vector parameterDataTypes) { +std::string CreateMasterFile(const std::string &filePath, + const std::string &fileNamePrefix, + const uint64_t fileIndex, + const bool overWriteEnable, const bool silentMode, + MasterAttributes *attr, std::mutex *hdf5LibMutex) { + + std::ostringstream os; + os << filePath << "/" << fileNamePrefix << "_master" + << "_" << fileIndex << ".h5"; + std::string fileName = os.str(); + + std::lock_guard lock(*hdf5LibMutex); + + std::unique_ptr fd{nullptr}; + try { + Exception::dontPrint(); // to handle errors + + FileAccPropList flist; + flist.setFcloseDegree(H5F_CLOSE_STRONG); + + unsigned int createFlags = H5F_ACC_EXCL; + if (overWriteEnable) { + createFlags = H5F_ACC_TRUNC; + } + fd = sls::make_unique(fileName.c_str(), createFlags, + FileCreatPropList::DEFAULT, flist); + + // attributes - version + double dValue = HDF5_WRITER_VERSION; + DataSpace dataspace_attr = DataSpace(H5S_SCALAR); + Attribute attribute = fd->createAttribute( + "version", PredType::NATIVE_DOUBLE, dataspace_attr); + attribute.write(PredType::NATIVE_DOUBLE, &dValue); + + // Create a group in the file + Group group1(fd->createGroup("entry")); + Group group2(group1.createGroup("data")); + Group group3(group1.createGroup("instrument")); + Group group4(group3.createGroup("beam")); + Group group5(group3.createGroup("detector")); + Group group6(group1.createGroup("sample")); + + attr->WriteHDF5Attributes(fd.get(), &group5); + fd->close(); + } catch (const Exception &error) { + error.printErrorStack(); + if (fd != nullptr) + fd->close(); + throw sls::RuntimeError( + "Could not create/overwrite master HDF5 handles"); + } + if (!silentMode) { + LOG(logINFO) << "Master File: " << fileName; + } + return fileName; +} + +std::array +CreateVirtualFile(const std::string &filePath, + const std::string &fileNamePrefix, const uint64_t fileIndex, + const bool overWriteEnable, const bool silentMode, + const int modulePos, const int numUnitsPerReadout, + const uint32_t maxFramesPerFile, const uint64_t numImages, + const uint32_t nPixelsX, const uint32_t nPixelsY, + const uint32_t dynamicRange, const uint64_t numImagesCaught, + const int numModX, const int numModY, const DataType dataType, + const std::vector parameterNames, + const std::vector parameterDataTypes, + std::mutex *hdf5LibMutex, bool gotthard25um) { + // virtual file name std::ostringstream osfn; osfn << filePath << "/" << fileNamePrefix << "_virtual" << "_" << fileIndex << ".h5"; - fileName_ = osfn.str(); + std::string fileName = osfn.str(); + + std::string dataSetName = "data"; unsigned int paraSize = parameterNames.size(); uint64_t numModZ = numModX; uint32_t nDimy = nPixelsY; uint32_t nDimz = ((dynamicRange == 4) ? (nPixelsX / 2) : nPixelsX); - std::lock_guard lock(*hdf5Lib_); + std::lock_guard lock(*hdf5LibMutex); + std::unique_ptr fd{nullptr}; try { Exception::dontPrint(); // to handle errors // file FileAccPropList fapl; fapl.setFcloseDegree(H5F_CLOSE_STRONG); - fd_ = nullptr; if (!overWriteEnable) - fd_ = new H5File(fileName_.c_str(), H5F_ACC_EXCL, - FileCreatPropList::DEFAULT, fapl); + fd = sls::make_unique(fileName.c_str(), H5F_ACC_EXCL, + FileCreatPropList::DEFAULT, fapl); else - fd_ = new H5File(fileName_.c_str(), H5F_ACC_TRUNC, - FileCreatPropList::DEFAULT, fapl); + fd = sls::make_unique(fileName.c_str(), H5F_ACC_TRUNC, + FileCreatPropList::DEFAULT, fapl); // attributes - version double dValue = HDF5_WRITER_VERSION; DataSpace dataspace_attr = DataSpace(H5S_SCALAR); - Attribute attribute = fd_->createAttribute( + Attribute attribute = fd->createAttribute( "version", PredType::NATIVE_DOUBLE, dataspace_attr); attribute.write(PredType::NATIVE_DOUBLE, &dValue); @@ -188,24 +285,28 @@ void HDF5VirtualFile::CreateVirtualFile( framesSaved += nDimx; } // datasets - dataSetName_ = "data"; - DataSet vdsDataSet(fd_->createDataSet(dataSetName_.c_str(), dataType, - vdsDataSpace, plist)); + DataSet vdsDataSet(fd->createDataSet(dataSetName.c_str(), dataType, + vdsDataSpace, plist)); for (unsigned int p = 0; p < paraSize; ++p) { - DataSet vdsDataSetPara(fd_->createDataSet( + DataSet vdsDataSetPara(fd->createDataSet( parameterNames[p].c_str(), parameterDataTypes[p], vdsDataSpacePara, plistPara[p])); } - fd_->close(); + fd->close(); } catch (const Exception &error) { error.printErrorStack(); - CloseFile(); + if (fd) { + fd->close(); + } throw sls::RuntimeError( "Could not create/overwrite virtual HDF5 handles"); } if (!silentMode) { - LOG(logINFO) << "Virtual File: " << fileName_; + LOG(logINFO) << "Virtual File: " << fileName; } + return std::array{fileName, dataSetName}; } + +} // namespace hdf5Utility diff --git a/slsReceiverSoftware/src/HDF5Utility.h b/slsReceiverSoftware/src/HDF5Utility.h new file mode 100644 index 000000000..c50e3129f --- /dev/null +++ b/slsReceiverSoftware/src/HDF5Utility.h @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: LGPL-3.0-or-other +// Copyright (C) 2021 Contributors to the SLS Detector Package +#pragma once + +#include "MasterAttributes.h" + +#ifdef HDF5C +#include "H5Cpp.h" +#ifndef H5_NO_NAMESPACE +using namespace H5; +#endif +#endif + +#include + +namespace hdf5Utility { + +void LinkFileInMaster(const std::string &masterFileName, + const std::string &dataFilename, + const std::string &dataSetname, + const std::vector parameterNames, + const bool silentMode, std::mutex *hdf5LibMutex); + +std::string CreateMasterFile(const std::string &filePath, + const std::string &fileNamePrefix, + const uint64_t fileIndex, + const bool overWriteEnable, const bool silentMode, + MasterAttributes *attr, std::mutex *hdf5LibMutex); + +std::array +CreateVirtualFile(const std::string &filePath, + const std::string &fileNamePrefix, const uint64_t fileIndex, + const bool overWriteEnable, const bool silentMode, + const int modulePos, const int numUnitsPerReadout, + const uint32_t maxFramesPerFile, const uint64_t numImages, + const uint32_t nPixelsX, const uint32_t nPixelsY, + const uint32_t dynamicRange, const uint64_t numImagesCaught, + const int numModX, const int numModY, const DataType dataType, + const std::vector parameterNames, + const std::vector parameterDataTypes, + std::mutex *hdf5LibMutex, bool gotthard25um); + +} // namespace hdf5Utility \ No newline at end of file diff --git a/slsReceiverSoftware/src/HDF5VirtualFile.h b/slsReceiverSoftware/src/HDF5VirtualFile.h deleted file mode 100644 index eee01dad5..000000000 --- a/slsReceiverSoftware/src/HDF5VirtualFile.h +++ /dev/null @@ -1,34 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-or-other -// Copyright (C) 2021 Contributors to the SLS Detector Package -#pragma once - -#include "File.h" - -#include - -class HDF5VirtualFile : private virtual slsDetectorDefs, public File { - - public: - HDF5VirtualFile(std::mutex *hdf5LibMutex, bool g25); - ~HDF5VirtualFile(); - - std::array GetFileAndDatasetName() const override; - void CloseFile() override; - void CreateVirtualFile( - const std::string &filePath, const std::string &fileNamePrefix, - const uint64_t fileIndex, const bool overWriteEnable, - const bool silentMode, const int modulePos, - const int numUnitsPerReadout, const uint32_t maxFramesPerFile, - const uint64_t numImages, const uint32_t nPixelsX, - const uint32_t nPixelsY, const uint32_t dynamicRange, - const uint64_t numImagesCaught, const int numModX, const int numModY, - const DataType dataType, const std::vector parameterNames, - const std::vector parameterDataTypes) override; - - private: - std::mutex *hdf5Lib_; - H5File *fd_{nullptr}; - std::string fileName_; - std::string dataSetName_; - bool gotthard25um; -}; \ No newline at end of file diff --git a/slsReceiverSoftware/src/Implementation.cpp b/slsReceiverSoftware/src/Implementation.cpp index ffa10e806..3ec1831d7 100644 --- a/slsReceiverSoftware/src/Implementation.cpp +++ b/slsReceiverSoftware/src/Implementation.cpp @@ -810,19 +810,22 @@ void Implementation::StartMasterWriter() { } #ifdef HDF5C if (fileFormatType == HDF5) { + std::array virtualFileAndDatasetNames; // create virtual hdf5 file (if multiple files) if (dataProcessor[0]->GetFilesInAcquisition() > 1 || (numModules.x * numModules.y) > 1) { - dataProcessor[0]->CreateVirtualFile( - filePath, fileName, fileIndex, overwriteEnable, silentMode, - modulePos, numUDPInterfaces, framesPerFile, - numberOfTotalFrames, dynamicRange, numModules.x, - numModules.y, &hdf5LibMutex); + virtualFileAndDatasetNames = + dataProcessor[0]->CreateVirtualFile( + filePath, fileName, fileIndex, overwriteEnable, + silentMode, modulePos, numUDPInterfaces, framesPerFile, + numberOfTotalFrames, numModules.x, numModules.y, + dynamicRange, &hdf5LibMutex); } // link file in master if (masterFileWriteEnable) { - dataProcessor[0]->LinkDataInMasterFile( - masterFileName, silentMode, &hdf5LibMutex); + dataProcessor[0]->LinkFileInMaster( + masterFileName, virtualFileAndDatasetNames[0], + virtualFileAndDatasetNames[1], silentMode, &hdf5LibMutex); } } #endif diff --git a/slsReceiverSoftware/src/MasterAttributes.cpp b/slsReceiverSoftware/src/MasterAttributes.cpp index 250806cb6..59de67ccd 100644 --- a/slsReceiverSoftware/src/MasterAttributes.cpp +++ b/slsReceiverSoftware/src/MasterAttributes.cpp @@ -385,6 +385,7 @@ void MasterAttributes::WriteHDF5ThresholdEnergy(H5File *fd, Group *group) { } void MasterAttributes::WriteHDF5ThresholdEnergies(H5File *fd, Group *group) { + char c[1024]{0}; DataSpace dataspace = DataSpace(H5S_SCALAR); StrType strdatatype(PredType::C_S1, 1024); DataSet dataset =