detectors/slsDetectorPackage
25
0
Fork 0
mirror of https://github.com/slsdetectorgroup/slsDetectorPackage.git synced 2025-04-24 15:20:02 +02:00
Code Issues Actions Packages Releases Activity

wip, hdf5 refactored

Browse Source
This commit is contained in:
maliakal_d 2022-03-29 13:02:57 +02:00
parent 6cd780ae99
commit b9aa0f46e4
11 changed files with 229 additions and 292 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
slsReceiverSoftware/CMakeLists.txt
View File

@ -28,8 +28,7 @@ if (SLS_USE_HDF5)
) )
list (APPEND SOURCES list (APPEND SOURCES
src/HDF5DataFile.cpp src/HDF5DataFile.cpp
src/HDF5MasterFile.cpp src/HDF5Utility.cpp
src/HDF5VirtualFile.cpp
) )
endif (SLS_USE_HDF5) endif (SLS_USE_HDF5)

48
slsReceiverSoftware/src/DataProcessor.cpp
View File

@ -15,8 +15,7 @@
#include "MasterAttributes.h" #include "MasterAttributes.h"
#ifdef HDF5C #ifdef HDF5C
#include "HDF5DataFile.h" #include "HDF5DataFile.h"
#include "HDF5MasterFile.h" #include "HDF5Utility.h"
#include "HDF5VirtualFile.h"
#endif #endif
#include "DataStreamer.h" #include "DataStreamer.h"
#include "sls/container_utils.h" #include "sls/container_utils.h"
@ -82,10 +81,6 @@ void DataProcessor::SetGeneralData(GeneralData *generalData) {
void DataProcessor::CloseFiles() { void DataProcessor::CloseFiles() {
if (dataFile_) if (dataFile_)
dataFile_->CloseFile(); dataFile_->CloseFile();
#ifdef HDF5C
if (virtualFile_)
virtualFile_->CloseFile();
#endif
} }
void DataProcessor::DeleteFiles() { void DataProcessor::DeleteFiles() {
@ -94,12 +89,6 @@ void DataProcessor::DeleteFiles() {
delete dataFile_; delete dataFile_;
dataFile_ = nullptr; dataFile_ = nullptr;
} }
#ifdef HDF5C
if (virtualFile_) {
delete virtualFile_;
virtualFile_ = nullptr;
}
#endif
} }
void DataProcessor::SetupFileWriter(const bool filewriteEnable, void DataProcessor::SetupFileWriter(const bool filewriteEnable,
const fileFormat fileFormatType, const fileFormat fileFormatType,
@ -168,21 +157,17 @@ uint32_t DataProcessor::GetFilesInAcquisition() const {
return dataFile_->GetFilesInAcquisition(); return dataFile_->GetFilesInAcquisition();
} }
void DataProcessor::CreateVirtualFile( std::array<std::string, 2> DataProcessor::CreateVirtualFile(
const std::string filePath, const std::string fileNamePrefix, const std::string &filePath, const std::string &fileNamePrefix,
const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode, const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode,
const int modulePos, const int numUnitsPerReadout, const int modulePos, const int numUnitsPerReadout,
const uint32_t maxFramesPerFile, const uint64_t numImages, const uint32_t maxFramesPerFile, const uint64_t numImages,
const uint32_t dynamicRange, const int numModX, const int numModY, const int numModX, const int numModY, const uint32_t dynamicRange,
std::mutex *hdf5LibMutex) { std::mutex *hdf5LibMutex) {
if (virtualFile_) {
delete virtualFile_;
}
bool gotthard25um = bool gotthard25um =
((detectorType_ == GOTTHARD || detectorType_ == GOTTHARD2) && ((detectorType_ == GOTTHARD || detectorType_ == GOTTHARD2) &&
(numModX * numModY) == 2); (numModX * numModY) == 2);
virtualFile_ = new HDF5VirtualFile(hdf5LibMutex, gotthard25um);
// maxframesperfile = 0 for infinite files // maxframesperfile = 0 for infinite files
uint32_t framesPerFile = uint32_t framesPerFile =
@ -192,31 +177,30 @@ void DataProcessor::CreateVirtualFile(
// files (they exist anyway) assumption2: virtual file max frame index // files (they exist anyway) assumption2: virtual file max frame index
// is from R0 P0 (difference from others when missing frames or for a // is from R0 P0 (difference from others when missing frames or for a
// stop acquisition) // stop acquisition)
virtualFile_->CreateVirtualFile( return hdf5Utility::CreateVirtualFile(
filePath, fileNamePrefix, fileIndex, overWriteEnable, silentMode, filePath, fileNamePrefix, fileIndex, overWriteEnable, silentMode,
modulePos, numUnitsPerReadout, framesPerFile, numImages, modulePos, numUnitsPerReadout, framesPerFile, numImages,
generalData_->nPixelsX, generalData_->nPixelsY, dynamicRange, generalData_->nPixelsX, generalData_->nPixelsY, dynamicRange,
numFramesCaught_, numModX, numModY, dataFile_->GetPDataType(), numFramesCaught_, numModX, numModY, dataFile_->GetPDataType(),
dataFile_->GetParameterNames(), dataFile_->GetParameterDataTypes()); dataFile_->GetParameterNames(), dataFile_->GetParameterDataTypes(),
hdf5LibMutex, gotthard25um);
} }
void DataProcessor::LinkDataInMasterFile(const std::string &masterFileName, void DataProcessor::LinkFileInMaster(const std::string &masterFileName,
const std::string &virtualFileName,
const std::string &virtualDatasetName,
const bool silentMode, const bool silentMode,
std::mutex *hdf5LibMutex) { std::mutex *hdf5LibMutex) {
std::string fname, datasetName; std::string fname{virtualFileName}, datasetName{virtualDatasetName};
if (virtualFile_) { // if no virtual file, link data file
auto res = virtualFile_->GetFileAndDatasetName(); if (virtualFileName.empty()) {
fname = res[0];
datasetName = res[1];
} else {
auto res = dataFile_->GetFileAndDatasetName(); auto res = dataFile_->GetFileAndDatasetName();
fname = res[0]; fname = res[0];
datasetName = res[1]; datasetName = res[1];
} }
// link in master hdf5Utility::LinkFileInMaster(masterFileName, fname, datasetName,
HDF5MasterFile::LinkDataFile(masterFileName, fname, datasetName,
dataFile_->GetParameterNames(), silentMode, dataFile_->GetParameterNames(), silentMode,
&hdf5LibMutex); hdf5LibMutex);
} }
#endif #endif
@ -232,7 +216,7 @@ std::string DataProcessor::CreateMasterFile(
switch (fileFormatType) { switch (fileFormatType) {
#ifdef HDF5C #ifdef HDF5C
case HDF5: case HDF5:
return HDF5MasterFile::CreateMasterFile(filePath, fileNamePrefix, return hdf5Utility::CreateMasterFile(filePath, fileNamePrefix,
fileIndex, overWriteEnable, fileIndex, overWriteEnable,
silentMode, attr, hdf5LibMutex); silentMode, attr, hdf5LibMutex);
#endif #endif

19
slsReceiverSoftware/src/DataProcessor.h
View File

@ -59,16 +59,16 @@ class DataProcessor : private virtual slsDetectorDefs, public ThreadObject {
const bool detectorDataStream); const bool detectorDataStream);
#ifdef HDF5C #ifdef HDF5C
uint32_t GetFilesInAcquisition() const; uint32_t GetFilesInAcquisition() const;
void CreateVirtualFile(const std::string &filePath, std::array<std::string, 2> CreateVirtualFile(
const std::string &fileNamePrefix, const std::string &filePath, const std::string &fileNamePrefix,
const uint64_t fileIndex, const bool overWriteEnable, const uint64_t fileIndex, const bool overWriteEnable,
const bool silentMode, const int modulePos, const bool silentMode, const int modulePos,
const int numUnitsPerReadout, const int numUnitsPerReadout, const uint32_t maxFramesPerFile,
const uint32_t maxFramesPerFile, const uint64_t numImages, const int numModX, const int numModY,
const uint64_t numImages, const uint32_t dynamicRange, std::mutex *hdf5LibMutex);
const uint32_t dynamicRange, const int numModX, void LinkFileInMaster(const std::string &masterFileName,
const int numModY, std::mutex *hdf5LibMutex); const std::string &virtualFileName,
void LinkDataInMasterFile(const std::string &masterFileName, const std::string &virtualDatasetName,
const bool silentMode, std::mutex *hdf5LibMutex); const bool silentMode, std::mutex *hdf5LibMutex);
#endif #endif
@ -186,9 +186,6 @@ class DataProcessor : private virtual slsDetectorDefs, public ThreadObject {
bool firstStreamerFrame_{false}; bool firstStreamerFrame_{false};
File *dataFile_{nullptr}; File *dataFile_{nullptr};
#ifdef HDF5C
File *virtualFile_{nullptr};
#endif
// call back // call back
/** /**

21
slsReceiverSoftware/src/File.h
View File

@ -60,20 +60,6 @@ class File : private virtual slsDetectorDefs {
return std::vector<DataType>{}; return std::vector<DataType>{};
}; };
virtual void CreateVirtualFile(
const std::string filePath, const std::string fileNamePrefix,
const uint64_t fileIndex, const bool overWriteEnable,
const bool silentMode, const int modulePos,
const int numUnitsPerReadout, const uint32_t maxFramesPerFile,
const uint64_t numImages, const uint32_t nPixelsX,
const uint32_t nPixelsY, const uint32_t dynamicRange,
const uint64_t numImagesCaught, const int numModX, const int numModY,
const DataType dataType, const std::vector<std::string> parameterNames,
const std::vector<DataType> parameterDataTypes) {
LOG(logERROR) << "This is a generic function CreateVirtualFile that "
"should be overloaded by a derived class";
}
virtual void CreateFirstHDF5DataFile( virtual void CreateFirstHDF5DataFile(
const std::string filePath, const std::string fileNamePrefix, const std::string filePath, const std::string fileNamePrefix,
const uint64_t fileIndex, const bool overWriteEnable, const uint64_t fileIndex, const bool overWriteEnable,
@ -85,13 +71,6 @@ class File : private virtual slsDetectorDefs {
LOG(logERROR) << "This is a generic function CreateFirstDataFile that " LOG(logERROR) << "This is a generic function CreateFirstDataFile that "
"should be overloaded by a derived class"; "should be overloaded by a derived class";
}; };
virtual void LinkDataFile(std::string dataFilename, std::string dataSetname,
const std::vector<std::string> parameterNames,
const bool silentMode) {
LOG(logERROR) << "This is a generic function LinkDataFile that "
"should be overloaded by a derived class";
};
#endif #endif
virtual void CreateFirstBinaryDataFile( virtual void CreateFirstBinaryDataFile(
const std::string filePath, const std::string fileNamePrefix, const std::string filePath, const std::string fileNamePrefix,

114
slsReceiverSoftware/src/HDF5MasterFile.cpp
View File

@ -1,114 +0,0 @@
// SPDX-License-Identifier: LGPL-3.0-or-other
// Copyright (C) 2021 Contributors to the SLS Detector Package
#include "HDF5MasterFile.h"
#include "MasterAttributes.h"
void HDF5MasterFile::LinkDataFile(const std::string &masterFileName,
const std::string &dataFilename,
const std::string &dataSetname,
const std::vector<std::string> parameterNames,
const bool silentMode,
std::mutex *hdf5LibMutex) {
std::lock_guard<std::mutex> lock(*hdf5Lib_);
try {
Exception::dontPrint(); // to handle errors
FileAccPropList flist;
flist.setFcloseDegree(H5F_CLOSE_STRONG);
// open master file
H5File masterfd(masterFileName.c_str(), H5F_ACC_RDWR,
FileCreatPropList::DEFAULT, flist);
// open data file
H5File fd(dataFilename.c_str(), H5F_ACC_RDONLY,
FileCreatPropList::DEFAULT, flist);
// create link for data dataset
DataSet dset = fd.openDataSet(dataSetname.c_str());
std::string linkname = std::string("/entry/data/") + dataSetname;
if (H5Lcreate_external(dataFilename.c_str(), dataSetname.c_str(),
masterfd.getLocId(), linkname.c_str(),
H5P_DEFAULT, H5P_DEFAULT) < 0) {
throw sls::RuntimeError(
"Could not create link to data dataset in master");
}
// create link for parameter datasets
for (unsigned int i = 0; i < parameterNames.size(); ++i) {
DataSet pDset = fd.openDataSet(parameterNames[i].c_str());
linkname = std::string("/entry/data/") + parameterNames[i];
if (H5Lcreate_external(dataFilename.c_str(),
parameterNames[i].c_str(),
masterfd.getLocId(), linkname.c_str(),
H5P_DEFAULT, H5P_DEFAULT) < 0) {
throw sls::RuntimeError(
"Could not create link to parameter dataset in master");
}
}
fd.close();
masterfd.close();
} catch (const Exception &error) {
error.printErrorStack();
fd.close();
throw sls::RuntimeError("Could not link in master hdf5 file");
}
if (!silentMode) {
LOG(logINFO) << "Linked in Master File: " << dataFilename;
}
}
std::string HDF5MasterFile::CreateMasterFile(
const std::string filePath, const std::string fileNamePrefix,
const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode,
MasterAttributes *attr, std::mutex *hdf5LibMutex) {
std::ostringstream os;
os << filePath << "/" << fileNamePrefix << "_master"
<< "_" << fileIndex << ".h5";
std::string fileName = os.str();
std::lock_guard<std::mutex> lock(*hdf5Lib_);
try {
Exception::dontPrint(); // to handle errors
FileAccPropList flist;
flist.setFcloseDegree(H5F_CLOSE_STRONG);
unsigned int createFlags = H5F_ACC_EXCL;
if (overWriteEnable) {
createFlags = H5F_ACC_TRUNC;
}
H5File fd(fileName.c_str(), createFlags, FileCreatPropList::DEFAULT,
flist);
// attributes - version
double dValue = HDF5_WRITER_VERSION;
DataSpace dataspace_attr = DataSpace(H5S_SCALAR);
Attribute attribute = fd.createAttribute(
"version", PredType::NATIVE_DOUBLE, dataspace_attr);
attribute.write(PredType::NATIVE_DOUBLE, &dValue);
// Create a group in the file
Group group1(fd.createGroup("entry"));
Group group2(group1.createGroup("data"));
Group group3(group1.createGroup("instrument"));
Group group4(group3.createGroup("beam"));
Group group5(group3.createGroup("detector"));
Group group6(group1.createGroup("sample"));
attr->WriteHDF5Attributes(&fd, &group5);
fd.close();
} catch (const Exception &error) {
error.printErrorStack();
fd.close();
throw sls::RuntimeError(
"Could not create/overwrite master HDF5 handles");
}
if (!silentMode) {
LOG(logINFO) << "Master File: " << fileName;
}
return fileName;
}

22
slsReceiverSoftware/src/HDF5MasterFile.h
View File

@ -1,22 +0,0 @@
// SPDX-License-Identifier: LGPL-3.0-or-other
// Copyright (C) 2021 Contributors to the SLS Detector Package
#pragma once
#include "MasterAttributes.h"
#include <mutex>
class HDF5MasterFile : private virtual slsDetectorDefs {
public:
void LinkDataFile(const std::string &masterFileName,
const std::string &dataFilename,
const std::string &dataSetname,
const std::vector<std::string> parameterNames,
const bool silentMode, std::mutex *hdf5LibMutex);
std::string CreateMasterFile(const std::string &filePath,
const std::string &fileNamePrefix,
const uint64_t fileIndex,
const bool overWriteEnable,
const bool silentMode, MasterAttributes *attr,
std::mutex *hdf5LibMutex);
};

165
slsReceiverSoftware/src/HDF5VirtualFile.cpp → slsReceiverSoftware/src/HDF5Utility.cpp
View File

@ -1,75 +1,172 @@
// SPDX-License-Identifier: LGPL-3.0-or-other // SPDX-License-Identifier: LGPL-3.0-or-other
// Copyright (C) 2021 Contributors to the SLS Detector Package // Copyright (C) 2021 Contributors to the SLS Detector Package
#include "HDF5VirtualFile.h" #include "HDF5Utility.h"
#include "receiver_defs.h" #include "sls/container_utils.h"
#include <iomanip> #include <iomanip>
HDF5VirtualFile::HDF5VirtualFile(std::mutex *hdf5LibMutex, bool g25) namespace hdf5Utility {
: File(HDF5), hdf5Lib_(hdf5LibMutex), gotthard25um(g25) {}
HDF5VirtualFile::~HDF5VirtualFile() { CloseFile(); } void LinkFileInMaster(const std::string &masterFileName,
const std::string &dataFilename,
const std::string &dataSetname,
const std::vector<std::string> parameterNames,
const bool silentMode, std::mutex *hdf5LibMutex) {
std::array<std::string, 2> HDF5VirtualFile::GetFileAndDatasetName() const { std::lock_guard<std::mutex> lock(*hdf5LibMutex);
return std::array<std::string, 2>{fileName_, dataSetName_}; std::unique_ptr<H5File> fd{nullptr};
}
void HDF5VirtualFile::CloseFile() {
std::lock_guard<std::mutex> lock(*hdf5Lib_);
try { try {
Exception::dontPrint(); // to handle errors Exception::dontPrint(); // to handle errors
if (fd_) {
fd_->close(); FileAccPropList flist;
delete fd_; flist.setFcloseDegree(H5F_CLOSE_STRONG);
fd_ = nullptr;
// open master file
H5File masterfd(masterFileName.c_str(), H5F_ACC_RDWR,
FileCreatPropList::DEFAULT, flist);
// open data file
fd = sls::make_unique<H5File>(dataFilename.c_str(), H5F_ACC_RDONLY,
FileCreatPropList::DEFAULT, flist);
// create link for data dataset
DataSet dset = fd->openDataSet(dataSetname.c_str());
std::string linkname = std::string("/entry/data/") + dataSetname;
if (H5Lcreate_external(dataFilename.c_str(), dataSetname.c_str(),
masterfd.getLocId(), linkname.c_str(),
H5P_DEFAULT, H5P_DEFAULT) < 0) {
throw sls::RuntimeError(
"Could not create link to data dataset in master");
} }
// create link for parameter datasets
for (unsigned int i = 0; i < parameterNames.size(); ++i) {
DataSet pDset = fd->openDataSet(parameterNames[i].c_str());
linkname = std::string("/entry/data/") + parameterNames[i];
if (H5Lcreate_external(dataFilename.c_str(),
parameterNames[i].c_str(),
masterfd.getLocId(), linkname.c_str(),
H5P_DEFAULT, H5P_DEFAULT) < 0) {
throw sls::RuntimeError(
"Could not create link to parameter dataset in master");
}
}
fd->close();
masterfd.close();
} catch (const Exception &error) { } catch (const Exception &error) {
LOG(logERROR) << "Could not close virtual HDF5 handles of index";
error.printErrorStack(); error.printErrorStack();
if (fd != nullptr)
fd->close();
throw sls::RuntimeError("Could not link in master hdf5 file");
}
if (!silentMode) {
LOG(logINFO) << "Linked in Master File: " << dataFilename;
} }
} }
void HDF5VirtualFile::CreateVirtualFile( std::string CreateMasterFile(const std::string &filePath,
const &std::string filePath, const &std::string fileNamePrefix, const std::string &fileNamePrefix,
const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode, const uint64_t fileIndex,
const bool overWriteEnable, const bool silentMode,
MasterAttributes *attr, std::mutex *hdf5LibMutex) {
std::ostringstream os;
os << filePath << "/" << fileNamePrefix << "_master"
<< "_" << fileIndex << ".h5";
std::string fileName = os.str();
std::lock_guard<std::mutex> lock(*hdf5LibMutex);
std::unique_ptr<H5File> fd{nullptr};
try {
Exception::dontPrint(); // to handle errors
FileAccPropList flist;
flist.setFcloseDegree(H5F_CLOSE_STRONG);
unsigned int createFlags = H5F_ACC_EXCL;
if (overWriteEnable) {
createFlags = H5F_ACC_TRUNC;
}
fd = sls::make_unique<H5File>(fileName.c_str(), createFlags,
FileCreatPropList::DEFAULT, flist);
// attributes - version
double dValue = HDF5_WRITER_VERSION;
DataSpace dataspace_attr = DataSpace(H5S_SCALAR);
Attribute attribute = fd->createAttribute(
"version", PredType::NATIVE_DOUBLE, dataspace_attr);
attribute.write(PredType::NATIVE_DOUBLE, &dValue);
// Create a group in the file
Group group1(fd->createGroup("entry"));
Group group2(group1.createGroup("data"));
Group group3(group1.createGroup("instrument"));
Group group4(group3.createGroup("beam"));
Group group5(group3.createGroup("detector"));
Group group6(group1.createGroup("sample"));
attr->WriteHDF5Attributes(fd.get(), &group5);
fd->close();
} catch (const Exception &error) {
error.printErrorStack();
if (fd != nullptr)
fd->close();
throw sls::RuntimeError(
"Could not create/overwrite master HDF5 handles");
}
if (!silentMode) {
LOG(logINFO) << "Master File: " << fileName;
}
return fileName;
}
std::array<std::string, 2>
CreateVirtualFile(const std::string &filePath,
const std::string &fileNamePrefix, const uint64_t fileIndex,
const bool overWriteEnable, const bool silentMode,
const int modulePos, const int numUnitsPerReadout, const int modulePos, const int numUnitsPerReadout,
const uint32_t maxFramesPerFile, const uint64_t numImages, const uint32_t maxFramesPerFile, const uint64_t numImages,
const uint32_t nPixelsX, const uint32_t nPixelsY, const uint32_t nPixelsX, const uint32_t nPixelsY,
const uint32_t dynamicRange, const uint64_t numImagesCaught, const uint32_t dynamicRange, const uint64_t numImagesCaught,
const int numModX, const int numModY, const DataType dataType, const int numModX, const int numModY, const DataType dataType,
const std::vector<std::string> parameterNames, const std::vector<std::string> parameterNames,
const std::vector<DataType> parameterDataTypes) { const std::vector<DataType> parameterDataTypes,
std::mutex *hdf5LibMutex, bool gotthard25um) {
// virtual file name // virtual file name
std::ostringstream osfn; std::ostringstream osfn;
osfn << filePath << "/" << fileNamePrefix << "_virtual" osfn << filePath << "/" << fileNamePrefix << "_virtual"
<< "_" << fileIndex << ".h5"; << "_" << fileIndex << ".h5";
fileName_ = osfn.str(); std::string fileName = osfn.str();
std::string dataSetName = "data";
unsigned int paraSize = parameterNames.size(); unsigned int paraSize = parameterNames.size();
uint64_t numModZ = numModX; uint64_t numModZ = numModX;
uint32_t nDimy = nPixelsY; uint32_t nDimy = nPixelsY;
uint32_t nDimz = ((dynamicRange == 4) ? (nPixelsX / 2) : nPixelsX); uint32_t nDimz = ((dynamicRange == 4) ? (nPixelsX / 2) : nPixelsX);
std::lock_guard<std::mutex> lock(*hdf5Lib_); std::lock_guard<std::mutex> lock(*hdf5LibMutex);
std::unique_ptr<H5File> fd{nullptr};
try { try {
Exception::dontPrint(); // to handle errors Exception::dontPrint(); // to handle errors
// file // file
FileAccPropList fapl; FileAccPropList fapl;
fapl.setFcloseDegree(H5F_CLOSE_STRONG); fapl.setFcloseDegree(H5F_CLOSE_STRONG);
fd_ = nullptr;
if (!overWriteEnable) if (!overWriteEnable)
fd_ = new H5File(fileName_.c_str(), H5F_ACC_EXCL, fd = sls::make_unique<H5File>(fileName.c_str(), H5F_ACC_EXCL,
FileCreatPropList::DEFAULT, fapl); FileCreatPropList::DEFAULT, fapl);
else else
fd_ = new H5File(fileName_.c_str(), H5F_ACC_TRUNC, fd = sls::make_unique<H5File>(fileName.c_str(), H5F_ACC_TRUNC,
FileCreatPropList::DEFAULT, fapl); FileCreatPropList::DEFAULT, fapl);
// attributes - version // attributes - version
double dValue = HDF5_WRITER_VERSION; double dValue = HDF5_WRITER_VERSION;
DataSpace dataspace_attr = DataSpace(H5S_SCALAR); DataSpace dataspace_attr = DataSpace(H5S_SCALAR);
Attribute attribute = fd_->createAttribute( Attribute attribute = fd->createAttribute(
"version", PredType::NATIVE_DOUBLE, dataspace_attr); "version", PredType::NATIVE_DOUBLE, dataspace_attr);
attribute.write(PredType::NATIVE_DOUBLE, &dValue); attribute.write(PredType::NATIVE_DOUBLE, &dValue);
@ -188,24 +285,28 @@ void HDF5VirtualFile::CreateVirtualFile(
framesSaved += nDimx; framesSaved += nDimx;
} }
// datasets // datasets
dataSetName_ = "data"; DataSet vdsDataSet(fd->createDataSet(dataSetName.c_str(), dataType,
DataSet vdsDataSet(fd_->createDataSet(dataSetName_.c_str(), dataType,
vdsDataSpace, plist)); vdsDataSpace, plist));
for (unsigned int p = 0; p < paraSize; ++p) { for (unsigned int p = 0; p < paraSize; ++p) {
DataSet vdsDataSetPara(fd_->createDataSet( DataSet vdsDataSetPara(fd->createDataSet(
parameterNames[p].c_str(), parameterDataTypes[p], parameterNames[p].c_str(), parameterDataTypes[p],
vdsDataSpacePara, plistPara[p])); vdsDataSpacePara, plistPara[p]));
} }
fd_->close(); fd->close();
} catch (const Exception &error) { } catch (const Exception &error) {
error.printErrorStack(); error.printErrorStack();
CloseFile(); if (fd) {
fd->close();
}
throw sls::RuntimeError( throw sls::RuntimeError(
"Could not create/overwrite virtual HDF5 handles"); "Could not create/overwrite virtual HDF5 handles");
} }
if (!silentMode) { if (!silentMode) {
LOG(logINFO) << "Virtual File: " << fileName_; LOG(logINFO) << "Virtual File: " << fileName;
} }
return std::array<std::string, 2>{fileName, dataSetName};
} }
} // namespace hdf5Utility

43
slsReceiverSoftware/src/HDF5Utility.h Normal file
View File

@ -0,0 +1,43 @@
// SPDX-License-Identifier: LGPL-3.0-or-other
// Copyright (C) 2021 Contributors to the SLS Detector Package
#pragma once
#include "MasterAttributes.h"
#ifdef HDF5C
#include "H5Cpp.h"
#ifndef H5_NO_NAMESPACE
using namespace H5;
#endif
#endif
#include <mutex>
namespace hdf5Utility {
void LinkFileInMaster(const std::string &masterFileName,
const std::string &dataFilename,
const std::string &dataSetname,
const std::vector<std::string> parameterNames,
const bool silentMode, std::mutex *hdf5LibMutex);
std::string CreateMasterFile(const std::string &filePath,
const std::string &fileNamePrefix,
const uint64_t fileIndex,
const bool overWriteEnable, const bool silentMode,
MasterAttributes *attr, std::mutex *hdf5LibMutex);
std::array<std::string, 2>
CreateVirtualFile(const std::string &filePath,
const std::string &fileNamePrefix, const uint64_t fileIndex,
const bool overWriteEnable, const bool silentMode,
const int modulePos, const int numUnitsPerReadout,
const uint32_t maxFramesPerFile, const uint64_t numImages,
const uint32_t nPixelsX, const uint32_t nPixelsY,
const uint32_t dynamicRange, const uint64_t numImagesCaught,
const int numModX, const int numModY, const DataType dataType,
const std::vector<std::string> parameterNames,
const std::vector<DataType> parameterDataTypes,
std::mutex *hdf5LibMutex, bool gotthard25um);
} // namespace hdf5Utility

34
slsReceiverSoftware/src/HDF5VirtualFile.h
View File

@ -1,34 +0,0 @@
// SPDX-License-Identifier: LGPL-3.0-or-other
// Copyright (C) 2021 Contributors to the SLS Detector Package
#pragma once
#include "File.h"
#include <mutex>
class HDF5VirtualFile : private virtual slsDetectorDefs, public File {
public:
HDF5VirtualFile(std::mutex *hdf5LibMutex, bool g25);
~HDF5VirtualFile();
std::array<std::string, 2> GetFileAndDatasetName() const override;
void CloseFile() override;
void CreateVirtualFile(
const std::string &filePath, const std::string &fileNamePrefix,
const uint64_t fileIndex, const bool overWriteEnable,
const bool silentMode, const int modulePos,
const int numUnitsPerReadout, const uint32_t maxFramesPerFile,
const uint64_t numImages, const uint32_t nPixelsX,
const uint32_t nPixelsY, const uint32_t dynamicRange,
const uint64_t numImagesCaught, const int numModX, const int numModY,
const DataType dataType, const std::vector<std::string> parameterNames,
const std::vector<DataType> parameterDataTypes) override;
private:
std::mutex *hdf5Lib_;
H5File *fd_{nullptr};
std::string fileName_;
std::string dataSetName_;
bool gotthard25um;
};

15
slsReceiverSoftware/src/Implementation.cpp
View File

@ -810,19 +810,22 @@ void Implementation::StartMasterWriter() {
} }
#ifdef HDF5C #ifdef HDF5C
if (fileFormatType == HDF5) { if (fileFormatType == HDF5) {
std::array<std::string, 2> virtualFileAndDatasetNames;
// create virtual hdf5 file (if multiple files) // create virtual hdf5 file (if multiple files)
if (dataProcessor[0]->GetFilesInAcquisition() > 1 || if (dataProcessor[0]->GetFilesInAcquisition() > 1 ||
(numModules.x * numModules.y) > 1) { (numModules.x * numModules.y) > 1) {
virtualFileAndDatasetNames =
dataProcessor[0]->CreateVirtualFile( dataProcessor[0]->CreateVirtualFile(
filePath, fileName, fileIndex, overwriteEnable, silentMode, filePath, fileName, fileIndex, overwriteEnable,
modulePos, numUDPInterfaces, framesPerFile, silentMode, modulePos, numUDPInterfaces, framesPerFile,
numberOfTotalFrames, dynamicRange, numModules.x, numberOfTotalFrames, numModules.x, numModules.y,
numModules.y, &hdf5LibMutex); dynamicRange, &hdf5LibMutex);
} }
// link file in master // link file in master
if (masterFileWriteEnable) { if (masterFileWriteEnable) {
dataProcessor[0]->LinkDataInMasterFile( dataProcessor[0]->LinkFileInMaster(
masterFileName, silentMode, &hdf5LibMutex); masterFileName, virtualFileAndDatasetNames[0],
virtualFileAndDatasetNames[1], silentMode, &hdf5LibMutex);
} }
} }
#endif #endif

1
slsReceiverSoftware/src/MasterAttributes.cpp
View File

@ -385,6 +385,7 @@ void MasterAttributes::WriteHDF5ThresholdEnergy(H5File *fd, Group *group) {
} }
void MasterAttributes::WriteHDF5ThresholdEnergies(H5File *fd, Group *group) { void MasterAttributes::WriteHDF5ThresholdEnergies(H5File *fd, Group *group) {
char c[1024]{0};
DataSpace dataspace = DataSpace(H5S_SCALAR); DataSpace dataspace = DataSpace(H5S_SCALAR);
StrType strdatatype(PredType::C_S1, 1024); StrType strdatatype(PredType::C_S1, 1024);
DataSet dataset = DataSet dataset =