detectors/slsDetectorPackage
25
0
Fork 0
mirror of https://github.com/slsdetectorgroup/slsDetectorPackage.git synced 2025-04-22 03:40:04 +02:00
Code Issues Actions Packages Releases Activity

wip

Browse Source
This commit is contained in:
maliakal_d 2021-07-02 19:49:37 +02:00
parent c30716db12
commit 2d913b901d
7 changed files with 124 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
slsReceiverSoftware/src/DataProcessor.cpp
View File

@ -129,7 +129,6 @@ void DataProcessor::SetupFileWriter(const bool filewriteEnable,
if (masterFilewriteEnable) {
masterFile_ = new HDF5MasterFile(hdf5Lib_);
}
virtualFile_ = new HDF5VirtualFile(hdf5Lib_);
}
break;
#endif
@ -185,33 +184,30 @@ void DataProcessor::CreateFirstFiles(
}
#ifdef HDF5C
uint32_t DataProcessor::GetFilesInAcquisition() const {
if (dataFile_ == nullptr) {
throw sls::RuntimeError("No data file object created to get number of "
"files in acquiistion");
}
return dataFile_->GetFilesInAcquisition();
}
void DataProcessor::CreateVirtualFile(
const std::string filePath, const std::string fileNamePrefix,
const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode,
const int modulePos, const int numUnitsPerReadout,
const uint32_t maxFramesPerFile, const uint64_t numImages,
const uint32_t dynamicRange, const int numModX, const int numModY) {
if (virtualFile_ == nullptr || dataFile_ == nullptr) {
throw sls::RuntimeError("No virtual or data file object created");
}
std::string fname, datasetName;
// only 1 file (subfile and number of files in acquisition) (to link in
// master)
if (dataFile_->GetFilesInAcquisition() == 1 && (numModX * numModY) == 1) {
auto res = dataFile_->GetFileAndDatasetName();
fname = res[0];
datasetName = res[1];
if (virtualFile_) {
delete virtualFile_;
}
// create virtual file and link that in master
else {
virtualFile_ = new HDF5VirtualFile(hdf5Lib_);
uint64_t numImagesProcessed = GetProcessedIndex() + 1;
// maxframesperfile = 0 for infinite files
uint32_t framesPerFile =
((maxFramesPerFile == 0) ? numImagesProcessed + 1
: maxFramesPerFile);
((maxFramesPerFile == 0) ? numImagesProcessed + 1 : maxFramesPerFile);
// TODO: assumption 1: create virtual file even if no data in other
// files (they exist anyway) assumption2: virtual file max frame index
@ -223,12 +219,22 @@ void DataProcessor::CreateVirtualFile(
generalData_->nPixelsX, generalData_->nPixelsY, dynamicRange,
numImagesProcessed, numModX, numModY, dataFile_->GetPDataType(),
dataFile_->GetParameterNames(), dataFile_->GetParameterDataTypes());
}
void DataProcessor::LinkDataInMasterFile(const bool silentMode) {
std::string fname, datasetName;
if (virtualFile_) {
auto res = virtualFile_->GetFileAndDatasetName();
fname = res[0];
datasetName = res[1];
} else {
auto res = dataFile_->GetFileAndDatasetName();
fname = res[0];
datasetName = res[1];
}
// link in master
masterFile_->LinkDataFile(fname, datasetName,
dataFile_->GetParameterNames(), silentMode);
}
#endif

2
slsReceiverSoftware/src/DataProcessor.h
View File

@ -62,6 +62,7 @@ class DataProcessor : private virtual slsDetectorDefs, public ThreadObject {
const uint64_t numImages,
const uint32_t dynamicRange);
#ifdef HDF5C
uint32_t GetFilesInAcquisition() const;
void CreateVirtualFile(const std::string filePath,
const std::string fileNamePrefix,
const uint64_t fileIndex, const bool overWriteEnable,
@ -71,6 +72,7 @@ class DataProcessor : private virtual slsDetectorDefs, public ThreadObject {
const uint64_t numImages,
const uint32_t dynamicRange, const int numModX,
const int numModY);
void LinkDataInMasterFile(const bool silentMode);
#endif
/**
* Call back for raw data

7
slsReceiverSoftware/src/File.h
View File

@ -83,6 +83,13 @@ class File : private virtual slsDetectorDefs {
LOG(logERROR) << "This is a generic function CreateFirstDataFile that "
"should be overloaded by a derived class";
};
virtual void LinkDataFile(std::string dataFilename, std::string dataSetname,
const std::vector<std::string> parameterNames,
const bool silentMode) {
LOG(logERROR) << "This is a generic function LinkDataFile that "
"should be overloaded by a derived class";
};
#endif
virtual void CreateFirstBinaryDataFile(
const std::string filePath, const std::string fileNamePrefix,

62
slsReceiverSoftware/src/HDF5MasterFile.cpp
View File

@ -21,6 +21,60 @@ void HDF5MasterFile::CloseFile() {
}
}
void HDF5MasterFile::LinkDataFile(std::string dataFilename,
std::string dataSetname,
const std::vector<std::string> parameterNames,
const bool silentMode) {
std::lock_guard<std::mutex> lock(*hdf5Lib_);
try {
Exception::dontPrint(); // to handle errors
FileAccPropList flist;
flist.setFcloseDegree(H5F_CLOSE_STRONG);
// open master file
H5File masterfd(fileName_.c_str(), H5F_ACC_RDWR,
FileCreatPropList::DEFAULT, flist);
// open data file
H5File fd(dataFilename.c_str(), H5F_ACC_RDONLY,
FileCreatPropList::DEFAULT, flist);
// create link for data dataset
DataSet dset = fd.openDataSet(dataSetname.c_str());
std::string linkname = std::string("/entry/data/") + dataSetname;
if (H5Lcreate_external(dataFilename.c_str(), dataSetname.c_str(),
masterfd.getLocId(), linkname.c_str(),
H5P_DEFAULT, H5P_DEFAULT) < 0) {
throw sls::RuntimeError(
"Could not create link to data dataset in master");
}
// create link for parameter datasets
for (unsigned int i = 0; i < parameterNames.size(); ++i) {
DataSet pDset = fd.openDataSet(parameterNames[i].c_str());
linkname = std::string("/entry/data/") + parameterNames[i];
if (H5Lcreate_external(dataFilename.c_str(),
parameterNames[i].c_str(),
masterfd.getLocId(), linkname.c_str(),
H5P_DEFAULT, H5P_DEFAULT) < 0) {
throw sls::RuntimeError(
"Could not create link to parameter dataset in master");
}
}
fd.close();
masterfd.close();
} catch (const Exception &error) {
error.printErrorStack();
CloseFile();
throw sls::RuntimeError("Could not link in master hdf5 file");
}
if (!silentMode) {
LOG(logINFO) << "Linked in Master File: " << dataFilename;
}
}
void HDF5MasterFile::CreateMasterFile(const std::string filePath,
const std::string fileNamePrefix,
const uint64_t fileIndex,
@ -31,7 +85,7 @@ void HDF5MasterFile::CreateMasterFile(const std::string filePath,
std::ostringstream os;
os << filePath << "/" << fileNamePrefix << "_master"
<< "_" << fileIndex << ".h5";
std::string fileName = os.str();
fileName_ = os.str();
std::lock_guard<std::mutex> lock(*hdf5Lib_);
@ -42,10 +96,10 @@ void HDF5MasterFile::CreateMasterFile(const std::string filePath,
flist.setFcloseDegree(H5F_CLOSE_STRONG);
fd_ = nullptr;
if (!(overWriteEnable))
fd_ = new H5File(fileName.c_str(), H5F_ACC_EXCL,
fd_ = new H5File(fileName_.c_str(), H5F_ACC_EXCL,
FileCreatPropList::DEFAULT, flist);
else
fd_ = new H5File(fileName.c_str(), H5F_ACC_TRUNC,
fd_ = new H5File(fileName_.c_str(), H5F_ACC_TRUNC,
FileCreatPropList::DEFAULT, flist);
// attributes - version
@ -73,6 +127,6 @@ void HDF5MasterFile::CreateMasterFile(const std::string filePath,
"Could not create/overwrite master HDF5 handles");
}
if (!silentMode) {
LOG(logINFO) << "Master File: " << fileName;
LOG(logINFO) << "Master File: " << fileName_;
}
}

4
slsReceiverSoftware/src/HDF5MasterFile.h
View File

@ -11,6 +11,9 @@ class HDF5MasterFile : private virtual slsDetectorDefs, public File {
~HDF5MasterFile();
void CloseFile() override;
void LinkDataFile(std::string dataFilename, std::string dataSetname,
const std::vector<std::string> parameterNames,
const bool silentMode) override;
void CreateMasterFile(const std::string filePath,
const std::string fileNamePrefix,
const uint64_t fileIndex, const bool overWriteEnable,
@ -20,4 +23,5 @@ class HDF5MasterFile : private virtual slsDetectorDefs, public File {
private:
std::mutex *hdf5Lib_;
H5File *fd_{nullptr};
std::string fileName_;
};

5
slsReceiverSoftware/src/HDF5VirtualFile.cpp
View File

@ -97,6 +97,7 @@ void HDF5VirtualFile::CreateVirtualFile(
if (numImagesCaught % maxFramesPerFile)
++numMajorHyperslab;
uint64_t framesSaved = 0;
// loop through files
for (int hyperSlab = 0; hyperSlab < numMajorHyperslab; ++hyperSlab) {
uint64_t nDimx =
@ -107,7 +108,7 @@ void HDF5VirtualFile::CreateVirtualFile(
hsize_t count[3] = {nDimx, nDimy, nDimz};
hsize_t startPara[2] = {framesSaved, 0};
hsize_t countPara[2] = {nDimx, 1};
// loop through readouts
for (unsigned int i = 0; i < numModY * numModZ; ++i) {
// setect data hyperslabs
@ -150,7 +151,7 @@ void HDF5VirtualFile::CreateVirtualFile(
// source parameter dataspace
hsize_t srcDimsPara[1] = {nDimx};
hsize_t srcDimsMaxPara[1] = {H5S_UNLIMITED};
DataSpace srcDataSpacePara(3, srcDimsPara, srcDimsMaxPara);
DataSpace srcDataSpacePara(1, srcDimsPara, srcDimsMaxPara);
// mapping of data property list
plist.setVirtual(vdsDataSpace, relative_srcFileName.c_str(),

8
slsReceiverSoftware/src/Implementation.cpp
View File

@ -557,11 +557,19 @@ void Implementation::stopReceiver() {
#ifdef HDF5C
if (fileWriteEnable && fileFormatType == HDF5) {
if (modulePos == 0) {
// more than 1 file, create virtual file
if (dataProcessor[0]->GetFilesInAcquisition() > 1 ||
(numMods[X] * numMods[Y]) > 1) {
dataProcessor[0]->CreateVirtualFile(
filePath, fileName, fileIndex, overwriteEnable, silentMode,
modulePos, numThreads, framesPerFile, numberOfTotalFrames,
dynamicRange, numMods[X], numMods[Y]);
}
// link file in master
dataProcessor[0]->LinkDataInMasterFile(silentMode);
}
}
#endif
// wait for the processes (dataStreamer) to be done