Rxclassmembers (#503)

* gui message doesnt show if it has a '>' symbol in error msg

* minor refactoring for readability (size_t calc fifo size)

* refactoring listening udp socket code: activated and datastream dont create udp sockets anyway, rc<=- should be discarded in any case

* wip

* refactoring memory structure access

* wip: bugfix write header + data to binary

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* portRoi no roi effecto on progress

* fail at receiver progress, wip

* segfaults for char pointer in struct

* reference to header to get header and data

* refactoring

* use const defined for size of header of fifo

* updated release notes

* remove pointer in callback for sls_receiver_header pointer

* rx same name arguments in constructors

* rx: same name arguments in constructor

* rx: removing the '_' suffix in class data members

* merge fix

* merge fix

* review fix refactoring
This commit is contained in:
Dhanya Thattil
2022-07-25 14:02:11 +02:00
committed by GitHub
parent d132ad8d02
commit 9ac8dab8af
22 changed files with 349 additions and 493 deletions

View File

@ -8,9 +8,9 @@
namespace sls {
HDF5DataFile::HDF5DataFile(int index, std::mutex *hdf5Lib)
: File(HDF5), index_(index), hdf5Lib_(hdf5Lib) {
: index(index), hdf5Lib(hdf5Lib) {
parameterNames_ = std::vector<std::string>{
parameterNames = std::vector<std::string>{
"frame number",
"exp length or sub exposure time",
"packets caught",
@ -27,7 +27,7 @@ HDF5DataFile::HDF5DataFile(int index, std::mutex *hdf5Lib)
"packets caught bit mask",
};
H5::StrType strdatatype(H5::PredType::C_S1, sizeof(bitset_storage));
parameterDataTypes_ = std::vector<H5::DataType>{
parameterDataTypes = std::vector<H5::DataType>{
H5::PredType::STD_U64LE, H5::PredType::STD_U32LE, H5::PredType::STD_U32LE,
H5::PredType::STD_U64LE, H5::PredType::STD_U64LE, H5::PredType::STD_U16LE,
H5::PredType::STD_U16LE, H5::PredType::STD_U16LE, H5::PredType::STD_U16LE,
@ -38,91 +38,95 @@ HDF5DataFile::HDF5DataFile(int index, std::mutex *hdf5Lib)
HDF5DataFile::~HDF5DataFile() { CloseFile(); }
std::string HDF5DataFile::GetFileName() const {
return fileName_;
return fileName;
}
uint32_t HDF5DataFile::GetFilesInAcquisition() const {
return numFilesInAcquisition_;
return numFilesInAcquisition;
}
H5::DataType HDF5DataFile::GetPDataType() const { return dataType_; }
H5::DataType HDF5DataFile::GetPDataType() const { return dataType; }
std::vector<std::string> HDF5DataFile::GetParameterNames() const {
return parameterNames_;
return parameterNames;
}
std::vector<H5::DataType> HDF5DataFile::GetParameterDataTypes() const {
return parameterDataTypes_;
return parameterDataTypes;
}
slsDetectorDefs::fileFormat HDF5DataFile::GetFileFormat() const {
return HDF5;
}
void HDF5DataFile::CloseFile() {
std::lock_guard<std::mutex> lock(*hdf5Lib_);
std::lock_guard<std::mutex> lock(*hdf5Lib);
try {
H5::Exception::dontPrint(); // to handle errors
if (fd_) {
fd_->close();
delete fd_;
fd_ = nullptr;
if (fd) {
fd->close();
delete fd;
fd = nullptr;
}
} catch (const H5::Exception &error) {
LOG(logERROR) << "Could not close data HDF5 handles of index "
<< index_;
<< index;
error.printErrorStack();
}
if (dataSpace_) {
delete dataSpace_;
dataSpace_ = nullptr;
if (dataSpace) {
delete dataSpace;
dataSpace = nullptr;
}
if (dataSet_) {
delete dataSet_;
dataSet_ = nullptr;
if (dataSet) {
delete dataSet;
dataSet = nullptr;
}
if (dataSpacePara_) {
delete dataSpacePara_;
dataSpacePara_ = nullptr;
if (dataSpacePara) {
delete dataSpacePara;
dataSpacePara = nullptr;
}
for (auto it : dataSetPara_)
for (auto it : dataSetPara)
delete it;
dataSetPara_.clear();
dataSetPara.clear();
}
void HDF5DataFile::CreateFirstHDF5DataFile(
const std::string filePath, const std::string fileNamePrefix,
const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode,
const int modulePos, const int numUnitsPerReadout,
const uint32_t udpPortNumber, const uint32_t maxFramesPerFile,
const uint64_t numImages, const uint32_t nPixelsX, const uint32_t nPixelsY,
const uint32_t dynamicRange) {
const std::string fPath, const std::string fNamePrefix,
const uint64_t fIndex, const bool owEnable, const bool sMode,
const int modulePos, const int nUnitsPerReadout,
const uint32_t uPortNumber, const uint32_t mFramesPerFile,
const uint64_t nImages, const uint32_t nX, const uint32_t nY,
const uint32_t dr) {
subFileIndex_ = 0;
numFramesInFile_ = 0;
extNumImages_ = numImages;
numFilesInAcquisition_ = 0;
subFileIndex = 0;
numFramesInFile = 0;
extNumImages = nImages;
numFilesInAcquisition = 0;
maxFramesPerFile_ = maxFramesPerFile;
numImages_ = numImages;
nPixelsX_ = nPixelsX;
nPixelsY_ = nPixelsY;
dynamicRange_ = dynamicRange;
maxFramesPerFile = mFramesPerFile;
numImages = nImages;
nPixelsX = nX;
nPixelsY = nY;
dynamicRange = dr;
filePath_ = filePath;
fileNamePrefix_ = fileNamePrefix;
fileIndex_ = fileIndex;
overWriteEnable_ = overWriteEnable;
silentMode_ = silentMode;
detIndex_ = modulePos;
numUnitsPerReadout_ = numUnitsPerReadout;
udpPortNumber_ = udpPortNumber;
filePath = fPath;
fileNamePrefix = fNamePrefix;
fileIndex = fIndex;
overWriteEnable = owEnable;
silentMode = sMode;
detIndex = modulePos;
numUnitsPerReadout = nUnitsPerReadout;
udpPortNumber = uPortNumber;
switch (dynamicRange_) {
switch (dynamicRange) {
case 12:
case 16:
dataType_ = H5::PredType::STD_U16LE;
dataType = H5::PredType::STD_U16LE;
break;
case 32:
dataType_ = H5::PredType::STD_U32LE;
dataType = H5::PredType::STD_U32LE;
break;
default:
dataType_ = H5::PredType::STD_U8LE;
dataType = H5::PredType::STD_U8LE;
break;
}
@ -130,27 +134,27 @@ void HDF5DataFile::CreateFirstHDF5DataFile(
}
void HDF5DataFile::CreateFile() {
numFramesInFile_ = 0;
numFilesInAcquisition_++;
numFramesInFile = 0;
numFilesInAcquisition++;
std::ostringstream os;
os << filePath_ << "/" << fileNamePrefix_ << "_d"
<< (detIndex_ * numUnitsPerReadout_ + index_) << "_f" << subFileIndex_
<< '_' << fileIndex_ << ".h5";
fileName_ = os.str();
os << filePath << "/" << fileNamePrefix << "_d"
<< (detIndex * numUnitsPerReadout + index) << "_f" << subFileIndex
<< '_' << fileIndex << ".h5";
fileName = os.str();
std::lock_guard<std::mutex> lock(*hdf5Lib_);
std::lock_guard<std::mutex> lock(*hdf5Lib);
uint64_t framestosave =
((maxFramesPerFile_ == 0) ? numImages_ : // infinite images
(((extNumImages_ - subFileIndex_) > maxFramesPerFile_)
((maxFramesPerFile == 0) ? numImages : // infinite images
(((extNumImages - subFileIndex) > maxFramesPerFile)
? // save up to maximum at a time
maxFramesPerFile_
: (extNumImages_ - subFileIndex_)));
maxFramesPerFile
: (extNumImages - subFileIndex)));
uint64_t nDimx = framestosave;
uint32_t nDimy = nPixelsY_;
uint32_t nDimz = ((dynamicRange_ == 4) ? (nPixelsX_ / 2) : nPixelsX_);
uint32_t nDimy = nPixelsY;
uint32_t nDimz = ((dynamicRange == 4) ? (nPixelsX / 2) : nPixelsX);
try {
H5::Exception::dontPrint(); // to handle errors
@ -158,45 +162,45 @@ void HDF5DataFile::CreateFile() {
// file
H5::FileAccPropList fapl;
fapl.setFcloseDegree(H5F_CLOSE_STRONG);
fd_ = nullptr;
if (!overWriteEnable_)
fd_ = new H5::H5File(fileName_.c_str(), H5F_ACC_EXCL,
fd = nullptr;
if (!overWriteEnable)
fd = new H5::H5File(fileName.c_str(), H5F_ACC_EXCL,
H5::FileCreatPropList::DEFAULT, fapl);
else
fd_ = new H5::H5File(fileName_.c_str(), H5F_ACC_TRUNC,
fd = new H5::H5File(fileName.c_str(), H5F_ACC_TRUNC,
H5::FileCreatPropList::DEFAULT, fapl);
// attributes - version
double dValue = HDF5_WRITER_VERSION;
H5::DataSpace dataspace_attr = H5::DataSpace(H5S_SCALAR);
H5::Attribute attribute = fd_->createAttribute(
H5::Attribute attribute = fd->createAttribute(
"version", H5::PredType::NATIVE_DOUBLE, dataspace_attr);
attribute.write(H5::PredType::NATIVE_DOUBLE, &dValue);
// dataspace
hsize_t srcdims[3] = {nDimx, nDimy, nDimz};
hsize_t srcdimsmax[3] = {H5S_UNLIMITED, nDimy, nDimz};
dataSpace_ = nullptr;
dataSpace_ = new H5::DataSpace(3, srcdims, srcdimsmax);
dataSpace = nullptr;
dataSpace = new H5::DataSpace(3, srcdims, srcdimsmax);
// dataset
// fill value
H5::DSetCreatPropList plist;
int fill_value = -1;
plist.setFillValue(dataType_, &fill_value);
plist.setFillValue(dataType, &fill_value);
// always create chunked dataset as unlimited is only
// supported with chunked layout
hsize_t chunk_dims[3] = {MAX_CHUNKED_IMAGES, nDimy, nDimz};
plist.setChunk(3, chunk_dims);
dataSet_ = nullptr;
dataSet_ = new H5::DataSet(fd_->createDataSet(
DATASET_NAME, dataType_, *dataSpace_, plist));
dataSet = nullptr;
dataSet = new H5::DataSet(fd->createDataSet(
DATASET_NAME, dataType, *dataSpace, plist));
// create parameter datasets
hsize_t dims[1] = {nDimx};
hsize_t dimsmax[1] = {H5S_UNLIMITED};
dataSpacePara_ = nullptr;
dataSpacePara_ = new H5::DataSpace(1, dims, dimsmax);
dataSpacePara = nullptr;
dataSpacePara = new H5::DataSpace(1, dims, dimsmax);
// always create chunked dataset as unlimited is only
// supported with chunked layout
@ -204,37 +208,37 @@ void HDF5DataFile::CreateFile() {
hsize_t chunkpara_dims[3] = {MAX_CHUNKED_IMAGES};
paralist.setChunk(1, chunkpara_dims);
for (unsigned int i = 0; i < parameterNames_.size(); ++i) {
H5::DataSet *ds = new H5::DataSet(fd_->createDataSet(
parameterNames_[i].c_str(), parameterDataTypes_[i],
*dataSpacePara_, paralist));
dataSetPara_.push_back(ds);
for (unsigned int i = 0; i < parameterNames.size(); ++i) {
H5::DataSet *ds = new H5::DataSet(fd->createDataSet(
parameterNames[i].c_str(), parameterDataTypes[i],
*dataSpacePara, paralist));
dataSetPara.push_back(ds);
}
} catch (const H5::Exception &error) {
error.printErrorStack();
CloseFile();
throw RuntimeError("Could not create HDF5 handles in object " +
index_);
index);
}
if (!silentMode_) {
LOG(logINFO) << "[" << udpPortNumber_
<< "]: HDF5 File created: " << fileName_;
if (!silentMode) {
LOG(logINFO) << "[" << udpPortNumber
<< "]: HDF5 File created: " << fileName;
}
}
void HDF5DataFile::WriteToFile(char *imageData, sls_receiver_header& header, const int imageSize, const uint64_t currentFrameNumber, const uint32_t numPacketsCaught) {
// check if maxframesperfile = 0 for infinite
if (maxFramesPerFile_ && (numFramesInFile_ >= maxFramesPerFile_)) {
if (maxFramesPerFile && (numFramesInFile >= maxFramesPerFile)) {
CloseFile();
++subFileIndex_;
++subFileIndex;
CreateFile();
}
++numFramesInFile_;
++numFramesInFile;
// extend dataset (when receiver start followed by many status starts
// (jungfrau)))
if (currentFrameNumber >= extNumImages_) {
if (currentFrameNumber >= extNumImages) {
ExtendDataset();
}
@ -256,23 +260,23 @@ void HDF5DataFile::WriteDataFile(const uint64_t currentFrameNumber,
char *buffer) {
// expand 12 bit to 16 bits
char *revBuffer = buffer;
if (dynamicRange_ == 12) {
if (dynamicRange == 12) {
revBuffer = (char *)malloc(EIGER_16_BIT_IMAGE_SIZE);
if (revBuffer == nullptr) {
throw RuntimeError("Could not allocate memory for 12 bit to "
"16 bit conversion in object " +
std::to_string(index_));
std::to_string(index));
}
Convert12to16Bit((uint16_t *)revBuffer, (uint8_t *)buffer);
}
std::lock_guard<std::mutex> lock(*hdf5Lib_);
std::lock_guard<std::mutex> lock(*hdf5Lib);
uint64_t nDimx =
((maxFramesPerFile_ == 0) ? currentFrameNumber
: currentFrameNumber % maxFramesPerFile_);
uint32_t nDimy = nPixelsY_;
uint32_t nDimz = ((dynamicRange_ == 4) ? (nPixelsX_ / 2) : nPixelsX_);
((maxFramesPerFile == 0) ? currentFrameNumber
: currentFrameNumber % maxFramesPerFile);
uint32_t nDimy = nPixelsY;
uint32_t nDimz = ((dynamicRange == 4) ? (nPixelsX / 2) : nPixelsX);
hsize_t count[3] = {1, nDimy, nDimz};
hsize_t start[3] = {nDimx, 0, 0};
@ -280,31 +284,31 @@ void HDF5DataFile::WriteDataFile(const uint64_t currentFrameNumber,
try {
H5::Exception::dontPrint(); // to handle errors
dataSpace_->selectHyperslab(H5S_SELECT_SET, count, start);
dataSpace->selectHyperslab(H5S_SELECT_SET, count, start);
H5::DataSpace memspace(2, dims2);
dataSet_->write(revBuffer, dataType_, memspace, *dataSpace_);
dataSet->write(revBuffer, dataType, memspace, *dataSpace);
memspace.close();
if (dynamicRange_ == 12) {
if (dynamicRange == 12) {
free(revBuffer);
}
} catch (const H5::Exception &error) {
if (dynamicRange_ == 12) {
if (dynamicRange == 12) {
free(revBuffer);
}
LOG(logERROR) << "Could not write to file in object " << index_;
LOG(logERROR) << "Could not write to file in object " << index;
error.printErrorStack();
throw RuntimeError("Could not write to file in object " +
std::to_string(index_));
std::to_string(index));
}
}
void HDF5DataFile::WriteParameterDatasets(const uint64_t currentFrameNumber,
sls_receiver_header rheader) {
std::lock_guard<std::mutex> lock(*hdf5Lib_);
std::lock_guard<std::mutex> lock(*hdf5Lib);
uint64_t fnum =
((maxFramesPerFile_ == 0) ? currentFrameNumber
: currentFrameNumber % maxFramesPerFile_);
((maxFramesPerFile == 0) ? currentFrameNumber
: currentFrameNumber % maxFramesPerFile);
sls_detector_header header = rheader.detHeader;
hsize_t count[1] = {1};
@ -312,53 +316,53 @@ void HDF5DataFile::WriteParameterDatasets(const uint64_t currentFrameNumber,
int i = 0;
try {
H5::Exception::dontPrint(); // to handle errors
dataSpacePara_->selectHyperslab(H5S_SELECT_SET, count, start);
dataSpacePara->selectHyperslab(H5S_SELECT_SET, count, start);
H5::DataSpace memspace(H5S_SCALAR);
dataSetPara_[0]->write(&header.frameNumber, parameterDataTypes_[0],
memspace, *dataSpacePara_);
dataSetPara[0]->write(&header.frameNumber, parameterDataTypes[0],
memspace, *dataSpacePara);
i = 1;
dataSetPara_[1]->write(&header.expLength, parameterDataTypes_[1],
memspace, *dataSpacePara_);
dataSetPara[1]->write(&header.expLength, parameterDataTypes[1],
memspace, *dataSpacePara);
i = 2;
dataSetPara_[2]->write(&header.packetNumber, parameterDataTypes_[2],
memspace, *dataSpacePara_);
dataSetPara[2]->write(&header.packetNumber, parameterDataTypes[2],
memspace, *dataSpacePara);
i = 3;
dataSetPara_[3]->write(&header.bunchId, parameterDataTypes_[3],
memspace, *dataSpacePara_);
dataSetPara[3]->write(&header.bunchId, parameterDataTypes[3],
memspace, *dataSpacePara);
i = 4;
dataSetPara_[4]->write(&header.timestamp, parameterDataTypes_[4],
memspace, *dataSpacePara_);
dataSetPara[4]->write(&header.timestamp, parameterDataTypes[4],
memspace, *dataSpacePara);
i = 5;
dataSetPara_[5]->write(&header.modId, parameterDataTypes_[5], memspace,
*dataSpacePara_);
dataSetPara[5]->write(&header.modId, parameterDataTypes[5], memspace,
*dataSpacePara);
i = 6;
dataSetPara_[6]->write(&header.row, parameterDataTypes_[6], memspace,
*dataSpacePara_);
dataSetPara[6]->write(&header.row, parameterDataTypes[6], memspace,
*dataSpacePara);
i = 7;
dataSetPara_[7]->write(&header.column, parameterDataTypes_[7], memspace,
*dataSpacePara_);
dataSetPara[7]->write(&header.column, parameterDataTypes[7], memspace,
*dataSpacePara);
i = 8;
dataSetPara_[8]->write(&header.reserved, parameterDataTypes_[8],
memspace, *dataSpacePara_);
dataSetPara[8]->write(&header.reserved, parameterDataTypes[8],
memspace, *dataSpacePara);
i = 9;
dataSetPara_[9]->write(&header.debug, parameterDataTypes_[9], memspace,
*dataSpacePara_);
dataSetPara[9]->write(&header.debug, parameterDataTypes[9], memspace,
*dataSpacePara);
i = 10;
dataSetPara_[10]->write(&header.roundRNumber, parameterDataTypes_[10],
memspace, *dataSpacePara_);
dataSetPara[10]->write(&header.roundRNumber, parameterDataTypes[10],
memspace, *dataSpacePara);
i = 11;
dataSetPara_[11]->write(&header.detType, parameterDataTypes_[11],
memspace, *dataSpacePara_);
dataSetPara[11]->write(&header.detType, parameterDataTypes[11],
memspace, *dataSpacePara);
i = 12;
dataSetPara_[12]->write(&header.version, parameterDataTypes_[12],
memspace, *dataSpacePara_);
dataSetPara[12]->write(&header.version, parameterDataTypes[12],
memspace, *dataSpacePara);
i = 13;
// contiguous bitset
if (sizeof(sls_bitset) == sizeof(bitset_storage)) {
dataSetPara_[13]->write((char *)&(rheader.packetsMask),
parameterDataTypes_[13], memspace,
*dataSpacePara_);
dataSetPara[13]->write((char *)&(rheader.packetsMask),
parameterDataTypes[13], memspace,
*dataSpacePara);
}
// not contiguous bitset
@ -370,50 +374,50 @@ void HDF5DataFile::WriteParameterDatasets(const uint64_t currentFrameNumber,
for (int i = 0; i < MAX_NUM_PACKETS; ++i)
storage[i >> 3] |= (bits[i] << (i & 7));
// write bitmask
dataSetPara_[13]->write((char *)storage, parameterDataTypes_[13],
memspace, *dataSpacePara_);
dataSetPara[13]->write((char *)storage, parameterDataTypes[13],
memspace, *dataSpacePara);
}
i = 14;
} catch (const H5::Exception &error) {
error.printErrorStack();
throw RuntimeError(
"Could not write parameters (index:" + std::to_string(i) +
") to file in object " + std::to_string(index_));
") to file in object " + std::to_string(index));
}
}
void HDF5DataFile::ExtendDataset() {
std::lock_guard<std::mutex> lock(*hdf5Lib_);
std::lock_guard<std::mutex> lock(*hdf5Lib);
try {
H5::Exception::dontPrint(); // to handle errors
hsize_t dims[3];
dataSpace_->getSimpleExtentDims(dims);
dims[0] += numImages_;
dataSpace->getSimpleExtentDims(dims);
dims[0] += numImages;
dataSet_->extend(dims);
delete dataSpace_;
dataSpace_ = nullptr;
dataSpace_ = new H5::DataSpace(dataSet_->getSpace());
dataSet->extend(dims);
delete dataSpace;
dataSpace = nullptr;
dataSpace = new H5::DataSpace(dataSet->getSpace());
hsize_t dims_para[1] = {dims[0]};
for (unsigned int i = 0; i < dataSetPara_.size(); ++i)
dataSetPara_[i]->extend(dims_para);
delete dataSpacePara_;
dataSpacePara_ = nullptr;
dataSpacePara_ = new H5::DataSpace(dataSetPara_[0]->getSpace());
for (unsigned int i = 0; i < dataSetPara.size(); ++i)
dataSetPara[i]->extend(dims_para);
delete dataSpacePara;
dataSpacePara = nullptr;
dataSpacePara = new H5::DataSpace(dataSetPara[0]->getSpace());
} catch (const H5::Exception &error) {
error.printErrorStack();
throw RuntimeError("Could not extend dataset in object " +
std::to_string(index_));
std::to_string(index));
}
if (!silentMode_) {
LOG(logINFO) << index_ << " Extending HDF5 dataset by " << extNumImages_
<< ", Total x Dimension: " << (extNumImages_ + numImages_);
if (!silentMode) {
LOG(logINFO) << index << " Extending HDF5 dataset by " << extNumImages
<< ", Total x Dimension: " << (extNumImages + numImages);
}
extNumImages_ += numImages_;
extNumImages += numImages;
}
} // namespace sls