detectors/slsDetectorPackage
25
0
Fork 0
mirror of https://github.com/slsdetectorgroup/slsDetectorPackage.git synced 2025-06-05 17:40:40 +02:00
Code Issues Actions Packages Releases Activity

Hdf5 para (#505)

Browse Source 
* incorrect dimensions for virtual hdf5 parameter set

* fix for corner case bug in hdf5 virtual parameter dataset when frames caught is not a multiple of framesperfile

* reafctoring for readability and error prone hard numbers
This commit is contained in:
Dhanya Thattil 2022-08-05 09:09:00 +02:00 committed by GitHub
parent 89e293cb5a
commit 7173785b29
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 71 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
slsReceiverSoftware/src/HDF5DataFile.cpp
View File

@ -172,41 +172,39 @@ void HDF5DataFile::CreateFile() {
"version", H5::PredType::NATIVE_DOUBLE, dataspace_attr); "version", H5::PredType::NATIVE_DOUBLE, dataspace_attr);
attribute.write(H5::PredType::NATIVE_DOUBLE, &dValue); attribute.write(H5::PredType::NATIVE_DOUBLE, &dValue);
// dataspace // dimensions
hsize_t srcdims[3] = {nDimx, nDimy, nDimz}; hsize_t dims[DATA_RANK] = {nDimx, nDimy, nDimz};
hsize_t srcdimsmax[3] = {H5S_UNLIMITED, nDimy, nDimz}; hsize_t dimsMax[DATA_RANK] = {H5S_UNLIMITED, nDimy, nDimz};
dataSpace = nullptr; hsize_t dimsPara[PARA_RANK] = {nDimx};
dataSpace = new H5::DataSpace(3, srcdims, srcdimsmax); hsize_t dimsMaxPara[PARA_RANK] = {H5S_UNLIMITED};
// dataset
// fill value
H5::DSetCreatPropList plist;
int fill_value = -1;
plist.setFillValue(dataType, &fill_value);
// always create chunked dataset as unlimited is only // always create chunked dataset as unlimited is only
// supported with chunked layout // supported with chunked layout
hsize_t chunk_dims[3] = {MAX_CHUNKED_IMAGES, nDimy, nDimz}; hsize_t dimsChunk[DATA_RANK] = {MAX_CHUNKED_IMAGES, nDimy, nDimz};
plist.setChunk(3, chunk_dims); hsize_t dimsChunkPara[PARA_RANK] = {MAX_CHUNKED_IMAGES};
// dataspace
dataSpace = nullptr;
dataSpace = new H5::DataSpace(DATA_RANK, dims, dimsMax);
dataSpacePara = nullptr;
dataSpacePara = new H5::DataSpace(PARA_RANK, dimsPara, dimsMaxPara);
// property list
H5::DSetCreatPropList plist;
H5::DSetCreatPropList plistPara;
int fill_value = -1;
plist.setFillValue(dataType, &fill_value);
//plistPara.setFillValue(dataType, &fill_value);
plist.setChunk(DATA_RANK, dimsChunk);
plistPara.setChunk(PARA_RANK, dimsChunkPara);
// dataset
dataSet = nullptr; dataSet = nullptr;
dataSet = new H5::DataSet(fd->createDataSet( dataSet = new H5::DataSet(fd->createDataSet(
DATASET_NAME, dataType, *dataSpace, plist)); DATASET_NAME, dataType, *dataSpace, plist));
// create parameter datasets
hsize_t dims[1] = {nDimx};
hsize_t dimsmax[1] = {H5S_UNLIMITED};
dataSpacePara = nullptr;
dataSpacePara = new H5::DataSpace(1, dims, dimsmax);
// always create chunked dataset as unlimited is only
// supported with chunked layout
H5::DSetCreatPropList paralist;
hsize_t chunkpara_dims[3] = {MAX_CHUNKED_IMAGES};
paralist.setChunk(1, chunkpara_dims);
for (unsigned int i = 0; i < parameterNames.size(); ++i) { for (unsigned int i = 0; i < parameterNames.size(); ++i) {
H5::DataSet *ds = new H5::DataSet(fd->createDataSet( H5::DataSet *ds = new H5::DataSet(fd->createDataSet(
parameterNames[i].c_str(), parameterDataTypes[i], parameterNames[i].c_str(), parameterDataTypes[i],
*dataSpacePara, paralist)); *dataSpacePara, plistPara));
dataSetPara.push_back(ds); dataSetPara.push_back(ds);
} }
} catch (const H5::Exception &error) { } catch (const H5::Exception &error) {
@ -237,7 +235,7 @@ void HDF5DataFile::WriteToFile(char *imageData, sls_receiver_header& header, con
ExtendDataset(); ExtendDataset();
} }
WriteDataFile(currentFrameNumber, imageData); WriteImageDatasets(currentFrameNumber, imageData);
WriteParameterDatasets(currentFrameNumber, header); WriteParameterDatasets(currentFrameNumber, header);
} }
@ -251,7 +249,7 @@ void HDF5DataFile::Convert12to16Bit(uint16_t *dst, uint8_t *src) {
} }
} }
void HDF5DataFile::WriteDataFile(const uint64_t currentFrameNumber, void HDF5DataFile::WriteImageDatasets(const uint64_t currentFrameNumber,
char *buffer) { char *buffer) {
// expand 12 bit to 16 bits // expand 12 bit to 16 bits
char *revBuffer = buffer; char *revBuffer = buffer;
@ -306,8 +304,8 @@ void HDF5DataFile::WriteParameterDatasets(const uint64_t currentFrameNumber,
: currentFrameNumber % maxFramesPerFile); : currentFrameNumber % maxFramesPerFile);
sls_detector_header header = rheader.detHeader; sls_detector_header header = rheader.detHeader;
hsize_t count[1] = {1}; hsize_t count[PARA_RANK] = {1};
hsize_t start[1] = {fnum}; hsize_t start[PARA_RANK] = {fnum};
int i = 0; int i = 0;
try { try {
H5::Exception::dontPrint(); // to handle errors H5::Exception::dontPrint(); // to handle errors
@ -387,18 +385,17 @@ void HDF5DataFile::ExtendDataset() {
try { try {
H5::Exception::dontPrint(); // to handle errors H5::Exception::dontPrint(); // to handle errors
hsize_t dims[3]; hsize_t dims[DATA_RANK];
dataSpace->getSimpleExtentDims(dims); dataSpace->getSimpleExtentDims(dims);
dims[0] += numImages; dims[0] += numImages;
dataSet->extend(dims); dataSet->extend(dims);
delete dataSpace; delete dataSpace;
dataSpace = nullptr; dataSpace = nullptr;
dataSpace = new H5::DataSpace(dataSet->getSpace()); dataSpace = new H5::DataSpace(dataSet->getSpace());
hsize_t dims_para[1] = {dims[0]}; hsize_t dimsPara[PARA_RANK] = {dims[0]};
for (unsigned int i = 0; i < dataSetPara.size(); ++i) for (unsigned int i = 0; i < dataSetPara.size(); ++i)
dataSetPara[i]->extend(dims_para); dataSetPara[i]->extend(dimsPara);
delete dataSpacePara; delete dataSpacePara;
dataSpacePara = nullptr; dataSpacePara = nullptr;
dataSpacePara = new H5::DataSpace(dataSetPara[0]->getSpace()); dataSpacePara = new H5::DataSpace(dataSetPara[0]->getSpace());

2
slsReceiverSoftware/src/HDF5DataFile.h
View File

@ -36,7 +36,7 @@ class HDF5DataFile : private virtual slsDetectorDefs, public File {
private: private:
void CreateFile(); void CreateFile();
void Convert12to16Bit(uint16_t *dst, uint8_t *src); void Convert12to16Bit(uint16_t *dst, uint8_t *src);
void WriteDataFile(const uint64_t currentFrameNumber, char *buffer); void WriteImageDatasets(const uint64_t currentFrameNumber, char *buffer);
void WriteParameterDatasets(const uint64_t currentFrameNumber, void WriteParameterDatasets(const uint64_t currentFrameNumber,
sls_receiver_header rheader); sls_receiver_header rheader);
void ExtendDataset(); void ExtendDataset();

60
slsReceiverSoftware/src/MasterFileUtility.cpp
View File

@ -207,21 +207,22 @@ std::string CreateVirtualHDF5File(
"version", H5::PredType::NATIVE_DOUBLE, dataspace_attr); "version", H5::PredType::NATIVE_DOUBLE, dataspace_attr);
attribute.write(H5::PredType::NATIVE_DOUBLE, &dValue); attribute.write(H5::PredType::NATIVE_DOUBLE, &dValue);
// virtual dataspace // dataspace
hsize_t vdsDims[3] = {numImagesCaught, numModY * nDimy, hsize_t vdsDims[DATA_RANK] = {numImagesCaught, numModY * nDimy,
numModZ * nDimz}; numModZ * nDimz};
H5::DataSpace vdsDataSpace(3, vdsDims, nullptr); hsize_t vdsDimsPara[VDS_PARA_RANK] = {numImagesCaught, numModY * numModZ};
hsize_t vdsDimsPara[2] = {numImagesCaught, H5::DataSpace vdsDataSpace(DATA_RANK, vdsDims, nullptr);
(unsigned int)numModY * numModZ}; H5::DataSpace vdsDataSpacePara(VDS_PARA_RANK, vdsDimsPara, nullptr);
H5::DataSpace vdsDataSpacePara(2, vdsDimsPara, nullptr);
// property list (fill value and datatype) // property list
int fill_value = -1;
H5::DSetCreatPropList plist; H5::DSetCreatPropList plist;
int fill_value = -1;
plist.setFillValue(dataType, &fill_value); plist.setFillValue(dataType, &fill_value);
// property list for parameters (datatype)
std::vector<H5::DSetCreatPropList> plistPara(paraSize); std::vector<H5::DSetCreatPropList> plistPara(paraSize);
// ignoring last fill (string)
for (unsigned int i = 0; i != plistPara.size() - 1; ++i) {
plistPara[i].setFillValue(parameterDataTypes[i], &fill_value);
}
// hyperslab (files) // hyperslab (files)
int numFiles = numImagesCaught / maxFramesPerFile; int numFiles = numImagesCaught / maxFramesPerFile;
@ -235,15 +236,15 @@ std::string CreateVirtualHDF5File(
? maxFramesPerFile ? maxFramesPerFile
: (numImagesCaught - framesSaved); : (numImagesCaught - framesSaved);
hsize_t startLocation[3] = {framesSaved, 0, 0}; hsize_t startLocation[DATA_RANK] = {framesSaved, 0, 0};
hsize_t strideBetweenBlocks[3] = {1, 1, 1}; hsize_t strideBetweenBlocks[DATA_RANK] = {1, 1, 1};
hsize_t numBlocks[3] = {nDimx, nDimy, nDimz}; hsize_t numBlocks[DATA_RANK] = {nDimx, nDimy, nDimz};
hsize_t blockSize[3] = {1, 1, 1}; hsize_t blockSize[DATA_RANK] = {1, 1, 1};
hsize_t startLocationPara[2] = {framesSaved, 0}; hsize_t startLocationPara[VDS_PARA_RANK] = {framesSaved, 0};
hsize_t strideBetweenBlocksPara[3] = {1, 1}; hsize_t strideBetweenBlocksPara[VDS_PARA_RANK] = {1, 1};
hsize_t numBlocksPara[2] = {1, 1}; hsize_t numBlocksPara[VDS_PARA_RANK] = {nDimx, 1};
hsize_t blockSizePara[3] = {nDimx, 1}; hsize_t blockSizePara[VDS_PARA_RANK] = {1, 1};
// interleaving for g2 // interleaving for g2
if (gotthard25um) { if (gotthard25um) {
@ -284,12 +285,21 @@ std::string CreateVirtualHDF5File(
} }
// source dataspace // source dataspace
hsize_t srcDims[3] = {nDimx, nDimy, nDimz}; hsize_t srcDims[DATA_RANK] = {nDimx, nDimy, nDimz};
hsize_t srcDimsMax[3] = {H5S_UNLIMITED, nDimy, nDimz}; hsize_t srcDimsMax[DATA_RANK] = {H5S_UNLIMITED, nDimy, nDimz};
H5::DataSpace srcDataSpace(3, srcDims, srcDimsMax); H5::DataSpace srcDataSpace(DATA_RANK, srcDims, srcDimsMax);
hsize_t srcDimsPara[1] = {nDimx}; hsize_t srcDimsPara[PARA_RANK] = {nDimx};
hsize_t srcDimsMaxPara[1] = {H5S_UNLIMITED}; hsize_t srcDimsMaxPara[PARA_RANK] = {H5S_UNLIMITED};
H5::DataSpace srcDataSpacePara(1, srcDimsPara, srcDimsMaxPara); H5::DataSpace srcDataSpacePara(PARA_RANK, srcDimsPara, srcDimsMaxPara);
// temporary fixfor corner case bug:
// (framescaught not multiple of framesperfile,
// virtual parameter datasets error loading (bad scalar value))
if (nDimx != maxFramesPerFile) {
hsize_t count[1] = {nDimx};
hsize_t start[1] = {0};
srcDataSpacePara.selectHyperslab(H5S_SELECT_SET, count, start, strideBetweenBlocksPara, blockSizePara);
}
// mapping of property list // mapping of property list
plist.setVirtual(vdsDataSpace, relative_srcFileName.c_str(), plist.setVirtual(vdsDataSpace, relative_srcFileName.c_str(),
@ -310,7 +320,7 @@ std::string CreateVirtualHDF5File(
startLocation[1] += nDimy; startLocation[1] += nDimy;
} }
} }
startLocationPara[1]++; ++startLocationPara[1];
} }
framesSaved += nDimx; framesSaved += nDimx;
} }

3
slsReceiverSoftware/src/receiver_defs.h
View File

@ -47,6 +47,9 @@ struct image_structure {
// hdf5 // hdf5
#define MAX_CHUNKED_IMAGES (1) #define MAX_CHUNKED_IMAGES (1)
#define DATA_RANK (3)
#define PARA_RANK (1)
#define VDS_PARA_RANK (2)
// parameters to calculate fifo depth // parameters to calculate fifo depth
#define SAMPLE_TIME_IN_NS (100000000) // 100ms #define SAMPLE_TIME_IN_NS (100000000) // 100ms