mirror of
https://github.com/slsdetectorgroup/slsDetectorPackage.git
synced 2025-06-04 00:50:42 +02:00
Hdf5 para (#505)
* incorrect dimensions for virtual hdf5 parameter set * fix for corner case bug in hdf5 virtual parameter dataset when frames caught is not a multiple of framesperfile * reafctoring for readability and error prone hard numbers
This commit is contained in:
Dhanya Thattil
GitHub
parent
89e293cb5a
commit
7173785b29
@ -172,41 +172,39 @@ void HDF5DataFile::CreateFile() {
|
||||
"version", H5::PredType::NATIVE_DOUBLE, dataspace_attr);
|
||||
attribute.write(H5::PredType::NATIVE_DOUBLE, &dValue);
|
||||
|
||||
// dataspace
|
||||
hsize_t srcdims[3] = {nDimx, nDimy, nDimz};
|
||||
hsize_t srcdimsmax[3] = {H5S_UNLIMITED, nDimy, nDimz};
|
||||
dataSpace = nullptr;
|
||||
dataSpace = new H5::DataSpace(3, srcdims, srcdimsmax);
|
||||
|
||||
// dataset
|
||||
// fill value
|
||||
H5::DSetCreatPropList plist;
|
||||
int fill_value = -1;
|
||||
plist.setFillValue(dataType, &fill_value);
|
||||
// dimensions
|
||||
hsize_t dims[DATA_RANK] = {nDimx, nDimy, nDimz};
|
||||
hsize_t dimsMax[DATA_RANK] = {H5S_UNLIMITED, nDimy, nDimz};
|
||||
hsize_t dimsPara[PARA_RANK] = {nDimx};
|
||||
hsize_t dimsMaxPara[PARA_RANK] = {H5S_UNLIMITED};
|
||||
// always create chunked dataset as unlimited is only
|
||||
// supported with chunked layout
|
||||
hsize_t chunk_dims[3] = {MAX_CHUNKED_IMAGES, nDimy, nDimz};
|
||||
plist.setChunk(3, chunk_dims);
|
||||
hsize_t dimsChunk[DATA_RANK] = {MAX_CHUNKED_IMAGES, nDimy, nDimz};
|
||||
hsize_t dimsChunkPara[PARA_RANK] = {MAX_CHUNKED_IMAGES};
|
||||
|
||||
// dataspace
|
||||
dataSpace = nullptr;
|
||||
dataSpace = new H5::DataSpace(DATA_RANK, dims, dimsMax);
|
||||
dataSpacePara = nullptr;
|
||||
dataSpacePara = new H5::DataSpace(PARA_RANK, dimsPara, dimsMaxPara);
|
||||
|
||||
// property list
|
||||
H5::DSetCreatPropList plist;
|
||||
H5::DSetCreatPropList plistPara;
|
||||
int fill_value = -1;
|
||||
plist.setFillValue(dataType, &fill_value);
|
||||
//plistPara.setFillValue(dataType, &fill_value);
|
||||
plist.setChunk(DATA_RANK, dimsChunk);
|
||||
plistPara.setChunk(PARA_RANK, dimsChunkPara);
|
||||
|
||||
// dataset
|
||||
dataSet = nullptr;
|
||||
dataSet = new H5::DataSet(fd->createDataSet(
|
||||
DATASET_NAME, dataType, *dataSpace, plist));
|
||||
|
||||
// create parameter datasets
|
||||
hsize_t dims[1] = {nDimx};
|
||||
hsize_t dimsmax[1] = {H5S_UNLIMITED};
|
||||
dataSpacePara = nullptr;
|
||||
dataSpacePara = new H5::DataSpace(1, dims, dimsmax);
|
||||
|
||||
// always create chunked dataset as unlimited is only
|
||||
// supported with chunked layout
|
||||
H5::DSetCreatPropList paralist;
|
||||
hsize_t chunkpara_dims[3] = {MAX_CHUNKED_IMAGES};
|
||||
paralist.setChunk(1, chunkpara_dims);
|
||||
|
||||
for (unsigned int i = 0; i < parameterNames.size(); ++i) {
|
||||
H5::DataSet *ds = new H5::DataSet(fd->createDataSet(
|
||||
parameterNames[i].c_str(), parameterDataTypes[i],
|
||||
*dataSpacePara, paralist));
|
||||
*dataSpacePara, plistPara));
|
||||
dataSetPara.push_back(ds);
|
||||
}
|
||||
} catch (const H5::Exception &error) {
|
||||
@ -237,7 +235,7 @@ void HDF5DataFile::WriteToFile(char *imageData, sls_receiver_header& header, con
|
||||
ExtendDataset();
|
||||
}
|
||||
|
||||
WriteDataFile(currentFrameNumber, imageData);
|
||||
WriteImageDatasets(currentFrameNumber, imageData);
|
||||
WriteParameterDatasets(currentFrameNumber, header);
|
||||
}
|
||||
|
||||
@ -251,7 +249,7 @@ void HDF5DataFile::Convert12to16Bit(uint16_t *dst, uint8_t *src) {
|
||||
}
|
||||
}
|
||||
|
||||
void HDF5DataFile::WriteDataFile(const uint64_t currentFrameNumber,
|
||||
void HDF5DataFile::WriteImageDatasets(const uint64_t currentFrameNumber,
|
||||
char *buffer) {
|
||||
// expand 12 bit to 16 bits
|
||||
char *revBuffer = buffer;
|
||||
@ -306,8 +304,8 @@ void HDF5DataFile::WriteParameterDatasets(const uint64_t currentFrameNumber,
|
||||
: currentFrameNumber % maxFramesPerFile);
|
||||
|
||||
sls_detector_header header = rheader.detHeader;
|
||||
hsize_t count[1] = {1};
|
||||
hsize_t start[1] = {fnum};
|
||||
hsize_t count[PARA_RANK] = {1};
|
||||
hsize_t start[PARA_RANK] = {fnum};
|
||||
int i = 0;
|
||||
try {
|
||||
H5::Exception::dontPrint(); // to handle errors
|
||||
@ -387,18 +385,17 @@ void HDF5DataFile::ExtendDataset() {
|
||||
try {
|
||||
H5::Exception::dontPrint(); // to handle errors
|
||||
|
||||
hsize_t dims[3];
|
||||
hsize_t dims[DATA_RANK];
|
||||
dataSpace->getSimpleExtentDims(dims);
|
||||
dims[0] += numImages;
|
||||
|
||||
dataSet->extend(dims);
|
||||
delete dataSpace;
|
||||
dataSpace = nullptr;
|
||||
dataSpace = new H5::DataSpace(dataSet->getSpace());
|
||||
|
||||
hsize_t dims_para[1] = {dims[0]};
|
||||
hsize_t dimsPara[PARA_RANK] = {dims[0]};
|
||||
for (unsigned int i = 0; i < dataSetPara.size(); ++i)
|
||||
dataSetPara[i]->extend(dims_para);
|
||||
dataSetPara[i]->extend(dimsPara);
|
||||
delete dataSpacePara;
|
||||
dataSpacePara = nullptr;
|
||||
dataSpacePara = new H5::DataSpace(dataSetPara[0]->getSpace());
|
||||
|
||||
@ -36,7 +36,7 @@ class HDF5DataFile : private virtual slsDetectorDefs, public File {
|
||||
private:
|
||||
void CreateFile();
|
||||
void Convert12to16Bit(uint16_t *dst, uint8_t *src);
|
||||
void WriteDataFile(const uint64_t currentFrameNumber, char *buffer);
|
||||
void WriteImageDatasets(const uint64_t currentFrameNumber, char *buffer);
|
||||
void WriteParameterDatasets(const uint64_t currentFrameNumber,
|
||||
sls_receiver_header rheader);
|
||||
void ExtendDataset();
|
||||
|
||||
@ -207,21 +207,22 @@ std::string CreateVirtualHDF5File(
|
||||
"version", H5::PredType::NATIVE_DOUBLE, dataspace_attr);
|
||||
attribute.write(H5::PredType::NATIVE_DOUBLE, &dValue);
|
||||
|
||||
// virtual dataspace
|
||||
hsize_t vdsDims[3] = {numImagesCaught, numModY * nDimy,
|
||||
// dataspace
|
||||
hsize_t vdsDims[DATA_RANK] = {numImagesCaught, numModY * nDimy,
|
||||
numModZ * nDimz};
|
||||
H5::DataSpace vdsDataSpace(3, vdsDims, nullptr);
|
||||
hsize_t vdsDimsPara[2] = {numImagesCaught,
|
||||
(unsigned int)numModY * numModZ};
|
||||
H5::DataSpace vdsDataSpacePara(2, vdsDimsPara, nullptr);
|
||||
hsize_t vdsDimsPara[VDS_PARA_RANK] = {numImagesCaught, numModY * numModZ};
|
||||
H5::DataSpace vdsDataSpace(DATA_RANK, vdsDims, nullptr);
|
||||
H5::DataSpace vdsDataSpacePara(VDS_PARA_RANK, vdsDimsPara, nullptr);
|
||||
|
||||
// property list (fill value and datatype)
|
||||
int fill_value = -1;
|
||||
// property list
|
||||
H5::DSetCreatPropList plist;
|
||||
int fill_value = -1;
|
||||
plist.setFillValue(dataType, &fill_value);
|
||||
|
||||
// property list for parameters (datatype)
|
||||
std::vector<H5::DSetCreatPropList> plistPara(paraSize);
|
||||
// ignoring last fill (string)
|
||||
for (unsigned int i = 0; i != plistPara.size() - 1; ++i) {
|
||||
plistPara[i].setFillValue(parameterDataTypes[i], &fill_value);
|
||||
}
|
||||
|
||||
// hyperslab (files)
|
||||
int numFiles = numImagesCaught / maxFramesPerFile;
|
||||
@ -235,15 +236,15 @@ std::string CreateVirtualHDF5File(
|
||||
? maxFramesPerFile
|
||||
: (numImagesCaught - framesSaved);
|
||||
|
||||
hsize_t startLocation[3] = {framesSaved, 0, 0};
|
||||
hsize_t strideBetweenBlocks[3] = {1, 1, 1};
|
||||
hsize_t numBlocks[3] = {nDimx, nDimy, nDimz};
|
||||
hsize_t blockSize[3] = {1, 1, 1};
|
||||
hsize_t startLocation[DATA_RANK] = {framesSaved, 0, 0};
|
||||
hsize_t strideBetweenBlocks[DATA_RANK] = {1, 1, 1};
|
||||
hsize_t numBlocks[DATA_RANK] = {nDimx, nDimy, nDimz};
|
||||
hsize_t blockSize[DATA_RANK] = {1, 1, 1};
|
||||
|
||||
hsize_t startLocationPara[2] = {framesSaved, 0};
|
||||
hsize_t strideBetweenBlocksPara[3] = {1, 1};
|
||||
hsize_t numBlocksPara[2] = {1, 1};
|
||||
hsize_t blockSizePara[3] = {nDimx, 1};
|
||||
hsize_t startLocationPara[VDS_PARA_RANK] = {framesSaved, 0};
|
||||
hsize_t strideBetweenBlocksPara[VDS_PARA_RANK] = {1, 1};
|
||||
hsize_t numBlocksPara[VDS_PARA_RANK] = {nDimx, 1};
|
||||
hsize_t blockSizePara[VDS_PARA_RANK] = {1, 1};
|
||||
|
||||
// interleaving for g2
|
||||
if (gotthard25um) {
|
||||
@ -284,12 +285,21 @@ std::string CreateVirtualHDF5File(
|
||||
}
|
||||
|
||||
// source dataspace
|
||||
hsize_t srcDims[3] = {nDimx, nDimy, nDimz};
|
||||
hsize_t srcDimsMax[3] = {H5S_UNLIMITED, nDimy, nDimz};
|
||||
H5::DataSpace srcDataSpace(3, srcDims, srcDimsMax);
|
||||
hsize_t srcDimsPara[1] = {nDimx};
|
||||
hsize_t srcDimsMaxPara[1] = {H5S_UNLIMITED};
|
||||
H5::DataSpace srcDataSpacePara(1, srcDimsPara, srcDimsMaxPara);
|
||||
hsize_t srcDims[DATA_RANK] = {nDimx, nDimy, nDimz};
|
||||
hsize_t srcDimsMax[DATA_RANK] = {H5S_UNLIMITED, nDimy, nDimz};
|
||||
H5::DataSpace srcDataSpace(DATA_RANK, srcDims, srcDimsMax);
|
||||
hsize_t srcDimsPara[PARA_RANK] = {nDimx};
|
||||
hsize_t srcDimsMaxPara[PARA_RANK] = {H5S_UNLIMITED};
|
||||
H5::DataSpace srcDataSpacePara(PARA_RANK, srcDimsPara, srcDimsMaxPara);
|
||||
// temporary fixfor corner case bug:
|
||||
// (framescaught not multiple of framesperfile,
|
||||
// virtual parameter datasets error loading (bad scalar value))
|
||||
if (nDimx != maxFramesPerFile) {
|
||||
hsize_t count[1] = {nDimx};
|
||||
hsize_t start[1] = {0};
|
||||
srcDataSpacePara.selectHyperslab(H5S_SELECT_SET, count, start, strideBetweenBlocksPara, blockSizePara);
|
||||
}
|
||||
|
||||
|
||||
// mapping of property list
|
||||
plist.setVirtual(vdsDataSpace, relative_srcFileName.c_str(),
|
||||
@ -310,7 +320,7 @@ std::string CreateVirtualHDF5File(
|
||||
startLocation[1] += nDimy;
|
||||
}
|
||||
}
|
||||
startLocationPara[1]++;
|
||||
++startLocationPara[1];
|
||||
}
|
||||
framesSaved += nDimx;
|
||||
}
|
||||
|
||||
@ -47,6 +47,9 @@ struct image_structure {
|
||||
|
||||
// hdf5
|
||||
#define MAX_CHUNKED_IMAGES (1)
|
||||
#define DATA_RANK (3)
|
||||
#define PARA_RANK (1)
|
||||
#define VDS_PARA_RANK (2)
|
||||
|
||||
// parameters to calculate fifo depth
|
||||
#define SAMPLE_TIME_IN_NS (100000000) // 100ms
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user