From 20f3fb19af44205d937baaa88e6633eef9b8d8fe Mon Sep 17 00:00:00 2001 From: Dhanya Thattil Date: Fri, 4 Feb 2022 15:09:18 +0100 Subject: [PATCH 1/8] g25 option passed to hdf5 --- slsReceiverSoftware/src/DataProcessor.cpp | 4 +++- slsReceiverSoftware/src/HDF5VirtualFile.cpp | 4 ++-- slsReceiverSoftware/src/HDF5VirtualFile.h | 3 ++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/slsReceiverSoftware/src/DataProcessor.cpp b/slsReceiverSoftware/src/DataProcessor.cpp index 700561924..b97da54df 100644 --- a/slsReceiverSoftware/src/DataProcessor.cpp +++ b/slsReceiverSoftware/src/DataProcessor.cpp @@ -213,7 +213,9 @@ void DataProcessor::CreateVirtualFile( if (virtualFile_) { delete virtualFile_; } - virtualFile_ = new HDF5VirtualFile(hdf5Lib); + gotthard25um = ((detectorType_ == GOTTHARD || detectorType_ == GOTTHARD2) && + (numModX * numModY) == 2); + virtualFile_ = new HDF5VirtualFile(hdf5Lib, gotthard25um); uint64_t numImagesProcessed = GetProcessedIndex() + 1; // maxframesperfile = 0 for infinite files diff --git a/slsReceiverSoftware/src/HDF5VirtualFile.cpp b/slsReceiverSoftware/src/HDF5VirtualFile.cpp index 71ef51099..8349291b9 100644 --- a/slsReceiverSoftware/src/HDF5VirtualFile.cpp +++ b/slsReceiverSoftware/src/HDF5VirtualFile.cpp @@ -5,8 +5,8 @@ #include -HDF5VirtualFile::HDF5VirtualFile(std::mutex *hdf5Lib) - : File(HDF5), hdf5Lib_(hdf5Lib) {} +HDF5VirtualFile::HDF5VirtualFile(std::mutex *hdf5Lib, bool g25) + : File(HDF5), hdf5Lib_(hdf5Lib), gotthard25um(g25) {} HDF5VirtualFile::~HDF5VirtualFile() { CloseFile(); } diff --git a/slsReceiverSoftware/src/HDF5VirtualFile.h b/slsReceiverSoftware/src/HDF5VirtualFile.h index a479bce2d..89b1c0b60 100644 --- a/slsReceiverSoftware/src/HDF5VirtualFile.h +++ b/slsReceiverSoftware/src/HDF5VirtualFile.h @@ -9,7 +9,7 @@ class HDF5VirtualFile : private virtual slsDetectorDefs, public File { public: - HDF5VirtualFile(std::mutex *hdf5Lib); + HDF5VirtualFile(std::mutex *hdf5Lib, bool g25); ~HDF5VirtualFile(); std::array GetFileAndDatasetName() const override; @@ -30,4 +30,5 @@ class HDF5VirtualFile : private virtual slsDetectorDefs, public File { H5File *fd_{nullptr}; std::string fileName_; std::string dataSetName_; + bool gotthard25um; }; \ No newline at end of file From f228fde6f77aeda2408393d8547d0c673abf75d8 Mon Sep 17 00:00:00 2001 From: Dhanya Thattil Date: Mon, 7 Feb 2022 14:01:19 +0100 Subject: [PATCH 2/8] including stride and block for selecting hyperslab --- slsReceiverSoftware/src/DataProcessor.cpp | 5 ++-- slsReceiverSoftware/src/HDF5VirtualFile.cpp | 26 +++++++++++++++++---- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/slsReceiverSoftware/src/DataProcessor.cpp b/slsReceiverSoftware/src/DataProcessor.cpp index b97da54df..6f817e974 100644 --- a/slsReceiverSoftware/src/DataProcessor.cpp +++ b/slsReceiverSoftware/src/DataProcessor.cpp @@ -213,8 +213,9 @@ void DataProcessor::CreateVirtualFile( if (virtualFile_) { delete virtualFile_; } - gotthard25um = ((detectorType_ == GOTTHARD || detectorType_ == GOTTHARD2) && - (numModX * numModY) == 2); + bool gotthard25um = + ((detectorType_ == GOTTHARD || detectorType_ == GOTTHARD2) && + (numModX * numModY) == 2); virtualFile_ = new HDF5VirtualFile(hdf5Lib, gotthard25um); uint64_t numImagesProcessed = GetProcessedIndex() + 1; diff --git a/slsReceiverSoftware/src/HDF5VirtualFile.cpp b/slsReceiverSoftware/src/HDF5VirtualFile.cpp index 8349291b9..3b15831c4 100644 --- a/slsReceiverSoftware/src/HDF5VirtualFile.cpp +++ b/slsReceiverSoftware/src/HDF5VirtualFile.cpp @@ -103,19 +103,35 @@ void HDF5VirtualFile::CreateVirtualFile( ((numImagesCaught - framesSaved) > maxFramesPerFile) ? maxFramesPerFile : (numImagesCaught - framesSaved); + // starting location hsize_t start[3] = {framesSaved, 0, 0}; - hsize_t count[3] = {nDimx, nDimy, nDimz}; + // number of elements separating each block + hsize_t stride[3] = {1, 1, 1}; + // number of blocks + hsize_t count[3] = {1, 1, 1}; + // block size + hsize_t block[3] = {nDimx, nDimy, nDimz}; + + // starting location hsize_t startPara[2] = {framesSaved, 0}; - hsize_t countPara[2] = {nDimx, 1}; - // loop through readouts + // number of elements separating each block + hsize_t stridePara[3] = {1, 1}; + // number of blocks + hsize_t countPara[2] = {1, 1}; + // block size + hsize_t blockPara[3] = {nDimx, 1}; + + // loop through readouts (image) for (unsigned int i = 0; i < numModY * numModZ; ++i) { // setect data hyperslabs - vdsDataSpace.selectHyperslab(H5S_SELECT_SET, count, start); + vdsDataSpace.selectHyperslab(H5S_SELECT_SET, count, start, + stride, block); // select parameter hyperslabs vdsDataSpacePara.selectHyperslab(H5S_SELECT_SET, countPara, - startPara); + startPara, stridePara, + blockPara); // source file name std::ostringstream os; From f2cca765be03d6bb9186c4eed3544abade861ef2 Mon Sep 17 00:00:00 2001 From: Dhanya Thattil Date: Thu, 10 Feb 2022 16:59:25 +0100 Subject: [PATCH 3/8] wip --- slsReceiverSoftware/src/HDF5VirtualFile.cpp | 170 +++++++++++++++++++- 1 file changed, 169 insertions(+), 1 deletion(-) diff --git a/slsReceiverSoftware/src/HDF5VirtualFile.cpp b/slsReceiverSoftware/src/HDF5VirtualFile.cpp index 3b15831c4..b52925fa1 100644 --- a/slsReceiverSoftware/src/HDF5VirtualFile.cpp +++ b/slsReceiverSoftware/src/HDF5VirtualFile.cpp @@ -28,7 +28,7 @@ void HDF5VirtualFile::CloseFile() { error.printErrorStack(); } } - +/* void HDF5VirtualFile::CreateVirtualFile( const std::string filePath, const std::string fileNamePrefix, const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode, @@ -212,4 +212,172 @@ void HDF5VirtualFile::CreateVirtualFile( if (!silentMode) { LOG(logINFO) << "Virtual File: " << fileName_; } +} +*/ + +void HDF5VirtualFile::CreateVirtualFile( + const std::string filePath, const std::string fileNamePrefix, + const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode, + const int modulePos, const int numUnitsPerReadout, + const uint32_t maxFramesPerFile, const uint64_t numImages, + const uint32_t nPixelsX, const uint32_t nPixelsY, + const uint32_t dynamicRange, const uint64_t numImagesCaught, + const int numModX, const int numModY, const DataType dataType, + const std::vector parameterNames, + const std::vector parameterDataTypes) { + // virtual file name + std::ostringstream osfn; + osfn << filePath << "/" << fileNamePrefix << "_virtual" + << "_" << fileIndex << ".h5"; + fileName_ = osfn.str(); + + uint64_t numModZ = numModX; + uint32_t nDimy = nPixelsY; + uint32_t nDimz = ((dynamicRange == 4) ? (nPixelsX / 2) : nPixelsX); + + std::lock_guard lock(*hdf5Lib_); + + try { + Exception::dontPrint(); // to handle errors + + // file + FileAccPropList fapl; + fapl.setFcloseDegree(H5F_CLOSE_STRONG); + fd_ = nullptr; + if (!overWriteEnable) + fd_ = new H5File(fileName_.c_str(), H5F_ACC_EXCL, + FileCreatPropList::DEFAULT, fapl); + else + fd_ = new H5File(fileName_.c_str(), H5F_ACC_TRUNC, + FileCreatPropList::DEFAULT, fapl); + + // attributes - version + double dValue = HDF5_WRITER_VERSION; + DataSpace dataspace_attr = DataSpace(H5S_SCALAR); + Attribute attribute = fd_->createAttribute( + "version", PredType::NATIVE_DOUBLE, dataspace_attr); + attribute.write(PredType::NATIVE_DOUBLE, &dValue); + + // virtual data dataspace + hsize_t vdsDims[3] = {numImagesCaught, numModY * nDimy, + numModZ * nDimz}; + DataSpace vdsDataSpace(3, vdsDims, nullptr); + + // property list (fill value and datatype) + int fill_value = -1; + DSetCreatPropList plist; + plist.setFillValue(dataType, &fill_value); + + // hyperslab + int numFiles = numImagesCaught / maxFramesPerFile; + if (numImagesCaught % maxFramesPerFile) + ++numFiles; + uint64_t framesSaved = 0; + // loop through files + for (int file = 0; file < numFiles; ++file) { + + uint64_t nDimx = + ((numImagesCaught - framesSaved) > maxFramesPerFile) + ? maxFramesPerFile + : (numImagesCaught - framesSaved); + + // static const int nImages = 1280 * nDimx; + // hsize_t coord[nImages][3]; + + // loop through readouts (image) + for (unsigned int iReadout = 0; iReadout < numModY * numModZ; + ++iReadout) { + + // memset(&coord, 0, sizeof(coord)); + /*for (int x = 0; x != (int)nDimx; ++x) { + for (int z = 0; z != 1280; ++z) { + coord[1280 * x + z][0] = x + iReadout * nDimx; + coord[1280 * x + z][2] = z; + } + }*/ + hsize_t coord[1280][3]; + memset(coord, 0, sizeof(coord)); + for (int z = 0; z != 1280; ++z) { + coord[z][0] = iReadout; + coord[z][2] = z; + } + /*for (int x = 0; x != nImages; ++x) { + for (int y = 0; y != 3; ++y) { + LOG(logDEBUG) << "coord[" << x << "][" << y << "]:\t" + << coord[x][y] << '\n'; + } + }*/ + + LOG(logINFO) << iReadout << " before selecting"; + vdsDataSpace.selectElements(H5S_SELECT_SET, 1280, + (const hsize_t *)coord); + LOG(logINFO) << iReadout << " after selecting"; + + LOG(logINFO) << "vds valid:" << vdsDataSpace.selectValid(); + + // source file name + std::ostringstream os; + os << filePath << "/" << fileNamePrefix << "_d" + << (modulePos * numUnitsPerReadout + iReadout) << "_f" + << file << '_' << fileIndex << ".h5"; + std::string srcFileName = os.str(); + LOG(logDEBUG1) << srcFileName; + + // find relative path + std::string relative_srcFileName = srcFileName; + { + size_t p = srcFileName.rfind('/', srcFileName.length()); + if (p != std::string::npos) + relative_srcFileName = (srcFileName.substr( + p + 1, srcFileName.length() - p)); + } + + // source dataset name + std::ostringstream osfn; + osfn << "/data"; + if (numImages > 1) + osfn << "_f" << std::setfill('0') << std::setw(12) << file; + std::string srcDatasetName = osfn.str(); + + // source data dataspace + hsize_t srcDims[3] = {nDimx, nDimy, nDimz}; + // hsize_t srcDimsMax[3] = {H5S_UNLIMITED, nDimy, nDimz}; + DataSpace srcDataSpace(3, srcDims); //, srcDimsMax); + /* + LOG(logINFO) << iReadout << " before selecting + src"; srcDataSpace.selectElements(H5S_SELECT_SET, 1280, + (const hsize_t + *)coord); LOG(logINFO) << iReadout << " after selecting src"; + LOG(logINFO) << "src valid:" << + srcDataSpace.selectValid(); + */ + // mapping of data property list + // int values[] = {53, 59, 61, 67}; /* New values to be written + // */ vdsDataSet->write(values, dataType, srcDataSpace, + // vdsDataSpace); + + LOG(logINFORED) << iReadout << " before vritual"; + plist.setVirtual(vdsDataSpace, relative_srcFileName.c_str(), + srcDatasetName.c_str(), srcDataSpace); + LOG(logINFORED) << iReadout << " before virtual"; + + // H5Sclose(srcDataspace); + } + framesSaved += nDimx; + } + // data dataset + dataSetName_ = "data"; + DataSet vdsDataSet(fd_->createDataSet(dataSetName_.c_str(), dataType, + vdsDataSpace, plist)); + + fd_->close(); + } catch (const Exception &error) { + error.printErrorStack(); + CloseFile(); + throw sls::RuntimeError( + "Could not create/overwrite virtual HDF5 handles"); + } + if (!silentMode) { + LOG(logINFO) << "Virtual File: " << fileName_; + } } \ No newline at end of file From 3e5b8840b467149b9d0af45a12daac151f5bfe9a Mon Sep 17 00:00:00 2001 From: Dhanya Thattil Date: Thu, 17 Mar 2022 12:21:29 +0100 Subject: [PATCH 4/8] wip --- RELEASE.txt | 2 +- slsReceiverSoftware/src/HDF5VirtualFile.cpp | 170 +------------------- 2 files changed, 2 insertions(+), 170 deletions(-) diff --git a/RELEASE.txt b/RELEASE.txt index 2b46be795..c72a50339 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -50,7 +50,7 @@ This document describes the differences between v7.0.0 and v6.x.x - added geometry to metadata - 10g eiger nextframenumber get fixed. - stop, able to set nextframenumber to a consistent (max + 1) for all modules if different (eiger/ctb/jungfrau/moench) - +- gotthard 25 um image reconstructed in gui and virtual hdf5 (firmware updated for slave to reverse channels) 2. Resolved Issues ================== diff --git a/slsReceiverSoftware/src/HDF5VirtualFile.cpp b/slsReceiverSoftware/src/HDF5VirtualFile.cpp index b52925fa1..6c5f3202c 100644 --- a/slsReceiverSoftware/src/HDF5VirtualFile.cpp +++ b/slsReceiverSoftware/src/HDF5VirtualFile.cpp @@ -28,7 +28,7 @@ void HDF5VirtualFile::CloseFile() { error.printErrorStack(); } } -/* + void HDF5VirtualFile::CreateVirtualFile( const std::string filePath, const std::string fileNamePrefix, const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode, @@ -213,171 +213,3 @@ void HDF5VirtualFile::CreateVirtualFile( LOG(logINFO) << "Virtual File: " << fileName_; } } -*/ - -void HDF5VirtualFile::CreateVirtualFile( - const std::string filePath, const std::string fileNamePrefix, - const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode, - const int modulePos, const int numUnitsPerReadout, - const uint32_t maxFramesPerFile, const uint64_t numImages, - const uint32_t nPixelsX, const uint32_t nPixelsY, - const uint32_t dynamicRange, const uint64_t numImagesCaught, - const int numModX, const int numModY, const DataType dataType, - const std::vector parameterNames, - const std::vector parameterDataTypes) { - // virtual file name - std::ostringstream osfn; - osfn << filePath << "/" << fileNamePrefix << "_virtual" - << "_" << fileIndex << ".h5"; - fileName_ = osfn.str(); - - uint64_t numModZ = numModX; - uint32_t nDimy = nPixelsY; - uint32_t nDimz = ((dynamicRange == 4) ? (nPixelsX / 2) : nPixelsX); - - std::lock_guard lock(*hdf5Lib_); - - try { - Exception::dontPrint(); // to handle errors - - // file - FileAccPropList fapl; - fapl.setFcloseDegree(H5F_CLOSE_STRONG); - fd_ = nullptr; - if (!overWriteEnable) - fd_ = new H5File(fileName_.c_str(), H5F_ACC_EXCL, - FileCreatPropList::DEFAULT, fapl); - else - fd_ = new H5File(fileName_.c_str(), H5F_ACC_TRUNC, - FileCreatPropList::DEFAULT, fapl); - - // attributes - version - double dValue = HDF5_WRITER_VERSION; - DataSpace dataspace_attr = DataSpace(H5S_SCALAR); - Attribute attribute = fd_->createAttribute( - "version", PredType::NATIVE_DOUBLE, dataspace_attr); - attribute.write(PredType::NATIVE_DOUBLE, &dValue); - - // virtual data dataspace - hsize_t vdsDims[3] = {numImagesCaught, numModY * nDimy, - numModZ * nDimz}; - DataSpace vdsDataSpace(3, vdsDims, nullptr); - - // property list (fill value and datatype) - int fill_value = -1; - DSetCreatPropList plist; - plist.setFillValue(dataType, &fill_value); - - // hyperslab - int numFiles = numImagesCaught / maxFramesPerFile; - if (numImagesCaught % maxFramesPerFile) - ++numFiles; - uint64_t framesSaved = 0; - // loop through files - for (int file = 0; file < numFiles; ++file) { - - uint64_t nDimx = - ((numImagesCaught - framesSaved) > maxFramesPerFile) - ? maxFramesPerFile - : (numImagesCaught - framesSaved); - - // static const int nImages = 1280 * nDimx; - // hsize_t coord[nImages][3]; - - // loop through readouts (image) - for (unsigned int iReadout = 0; iReadout < numModY * numModZ; - ++iReadout) { - - // memset(&coord, 0, sizeof(coord)); - /*for (int x = 0; x != (int)nDimx; ++x) { - for (int z = 0; z != 1280; ++z) { - coord[1280 * x + z][0] = x + iReadout * nDimx; - coord[1280 * x + z][2] = z; - } - }*/ - hsize_t coord[1280][3]; - memset(coord, 0, sizeof(coord)); - for (int z = 0; z != 1280; ++z) { - coord[z][0] = iReadout; - coord[z][2] = z; - } - /*for (int x = 0; x != nImages; ++x) { - for (int y = 0; y != 3; ++y) { - LOG(logDEBUG) << "coord[" << x << "][" << y << "]:\t" - << coord[x][y] << '\n'; - } - }*/ - - LOG(logINFO) << iReadout << " before selecting"; - vdsDataSpace.selectElements(H5S_SELECT_SET, 1280, - (const hsize_t *)coord); - LOG(logINFO) << iReadout << " after selecting"; - - LOG(logINFO) << "vds valid:" << vdsDataSpace.selectValid(); - - // source file name - std::ostringstream os; - os << filePath << "/" << fileNamePrefix << "_d" - << (modulePos * numUnitsPerReadout + iReadout) << "_f" - << file << '_' << fileIndex << ".h5"; - std::string srcFileName = os.str(); - LOG(logDEBUG1) << srcFileName; - - // find relative path - std::string relative_srcFileName = srcFileName; - { - size_t p = srcFileName.rfind('/', srcFileName.length()); - if (p != std::string::npos) - relative_srcFileName = (srcFileName.substr( - p + 1, srcFileName.length() - p)); - } - - // source dataset name - std::ostringstream osfn; - osfn << "/data"; - if (numImages > 1) - osfn << "_f" << std::setfill('0') << std::setw(12) << file; - std::string srcDatasetName = osfn.str(); - - // source data dataspace - hsize_t srcDims[3] = {nDimx, nDimy, nDimz}; - // hsize_t srcDimsMax[3] = {H5S_UNLIMITED, nDimy, nDimz}; - DataSpace srcDataSpace(3, srcDims); //, srcDimsMax); - /* - LOG(logINFO) << iReadout << " before selecting - src"; srcDataSpace.selectElements(H5S_SELECT_SET, 1280, - (const hsize_t - *)coord); LOG(logINFO) << iReadout << " after selecting src"; - LOG(logINFO) << "src valid:" << - srcDataSpace.selectValid(); - */ - // mapping of data property list - // int values[] = {53, 59, 61, 67}; /* New values to be written - // */ vdsDataSet->write(values, dataType, srcDataSpace, - // vdsDataSpace); - - LOG(logINFORED) << iReadout << " before vritual"; - plist.setVirtual(vdsDataSpace, relative_srcFileName.c_str(), - srcDatasetName.c_str(), srcDataSpace); - LOG(logINFORED) << iReadout << " before virtual"; - - // H5Sclose(srcDataspace); - } - framesSaved += nDimx; - } - // data dataset - dataSetName_ = "data"; - DataSet vdsDataSet(fd_->createDataSet(dataSetName_.c_str(), dataType, - vdsDataSpace, plist)); - - fd_->close(); - } catch (const Exception &error) { - error.printErrorStack(); - CloseFile(); - throw sls::RuntimeError( - "Could not create/overwrite virtual HDF5 handles"); - } - if (!silentMode) { - LOG(logINFO) << "Virtual File: " << fileName_; - } -} \ No newline at end of file From ca0aa7144c550d4dc6ff4ef53005d6bf350c5bc3 Mon Sep 17 00:00:00 2001 From: Dhanya Thattil Date: Thu, 17 Mar 2022 12:53:32 +0100 Subject: [PATCH 5/8] hdf5 for g25um --- slsReceiverSoftware/src/HDF5VirtualFile.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/slsReceiverSoftware/src/HDF5VirtualFile.cpp b/slsReceiverSoftware/src/HDF5VirtualFile.cpp index 6c5f3202c..519dfe1f0 100644 --- a/slsReceiverSoftware/src/HDF5VirtualFile.cpp +++ b/slsReceiverSoftware/src/HDF5VirtualFile.cpp @@ -103,14 +103,13 @@ void HDF5VirtualFile::CreateVirtualFile( ((numImagesCaught - framesSaved) > maxFramesPerFile) ? maxFramesPerFile : (numImagesCaught - framesSaved); - // starting location - hsize_t start[3] = {framesSaved, 0, 0}; + // number of elements separating each block - hsize_t stride[3] = {1, 1, 1}; + hsize_t stride[3] = {1, 1, 2}; // number of blocks - hsize_t count[3] = {1, 1, 1}; + hsize_t count[3] = {nDimx, nDimy, nDimz}; // block size - hsize_t block[3] = {nDimx, nDimy, nDimz}; + hsize_t block[3] = {1, 1, 1}; // starting location hsize_t startPara[2] = {framesSaved, 0}; @@ -124,6 +123,9 @@ void HDF5VirtualFile::CreateVirtualFile( // loop through readouts (image) for (unsigned int i = 0; i < numModY * numModZ; ++i) { + // starting location + hsize_t start[3] = {framesSaved, 0, i}; + // setect data hyperslabs vdsDataSpace.selectHyperslab(H5S_SELECT_SET, count, start, stride, block); From afbc414afeebaabdc4580cf8203fcd810490ff0b Mon Sep 17 00:00:00 2001 From: Dhanya Thattil Date: Thu, 17 Mar 2022 13:08:40 +0100 Subject: [PATCH 6/8] works for both g25 and normal --- slsReceiverSoftware/src/HDF5VirtualFile.cpp | 26 +++++++++++++++------ 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/slsReceiverSoftware/src/HDF5VirtualFile.cpp b/slsReceiverSoftware/src/HDF5VirtualFile.cpp index 519dfe1f0..03ab02008 100644 --- a/slsReceiverSoftware/src/HDF5VirtualFile.cpp +++ b/slsReceiverSoftware/src/HDF5VirtualFile.cpp @@ -104,8 +104,10 @@ void HDF5VirtualFile::CreateVirtualFile( ? maxFramesPerFile : (numImagesCaught - framesSaved); + // starting location + hsize_t start[3] = {framesSaved, 0, 0}; // number of elements separating each block - hsize_t stride[3] = {1, 1, 2}; + hsize_t stride[3] = {1, 1, 1}; // number of blocks hsize_t count[3] = {nDimx, nDimy, nDimz}; // block size @@ -120,11 +122,18 @@ void HDF5VirtualFile::CreateVirtualFile( // block size hsize_t blockPara[3] = {nDimx, 1}; + // interleaving for g2 + if (gotthard25um) { + stride[2] = 2; + } + // loop through readouts (image) for (unsigned int i = 0; i < numModY * numModZ; ++i) { - // starting location - hsize_t start[3] = {framesSaved, 0, i}; + // interleaving for g2 + if (gotthard25um) { + start[2] = i; + } // setect data hyperslabs vdsDataSpace.selectHyperslab(H5S_SELECT_SET, count, start, @@ -183,10 +192,13 @@ void HDF5VirtualFile::CreateVirtualFile( // H5Sclose(srcDataspace); // H5Sclose(srcDataspace_para); - start[2] += nDimz; - if (start[2] >= (numModZ * nDimz)) { - start[2] = 0; - start[1] += nDimy; + + if (!gotthard25um) { + start[2] += nDimz; + if (start[2] >= (numModZ * nDimz)) { + start[2] = 0; + start[1] += nDimy; + } } startPara[1]++; } From 570651a9f8db8f85488dd32d0a2db12aa567bc6b Mon Sep 17 00:00:00 2001 From: Dhanya Thattil Date: Thu, 17 Mar 2022 13:09:54 +0100 Subject: [PATCH 7/8] minor --- slsReceiverSoftware/src/HDF5VirtualFile.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slsReceiverSoftware/src/HDF5VirtualFile.cpp b/slsReceiverSoftware/src/HDF5VirtualFile.cpp index 03ab02008..7ee6f94e8 100644 --- a/slsReceiverSoftware/src/HDF5VirtualFile.cpp +++ b/slsReceiverSoftware/src/HDF5VirtualFile.cpp @@ -130,7 +130,7 @@ void HDF5VirtualFile::CreateVirtualFile( // loop through readouts (image) for (unsigned int i = 0; i < numModY * numModZ; ++i) { - // interleaving for g2 + // interleaving for g2 (start is 0 and 1) if (gotthard25um) { start[2] = i; } From fc21a6763d1452effabe7376e5023f69c5fccc3d Mon Sep 17 00:00:00 2001 From: Dhanya Thattil Date: Wed, 23 Mar 2022 12:03:51 +0100 Subject: [PATCH 8/8] fix and minor removing comments --- slsReceiverSoftware/src/DataProcessor.cpp | 2 +- slsReceiverSoftware/src/HDF5VirtualFile.cpp | 88 ++++++++------------- 2 files changed, 36 insertions(+), 54 deletions(-) diff --git a/slsReceiverSoftware/src/DataProcessor.cpp b/slsReceiverSoftware/src/DataProcessor.cpp index 5ad3b6707..a22abdfc1 100644 --- a/slsReceiverSoftware/src/DataProcessor.cpp +++ b/slsReceiverSoftware/src/DataProcessor.cpp @@ -217,7 +217,7 @@ void DataProcessor::CreateVirtualFile( filePath, fileNamePrefix, fileIndex, overWriteEnable, silentMode, modulePos, numUnitsPerReadout, framesPerFile, numImages, generalData_->nPixelsX, generalData_->nPixelsY, dynamicRange, - numImagesProcessed, numModX, numModY, dataFile_->GetPDataType(), + numFramesCaught_, numModX, numModY, dataFile_->GetPDataType(), dataFile_->GetParameterNames(), dataFile_->GetParameterDataTypes()); } diff --git a/slsReceiverSoftware/src/HDF5VirtualFile.cpp b/slsReceiverSoftware/src/HDF5VirtualFile.cpp index 7ee6f94e8..015ad0ae3 100644 --- a/slsReceiverSoftware/src/HDF5VirtualFile.cpp +++ b/slsReceiverSoftware/src/HDF5VirtualFile.cpp @@ -73,12 +73,10 @@ void HDF5VirtualFile::CreateVirtualFile( "version", PredType::NATIVE_DOUBLE, dataspace_attr); attribute.write(PredType::NATIVE_DOUBLE, &dValue); - // virtual data dataspace + // virtual dataspace hsize_t vdsDims[3] = {numImagesCaught, numModY * nDimy, numModZ * nDimz}; DataSpace vdsDataSpace(3, vdsDims, nullptr); - - // virtual parameter dataspace hsize_t vdsDimsPara[2] = {numImagesCaught, (unsigned int)numModY * numModZ}; DataSpace vdsDataSpacePara(2, vdsDimsPara, nullptr); @@ -91,64 +89,54 @@ void HDF5VirtualFile::CreateVirtualFile( // property list for parameters (datatype) std::vector plistPara(paraSize); - // hyperslab - int numMajorHyperslab = numImagesCaught / maxFramesPerFile; + // hyperslab (files) + int numFiles = numImagesCaught / maxFramesPerFile; if (numImagesCaught % maxFramesPerFile) - ++numMajorHyperslab; + ++numFiles; uint64_t framesSaved = 0; - // loop through files - for (int hyperSlab = 0; hyperSlab < numMajorHyperslab; ++hyperSlab) { + for (int iFile = 0; iFile < numFiles; ++iFile) { uint64_t nDimx = ((numImagesCaught - framesSaved) > maxFramesPerFile) ? maxFramesPerFile : (numImagesCaught - framesSaved); - // starting location - hsize_t start[3] = {framesSaved, 0, 0}; - // number of elements separating each block - hsize_t stride[3] = {1, 1, 1}; - // number of blocks - hsize_t count[3] = {nDimx, nDimy, nDimz}; - // block size - hsize_t block[3] = {1, 1, 1}; + hsize_t startLocation[3] = {framesSaved, 0, 0}; + hsize_t strideBetweenBlocks[3] = {1, 1, 1}; + hsize_t numBlocks[3] = {nDimx, nDimy, nDimz}; + hsize_t blockSize[3] = {1, 1, 1}; - // starting location - hsize_t startPara[2] = {framesSaved, 0}; - // number of elements separating each block - hsize_t stridePara[3] = {1, 1}; - // number of blocks - hsize_t countPara[2] = {1, 1}; - // block size - hsize_t blockPara[3] = {nDimx, 1}; + hsize_t startLocationPara[2] = {framesSaved, 0}; + hsize_t strideBetweenBlocksPara[3] = {1, 1}; + hsize_t numBlocksPara[2] = {1, 1}; + hsize_t blockSizePara[3] = {nDimx, 1}; // interleaving for g2 if (gotthard25um) { - stride[2] = 2; + strideBetweenBlocks[2] = 2; } - // loop through readouts (image) - for (unsigned int i = 0; i < numModY * numModZ; ++i) { + for (unsigned int iReadout = 0; iReadout < numModY * numModZ; + ++iReadout) { - // interleaving for g2 (start is 0 and 1) + // interleaving for g2 (startLocation is 0 and 1) if (gotthard25um) { - start[2] = i; + startLocation[2] = iReadout; } - // setect data hyperslabs - vdsDataSpace.selectHyperslab(H5S_SELECT_SET, count, start, - stride, block); + vdsDataSpace.selectHyperslab(H5S_SELECT_SET, numBlocks, + startLocation, strideBetweenBlocks, + blockSize); - // select parameter hyperslabs - vdsDataSpacePara.selectHyperslab(H5S_SELECT_SET, countPara, - startPara, stridePara, - blockPara); + vdsDataSpacePara.selectHyperslab( + H5S_SELECT_SET, numBlocksPara, startLocationPara, + strideBetweenBlocksPara, blockSizePara); // source file name std::ostringstream os; os << filePath << "/" << fileNamePrefix << "_d" - << (modulePos * numUnitsPerReadout + i) << "_f" << hyperSlab - << '_' << fileIndex << ".h5"; + << (modulePos * numUnitsPerReadout + iReadout) << "_f" + << iFile << '_' << fileIndex << ".h5"; std::string srcFileName = os.str(); LOG(logDEBUG1) << srcFileName; @@ -165,25 +153,20 @@ void HDF5VirtualFile::CreateVirtualFile( std::ostringstream osfn; osfn << "/data"; if (numImages > 1) - osfn << "_f" << std::setfill('0') << std::setw(12) - << hyperSlab; + osfn << "_f" << std::setfill('0') << std::setw(12) << iFile; std::string srcDatasetName = osfn.str(); - // source data dataspace + // source dataspace hsize_t srcDims[3] = {nDimx, nDimy, nDimz}; hsize_t srcDimsMax[3] = {H5S_UNLIMITED, nDimy, nDimz}; DataSpace srcDataSpace(3, srcDims, srcDimsMax); - - // source parameter dataspace hsize_t srcDimsPara[1] = {nDimx}; hsize_t srcDimsMaxPara[1] = {H5S_UNLIMITED}; DataSpace srcDataSpacePara(1, srcDimsPara, srcDimsMaxPara); - // mapping of data property list + // mapping of property list plist.setVirtual(vdsDataSpace, relative_srcFileName.c_str(), srcDatasetName.c_str(), srcDataSpace); - - // mapping of parameter property list for (unsigned int p = 0; p < paraSize; ++p) { plistPara[p].setVirtual( vdsDataSpacePara, relative_srcFileName.c_str(), @@ -194,22 +177,21 @@ void HDF5VirtualFile::CreateVirtualFile( // H5Sclose(srcDataspace_para); if (!gotthard25um) { - start[2] += nDimz; - if (start[2] >= (numModZ * nDimz)) { - start[2] = 0; - start[1] += nDimy; + startLocation[2] += nDimz; + if (startLocation[2] >= (numModZ * nDimz)) { + startLocation[2] = 0; + startLocation[1] += nDimy; } } - startPara[1]++; + startLocationPara[1]++; } framesSaved += nDimx; } - // data dataset + // datasets dataSetName_ = "data"; DataSet vdsDataSet(fd_->createDataSet(dataSetName_.c_str(), dataType, vdsDataSpace, plist)); - // parameter dataset for (unsigned int p = 0; p < paraSize; ++p) { DataSet vdsDataSetPara(fd_->createDataSet( parameterNames[p].c_str(), parameterDataTypes[p],