detectors/slsDetectorPackage
25
0
Fork 0
mirror of https://github.com/slsdetectorgroup/slsDetectorPackage.git synced 2025-06-04 09:00:41 +02:00
Code Issues Actions Packages Releases Activity

Wip

Browse Source
This commit is contained in:
maliakal_d 2021-07-02 16:09:57 +02:00
parent 0e4fd598b1
commit c30716db12
13 changed files with 422 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
slsReceiverSoftware/src/DataProcessor.cpp
View File

@ -92,12 +92,10 @@ void DataProcessor::CloseFiles() {
dataFile_->CloseFile();
if (masterFile_)
masterFile_->CloseFile();
/*
#ifdef HDF5C
if(virtualFile_)
virtualFile_->CloseFile();
#endif
*/
#ifdef HDF5C
if (virtualFile_)
virtualFile_->CloseFile();
#endif
}
void DataProcessor::DeleteFiles() {
@ -110,14 +108,12 @@ void DataProcessor::DeleteFiles() {
delete masterFile_;
masterFile_ = nullptr;
}
/*
#ifdef HDF5C
if(virtualFile_) {
delete virtualFile_;
virtualFile_ = nullptr;
}
#endif
*/
#ifdef HDF5C
if (virtualFile_) {
delete virtualFile_;
virtualFile_ = nullptr;
}
#endif
}
void DataProcessor::SetupFileWriter(const bool filewriteEnable,
const bool masterFilewriteEnable,
@ -133,7 +129,7 @@ void DataProcessor::SetupFileWriter(const bool filewriteEnable,
if (masterFilewriteEnable) {
masterFile_ = new HDF5MasterFile(hdf5Lib_);
}
// virtual file
virtualFile_ = new HDF5VirtualFile(hdf5Lib_);
}
break;
#endif
@ -176,9 +172,6 @@ void DataProcessor::CreateFirstFiles(
modulePos, numUnitsPerReadout, udpPortNumber, maxFramesPerFile,
numImages, generalData_->nPixelsX, generalData_->nPixelsY,
dynamicRange);
/*if (virtualFile_) {
virtualFile_->CreateFile();
}*/
break;
#endif
case BINARY:
@ -191,6 +184,54 @@ void DataProcessor::CreateFirstFiles(
}
}
#ifdef HDF5C
void DataProcessor::CreateVirtualFile(
const std::string filePath, const std::string fileNamePrefix,
const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode,
const int modulePos, const int numUnitsPerReadout,
const uint32_t maxFramesPerFile, const uint64_t numImages,
const uint32_t dynamicRange, const int numModX, const int numModY) {
if (virtualFile_ == nullptr || dataFile_ == nullptr) {
throw sls::RuntimeError("No virtual or data file object created");
}
std::string fname, datasetName;
// only 1 file (subfile and number of files in acquisition) (to link in
// master)
if (dataFile_->GetFilesInAcquisition() == 1 && (numModX * numModY) == 1) {
auto res = dataFile_->GetFileAndDatasetName();
fname = res[0];
datasetName = res[1];
}
// create virtual file and link that in master
else {
uint64_t numImagesProcessed = GetProcessedIndex() + 1;
// maxframesperfile = 0 for infinite files
uint32_t framesPerFile =
((maxFramesPerFile == 0) ? numImagesProcessed + 1
: maxFramesPerFile);
// TODO: assumption 1: create virtual file even if no data in other
// files (they exist anyway) assumption2: virtual file max frame index
// is from R0 P0 (difference from others when missing frames or for a
// stop acquisition)
virtualFile_->CreateVirtualFile(
filePath, fileNamePrefix, fileIndex, overWriteEnable, silentMode,
modulePos, numUnitsPerReadout, framesPerFile, numImages,
generalData_->nPixelsX, generalData_->nPixelsY, dynamicRange,
numImagesProcessed, numModX, numModY, dataFile_->GetPDataType(),
dataFile_->GetParameterNames(), dataFile_->GetParameterDataTypes());
auto res = virtualFile_->GetFileAndDatasetName();
fname = res[0];
datasetName = res[1];
}
// link in master
}
#endif
void DataProcessor::ThreadExecution() {
char *buffer = nullptr;
fifo_->PopAddress(buffer);

15
slsReceiverSoftware/src/DataProcessor.h
View File

@ -61,7 +61,17 @@ class DataProcessor : private virtual slsDetectorDefs, public ThreadObject {
const uint32_t maxFramesPerFile,
const uint64_t numImages,
const uint32_t dynamicRange);
#ifdef HDF5C
void CreateVirtualFile(const std::string filePath,
const std::string fileNamePrefix,
const uint64_t fileIndex, const bool overWriteEnable,
const bool silentMode, const int modulePos,
const int numUnitsPerReadout,
const uint32_t maxFramesPerFile,
const uint64_t numImages,
const uint32_t dynamicRange, const int numModX,
const int numModY);
#endif
/**
* Call back for raw data
* args to raw data ready callback are
@ -172,6 +182,9 @@ class DataProcessor : private virtual slsDetectorDefs, public ThreadObject {
File *dataFile_{nullptr};
File *masterFile_{nullptr};
std::mutex *hdf5Lib_;
#ifdef HDF5C
File *virtualFile_{nullptr};
#endif
// call back
/**

16
slsReceiverSoftware/src/DataStreamer.cpp
View File

@ -15,11 +15,11 @@
const std::string DataStreamer::TypeName = "DataStreamer";
DataStreamer::DataStreamer(int ind, Fifo *f, uint32_t *dr, ROI *r, uint64_t *fi,
int fd, int *nd, bool *qe, uint64_t *tot)
int fd, int *nm, bool *qe, uint64_t *tot)
: ThreadObject(ind, TypeName), fifo(f), dynamicRange(dr), roi(r),
fileIndex(fi), flippedDataX(fd), quadEnable(qe), totalNumFrames(tot) {
numDet[0] = nd[0];
numDet[1] = nd[1];
numMods[0] = nm[0];
numMods[1] = nm[1];
LOG(logDEBUG) << "DataStreamer " << ind << " created";
}
@ -60,9 +60,9 @@ void DataStreamer::RecordFirstIndex(uint64_t fnum, char *buf) {
void DataStreamer::SetGeneralData(GeneralData *g) { generalData = g; }
void DataStreamer::SetNumberofDetectors(int *nd) {
numDet[0] = nd[0];
numDet[1] = nd[1];
void DataStreamer::SetNumberofModules(int *nm) {
numMods[0] = nm[0];
numMods[1] = nm[1];
}
void DataStreamer::SetFlippedDataX(int fd) { flippedDataX = fd; }
@ -217,8 +217,8 @@ int DataStreamer::SendHeader(sls_receiver_header *rheader, uint32_t size,
zHeader.dynamicRange = *dynamicRange;
zHeader.fileIndex = *fileIndex;
zHeader.ndetx = numDet[0];
zHeader.ndety = numDet[1];
zHeader.ndetx = numMods[0];
zHeader.ndety = numMods[1];
zHeader.npixelsx = nx;
zHeader.npixelsy = ny;
zHeader.imageSize = size;

12
slsReceiverSoftware/src/DataStreamer.h
View File

@ -31,12 +31,12 @@ class DataStreamer : private virtual slsDetectorDefs, public ThreadObject {
* @param r roi
* @param fi pointer to file index
* @param fd flipped data enable for x dimension
* @param nd pointer to number of detectors in each dimension
* @param nm pointer to number of modules in each dimension
* @param qe pointer to quad Enable
* @param tot pointer to total number of frames
*/
DataStreamer(int ind, Fifo *f, uint32_t *dr, ROI *r, uint64_t *fi, int fd,
int *nd, bool *qe, uint64_t *tot);
int *nm, bool *qe, uint64_t *tot);
/**
* Destructor
@ -63,9 +63,9 @@ class DataStreamer : private virtual slsDetectorDefs, public ThreadObject {
/**
* Set number of detectors
* @param nd number of detectors in both dimensions
* @param nm number of modules in both dimensions
*/
void SetNumberofDetectors(int *nd);
void SetNumberofModules(int *nm);
/**
* Set Flipped data enable across x dimension
@ -193,8 +193,8 @@ class DataStreamer : private virtual slsDetectorDefs, public ThreadObject {
/** Complete buffer used for roi, eg. shortGotthard */
char *completeBuffer{nullptr};
/** Number of Detectors in X and Y dimension */
int numDet[2];
/** Number of Modules in X and Y dimension */
int numMods[2];
/** Quad Enable */
bool *quadEnable;

97
slsReceiverSoftware/src/File.h
View File

@ -5,6 +5,15 @@
struct MasterAttributes;
#ifdef HDF5C
#include "H5Cpp.h"
#ifndef H5_NO_NAMESPACE
using namespace H5;
#endif
#endif
#include <array>
class File : private virtual slsDetectorDefs {
public:
@ -14,6 +23,14 @@ class File : private virtual slsDetectorDefs {
fileFormat GetFileFormat() const;
virtual void CloseFile() = 0;
#ifdef HDF5C
virtual std::array<std::string, 2> GetFileAndDatasetName() const {
LOG(logERROR)
<< "This is a generic function GetFilesInAcquisition that "
"should be overloaded by a derived class";
return std::array<std::string, 2>{};
}
virtual uint32_t GetFilesInAcquisition() const {
LOG(logERROR)
<< "This is a generic function GetFilesInAcquisition that "
@ -21,6 +38,62 @@ class File : private virtual slsDetectorDefs {
return 0;
};
virtual DataType GetPDataType() const {
LOG(logERROR) << "This is a generic function GetPDataType that "
"should be overloaded by a derived class";
return PredType::STD_U16LE;
}
virtual std::vector<std::string> GetParameterNames() const {
LOG(logERROR)
<< "This is a generic function GetFilesInAcquisition that "
"should be overloaded by a derived class";
return std::vector<std::string>{};
};
virtual std::vector<DataType> GetParameterDataTypes() const {
LOG(logERROR)
<< "This is a generic function GetFilesInAcquisition that "
"should be overloaded by a derived class";
return std::vector<DataType>{};
};
virtual void CreateVirtualFile(
const std::string filePath, const std::string fileNamePrefix,
const uint64_t fileIndex, const bool overWriteEnable,
const bool silentMode, const int modulePos,
const int numUnitsPerReadout, const uint32_t maxFramesPerFile,
const uint64_t numImages, const uint32_t nPixelsX,
const uint32_t nPixelsY, const uint32_t dynamicRange,
const uint64_t numImagesCaught, const int numModX, const int numModY,
const DataType dataType, const std::vector<std::string> parameterNames,
const std::vector<DataType> parameterDataTypes) {
LOG(logERROR) << "This is a generic function CreateVirtualFile that "
"should be overloaded by a derived class";
}
virtual void CreateFirstHDF5DataFile(
const std::string filePath, const std::string fileNamePrefix,
const uint64_t fileIndex, const bool overWriteEnable,
const bool silentMode, const int modulePos,
const int numUnitsPerReadout, const uint32_t udpPortNumber,
const uint32_t maxFramesPerFile, const uint64_t numImages,
const uint32_t nPixelsX, const uint32_t nPixelsY,
const uint32_t dynamicRange) {
LOG(logERROR) << "This is a generic function CreateFirstDataFile that "
"should be overloaded by a derived class";
};
#endif
virtual void CreateFirstBinaryDataFile(
const std::string filePath, const std::string fileNamePrefix,
const uint64_t fileIndex, const bool overWriteEnable,
const bool silentMode, const int modulePos,
const int numUnitsPerReadout, const uint32_t udpPortNumber,
const uint32_t maxFramesPerFile) {
LOG(logERROR) << "This is a generic function CreateFirstDataFile that "
"should be overloaded by a derived class";
};
virtual void CreateMasterFile(const std::string filePath,
const std::string fileNamePrefix,
const uint64_t fileIndex,
@ -31,30 +104,6 @@ class File : private virtual slsDetectorDefs {
"should be overloaded by a derived class";
};
virtual void CreateFirstBinaryDataFile(
const std::string filePath, const std::string fileNamePrefix,
const uint64_t fileIndex, const bool overWriteEnable,
const bool silentMode, const int modulePos,
const int numUnitsPerReadout, const uint32_t udpPortNumber,
const uint32_t maxFramesPerFile) {
LOG(logERROR)
<< "This is a generic function CreateFirstBinaryDataFile that "
"should be overloaded by a derived class";
};
virtual void CreateFirstHDF5DataFile(
const std::string filePath, const std::string fileNamePrefix,
const uint64_t fileIndex, const bool overWriteEnable,
const bool silentMode, const int modulePos,
const int numUnitsPerReadout, const uint32_t udpPortNumber,
const uint32_t maxFramesPerFile, const uint64_t numImages,
const uint32_t nPIxelsX, const uint32_t nPIxelsY,
const uint32_t dynamicRange) {
LOG(logERROR)
<< "This is a generic function CreateFirstHDF5DataFile that "
"should be overloaded by a derived class";
};
virtual void WriteToFile(char *buffer, const int buffersize,
const uint64_t currentFrameNumber,
const uint32_t numPacketsCaught) {

25
slsReceiverSoftware/src/HDF5DataFile.cpp
View File

@ -33,10 +33,23 @@ HDF5DataFile::HDF5DataFile(int index, std::mutex *hdf5Lib)
HDF5DataFile::~HDF5DataFile() { CloseFile(); }
std::array<std::string, 2> HDF5DataFile::GetFileAndDatasetName() const {
return std::array<std::string, 2>{fileName_, dataSetName_};
}
uint32_t HDF5DataFile::GetFilesInAcquisition() const {
return numFilesInAcquisition_;
}
DataType HDF5DataFile::GetPDataType() const { return dataType_; }
std::vector<std::string> HDF5DataFile::GetParameterNames() const {
return parameterNames_;
}
std::vector<DataType> HDF5DataFile::GetParameterDataTypes() const {
return parameterDataTypes_;
}
void HDF5DataFile::CloseFile() {
std::lock_guard<std::mutex> lock(*hdf5Lib_);
try {
@ -73,7 +86,7 @@ void HDF5DataFile::CreateFirstHDF5DataFile(
const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode,
const int modulePos, const int numUnitsPerReadout,
const uint32_t udpPortNumber, const uint32_t maxFramesPerFile,
const uint64_t numImages, const uint32_t nPIxelsX, const uint32_t nPIxelsY,
const uint64_t numImages, const uint32_t nPixelsX, const uint32_t nPixelsY,
const uint32_t dynamicRange) {
subFileIndex_ = 0;
@ -83,8 +96,8 @@ void HDF5DataFile::CreateFirstHDF5DataFile(
maxFramesPerFile_ = maxFramesPerFile;
numImages_ = numImages;
nPixelsX_ = nPIxelsX;
nPixelsY_ = nPIxelsY;
nPixelsX_ = nPixelsX;
nPixelsY_ = nPixelsY;
dynamicRange_ = dynamicRange;
filePath_ = filePath;
@ -166,7 +179,7 @@ void HDF5DataFile::CreateFile() {
osfn << "/data";
if (numImages_ > 1)
osfn << "_f" << std::setfill('0') << std::setw(12) << subFileIndex_;
std::string dsetname = osfn.str();
dataSetName_ = osfn.str();
// dataset
// fill value
@ -178,8 +191,8 @@ void HDF5DataFile::CreateFile() {
hsize_t chunk_dims[3] = {MAX_CHUNKED_IMAGES, nDimy, nDimz};
plist.setChunk(3, chunk_dims);
dataSet_ = nullptr;
dataSet_ = new DataSet(fd_->createDataSet(dsetname.c_str(), dataType_,
*dataSpace_, plist));
dataSet_ = new DataSet(fd_->createDataSet(
dataSetName_.c_str(), dataType_, *dataSpace_, plist));
// create parameter datasets
hsize_t dims[1] = {nDimx};

12
slsReceiverSoftware/src/HDF5DataFile.h
View File

@ -2,11 +2,6 @@
#include "File.h"
#include "H5Cpp.h"
#ifndef H5_NO_NAMESPACE
using namespace H5;
#endif
#include <mutex>
class HDF5DataFile : private virtual slsDetectorDefs, public File {
@ -15,7 +10,11 @@ class HDF5DataFile : private virtual slsDetectorDefs, public File {
HDF5DataFile(const int index, std::mutex *hdf5Lib);
~HDF5DataFile();
std::array<std::string, 2> GetFileAndDatasetName() const override;
uint32_t GetFilesInAcquisition() const override;
DataType GetPDataType() const override;
std::vector<std::string> GetParameterNames() const override;
std::vector<DataType> GetParameterDataTypes() const override;
void CloseFile() override;
@ -25,7 +24,7 @@ class HDF5DataFile : private virtual slsDetectorDefs, public File {
const bool silentMode, const int modulePos,
const int numUnitsPerReadout, const uint32_t udpPortNumber,
const uint32_t maxFramesPerFile, const uint64_t numImages,
const uint32_t nPIxelsX, const uint32_t nPIxelsY,
const uint32_t nPixelsX, const uint32_t nPixelsY,
const uint32_t dynamicRange) override;
void WriteToFile(char *buffer, const int buffersize,
@ -43,6 +42,7 @@ class HDF5DataFile : private virtual slsDetectorDefs, public File {
std::mutex *hdf5Lib_;
H5File *fd_{nullptr};
std::string fileName_;
std::string dataSetName_;
DataSpace *dataSpace_{nullptr};
DataSet *dataSet_{nullptr};
DataType dataType_{PredType::STD_U16LE};

20
slsReceiverSoftware/src/HDF5MasterFile.cpp
View File

@ -31,11 +31,7 @@ void HDF5MasterFile::CreateMasterFile(const std::string filePath,
std::ostringstream os;
os << filePath << "/" << fileNamePrefix << "_master"
<< "_" << fileIndex << ".h5";
fileName_ = os.str();
if (!(silentMode)) {
LOG(logINFO) << "Master File: " << fileName_;
}
std::string fileName = os.str();
std::lock_guard<std::mutex> lock(*hdf5Lib_);
@ -46,12 +42,19 @@ void HDF5MasterFile::CreateMasterFile(const std::string filePath,
flist.setFcloseDegree(H5F_CLOSE_STRONG);
fd_ = nullptr;
if (!(overWriteEnable))
fd_ = new H5File(fileName_.c_str(), H5F_ACC_EXCL,
fd_ = new H5File(fileName.c_str(), H5F_ACC_EXCL,
FileCreatPropList::DEFAULT, flist);
else
fd_ = new H5File(fileName_.c_str(), H5F_ACC_TRUNC,
fd_ = new H5File(fileName.c_str(), H5F_ACC_TRUNC,
FileCreatPropList::DEFAULT, flist);
// attributes - version
double dValue = HDF5_WRITER_VERSION;
DataSpace dataspace_attr = DataSpace(H5S_SCALAR);
Attribute attribute = fd_->createAttribute(
"version", PredType::NATIVE_DOUBLE, dataspace_attr);
attribute.write(PredType::NATIVE_DOUBLE, &dValue);
// Create a group in the file
Group group1(fd_->createGroup("entry"));
Group group2(group1.createGroup("data"));
@ -69,4 +72,7 @@ void HDF5MasterFile::CreateMasterFile(const std::string filePath,
throw sls::RuntimeError(
"Could not create/overwrite master HDF5 handles");
}
if (!silentMode) {
LOG(logINFO) << "Master File: " << fileName;
}
}

6
slsReceiverSoftware/src/HDF5MasterFile.h
View File

@ -2,11 +2,6 @@
#include "File.h"
#include "H5Cpp.h"
#ifndef H5_NO_NAMESPACE
using namespace H5;
#endif
#include <mutex>
class HDF5MasterFile : private virtual slsDetectorDefs, public File {
@ -25,5 +20,4 @@ class HDF5MasterFile : private virtual slsDetectorDefs, public File {
private:
std::mutex *hdf5Lib_;
H5File *fd_{nullptr};
std::string fileName_;
};

184
slsReceiverSoftware/src/HDF5VirtualFile.cpp
View File

@ -1,18 +1,19 @@
#include "HDF5VirtualFile.h"
#include "receiver_defs.h"
#include <iomanip>
HDF5VirtualFile::HDF5VirtualFile(std::mutex *hdf5Lib)
: File(HDF5), hdf5Lib_(hdf5Lib) {}
HDF5VirtualFile::~HDF5VirtualFile() { CloseFile(); }
std::array<std::string, 2> HDF5VirtualFile::GetFileAndDatasetName() const {
return std::array<std::string, 2>{fileName_, dataSetName_};
}
void HDF5VirtualFile::CloseFile() {
std::lock_guard<std::mutex> lock(*hdf5Lib_);
/*if (fd_ != 0) {
if (H5Fclose(fd_) < 0) {
LOG(logERROR) << "Could not close virtual HDF5 handles";
}
fd_ = 0;
}*/
try {
Exception::dontPrint(); // to handle errors
if (fd_) {
@ -24,4 +25,175 @@ void HDF5VirtualFile::CloseFile() {
LOG(logERROR) << "Could not close virtual HDF5 handles of index";
error.printErrorStack();
}
}
void HDF5VirtualFile::CreateVirtualFile(
const std::string filePath, const std::string fileNamePrefix,
const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode,
const int modulePos, const int numUnitsPerReadout,
const uint32_t maxFramesPerFile, const uint64_t numImages,
const uint32_t nPixelsX, const uint32_t nPixelsY,
const uint32_t dynamicRange, const uint64_t numImagesCaught,
const int numModX, const int numModY, const DataType dataType,
const std::vector<std::string> parameterNames,
const std::vector<DataType> parameterDataTypes) {
// virtual file name
std::ostringstream osfn;
osfn << filePath << "/" << fileNamePrefix << "_virtual"
<< "_" << fileIndex << ".h5";
fileName_ = osfn.str();
unsigned int paraSize = parameterNames.size();
uint64_t numModZ = numModX;
uint32_t nDimy = nPixelsY;
uint32_t nDimz = ((dynamicRange == 4) ? (nPixelsX / 2) : nPixelsX);
std::lock_guard<std::mutex> lock(*hdf5Lib_);
try {
Exception::dontPrint(); // to handle errors
// file
FileAccPropList fapl;
fapl.setFcloseDegree(H5F_CLOSE_STRONG);
fd_ = nullptr;
if (!overWriteEnable)
fd_ = new H5File(fileName_.c_str(), H5F_ACC_EXCL,
FileCreatPropList::DEFAULT, fapl);
else
fd_ = new H5File(fileName_.c_str(), H5F_ACC_TRUNC,
FileCreatPropList::DEFAULT, fapl);
// attributes - version
double dValue = HDF5_WRITER_VERSION;
DataSpace dataspace_attr = DataSpace(H5S_SCALAR);
Attribute attribute = fd_->createAttribute(
"version", PredType::NATIVE_DOUBLE, dataspace_attr);
attribute.write(PredType::NATIVE_DOUBLE, &dValue);
// virtual data dataspace
hsize_t vdsDims[3] = {numImagesCaught, numModY * nDimy,
numModZ * nDimz};
DataSpace vdsDataSpace(3, vdsDims, nullptr);
// virtual parameter dataspace
hsize_t vdsDimsPara[2] = {numImagesCaught,
(unsigned int)numModY * numModZ};
DataSpace vdsDataSpacePara(2, vdsDimsPara, nullptr);
// property list (fill value and datatype)
int fill_value = -1;
DSetCreatPropList plist;
plist.setFillValue(dataType, &fill_value);
// property list for parameters (fill value and datatype)
std::vector<DSetCreatPropList> plistPara(paraSize);
for (unsigned int p = 0; p < paraSize; ++p) {
plistPara[p].setFillValue(parameterDataTypes[p], &fill_value);
}
// hyperslab
int numMajorHyperslab = numImagesCaught / maxFramesPerFile;
if (numImagesCaught % maxFramesPerFile)
++numMajorHyperslab;
uint64_t framesSaved = 0;
for (int hyperSlab = 0; hyperSlab < numMajorHyperslab; ++hyperSlab) {
uint64_t nDimx =
((numImagesCaught - framesSaved) > maxFramesPerFile)
? maxFramesPerFile
: (numImagesCaught - framesSaved);
hsize_t start[3] = {framesSaved, 0, 0};
hsize_t count[3] = {nDimx, nDimy, nDimz};
hsize_t startPara[2] = {framesSaved, 0};
hsize_t countPara[2] = {nDimx, 1};
for (unsigned int i = 0; i < numModY * numModZ; ++i) {
// setect data hyperslabs
vdsDataSpace.selectHyperslab(H5S_SELECT_SET, count, start);
// select parameter hyperslabs
vdsDataSpacePara.selectHyperslab(H5S_SELECT_SET, countPara,
startPara);
// source file name
std::ostringstream os;
os << filePath << "/" << fileNamePrefix << "_d"
<< (modulePos * numUnitsPerReadout + i) << "_f" << hyperSlab
<< '_' << fileIndex << ".h5";
std::string srcFileName = os.str();
LOG(logDEBUG1) << srcFileName;
// find relative path
std::string relative_srcFileName = srcFileName;
{
size_t p = srcFileName.rfind('/', srcFileName.length());
if (p != std::string::npos)
relative_srcFileName = (srcFileName.substr(
p + 1, srcFileName.length() - p));
}
// source dataset name
std::ostringstream osfn;
osfn << "/data";
if (numImages > 1)
osfn << "_f" << std::setfill('0') << std::setw(12)
<< hyperSlab;
std::string srcDatasetName = osfn.str();
// source data dataspace
hsize_t srcDims[3] = {nDimx, nDimy, nDimz};
hsize_t srcDimsMax[3] = {H5S_UNLIMITED, nDimy, nDimz};
DataSpace srcDataSpace(3, srcDims, srcDimsMax);
// source parameter dataspace
hsize_t srcDimsPara[1] = {nDimx};
hsize_t srcDimsMaxPara[1] = {H5S_UNLIMITED};
DataSpace srcDataSpacePara(3, srcDimsPara, srcDimsMaxPara);
// mapping of data property list
plist.setVirtual(vdsDataSpace, relative_srcFileName.c_str(),
srcDatasetName.c_str(), srcDataSpace);
// mapping of parameter property list
for (unsigned int p = 0; p < paraSize; ++p) {
plistPara[p].setVirtual(
vdsDataSpacePara, relative_srcFileName.c_str(),
parameterNames[p].c_str(), srcDataSpacePara);
}
// H5Sclose(srcDataspace);
// H5Sclose(srcDataspace_para);
start[2] += nDimz;
if (start[2] >= (numModZ * nDimz)) {
start[2] = 0;
start[1] += nDimy;
}
startPara[1]++;
}
framesSaved += nDimx;
}
// data dataset
dataSetName_ = "data";
DataSet vdsDataSet(fd_->createDataSet(dataSetName_.c_str(), dataType,
vdsDataSpace, plist));
// parameter dataset
for (unsigned int p = 0; p < paraSize; ++p) {
DataSet vdsDataSetPara(fd_->createDataSet(
parameterNames[p].c_str(), parameterDataTypes[p],
vdsDataSpacePara, plistPara[p]));
}
fd_->close();
} catch (const Exception &error) {
error.printErrorStack();
CloseFile();
throw sls::RuntimeError(
"Could not create/overwrite virtual HDF5 handles");
}
if (!silentMode) {
LOG(logINFO) << "Virtual File: " << fileName_;
}
}

19
slsReceiverSoftware/src/HDF5VirtualFile.h
View File

@ -2,11 +2,6 @@
#include "File.h"
#include "H5Cpp.h"
#ifndef H5_NO_NAMESPACE
using namespace H5;
#endif
#include <mutex>
class HDF5VirtualFile : private virtual slsDetectorDefs, public File {
@ -15,10 +10,22 @@ class HDF5VirtualFile : private virtual slsDetectorDefs, public File {
HDF5VirtualFile(std::mutex *hdf5Lib);
~HDF5VirtualFile();
std::array<std::string, 2> GetFileAndDatasetName() const override;
void CloseFile() override;
void CreateVirtualFile(
const std::string filePath, const std::string fileNamePrefix,
const uint64_t fileIndex, const bool overWriteEnable,
const bool silentMode, const int modulePos,
const int numUnitsPerReadout, const uint32_t maxFramesPerFile,
const uint64_t numImages, const uint32_t nPixelsX,
const uint32_t nPixelsY, const uint32_t dynamicRange,
const uint64_t numImagesCaught, const int numModX, const int numModY,
const DataType dataType, const std::vector<std::string> parameterNames,
const std::vector<DataType> parameterDataTypes) override;
private:
std::mutex *hdf5Lib_;
// hid_t fd_;
H5File *fd_{nullptr};
std::string fileName_;
std::string dataSetName_;
};

82
slsReceiverSoftware/src/Implementation.cpp
View File

@ -192,32 +192,32 @@ void Implementation::setDetectorType(const detectorType d) {
LOG(logDEBUG) << " Detector type set to " << sls::ToString(d);
}
int *Implementation::getDetectorSize() const { return (int *)numDet; }
int *Implementation::getDetectorSize() const { return (int *)numMods; }
void Implementation::setDetectorSize(const int *size) {
std::string log_message = "Detector Size (ports): (";
for (int i = 0; i < MAX_DIMENSIONS; ++i) {
// x dir (colums) each udp port
if (detType == EIGER && i == X)
numDet[i] = size[i] * 2;
numMods[i] = size[i] * 2;
// y dir (rows) each udp port
else if (numUDPInterfaces == 2 && i == Y)
numDet[i] = size[i] * 2;
numMods[i] = size[i] * 2;
else
numDet[i] = size[i];
log_message += std::to_string(numDet[i]);
numMods[i] = size[i];
log_message += std::to_string(numMods[i]);
if (i < MAX_DIMENSIONS - 1)
log_message += ", ";
}
log_message += ")";
int nd[2] = {numDet[0], numDet[1]};
int nm[2] = {numMods[0], numMods[1]};
if (quadEnable) {
nd[0] = 1;
nd[1] = 2;
nm[0] = 1;
nm[1] = 2;
}
for (const auto &it : dataStreamer) {
it->SetNumberofDetectors(nd);
it->SetNumberofModules(nm);
}
LOG(logINFO) << log_message;
@ -236,12 +236,12 @@ void Implementation::setModulePositionId(const int id) {
for (const auto &it : dataProcessor)
it->SetupFileWriter(fileWriteEnable, masterFileWriteEnable,
fileFormatType, modulePos);
assert(numDet[1] != 0);
assert(numMods[1] != 0);
for (unsigned int i = 0; i < listener.size(); ++i) {
uint16_t row = 0, col = 0;
row =
(modulePos % numDet[1]) * ((numUDPInterfaces == 2) ? 2 : 1); // row
col = (modulePos / numDet[1]) * ((detType == EIGER) ? 2 : 1) +
(modulePos % numMods[1]) * ((numUDPInterfaces == 2) ? 2 : 1); // row
col = (modulePos / numMods[1]) * ((detType == EIGER) ? 2 : 1) +
i; // col for horiz. udp ports
listener[i]->SetHardCodedPosition(row, col);
}
@ -555,19 +555,14 @@ void Implementation::stopReceiver() {
std::this_thread::sleep_for(std::chrono::milliseconds(5));
}
// create virtual file
#ifdef HDF5C
if (fileWriteEnable && fileFormatType == HDF5) {
uint64_t maxIndexCaught = 0;
bool anycaught = false;
for (const auto &it : dataProcessor) {
maxIndexCaught = std::max(maxIndexCaught, it->GetProcessedIndex());
if (it->GetStartedFlag())
anycaught = true;
}
// to create virtual file & set files/acquisition to 0 (only hdf5 at the
// moment)
/*dataProcessor[0]->EndofAcquisition(anycaught, maxIndexCaught);*/
dataProcessor[0]->CreateVirtualFile(
filePath, fileName, fileIndex, overwriteEnable, silentMode,
modulePos, numThreads, framesPerFile, numberOfTotalFrames,
dynamicRange, numMods[X], numMods[Y]);
}
#endif
// wait for the processes (dataStreamer) to be done
running = true;
@ -672,20 +667,6 @@ void Implementation::shutDownUDPSockets() {
it->ShutDownUDPSocket();
}
void Implementation::closeFiles() {
uint64_t maxIndexCaught = 0;
bool anycaught = false;
for (const auto &it : dataProcessor) {
it->CloseFiles();
maxIndexCaught = std::max(maxIndexCaught, it->GetProcessedIndex());
if (it->GetStartedFlag())
anycaught = true;
}
// to create virtual file & set files/acquisition to 0 (only hdf5 at the
// moment)
/*dataProcessor[0]->EndofAcquisition(anycaught, maxIndexCaught);*/
}
void Implementation::restreamStop() {
for (const auto &it : dataStreamer)
it->RestreamStop();
@ -811,7 +792,8 @@ void Implementation::SetupWriter() {
}
} catch (const sls::RuntimeError &e) {
shutDownUDPSockets();
closeFiles();
for (const auto &it : dataProcessor)
it->CloseFiles();
throw sls::RuntimeError("Could not create first data file.");
}
}
@ -850,7 +832,7 @@ void Implementation::setNumberofUDPInterfaces(const int n) {
// reduce number of detectors in y dir (rows) if it had 2 interfaces
// before
if (numUDPInterfaces == 2)
numDet[Y] /= 2;
numMods[Y] /= 2;
numUDPInterfaces = n;
@ -898,15 +880,15 @@ void Implementation::setNumberofUDPInterfaces(const int n) {
if (dataStreamEnable) {
try {
int fd = flippedDataX;
int nd[2] = {numDet[0], numDet[1]};
int nm[2] = {numMods[0], numMods[1]};
if (quadEnable) {
fd = i;
nd[0] = 1;
nd[1] = 2;
nm[0] = 1;
nm[1] = 2;
}
dataStreamer.push_back(sls::make_unique<DataStreamer>(
i, fifo[i].get(), &dynamicRange, &roi, &fileIndex, fd,
(int *)nd, &quadEnable, &numberOfTotalFrames));
(int *)nm, &quadEnable, &numberOfTotalFrames));
dataStreamer[i]->SetGeneralData(generalData);
dataStreamer[i]->CreateZmqSockets(
&numThreads, streamingPort, streamingSrcIP,
@ -929,7 +911,7 @@ void Implementation::setNumberofUDPInterfaces(const int n) {
SetThreadPriorities();
// update (from 1 to 2 interface) & also for printout
setDetectorSize(numDet);
setDetectorSize(numMods);
// update row and column in dataprocessor
setModulePositionId(modulePos);
@ -1027,15 +1009,15 @@ void Implementation::setDataStreamEnable(const bool enable) {
for (int i = 0; i < numThreads; ++i) {
try {
int fd = flippedDataX;
int nd[2] = {numDet[0], numDet[1]};
int nm[2] = {numMods[0], numMods[1]};
if (quadEnable) {
fd = i;
nd[0] = 1;
nd[1] = 2;
nm[0] = 1;
nm[1] = 2;
}
dataStreamer.push_back(sls::make_unique<DataStreamer>(
i, fifo[i].get(), &dynamicRange, &roi, &fileIndex, fd,
(int *)nd, &quadEnable, &numberOfTotalFrames));
(int *)nm, &quadEnable, &numberOfTotalFrames));
dataStreamer[i]->SetGeneralData(generalData);
dataStreamer[i]->CreateZmqSockets(
&numThreads, streamingPort, streamingSrcIP,
@ -1488,13 +1470,13 @@ void Implementation::setQuad(const bool b) {
if (!quadEnable) {
for (const auto &it : dataStreamer) {
it->SetNumberofDetectors(numDet);
it->SetNumberofModules(numMods);
it->SetFlippedDataX(flippedDataX);
}
} else {
int size[2] = {1, 2};
for (const auto &it : dataStreamer) {
it->SetNumberofDetectors(size);
it->SetNumberofModules(size);
}
if (dataStreamer.size() == 2) {
dataStreamer[0]->SetFlippedDataX(0);

3
slsReceiverSoftware/src/Implementation.h
View File

@ -88,7 +88,6 @@ class Implementation : private virtual slsDetectorDefs {
void stopReceiver();
void startReadout();
void shutDownUDPSockets();
void closeFiles();
void restreamStop();
/**************************************************
@ -276,7 +275,7 @@ class Implementation : private virtual slsDetectorDefs {
// config parameters
int numThreads{1};
detectorType detType{GENERIC};
int numDet[MAX_DIMENSIONS] = {0, 0};
int numMods[MAX_DIMENSIONS] = {0, 0};
int modulePos{0};
std::string detHostname;
bool silentMode{false};