detectors/slsDetectorPackage
25
0
Fork 0
mirror of https://github.com/slsdetectorgroup/slsDetectorPackage.git synced 2025-04-26 00:00:02 +02:00
Code Issues Actions Packages Releases Activity

WIP

Browse Source
This commit is contained in:
maliakal_d 2020-04-30 14:04:50 +02:00
parent 3238ecfc8d
commit 7224ad989a
8 changed files with 241 additions and 401 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
slsReceiverSoftware/src/BinaryFile.cpp
View File

@ -18,7 +18,7 @@ BinaryFile::BinaryFile(int ind, uint32_t* maxf,
int* nd, std::string* fname, std::string* fpath, uint64_t* findex, bool* owenable,
int* dindex, int* nunits, uint64_t* nf, uint32_t* dr, uint32_t* portno,
bool* smode):
File(ind, maxf, nd, fname, fpath, findex, owenable, dindex, nunits, nf, dr, portno, smode),
File(ind, BINARY, maxf, nd, fname, fpath, findex, owenable, dindex, nunits, nf, dr, portno, smode),
filefd(nullptr),
numFramesInFile(0),
numActualPacketsInFile(0),
@ -39,10 +39,6 @@ void BinaryFile::PrintMembers(TLogLevel level) {
LOG(logINFO) << "Number of Frames in File: " << numFramesInFile;
}
slsDetectorDefs::fileFormat BinaryFile::GetFileType() {
return BINARY;
}
void BinaryFile::CreateFile() {
numFramesInFile = 0;
numActualPacketsInFile = 0;

1
slsReceiverSoftware/src/BinaryFile.h
View File

@ -60,7 +60,6 @@ class BinaryFile : private virtual slsDetectorDefs, public File {
private:
fileFormat GetFileType() override;
int WriteData(char* buf, int bsize);
FILE* filefd;

5
slsReceiverSoftware/src/DataProcessor.cpp
View File

@ -115,8 +115,9 @@ void DataProcessor::SetFileFormat(const fileFormat f) {
//remember the pointer values before they are destroyed
int nd[MAX_DIMENSIONS];nd[0] = 0; nd[1] = 0;
uint32_t* maxf = nullptr;
std::string* fname=nullptr; std::string* fpath=nullptr; uint64_t* findex=nullptr;
bool* owenable=nullptr; int* dindex=nullptr; int* nunits=nullptr; uint64_t* nf = nullptr;
std::string* fname=nullptr; std::string* fpath=nullptr;
uint64_t* findex=nullptr; bool* owenable=nullptr; int* dindex=nullptr;
int* nunits=nullptr; uint64_t* nf = nullptr;
uint32_t* dr = nullptr; uint32_t* port = nullptr;
file->GetMemberPointerValues(nd, maxf, fname, fpath, findex,
owenable, dindex, nunits, nf, dr, port);

15
slsReceiverSoftware/src/File.cpp
View File

@ -9,11 +9,12 @@
#include <iostream>
File::File(int ind, uint32_t* maxf,
int* nd, std::string* fname, std::string* fpath, uint64_t* findex, bool* owenable,
int* dindex, int* nunits, uint64_t* nf, uint32_t* dr, uint32_t* portno,
bool* smode):
File::File(int ind, slsDetectorDefs::fileFormat type, uint32_t* maxf,
int* nd, std::string* fname, std::string* fpath, uint64_t* findex,
bool* owenable, int* dindex, int* nunits, uint64_t* nf, uint32_t* dr,
uint32_t* portno, bool* smode):
index(ind),
formatType(type),
maxFramesPerFile(maxf),
numDetX(nd[0]),
numDetY(nd[1]),
@ -29,11 +30,15 @@ File::File(int ind, uint32_t* maxf,
silentMode(smode)
{
master = index?false:true;
master = ((index == 0) && (*detIndex == 0)) ? true : false;
}
File::~File() {}
slsDetectorDefs::fileFormat File::GetFileType() {
return formatType;
}
std::string File::GetCurrentFileName() {
return currentFileName;
}

85
slsReceiverSoftware/src/File.h
View File

@ -22,6 +22,7 @@ class File : private virtual slsDetectorDefs {
* Constructor
* creates the File Writer
* @param ind self index
* @param type file format type
* @param maxf pointer to max frames per file
* @param nd pointer to number of detectors in each dimension
* @param fname pointer to file name prefix
@ -35,35 +36,17 @@ class File : private virtual slsDetectorDefs {
* @param portno pointer to udp port number for logging
* @param smode pointer to silent mode
*/
File(int ind, uint32_t* maxf,
int* nd, std::string* fname, std::string* fpath, uint64_t* findex, bool* owenable,
int* dindex, int* nunits, uint64_t* nf, uint32_t* dr, uint32_t* portno,
bool* smode);
File(int ind, slsDetectorDefs::fileFormat type, uint32_t* maxf,
int* nd, std::string* fname, std::string* fpath, uint64_t* findex,
bool* owenable, int* dindex, int* nunits, uint64_t* nf, uint32_t* dr,
uint32_t* portno, bool* smode);
/**
* Destructor
*/
virtual ~File();
/**
* Get Current File Name
* @returns current file name
*/
fileFormat GetFileType();
std::string GetCurrentFileName();
void resetSubFileIndex();
/**
* Print all member values
*/
virtual void PrintMembers(TLogLevel level = logDEBUG1);
/**
* Get Type
* @return type
*/
virtual fileFormat GetFileType() = 0;
/**
* Get Member Pointer Values before the object is destroyed
* @param nd pointer to number of detectors in each dimension
@ -78,23 +61,12 @@ class File : private virtual slsDetectorDefs {
* @param dr pointer to dynamic range
* @param portno pointer to dynamic range
*/
void GetMemberPointerValues(int* nd, uint32_t*& maxf, std::string*& fname, std::string*& fpath,
uint64_t*& findex, bool*& owenable,
int*& dindex, int*& nunits, uint64_t*& nf, uint32_t*& dr, uint32_t*& portno);
void GetMemberPointerValues(int* nd, uint32_t*& maxf, std::string*& fname,
std::string*& fpath, uint64_t*& findex, bool*& owenable, int*& dindex,
int*& nunits, uint64_t*& nf, uint32_t*& dr, uint32_t*& portno);
/**
* Create file
*/
virtual void CreateFile() = 0;
/**
* Close Current File
*/
virtual void CloseCurrentFile() = 0;
/**
* Close Files
*/
virtual void CloseAllFiles() = 0;
/**
@ -123,7 +95,6 @@ class File : private virtual slsDetectorDefs {
"should be overloaded by a derived class";
}
/**
* End of Acquisition
* @param anyPacketsCaught true if any packets are caught, else false
@ -135,60 +106,24 @@ class File : private virtual slsDetectorDefs {
}
protected:
/** master file writer/reader */
bool master;
/** Self Index */
int index;
/** Maximum frames per file */
slsDetectorDefs::fileFormat formatType;
uint32_t* maxFramesPerFile;
/** Master File Name */
std::string masterFileName;
/** Current File Name */
std::string currentFileName;
/** Number of Detectors in X dimension */
int numDetX;
/** Number of Detectors in Y dimension */
int numDetY;
/** File Name Prefix */
std::string* fileNamePrefix;
/** File Path */
std::string* filePath;
/** File Index */
uint64_t* fileIndex;
/** Sub file index */
uint64_t subFileIndex{0};
/** Over write enable */
bool* overWriteEnable;
/** Detector Index */
int* detIndex;
/** Number of units per detector. Eg. Eiger has 2, others 1 */
int* numUnitsPerDetector;
/** Number of images in acquisition */
uint64_t* numImages;
/** Dynamic Range */
uint32_t* dynamicRange;
/** UDP Port Number for logging */
uint32_t* udpPortNumber;
/** Silent Mode */
bool* silentMode;
};

442
slsReceiverSoftware/src/HDF5File.cpp
View File

@ -13,9 +13,7 @@
#include <string.h>
H5File* HDF5File::masterfd = 0;
hid_t HDF5File::virtualfd = 0;
std::mutex HDF5File::hdf5Lib;
HDF5File::HDF5File(int ind, uint32_t* maxf,
@ -24,7 +22,9 @@ HDF5File::HDF5File(int ind, uint32_t* maxf,
uint32_t nx, uint32_t ny,
bool* smode):
File(ind, maxf, nd, fname, fpath, findex, owenable, dindex, nunits, nf, dr, portno, smode),
File(ind, HDF5, maxf, nd, fname, fpath, findex, owenable, dindex, nunits, nf, dr, portno, smode),
masterfd(0),
virtualfd(0),
filefd(0),
dataspace(0),
dataset(0),
@ -84,90 +84,65 @@ HDF5File::HDF5File(int ind, uint32_t* maxf,
parameterNames.push_back("packets caught bit mask");
StrType strdatatype(PredType::C_S1, sizeof(bitset_storage));
parameterDataTypes.push_back(strdatatype);
}
HDF5File::~HDF5File() {
CloseAllFiles();
}
void HDF5File::PrintMembers(TLogLevel level) {
File::PrintMembers();
UpdateDataTypeFromDr();
if (datatype == PredType::STD_U8LE) {
LOG(level) << "Data Type: 4 or 8";
} else if (datatype == PredType::STD_U16LE) {
LOG(level) << "Data Type: 16";
} else if (datatype == PredType::STD_U32LE) {
LOG(level) << "Data Type: 32";
} else {
LOG(logERROR) << "unknown data type";
}
}
void HDF5File::SetNumberofPixels(uint32_t nx, uint32_t ny) {
nPixelsX = nx;
nPixelsY = ny;
}
slsDetectorDefs::fileFormat HDF5File::GetFileType() {
return HDF5;
}
void HDF5File::UpdateDataTypeFromDr() {
switch(*dynamicRange){
case 16: datatype = PredType::STD_U16LE; break;
case 32: datatype = PredType::STD_U32LE; break;
default: datatype = PredType::STD_U8LE; break;
}
}
void HDF5File::CreateFile() {
numFilesinAcquisition++;
numFramesInFile = 0;
numActualPacketsInFile = 0;
std::ostringstream os;
os << *filePath << "/" << *fileNamePrefix << "_d"
<< (*detIndex * (*numUnitsPerDetector) + index) << "_f" << subFileIndex << '_'
<< *fileIndex << ".h5";
currentFileName = os.str();
//first time
if(subFileIndex == 0u)
UpdateDataTypeFromDr();
CreateDataFile();
if(!(*silentMode)) {
LOG(logINFO) << *udpPortNumber << ": HDF5 File created: " << currentFileName;
if(subFileIndex == 0u) {
switch(*dynamicRange){
case 16:
datatype = PredType::STD_U16LE;
break;
case 32:
datatype = PredType::STD_U32LE;
break;
default:
datatype = PredType::STD_U8LE;
break;
}
}
CreateDataFile();
}
void HDF5File::CloseCurrentFile() {
CloseFile(filefd, nullptr, false);
CloseFile(filefd, false, false);
for (unsigned int i = 0; i < dataset_para.size(); ++i)
delete dataset_para[i];
dataset_para.clear();
if(dataspace_para) {delete dataspace_para;dataspace_para=0;}
if(dataset) {delete dataset;dataset=0;}
if(dataspace) {delete dataspace;dataspace=0;}
if(filefd) {delete filefd;filefd=0;}
if(dataspace_para) {
delete dataspace_para;
dataspace_para = 0;
}
if(dataset) {
delete dataset;
dataset = 0;
}
if(dataspace) {
delete dataspace;
dataspace = 0;
}
}
void HDF5File::CloseAllFiles() {
numFilesinAcquisition = 0;
{
CloseFile(filefd, nullptr, false);
if (master && (*detIndex==0)) {
CloseFile(masterfd, nullptr, true);
CloseFile(nullptr, &virtualfd, false);
CloseFile(filefd, false, false);
if (master) {
CloseFile(masterfd, false, true);
CloseFile(nullptr, true, false);
}
}
for (unsigned int i = 0; i < dataset_para.size(); ++i)
@ -179,7 +154,6 @@ void HDF5File::CloseAllFiles() {
if(filefd) delete filefd;
}
void HDF5File::WriteToFile(char* buffer, int bufferSize, uint64_t currentFrameNumber, uint32_t numPacketsCaught) {
// check if maxframesperfile = 0 for infinite
@ -200,7 +174,6 @@ void HDF5File::WriteToFile(char* buffer, int bufferSize, uint64_t currentFrameNu
WriteParameterDatasets(currentFrameNumber, (sls_receiver_header*) (buffer));
}
void HDF5File::CreateMasterFile(bool masterFileWriteEnable, masterAttributes& masterFileAttributes) {
//beginning of every acquisition
@ -208,62 +181,63 @@ void HDF5File::CreateMasterFile(bool masterFileWriteEnable, masterAttributes& ma
numActualPacketsInFile = 0;
extNumImages = *numImages;
if (masterFileWriteEnable && master && (*detIndex==0)) {
if (masterFileWriteEnable && master) {
virtualfd = 0;
CreateMasterDataFile();
CreateMasterDataFile(masterFileAttributes);
}
}
void HDF5File::EndofAcquisition(bool anyPacketsCaught, uint64_t numImagesCaught) {
//not created before
if (!virtualfd && anyPacketsCaught) {
// called only by the one maser receiver
if (master && (*detIndex==0)) {
if (master && masterfd != nullptr) {
//only one file and one sub image (link current file in master)
if (((numFilesinAcquisition == 1) && (numDetY*numDetX) == 1)) {
LinkVirtualInMaster();
//dataset name
std::ostringstream oss;
oss << "/data";
if ((*numImages > 1)) {
oss << "_f" << std::setfill('0') << std::setw(12) << 0;
}
std::string dsetname = oss.str();
LinkVirtualInMaster(currentFileName, dsetname);
}
//create virutal file
else{
CreateVirtualFile(numImagesCaught);}
CreateVirtualDataFile(
// infinite images in 1 file, then maxfrperfile = numImagesCaught
((*maxFramesPerFile == 0) ?
numImagesCaught+1 : *maxFramesPerFile),
numImagesCaught + 1);
}
}
}
numFilesinAcquisition = 0;
}
// called only by the one maser receiver
void HDF5File::CreateVirtualFile(uint64_t numImagesCaught) {
HDF5FileStatic::CreateVirtualDataFile(
// infinite images in 1 file, then maxfrperfile = numImagesCaught
((*maxFramesPerFile == 0) ? numImagesCaught+1 : *maxFramesPerFile),
numImagesCaught + 1);
}
void HDF5File::CloseFile(H5File* fd, hid_t* cfd, bool master) {
std::lock_guard<std::mutex> lock(mutex);
void HDF5File::CloseFile(H5File* fd, bool virtualFile, bool masterFile) {
std::lock_guard<std::mutex> lock(HDF5File::hdf5Lib);
// c code due to only c implementation of H5Pset_virtual available
if (cfd != nullptr) {
if(*cfd) {
if (H5Fclose(*cfd) < 0 ) {
LOG(logERROR) << "Could not close virtual HDF5 handles";
}
*cfd = 0;
if (virtualFile) {
if (virtualfd != 0) {
if (H5Fclose(virtualfd) < 0 ) {
LOG(logERROR) << "Could not close virtual HDF5 handles";
}
virtualfd = 0;
}
} else {
Exception::dontPrint(); // to handle errors
try {
Exception::dontPrint(); // to handle errors
if (fd) {
delete fd;
fd = 0;
}
} catch(const Exception& error) {
LOG(logERROR) << "Could not close "
<< master ? "master" : "data"
<< (masterFile ? "master" : "data")
<< " HDF5 handles of index " << index;
error.printErrorStack();
}
@ -271,7 +245,7 @@ void HDF5File::CloseFile(H5File* fd, hid_t* cfd, bool master) {
}
void HDF5File::WriteDataFile(uint64_t currentFrameNumber, char* buffer) {
std::lock_guard<std::mutex> lock(mutex);
std::lock_guard<std::mutex> lock(HDF5File::hdf5Lib);
uint64_t nDimx = ((*maxFramesPerFile == 0) ?
currentFrameNumber : currentFrameNumber % (*maxFramesPerFile));
@ -292,14 +266,14 @@ void HDF5File::WriteDataFile(uint64_t currentFrameNumber, char* buffer) {
catch(const Exception& error){
LOG(logERROR) << "Could not write to file in object " << index;
error.printErrorStack();
throw RuntimeError("Could not write to file in object " +
throw sls::RuntimeError("Could not write to file in object " +
std::to_string(index));
}
}
void HDF5File::WriteParameterDatasets(uint64_t currentFrameNumber,
sls_receiver_header* rheader) {
std::lock_guard<std::mutex> lock(mutex);
std::lock_guard<std::mutex> lock(HDF5File::hdf5Lib);
uint64_t fnum = ((*maxFramesPerFile == 0) ?
currentFrameNumber : currentFrameNumber % (*maxFramesPerFile));
@ -360,13 +334,13 @@ void HDF5File::WriteParameterDatasets(uint64_t currentFrameNumber,
}
catch(const Exception& error){
error.printErrorStack();
throw RuntimeError("Could not write parameters (index:" +
throw sls::RuntimeError("Could not write parameters (index:" +
std::to_string(i) + ") to file in object " + std::to_string(index));
}
}
void HDF5File::ExtendDataset() {
std::lock_guard<std::mutex> lock(mutex);
std::lock_guard<std::mutex> lock(HDF5File::hdf5Lib);
try{
Exception::dontPrint(); //to handle errors
@ -385,12 +359,12 @@ void HDF5File::ExtendDataset() {
dataset_para[i]->extend(dims_para);
delete dataspace_para;
dataspace_para = 0;
dspace_dataspace_parapara = new DataSpace(dataset_para[0]->getSpace());
dataspace_para = new DataSpace(dataset_para[0]->getSpace());
}
catch(const Exception& error){
error.printErrorStack();
throw RuntimeError("Could not extend dataset in object " +
throw sls::RuntimeError("Could not extend dataset in object " +
std::to_string(index));
}
if (!(*silentMode)) {
@ -400,8 +374,103 @@ void HDF5File::ExtendDataset() {
extNumImages += *numImages;
}
void HDF5File::CreateMasterDataFile() {
std::lock_guard<std::mutex> lock(mutex);
void HDF5File::CreateDataFile() {
std::ostringstream os;
os << *filePath << "/" << *fileNamePrefix << "_d"
<< (*detIndex * (*numUnitsPerDetector) + index) << "_f" << subFileIndex << '_'
<< *fileIndex << ".h5";
currentFileName = os.str();
std::lock_guard<std::mutex> lock(HDF5File::hdf5Lib);
uint64_t framestosave = ((*maxFramesPerFile == 0) ? *numImages : // infinite images
(((extNumImages - subFileIndex) > (*maxFramesPerFile)) ? // save up to maximum at a time
(*maxFramesPerFile) : (extNumImages-subFileIndex)));
uint64_t nDimx = framestosave;
uint32_t nDimy = nPixelsY;
uint32_t nDimz = ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX);
try {
Exception::dontPrint(); //to handle errors
//file
FileAccPropList fapl;
fapl.setFcloseDegree(H5F_CLOSE_STRONG);
filefd = 0;
if(!(*overWriteEnable))
filefd = new H5File( currentFileName.c_str(), H5F_ACC_EXCL,
FileCreatPropList::DEFAULT,
fapl );
else
filefd = new H5File( currentFileName.c_str(), H5F_ACC_TRUNC,
FileCreatPropList::DEFAULT,
fapl );
//attributes - version
double dValue = HDF5_WRITER_VERSION;
DataSpace dataspace_attr = DataSpace (H5S_SCALAR);
Attribute attribute = filefd->createAttribute("version",
PredType::NATIVE_DOUBLE, dataspace_attr);
attribute.write(PredType::NATIVE_DOUBLE, &dValue);
//dataspace
hsize_t srcdims[3] = {nDimx, nDimy, nDimz};
hsize_t srcdimsmax[3] = {H5S_UNLIMITED, nDimy, nDimz};
dataspace = 0;
dataspace = new DataSpace (3,srcdims,srcdimsmax);
//dataset name
std::ostringstream osfn;
osfn << "/data";
if (*numImages > 1)
osfn << "_f" << std::setfill('0') << std::setw(12) << subFileIndex;
std::string dsetname = osfn.str();
//dataset
//fill value
DSetCreatPropList plist;
int fill_value = -1;
plist.setFillValue(datatype, &fill_value);
// always create chunked dataset as unlimited is only
// supported with chunked layout
hsize_t chunk_dims[3] = {MAX_CHUNKED_IMAGES, nDimy, nDimz};
plist.setChunk(3, chunk_dims);
dataset = 0;
dataset = new DataSet (filefd->createDataSet(dsetname.c_str(), datatype,
*dataspace, plist));
//create parameter datasets
hsize_t dims[1] = {nDimx};
hsize_t dimsmax[1] = {H5S_UNLIMITED};
dataspace_para = 0;
dataspace_para = new DataSpace (1,dims,dimsmax);
// always create chunked dataset as unlimited is only
// supported with chunked layout
DSetCreatPropList paralist;
hsize_t chunkpara_dims[3] = {MAX_CHUNKED_IMAGES};
paralist.setChunk(1, chunkpara_dims);
for (unsigned int i = 0; i < parameterNames.size(); ++i){
DataSet* ds = new DataSet(filefd->createDataSet(parameterNames[i],
parameterDataTypes[i], *dataspace_para, paralist));
dataset_para.push_back(ds);
}
}
catch(const Exception& error){
error.printErrorStack();
if (filefd) filefd->close();
throw sls::RuntimeError("Could not create HDF5 handles in object " + index);
}
if(!(*silentMode)) {
LOG(logINFO) << *udpPortNumber << ": HDF5 File created: " << currentFileName;
}
}
void HDF5File::CreateMasterDataFile(masterAttributes& masterFileAttributes) {
std::ostringstream os;
os << *filePath << "/" << *fileNamePrefix << "_master"
@ -413,6 +482,8 @@ void HDF5File::CreateMasterDataFile() {
}
masterFileAttributes.version = HDF5_WRITER_VERSION;
std::lock_guard<std::mutex> lock(HDF5File::hdf5Lib);
try {
Exception::dontPrint(); //to handle errors
@ -454,7 +525,7 @@ void HDF5File::CreateMasterDataFile() {
dataset = group5.createDataSet ( "dynamic range", PredType::NATIVE_INT, dataspace );
dataset.write ( &(masterFileAttributes.dynamicRange), PredType::NATIVE_INT);
attribute = dataset.createAttribute("unit",strdatatype, dataspace);
attribute.write(strdatatype, std::string("bits"));
attribute.write(strdatatype, std::string("bits").c_str());
//Ten Giga
iValue = masterFileAttributes.tenGiga;
@ -549,98 +620,11 @@ void HDF5File::CreateMasterDataFile() {
} catch(const Exception& error) {
error.printErrorStack();
if (masterfd) masterfd->close();
throw RuntimeError("Could not create master HDF5 handles");
}
}
void HDF5File::CreateDataFile() {
std::lock_guard<std::mutex> lock(mutex);
uint64_t framestosave = ((*maxFramesPerFile == 0) ? *numImages : // infinite images
(((extNumImages - subFileIndex) > (*maxFramesPerFile)) ? // save up to maximum at a time
(*maxFramesPerFile) : (extNumImages-subFileIndex)));
uint64_t nDimx = framestosave;
uint32_t nDimy = nPixelsY;
uint32_t nDimz = ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX);
try {
Exception::dontPrint(); //to handle errors
//file
FileAccPropList fapl;
fapl.setFcloseDegree(H5F_CLOSE_STRONG);
filefd = 0;
if(!(*overWriteEnable))
filefd = new H5File( currentFileName.c_str(), H5F_ACC_EXCL,
FileCreatPropList::DEFAULT,
fapl );
else
filefd = new H5File( currentFileName.c_str(), H5F_ACC_TRUNC,
FileCreatPropList::DEFAULT,
fapl );
//attributes - version
double dValue = HDF5_WRITER_VERSION;
DataSpace dataspace_attr = DataSpace (H5S_SCALAR);
Attribute attribute = filefd->createAttribute("version",
PredType::NATIVE_DOUBLE, dataspace_attr);
attribute.write(PredType::NATIVE_DOUBLE, &dValue);
//dataspace
hsize_t srcdims[3] = {nDimx, nDimy, nDimz};
hsize_t srcdimsmax[3] = {H5S_UNLIMITED, nDimy, nDimz};
dataspace = 0;
dataspace = new DataSpace (3,srcdims,srcdimsmax);
//dataset name
std::ostringstream osfn;
osfn << "/data";
if (*numImages > 1)
osfn << "_f" << std::setfill('0') << std::setw(12) << subFileIndex;
std::string dsetname = osfn.str();
//dataset
//fill value
DSetCreatPropList plist;
int fill_value = -1;
plist.setFillValue(datatype, &fill_value);
// always create chunked dataset as unlimited is only
// supported with chunked layout
hsize_t chunk_dims[3] = {MAX_CHUNKED_IMAGES, nDimy, nDimz};
plist.setChunk(3, chunk_dims);
dataset = 0;
dataset = new DataSet (filefd->createDataSet(dsetname.c_str(), datatype,
*dataspace, plist));
//create parameter datasets
hsize_t dims[1] = {nDimx};
hsize_t dimsmax[1] = {H5S_UNLIMITED};
dataspace_para = 0;
dataspace_para = new DataSpace (1,dims,dimsmax);
// always create chunked dataset as unlimited is only
// supported with chunked layout
DSetCreatPropList paralist;
hsize_t chunkpara_dims[3] = {MAX_CHUNKED_IMAGES};
paralist.setChunk(1, chunkpara_dims);
for (unsigned int i = 0; i < parameterNames.size(); ++i){
DataSet* ds = new DataSet(filefd->createDataSet(parameterNames[i],
parameterDataTypes[i], *dataspace_para, paralist));
dataset_para.push_back(ds);
}
}
catch(const Exception& error){
error.printErrorStack();
if (filefd) filefd->close();
throw RuntimeError("Could not create HDF5 handles in object " + index);
throw sls::RuntimeError("Could not create master HDF5 handles");
}
}
void HDF5File::CreateVirtualDataFile(uint32_t maxFramesPerFile, uint64_t numf) {
std::lock_guard<std::mutex> lock(mutex);
std::ostringstream osfn;
osfn << *filePath << "/" << *fileNamePrefix;
@ -657,56 +641,58 @@ void HDF5File::CreateVirtualDataFile(uint32_t maxFramesPerFile, uint64_t numf) {
uint32_t nDimy = nPixelsY;
uint32_t nDimz = ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX);
std::lock_guard<std::mutex> lock(HDF5File::hdf5Lib);
try {
//file
hid_t dfal = H5Pcreate (H5P_FILE_ACCESS);
if (dfal < 0)
throw RuntimeError("Could not create file access property for virtual file " + vname);
throw sls::RuntimeError("Could not create file access property for virtual file " + vname);
if (H5Pset_fclose_degree (dfal, H5F_CLOSE_STRONG) < 0)
throw RuntimeError("Could not set strong file close degree for virtual file " + vname);
throw sls::RuntimeError("Could not set strong file close degree for virtual file " + vname);
virtualfd = H5Fcreate( vname.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, dfal);
if (virtualfd < 0)
throw RuntimeError("Could not create virtual file " + vname);
throw sls::RuntimeError("Could not create virtual file " + vname);
//attributes - version
hid_t dataspace_attr = H5Screate (H5S_SCALAR);
if (dataspace_attr < 0)
throw RuntimeError("Could not create dataspace for attribute in virtual file " + vname);
throw sls::RuntimeError("Could not create dataspace for attribute in virtual file " + vname);
hid_t attrid = H5Acreate2 (virtualfd, "version", H5T_NATIVE_DOUBLE, dataspace_attr, H5P_DEFAULT, H5P_DEFAULT);
if (attrid < 0)
throw RuntimeError("Could not create attribute in virtual file " + vname);
throw sls::RuntimeError("Could not create attribute in virtual file " + vname);
double attr_data = HDF5_WRITER_VERSION;
if (H5Awrite (attrid, H5T_NATIVE_DOUBLE, &attr_data) < 0)
throw RuntimeError("Could not write attribute in virtual file " + vname);
throw sls::RuntimeError("Could not write attribute in virtual file " + vname);
if (H5Aclose (attrid) < 0)
throw RuntimeError("Could not close attribute in virtual file " + vname);
throw sls::RuntimeError("Could not close attribute in virtual file " + vname);
//virtual dataspace
hsize_t vdsdims[3] = {numf, numDetY * nDimy, numDetz * nDimz};
hid_t vdsDataspace = H5Screate_simple(3, vdsdims ,NULL);
if (vdsDataspace < 0)
throw RuntimeError("Could not create virtual dataspace in virtual file " + vname);
throw sls::RuntimeError("Could not create virtual dataspace in virtual file " + vname);
hsize_t vdsdims_para[2] = {numf, (unsigned int) numDetY * numDetz};
hid_t vdsDataspace_para = H5Screate_simple(2, vdsdims_para, NULL);
if (vdsDataspace_para < 0)
throw RuntimeError("Could not create virtual dataspace (parameters) in virtual file " + vname);
throw sls::RuntimeError("Could not create virtual dataspace (parameters) in virtual file " + vname);
//fill values
hid_t dcpl = H5Pcreate (H5P_DATASET_CREATE);
if (dcpl < 0)
throw RuntimeError("Could not create file creation properties in virtual file " + vname);
throw sls::RuntimeError("Could not create file creation properties in virtual file " + vname);
int fill_value = -1;
if (H5Pset_fill_value (dcpl, GetDataTypeinC(datatype), &fill_value) < 0)
throw RuntimeError("Could not create fill value in virtual file " + vname);
throw sls::RuntimeError("Could not create fill value in virtual file " + vname);
hid_t dcpl_para[parameterNames.size()];
for (unsigned int i = 0; i < parameterNames.size(); ++i) {
dcpl_para[i] = H5Pcreate (H5P_DATASET_CREATE);
if (dcpl_para[i] < 0)
throw RuntimeError("Could not create file creation properties (parameters) in virtual file " + vname);
throw sls::RuntimeError("Could not create file creation properties (parameters) in virtual file " + vname);
if (H5Pset_fill_value (dcpl_para[i], GetDataTypeinC(parameterDataTypes[i]), &fill_value) < 0)
throw RuntimeError("Could not create fill value (parameters) in virtual file " + vname);
throw sls::RuntimeError("Could not create fill value (parameters) in virtual file " + vname);
}
//hyperslab
@ -726,11 +712,11 @@ void HDF5File::CreateVirtualDataFile(uint32_t maxFramesPerFile, uint64_t numf) {
//setect hyperslabs
if (H5Sselect_hyperslab (vdsDataspace, H5S_SELECT_SET, offset, NULL, count, NULL) < 0) {
throw RuntimeError("Could not select hyperslab");
throw sls::RuntimeError("Could not select hyperslab");
}
if (H5Sselect_hyperslab (vdsDataspace_para, H5S_SELECT_SET,
offset_para, NULL, count_para, NULL) < 0) {
throw RuntimeError("Could not select hyperslab for parameters");
throw sls::RuntimeError("Could not select hyperslab for parameters");
}
//source file name
@ -761,23 +747,23 @@ void HDF5File::CreateVirtualDataFile(uint32_t maxFramesPerFile, uint64_t numf) {
hsize_t srcdimsmax[3] = {H5S_UNLIMITED, nDimy, nDimz};
hid_t srcDataspace = H5Screate_simple(3, srcdims, srcdimsmax);
if (srcDataspace < 0)
throw RuntimeError("Could not create source dataspace in virtual file " + vname);
throw sls::RuntimeError("Could not create source dataspace in virtual file " + vname);
hsize_t srcdims_para[1] = {nDimx};
hsize_t srcdimsmax_para[1] = {H5S_UNLIMITED};
hid_t srcDataspace_para = H5Screate_simple(1, srcdims_para, srcdimsmax_para);
if (srcDataspace_para < 0)
throw RuntimeError("Could not create source dataspace (parameters) in virtual file " + vname);
throw sls::RuntimeError("Could not create source dataspace (parameters) in virtual file " + vname);
//mapping
if (H5Pset_virtual(dcpl, vdsDataspace, relative_srcFileName.c_str(),
srcDatasetName.c_str(), srcDataspace) < 0) {
throw RuntimeError("Could not set mapping for paramter 1");
throw sls::RuntimeError("Could not set mapping for paramter 1");
}
for (unsigned int k = 0; k < parameterNames.size(); ++k) {
if (H5Pset_virtual(dcpl_para[k], vdsDataspace_para, relative_srcFileName.c_str(),
parameterNames[k], srcDataspace_para) < 0) {
throw RuntimeError("Could not set mapping for paramter " + std::to_string(k));
throw sls::RuntimeError("Could not set mapping for paramter " + std::to_string(k));
}
}
@ -798,7 +784,7 @@ void HDF5File::CreateVirtualDataFile(uint32_t maxFramesPerFile, uint64_t numf) {
hid_t vdsdataset = H5Dcreate2 (virtualfd, virtualDatasetName.c_str(),
GetDataTypeinC(datatype), vdsDataspace, H5P_DEFAULT, dcpl, H5P_DEFAULT);
if (vdsdataset < 0)
throw RuntimeError("Could not create virutal dataset in virtual file " + vname);
throw sls::RuntimeError("Could not create virutal dataset in virtual file " + vname);
//virtual parameter dataset
@ -808,7 +794,7 @@ void HDF5File::CreateVirtualDataFile(uint32_t maxFramesPerFile, uint64_t numf) {
GetDataTypeinC(parameterDataTypes[i]), vdsDataspace_para,
H5P_DEFAULT, dcpl_para[i], H5P_DEFAULT);
if (vdsdataset_para < 0)
throw RuntimeError("Could not create virutal dataset (parameters) in virtual file " + vname);
throw sls::RuntimeError("Could not create virutal dataset (parameters) in virtual file " + vname);
}
//close
@ -816,22 +802,16 @@ void HDF5File::CreateVirtualDataFile(uint32_t maxFramesPerFile, uint64_t numf) {
virtualfd = 0;
//link
LinkVirtualInMaster(masterFileName, vname, virtualDatasetName, parameterNames);
} catch (const RuntimeError &e) {
LinkVirtualInMaster(vname, virtualDatasetName);
} catch (const sls::RuntimeError &e) {
if (virtualfd > 0)
H5Fclose(virtualfd);
virtualfd = 0;
}
}
void HDF5File::LinkVirtualInMaster() {
std::lock_guard<std::mutex> lock(mutex);
//dataset name
std::ostringstream osfn;
osfn << "/data";
if ((*numImages > 1)) osfn << "_f" << std::setfill('0') << std::setw(12) << 0;
std::string dsetname = osfn.str();
void HDF5File::LinkVirtualInMaster(std::string fname, std::string dsetname) {
std::lock_guard<std::mutex> lock(HDF5File::hdf5Lib);
char linkname[100];
hid_t vfd = 0;
@ -839,41 +819,41 @@ void HDF5File::LinkVirtualInMaster() {
try {
hid_t dfal = H5Pcreate (H5P_FILE_ACCESS);
if (dfal < 0)
throw RuntimeError("Could not create file access property for link");
throw sls::RuntimeError("Could not create file access property for link");
if (H5Pset_fclose_degree (dfal, H5F_CLOSE_STRONG) < 0)
throw RuntimeError("Could not set strong file close degree for link");
throw sls::RuntimeError("Could not set strong file close degree for link");
//open master file
hid_t mfd = H5Fopen( masterFileName.c_str(), H5F_ACC_RDWR, dfal);
if (mfd < 0)
throw RuntimeError("Could not open master file");
throw sls::RuntimeError("Could not open master file");
//open virtual file
vfd = H5Fopen( currentFileName.c_str(), H5F_ACC_RDWR, dfal);
vfd = H5Fopen( fname.c_str(), H5F_ACC_RDWR, dfal);
if (vfd < 0) {
H5Fclose(mfd); mfd = 0;
throw RuntimeError("Could not open virtual file");
throw sls::RuntimeError("Could not open virtual file");
}
// find relative path
std::string relative_virtualfname = currentFileName;
std::string relative_virtualfname = fname;
{
size_t i = currentFileName.rfind('/', currentFileName.length());
size_t i = fname.rfind('/', fname.length());
if (i != std::string::npos)
relative_virtualfname = (currentFileName.substr(i+1, currentFileName.length() - i));
relative_virtualfname = (fname.substr(i+1, fname.length() - i));
}
//**data dataset**
hid_t vdset = H5Dopen2( vfd, dsetname.c_str(), H5P_DEFAULT);
if (vdset < 0) {
H5Fclose(mfd);
throw RuntimeError("Could not open virtual data dataset");
throw sls::RuntimeError("Could not open virtual data dataset");
}
sprintf(linkname, "/entry/data/%s",dsetname.c_str());
if(H5Lcreate_external( relative_virtualfname.c_str(), dsetname.c_str(),
mfd, linkname, H5P_DEFAULT, H5P_DEFAULT) < 0) {
H5Fclose(mfd); mfd = 0;
throw RuntimeError("Could not create link to data dataset");
throw sls::RuntimeError("Could not create link to data dataset");
}
H5Dclose(vdset);
@ -882,20 +862,20 @@ void HDF5File::LinkVirtualInMaster() {
hid_t vdset_para = H5Dopen2( vfd, (std::string (parameterNames[i])).c_str(), H5P_DEFAULT);
if (vdset_para < 0) {
H5Fclose(mfd); mfd = 0;
throw RuntimeError("Could not open virtual parameter dataset to create link");
throw sls::RuntimeError("Could not open virtual parameter dataset to create link");
}
sprintf(linkname, "/entry/data/%s",(std::string (parameterNames[i])).c_str());
if(H5Lcreate_external( relative_virtualfname.c_str(), (std::string (parameterNames[i])).c_str(),
mfd, linkname, H5P_DEFAULT, H5P_DEFAULT) < 0) {
H5Fclose(mfd); mfd = 0;
throw RuntimeError("Could not create link to virtual parameter dataset");
throw sls::RuntimeError("Could not create link to virtual parameter dataset");
}
}
H5Fclose(mfd); mfd = 0;
H5Fclose(vfd); vfd = 0;
} catch () {
} catch (...) {
if(vfd > 0)
H5Fclose(vfd);
vfd = 0;

18
slsReceiverSoftware/src/HDF5File.h
View File

@ -47,7 +47,6 @@ class HDF5File : private virtual slsDetectorDefs, public File {
uint32_t nx, uint32_t ny,
bool* smode);
~HDF5File();
void PrintMembers(TLogLevel level = logDEBUG1);
void SetNumberofPixels(uint32_t nx, uint32_t ny);
void CreateFile();
void CloseCurrentFile();
@ -56,29 +55,24 @@ class HDF5File : private virtual slsDetectorDefs, public File {
void CreateMasterFile(bool masterFileWriteEnable, masterAttributes& masterFileAttributes);
void EndofAcquisition(bool anyPacketsCaught, uint64_t numImagesCaught);
private:
void CreateVirtualFile(uint64_t numImagesCaught);
fileFormat GetFileType();
void UpdateDataTypeFromDr();
void CloseFile(H5File* fd, hid_t* cfd, bool master);
void CloseFile(H5File* fd, bool virtualFile, bool masterFile);
void WriteDataFile(uint64_t currentFrameNumber, char* buffer);
void WriteParameterDatasets(uint64_t currentFrameNumber, sls_receiver_header* rheader);
void ExtendDataset();
void CreateMasterDataFile();
void CreateDataFile();
void CreateMasterDataFile(masterAttributes& masterFileAttributes);
void CreateVirtualDataFile(uint32_t maxFramesPerFile, uint64_t numf);
void LinkVirtualInMaster();
void LinkVirtualInMaster(std::string fname, std::string dsetname);
hid_t GetDataTypeinC(DataType dtype);
static std::mutex hdf5Lib;
static mutable std::mutex mutex;
static H5File* masterfd;
H5File* masterfd;
/** Virtual File handle ( only file name because
code in C as H5Pset_virtual doesnt exist yet in C++) */
static hid_t virtualfd;
hid_t virtualfd;
H5File* filefd;
DataSpace* dataspace;
DataSet* dataset;

70
slsReceiverSoftware/src/HDF5FileStatic.h
View File

@ -1,70 +0,0 @@
#ifdef HDF5C
#pragma once
/************************************************
* @file HDF5FileStatic.h
* @short creating, closing, writing and reading
* from HDF5 files
***********************************************/
/**
*@short creating, closing, writing and reading from HDF5 files
*/
#include "H5Cpp.h"
#ifndef H5_NO_NAMESPACE
using namespace H5;
#endif
#include "sls_detector_defs.h"
#include "logger.h"
#include <iostream>
#include <string>
#include <vector>
#include <iomanip>
#include <stdlib.h> //malloc
#include <sstream>
#include <cstring> //memset
class HDF5FileStatic: public virtual slsDetectorDefs {
public:
/**
* Create virtual file
* (in C because H5Pset_virtual doesnt exist yet in C++)
* @param virtualFileName virtual file name
* @param fd virtual file handle
* @param masterFileName master file name
* @param fpath file path
* @param fnameprefix file name prefix
* @param findex file index
* @param frindexenable frame index enable
* @param dindex readout index
* @param numunits number of units per readout. eg. eiger has 2 udp units per readout
* @param maxFramesPerFile maximum frames per file
* @param numf number of frames caught
* @param srcDataseName source dataset name
* @param dataType datatype of data dataset
* @param numDety number of readouts in Y dir
* @param numDetz number of readouts in Z dir
* @param nDimy number of objects in y dimension in source file (Number of pixels in y dir)
* @param nDimz number of objects in z dimension in source file (Number of pixels in x dir)
* @param version version of software for hdf5 writing
* @param parameterNames parameter names
* @param parameterDataTypes parameter datatypes
*/
};
#endif