detectors/slsDetectorPackage
25
0
Fork 0
mirror of https://github.com/slsdetectorgroup/slsDetectorPackage.git synced 2025-04-22 22:40:02 +02:00
Code Issues Actions Packages Releases Activity

WIP

Browse Source
This commit is contained in:
maliakal_d 2020-04-29 18:02:57 +02:00
parent 395d7ba98a
commit 3238ecfc8d
6 changed files with 829 additions and 1206 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

99
slsReceiverSoftware/src/BinaryFile.cpp
View File

@ -10,6 +10,7 @@
#include <iostream>
#include <iomanip>
#include <string.h>
FILE* BinaryFile::masterfd = nullptr;
@ -106,13 +107,13 @@ void BinaryFile::WriteToFile(char* buffer, int buffersize, uint64_t fnum, uint32
// contiguous bitset
if (sizeof(sls_bitset) == sizeof(bitset_storage)) {
ret = BinaryFileStatic::WriteData(buffer, buffersize);
ret = WriteData(buffer, buffersize);
}
// not contiguous bitset
else {
// write detector header
ret = BinaryFileStatic::WriteData(buffer, sizeof(sls_detector_header));
ret = WriteData(buffer, sizeof(sls_detector_header));
// get contiguous representation of bit mask
bitset_storage storage;
@ -121,10 +122,10 @@ void BinaryFile::WriteToFile(char* buffer, int buffersize, uint64_t fnum, uint32
for (int i = 0; i < MAX_NUM_PACKETS; ++i)
storage[i >> 3] |= (bits[i] << (i & 7));
// write bitmask
ret += BinaryFileStatic::WriteData((char*)storage, sizeof(bitset_storage));
ret += WriteData((char*)storage, sizeof(bitset_storage));
// write data
ret += BinaryFileStatic::WriteData(buffer + sizeof(sls_detector_header),
ret += WriteData(buffer + sizeof(sls_detector_header),
buffersize - sizeof(sls_receiver_header));
}
@ -150,8 +151,94 @@ void BinaryFile::CreateMasterFile(bool mfwenable, masterAttributes& attr) {
LOG(logINFO) << "Master File: " << masterFileName;
}
attr.version = BINARY_WRITER_VERSION;
BinaryFileStatic::CreateMasterDataFile(masterfd, masterFileName,
*overWriteEnable, attr);
// create master file
if (!(*overWriteEnable)){
if (NULL == (masterfd = fopen((const char *) masterFileName.c_str(), "wx"))) {
masterfd = 0;
throw sls::RuntimeError("Could not create binary master file "
"(without overwrite enable) " + masterFileName);
}
}else if (NULL == (masterfd = fopen((const char *) masterFileName.c_str(), "w"))) {
masterfd = 0;
throw sls::RuntimeError("Could not create binary master file "
"(with overwrite enable) " + masterFileName);
}
// create master file data
time_t t = time(0);
char message[maxMasterFileSize];
sprintf(message,
"Version : %.1f\n"
"Detector Type : %d\n"
"Dynamic Range : %d\n"
"Ten Giga : %d\n"
"Image Size : %d bytes\n"
"nPixelsX : %d pixels\n"
"nPixelsY : %d pixels\n"
"Max Frames Per File : %u\n"
"Total Frames : %lld\n"
"Exptime (ns) : %lld\n"
"SubExptime (ns) : %lld\n"
"SubPeriod(ns) : %lld\n"
"Period (ns) : %lld\n"
"Quad Enable : %d\n"
"Analog Flag : %d\n"
"Digital Flag : %d\n"
"ADC Mask : %d\n"
"Dbit Offset : %d\n"
"Dbit Bitset : %lld\n"
"Roi (xmin, xmax) : %d %d\n"
"Timestamp : %s\n\n"
"#Frame Header\n"
"Frame Number : 8 bytes\n"
"SubFrame Number/ExpLength : 4 bytes\n"
"Packet Number : 4 bytes\n"
"Bunch ID : 8 bytes\n"
"Timestamp : 8 bytes\n"
"Module Id : 2 bytes\n"
"Row : 2 bytes\n"
"Column : 2 bytes\n"
"Reserved : 2 bytes\n"
"Debug : 4 bytes\n"
"Round Robin Number : 2 bytes\n"
"Detector Type : 1 byte\n"
"Header Version : 1 byte\n"
"Packets Caught Mask : 64 bytes\n"
,
attr.version,
attr.detectorType,
attr.dynamicRange,
attr.tenGiga,
attr.imageSize,
attr.nPixelsX,
attr.nPixelsY,
attr.maxFramesPerFile,
(long long int)attr.totalFrames,
(long long int)attr.exptimeNs,
(long long int)attr.subExptimeNs,
(long long int)attr.subPeriodNs,
(long long int)attr.periodNs,
attr.quadEnable,
attr.analogFlag,
attr.digitalFlag,
attr.adcmask,
attr.dbitoffset,
(long long int)attr.dbitlist,
attr.roiXmin,
attr.roiXmax,
ctime(&t));
if (strlen(message) > maxMasterFileSize) {
throw sls::RuntimeError("Master File Size " + std::to_string(strlen(message)) +
" is greater than max str size " + std::to_string(maxMasterFileSize));
}
// write and close file
if (fwrite((void*)message, 1, strlen(message), masterfd) != strlen(message)) {
throw sls::RuntimeError("Master binary file incorrect number of bytes written to file");
}
if (masterfd)
fclose(masterfd);
masterfd = 0;
}
}

4
slsReceiverSoftware/src/BinaryFile.h
View File

@ -13,7 +13,7 @@
#include <string>
class BinaryFile : private virtual slsDetectorDefs, public File, public BinaryFileStatic {
class BinaryFile : private virtual slsDetectorDefs, public File {
public:
/**
@ -67,7 +67,7 @@ class BinaryFile : private virtual slsDetectorDefs, public File, public BinaryFi
static FILE* masterfd;
uint32_t numFramesInFile;
uint64_t numActualPacketsInFile;
const int maxMasterFileSize;
const size_t maxMasterFileSize;
};

122
slsReceiverSoftware/src/BinaryFileStatic.h
View File

@ -1,122 +0,0 @@
#pragma once
/************************************************
* @file BinaryFileStatic.h
* @short creating, closing, writing and reading
* from binary files
***********************************************/
/**
*@short creating, closing, writing and reading from binary files
*/
#include "logger.h"
#include <string>
#include <iomanip>
#include <string.h>
class BinaryFileStatic {
public:
/**
* Create master files
* @param fd pointer to file handle
* @param fname master file name
* @param owenable overwrite enable
* @param attr master file attributes
*/
static void CreateMasterDataFile(FILE*& fd, std::string fname, bool owenable,
masterAttributes& attr)
{
if(!owenable){
if (NULL == (fd = fopen((const char *) fname.c_str(), "wx"))){
fd = 0;
throw sls::RuntimeError("Could not create binary master file "
"(without overwrite enable) " + fname);
}
}else if (NULL == (fd = fopen((const char *) fname.c_str(), "w"))){
fd = 0;
throw sls::RuntimeError("Could not create binary master file "
"(with overwrite enable) " + fname);
}
time_t t = time(0);
char message[MAX_MASTER_FILE_LENGTH];
sprintf(message,
"Version : %.1f\n"
"Detector Type : %d\n"
"Dynamic Range : %d\n"
"Ten Giga : %d\n"
"Image Size : %d bytes\n"
"nPixelsX : %d pixels\n"
"nPixelsY : %d pixels\n"
"Max Frames Per File : %u\n"
"Total Frames : %lld\n"
"Exptime (ns) : %lld\n"
"SubExptime (ns) : %lld\n"
"SubPeriod(ns) : %lld\n"
"Period (ns) : %lld\n"
"Quad Enable : %d\n"
"Analog Flag : %d\n"
"Digital Flag : %d\n"
"ADC Mask : %d\n"
"Dbit Offset : %d\n"
"Dbit Bitset : %lld\n"
"Roi (xmin, xmax) : %d %d\n"
"Timestamp : %s\n\n"
"#Frame Header\n"
"Frame Number : 8 bytes\n"
"SubFrame Number/ExpLength : 4 bytes\n"
"Packet Number : 4 bytes\n"
"Bunch ID : 8 bytes\n"
"Timestamp : 8 bytes\n"
"Module Id : 2 bytes\n"
"Row : 2 bytes\n"
"Column : 2 bytes\n"
"Reserved : 2 bytes\n"
"Debug : 4 bytes\n"
"Round Robin Number : 2 bytes\n"
"Detector Type : 1 byte\n"
"Header Version : 1 byte\n"
"Packets Caught Mask : 64 bytes\n"
,
attr.version,
attr.detectorType,
attr.dynamicRange,
attr.tenGiga,
attr.imageSize,
attr.nPixelsX,
attr.nPixelsY,
attr.maxFramesPerFile,
(long long int)attr.totalFrames,
(long long int)attr.exptimeNs,
(long long int)attr.subExptimeNs,
(long long int)attr.subPeriodNs,
(long long int)attr.periodNs,
attr.quadEnable,
attr.analogFlag,
attr.digitalFlag,
attr.adcmask,
attr.dbitoffset,
(long long int)attr.dbitlist,
attr.roiXmin,
attr.roiXmax,
ctime(&t));
if (strlen(message) > MAX_MASTER_FILE_LENGTH) {
throw sls::RuntimeError("Master File Size " + std::to_string(strlen(message)) +
" is greater than max str size " + std::to_string(MAX_MASTER_FILE_LENGTH));
}
if (fwrite((void*)message, 1, strlen(message), fd) != strlen(message)) {
throw sls::RuntimeError("Master binary file incorrect number of bytes written to file");
}
BinaryFileStatic::CloseDataFile(fd);
}

773
slsReceiverSoftware/src/HDF5File.cpp
View File

@ -94,7 +94,7 @@ HDF5File::~HDF5File() {
void HDF5File::PrintMembers(TLogLevel level) {
File::PrintMembers();
UpdateDataType();
UpdateDataTypeFromDr();
if (datatype == PredType::STD_U8LE) {
LOG(level) << "Data Type: 4 or 8";
} else if (datatype == PredType::STD_U16LE) {
@ -118,7 +118,7 @@ slsDetectorDefs::fileFormat HDF5File::GetFileType() {
}
void HDF5File::UpdateDataType() {
void HDF5File::UpdateDataTypeFromDr() {
switch(*dynamicRange){
case 16: datatype = PredType::STD_U16LE; break;
case 32: datatype = PredType::STD_U32LE; break;
@ -131,36 +131,26 @@ void HDF5File::CreateFile() {
numFilesinAcquisition++;
numFramesInFile = 0;
numActualPacketsInFile = 0;
currentFileName = HDF5FileStatic::CreateFileName(*filePath, *fileNamePrefix, *fileIndex,
subFileIndex, *detIndex, *numUnitsPerDetector, index);
std::ostringstream os;
os << *filePath << "/" << *fileNamePrefix << "_d"
<< (*detIndex * (*numUnitsPerDetector) + index) << "_f" << subFileIndex << '_'
<< *fileIndex << ".h5";
currentFileName = os.str();
//first time
if(subFileIndex == 0u)
UpdateDataType();
UpdateDataTypeFromDr();
uint64_t framestosave = ((*maxFramesPerFile == 0) ? *numImages : // infinite images
(((extNumImages - subFileIndex) > (*maxFramesPerFile)) ? // save up to maximum at a time
(*maxFramesPerFile) : (extNumImages-subFileIndex)));
std::lock_guard<std::mutex> lock(mutex);
HDF5FileStatic::CreateDataFile(index, *overWriteEnable, currentFileName, (*numImages > 1),
subFileIndex, framestosave, nPixelsY, ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX),
datatype, filefd, dataspace, dataset,
HDF5_WRITER_VERSION, MAX_CHUNKED_IMAGES,
dataspace_para, dataset_para,
parameterNames, parameterDataTypes);
CreateDataFile();
if(!(*silentMode)) {
LOG(logINFO) << *udpPortNumber << ": HDF5 File created: " << currentFileName;
}
}
void HDF5File::CloseCurrentFile() {
{
std::lock_guard<std::mutex> lock(mutex);
HDF5FileStatic::CloseDataFile(index, filefd);
}
CloseFile(filefd, nullptr, false);
for (unsigned int i = 0; i < dataset_para.size(); ++i)
delete dataset_para[i];
dataset_para.clear();
@ -174,11 +164,10 @@ void HDF5File::CloseCurrentFile() {
void HDF5File::CloseAllFiles() {
numFilesinAcquisition = 0;
{
std::lock_guard<std::mutex> lock(mutex);
HDF5FileStatic::CloseDataFile(index, filefd);
CloseFile(filefd, nullptr, false);
if (master && (*detIndex==0)) {
HDF5FileStatic::CloseMasterDataFile(masterfd);
HDF5FileStatic::CloseVirtualDataFile(virtualfd);
CloseFile(masterfd, nullptr, true);
CloseFile(nullptr, &virtualfd, false);
}
}
for (unsigned int i = 0; i < dataset_para.size(); ++i)
@ -191,7 +180,7 @@ void HDF5File::CloseAllFiles() {
}
void HDF5File::WriteToFile(char* buffer, int buffersize, uint64_t fnum, uint32_t nump) {
void HDF5File::WriteToFile(char* buffer, int bufferSize, uint64_t currentFrameNumber, uint32_t numPacketsCaught) {
// check if maxframesperfile = 0 for infinite
if ((*maxFramesPerFile) && (numFramesInFile >= (*maxFramesPerFile))) {
@ -200,58 +189,33 @@ void HDF5File::WriteToFile(char* buffer, int buffersize, uint64_t fnum, uint32_t
CreateFile();
}
numFramesInFile++;
numActualPacketsInFile += nump;
std::lock_guard<std::mutex> lock(mutex);
numActualPacketsInFile += numPacketsCaught;
// extend dataset (when receiver start followed by many status starts (jungfrau)))
if (fnum >= extNumImages) {
HDF5FileStatic::ExtendDataset(index, dataspace, dataset,
dataspace_para, dataset_para, *numImages);
if (!(*silentMode)) {
LOG(logINFO) << index << " Extending HDF5 dataset by " <<
extNumImages << ", Total x Dimension: " << (extNumImages + *numImages);
}
extNumImages += *numImages;
if (currentFrameNumber >= extNumImages) {
ExtendDataset();
}
HDF5FileStatic::WriteDataFile(index, buffer + sizeof(sls_receiver_header),
// infinite then no need for %maxframesperfile
((*maxFramesPerFile == 0) ? fnum : fnum%(*maxFramesPerFile)),
nPixelsY, ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX),
dataspace, dataset, datatype);
HDF5FileStatic::WriteParameterDatasets(index, dataspace_para,
// infinite then no need for %maxframesperfile
((*maxFramesPerFile == 0) ? fnum : fnum%(*maxFramesPerFile)),
dataset_para, (sls_receiver_header*) (buffer),
parameterDataTypes);
WriteDataFile(currentFrameNumber, buffer + sizeof(sls_receiver_header));
WriteParameterDatasets(currentFrameNumber, (sls_receiver_header*) (buffer));
}
void HDF5File::CreateMasterFile(bool mfwenable, masterAttributes& attr) {
void HDF5File::CreateMasterFile(bool masterFileWriteEnable, masterAttributes& masterFileAttributes) {
//beginning of every acquisition
numFramesInFile = 0;
numActualPacketsInFile = 0;
extNumImages = *numImages;
if (mfwenable && master && (*detIndex==0)) {
if (masterFileWriteEnable && master && (*detIndex==0)) {
virtualfd = 0;
masterFileName = HDF5FileStatic::CreateMasterFileName(*filePath,
*fileNamePrefix, *fileIndex);
if(!(*silentMode)) {
LOG(logINFO) << "Master File: " << masterFileName;
}
std::lock_guard<std::mutex> lock(mutex);
attr.version = HDF5_WRITER_VERSION;
HDF5FileStatic::CreateMasterDataFile(masterfd, masterFileName,
*overWriteEnable, attr);
CreateMasterDataFile();
}
}
void HDF5File::EndofAcquisition(bool anyPacketsCaught, uint64_t numf) {
void HDF5File::EndofAcquisition(bool anyPacketsCaught, uint64_t numImagesCaught) {
//not created before
if (!virtualfd && anyPacketsCaught) {
@ -260,11 +224,11 @@ void HDF5File::EndofAcquisition(bool anyPacketsCaught, uint64_t numf) {
//only one file and one sub image (link current file in master)
if (((numFilesinAcquisition == 1) && (numDetY*numDetX) == 1)) {
LinkVirtualFileinMasterFile();
LinkVirtualInMaster();
}
//create virutal file
else{
CreateVirtualFile(numf);}
CreateVirtualFile(numImagesCaught);}
}
}
numFilesinAcquisition = 0;
@ -272,35 +236,684 @@ void HDF5File::EndofAcquisition(bool anyPacketsCaught, uint64_t numf) {
// called only by the one maser receiver
void HDF5File::CreateVirtualFile(uint64_t numf) {
void HDF5File::CreateVirtualFile(uint64_t numImagesCaught) {
HDF5FileStatic::CreateVirtualDataFile(
// infinite images in 1 file, then maxfrperfile = numImagesCaught
((*maxFramesPerFile == 0) ? numImagesCaught+1 : *maxFramesPerFile),
numImagesCaught + 1);
}
void HDF5File::CloseFile(H5File* fd, hid_t* cfd, bool master) {
std::lock_guard<std::mutex> lock(mutex);
std::string vname = HDF5FileStatic::CreateVirtualFileName(*filePath, *fileNamePrefix, *fileIndex);
// c code due to only c implementation of H5Pset_virtual available
if (cfd != nullptr) {
if(*cfd) {
if (H5Fclose(*cfd) < 0 ) {
LOG(logERROR) << "Could not close virtual HDF5 handles";
}
*cfd = 0;
}
} else {
Exception::dontPrint(); // to handle errors
if (fd) {
delete fd;
fd = 0;
}
} catch(const Exception& error) {
LOG(logERROR) << "Could not close "
<< master ? "master" : "data"
<< " HDF5 handles of index " << index;
error.printErrorStack();
}
}
}
void HDF5File::WriteDataFile(uint64_t currentFrameNumber, char* buffer) {
std::lock_guard<std::mutex> lock(mutex);
uint64_t nDimx = ((*maxFramesPerFile == 0) ?
currentFrameNumber : currentFrameNumber % (*maxFramesPerFile));
uint32_t nDimy = nPixelsY;
uint32_t nDimz = ((*dynamicRange == 4) ? (nPixelsX / 2) : nPixelsX);
hsize_t count[3] = {1, nDimy, nDimz};
hsize_t start[3] = {nDimx, 0, 0};
hsize_t dims2[2] = {nDimy, nDimz};
try{
Exception::dontPrint(); //to handle errors
dataspace->selectHyperslab( H5S_SELECT_SET, count, start);
DataSpace memspace(2,dims2);
dataset->write(buffer, datatype, memspace, *dataspace);
memspace.close();
}
catch(const Exception& error){
LOG(logERROR) << "Could not write to file in object " << index;
error.printErrorStack();
throw RuntimeError("Could not write to file in object " +
std::to_string(index));
}
}
void HDF5File::WriteParameterDatasets(uint64_t currentFrameNumber,
sls_receiver_header* rheader) {
std::lock_guard<std::mutex> lock(mutex);
uint64_t fnum = ((*maxFramesPerFile == 0) ?
currentFrameNumber : currentFrameNumber % (*maxFramesPerFile));
sls_detector_header header = rheader->detHeader;
hsize_t count[1] = {1};
hsize_t start[1] = {fnum};
int i = 0;
try{
Exception::dontPrint(); //to handle errors
dataspace_para->selectHyperslab( H5S_SELECT_SET, count, start);
DataSpace memspace(H5S_SCALAR);
dataset_para[0]->write(&header.frameNumber,
parameterDataTypes[0], memspace, *dataspace_para);i=1;
dataset_para[1]->write(&header.expLength,
parameterDataTypes[1], memspace, *dataspace_para);i=2;
dataset_para[2]->write(&header.packetNumber,
parameterDataTypes[2], memspace, *dataspace_para);i=3;
dataset_para[3]->write(&header.bunchId,
parameterDataTypes[3], memspace, *dataspace_para);i=4;
dataset_para[4]->write(&header.timestamp,
parameterDataTypes[4], memspace, *dataspace_para);i=5;
dataset_para[5]->write(&header.modId,
parameterDataTypes[5], memspace, *dataspace_para);i=6;
dataset_para[6]->write(&header.row,
parameterDataTypes[6], memspace, *dataspace_para);i=7;
dataset_para[7]->write(&header.column,
parameterDataTypes[7], memspace, *dataspace_para);i=8;
dataset_para[8]->write(&header.reserved,
parameterDataTypes[8], memspace, *dataspace_para);i=9;
dataset_para[9]->write(&header.debug,
parameterDataTypes[9], memspace, *dataspace_para);i=10;
dataset_para[10]->write(&header.roundRNumber,
parameterDataTypes[10], memspace, *dataspace_para);i=11;
dataset_para[11]->write(&header.detType,
parameterDataTypes[11], memspace, *dataspace_para);i=12;
dataset_para[12]->write(&header.version,
parameterDataTypes[12], memspace, *dataspace_para);i=13;
// contiguous bitset
if (sizeof(sls_bitset) == sizeof(bitset_storage)) {
dataset_para[13]->write((char*)&(rheader->packetsMask),
parameterDataTypes[13], memspace, *dataspace_para);
}
// not contiguous bitset
else {
// get contiguous representation of bit mask
bitset_storage storage;
memset(storage, 0 , sizeof(bitset_storage));
sls_bitset bits = rheader->packetsMask;
for (int i = 0; i < MAX_NUM_PACKETS; ++i)
storage[i >> 3] |= (bits[i] << (i & 7));
// write bitmask
dataset_para[13]->write((char*)storage, parameterDataTypes[13],
memspace, *dataspace_para);
}i=14;
}
catch(const Exception& error){
error.printErrorStack();
throw RuntimeError("Could not write parameters (index:" +
std::to_string(i) + ") to file in object " + std::to_string(index));
}
}
void HDF5File::ExtendDataset() {
std::lock_guard<std::mutex> lock(mutex);
try{
Exception::dontPrint(); //to handle errors
hsize_t dims[3];
dataspace->getSimpleExtentDims(dims);
dims[0] += *numImages;
dataset->extend(dims);
delete dataspace;
dataspace = 0;
dataspace = new DataSpace(dataset->getSpace());
hsize_t dims_para[1] = {dims[0]};
for (unsigned int i = 0; i < dataset_para.size(); ++i)
dataset_para[i]->extend(dims_para);
delete dataspace_para;
dataspace_para = 0;
dspace_dataspace_parapara = new DataSpace(dataset_para[0]->getSpace());
}
catch(const Exception& error){
error.printErrorStack();
throw RuntimeError("Could not extend dataset in object " +
std::to_string(index));
}
if (!(*silentMode)) {
LOG(logINFO) << index << " Extending HDF5 dataset by " <<
extNumImages << ", Total x Dimension: " << (extNumImages + *numImages);
}
extNumImages += *numImages;
}
void HDF5File::CreateMasterDataFile() {
std::lock_guard<std::mutex> lock(mutex);
std::ostringstream os;
os << *filePath << "/" << *fileNamePrefix << "_master"
<< "_" << *fileIndex << ".h5";
masterFileName = os.str();
if(!(*silentMode)) {
LOG(logINFO) << "Master File: " << masterFileName;
}
masterFileAttributes.version = HDF5_WRITER_VERSION;
try {
Exception::dontPrint(); //to handle errors
FileAccPropList flist;
flist.setFcloseDegree(H5F_CLOSE_STRONG);
masterfd = 0;
if(!(*overWriteEnable))
masterfd = new H5File( masterFileName.c_str(), H5F_ACC_EXCL,
FileCreatPropList::DEFAULT,
flist );
else
masterfd = new H5File( masterFileName.c_str(), H5F_ACC_TRUNC,
FileCreatPropList::DEFAULT,
flist );
//variables
DataSpace dataspace = DataSpace (H5S_SCALAR);
Attribute attribute;
DataSet dataset;
int iValue=0;
double dValue=0;
StrType strdatatype(PredType::C_S1,256);
//create attributes
//version
dValue = masterFileAttributes.version;
attribute = masterfd->createAttribute("version",PredType::NATIVE_DOUBLE, dataspace);
attribute.write(PredType::NATIVE_DOUBLE, &dValue);
//Create a group in the file
Group group1( masterfd->createGroup( "entry" ) );
Group group2( group1.createGroup("data") );
Group group3( group1.createGroup("instrument") );
Group group4( group3.createGroup("beam") );
Group group5( group3.createGroup("detector") );
Group group6( group1.createGroup("sample") );
//Dynamic Range
dataset = group5.createDataSet ( "dynamic range", PredType::NATIVE_INT, dataspace );
dataset.write ( &(masterFileAttributes.dynamicRange), PredType::NATIVE_INT);
attribute = dataset.createAttribute("unit",strdatatype, dataspace);
attribute.write(strdatatype, std::string("bits"));
//Ten Giga
iValue = masterFileAttributes.tenGiga;
dataset = group5.createDataSet ( "ten giga enable", PredType::NATIVE_INT, dataspace );
dataset.write ( &iValue, PredType::NATIVE_INT);
//Image Size
dataset = group5.createDataSet ( "image size", PredType::NATIVE_INT, dataspace );
dataset.write ( &(masterFileAttributes.imageSize), PredType::NATIVE_INT);
attribute = dataset.createAttribute("unit",strdatatype, dataspace);
attribute.write(strdatatype, std::string("bytes"));
//x
dataset = group5.createDataSet ( "number of pixels in x axis", PredType::NATIVE_INT, dataspace );
dataset.write ( &(masterFileAttributes.nPixelsX), PredType::NATIVE_INT);
//y
dataset = group5.createDataSet ( "number of pixels in y axis", PredType::NATIVE_INT, dataspace );
dataset.write ( &(masterFileAttributes.nPixelsY), PredType::NATIVE_INT);
//Maximum frames per file
dataset = group5.createDataSet ( "maximum frames per file", PredType::NATIVE_INT, dataspace );
dataset.write ( &(masterFileAttributes.maxFramesPerFile), PredType::NATIVE_INT);
//Total Frames
dataset = group5.createDataSet ( "total frames", PredType::STD_U64LE, dataspace );
dataset.write ( &(masterFileAttributes.totalFrames), PredType::STD_U64LE);
//Exptime
dataset = group5.createDataSet ( "exposure time", PredType::STD_U64LE, dataspace );
dataset.write ( &(masterFileAttributes.exptimeNs), PredType::STD_U64LE);
attribute = dataset.createAttribute("unit",strdatatype, dataspace);
attribute.write(strdatatype, std::string("ns"));
//SubExptime
dataset = group5.createDataSet ( "sub exposure time", PredType::STD_U64LE, dataspace );
dataset.write ( &(masterFileAttributes.subExptimeNs), PredType::STD_U64LE);
attribute = dataset.createAttribute("unit",strdatatype, dataspace);
attribute.write(strdatatype, std::string("ns"));
//SubPeriod
dataset = group5.createDataSet ( "sub period", PredType::STD_U64LE, dataspace );
dataset.write ( &(masterFileAttributes.subPeriodNs), PredType::STD_U64LE);
attribute = dataset.createAttribute("unit",strdatatype, dataspace);
attribute.write(strdatatype, std::string("ns"));
//Period
dataset = group5.createDataSet ( "acquisition period", PredType::STD_U64LE, dataspace );
dataset.write ( &(masterFileAttributes.periodNs), PredType::STD_U64LE);
attribute = dataset.createAttribute("unit",strdatatype, dataspace);
attribute.write(strdatatype, std::string("ns"));
//Quad Enable
dataset = group5.createDataSet ( "quad enable", PredType::NATIVE_INT, dataspace );
dataset.write ( &(masterFileAttributes.quadEnable), PredType::NATIVE_INT);
//Analog Flag
dataset = group5.createDataSet ( "analog flag", PredType::NATIVE_INT, dataspace );
dataset.write ( &(masterFileAttributes.analogFlag), PredType::NATIVE_INT);
//Digital Flag
dataset = group5.createDataSet ( "digital flag", PredType::NATIVE_INT, dataspace );
dataset.write ( &(masterFileAttributes.digitalFlag), PredType::NATIVE_INT);
//ADC Mask
dataset = group5.createDataSet ( "adc mask", PredType::NATIVE_INT, dataspace );
dataset.write ( &(masterFileAttributes.adcmask), PredType::NATIVE_INT);
//Dbit Offset
dataset = group5.createDataSet ( "dbit offset", PredType::NATIVE_INT, dataspace );
dataset.write ( &(masterFileAttributes.dbitoffset), PredType::NATIVE_INT);
// Dbit List
dataset = group5.createDataSet ( "dbit bitset list", PredType::STD_U64LE, dataspace );
dataset.write ( &(masterFileAttributes.dbitlist), PredType::STD_U64LE);
// Roi xmin
dataset = group5.createDataSet ( "roi xmin", PredType::NATIVE_INT, dataspace );
dataset.write ( &(masterFileAttributes.roiXmin), PredType::NATIVE_INT);
// Roi xmax
dataset = group5.createDataSet ( "roi xmax", PredType::NATIVE_INT, dataspace );
dataset.write ( &(masterFileAttributes.roiXmax), PredType::NATIVE_INT);
//Timestamp
time_t t = time(0);
dataset = group5.createDataSet ( "timestamp", strdatatype, dataspace );
dataset.write ( std::string(ctime(&t)), strdatatype );
masterfd->close();
} catch(const Exception& error) {
error.printErrorStack();
if (masterfd) masterfd->close();
throw RuntimeError("Could not create master HDF5 handles");
}
}
void HDF5File::CreateDataFile() {
std::lock_guard<std::mutex> lock(mutex);
uint64_t framestosave = ((*maxFramesPerFile == 0) ? *numImages : // infinite images
(((extNumImages - subFileIndex) > (*maxFramesPerFile)) ? // save up to maximum at a time
(*maxFramesPerFile) : (extNumImages-subFileIndex)));
uint64_t nDimx = framestosave;
uint32_t nDimy = nPixelsY;
uint32_t nDimz = ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX);
try {
Exception::dontPrint(); //to handle errors
//file
FileAccPropList fapl;
fapl.setFcloseDegree(H5F_CLOSE_STRONG);
filefd = 0;
if(!(*overWriteEnable))
filefd = new H5File( currentFileName.c_str(), H5F_ACC_EXCL,
FileCreatPropList::DEFAULT,
fapl );
else
filefd = new H5File( currentFileName.c_str(), H5F_ACC_TRUNC,
FileCreatPropList::DEFAULT,
fapl );
//attributes - version
double dValue = HDF5_WRITER_VERSION;
DataSpace dataspace_attr = DataSpace (H5S_SCALAR);
Attribute attribute = filefd->createAttribute("version",
PredType::NATIVE_DOUBLE, dataspace_attr);
attribute.write(PredType::NATIVE_DOUBLE, &dValue);
//dataspace
hsize_t srcdims[3] = {nDimx, nDimy, nDimz};
hsize_t srcdimsmax[3] = {H5S_UNLIMITED, nDimy, nDimz};
dataspace = 0;
dataspace = new DataSpace (3,srcdims,srcdimsmax);
//dataset name
std::ostringstream osfn;
osfn << "/data";
if (*numImages > 1)
osfn << "_f" << std::setfill('0') << std::setw(12) << subFileIndex;
std::string dsetname = osfn.str();
//dataset
//fill value
DSetCreatPropList plist;
int fill_value = -1;
plist.setFillValue(datatype, &fill_value);
// always create chunked dataset as unlimited is only
// supported with chunked layout
hsize_t chunk_dims[3] = {MAX_CHUNKED_IMAGES, nDimy, nDimz};
plist.setChunk(3, chunk_dims);
dataset = 0;
dataset = new DataSet (filefd->createDataSet(dsetname.c_str(), datatype,
*dataspace, plist));
//create parameter datasets
hsize_t dims[1] = {nDimx};
hsize_t dimsmax[1] = {H5S_UNLIMITED};
dataspace_para = 0;
dataspace_para = new DataSpace (1,dims,dimsmax);
// always create chunked dataset as unlimited is only
// supported with chunked layout
DSetCreatPropList paralist;
hsize_t chunkpara_dims[3] = {MAX_CHUNKED_IMAGES};
paralist.setChunk(1, chunkpara_dims);
for (unsigned int i = 0; i < parameterNames.size(); ++i){
DataSet* ds = new DataSet(filefd->createDataSet(parameterNames[i],
parameterDataTypes[i], *dataspace_para, paralist));
dataset_para.push_back(ds);
}
}
catch(const Exception& error){
error.printErrorStack();
if (filefd) filefd->close();
throw RuntimeError("Could not create HDF5 handles in object " + index);
}
}
void HDF5File::CreateVirtualDataFile(uint32_t maxFramesPerFile, uint64_t numf) {
std::lock_guard<std::mutex> lock(mutex);
std::ostringstream osfn;
osfn << *filePath << "/" << *fileNamePrefix;
osfn << "_virtual";
osfn << "_" << *fileIndex;
osfn << ".h5";
std::string vname = osfn.str();
if(!(*silentMode)) {
LOG(logINFO) << "Virtual File: " << vname;
}
HDF5FileStatic::CreateVirtualDataFile(vname,
virtualfd, masterFileName,
*filePath, *fileNamePrefix, *fileIndex, (*numImages > 1),
*detIndex, *numUnitsPerDetector,
// infinite images in 1 file, then maxfrperfile = numf
((*maxFramesPerFile == 0) ? numf+1 : *maxFramesPerFile),
numf+1,
"data", datatype,
numDetY, numDetX, nPixelsY, ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX),
HDF5_WRITER_VERSION,
parameterNames, parameterDataTypes);
int numDetz = numDetX;
uint32_t nDimy = nPixelsY;
uint32_t nDimz = ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX);
try {
//file
hid_t dfal = H5Pcreate (H5P_FILE_ACCESS);
if (dfal < 0)
throw RuntimeError("Could not create file access property for virtual file " + vname);
if (H5Pset_fclose_degree (dfal, H5F_CLOSE_STRONG) < 0)
throw RuntimeError("Could not set strong file close degree for virtual file " + vname);
virtualfd = H5Fcreate( vname.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, dfal);
if (virtualfd < 0)
throw RuntimeError("Could not create virtual file " + vname);
//attributes - version
hid_t dataspace_attr = H5Screate (H5S_SCALAR);
if (dataspace_attr < 0)
throw RuntimeError("Could not create dataspace for attribute in virtual file " + vname);
hid_t attrid = H5Acreate2 (virtualfd, "version", H5T_NATIVE_DOUBLE, dataspace_attr, H5P_DEFAULT, H5P_DEFAULT);
if (attrid < 0)
throw RuntimeError("Could not create attribute in virtual file " + vname);
double attr_data = HDF5_WRITER_VERSION;
if (H5Awrite (attrid, H5T_NATIVE_DOUBLE, &attr_data) < 0)
throw RuntimeError("Could not write attribute in virtual file " + vname);
if (H5Aclose (attrid) < 0)
throw RuntimeError("Could not close attribute in virtual file " + vname);
//virtual dataspace
hsize_t vdsdims[3] = {numf, numDetY * nDimy, numDetz * nDimz};
hid_t vdsDataspace = H5Screate_simple(3, vdsdims ,NULL);
if (vdsDataspace < 0)
throw RuntimeError("Could not create virtual dataspace in virtual file " + vname);
hsize_t vdsdims_para[2] = {numf, (unsigned int) numDetY * numDetz};
hid_t vdsDataspace_para = H5Screate_simple(2, vdsdims_para, NULL);
if (vdsDataspace_para < 0)
throw RuntimeError("Could not create virtual dataspace (parameters) in virtual file " + vname);
//fill values
hid_t dcpl = H5Pcreate (H5P_DATASET_CREATE);
if (dcpl < 0)
throw RuntimeError("Could not create file creation properties in virtual file " + vname);
int fill_value = -1;
if (H5Pset_fill_value (dcpl, GetDataTypeinC(datatype), &fill_value) < 0)
throw RuntimeError("Could not create fill value in virtual file " + vname);
hid_t dcpl_para[parameterNames.size()];
for (unsigned int i = 0; i < parameterNames.size(); ++i) {
dcpl_para[i] = H5Pcreate (H5P_DATASET_CREATE);
if (dcpl_para[i] < 0)
throw RuntimeError("Could not create file creation properties (parameters) in virtual file " + vname);
if (H5Pset_fill_value (dcpl_para[i], GetDataTypeinC(parameterDataTypes[i]), &fill_value) < 0)
throw RuntimeError("Could not create fill value (parameters) in virtual file " + vname);
}
//hyperslab
int numMajorHyperslab = numf/maxFramesPerFile;
if (numf%maxFramesPerFile) numMajorHyperslab++;
uint64_t framesSaved = 0;
for (int j = 0; j < numMajorHyperslab; j++) {
uint64_t nDimx = ((numf - framesSaved) > maxFramesPerFile)
? maxFramesPerFile : (numf-framesSaved);
hsize_t offset[3] = {framesSaved, 0, 0};
hsize_t count[3] = {nDimx, nDimy, nDimz};
hsize_t offset_para[2] = {framesSaved, 0};
hsize_t count_para[2] = {nDimx, 1};
for (int i = 0; i < numDetY * numDetz; ++i) {
//setect hyperslabs
if (H5Sselect_hyperslab (vdsDataspace, H5S_SELECT_SET, offset, NULL, count, NULL) < 0) {
throw RuntimeError("Could not select hyperslab");
}
if (H5Sselect_hyperslab (vdsDataspace_para, H5S_SELECT_SET,
offset_para, NULL, count_para, NULL) < 0) {
throw RuntimeError("Could not select hyperslab for parameters");
}
//source file name
std::ostringstream os;
os << *filePath << "/" << *fileNamePrefix << "_d"
<< (*detIndex * (*numUnitsPerDetector) + i) << "_f" << j << '_'
<< *fileIndex << ".h5";
std::string srcFileName = os.str();
LOG(logERROR) << srcFileName;
// find relative path
std::string relative_srcFileName = srcFileName;
{
size_t i = srcFileName.rfind('/', srcFileName.length());
if (i != std::string::npos)
relative_srcFileName = (srcFileName.substr(i+1, srcFileName.length() - i));
}
//source dataset name
std::ostringstream osfn;
osfn << "/data";
if (*numImages > 1)
osfn << "_f" << std::setfill('0') << std::setw(12) << j;
std::string srcDatasetName = osfn.str();
//source dataspace
hsize_t srcdims[3] = {nDimx, nDimy, nDimz};
hsize_t srcdimsmax[3] = {H5S_UNLIMITED, nDimy, nDimz};
hid_t srcDataspace = H5Screate_simple(3, srcdims, srcdimsmax);
if (srcDataspace < 0)
throw RuntimeError("Could not create source dataspace in virtual file " + vname);
hsize_t srcdims_para[1] = {nDimx};
hsize_t srcdimsmax_para[1] = {H5S_UNLIMITED};
hid_t srcDataspace_para = H5Screate_simple(1, srcdims_para, srcdimsmax_para);
if (srcDataspace_para < 0)
throw RuntimeError("Could not create source dataspace (parameters) in virtual file " + vname);
//mapping
if (H5Pset_virtual(dcpl, vdsDataspace, relative_srcFileName.c_str(),
srcDatasetName.c_str(), srcDataspace) < 0) {
throw RuntimeError("Could not set mapping for paramter 1");
}
for (unsigned int k = 0; k < parameterNames.size(); ++k) {
if (H5Pset_virtual(dcpl_para[k], vdsDataspace_para, relative_srcFileName.c_str(),
parameterNames[k], srcDataspace_para) < 0) {
throw RuntimeError("Could not set mapping for paramter " + std::to_string(k));
}
}
//H5Sclose(srcDataspace);
//H5Sclose(srcDataspace_para);
offset[2] += nDimz;
if (offset[2] >= (numDetz * nDimz)) {
offset[2] = 0;
offset[1] += nDimy;
}
offset_para[1]++;
}
framesSaved += nDimx;
}
//dataset
std::string virtualDatasetName = "data";
hid_t vdsdataset = H5Dcreate2 (virtualfd, virtualDatasetName.c_str(),
GetDataTypeinC(datatype), vdsDataspace, H5P_DEFAULT, dcpl, H5P_DEFAULT);
if (vdsdataset < 0)
throw RuntimeError("Could not create virutal dataset in virtual file " + vname);
//virtual parameter dataset
for (unsigned int i = 0; i < parameterNames.size(); ++i) {
hid_t vdsdataset_para = H5Dcreate2 (virtualfd,
parameterNames[i],
GetDataTypeinC(parameterDataTypes[i]), vdsDataspace_para,
H5P_DEFAULT, dcpl_para[i], H5P_DEFAULT);
if (vdsdataset_para < 0)
throw RuntimeError("Could not create virutal dataset (parameters) in virtual file " + vname);
}
//close
H5Fclose(virtualfd);
virtualfd = 0;
//link
LinkVirtualInMaster(masterFileName, vname, virtualDatasetName, parameterNames);
} catch (const RuntimeError &e) {
if (virtualfd > 0)
H5Fclose(virtualfd);
virtualfd = 0;
}
}
// called only by the one maser receiver
void HDF5File::LinkVirtualFileinMasterFile() {
void HDF5File::LinkVirtualInMaster() {
std::lock_guard<std::mutex> lock(mutex);
//dataset name
std::ostringstream osfn;
osfn << "/data";
if ((*numImages > 1)) osfn << "_f" << std::setfill('0') << std::setw(12) << 0;
std::string dsetname = osfn.str();
std::lock_guard<std::mutex> lock(mutex);
HDF5FileStatic::LinkVirtualInMaster(masterFileName, currentFileName,
dsetname, parameterNames);
char linkname[100];
hid_t vfd = 0;
try {
hid_t dfal = H5Pcreate (H5P_FILE_ACCESS);
if (dfal < 0)
throw RuntimeError("Could not create file access property for link");
if (H5Pset_fclose_degree (dfal, H5F_CLOSE_STRONG) < 0)
throw RuntimeError("Could not set strong file close degree for link");
//open master file
hid_t mfd = H5Fopen( masterFileName.c_str(), H5F_ACC_RDWR, dfal);
if (mfd < 0)
throw RuntimeError("Could not open master file");
//open virtual file
vfd = H5Fopen( currentFileName.c_str(), H5F_ACC_RDWR, dfal);
if (vfd < 0) {
H5Fclose(mfd); mfd = 0;
throw RuntimeError("Could not open virtual file");
}
// find relative path
std::string relative_virtualfname = currentFileName;
{
size_t i = currentFileName.rfind('/', currentFileName.length());
if (i != std::string::npos)
relative_virtualfname = (currentFileName.substr(i+1, currentFileName.length() - i));
}
//**data dataset**
hid_t vdset = H5Dopen2( vfd, dsetname.c_str(), H5P_DEFAULT);
if (vdset < 0) {
H5Fclose(mfd);
throw RuntimeError("Could not open virtual data dataset");
}
sprintf(linkname, "/entry/data/%s",dsetname.c_str());
if(H5Lcreate_external( relative_virtualfname.c_str(), dsetname.c_str(),
mfd, linkname, H5P_DEFAULT, H5P_DEFAULT) < 0) {
H5Fclose(mfd); mfd = 0;
throw RuntimeError("Could not create link to data dataset");
}
H5Dclose(vdset);
//**paramter datasets**
for (unsigned int i = 0; i < parameterNames.size(); ++i){
hid_t vdset_para = H5Dopen2( vfd, (std::string (parameterNames[i])).c_str(), H5P_DEFAULT);
if (vdset_para < 0) {
H5Fclose(mfd); mfd = 0;
throw RuntimeError("Could not open virtual parameter dataset to create link");
}
sprintf(linkname, "/entry/data/%s",(std::string (parameterNames[i])).c_str());
if(H5Lcreate_external( relative_virtualfname.c_str(), (std::string (parameterNames[i])).c_str(),
mfd, linkname, H5P_DEFAULT, H5P_DEFAULT) < 0) {
H5Fclose(mfd); mfd = 0;
throw RuntimeError("Could not create link to virtual parameter dataset");
}
}
H5Fclose(mfd); mfd = 0;
H5Fclose(vfd); vfd = 0;
} catch () {
if(vfd > 0)
H5Fclose(vfd);
vfd = 0;
}
}
hid_t HDF5File::GetDataTypeinC(DataType dtype) {
if (dtype == PredType::STD_U8LE)
return H5T_STD_U8LE;
else if (dtype == PredType::STD_U16LE)
return H5T_STD_U16LE;
else if (dtype == PredType::STD_U32LE)
return H5T_STD_U32LE;
else if (dtype == PredType::STD_U64LE)
return H5T_STD_U64LE;
else {
hid_t s = H5Tcopy(H5T_C_S1);
H5Tset_size(s, MAX_NUM_PACKETS);
return s;
}
}

119
slsReceiverSoftware/src/HDF5File.h
View File

@ -11,7 +11,6 @@
#include "File.h"
#include "HDF5FileStatic.h"
#include "H5Cpp.h"
#ifndef H5_NO_NAMESPACE
@ -20,7 +19,7 @@
#include <mutex>
class HDF5File : private virtual slsDetectorDefs, public File, public HDF5FileStatic {
class HDF5File : private virtual slsDetectorDefs, public File {
public:
/**
@ -47,139 +46,55 @@ class HDF5File : private virtual slsDetectorDefs, public File, public HDF5FileSt
int* dindex, int* nunits, uint64_t* nf, uint32_t* dr, uint32_t* portno,
uint32_t nx, uint32_t ny,
bool* smode);
/**
* Destructor
*/
~HDF5File();
/**
* Print all member values
*/
void PrintMembers(TLogLevel level = logDEBUG1);
/**
* Set Number of pixels
* @param nx number of pixels in x direction
* @param ny number of pixels in y direction
*/
void SetNumberofPixels(uint32_t nx, uint32_t ny);
/**
* Create file
* @param fnum current frame index to include in file name
*/
void CreateFile();
/**
* Close Current File
*/
void CloseCurrentFile();
/**
* Close all Files
*/
void CloseAllFiles();
/**
* Write data to file
* @param buffer buffer to write from
* @param bsize size of buffer (not used)
* @param fnum current image number
* @param nump number of packets caught
*/
void WriteToFile(char* buffer, int bsize, uint64_t fnum, uint32_t nump);
/**
* Create master file
* @param mfwenable master file write enable
* @param attr master file attributes
*/
void CreateMasterFile(bool mfwenable, masterAttributes& attr);
/**
* End of Acquisition
* @param anyPacketsCaught true if any packets are caught, else false
* @param numf number of images caught
*/
void EndofAcquisition(bool anyPacketsCaught, uint64_t numf);
void WriteToFile(char* buffer, int bufferSize, uint64_t currentFrameNumber, uint32_t numPacketsCaught);
void CreateMasterFile(bool masterFileWriteEnable, masterAttributes& masterFileAttributes);
void EndofAcquisition(bool anyPacketsCaught, uint64_t numImagesCaught);
private:
/**
* Create Virtual File
* @param numf number of images caught
*/
void CreateVirtualFile(uint64_t numf);
/**
* Link virtual file in master file
* Only for Jungfrau at the moment for 1 module and 1 data file
*/
void LinkVirtualFileinMasterFile();
/**
* Get Type
* @return type
*/
void CreateVirtualFile(uint64_t numImagesCaught);
fileFormat GetFileType();
/**
* Updates data type depending on current dynamic range
*/
void UpdateDataType();
void UpdateDataTypeFromDr();
void CloseFile(H5File* fd, hid_t* cfd, bool master);
void WriteDataFile(uint64_t currentFrameNumber, char* buffer);
void WriteParameterDatasets(uint64_t currentFrameNumber, sls_receiver_header* rheader);
void ExtendDataset();
void CreateMasterDataFile();
void CreateDataFile();
void CreateVirtualDataFile(uint32_t maxFramesPerFile, uint64_t numf);
void LinkVirtualInMaster();
hid_t GetDataTypeinC(DataType dtype);
/** mutex to update static items among objects (threads)*/
static mutable std::mutex mutex;
/** Master File handle */
static H5File* masterfd;
/** Virtual File handle ( only file name because code in C as H5Pset_virtual doesnt exist yet in C++) */
/** Virtual File handle ( only file name because
code in C as H5Pset_virtual doesnt exist yet in C++) */
static hid_t virtualfd;
/** File handle */
H5File* filefd;
/** Dataspace handle */
DataSpace* dataspace;
/** DataSet handle */
DataSet* dataset;
/** Datatype of dataset */
DataType datatype;
/** Number of pixels in x direction */
uint32_t nPixelsX;
/** Number of pixels in y direction */
uint32_t nPixelsY;
/** Number of frames in file */
uint32_t numFramesInFile;
/** Number of actual packets caught in file */
uint64_t numActualPacketsInFile;
/** Number of files in an acquisition - to verify need of virtual file */
int numFilesinAcquisition;
/** parameter names */
std::vector <const char*> parameterNames;
/** parameter data types */
std::vector <DataType> parameterDataTypes;
/** Dataspace of parameters */
DataSpace* dataspace_para;
/** Dataset array for parameters */
std::vector <DataSet*> dataset_para;
/** Number of Images (including extended during acquisition) */
uint64_t extNumImages;
};

918
slsReceiverSoftware/src/HDF5FileStatic.h
View File

@ -28,910 +28,40 @@ class HDF5FileStatic: public virtual slsDetectorDefs {
public:
/** Constructor */
HDF5FileStatic(){};
/** Destructor */
virtual ~HDF5FileStatic(){};
/**
* Create File Name in format fpath/fnameprefix_fx_dy_z.raw,
* where x is fnum, y is (dindex * numunits + unitindex) and z is findex
* @param fpath file path
* @param fnameprefix file name prefix
* @param findex file index
* @param subfindex sub file index
* @param dindex readout index
* @param numunits number of units per readout. eg. eiger has 2 udp units per readout
* @param unitindex unit index
* @returns complete file name created
*/
static std::string CreateFileName(std::string fpath, std::string fprefix,
uint64_t findex, uint64_t subfindex,
int dindex, int numunits = 1,
int unitindex = 0) {
std::ostringstream os;
os << fpath << "/" << fprefix << "_d"
<< (dindex * numunits + unitindex) << "_f" << subfindex << '_'
<< findex << ".h5";
return os.str();
}
/**
* Create master file name
* @param fpath file path
* @param fnameprefix file name prefix
* @param findex file index
* @returns master file name
*/
static std::string CreateMasterFileName(std::string fpath, std::string fnameprefix, uint64_t findex)
{
std::ostringstream os;
os << fpath << "/" << fnameprefix << "_master"
<< "_" << findex << ".h5";
return os.str();
}
/**
* Create virtual file name
* @param fpath file path
* @param fnameprefix file name prefix
* @param fnum current frame number
* @param findex file index
* @returns virtual file name
*/
static std::string CreateVirtualFileName(std::string fpath, std::string fnameprefix, uint64_t findex)
{
std::ostringstream osfn;
osfn << fpath << "/" << fnameprefix;
osfn << "_virtual";
osfn << "_" << findex;
osfn << ".h5";
return osfn.str();
}
/**
* Close File
* @param ind index for debugging
* @param fd file pointer
* @param dp dataspace pointer
* @param ds dataset pointer
* @param ds_para pointer to array of parameter datasets
*/
static void CloseDataFile(int ind, H5File*& fd)
{
try {
Exception::dontPrint(); //to handle errors
if (fd) {
delete fd;
fd = 0;
}
} catch(const Exception& error) {
LOG(logERROR) << "Could not close HDF5 handles of index " << ind;
error.printErrorStack();
}
}
/*
* Close master file
* @param fd master hdf5 file object
*/
static void CloseMasterDataFile(H5File*& fd)
{
try {
Exception::dontPrint(); //to handle errors
if (fd) {
delete fd;
fd = 0;
}
} catch(const Exception& error) {
LOG(logERROR) << "Could not close master HDF5 handles";
error.printErrorStack();
}
}
/*
* Close virtual file
* (in C because H5Pset_virtual doesnt exist yet in C++)
* @param fd virtual hdf5 file handle
*/
static void CloseVirtualDataFile(hid_t& fd)
{
if(fd) {
if (H5Fclose(fd) < 0 ) {
LOG(logERROR) << "Could not close virtual HDF5 handles";
}
fd = 0;
}
}
/**
* Write data to file
* @param ind object index for debugging
* @param buf buffer to write from
* @param nDimx image number in file (imagenumber%maxframesinfile)
* @param nDimy number of pixels in y direction
* @param nDimz number of pixels in x direction
* @param dspace dataspace pointer
* @param dset dataset pointer
* @param dtype datatype
*/
static void WriteDataFile(int ind, char* buf,
uint64_t nDimx, uint32_t nDimy, uint32_t nDimz,
DataSpace* dspace, DataSet* dset, DataType dtype)
{
hsize_t count[3] = {1, nDimy, nDimz};
hsize_t start[3] = {nDimx, 0, 0};
hsize_t dims2[2] = {nDimy, nDimz};
try{
Exception::dontPrint(); //to handle errors
/**
* Create virtual file
* (in C because H5Pset_virtual doesnt exist yet in C++)
* @param virtualFileName virtual file name
* @param fd virtual file handle
* @param masterFileName master file name
* @param fpath file path
* @param fnameprefix file name prefix
* @param findex file index
* @param frindexenable frame index enable
* @param dindex readout index
* @param numunits number of units per readout. eg. eiger has 2 udp units per readout
* @param maxFramesPerFile maximum frames per file
* @param numf number of frames caught
* @param srcDataseName source dataset name
* @param dataType datatype of data dataset
* @param numDety number of readouts in Y dir
* @param numDetz number of readouts in Z dir
* @param nDimy number of objects in y dimension in source file (Number of pixels in y dir)
* @param nDimz number of objects in z dimension in source file (Number of pixels in x dir)
* @param version version of software for hdf5 writing
* @param parameterNames parameter names
* @param parameterDataTypes parameter datatypes
*/
dspace->selectHyperslab( H5S_SELECT_SET, count, start);
DataSpace memspace(2,dims2);
dset->write(buf, dtype, memspace, *dspace);
memspace.close();
}
catch(const Exception& error){
LOG(logERROR) << "Could not write to file in object " << ind;
error.printErrorStack();
throw RuntimeError("Could not write to file in object " + std::to_string(ind));
}
}
/**
* Write Parameter Arrays as datasets (to virtual file)
* @param ind self index
* @param dspace_para parameter dataspace
* @param fnum current frame number
* @param dset_para vector or dataset pointers of parameters
* @param rheader sls_receiver_header pointer
* @param parameterDataTypes parameter datatypes
*/
static void WriteParameterDatasets(int ind, DataSpace* dspace_para, uint64_t fnum,
std::vector <DataSet*> dset_para,sls_receiver_header* rheader,
std::vector <DataType> parameterDataTypes)
{
sls_detector_header header = rheader->detHeader;
hsize_t count[1] = {1};
hsize_t start[1] = {fnum};
int i = 0;
try{
Exception::dontPrint(); //to handle errors
dspace_para->selectHyperslab( H5S_SELECT_SET, count, start);
DataSpace memspace(H5S_SCALAR);
dset_para[0]->write(&header.frameNumber, parameterDataTypes[0], memspace, *dspace_para);i=1;
dset_para[1]->write(&header.expLength, parameterDataTypes[1], memspace, *dspace_para);i=2;
dset_para[2]->write(&header.packetNumber, parameterDataTypes[2], memspace, *dspace_para);i=3;
dset_para[3]->write(&header.bunchId, parameterDataTypes[3], memspace, *dspace_para);i=4;
dset_para[4]->write(&header.timestamp, parameterDataTypes[4], memspace, *dspace_para);i=5;
dset_para[5]->write(&header.modId, parameterDataTypes[5], memspace, *dspace_para);i=6;
dset_para[6]->write(&header.row, parameterDataTypes[6], memspace, *dspace_para);i=7;
dset_para[7]->write(&header.column, parameterDataTypes[7], memspace, *dspace_para);i=8;
dset_para[8]->write(&header.reserved, parameterDataTypes[8], memspace, *dspace_para);i=9;
dset_para[9]->write(&header.debug, parameterDataTypes[9], memspace, *dspace_para);i=10;
dset_para[10]->write(&header.roundRNumber, parameterDataTypes[10], memspace, *dspace_para);i=11;
dset_para[11]->write(&header.detType, parameterDataTypes[11], memspace, *dspace_para);i=12;
dset_para[12]->write(&header.version, parameterDataTypes[12], memspace, *dspace_para);i=13;
// contiguous bitset
if (sizeof(sls_bitset) == sizeof(bitset_storage)) {
dset_para[13]->write((char*)&(rheader->packetsMask), parameterDataTypes[13], memspace, *dspace_para);
}
// not contiguous bitset
else {
// get contiguous representation of bit mask
bitset_storage storage;
memset(storage, 0 , sizeof(bitset_storage));
sls_bitset bits = rheader->packetsMask;
for (int i = 0; i < MAX_NUM_PACKETS; ++i)
storage[i >> 3] |= (bits[i] << (i & 7));
// write bitmask
dset_para[13]->write((char*)storage, parameterDataTypes[13], memspace, *dspace_para);
}i=14;
}
catch(const Exception& error){
error.printErrorStack();
throw RuntimeError("Could not write parameters (index:" + std::to_string(i) + ") to file in object " + std::to_string(ind));
}
}
/**
* Extend datasets in #images dimension (x dimension)
* @param ind self index
* @param dpace data space pointer address
* @param dset data set pointer
* @param dspace_para parameter dataspace address pointer
* @param dset dataset parameter pointer
* @param initialNumImages initial number of images
*/
static void ExtendDataset(int ind, DataSpace*& dspace, DataSet* dset,
DataSpace*& dspace_para, std::vector <DataSet*> dset_para,
uint64_t initialNumImages) {
try{
Exception::dontPrint(); //to handle errors
hsize_t dims[3];
dspace->getSimpleExtentDims(dims);
dims[0] += initialNumImages;
dset->extend(dims);
delete dspace;
dspace = 0;
dspace = new DataSpace(dset->getSpace());
hsize_t dims_para[1] = {dims[0]};
for (unsigned int i = 0; i < dset_para.size(); ++i)
dset_para[i]->extend(dims_para);
delete dspace_para;
dspace_para = 0;
dspace_para = new DataSpace(dset_para[0]->getSpace());
}
catch(const Exception& error){
error.printErrorStack();
throw RuntimeError("Could not extend dataset in object " + std::to_string(ind));
}
}
/**
* Create master file
* @param fname master file name
* @param owenable overwrite enable
* @param attr master file attributes
*/
static void CreateMasterDataFile(H5File*& fd, std::string fname, bool owenable,
masterAttributes& attr)
{
try {
Exception::dontPrint(); //to handle errors
FileAccPropList flist;
flist.setFcloseDegree(H5F_CLOSE_STRONG);
fd = 0;
if(!owenable)
fd = new H5File( fname.c_str(), H5F_ACC_EXCL,
FileCreatPropList::DEFAULT,
flist );
else
fd = new H5File( fname.c_str(), H5F_ACC_TRUNC,
FileCreatPropList::DEFAULT,
flist );
//variables
DataSpace dataspace = DataSpace (H5S_SCALAR);
Attribute attribute;
DataSet dataset;
int iValue=0;
double dValue=0;
StrType strdatatype(PredType::C_S1,256);
//create attributes
//version
dValue = attr.version;
attribute = fd->createAttribute("version",PredType::NATIVE_DOUBLE, dataspace);
attribute.write(PredType::NATIVE_DOUBLE, &dValue);
//Create a group in the file
Group group1( fd->createGroup( "entry" ) );
Group group2( group1.createGroup("data") );
Group group3( group1.createGroup("instrument") );
Group group4( group3.createGroup("beam") );
Group group5( group3.createGroup("detector") );
Group group6( group1.createGroup("sample") );
//Dynamic Range
dataset = group5.createDataSet ( "dynamic range", PredType::NATIVE_INT, dataspace );
dataset.write ( &(attr.dynamicRange), PredType::NATIVE_INT);
attribute = dataset.createAttribute("unit",strdatatype, dataspace);
attribute.write(strdatatype, std::string("bits"));
//Ten Giga
iValue = attr.tenGiga;
dataset = group5.createDataSet ( "ten giga enable", PredType::NATIVE_INT, dataspace );
dataset.write ( &iValue, PredType::NATIVE_INT);
//Image Size
dataset = group5.createDataSet ( "image size", PredType::NATIVE_INT, dataspace );
dataset.write ( &(attr.imageSize), PredType::NATIVE_INT);
attribute = dataset.createAttribute("unit",strdatatype, dataspace);
attribute.write(strdatatype, std::string("bytes"));
//x
dataset = group5.createDataSet ( "number of pixels in x axis", PredType::NATIVE_INT, dataspace );
dataset.write ( &(attr.nPixelsX), PredType::NATIVE_INT);
//y
dataset = group5.createDataSet ( "number of pixels in y axis", PredType::NATIVE_INT, dataspace );
dataset.write ( &(attr.nPixelsY), PredType::NATIVE_INT);
//Maximum frames per file
dataset = group5.createDataSet ( "maximum frames per file", PredType::NATIVE_INT, dataspace );
dataset.write ( &(attr.maxFramesPerFile), PredType::NATIVE_INT);
//Total Frames
dataset = group5.createDataSet ( "total frames", PredType::STD_U64LE, dataspace );
dataset.write ( &(attr.totalFrames), PredType::STD_U64LE);
//Exptime
dataset = group5.createDataSet ( "exposure time", PredType::STD_U64LE, dataspace );
dataset.write ( &(attr.exptimeNs), PredType::STD_U64LE);
attribute = dataset.createAttribute("unit",strdatatype, dataspace);
attribute.write(strdatatype, std::string("ns"));
//SubExptime
dataset = group5.createDataSet ( "sub exposure time", PredType::STD_U64LE, dataspace );
dataset.write ( &(attr.subExptimeNs), PredType::STD_U64LE);
attribute = dataset.createAttribute("unit",strdatatype, dataspace);
attribute.write(strdatatype, std::string("ns"));
//SubPeriod
dataset = group5.createDataSet ( "sub period", PredType::STD_U64LE, dataspace );
dataset.write ( &(attr.subPeriodNs), PredType::STD_U64LE);
attribute = dataset.createAttribute("unit",strdatatype, dataspace);
attribute.write(strdatatype, std::string("ns"));
//Period
dataset = group5.createDataSet ( "acquisition period", PredType::STD_U64LE, dataspace );
dataset.write ( &(attr.periodNs), PredType::STD_U64LE);
attribute = dataset.createAttribute("unit",strdatatype, dataspace);
attribute.write(strdatatype, std::string("ns"));
//Quad Enable
dataset = group5.createDataSet ( "quad enable", PredType::NATIVE_INT, dataspace );
dataset.write ( &(attr.quadEnable), PredType::NATIVE_INT);
//Analog Flag
dataset = group5.createDataSet ( "analog flag", PredType::NATIVE_INT, dataspace );
dataset.write ( &(attr.analogFlag), PredType::NATIVE_INT);
//Digital Flag
dataset = group5.createDataSet ( "digital flag", PredType::NATIVE_INT, dataspace );
dataset.write ( &(attr.digitalFlag), PredType::NATIVE_INT);
//ADC Mask
dataset = group5.createDataSet ( "adc mask", PredType::NATIVE_INT, dataspace );
dataset.write ( &(attr.adcmask), PredType::NATIVE_INT);
//Dbit Offset
dataset = group5.createDataSet ( "dbit offset", PredType::NATIVE_INT, dataspace );
dataset.write ( &(attr.dbitoffset), PredType::NATIVE_INT);
// Dbit List
dataset = group5.createDataSet ( "dbit bitset list", PredType::STD_U64LE, dataspace );
dataset.write ( &(attr.dbitlist), PredType::STD_U64LE);
// Roi xmin
dataset = group5.createDataSet ( "roi xmin", PredType::NATIVE_INT, dataspace );
dataset.write ( &(attr.roiXmin), PredType::NATIVE_INT);
// Roi xmax
dataset = group5.createDataSet ( "roi xmax", PredType::NATIVE_INT, dataspace );
dataset.write ( &(attr.roiXmax), PredType::NATIVE_INT);
//Timestamp
time_t t = time(0);
dataset = group5.createDataSet ( "timestamp", strdatatype, dataspace );
dataset.write ( std::string(ctime(&t)), strdatatype );
fd->close();
} catch(const Exception& error) {
error.printErrorStack();
if (fd) fd->close();
throw RuntimeError("Could not create master HDF5 handles");
}
}
/**
* Create File
* @param ind object index for debugging
* @param owenable overwrite enable
* @param fname complete file name
* @param frindexenable frame index enable
* @param fnum current image number
* @param nDimx number of pixels in x dim (#frames)
* @param nDimy number of pixels in y dim (height y dir)
* @param nDimz number of pixels in z dim (width x dir)
* @param dtype data type
* @param fd file pointer
* @param dspace dataspace pointer
* @param dset dataset pointer
* @param version version of software for hdf5 writing
* @param maxchunkedimages maximum chunked images
* @param dspace_para dataspace of parameters
* @param dset_para vector of datasets of parameters
* @param parameterNames parameter names
* @param parameterDataTypes parameter datatypes
*/
static void CreateDataFile(int ind, bool owenable, std::string fname, bool frindexenable,
uint64_t fnum, uint64_t nDimx, uint32_t nDimy, uint32_t nDimz,
DataType dtype, H5File*& fd, DataSpace*& dspace, DataSet*& dset,
double version, uint64_t maxchunkedimages,
DataSpace*& dspace_para, std::vector<DataSet*>& dset_para,
std::vector <const char*> parameterNames,
std::vector <DataType> parameterDataTypes)
{
try {
Exception::dontPrint(); //to handle errors
//file
FileAccPropList fapl;
fapl.setFcloseDegree(H5F_CLOSE_STRONG);
fd = 0;
if(!owenable)
fd = new H5File( fname.c_str(), H5F_ACC_EXCL,
FileCreatPropList::DEFAULT,
fapl );
else
fd = new H5File( fname.c_str(), H5F_ACC_TRUNC,
FileCreatPropList::DEFAULT,
fapl );
//attributes - version
double dValue=version;
DataSpace dataspace_attr = DataSpace (H5S_SCALAR);
Attribute attribute = fd->createAttribute("version",PredType::NATIVE_DOUBLE, dataspace_attr);
attribute.write(PredType::NATIVE_DOUBLE, &dValue);
//dataspace
hsize_t srcdims[3] = {nDimx, nDimy, nDimz};
hsize_t srcdimsmax[3] = {H5S_UNLIMITED, nDimy, nDimz};
dspace = 0;
dspace = new DataSpace (3,srcdims,srcdimsmax);
//dataset name
std::ostringstream osfn;
osfn << "/data";
if (frindexenable) osfn << "_f" << std::setfill('0') << std::setw(12) << fnum;
std::string dsetname = osfn.str();
//dataset
//fill value
DSetCreatPropList plist;
int fill_value = -1;
plist.setFillValue(dtype, &fill_value);
// always create chunked dataset as unlimited is only supported with chunked layout
hsize_t chunk_dims[3] ={maxchunkedimages, nDimy, nDimz};
plist.setChunk(3, chunk_dims);
dset = 0;
dset = new DataSet (fd->createDataSet(dsetname.c_str(), dtype, *dspace, plist));
//create parameter datasets
hsize_t dims[1] = {nDimx};
hsize_t dimsmax[1] = {H5S_UNLIMITED};
dspace_para = 0;
dspace_para = new DataSpace (1,dims,dimsmax);
// always create chunked dataset as unlimited is only supported with chunked layout
DSetCreatPropList paralist;
hsize_t chunkpara_dims[3] = {maxchunkedimages};
paralist.setChunk(1, chunkpara_dims);
for (unsigned int i = 0; i < parameterNames.size(); ++i){
DataSet* ds = new DataSet(fd->createDataSet(parameterNames[i],
parameterDataTypes[i], *dspace_para, paralist));
dset_para.push_back(ds);
}
}
catch(const Exception& error){
error.printErrorStack();
if (fd) fd->close();
throw RuntimeError("Could not create HDF5 handles in object " + ind);
}
}
/**
* Create virtual file
* (in C because H5Pset_virtual doesnt exist yet in C++)
* @param virtualFileName virtual file name
* @param fd virtual file handle
* @param masterFileName master file name
* @param fpath file path
* @param fnameprefix file name prefix
* @param findex file index
* @param frindexenable frame index enable
* @param dindex readout index
* @param numunits number of units per readout. eg. eiger has 2 udp units per readout
* @param maxFramesPerFile maximum frames per file
* @param numf number of frames caught
* @param srcDataseName source dataset name
* @param dataType datatype of data dataset
* @param numDety number of readouts in Y dir
* @param numDetz number of readouts in Z dir
* @param nDimy number of objects in y dimension in source file (Number of pixels in y dir)
* @param nDimz number of objects in z dimension in source file (Number of pixels in x dir)
* @param version version of software for hdf5 writing
* @param parameterNames parameter names
* @param parameterDataTypes parameter datatypes
*/
static void CreateVirtualDataFile(
std::string virtualFileName,
hid_t& fd, std::string masterFileName,
std::string fpath, std::string fnameprefix, uint64_t findex, bool frindexenable,
int dindex, int numunits,
uint32_t maxFramesPerFile, uint64_t numf,
std::string srcDataseName, DataType dataType,
int numDety, int numDetz, uint32_t nDimy, uint32_t nDimz,
double version,
std::vector <const char*> parameterNames,
std::vector <DataType> parameterDataTypes)
{
try {
//file
hid_t dfal = H5Pcreate (H5P_FILE_ACCESS);
if (dfal < 0)
throw RuntimeError("Could not create file access property for virtual file " + virtualFileName);
if (H5Pset_fclose_degree (dfal, H5F_CLOSE_STRONG) < 0)
throw RuntimeError("Could not set strong file close degree for virtual file " + virtualFileName);
fd = H5Fcreate( virtualFileName.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, dfal);
if (fd < 0)
throw RuntimeError("Could not create virtual file " + virtualFileName);
//attributes - version
hid_t dataspace_attr = H5Screate (H5S_SCALAR);
if (dataspace_attr < 0)
throw RuntimeError("Could not create dataspace for attribute in virtual file " + virtualFileName);
hid_t attrid = H5Acreate2 (fd, "version", H5T_NATIVE_DOUBLE, dataspace_attr, H5P_DEFAULT, H5P_DEFAULT);
if (attrid < 0)
throw RuntimeError("Could not create attribute in virtual file " + virtualFileName);
double attr_data = version;
if (H5Awrite (attrid, H5T_NATIVE_DOUBLE, &attr_data) < 0)
throw RuntimeError("Could not write attribute in virtual file " + virtualFileName);
if (H5Aclose (attrid) < 0)
throw RuntimeError("Could not close attribute in virtual file " + virtualFileName);
//virtual dataspace
hsize_t vdsdims[3] = {numf, numDety * nDimy, numDetz * nDimz};
hid_t vdsDataspace = H5Screate_simple(3, vdsdims ,NULL);
if (vdsDataspace < 0)
throw RuntimeError("Could not create virtual dataspace in virtual file " + virtualFileName);
hsize_t vdsdims_para[2] = {numf, (unsigned int) numDety * numDetz};
hid_t vdsDataspace_para = H5Screate_simple(2, vdsdims_para, NULL);
if (vdsDataspace_para < 0)
throw RuntimeError("Could not create virtual dataspace (parameters) in virtual file " + virtualFileName);
//fill values
hid_t dcpl = H5Pcreate (H5P_DATASET_CREATE);
if (dcpl < 0)
throw RuntimeError("Could not create file creation properties in virtual file " + virtualFileName);
int fill_value = -1;
if (H5Pset_fill_value (dcpl, GetDataTypeinC(dataType), &fill_value) < 0)
throw RuntimeError("Could not create fill value in virtual file " + virtualFileName);
hid_t dcpl_para[parameterNames.size()];
for (unsigned int i = 0; i < parameterNames.size(); ++i) {
dcpl_para[i] = H5Pcreate (H5P_DATASET_CREATE);
if (dcpl_para[i] < 0)
throw RuntimeError("Could not create file creation properties (parameters) in virtual file " + virtualFileName);
if (H5Pset_fill_value (dcpl_para[i], GetDataTypeinC(parameterDataTypes[i]), &fill_value) < 0)
throw RuntimeError("Could not create fill value (parameters) in virtual file " + virtualFileName);
}
//hyperslab
int numMajorHyperslab = numf/maxFramesPerFile;
if (numf%maxFramesPerFile) numMajorHyperslab++;
uint64_t framesSaved = 0;
for (int j = 0; j < numMajorHyperslab; j++) {
uint64_t nDimx = ((numf - framesSaved) > maxFramesPerFile)
? maxFramesPerFile : (numf-framesSaved);
hsize_t offset[3] = {framesSaved, 0, 0};
hsize_t count[3] = {nDimx, nDimy, nDimz};
hsize_t offset_para[2] = {framesSaved, 0};
hsize_t count_para[2] = {nDimx, 1};
for (int i = 0; i < numDety * numDetz; ++i) {
//setect hyperslabs
if (H5Sselect_hyperslab (vdsDataspace, H5S_SELECT_SET, offset, NULL, count, NULL) < 0) {
throw RuntimeError("Could not select hyperslab");
}
if (H5Sselect_hyperslab (vdsDataspace_para, H5S_SELECT_SET,
offset_para, NULL, count_para, NULL) < 0) {
throw RuntimeError("Could not select hyperslab for parameters");
}
//source file name
std::string srcFileName = HDF5FileStatic::CreateFileName(fpath, fnameprefix, findex,
j, dindex, numunits, i);
LOG(logERROR) << srcFileName;
// find relative path
std::string relative_srcFileName = srcFileName;
{
size_t i = srcFileName.rfind('/', srcFileName.length());
if (i != std::string::npos)
relative_srcFileName = (srcFileName.substr(i+1, srcFileName.length() - i));
}
//source dataset name
std::ostringstream osfn;
osfn << "/data";
if (frindexenable) osfn << "_f" << std::setfill('0') << std::setw(12) << j;
std::string srcDatasetName = osfn.str();
//source dataspace
hsize_t srcdims[3] = {nDimx, nDimy, nDimz};
hsize_t srcdimsmax[3] = {H5S_UNLIMITED, nDimy, nDimz};
hid_t srcDataspace = H5Screate_simple(3, srcdims, srcdimsmax);
if (srcDataspace < 0)
throw RuntimeError("Could not create source dataspace in virtual file " + virtualFileName);
hsize_t srcdims_para[1] = {nDimx};
hsize_t srcdimsmax_para[1] = {H5S_UNLIMITED};
hid_t srcDataspace_para = H5Screate_simple(1, srcdims_para, srcdimsmax_para);
if (srcDataspace_para < 0)
throw RuntimeError("Could not create source dataspace (parameters) in virtual file " + virtualFileName);
//mapping
if (H5Pset_virtual(dcpl, vdsDataspace, relative_srcFileName.c_str(),
srcDatasetName.c_str(), srcDataspace) < 0) {
throw RuntimeError("Could not set mapping for paramter 1");
}
for (unsigned int k = 0; k < parameterNames.size(); ++k) {
if (H5Pset_virtual(dcpl_para[k], vdsDataspace_para, relative_srcFileName.c_str(),
parameterNames[k], srcDataspace_para) < 0) {
throw RuntimeError("Could not set mapping for paramter " + std::to_string(k));
}
}
//H5Sclose(srcDataspace);
//H5Sclose(srcDataspace_para);
offset[2] += nDimz;
if (offset[2] >= (numDetz * nDimz)) {
offset[2] = 0;
offset[1] += nDimy;
}
offset_para[1]++;
}
framesSaved += nDimx;
}
//dataset
std::string virtualDatasetName = srcDataseName;
hid_t vdsdataset = H5Dcreate2 (fd, virtualDatasetName.c_str(),
GetDataTypeinC(dataType), vdsDataspace, H5P_DEFAULT, dcpl, H5P_DEFAULT);
if (vdsdataset < 0)
throw RuntimeError("Could not create virutal dataset in virtual file " + virtualFileName);
//virtual parameter dataset
for (unsigned int i = 0; i < parameterNames.size(); ++i) {
hid_t vdsdataset_para = H5Dcreate2 (fd,
parameterNames[i],
GetDataTypeinC(parameterDataTypes[i]), vdsDataspace_para,
H5P_DEFAULT, dcpl_para[i], H5P_DEFAULT);
if (vdsdataset_para < 0)
throw RuntimeError("Could not create virutal dataset (parameters) in virtual file " + virtualFileName);
}
//close
H5Fclose(fd);
fd = 0;
//link
LinkVirtualInMaster(masterFileName, virtualFileName, virtualDatasetName, parameterNames);
} catch (const RuntimeError &e) {
if (fd > 0)
H5Fclose(fd);
fd = 0;
}
}
/**
* Copy file to another file (mainly to view virutal files in hdfviewer)
* @param dataType data type
* @param owenable overwrite enable
* @param oldFileName file name including path of file to copy
* @param oldDatasetName dataset name to copy
* @param newFileName file name including path of file to copy to
* @param newDatasetName dataset name to copy to
* @param rank rank
* @param nDimx Number of objects in x dimension
* @param nDimy Number of objects in y dimension
* @param nDimz Number of objects in z dimension
*/
template <typename T>
static void CopyVirtualFile(T datatype, bool owenable, std::string oldFileName, std::string oldDatasetName,
std::string newFileName, std::string newDatasetName, int rank,
uint64_t nDimx, uint32_t nDimy, uint32_t nDimz=0)
{
T *data_out;
switch (rank) {
case 2:
data_out = (T*)malloc(sizeof(T)*(nDimx*nDimy));
break;
case 3:
data_out = (T*)malloc(sizeof(T)*(nDimx*nDimy*nDimz));
break;
default:
throw RuntimeError("Invalid rank. Options: 2 or 3");
}
if (datatype == PredType::STD_U16LE) {
LOG(logINFO) << "datatype:16";
} else if (datatype == PredType::STD_U32LE) {
LOG(logINFO) << "datatype:32";
} else if (datatype == PredType::STD_U64LE) {
LOG(logINFO) << "datatype:64";
} else if (datatype == PredType::STD_U8LE) {
LOG(logINFO) << "datatype:8";
} else {
throw RuntimeError("Unknown datatype:" + std::to_string(datatype));
}
LOG(logINFO) << "owenable:" << (owenable?1:0) << std::endl
<< "oldFileName:" << oldFileName << std::endl
<< "oldDatasetName:" << oldDatasetName << std::endl
<< "newFileName:" << newFileName << std::endl
<< "newDatasetName:" << newDatasetName << std::endl
<< "rank:" << rank << std::endl
<< "nDimx:" << nDimx << std::endl
<< "nDimy:" << nDimy << std::endl
<< "nDimz:" << nDimz;
H5File* oldfd;
H5File* newfd;
try {
Exception::dontPrint(); //to handle errors
//open old file
oldfd = new H5File( oldFileName.c_str(), H5F_ACC_RDONLY);
DataSet oldDataset = oldfd->openDataSet( oldDatasetName.c_str());
//read dataset
oldDataset.read( data_out, datatype);
//new file
FileAccPropList fapl;
fapl.setFcloseDegree(H5F_CLOSE_STRONG);
newfd = 0;
if(!owenable)
newfd = new H5File( newFileName.c_str(), H5F_ACC_EXCL,
FileCreatPropList::DEFAULT,
fapl );
else
newfd = new H5File( newFileName.c_str(), H5F_ACC_TRUNC,
FileCreatPropList::DEFAULT,
fapl );
//dataspace and dataset
DataSpace* newDataspace = 0;
if (rank == 3) {
hsize_t dims[3] = {nDimx, nDimy, nDimz};
newDataspace = new DataSpace (3,dims);
} else if (rank == 2) {
hsize_t dims[2] = {nDimx, nDimy};
newDataspace = new DataSpace (2,dims);
}
DataSet* newDataset = 0;
newDataset = new DataSet( newfd->createDataSet(newDatasetName.c_str(), datatype, *newDataspace));
//write and close
newDataset->write(data_out,datatype);
newfd->close();
oldfd->close();
} catch(const Exception& error){
error.printErrorStack();
free(data_out);
oldfd->close();
newfd->close();
throw RuntimeError("Could not copy virtual files");
}
free(data_out);
}
/**
* Link the Virtual File in the Master File
* In C because H5Lcreate_external exists only in C
* @param masterFileName master file name
* @param virtualfname virtual file name
* @param virtualDatasetname virtual dataset name
* @param parameterNames parameter names
*/
static void LinkVirtualInMaster(std::string masterFileName, std::string virtualfname,
std::string virtualDatasetname, std::vector <const char*> parameterNames) {
char linkname[100];
hid_t vfd = 0;
try {
hid_t dfal = H5Pcreate (H5P_FILE_ACCESS);
if (dfal < 0)
throw RuntimeError("Could not create file access property for link");
if (H5Pset_fclose_degree (dfal, H5F_CLOSE_STRONG) < 0)
throw RuntimeError("Could not set strong file close degree for link");
//open master file
hid_t mfd = H5Fopen( masterFileName.c_str(), H5F_ACC_RDWR, dfal);
if (mfd < 0)
throw RuntimeError("Could not open master file");
//open virtual file
vfd = H5Fopen( virtualfname.c_str(), H5F_ACC_RDWR, dfal);
if (vfd < 0) {
H5Fclose(mfd); mfd = 0;
throw RuntimeError("Could not open virtual file");
}
// find relative path
std::string relative_virtualfname = virtualfname;
{
size_t i = virtualfname.rfind('/', virtualfname.length());
if (i != std::string::npos)
relative_virtualfname = (virtualfname.substr(i+1, virtualfname.length() - i));
}
//**data dataset**
hid_t vdset = H5Dopen2( vfd, virtualDatasetname.c_str(), H5P_DEFAULT);
if (vdset < 0) {
H5Fclose(mfd);
throw RuntimeError("Could not open virtual data dataset");
}
sprintf(linkname, "/entry/data/%s",virtualDatasetname.c_str());
if(H5Lcreate_external( relative_virtualfname.c_str(), virtualDatasetname.c_str(),
mfd, linkname, H5P_DEFAULT, H5P_DEFAULT) < 0) {
H5Fclose(mfd); mfd = 0;
throw RuntimeError("Could not create link to data dataset");
}
H5Dclose(vdset);
//**paramter datasets**
for (unsigned int i = 0; i < parameterNames.size(); ++i){
hid_t vdset_para = H5Dopen2( vfd, (std::string (parameterNames[i])).c_str(), H5P_DEFAULT);
if (vdset_para < 0) {
H5Fclose(mfd); mfd = 0;
throw RuntimeError("Could not open virtual parameter dataset to create link");
}
sprintf(linkname, "/entry/data/%s",(std::string (parameterNames[i])).c_str());
if(H5Lcreate_external( relative_virtualfname.c_str(), (std::string (parameterNames[i])).c_str(),
mfd, linkname, H5P_DEFAULT, H5P_DEFAULT) < 0) {
H5Fclose(mfd); mfd = 0;
throw RuntimeError("Could not create link to virtual parameter dataset");
}
}
H5Fclose(mfd); mfd = 0;
H5Fclose(vfd); vfd = 0;
} catch () {
if(vfd > 0)
H5Fclose(vfd);
vfd = 0;
}
}
/**
* Get Data type in C
* @param dtype datatype in C++
* @returns datatype in C
*/
static hid_t GetDataTypeinC(DataType dtype) {
if (dtype == PredType::STD_U8LE)
return H5T_STD_U8LE;
else if (dtype == PredType::STD_U16LE)
return H5T_STD_U16LE;
else if (dtype == PredType::STD_U32LE)
return H5T_STD_U32LE;
else if (dtype == PredType::STD_U64LE)
return H5T_STD_U64LE;
else {
hid_t s = H5Tcopy(H5T_C_S1);
H5Tset_size(s, MAX_NUM_PACKETS);
return s;
}
}
};