diff --git a/slsReceiverSoftware/src/BinaryFile.cpp b/slsReceiverSoftware/src/BinaryFile.cpp index 28929e309..1d76342c4 100755 --- a/slsReceiverSoftware/src/BinaryFile.cpp +++ b/slsReceiverSoftware/src/BinaryFile.cpp @@ -9,7 +9,8 @@ #include "receiver_defs.h" #include - +#include +#include FILE* BinaryFile::masterfd = nullptr; @@ -17,10 +18,11 @@ BinaryFile::BinaryFile(int ind, uint32_t* maxf, int* nd, std::string* fname, std::string* fpath, uint64_t* findex, bool* owenable, int* dindex, int* nunits, uint64_t* nf, uint32_t* dr, uint32_t* portno, bool* smode): - File(ind, maxf, nd, fname, fpath, findex, owenable, dindex, nunits, nf, dr, portno, smode), + File(ind, BINARY, maxf, nd, fname, fpath, findex, owenable, dindex, nunits, nf, dr, portno, smode), filefd(nullptr), numFramesInFile(0), - numActualPacketsInFile(0) + numActualPacketsInFile(0), + maxMasterFileSize(2000) { #ifdef VERBOSE PrintMembers(); @@ -37,19 +39,27 @@ void BinaryFile::PrintMembers(TLogLevel level) { LOG(logINFO) << "Number of Frames in File: " << numFramesInFile; } -slsDetectorDefs::fileFormat BinaryFile::GetFileType() { - return BINARY; -} - - void BinaryFile::CreateFile() { numFramesInFile = 0; numActualPacketsInFile = 0; - currentFileName = BinaryFileStatic::CreateFileName(*filePath, *fileNamePrefix, *fileIndex, - subFileIndex, *detIndex, *numUnitsPerDetector, index); + std::ostringstream os; + os << *filePath << "/" << *fileNamePrefix << "_d" + << (*detIndex * (*numUnitsPerDetector) + index) << "_f" << subFileIndex << '_' + << *fileIndex << ".raw"; + currentFileName = os.str(); - BinaryFileStatic::CreateDataFile(filefd, *overWriteEnable, currentFileName); + if (!(*overWriteEnable)){ + if (NULL == (filefd = fopen((const char *) currentFileName.c_str(), "wx"))){ + filefd = 0; + throw sls::RuntimeError("Could not create/overwrite file " + currentFileName); + } + } else if (NULL == (filefd = fopen((const char *) currentFileName.c_str(), "w"))){ + filefd = 0; + throw sls::RuntimeError("Could not create file " + currentFileName); + } + //setting to no file buffering + setvbuf(filefd, NULL, _IONBF, 0); if(!(*silentMode)) { LOG(logINFO) << "[" << *udpPortNumber << "]: Binary File created: " << currentFileName; @@ -57,16 +67,29 @@ void BinaryFile::CreateFile() { } void BinaryFile::CloseCurrentFile() { - BinaryFileStatic::CloseDataFile(filefd); + if (filefd) + fclose(filefd); + filefd = 0; } void BinaryFile::CloseAllFiles() { - BinaryFileStatic::CloseDataFile(filefd); - if (master && (*detIndex==0)) - BinaryFileStatic::CloseDataFile(masterfd); + CloseCurrentFile(); + if (master) { + if (masterfd) + fclose(masterfd); + masterfd = 0; + } } -void BinaryFile::WriteToFile(char* buffer, int buffersize, uint64_t fnum, uint32_t nump) { +int BinaryFile::WriteData(char* buf, int bsize) { + if (!filefd) + return 0; + return fwrite(buf, 1, bsize, filefd); +} + + +void BinaryFile::WriteToFile(char* buffer, int buffersize, uint64_t currentFrameNumber, + uint32_t numPacketsCaught) { // check if maxframesperfile = 0 for infinite if ((*maxFramesPerFile) && (numFramesInFile >= (*maxFramesPerFile))) { CloseCurrentFile(); @@ -74,20 +97,20 @@ void BinaryFile::WriteToFile(char* buffer, int buffersize, uint64_t fnum, uint32 CreateFile(); } numFramesInFile++; - numActualPacketsInFile += nump; + numActualPacketsInFile += numPacketsCaught; // write to file int ret = 0; // contiguous bitset if (sizeof(sls_bitset) == sizeof(bitset_storage)) { - ret = BinaryFileStatic::WriteDataFile(filefd, buffer, buffersize); + ret = WriteData(buffer, buffersize); } // not contiguous bitset else { // write detector header - ret = BinaryFileStatic::WriteDataFile(filefd, buffer, sizeof(sls_detector_header)); + ret = WriteData(buffer, sizeof(sls_detector_header)); // get contiguous representation of bit mask bitset_storage storage; @@ -96,34 +119,124 @@ void BinaryFile::WriteToFile(char* buffer, int buffersize, uint64_t fnum, uint32 for (int i = 0; i < MAX_NUM_PACKETS; ++i) storage[i >> 3] |= (bits[i] << (i & 7)); // write bitmask - ret += BinaryFileStatic::WriteDataFile(filefd, (char*)storage, sizeof(bitset_storage)); + ret += WriteData((char*)storage, sizeof(bitset_storage)); // write data - ret += BinaryFileStatic::WriteDataFile(filefd, - buffer + sizeof(sls_detector_header), buffersize - sizeof(sls_receiver_header)); + ret += WriteData(buffer + sizeof(sls_detector_header), + buffersize - sizeof(sls_receiver_header)); } // if write error if (ret != buffersize) { - throw sls::RuntimeError(std::to_string(index) + " : Write to file failed for image number " + std::to_string(fnum)); + throw sls::RuntimeError(std::to_string(index) + " : Write to file failed for image number " + + std::to_string(currentFrameNumber)); } } -void BinaryFile::CreateMasterFile(bool mfwenable, masterAttributes& attr) { +void BinaryFile::CreateMasterFile(bool masterFileWriteEnable, masterAttributes& masterFileAttributes) { //beginning of every acquisition numFramesInFile = 0; numActualPacketsInFile = 0; - if (mfwenable && master && (*detIndex==0)) { - masterFileName = BinaryFileStatic::CreateMasterFileName(*filePath, - *fileNamePrefix, *fileIndex); + if (masterFileWriteEnable && master) { + + std::ostringstream os; + os << *filePath << "/" << *fileNamePrefix << "_master" + << "_" << *fileIndex << ".raw"; + masterFileName = os.str(); if(!(*silentMode)) { LOG(logINFO) << "Master File: " << masterFileName; } - attr.version = BINARY_WRITER_VERSION; - BinaryFileStatic::CreateMasterDataFile(masterfd, masterFileName, - *overWriteEnable, attr); + masterFileAttributes.version = BINARY_WRITER_VERSION; + + // create master file + if (!(*overWriteEnable)){ + if (NULL == (masterfd = fopen((const char *) masterFileName.c_str(), "wx"))) { + masterfd = 0; + throw sls::RuntimeError("Could not create binary master file " + "(without overwrite enable) " + masterFileName); + } + }else if (NULL == (masterfd = fopen((const char *) masterFileName.c_str(), "w"))) { + masterfd = 0; + throw sls::RuntimeError("Could not create binary master file " + "(with overwrite enable) " + masterFileName); + } + // create master file data + time_t t = time(0); + char message[maxMasterFileSize]; + sprintf(message, + "Version : %.1f\n" + "Detector Type : %d\n" + "Dynamic Range : %d\n" + "Ten Giga : %d\n" + "Image Size : %d bytes\n" + "nPixelsX : %d pixels\n" + "nPixelsY : %d pixels\n" + "Max Frames Per File : %u\n" + "Total Frames : %lld\n" + "Exptime (ns) : %lld\n" + "SubExptime (ns) : %lld\n" + "SubPeriod(ns) : %lld\n" + "Period (ns) : %lld\n" + "Quad Enable : %d\n" + "Analog Flag : %d\n" + "Digital Flag : %d\n" + "ADC Mask : %d\n" + "Dbit Offset : %d\n" + "Dbit Bitset : %lld\n" + "Roi (xmin, xmax) : %d %d\n" + "Timestamp : %s\n\n" + + "#Frame Header\n" + "Frame Number : 8 bytes\n" + "SubFrame Number/ExpLength : 4 bytes\n" + "Packet Number : 4 bytes\n" + "Bunch ID : 8 bytes\n" + "Timestamp : 8 bytes\n" + "Module Id : 2 bytes\n" + "Row : 2 bytes\n" + "Column : 2 bytes\n" + "Reserved : 2 bytes\n" + "Debug : 4 bytes\n" + "Round Robin Number : 2 bytes\n" + "Detector Type : 1 byte\n" + "Header Version : 1 byte\n" + "Packets Caught Mask : 64 bytes\n" + , + masterFileAttributes.version, + masterFileAttributes.detectorType, + masterFileAttributes.dynamicRange, + masterFileAttributes.tenGiga, + masterFileAttributes.imageSize, + masterFileAttributes.nPixelsX, + masterFileAttributes.nPixelsY, + masterFileAttributes.maxFramesPerFile, + (long long int)masterFileAttributes.totalFrames, + (long long int)masterFileAttributes.exptimeNs, + (long long int)masterFileAttributes.subExptimeNs, + (long long int)masterFileAttributes.subPeriodNs, + (long long int)masterFileAttributes.periodNs, + masterFileAttributes.quadEnable, + masterFileAttributes.analogFlag, + masterFileAttributes.digitalFlag, + masterFileAttributes.adcmask, + masterFileAttributes.dbitoffset, + (long long int)masterFileAttributes.dbitlist, + masterFileAttributes.roiXmin, + masterFileAttributes.roiXmax, + ctime(&t)); + if (strlen(message) > maxMasterFileSize) { + throw sls::RuntimeError("Master File Size " + std::to_string(strlen(message)) + + " is greater than max str size " + std::to_string(maxMasterFileSize)); + } + // write and close file + if (fwrite((void*)message, 1, strlen(message), masterfd) != strlen(message)) { + throw sls::RuntimeError("Master binary file incorrect number of bytes written to file"); + } + if (masterfd) + fclose(masterfd); + masterfd = 0; } } diff --git a/slsReceiverSoftware/src/BinaryFile.h b/slsReceiverSoftware/src/BinaryFile.h index a5337bbaf..62a724182 100755 --- a/slsReceiverSoftware/src/BinaryFile.h +++ b/slsReceiverSoftware/src/BinaryFile.h @@ -9,12 +9,11 @@ */ #include "File.h" -#include "BinaryFileStatic.h" #include -class BinaryFile : private virtual slsDetectorDefs, public File, public BinaryFileStatic { +class BinaryFile : private virtual slsDetectorDefs, public File { public: /** @@ -38,71 +37,25 @@ class BinaryFile : private virtual slsDetectorDefs, public File, public BinaryFi int* nd, std::string* fname, std::string* fpath, uint64_t* findex, bool* owenable, int* dindex, int* nunits, uint64_t* nf, uint32_t* dr, uint32_t* portno, bool* smode); - - /** - * Destructor - */ ~BinaryFile(); - /** - * Print all member values - */ void PrintMembers(TLogLevel level = logDEBUG1) override; - - /** - * Create file - */ void CreateFile() override; - - /** - * Create master file - * @param mfwenable master file write enable - * @param attr master file attributes - */ - void CreateMasterFile(bool mfwenable, masterAttributes& attr) override; - - /** - * Close Current File - */ + void CreateMasterFile(bool masterFileWriteEnable, + masterAttributes& masterFileAttributes) override; void CloseCurrentFile() override; - - /** - * Close all Files - */ void CloseAllFiles() override; - - /** - * Write data to file - * @param buffer buffer to write from - * @param buffersize size of buffer - * @param fnum current image number - * @param nump number of packets caught - */ - void WriteToFile(char* buffer, int buffersize, uint64_t fnum, uint32_t nump) override; - - + void WriteToFile(char* buffer, int buffersize, uint64_t currentFrameNumber, + uint32_t numPacketsCaught) override; private: + int WriteData(char* buf, int bsize); - /** - * Get Type - * @return type - */ - fileFormat GetFileType() override; - - - - /** File Descriptor */ FILE* filefd; - - /** Master File Descriptor */ static FILE* masterfd; - - /** Number of frames in file */ uint32_t numFramesInFile; - - /** Number of actual packets caught in file */ uint64_t numActualPacketsInFile; + const size_t maxMasterFileSize; }; diff --git a/slsReceiverSoftware/src/BinaryFileStatic.h b/slsReceiverSoftware/src/BinaryFileStatic.h deleted file mode 100755 index 80f86fbc2..000000000 --- a/slsReceiverSoftware/src/BinaryFileStatic.h +++ /dev/null @@ -1,214 +0,0 @@ -#pragma once -/************************************************ - * @file BinaryFileStatic.h - * @short creating, closing, writing and reading - * from binary files - ***********************************************/ -/** - *@short creating, closing, writing and reading from binary files - */ - - -#include "logger.h" - -#include -#include -#include - -#define MAX_MASTER_FILE_LENGTH 2000 - - -class BinaryFileStatic { - - public: - - /** Constructor */ - BinaryFileStatic(){}; - /** Destructor */ - virtual ~BinaryFileStatic(){}; - - /** - * Create File Name in format fpath/fnameprefix_fx_dy_z.raw, - * where x is fnum, y is (dindex * numunits + unitindex) and z is findex - * @param fpath file path - * @param fnameprefix file name prefix - * @param findex file index - * @param subfindex sub file index - * @param dindex readout index - * @param numunits number of units per readout. eg. eiger has 2 udp units per readout - * @param unitindex unit index - * @returns complete file name created - */ - static std::string CreateFileName(std::string fpath, std::string fprefix, - uint64_t findex, uint64_t subfindex, - int dindex, int numunits = 1, - int unitindex = 0) { - std::ostringstream os; - os << fpath << "/" << fprefix << "_d" - << (dindex * numunits + unitindex) << "_f" << subfindex << '_' - << findex << ".raw"; - return os.str(); - } - - /** - * Create file names for master file - * @param fpath file path - * @param fnameprefix file name prefix - * @param findex file index - * @returns master file name - */ - static std::string CreateMasterFileName(std::string fpath, std::string fnameprefix, - uint64_t findex) { - std::ostringstream os; - os << fpath << "/" << fnameprefix << "_master" - << "_" << findex << ".raw"; - return os.str(); - } - - /** - * Close File - * @param fd file pointer - */ - static void CloseDataFile(FILE*& fd) - { - if (fd) - fclose(fd); - fd = 0; - } - - /** - * Write data to file - * @param fd file pointer - * @param buf buffer to write from - * @param bsize size of buffer - * @returns number of elements written - */ - static int WriteDataFile(FILE* fd, char* buf, int bsize) - { - if (!fd) - return 0; - return fwrite(buf, 1, bsize, fd); - } - - - /** - * Create master files - * @param fd pointer to file handle - * @param fname master file name - * @param owenable overwrite enable - * @param attr master file attributes - */ - static void CreateMasterDataFile(FILE*& fd, std::string fname, bool owenable, - masterAttributes& attr) - { - if(!owenable){ - if (NULL == (fd = fopen((const char *) fname.c_str(), "wx"))){ - fd = 0; - throw sls::RuntimeError("Could not create binary master file " - "(without overwrite enable) " + fname); - } - }else if (NULL == (fd = fopen((const char *) fname.c_str(), "w"))){ - fd = 0; - throw sls::RuntimeError("Could not create binary master file " - "(with overwrite enable) " + fname); - } - time_t t = time(0); - char message[MAX_MASTER_FILE_LENGTH]; - sprintf(message, - "Version : %.1f\n" - "Detector Type : %d\n" - "Dynamic Range : %d\n" - "Ten Giga : %d\n" - "Image Size : %d bytes\n" - "nPixelsX : %d pixels\n" - "nPixelsY : %d pixels\n" - "Max Frames Per File : %u\n" - "Total Frames : %lld\n" - "Exptime (ns) : %lld\n" - "SubExptime (ns) : %lld\n" - "SubPeriod(ns) : %lld\n" - "Period (ns) : %lld\n" - "Quad Enable : %d\n" - "Analog Flag : %d\n" - "Digital Flag : %d\n" - "ADC Mask : %d\n" - "Dbit Offset : %d\n" - "Dbit Bitset : %lld\n" - "Roi (xmin, xmax) : %d %d\n" - "Timestamp : %s\n\n" - - "#Frame Header\n" - "Frame Number : 8 bytes\n" - "SubFrame Number/ExpLength : 4 bytes\n" - "Packet Number : 4 bytes\n" - "Bunch ID : 8 bytes\n" - "Timestamp : 8 bytes\n" - "Module Id : 2 bytes\n" - "Row : 2 bytes\n" - "Column : 2 bytes\n" - "Reserved : 2 bytes\n" - "Debug : 4 bytes\n" - "Round Robin Number : 2 bytes\n" - "Detector Type : 1 byte\n" - "Header Version : 1 byte\n" - "Packets Caught Mask : 64 bytes\n" - , - attr.version, - attr.detectorType, - attr.dynamicRange, - attr.tenGiga, - attr.imageSize, - attr.nPixelsX, - attr.nPixelsY, - attr.maxFramesPerFile, - (long long int)attr.totalFrames, - (long long int)attr.exptimeNs, - (long long int)attr.subExptimeNs, - (long long int)attr.subPeriodNs, - (long long int)attr.periodNs, - attr.quadEnable, - attr.analogFlag, - attr.digitalFlag, - attr.adcmask, - attr.dbitoffset, - (long long int)attr.dbitlist, - attr.roiXmin, - attr.roiXmax, - ctime(&t)); - if (strlen(message) > MAX_MASTER_FILE_LENGTH) { - throw sls::RuntimeError("Master File Size " + std::to_string(strlen(message)) + - " is greater than max str size " + std::to_string(MAX_MASTER_FILE_LENGTH)); - } - - if (fwrite((void*)message, 1, strlen(message), fd) != strlen(message)) { - throw sls::RuntimeError("Master binary file incorrect number of bytes written to file"); - } - - BinaryFileStatic::CloseDataFile(fd); - } - - - /** - * Create File - * @param fd file pointer - * @param owenable overwrite enable - * @param fname complete file name - * @returns 0 for success and 1 for fail - */ - static void CreateDataFile(FILE*& fd, bool owenable, std::string fname) - { - if(!owenable){ - if (NULL == (fd = fopen((const char *) fname.c_str(), "wx"))){ - fd = 0; - throw sls::RuntimeError("Could not create/overwrite file " + fname); - } - } else if (NULL == (fd = fopen((const char *) fname.c_str(), "w"))){ - fd = 0; - throw sls::RuntimeError("Could not create file " + fname); - } - //setting to no file buffering - setvbuf(fd, NULL, _IONBF, 0); - } - -}; - diff --git a/slsReceiverSoftware/src/DataProcessor.cpp b/slsReceiverSoftware/src/DataProcessor.cpp index a3fe3e8d5..4d53b4cd3 100755 --- a/slsReceiverSoftware/src/DataProcessor.cpp +++ b/slsReceiverSoftware/src/DataProcessor.cpp @@ -115,8 +115,9 @@ void DataProcessor::SetFileFormat(const fileFormat f) { //remember the pointer values before they are destroyed int nd[MAX_DIMENSIONS];nd[0] = 0; nd[1] = 0; uint32_t* maxf = nullptr; - std::string* fname=nullptr; std::string* fpath=nullptr; uint64_t* findex=nullptr; - bool* owenable=nullptr; int* dindex=nullptr; int* nunits=nullptr; uint64_t* nf = nullptr; + std::string* fname=nullptr; std::string* fpath=nullptr; + uint64_t* findex=nullptr; bool* owenable=nullptr; int* dindex=nullptr; + int* nunits=nullptr; uint64_t* nf = nullptr; uint32_t* dr = nullptr; uint32_t* port = nullptr; file->GetMemberPointerValues(nd, maxf, fname, fpath, findex, owenable, dindex, nunits, nf, dr, port); @@ -139,7 +140,8 @@ void DataProcessor::SetupFileWriter(bool fwe, int* nd, uint32_t* maxf, if (file != nullptr) { - delete file; file = nullptr; + delete file; + file = nullptr; } if (fileWriteEnable) { diff --git a/slsReceiverSoftware/src/File.cpp b/slsReceiverSoftware/src/File.cpp index 83bfb0d59..ff04ae379 100755 --- a/slsReceiverSoftware/src/File.cpp +++ b/slsReceiverSoftware/src/File.cpp @@ -9,11 +9,12 @@ #include -File::File(int ind, uint32_t* maxf, - int* nd, std::string* fname, std::string* fpath, uint64_t* findex, bool* owenable, - int* dindex, int* nunits, uint64_t* nf, uint32_t* dr, uint32_t* portno, - bool* smode): +File::File(int ind, slsDetectorDefs::fileFormat type, uint32_t* maxf, + int* nd, std::string* fname, std::string* fpath, uint64_t* findex, + bool* owenable, int* dindex, int* nunits, uint64_t* nf, uint32_t* dr, + uint32_t* portno, bool* smode): index(ind), + formatType(type), maxFramesPerFile(maxf), numDetX(nd[0]), numDetY(nd[1]), @@ -29,11 +30,15 @@ File::File(int ind, uint32_t* maxf, silentMode(smode) { - master = index?false:true; + master = ((index == 0) && (*detIndex == 0)) ? true : false; } File::~File() {} +slsDetectorDefs::fileFormat File::GetFileType() { + return formatType; +} + std::string File::GetCurrentFileName() { return currentFileName; } diff --git a/slsReceiverSoftware/src/File.h b/slsReceiverSoftware/src/File.h index ea64aab33..5ef3b549c 100755 --- a/slsReceiverSoftware/src/File.h +++ b/slsReceiverSoftware/src/File.h @@ -22,6 +22,7 @@ class File : private virtual slsDetectorDefs { * Constructor * creates the File Writer * @param ind self index + * @param type file format type * @param maxf pointer to max frames per file * @param nd pointer to number of detectors in each dimension * @param fname pointer to file name prefix @@ -35,35 +36,17 @@ class File : private virtual slsDetectorDefs { * @param portno pointer to udp port number for logging * @param smode pointer to silent mode */ - File(int ind, uint32_t* maxf, - int* nd, std::string* fname, std::string* fpath, uint64_t* findex, bool* owenable, - int* dindex, int* nunits, uint64_t* nf, uint32_t* dr, uint32_t* portno, - bool* smode); + File(int ind, slsDetectorDefs::fileFormat type, uint32_t* maxf, + int* nd, std::string* fname, std::string* fpath, uint64_t* findex, + bool* owenable, int* dindex, int* nunits, uint64_t* nf, uint32_t* dr, + uint32_t* portno, bool* smode); - /** - * Destructor - */ virtual ~File(); - - /** - * Get Current File Name - * @returns current file name - */ + fileFormat GetFileType(); std::string GetCurrentFileName(); - - void resetSubFileIndex(); - /** - * Print all member values - */ virtual void PrintMembers(TLogLevel level = logDEBUG1); - /** - * Get Type - * @return type - */ - virtual fileFormat GetFileType() = 0; - /** * Get Member Pointer Values before the object is destroyed * @param nd pointer to number of detectors in each dimension @@ -78,23 +61,12 @@ class File : private virtual slsDetectorDefs { * @param dr pointer to dynamic range * @param portno pointer to dynamic range */ - void GetMemberPointerValues(int* nd, uint32_t*& maxf, std::string*& fname, std::string*& fpath, - uint64_t*& findex, bool*& owenable, - int*& dindex, int*& nunits, uint64_t*& nf, uint32_t*& dr, uint32_t*& portno); + void GetMemberPointerValues(int* nd, uint32_t*& maxf, std::string*& fname, + std::string*& fpath, uint64_t*& findex, bool*& owenable, int*& dindex, + int*& nunits, uint64_t*& nf, uint32_t*& dr, uint32_t*& portno); - /** - * Create file - */ virtual void CreateFile() = 0; - - /** - * Close Current File - */ virtual void CloseCurrentFile() = 0; - - /** - * Close Files - */ virtual void CloseAllFiles() = 0; /** @@ -123,7 +95,6 @@ class File : private virtual slsDetectorDefs { "should be overloaded by a derived class"; } - /** * End of Acquisition * @param anyPacketsCaught true if any packets are caught, else false @@ -135,60 +106,24 @@ class File : private virtual slsDetectorDefs { } protected: - - /** master file writer/reader */ bool master; - - /** Self Index */ int index; - - /** Maximum frames per file */ + slsDetectorDefs::fileFormat formatType; uint32_t* maxFramesPerFile; - - /** Master File Name */ std::string masterFileName; - - /** Current File Name */ std::string currentFileName; - - /** Number of Detectors in X dimension */ int numDetX; - - /** Number of Detectors in Y dimension */ int numDetY; - - /** File Name Prefix */ std::string* fileNamePrefix; - - /** File Path */ std::string* filePath; - - /** File Index */ uint64_t* fileIndex; - - /** Sub file index */ uint64_t subFileIndex{0}; - - /** Over write enable */ bool* overWriteEnable; - - /** Detector Index */ int* detIndex; - - /** Number of units per detector. Eg. Eiger has 2, others 1 */ int* numUnitsPerDetector; - - /** Number of images in acquisition */ uint64_t* numImages; - - /** Dynamic Range */ uint32_t* dynamicRange; - - /** UDP Port Number for logging */ uint32_t* udpPortNumber; - - /** Silent Mode */ bool* silentMode; - }; diff --git a/slsReceiverSoftware/src/HDF5File.cpp b/slsReceiverSoftware/src/HDF5File.cpp index e27d78fe7..ef319754b 100755 --- a/slsReceiverSoftware/src/HDF5File.cpp +++ b/slsReceiverSoftware/src/HDF5File.cpp @@ -13,9 +13,7 @@ #include -H5File* HDF5File::masterfd = 0; -hid_t HDF5File::virtualfd = 0; - +std::mutex HDF5File::hdf5Lib; HDF5File::HDF5File(int ind, uint32_t* maxf, @@ -24,7 +22,9 @@ HDF5File::HDF5File(int ind, uint32_t* maxf, uint32_t nx, uint32_t ny, bool* smode): - File(ind, maxf, nd, fname, fpath, findex, owenable, dindex, nunits, nf, dr, portno, smode), + File(ind, HDF5, maxf, nd, fname, fpath, findex, owenable, dindex, nunits, nf, dr, portno, smode), + masterfd(0), + virtualfd(0), filefd(0), dataspace(0), dataset(0), @@ -84,101 +84,72 @@ HDF5File::HDF5File(int ind, uint32_t* maxf, parameterNames.push_back("packets caught bit mask"); StrType strdatatype(PredType::C_S1, sizeof(bitset_storage)); parameterDataTypes.push_back(strdatatype); - } - HDF5File::~HDF5File() { CloseAllFiles(); } -void HDF5File::PrintMembers(TLogLevel level) { - File::PrintMembers(); - UpdateDataType(); - if (datatype == PredType::STD_U8LE) { - LOG(level) << "Data Type: 4 or 8"; - } else if (datatype == PredType::STD_U16LE) { - LOG(level) << "Data Type: 16"; - } else if (datatype == PredType::STD_U32LE) { - LOG(level) << "Data Type: 32"; - } else { - LOG(logERROR) << "unknown data type"; - } -} - - void HDF5File::SetNumberofPixels(uint32_t nx, uint32_t ny) { nPixelsX = nx; nPixelsY = ny; } - -slsDetectorDefs::fileFormat HDF5File::GetFileType() { - return HDF5; -} - - -void HDF5File::UpdateDataType() { - switch(*dynamicRange){ - case 16: datatype = PredType::STD_U16LE; break; - case 32: datatype = PredType::STD_U32LE; break; - default: datatype = PredType::STD_U8LE; break; - } -} - - void HDF5File::CreateFile() { numFilesinAcquisition++; numFramesInFile = 0; numActualPacketsInFile = 0; - currentFileName = HDF5FileStatic::CreateFileName(*filePath, *fileNamePrefix, *fileIndex, - subFileIndex, *detIndex, *numUnitsPerDetector, index); //first time - if(subFileIndex == 0u) - UpdateDataType(); - - uint64_t framestosave = ((*maxFramesPerFile == 0) ? *numImages : // infinite images - (((extNumImages - subFileIndex) > (*maxFramesPerFile)) ? // save up to maximum at a time - (*maxFramesPerFile) : (extNumImages-subFileIndex))); - - std::lock_guard lock(mutex); - HDF5FileStatic::CreateDataFile(index, *overWriteEnable, currentFileName, (*numImages > 1), - subFileIndex, framestosave, nPixelsY, ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX), - datatype, filefd, dataspace, dataset, - HDF5_WRITER_VERSION, MAX_CHUNKED_IMAGES, - dataspace_para, dataset_para, - parameterNames, parameterDataTypes); - - if(!(*silentMode)) { - LOG(logINFO) << *udpPortNumber << ": HDF5 File created: " << currentFileName; + if(subFileIndex == 0u) { + switch(*dynamicRange){ + case 16: + datatype = PredType::STD_U16LE; + break; + case 32: + datatype = PredType::STD_U32LE; + break; + default: + datatype = PredType::STD_U8LE; + break; + } } + CreateDataFile(); } - void HDF5File::CloseCurrentFile() { - { - std::lock_guard lock(mutex); - HDF5FileStatic::CloseDataFile(index, filefd); - } + CloseFile(filefd, false); for (unsigned int i = 0; i < dataset_para.size(); ++i) delete dataset_para[i]; dataset_para.clear(); - if(dataspace_para) {delete dataspace_para;dataspace_para=0;} - if(dataset) {delete dataset;dataset=0;} - if(dataspace) {delete dataspace;dataspace=0;} - if(filefd) {delete filefd;filefd=0;} + if(dataspace_para) { + delete dataspace_para; + dataspace_para = 0; + } + if(dataset) { + delete dataset; + dataset = 0; + } + if(dataspace) { + delete dataspace; + dataspace = 0; + } } - void HDF5File::CloseAllFiles() { numFilesinAcquisition = 0; { - std::lock_guard lock(mutex); - HDF5FileStatic::CloseDataFile(index, filefd); - if (master && (*detIndex==0)) { - HDF5FileStatic::CloseMasterDataFile(masterfd); - HDF5FileStatic::CloseVirtualDataFile(virtualfd); + CloseFile(filefd, false); + if (master) { + CloseFile(masterfd, true); + // close virtual file + // c code due to only c implementation of H5Pset_virtual available + if (virtualfd != 0) { + if (H5Fclose(virtualfd) < 0 ) { + LOG(logERROR) << "Could not close virtual HDF5 handles"; + } + virtualfd = 0; + } } } for (unsigned int i = 0; i < dataset_para.size(); ++i) @@ -187,11 +158,9 @@ void HDF5File::CloseAllFiles() { if(dataspace_para) delete dataspace_para; if(dataset) delete dataset; if(dataspace) delete dataspace; - if(filefd) delete filefd; } - -void HDF5File::WriteToFile(char* buffer, int buffersize, uint64_t fnum, uint32_t nump) { +void HDF5File::WriteToFile(char* buffer, int bufferSize, uint64_t currentFrameNumber, uint32_t numPacketsCaught) { // check if maxframesperfile = 0 for infinite if ((*maxFramesPerFile) && (numFramesInFile >= (*maxFramesPerFile))) { @@ -200,107 +169,801 @@ void HDF5File::WriteToFile(char* buffer, int buffersize, uint64_t fnum, uint32_t CreateFile(); } numFramesInFile++; - numActualPacketsInFile += nump; - - std::lock_guard lock(mutex); + numActualPacketsInFile += numPacketsCaught; // extend dataset (when receiver start followed by many status starts (jungfrau))) - if (fnum >= extNumImages) { - HDF5FileStatic::ExtendDataset(index, dataspace, dataset, - dataspace_para, dataset_para, *numImages); - if (!(*silentMode)) { - LOG(logINFO) << index << " Extending HDF5 dataset by " << - extNumImages << ", Total x Dimension: " << (extNumImages + *numImages); - } - extNumImages += *numImages; + if (currentFrameNumber >= extNumImages) { + ExtendDataset(); } - HDF5FileStatic::WriteDataFile(index, buffer + sizeof(sls_receiver_header), - // infinite then no need for %maxframesperfile - ((*maxFramesPerFile == 0) ? fnum : fnum%(*maxFramesPerFile)), - nPixelsY, ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX), - dataspace, dataset, datatype); - - HDF5FileStatic::WriteParameterDatasets(index, dataspace_para, - // infinite then no need for %maxframesperfile - ((*maxFramesPerFile == 0) ? fnum : fnum%(*maxFramesPerFile)), - dataset_para, (sls_receiver_header*) (buffer), - parameterDataTypes); + WriteDataFile(currentFrameNumber, buffer + sizeof(sls_receiver_header)); + WriteParameterDatasets(currentFrameNumber, (sls_receiver_header*) (buffer)); } - -void HDF5File::CreateMasterFile(bool mfwenable, masterAttributes& attr) { +void HDF5File::CreateMasterFile(bool masterFileWriteEnable, masterAttributes& masterFileAttributes) { //beginning of every acquisition numFramesInFile = 0; numActualPacketsInFile = 0; extNumImages = *numImages; - if (mfwenable && master && (*detIndex==0)) { + if (masterFileWriteEnable && master) { virtualfd = 0; - masterFileName = HDF5FileStatic::CreateMasterFileName(*filePath, - *fileNamePrefix, *fileIndex); - if(!(*silentMode)) { - LOG(logINFO) << "Master File: " << masterFileName; - } - std::lock_guard lock(mutex); - attr.version = HDF5_WRITER_VERSION; - HDF5FileStatic::CreateMasterDataFile(masterfd, masterFileName, - *overWriteEnable, attr); + CreateMasterDataFile(masterFileAttributes); } } - -void HDF5File::EndofAcquisition(bool anyPacketsCaught, uint64_t numf) { +void HDF5File::EndofAcquisition(bool anyPacketsCaught, uint64_t numImagesCaught) { //not created before if (!virtualfd && anyPacketsCaught) { - // called only by the one maser receiver - if (master && (*detIndex==0)) { - + if (master && masterfd != nullptr) { //only one file and one sub image (link current file in master) if (((numFilesinAcquisition == 1) && (numDetY*numDetX) == 1)) { - LinkVirtualFileinMasterFile(); + //dataset name + std::ostringstream oss; + oss << "/data"; + if ((*numImages > 1)) { + oss << "_f" << std::setfill('0') << std::setw(12) << 0; + } + std::string dsetname = oss.str(); + + LinkVirtualInMaster(currentFileName, dsetname); } //create virutal file else{ - CreateVirtualFile(numf);} + CreateVirtualDataFile( + // infinite images in 1 file, then maxfrperfile = numImagesCaught + ((*maxFramesPerFile == 0) ? + numImagesCaught+1 : *maxFramesPerFile), + numImagesCaught + 1); + } } } numFilesinAcquisition = 0; } +void HDF5File::CloseFile(H5File*& fd, bool masterFile) { + std::lock_guard lock(HDF5File::hdf5Lib); + try { + Exception::dontPrint(); // to handle errors + if (fd) { + fd->close(); + delete fd; + fd = 0; + } + } catch(const Exception& error) { + LOG(logERROR) << "Could not close " + << (masterFile ? "master" : "data") + << " HDF5 handles of index " << index; + error.printErrorStack(); + } +} -// called only by the one maser receiver -void HDF5File::CreateVirtualFile(uint64_t numf) { - std::lock_guard lock(mutex); +void HDF5File::WriteDataFile(uint64_t currentFrameNumber, char* buffer) { + std::lock_guard lock(HDF5File::hdf5Lib); + + uint64_t nDimx = ((*maxFramesPerFile == 0) ? + currentFrameNumber : currentFrameNumber % (*maxFramesPerFile)); + uint32_t nDimy = nPixelsY; + uint32_t nDimz = ((*dynamicRange == 4) ? (nPixelsX / 2) : nPixelsX); + + hsize_t count[3] = {1, nDimy, nDimz}; + hsize_t start[3] = {nDimx, 0, 0}; + hsize_t dims2[2] = {nDimy, nDimz}; + try{ + Exception::dontPrint(); //to handle errors + + dataspace->selectHyperslab( H5S_SELECT_SET, count, start); + DataSpace memspace(2,dims2); + dataset->write(buffer, datatype, memspace, *dataspace); + memspace.close(); + } + catch(const Exception& error){ + LOG(logERROR) << "Could not write to file in object " << index; + error.printErrorStack(); + throw sls::RuntimeError("Could not write to file in object " + + std::to_string(index)); + } +} + +void HDF5File::WriteParameterDatasets(uint64_t currentFrameNumber, + sls_receiver_header* rheader) { + std::lock_guard lock(HDF5File::hdf5Lib); + + uint64_t fnum = ((*maxFramesPerFile == 0) ? + currentFrameNumber : currentFrameNumber % (*maxFramesPerFile)); + + sls_detector_header header = rheader->detHeader; + hsize_t count[1] = {1}; + hsize_t start[1] = {fnum}; + int i = 0; + try{ + Exception::dontPrint(); //to handle errors + dataspace_para->selectHyperslab( H5S_SELECT_SET, count, start); + DataSpace memspace(H5S_SCALAR); + dataset_para[0]->write(&header.frameNumber, + parameterDataTypes[0], memspace, *dataspace_para);i=1; + dataset_para[1]->write(&header.expLength, + parameterDataTypes[1], memspace, *dataspace_para);i=2; + dataset_para[2]->write(&header.packetNumber, + parameterDataTypes[2], memspace, *dataspace_para);i=3; + dataset_para[3]->write(&header.bunchId, + parameterDataTypes[3], memspace, *dataspace_para);i=4; + dataset_para[4]->write(&header.timestamp, + parameterDataTypes[4], memspace, *dataspace_para);i=5; + dataset_para[5]->write(&header.modId, + parameterDataTypes[5], memspace, *dataspace_para);i=6; + dataset_para[6]->write(&header.row, + parameterDataTypes[6], memspace, *dataspace_para);i=7; + dataset_para[7]->write(&header.column, + parameterDataTypes[7], memspace, *dataspace_para);i=8; + dataset_para[8]->write(&header.reserved, + parameterDataTypes[8], memspace, *dataspace_para);i=9; + dataset_para[9]->write(&header.debug, + parameterDataTypes[9], memspace, *dataspace_para);i=10; + dataset_para[10]->write(&header.roundRNumber, + parameterDataTypes[10], memspace, *dataspace_para);i=11; + dataset_para[11]->write(&header.detType, + parameterDataTypes[11], memspace, *dataspace_para);i=12; + dataset_para[12]->write(&header.version, + parameterDataTypes[12], memspace, *dataspace_para);i=13; + + // contiguous bitset + if (sizeof(sls_bitset) == sizeof(bitset_storage)) { + dataset_para[13]->write((char*)&(rheader->packetsMask), + parameterDataTypes[13], memspace, *dataspace_para); + } + + // not contiguous bitset + else { + // get contiguous representation of bit mask + bitset_storage storage; + memset(storage, 0 , sizeof(bitset_storage)); + sls_bitset bits = rheader->packetsMask; + for (int i = 0; i < MAX_NUM_PACKETS; ++i) + storage[i >> 3] |= (bits[i] << (i & 7)); + // write bitmask + dataset_para[13]->write((char*)storage, parameterDataTypes[13], + memspace, *dataspace_para); + }i=14; + } + catch(const Exception& error){ + error.printErrorStack(); + throw sls::RuntimeError("Could not write parameters (index:" + + std::to_string(i) + ") to file in object " + std::to_string(index)); + } +} + +void HDF5File::ExtendDataset() { + std::lock_guard lock(HDF5File::hdf5Lib); + + try{ + Exception::dontPrint(); //to handle errors + + hsize_t dims[3]; + dataspace->getSimpleExtentDims(dims); + dims[0] += *numImages; + + dataset->extend(dims); + delete dataspace; + dataspace = 0; + dataspace = new DataSpace(dataset->getSpace()); + + hsize_t dims_para[1] = {dims[0]}; + for (unsigned int i = 0; i < dataset_para.size(); ++i) + dataset_para[i]->extend(dims_para); + delete dataspace_para; + dataspace_para = 0; + dataspace_para = new DataSpace(dataset_para[0]->getSpace()); + + } + catch(const Exception& error){ + error.printErrorStack(); + throw sls::RuntimeError("Could not extend dataset in object " + + std::to_string(index)); + } + if (!(*silentMode)) { + LOG(logINFO) << index << " Extending HDF5 dataset by " << + extNumImages << ", Total x Dimension: " << (extNumImages + *numImages); + } + extNumImages += *numImages; +} + +void HDF5File::CreateDataFile() { + + std::ostringstream os; + os << *filePath << "/" << *fileNamePrefix << "_d" + << (*detIndex * (*numUnitsPerDetector) + index) << "_f" << subFileIndex << '_' + << *fileIndex << ".h5"; + currentFileName = os.str(); + + std::lock_guard lock(HDF5File::hdf5Lib); + + uint64_t framestosave = ((*maxFramesPerFile == 0) ? *numImages : // infinite images + (((extNumImages - subFileIndex) > (*maxFramesPerFile)) ? // save up to maximum at a time + (*maxFramesPerFile) : (extNumImages-subFileIndex))); + + uint64_t nDimx = framestosave; + uint32_t nDimy = nPixelsY; + uint32_t nDimz = ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX); + + try { + Exception::dontPrint(); //to handle errors + + //file + FileAccPropList fapl; + fapl.setFcloseDegree(H5F_CLOSE_STRONG); + filefd = 0; + if(!(*overWriteEnable)) + filefd = new H5File( currentFileName.c_str(), H5F_ACC_EXCL, + FileCreatPropList::DEFAULT, + fapl ); + else + filefd = new H5File( currentFileName.c_str(), H5F_ACC_TRUNC, + FileCreatPropList::DEFAULT, + fapl ); + + //attributes - version + double dValue = HDF5_WRITER_VERSION; + DataSpace dataspace_attr = DataSpace (H5S_SCALAR); + Attribute attribute = filefd->createAttribute("version", + PredType::NATIVE_DOUBLE, dataspace_attr); + attribute.write(PredType::NATIVE_DOUBLE, &dValue); + + //dataspace + hsize_t srcdims[3] = {nDimx, nDimy, nDimz}; + hsize_t srcdimsmax[3] = {H5S_UNLIMITED, nDimy, nDimz}; + dataspace = 0; + dataspace = new DataSpace (3,srcdims,srcdimsmax); + + + //dataset name + std::ostringstream osfn; + osfn << "/data"; + if (*numImages > 1) + osfn << "_f" << std::setfill('0') << std::setw(12) << subFileIndex; + std::string dsetname = osfn.str(); + + //dataset + //fill value + DSetCreatPropList plist; + int fill_value = -1; + plist.setFillValue(datatype, &fill_value); + // always create chunked dataset as unlimited is only + // supported with chunked layout + hsize_t chunk_dims[3] = {MAX_CHUNKED_IMAGES, nDimy, nDimz}; + plist.setChunk(3, chunk_dims); + dataset = 0; + dataset = new DataSet (filefd->createDataSet(dsetname.c_str(), datatype, + *dataspace, plist)); + + //create parameter datasets + hsize_t dims[1] = {nDimx}; + hsize_t dimsmax[1] = {H5S_UNLIMITED}; + dataspace_para = 0; + dataspace_para = new DataSpace (1,dims,dimsmax); + + // always create chunked dataset as unlimited is only + // supported with chunked layout + DSetCreatPropList paralist; + hsize_t chunkpara_dims[3] = {MAX_CHUNKED_IMAGES}; + paralist.setChunk(1, chunkpara_dims); + + for (unsigned int i = 0; i < parameterNames.size(); ++i){ + DataSet* ds = new DataSet(filefd->createDataSet(parameterNames[i], + parameterDataTypes[i], *dataspace_para, paralist)); + dataset_para.push_back(ds); + } + } + catch(const Exception& error){ + error.printErrorStack(); + if (filefd) { + filefd->close(); + } + throw sls::RuntimeError("Could not create HDF5 handles in object " + index); + } + if(!(*silentMode)) { + LOG(logINFO) << *udpPortNumber << ": HDF5 File created: " << currentFileName; + } +} + +void HDF5File::CreateMasterDataFile(masterAttributes& masterFileAttributes) { + + std::ostringstream os; + os << *filePath << "/" << *fileNamePrefix << "_master" + << "_" << *fileIndex << ".h5"; + masterFileName = os.str(); + + if(!(*silentMode)) { + LOG(logINFO) << "Master File: " << masterFileName; + } + masterFileAttributes.version = HDF5_WRITER_VERSION; + + std::lock_guard lock(HDF5File::hdf5Lib); + + try { + Exception::dontPrint(); //to handle errors + + FileAccPropList flist; + flist.setFcloseDegree(H5F_CLOSE_STRONG); + masterfd = 0; + if(!(*overWriteEnable)) + masterfd = new H5File( masterFileName.c_str(), H5F_ACC_EXCL, + FileCreatPropList::DEFAULT, + flist ); + else + masterfd = new H5File( masterFileName.c_str(), H5F_ACC_TRUNC, + FileCreatPropList::DEFAULT, + flist ); + + //create attributes + //version + { + double dValue = masterFileAttributes.version; + DataSpace dataspace = DataSpace (H5S_SCALAR); + Attribute attribute = masterfd->createAttribute("version",PredType::NATIVE_DOUBLE, dataspace); + attribute.write(PredType::NATIVE_DOUBLE, &dValue); + } + //Create a group in the file + Group group1( masterfd->createGroup( "entry" ) ); + Group group2( group1.createGroup("data") ); + Group group3( group1.createGroup("instrument") ); + Group group4( group3.createGroup("beam") ); + Group group5( group3.createGroup("detector") ); + Group group6( group1.createGroup("sample") ); + + //Dynamic Range + { + DataSpace dataspace = DataSpace (H5S_SCALAR); + DataSet dataset = group5.createDataSet ( "dynamic range", PredType::NATIVE_INT, dataspace ); + dataset.write ( &(masterFileAttributes.dynamicRange), PredType::NATIVE_INT); + DataSpace dataspaceAttr = DataSpace (H5S_SCALAR); + StrType strdatatype(PredType::C_S1,256); + Attribute attribute = dataset.createAttribute("unit",strdatatype, dataspaceAttr); + attribute.write(strdatatype, std::string("bits")); + } + + //Ten Giga + { + DataSpace dataspace = DataSpace (H5S_SCALAR); + int iValue = masterFileAttributes.tenGiga; + DataSet dataset = group5.createDataSet ( "ten giga enable", PredType::NATIVE_INT, dataspace ); + dataset.write ( &iValue, PredType::NATIVE_INT); + } + + //Image Size + { + DataSpace dataspace = DataSpace (H5S_SCALAR); + DataSet dataset = group5.createDataSet ( "image size", PredType::NATIVE_INT, dataspace ); + dataset.write ( &(masterFileAttributes.imageSize), PredType::NATIVE_INT); + DataSpace dataspaceAttr = DataSpace (H5S_SCALAR); + StrType strdatatype(PredType::C_S1,256); + Attribute attribute = dataset.createAttribute("unit",strdatatype, dataspaceAttr); + attribute.write(strdatatype, std::string("bytes")); + } + + //x + { + DataSpace dataspace = DataSpace (H5S_SCALAR); + DataSet dataset = group5.createDataSet ( "number of pixels in x axis", PredType::NATIVE_INT, dataspace ); + dataset.write ( &(masterFileAttributes.nPixelsX), PredType::NATIVE_INT); + } + + //y + { + DataSpace dataspace = DataSpace (H5S_SCALAR); + DataSet dataset = group5.createDataSet ( "number of pixels in y axis", PredType::NATIVE_INT, dataspace ); + dataset.write ( &(masterFileAttributes.nPixelsY), PredType::NATIVE_INT); + } + + //Maximum frames per file + { + DataSpace dataspace = DataSpace (H5S_SCALAR); + DataSet dataset = group5.createDataSet ( "maximum frames per file", PredType::NATIVE_INT, dataspace ); + dataset.write ( &(masterFileAttributes.maxFramesPerFile), PredType::NATIVE_INT); + } + + //Total Frames + { + DataSpace dataspace = DataSpace (H5S_SCALAR); + DataSet dataset = group5.createDataSet ( "total frames", PredType::STD_U64LE, dataspace ); + dataset.write ( &(masterFileAttributes.totalFrames), PredType::STD_U64LE); + } + + //Exptime + { + DataSpace dataspace = DataSpace (H5S_SCALAR); + DataSet dataset = group5.createDataSet ( "exposure time", PredType::STD_U64LE, dataspace ); + dataset.write ( &(masterFileAttributes.exptimeNs), PredType::STD_U64LE); + DataSpace dataspaceAttr = DataSpace (H5S_SCALAR); + StrType strdatatype(PredType::C_S1,256); + Attribute attribute = dataset.createAttribute("unit",strdatatype, dataspaceAttr); + attribute.write(strdatatype, std::string("ns")); + } + + //SubExptime + { + DataSpace dataspace = DataSpace (H5S_SCALAR); + DataSet dataset = group5.createDataSet ( "sub exposure time", PredType::STD_U64LE, dataspace ); + dataset.write ( &(masterFileAttributes.subExptimeNs), PredType::STD_U64LE); + DataSpace dataspaceAttr = DataSpace (H5S_SCALAR); + StrType strdatatype(PredType::C_S1,256); + Attribute attribute = dataset.createAttribute("unit",strdatatype, dataspaceAttr); + attribute.write(strdatatype, std::string("ns")); + } + + //SubPeriod + { + DataSpace dataspace = DataSpace (H5S_SCALAR); + DataSet dataset = group5.createDataSet ( "sub period", PredType::STD_U64LE, dataspace ); + dataset.write ( &(masterFileAttributes.subPeriodNs), PredType::STD_U64LE); + DataSpace dataspaceAttr = DataSpace (H5S_SCALAR); + StrType strdatatype(PredType::C_S1,256); + Attribute attribute = dataset.createAttribute("unit",strdatatype, dataspaceAttr); + attribute.write(strdatatype, std::string("ns")); + } + + //Period + { + DataSpace dataspace = DataSpace (H5S_SCALAR); + DataSet dataset = group5.createDataSet ( "acquisition period", PredType::STD_U64LE, dataspace ); + dataset.write ( &(masterFileAttributes.periodNs), PredType::STD_U64LE); + DataSpace dataspaceAttr = DataSpace (H5S_SCALAR); + StrType strdatatype(PredType::C_S1,256); + Attribute attribute = dataset.createAttribute("unit",strdatatype, dataspaceAttr); + attribute.write(strdatatype, std::string("ns")); + } + + //Quad Enable + { + DataSpace dataspace = DataSpace (H5S_SCALAR); + DataSet dataset = group5.createDataSet ( "quad enable", PredType::NATIVE_INT, dataspace ); + dataset.write ( &(masterFileAttributes.quadEnable), PredType::NATIVE_INT); + } + + //Analog Flag + { + DataSpace dataspace = DataSpace (H5S_SCALAR); + DataSet dataset = group5.createDataSet ( "analog flag", PredType::NATIVE_INT, dataspace ); + dataset.write ( &(masterFileAttributes.analogFlag), PredType::NATIVE_INT); + } + + //Digital Flag + { + DataSpace dataspace = DataSpace (H5S_SCALAR); + DataSet dataset = group5.createDataSet ( "digital flag", PredType::NATIVE_INT, dataspace ); + dataset.write ( &(masterFileAttributes.digitalFlag), PredType::NATIVE_INT); + } + + //ADC Mask + { + DataSpace dataspace = DataSpace (H5S_SCALAR); + DataSet dataset = group5.createDataSet ( "adc mask", PredType::NATIVE_INT, dataspace ); + dataset.write ( &(masterFileAttributes.adcmask), PredType::NATIVE_INT); + } + + //Dbit Offset + { + DataSpace dataspace = DataSpace (H5S_SCALAR); + DataSet dataset = group5.createDataSet ( "dbit offset", PredType::NATIVE_INT, dataspace ); + dataset.write ( &(masterFileAttributes.dbitoffset), PredType::NATIVE_INT); + } + + // Dbit List + { + DataSpace dataspace = DataSpace (H5S_SCALAR); + DataSet dataset = group5.createDataSet ( "dbit bitset list", PredType::STD_U64LE, dataspace ); + dataset.write ( &(masterFileAttributes.dbitlist), PredType::STD_U64LE); + } + + // Roi xmin + { + DataSpace dataspace = DataSpace (H5S_SCALAR); + DataSet dataset = group5.createDataSet ( "roi xmin", PredType::NATIVE_INT, dataspace ); + dataset.write ( &(masterFileAttributes.roiXmin), PredType::NATIVE_INT); + } + + // Roi xmax + { + DataSpace dataspace = DataSpace (H5S_SCALAR); + DataSet dataset = group5.createDataSet ( "roi xmax", PredType::NATIVE_INT, dataspace ); + dataset.write ( &(masterFileAttributes.roiXmax), PredType::NATIVE_INT); + } + + //Timestamp + { + time_t t = time(0); + StrType strdatatype(PredType::C_S1,256); + DataSpace dataspace = DataSpace (H5S_SCALAR); + DataSet dataset = group5.createDataSet ( "timestamp", strdatatype, dataspace ); + dataset.write ( std::string(ctime(&t)), strdatatype ); + } + + masterfd->close(); + + } catch(const Exception& error) { + error.printErrorStack(); + if (masterfd) { + masterfd->close(); + } + throw sls::RuntimeError("Could not create master HDF5 handles"); + } +} + +void HDF5File::CreateVirtualDataFile(uint32_t maxFramesPerFile, uint64_t numf) { + + std::ostringstream osfn; + osfn << *filePath << "/" << *fileNamePrefix; + osfn << "_virtual"; + osfn << "_" << *fileIndex; + osfn << ".h5"; + std::string vname = osfn.str(); - std::string vname = HDF5FileStatic::CreateVirtualFileName(*filePath, *fileNamePrefix, *fileIndex); if(!(*silentMode)) { LOG(logINFO) << "Virtual File: " << vname; } - HDF5FileStatic::CreateVirtualDataFile(vname, - virtualfd, masterFileName, - *filePath, *fileNamePrefix, *fileIndex, (*numImages > 1), - *detIndex, *numUnitsPerDetector, - // infinite images in 1 file, then maxfrperfile = numf - ((*maxFramesPerFile == 0) ? numf+1 : *maxFramesPerFile), - numf+1, - "data", datatype, - numDetY, numDetX, nPixelsY, ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX), - HDF5_WRITER_VERSION, - parameterNames, parameterDataTypes); + + int numDetz = numDetX; + uint32_t nDimy = nPixelsY; + uint32_t nDimz = ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX); + + std::lock_guard lock(HDF5File::hdf5Lib); + + try { + //file + hid_t dfal = H5Pcreate (H5P_FILE_ACCESS); + if (dfal < 0) + throw sls::RuntimeError("Could not create file access property for virtual file " + vname); + if (H5Pset_fclose_degree (dfal, H5F_CLOSE_STRONG) < 0) + throw sls::RuntimeError("Could not set strong file close degree for virtual file " + vname); + virtualfd = H5Fcreate( vname.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, dfal); + if (virtualfd < 0) + throw sls::RuntimeError("Could not create virtual file " + vname); + + //attributes - version + hid_t dataspace_attr = H5Screate (H5S_SCALAR); + if (dataspace_attr < 0) + throw sls::RuntimeError("Could not create dataspace for attribute in virtual file " + vname); + hid_t attrid = H5Acreate2 (virtualfd, "version", H5T_NATIVE_DOUBLE, dataspace_attr, H5P_DEFAULT, H5P_DEFAULT); + if (attrid < 0) + throw sls::RuntimeError("Could not create attribute in virtual file " + vname); + double attr_data = HDF5_WRITER_VERSION; + if (H5Awrite (attrid, H5T_NATIVE_DOUBLE, &attr_data) < 0) + throw sls::RuntimeError("Could not write attribute in virtual file " + vname); + if (H5Aclose (attrid) < 0) + throw sls::RuntimeError("Could not close attribute in virtual file " + vname); + + + //virtual dataspace + hsize_t vdsdims[3] = {numf, numDetY * nDimy, numDetz * nDimz}; + hid_t vdsDataspace = H5Screate_simple(3, vdsdims ,NULL); + if (vdsDataspace < 0) + throw sls::RuntimeError("Could not create virtual dataspace in virtual file " + vname); + hsize_t vdsdims_para[2] = {numf, (unsigned int) numDetY * numDetz}; + hid_t vdsDataspace_para = H5Screate_simple(2, vdsdims_para, NULL); + if (vdsDataspace_para < 0) + throw sls::RuntimeError("Could not create virtual dataspace (parameters) in virtual file " + vname); + + + //fill values + hid_t dcpl = H5Pcreate (H5P_DATASET_CREATE); + if (dcpl < 0) + throw sls::RuntimeError("Could not create file creation properties in virtual file " + vname); + int fill_value = -1; + if (H5Pset_fill_value (dcpl, GetDataTypeinC(datatype), &fill_value) < 0) + throw sls::RuntimeError("Could not create fill value in virtual file " + vname); + hid_t dcpl_para[parameterNames.size()]; + for (unsigned int i = 0; i < parameterNames.size(); ++i) { + dcpl_para[i] = H5Pcreate (H5P_DATASET_CREATE); + if (dcpl_para[i] < 0) + throw sls::RuntimeError("Could not create file creation properties (parameters) in virtual file " + vname); + if (H5Pset_fill_value (dcpl_para[i], GetDataTypeinC(parameterDataTypes[i]), &fill_value) < 0) + throw sls::RuntimeError("Could not create fill value (parameters) in virtual file " + vname); + } + + //hyperslab + int numMajorHyperslab = numf/maxFramesPerFile; + if (numf%maxFramesPerFile) numMajorHyperslab++; + uint64_t framesSaved = 0; + for (int j = 0; j < numMajorHyperslab; j++) { + + uint64_t nDimx = ((numf - framesSaved) > maxFramesPerFile) + ? maxFramesPerFile : (numf-framesSaved); + hsize_t offset[3] = {framesSaved, 0, 0}; + hsize_t count[3] = {nDimx, nDimy, nDimz}; + hsize_t offset_para[2] = {framesSaved, 0}; + hsize_t count_para[2] = {nDimx, 1}; + + for (int i = 0; i < numDetY * numDetz; ++i) { + + //setect hyperslabs + if (H5Sselect_hyperslab (vdsDataspace, H5S_SELECT_SET, offset, NULL, count, NULL) < 0) { + throw sls::RuntimeError("Could not select hyperslab"); + } + if (H5Sselect_hyperslab (vdsDataspace_para, H5S_SELECT_SET, + offset_para, NULL, count_para, NULL) < 0) { + throw sls::RuntimeError("Could not select hyperslab for parameters"); + } + + //source file name + std::ostringstream os; + os << *filePath << "/" << *fileNamePrefix << "_d" + << (*detIndex * (*numUnitsPerDetector) + i) << "_f" << j << '_' + << *fileIndex << ".h5"; + std::string srcFileName = os.str(); + + LOG(logDEBUG1) << srcFileName; + // find relative path + std::string relative_srcFileName = srcFileName; + { + size_t i = srcFileName.rfind('/', srcFileName.length()); + if (i != std::string::npos) + relative_srcFileName = (srcFileName.substr(i+1, srcFileName.length() - i)); + } + + //source dataset name + std::ostringstream osfn; + osfn << "/data"; + if (*numImages > 1) + osfn << "_f" << std::setfill('0') << std::setw(12) << j; + std::string srcDatasetName = osfn.str(); + + //source dataspace + hsize_t srcdims[3] = {nDimx, nDimy, nDimz}; + hsize_t srcdimsmax[3] = {H5S_UNLIMITED, nDimy, nDimz}; + hid_t srcDataspace = H5Screate_simple(3, srcdims, srcdimsmax); + if (srcDataspace < 0) + throw sls::RuntimeError("Could not create source dataspace in virtual file " + vname); + hsize_t srcdims_para[1] = {nDimx}; + hsize_t srcdimsmax_para[1] = {H5S_UNLIMITED}; + hid_t srcDataspace_para = H5Screate_simple(1, srcdims_para, srcdimsmax_para); + if (srcDataspace_para < 0) + throw sls::RuntimeError("Could not create source dataspace (parameters) in virtual file " + vname); + + //mapping + if (H5Pset_virtual(dcpl, vdsDataspace, relative_srcFileName.c_str(), + srcDatasetName.c_str(), srcDataspace) < 0) { + throw sls::RuntimeError("Could not set mapping for paramter 1"); + } + + for (unsigned int k = 0; k < parameterNames.size(); ++k) { + if (H5Pset_virtual(dcpl_para[k], vdsDataspace_para, relative_srcFileName.c_str(), + parameterNames[k], srcDataspace_para) < 0) { + throw sls::RuntimeError("Could not set mapping for paramter " + std::to_string(k)); + } + } + + //H5Sclose(srcDataspace); + //H5Sclose(srcDataspace_para); + offset[2] += nDimz; + if (offset[2] >= (numDetz * nDimz)) { + offset[2] = 0; + offset[1] += nDimy; + } + offset_para[1]++; + } + framesSaved += nDimx; + } + + //dataset + std::string virtualDatasetName = "data"; + hid_t vdsdataset = H5Dcreate2 (virtualfd, virtualDatasetName.c_str(), + GetDataTypeinC(datatype), vdsDataspace, H5P_DEFAULT, dcpl, H5P_DEFAULT); + if (vdsdataset < 0) + throw sls::RuntimeError("Could not create virutal dataset in virtual file " + vname); + + + //virtual parameter dataset + for (unsigned int i = 0; i < parameterNames.size(); ++i) { + hid_t vdsdataset_para = H5Dcreate2 (virtualfd, + parameterNames[i], + GetDataTypeinC(parameterDataTypes[i]), vdsDataspace_para, + H5P_DEFAULT, dcpl_para[i], H5P_DEFAULT); + if (vdsdataset_para < 0) + throw sls::RuntimeError("Could not create virutal dataset (parameters) in virtual file " + vname); + } + + //close + H5Fclose(virtualfd); + virtualfd = 0; + + //link + LinkVirtualInMaster(vname, virtualDatasetName); + } catch (const sls::RuntimeError &e) { + if (virtualfd > 0) + H5Fclose(virtualfd); + virtualfd = 0; + } } -// called only by the one maser receiver -void HDF5File::LinkVirtualFileinMasterFile() { - //dataset name - std::ostringstream osfn; - osfn << "/data"; - if ((*numImages > 1)) osfn << "_f" << std::setfill('0') << std::setw(12) << 0; - std::string dsetname = osfn.str(); +void HDF5File::LinkVirtualInMaster(std::string fname, std::string dsetname) { - std::lock_guard lock(mutex); - HDF5FileStatic::LinkVirtualInMaster(masterFileName, currentFileName, - dsetname, parameterNames); + if (fname == currentFileName) { + std::lock_guard lock(HDF5File::hdf5Lib); + } + + char linkname[100]; + hid_t vfd = 0; + + try { + hid_t dfal = H5Pcreate (H5P_FILE_ACCESS); + if (dfal < 0) + throw sls::RuntimeError("Could not create file access property for link"); + if (H5Pset_fclose_degree (dfal, H5F_CLOSE_STRONG) < 0) + throw sls::RuntimeError("Could not set strong file close degree for link"); + + //open master file + hid_t mfd = H5Fopen( masterFileName.c_str(), H5F_ACC_RDWR, dfal); + if (mfd < 0) + throw sls::RuntimeError("Could not open master file"); + + //open virtual file + vfd = H5Fopen( fname.c_str(), H5F_ACC_RDWR, dfal); + if (vfd < 0) { + H5Fclose(mfd); mfd = 0; + throw sls::RuntimeError("Could not open virtual file"); + } + + // find relative path + std::string relative_virtualfname = fname; + { + size_t i = fname.rfind('/', fname.length()); + if (i != std::string::npos) + relative_virtualfname = (fname.substr(i+1, fname.length() - i)); + } + + //**data dataset** + hid_t vdset = H5Dopen2( vfd, dsetname.c_str(), H5P_DEFAULT); + if (vdset < 0) { + H5Fclose(mfd); + throw sls::RuntimeError("Could not open virtual data dataset"); + } + sprintf(linkname, "/entry/data/%s",dsetname.c_str()); + if(H5Lcreate_external( relative_virtualfname.c_str(), dsetname.c_str(), + mfd, linkname, H5P_DEFAULT, H5P_DEFAULT) < 0) { + H5Fclose(mfd); mfd = 0; + throw sls::RuntimeError("Could not create link to data dataset"); + } + H5Dclose(vdset); + + //**paramter datasets** + for (unsigned int i = 0; i < parameterNames.size(); ++i){ + hid_t vdset_para = H5Dopen2( vfd, (std::string (parameterNames[i])).c_str(), H5P_DEFAULT); + if (vdset_para < 0) { + H5Fclose(mfd); mfd = 0; + throw sls::RuntimeError("Could not open virtual parameter dataset to create link"); + } + sprintf(linkname, "/entry/data/%s",(std::string (parameterNames[i])).c_str()); + + if(H5Lcreate_external( relative_virtualfname.c_str(), (std::string (parameterNames[i])).c_str(), + mfd, linkname, H5P_DEFAULT, H5P_DEFAULT) < 0) { + H5Fclose(mfd); mfd = 0; + throw sls::RuntimeError("Could not create link to virtual parameter dataset"); + } + } + + H5Fclose(mfd); mfd = 0; + H5Fclose(vfd); vfd = 0; + } catch (...) { + if(vfd > 0) + H5Fclose(vfd); + vfd = 0; + } } + +hid_t HDF5File::GetDataTypeinC(DataType dtype) { + if (dtype == PredType::STD_U8LE) + return H5T_STD_U8LE; + else if (dtype == PredType::STD_U16LE) + return H5T_STD_U16LE; + else if (dtype == PredType::STD_U32LE) + return H5T_STD_U32LE; + else if (dtype == PredType::STD_U64LE) + return H5T_STD_U64LE; + else { + hid_t s = H5Tcopy(H5T_C_S1); + H5Tset_size(s, MAX_NUM_PACKETS); + return s; + } +} \ No newline at end of file diff --git a/slsReceiverSoftware/src/HDF5File.h b/slsReceiverSoftware/src/HDF5File.h index 722758cc0..5db6596d8 100755 --- a/slsReceiverSoftware/src/HDF5File.h +++ b/slsReceiverSoftware/src/HDF5File.h @@ -11,7 +11,6 @@ #include "File.h" -#include "HDF5FileStatic.h" #include "H5Cpp.h" #ifndef H5_NO_NAMESPACE @@ -20,7 +19,7 @@ #include -class HDF5File : private virtual slsDetectorDefs, public File, public HDF5FileStatic { +class HDF5File : private virtual slsDetectorDefs, public File { public: /** @@ -47,139 +46,49 @@ class HDF5File : private virtual slsDetectorDefs, public File, public HDF5FileSt int* dindex, int* nunits, uint64_t* nf, uint32_t* dr, uint32_t* portno, uint32_t nx, uint32_t ny, bool* smode); - - /** - * Destructor - */ ~HDF5File(); - - /** - * Print all member values - */ - void PrintMembers(TLogLevel level = logDEBUG1); - - /** - * Set Number of pixels - * @param nx number of pixels in x direction - * @param ny number of pixels in y direction - */ void SetNumberofPixels(uint32_t nx, uint32_t ny); - - /** - * Create file - * @param fnum current frame index to include in file name - */ void CreateFile(); - - /** - * Close Current File - */ void CloseCurrentFile(); - - /** - * Close all Files - */ void CloseAllFiles(); - - /** - * Write data to file - * @param buffer buffer to write from - * @param bsize size of buffer (not used) - * @param fnum current image number - * @param nump number of packets caught - */ - void WriteToFile(char* buffer, int bsize, uint64_t fnum, uint32_t nump); - - /** - * Create master file - * @param mfwenable master file write enable - * @param attr master file attributes - */ - void CreateMasterFile(bool mfwenable, masterAttributes& attr); - - /** - * End of Acquisition - * @param anyPacketsCaught true if any packets are caught, else false - * @param numf number of images caught - */ - void EndofAcquisition(bool anyPacketsCaught, uint64_t numf); - + void WriteToFile(char* buffer, int bufferSize, uint64_t currentFrameNumber, uint32_t numPacketsCaught); + void CreateMasterFile(bool masterFileWriteEnable, masterAttributes& masterFileAttributes); + void EndofAcquisition(bool anyPacketsCaught, uint64_t numImagesCaught); private: - /** - * Create Virtual File - * @param numf number of images caught - */ - void CreateVirtualFile(uint64_t numf); + void CloseFile(H5File*& fd, bool masterFile); + void WriteDataFile(uint64_t currentFrameNumber, char* buffer); + void WriteParameterDatasets(uint64_t currentFrameNumber, sls_receiver_header* rheader); + void ExtendDataset(); + void CreateDataFile(); + void CreateMasterDataFile(masterAttributes& masterFileAttributes); + void CreateVirtualDataFile(uint32_t maxFramesPerFile, uint64_t numf); + void LinkVirtualInMaster(std::string fname, std::string dsetname); + hid_t GetDataTypeinC(DataType dtype); - /** - * Link virtual file in master file - * Only for Jungfrau at the moment for 1 module and 1 data file - */ - void LinkVirtualFileinMasterFile(); + static std::mutex hdf5Lib; - /** - * Get Type - * @return type - */ - fileFormat GetFileType(); - - /** - * Updates data type depending on current dynamic range - */ - void UpdateDataType(); - - - /** mutex to update static items among objects (threads)*/ - static mutable std::mutex mutex; - - /** Master File handle */ - static H5File* masterfd; - - /** Virtual File handle ( only file name because code in C as H5Pset_virtual doesnt exist yet in C++) */ - static hid_t virtualfd; - - /** File handle */ + H5File* masterfd; + /** Virtual File handle ( only file name because + code in C as H5Pset_virtual doesnt exist yet in C++) */ + hid_t virtualfd; H5File* filefd; - - /** Dataspace handle */ DataSpace* dataspace; - - /** DataSet handle */ DataSet* dataset; - - /** Datatype of dataset */ DataType datatype; - /** Number of pixels in x direction */ uint32_t nPixelsX; - - /** Number of pixels in y direction */ uint32_t nPixelsY; - - /** Number of frames in file */ uint32_t numFramesInFile; - - /** Number of actual packets caught in file */ uint64_t numActualPacketsInFile; - - /** Number of files in an acquisition - to verify need of virtual file */ int numFilesinAcquisition; - /** parameter names */ std::vector parameterNames; - - /** parameter data types */ std::vector parameterDataTypes; - - /** Dataspace of parameters */ DataSpace* dataspace_para; - - /** Dataset array for parameters */ std::vector dataset_para; - /** Number of Images (including extended during acquisition) */ uint64_t extNumImages; }; diff --git a/slsReceiverSoftware/src/HDF5FileStatic.h b/slsReceiverSoftware/src/HDF5FileStatic.h deleted file mode 100755 index a2d4cb89a..000000000 --- a/slsReceiverSoftware/src/HDF5FileStatic.h +++ /dev/null @@ -1,940 +0,0 @@ -#ifdef HDF5C -#pragma once -/************************************************ - * @file HDF5FileStatic.h - * @short creating, closing, writing and reading - * from HDF5 files - ***********************************************/ -/** - *@short creating, closing, writing and reading from HDF5 files - */ - -#include "H5Cpp.h" -#ifndef H5_NO_NAMESPACE -using namespace H5; -#endif -#include "sls_detector_defs.h" -#include "logger.h" - -#include -#include -#include -#include -#include //malloc -#include -#include //memset - -class HDF5FileStatic: public virtual slsDetectorDefs { - -public: - - /** Constructor */ - HDF5FileStatic(){}; - /** Destructor */ - virtual ~HDF5FileStatic(){}; - - - /** - * Create File Name in format fpath/fnameprefix_fx_dy_z.raw, - * where x is fnum, y is (dindex * numunits + unitindex) and z is findex - * @param fpath file path - * @param fnameprefix file name prefix - * @param findex file index - * @param subfindex sub file index - * @param dindex readout index - * @param numunits number of units per readout. eg. eiger has 2 udp units per readout - * @param unitindex unit index - * @returns complete file name created - */ - static std::string CreateFileName(std::string fpath, std::string fprefix, - uint64_t findex, uint64_t subfindex, - int dindex, int numunits = 1, - int unitindex = 0) { - std::ostringstream os; - os << fpath << "/" << fprefix << "_d" - << (dindex * numunits + unitindex) << "_f" << subfindex << '_' - << findex << ".h5"; - return os.str(); - } - - /** - * Create master file name - * @param fpath file path - * @param fnameprefix file name prefix - * @param findex file index - * @returns master file name - */ - static std::string CreateMasterFileName(std::string fpath, std::string fnameprefix, uint64_t findex) - { - std::ostringstream os; - os << fpath << "/" << fnameprefix << "_master" - << "_" << findex << ".h5"; - return os.str(); - } - - /** - * Create virtual file name - * @param fpath file path - * @param fnameprefix file name prefix - * @param fnum current frame number - * @param findex file index - * @returns virtual file name - */ - static std::string CreateVirtualFileName(std::string fpath, std::string fnameprefix, uint64_t findex) - { - std::ostringstream osfn; - osfn << fpath << "/" << fnameprefix; - osfn << "_virtual"; - osfn << "_" << findex; - osfn << ".h5"; - return osfn.str(); - } - - /** - * Close File - * @param ind index for debugging - * @param fd file pointer - * @param dp dataspace pointer - * @param ds dataset pointer - * @param ds_para pointer to array of parameter datasets - */ - - static void CloseDataFile(int ind, H5File*& fd) - { - try { - Exception::dontPrint(); //to handle errors - if (fd) { - delete fd; - fd = 0; - } - } catch(const Exception& error) { - LOG(logERROR) << "Could not close HDF5 handles of index " << ind; - error.printErrorStack(); - } - } - - /* - * Close master file - * @param fd master hdf5 file object - */ - static void CloseMasterDataFile(H5File*& fd) - { - try { - Exception::dontPrint(); //to handle errors - if (fd) { - delete fd; - fd = 0; - } - } catch(const Exception& error) { - LOG(logERROR) << "Could not close master HDF5 handles"; - error.printErrorStack(); - } - } - - /* - * Close virtual file - * (in C because H5Pset_virtual doesnt exist yet in C++) - * @param fd virtual hdf5 file handle - */ - static void CloseVirtualDataFile(hid_t& fd) - { - if(fd) { - if (H5Fclose(fd) < 0 ) { - LOG(logERROR) << "Could not close virtual HDF5 handles"; - } - fd = 0; - } - } - - /** - * Write data to file - * @param ind object index for debugging - * @param buf buffer to write from - * @param nDimx image number in file (imagenumber%maxframesinfile) - * @param nDimy number of pixels in y direction - * @param nDimz number of pixels in x direction - * @param dspace dataspace pointer - * @param dset dataset pointer - * @param dtype datatype - */ - static void WriteDataFile(int ind, char* buf, - uint64_t nDimx, uint32_t nDimy, uint32_t nDimz, - DataSpace* dspace, DataSet* dset, DataType dtype) - { - hsize_t count[3] = {1, nDimy, nDimz}; - hsize_t start[3] = {nDimx, 0, 0}; - hsize_t dims2[2] = {nDimy, nDimz}; - try{ - Exception::dontPrint(); //to handle errors - - dspace->selectHyperslab( H5S_SELECT_SET, count, start); - DataSpace memspace(2,dims2); - dset->write(buf, dtype, memspace, *dspace); - memspace.close(); - } - catch(const Exception& error){ - LOG(logERROR) << "Could not write to file in object " << ind; - error.printErrorStack(); - throw RuntimeError("Could not write to file in object " + std::to_string(ind)); - } - } - - - - /** - * Write Parameter Arrays as datasets (to virtual file) - * @param ind self index - * @param dspace_para parameter dataspace - * @param fnum current frame number - * @param dset_para vector or dataset pointers of parameters - * @param rheader sls_receiver_header pointer - * @param parameterDataTypes parameter datatypes - */ - static void WriteParameterDatasets(int ind, DataSpace* dspace_para, uint64_t fnum, - std::vector dset_para,sls_receiver_header* rheader, - std::vector parameterDataTypes) - { - sls_detector_header header = rheader->detHeader; - hsize_t count[1] = {1}; - hsize_t start[1] = {fnum}; - int i = 0; - try{ - Exception::dontPrint(); //to handle errors - dspace_para->selectHyperslab( H5S_SELECT_SET, count, start); - DataSpace memspace(H5S_SCALAR); - dset_para[0]->write(&header.frameNumber, parameterDataTypes[0], memspace, *dspace_para);i=1; - dset_para[1]->write(&header.expLength, parameterDataTypes[1], memspace, *dspace_para);i=2; - dset_para[2]->write(&header.packetNumber, parameterDataTypes[2], memspace, *dspace_para);i=3; - dset_para[3]->write(&header.bunchId, parameterDataTypes[3], memspace, *dspace_para);i=4; - dset_para[4]->write(&header.timestamp, parameterDataTypes[4], memspace, *dspace_para);i=5; - dset_para[5]->write(&header.modId, parameterDataTypes[5], memspace, *dspace_para);i=6; - dset_para[6]->write(&header.row, parameterDataTypes[6], memspace, *dspace_para);i=7; - dset_para[7]->write(&header.column, parameterDataTypes[7], memspace, *dspace_para);i=8; - dset_para[8]->write(&header.reserved, parameterDataTypes[8], memspace, *dspace_para);i=9; - dset_para[9]->write(&header.debug, parameterDataTypes[9], memspace, *dspace_para);i=10; - dset_para[10]->write(&header.roundRNumber, parameterDataTypes[10], memspace, *dspace_para);i=11; - dset_para[11]->write(&header.detType, parameterDataTypes[11], memspace, *dspace_para);i=12; - dset_para[12]->write(&header.version, parameterDataTypes[12], memspace, *dspace_para);i=13; - - // contiguous bitset - if (sizeof(sls_bitset) == sizeof(bitset_storage)) { - dset_para[13]->write((char*)&(rheader->packetsMask), parameterDataTypes[13], memspace, *dspace_para); - } - - // not contiguous bitset - else { - // get contiguous representation of bit mask - bitset_storage storage; - memset(storage, 0 , sizeof(bitset_storage)); - sls_bitset bits = rheader->packetsMask; - for (int i = 0; i < MAX_NUM_PACKETS; ++i) - storage[i >> 3] |= (bits[i] << (i & 7)); - // write bitmask - dset_para[13]->write((char*)storage, parameterDataTypes[13], memspace, *dspace_para); - }i=14; - } - catch(const Exception& error){ - error.printErrorStack(); - throw RuntimeError("Could not write parameters (index:" + std::to_string(i) + ") to file in object " + std::to_string(ind)); - } - } - - - - /** - * Extend datasets in #images dimension (x dimension) - * @param ind self index - * @param dpace data space pointer address - * @param dset data set pointer - * @param dspace_para parameter dataspace address pointer - * @param dset dataset parameter pointer - * @param initialNumImages initial number of images - */ - static void ExtendDataset(int ind, DataSpace*& dspace, DataSet* dset, - DataSpace*& dspace_para, std::vector dset_para, - uint64_t initialNumImages) { - try{ - Exception::dontPrint(); //to handle errors - - hsize_t dims[3]; - dspace->getSimpleExtentDims(dims); - dims[0] += initialNumImages; - - dset->extend(dims); - delete dspace; - dspace = 0; - dspace = new DataSpace(dset->getSpace()); - - hsize_t dims_para[1] = {dims[0]}; - for (unsigned int i = 0; i < dset_para.size(); ++i) - dset_para[i]->extend(dims_para); - delete dspace_para; - dspace_para = 0; - dspace_para = new DataSpace(dset_para[0]->getSpace()); - - } - catch(const Exception& error){ - error.printErrorStack(); - throw RuntimeError("Could not extend dataset in object " + std::to_string(ind)); - } - } - - - /** - * Create master file - * @param fname master file name - * @param owenable overwrite enable - * @param attr master file attributes - */ - static void CreateMasterDataFile(H5File*& fd, std::string fname, bool owenable, - masterAttributes& attr) - { - try { - Exception::dontPrint(); //to handle errors - - FileAccPropList flist; - flist.setFcloseDegree(H5F_CLOSE_STRONG); - fd = 0; - if(!owenable) - fd = new H5File( fname.c_str(), H5F_ACC_EXCL, - FileCreatPropList::DEFAULT, - flist ); - else - fd = new H5File( fname.c_str(), H5F_ACC_TRUNC, - FileCreatPropList::DEFAULT, - flist ); - - //variables - DataSpace dataspace = DataSpace (H5S_SCALAR); - Attribute attribute; - DataSet dataset; - int iValue=0; - double dValue=0; - StrType strdatatype(PredType::C_S1,256); - - //create attributes - //version - dValue = attr.version; - attribute = fd->createAttribute("version",PredType::NATIVE_DOUBLE, dataspace); - attribute.write(PredType::NATIVE_DOUBLE, &dValue); - - //Create a group in the file - Group group1( fd->createGroup( "entry" ) ); - Group group2( group1.createGroup("data") ); - Group group3( group1.createGroup("instrument") ); - Group group4( group3.createGroup("beam") ); - Group group5( group3.createGroup("detector") ); - Group group6( group1.createGroup("sample") ); - - //Dynamic Range - dataset = group5.createDataSet ( "dynamic range", PredType::NATIVE_INT, dataspace ); - dataset.write ( &(attr.dynamicRange), PredType::NATIVE_INT); - attribute = dataset.createAttribute("unit",strdatatype, dataspace); - attribute.write(strdatatype, std::string("bits")); - - //Ten Giga - iValue = attr.tenGiga; - dataset = group5.createDataSet ( "ten giga enable", PredType::NATIVE_INT, dataspace ); - dataset.write ( &iValue, PredType::NATIVE_INT); - - //Image Size - dataset = group5.createDataSet ( "image size", PredType::NATIVE_INT, dataspace ); - dataset.write ( &(attr.imageSize), PredType::NATIVE_INT); - attribute = dataset.createAttribute("unit",strdatatype, dataspace); - attribute.write(strdatatype, std::string("bytes")); - - //x - dataset = group5.createDataSet ( "number of pixels in x axis", PredType::NATIVE_INT, dataspace ); - dataset.write ( &(attr.nPixelsX), PredType::NATIVE_INT); - - //y - dataset = group5.createDataSet ( "number of pixels in y axis", PredType::NATIVE_INT, dataspace ); - dataset.write ( &(attr.nPixelsY), PredType::NATIVE_INT); - - //Maximum frames per file - dataset = group5.createDataSet ( "maximum frames per file", PredType::NATIVE_INT, dataspace ); - dataset.write ( &(attr.maxFramesPerFile), PredType::NATIVE_INT); - - //Total Frames - dataset = group5.createDataSet ( "total frames", PredType::STD_U64LE, dataspace ); - dataset.write ( &(attr.totalFrames), PredType::STD_U64LE); - - //Exptime - dataset = group5.createDataSet ( "exposure time", PredType::STD_U64LE, dataspace ); - dataset.write ( &(attr.exptimeNs), PredType::STD_U64LE); - attribute = dataset.createAttribute("unit",strdatatype, dataspace); - attribute.write(strdatatype, std::string("ns")); - - //SubExptime - dataset = group5.createDataSet ( "sub exposure time", PredType::STD_U64LE, dataspace ); - dataset.write ( &(attr.subExptimeNs), PredType::STD_U64LE); - attribute = dataset.createAttribute("unit",strdatatype, dataspace); - attribute.write(strdatatype, std::string("ns")); - - //SubPeriod - dataset = group5.createDataSet ( "sub period", PredType::STD_U64LE, dataspace ); - dataset.write ( &(attr.subPeriodNs), PredType::STD_U64LE); - attribute = dataset.createAttribute("unit",strdatatype, dataspace); - attribute.write(strdatatype, std::string("ns")); - - //Period - dataset = group5.createDataSet ( "acquisition period", PredType::STD_U64LE, dataspace ); - dataset.write ( &(attr.periodNs), PredType::STD_U64LE); - attribute = dataset.createAttribute("unit",strdatatype, dataspace); - attribute.write(strdatatype, std::string("ns")); - - //Quad Enable - dataset = group5.createDataSet ( "quad enable", PredType::NATIVE_INT, dataspace ); - dataset.write ( &(attr.quadEnable), PredType::NATIVE_INT); - - //Analog Flag - dataset = group5.createDataSet ( "analog flag", PredType::NATIVE_INT, dataspace ); - dataset.write ( &(attr.analogFlag), PredType::NATIVE_INT); - - //Digital Flag - dataset = group5.createDataSet ( "digital flag", PredType::NATIVE_INT, dataspace ); - dataset.write ( &(attr.digitalFlag), PredType::NATIVE_INT); - - //ADC Mask - dataset = group5.createDataSet ( "adc mask", PredType::NATIVE_INT, dataspace ); - dataset.write ( &(attr.adcmask), PredType::NATIVE_INT); - - //Dbit Offset - dataset = group5.createDataSet ( "dbit offset", PredType::NATIVE_INT, dataspace ); - dataset.write ( &(attr.dbitoffset), PredType::NATIVE_INT); - - // Dbit List - dataset = group5.createDataSet ( "dbit bitset list", PredType::STD_U64LE, dataspace ); - dataset.write ( &(attr.dbitlist), PredType::STD_U64LE); - - // Roi xmin - dataset = group5.createDataSet ( "roi xmin", PredType::NATIVE_INT, dataspace ); - dataset.write ( &(attr.roiXmin), PredType::NATIVE_INT); - - // Roi xmax - dataset = group5.createDataSet ( "roi xmax", PredType::NATIVE_INT, dataspace ); - dataset.write ( &(attr.roiXmax), PredType::NATIVE_INT); - - //Timestamp - time_t t = time(0); - dataset = group5.createDataSet ( "timestamp", strdatatype, dataspace ); - dataset.write ( std::string(ctime(&t)), strdatatype ); - - fd->close(); - - } catch(const Exception& error) { - error.printErrorStack(); - if (fd) fd->close(); - throw RuntimeError("Could not create master HDF5 handles"); - } - } - - - - /** - * Create File - * @param ind object index for debugging - * @param owenable overwrite enable - * @param fname complete file name - * @param frindexenable frame index enable - * @param fnum current image number - * @param nDimx number of pixels in x dim (#frames) - * @param nDimy number of pixels in y dim (height y dir) - * @param nDimz number of pixels in z dim (width x dir) - * @param dtype data type - * @param fd file pointer - * @param dspace dataspace pointer - * @param dset dataset pointer - * @param version version of software for hdf5 writing - * @param maxchunkedimages maximum chunked images - * @param dspace_para dataspace of parameters - * @param dset_para vector of datasets of parameters - * @param parameterNames parameter names - * @param parameterDataTypes parameter datatypes - */ - static void CreateDataFile(int ind, bool owenable, std::string fname, bool frindexenable, - uint64_t fnum, uint64_t nDimx, uint32_t nDimy, uint32_t nDimz, - DataType dtype, H5File*& fd, DataSpace*& dspace, DataSet*& dset, - double version, uint64_t maxchunkedimages, - DataSpace*& dspace_para, std::vector& dset_para, - std::vector parameterNames, - std::vector parameterDataTypes) - { - try { - Exception::dontPrint(); //to handle errors - - //file - FileAccPropList fapl; - fapl.setFcloseDegree(H5F_CLOSE_STRONG); - fd = 0; - if(!owenable) - fd = new H5File( fname.c_str(), H5F_ACC_EXCL, - FileCreatPropList::DEFAULT, - fapl ); - else - fd = new H5File( fname.c_str(), H5F_ACC_TRUNC, - FileCreatPropList::DEFAULT, - fapl ); - - //attributes - version - double dValue=version; - DataSpace dataspace_attr = DataSpace (H5S_SCALAR); - Attribute attribute = fd->createAttribute("version",PredType::NATIVE_DOUBLE, dataspace_attr); - attribute.write(PredType::NATIVE_DOUBLE, &dValue); - - //dataspace - hsize_t srcdims[3] = {nDimx, nDimy, nDimz}; - hsize_t srcdimsmax[3] = {H5S_UNLIMITED, nDimy, nDimz}; - dspace = 0; - dspace = new DataSpace (3,srcdims,srcdimsmax); - - - //dataset name - std::ostringstream osfn; - osfn << "/data"; - if (frindexenable) osfn << "_f" << std::setfill('0') << std::setw(12) << fnum; - std::string dsetname = osfn.str(); - - //dataset - //fill value - DSetCreatPropList plist; - int fill_value = -1; - plist.setFillValue(dtype, &fill_value); - // always create chunked dataset as unlimited is only supported with chunked layout - hsize_t chunk_dims[3] ={maxchunkedimages, nDimy, nDimz}; - plist.setChunk(3, chunk_dims); - dset = 0; - dset = new DataSet (fd->createDataSet(dsetname.c_str(), dtype, *dspace, plist)); - - //create parameter datasets - hsize_t dims[1] = {nDimx}; - hsize_t dimsmax[1] = {H5S_UNLIMITED}; - dspace_para = 0; - dspace_para = new DataSpace (1,dims,dimsmax); - - // always create chunked dataset as unlimited is only supported with chunked layout - DSetCreatPropList paralist; - hsize_t chunkpara_dims[3] = {maxchunkedimages}; - paralist.setChunk(1, chunkpara_dims); - - for (unsigned int i = 0; i < parameterNames.size(); ++i){ - DataSet* ds = new DataSet(fd->createDataSet(parameterNames[i], - parameterDataTypes[i], *dspace_para, paralist)); - dset_para.push_back(ds); - } - } - catch(const Exception& error){ - error.printErrorStack(); - if (fd) fd->close(); - throw RuntimeError("Could not create HDF5 handles in object " + ind); - } - } - - - /** - * Create virtual file - * (in C because H5Pset_virtual doesnt exist yet in C++) - * @param virtualFileName virtual file name - * @param fd virtual file handle - * @param masterFileName master file name - * @param fpath file path - * @param fnameprefix file name prefix - * @param findex file index - * @param frindexenable frame index enable - * @param dindex readout index - * @param numunits number of units per readout. eg. eiger has 2 udp units per readout - * @param maxFramesPerFile maximum frames per file - * @param numf number of frames caught - * @param srcDataseName source dataset name - * @param dataType datatype of data dataset - * @param numDety number of readouts in Y dir - * @param numDetz number of readouts in Z dir - * @param nDimy number of objects in y dimension in source file (Number of pixels in y dir) - * @param nDimz number of objects in z dimension in source file (Number of pixels in x dir) - * @param version version of software for hdf5 writing - * @param parameterNames parameter names - * @param parameterDataTypes parameter datatypes - */ - static void CreateVirtualDataFile( - std::string virtualFileName, - hid_t& fd, std::string masterFileName, - std::string fpath, std::string fnameprefix, uint64_t findex, bool frindexenable, - int dindex, int numunits, - uint32_t maxFramesPerFile, uint64_t numf, - std::string srcDataseName, DataType dataType, - int numDety, int numDetz, uint32_t nDimy, uint32_t nDimz, - double version, - std::vector parameterNames, - std::vector parameterDataTypes) - { - try { - //file - hid_t dfal = H5Pcreate (H5P_FILE_ACCESS); - if (dfal < 0) - throw RuntimeError("Could not create file access property for virtual file " + virtualFileName); - if (H5Pset_fclose_degree (dfal, H5F_CLOSE_STRONG) < 0) - throw RuntimeError("Could not set strong file close degree for virtual file " + virtualFileName); - fd = H5Fcreate( virtualFileName.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, dfal); - if (fd < 0) - throw RuntimeError("Could not create virtual file " + virtualFileName); - - //attributes - version - hid_t dataspace_attr = H5Screate (H5S_SCALAR); - if (dataspace_attr < 0) - throw RuntimeError("Could not create dataspace for attribute in virtual file " + virtualFileName); - hid_t attrid = H5Acreate2 (fd, "version", H5T_NATIVE_DOUBLE, dataspace_attr, H5P_DEFAULT, H5P_DEFAULT); - if (attrid < 0) - throw RuntimeError("Could not create attribute in virtual file " + virtualFileName); - double attr_data = version; - if (H5Awrite (attrid, H5T_NATIVE_DOUBLE, &attr_data) < 0) - throw RuntimeError("Could not write attribute in virtual file " + virtualFileName); - if (H5Aclose (attrid) < 0) - throw RuntimeError("Could not close attribute in virtual file " + virtualFileName); - - - //virtual dataspace - hsize_t vdsdims[3] = {numf, numDety * nDimy, numDetz * nDimz}; - hid_t vdsDataspace = H5Screate_simple(3, vdsdims ,NULL); - if (vdsDataspace < 0) - throw RuntimeError("Could not create virtual dataspace in virtual file " + virtualFileName); - hsize_t vdsdims_para[2] = {numf, (unsigned int) numDety * numDetz}; - hid_t vdsDataspace_para = H5Screate_simple(2, vdsdims_para, NULL); - if (vdsDataspace_para < 0) - throw RuntimeError("Could not create virtual dataspace (parameters) in virtual file " + virtualFileName); - - - //fill values - hid_t dcpl = H5Pcreate (H5P_DATASET_CREATE); - if (dcpl < 0) - throw RuntimeError("Could not create file creation properties in virtual file " + virtualFileName); - int fill_value = -1; - if (H5Pset_fill_value (dcpl, GetDataTypeinC(dataType), &fill_value) < 0) - throw RuntimeError("Could not create fill value in virtual file " + virtualFileName); - hid_t dcpl_para[parameterNames.size()]; - for (unsigned int i = 0; i < parameterNames.size(); ++i) { - dcpl_para[i] = H5Pcreate (H5P_DATASET_CREATE); - if (dcpl_para[i] < 0) - throw RuntimeError("Could not create file creation properties (parameters) in virtual file " + virtualFileName); - if (H5Pset_fill_value (dcpl_para[i], GetDataTypeinC(parameterDataTypes[i]), &fill_value) < 0) - throw RuntimeError("Could not create fill value (parameters) in virtual file " + virtualFileName); - } - - //hyperslab - int numMajorHyperslab = numf/maxFramesPerFile; - if (numf%maxFramesPerFile) numMajorHyperslab++; - uint64_t framesSaved = 0; - for (int j = 0; j < numMajorHyperslab; j++) { - - uint64_t nDimx = ((numf - framesSaved) > maxFramesPerFile) - ? maxFramesPerFile : (numf-framesSaved); - hsize_t offset[3] = {framesSaved, 0, 0}; - hsize_t count[3] = {nDimx, nDimy, nDimz}; - hsize_t offset_para[2] = {framesSaved, 0}; - hsize_t count_para[2] = {nDimx, 1}; - - for (int i = 0; i < numDety * numDetz; ++i) { - - //setect hyperslabs - if (H5Sselect_hyperslab (vdsDataspace, H5S_SELECT_SET, offset, NULL, count, NULL) < 0) { - throw RuntimeError("Could not select hyperslab"); - } - if (H5Sselect_hyperslab (vdsDataspace_para, H5S_SELECT_SET, - offset_para, NULL, count_para, NULL) < 0) { - throw RuntimeError("Could not select hyperslab for parameters"); - } - - //source file name - std::string srcFileName = HDF5FileStatic::CreateFileName(fpath, fnameprefix, findex, - j, dindex, numunits, i); - - LOG(logERROR) << srcFileName; - // find relative path - std::string relative_srcFileName = srcFileName; - { - size_t i = srcFileName.rfind('/', srcFileName.length()); - if (i != std::string::npos) - relative_srcFileName = (srcFileName.substr(i+1, srcFileName.length() - i)); - } - - //source dataset name - std::ostringstream osfn; - osfn << "/data"; - if (frindexenable) osfn << "_f" << std::setfill('0') << std::setw(12) << j; - std::string srcDatasetName = osfn.str(); - - //source dataspace - hsize_t srcdims[3] = {nDimx, nDimy, nDimz}; - hsize_t srcdimsmax[3] = {H5S_UNLIMITED, nDimy, nDimz}; - hid_t srcDataspace = H5Screate_simple(3, srcdims, srcdimsmax); - if (srcDataspace < 0) - throw RuntimeError("Could not create source dataspace in virtual file " + virtualFileName); - hsize_t srcdims_para[1] = {nDimx}; - hsize_t srcdimsmax_para[1] = {H5S_UNLIMITED}; - hid_t srcDataspace_para = H5Screate_simple(1, srcdims_para, srcdimsmax_para); - if (srcDataspace_para < 0) - throw RuntimeError("Could not create source dataspace (parameters) in virtual file " + virtualFileName); - - //mapping - if (H5Pset_virtual(dcpl, vdsDataspace, relative_srcFileName.c_str(), - srcDatasetName.c_str(), srcDataspace) < 0) { - throw RuntimeError("Could not set mapping for paramter 1"); - } - - for (unsigned int k = 0; k < parameterNames.size(); ++k) { - if (H5Pset_virtual(dcpl_para[k], vdsDataspace_para, relative_srcFileName.c_str(), - parameterNames[k], srcDataspace_para) < 0) { - throw RuntimeError("Could not set mapping for paramter " + std::to_string(k)); - } - } - - //H5Sclose(srcDataspace); - //H5Sclose(srcDataspace_para); - offset[2] += nDimz; - if (offset[2] >= (numDetz * nDimz)) { - offset[2] = 0; - offset[1] += nDimy; - } - offset_para[1]++; - } - framesSaved += nDimx; - } - - //dataset - std::string virtualDatasetName = srcDataseName; - hid_t vdsdataset = H5Dcreate2 (fd, virtualDatasetName.c_str(), - GetDataTypeinC(dataType), vdsDataspace, H5P_DEFAULT, dcpl, H5P_DEFAULT); - if (vdsdataset < 0) - throw RuntimeError("Could not create virutal dataset in virtual file " + virtualFileName); - - - //virtual parameter dataset - for (unsigned int i = 0; i < parameterNames.size(); ++i) { - hid_t vdsdataset_para = H5Dcreate2 (fd, - parameterNames[i], - GetDataTypeinC(parameterDataTypes[i]), vdsDataspace_para, - H5P_DEFAULT, dcpl_para[i], H5P_DEFAULT); - if (vdsdataset_para < 0) - throw RuntimeError("Could not create virutal dataset (parameters) in virtual file " + virtualFileName); - } - - //close - H5Fclose(fd); - fd = 0; - - //link - LinkVirtualInMaster(masterFileName, virtualFileName, virtualDatasetName, parameterNames); - } catch (const RuntimeError &e) { - if (fd > 0) - H5Fclose(fd); - fd = 0; - } - } - - - - /** - * Copy file to another file (mainly to view virutal files in hdfviewer) - * @param dataType data type - * @param owenable overwrite enable - * @param oldFileName file name including path of file to copy - * @param oldDatasetName dataset name to copy - * @param newFileName file name including path of file to copy to - * @param newDatasetName dataset name to copy to - * @param rank rank - * @param nDimx Number of objects in x dimension - * @param nDimy Number of objects in y dimension - * @param nDimz Number of objects in z dimension - */ - template - static void CopyVirtualFile(T datatype, bool owenable, std::string oldFileName, std::string oldDatasetName, - std::string newFileName, std::string newDatasetName, int rank, - uint64_t nDimx, uint32_t nDimy, uint32_t nDimz=0) - { - T *data_out; - switch (rank) { - case 2: - data_out = (T*)malloc(sizeof(T)*(nDimx*nDimy)); - break; - case 3: - data_out = (T*)malloc(sizeof(T)*(nDimx*nDimy*nDimz)); - break; - default: - throw RuntimeError("Invalid rank. Options: 2 or 3"); - } - if (datatype == PredType::STD_U16LE) { - LOG(logINFO) << "datatype:16"; - } else if (datatype == PredType::STD_U32LE) { - LOG(logINFO) << "datatype:32"; - } else if (datatype == PredType::STD_U64LE) { - LOG(logINFO) << "datatype:64"; - } else if (datatype == PredType::STD_U8LE) { - LOG(logINFO) << "datatype:8"; - } else { - throw RuntimeError("Unknown datatype:" + std::to_string(datatype)); - } - LOG(logINFO) << "owenable:" << (owenable?1:0) << std::endl - << "oldFileName:" << oldFileName << std::endl - << "oldDatasetName:" << oldDatasetName << std::endl - << "newFileName:" << newFileName << std::endl - << "newDatasetName:" << newDatasetName << std::endl - << "rank:" << rank << std::endl - << "nDimx:" << nDimx << std::endl - << "nDimy:" << nDimy << std::endl - << "nDimz:" << nDimz; - - H5File* oldfd; - H5File* newfd; - try { - Exception::dontPrint(); //to handle errors - - //open old file - oldfd = new H5File( oldFileName.c_str(), H5F_ACC_RDONLY); - DataSet oldDataset = oldfd->openDataSet( oldDatasetName.c_str()); - //read dataset - oldDataset.read( data_out, datatype); - //new file - FileAccPropList fapl; - fapl.setFcloseDegree(H5F_CLOSE_STRONG); - newfd = 0; - if(!owenable) - newfd = new H5File( newFileName.c_str(), H5F_ACC_EXCL, - FileCreatPropList::DEFAULT, - fapl ); - else - newfd = new H5File( newFileName.c_str(), H5F_ACC_TRUNC, - FileCreatPropList::DEFAULT, - fapl ); - //dataspace and dataset - DataSpace* newDataspace = 0; - if (rank == 3) { - hsize_t dims[3] = {nDimx, nDimy, nDimz}; - newDataspace = new DataSpace (3,dims); - } else if (rank == 2) { - hsize_t dims[2] = {nDimx, nDimy}; - newDataspace = new DataSpace (2,dims); - } - DataSet* newDataset = 0; - newDataset = new DataSet( newfd->createDataSet(newDatasetName.c_str(), datatype, *newDataspace)); - //write and close - newDataset->write(data_out,datatype); - newfd->close(); - oldfd->close(); - } catch(const Exception& error){ - error.printErrorStack(); - free(data_out); - oldfd->close(); - newfd->close(); - throw RuntimeError("Could not copy virtual files"); - } - free(data_out); - } - - - - /** - * Link the Virtual File in the Master File - * In C because H5Lcreate_external exists only in C - * @param masterFileName master file name - * @param virtualfname virtual file name - * @param virtualDatasetname virtual dataset name - * @param parameterNames parameter names - */ - static void LinkVirtualInMaster(std::string masterFileName, std::string virtualfname, - std::string virtualDatasetname, std::vector parameterNames) { - char linkname[100]; - hid_t vfd = 0; - - try { - hid_t dfal = H5Pcreate (H5P_FILE_ACCESS); - if (dfal < 0) - throw RuntimeError("Could not create file access property for link"); - if (H5Pset_fclose_degree (dfal, H5F_CLOSE_STRONG) < 0) - throw RuntimeError("Could not set strong file close degree for link"); - - //open master file - hid_t mfd = H5Fopen( masterFileName.c_str(), H5F_ACC_RDWR, dfal); - if (mfd < 0) - throw RuntimeError("Could not open master file"); - - //open virtual file - vfd = H5Fopen( virtualfname.c_str(), H5F_ACC_RDWR, dfal); - if (vfd < 0) { - H5Fclose(mfd); mfd = 0; - throw RuntimeError("Could not open virtual file"); - } - - // find relative path - std::string relative_virtualfname = virtualfname; - { - size_t i = virtualfname.rfind('/', virtualfname.length()); - if (i != std::string::npos) - relative_virtualfname = (virtualfname.substr(i+1, virtualfname.length() - i)); - } - - //**data dataset** - hid_t vdset = H5Dopen2( vfd, virtualDatasetname.c_str(), H5P_DEFAULT); - if (vdset < 0) { - H5Fclose(mfd); - throw RuntimeError("Could not open virtual data dataset"); - } - sprintf(linkname, "/entry/data/%s",virtualDatasetname.c_str()); - if(H5Lcreate_external( relative_virtualfname.c_str(), virtualDatasetname.c_str(), - mfd, linkname, H5P_DEFAULT, H5P_DEFAULT) < 0) { - H5Fclose(mfd); mfd = 0; - throw RuntimeError("Could not create link to data dataset"); - } - H5Dclose(vdset); - - //**paramter datasets** - for (unsigned int i = 0; i < parameterNames.size(); ++i){ - hid_t vdset_para = H5Dopen2( vfd, (std::string (parameterNames[i])).c_str(), H5P_DEFAULT); - if (vdset_para < 0) { - H5Fclose(mfd); mfd = 0; - throw RuntimeError("Could not open virtual parameter dataset to create link"); - } - sprintf(linkname, "/entry/data/%s",(std::string (parameterNames[i])).c_str()); - - if(H5Lcreate_external( relative_virtualfname.c_str(), (std::string (parameterNames[i])).c_str(), - mfd, linkname, H5P_DEFAULT, H5P_DEFAULT) < 0) { - H5Fclose(mfd); mfd = 0; - throw RuntimeError("Could not create link to virtual parameter dataset"); - } - } - - H5Fclose(mfd); mfd = 0; - H5Fclose(vfd); vfd = 0; - } catch () { - if(vfd > 0) - H5Fclose(vfd); - vfd = 0; - } - } - - - - /** - * Get Data type in C - * @param dtype datatype in C++ - * @returns datatype in C - */ - static hid_t GetDataTypeinC(DataType dtype) { - if (dtype == PredType::STD_U8LE) - return H5T_STD_U8LE; - else if (dtype == PredType::STD_U16LE) - return H5T_STD_U16LE; - else if (dtype == PredType::STD_U32LE) - return H5T_STD_U32LE; - else if (dtype == PredType::STD_U64LE) - return H5T_STD_U64LE; - else { - hid_t s = H5Tcopy(H5T_C_S1); - H5Tset_size(s, MAX_NUM_PACKETS); - return s; - } - } - -}; - - - -#endif