diff --git a/slsReceiverSoftware/src/BinaryFile.cpp b/slsReceiverSoftware/src/BinaryFile.cpp index 091f33b33..867d7ee12 100755 --- a/slsReceiverSoftware/src/BinaryFile.cpp +++ b/slsReceiverSoftware/src/BinaryFile.cpp @@ -18,7 +18,7 @@ BinaryFile::BinaryFile(int ind, uint32_t* maxf, int* nd, std::string* fname, std::string* fpath, uint64_t* findex, bool* owenable, int* dindex, int* nunits, uint64_t* nf, uint32_t* dr, uint32_t* portno, bool* smode): - File(ind, maxf, nd, fname, fpath, findex, owenable, dindex, nunits, nf, dr, portno, smode), + File(ind, BINARY, maxf, nd, fname, fpath, findex, owenable, dindex, nunits, nf, dr, portno, smode), filefd(nullptr), numFramesInFile(0), numActualPacketsInFile(0), @@ -39,10 +39,6 @@ void BinaryFile::PrintMembers(TLogLevel level) { LOG(logINFO) << "Number of Frames in File: " << numFramesInFile; } -slsDetectorDefs::fileFormat BinaryFile::GetFileType() { - return BINARY; -} - void BinaryFile::CreateFile() { numFramesInFile = 0; numActualPacketsInFile = 0; diff --git a/slsReceiverSoftware/src/BinaryFile.h b/slsReceiverSoftware/src/BinaryFile.h index 98b88ad12..2d0da89e9 100755 --- a/slsReceiverSoftware/src/BinaryFile.h +++ b/slsReceiverSoftware/src/BinaryFile.h @@ -60,7 +60,6 @@ class BinaryFile : private virtual slsDetectorDefs, public File { private: - fileFormat GetFileType() override; int WriteData(char* buf, int bsize); FILE* filefd; diff --git a/slsReceiverSoftware/src/DataProcessor.cpp b/slsReceiverSoftware/src/DataProcessor.cpp index a3fe3e8d5..7cdba016c 100755 --- a/slsReceiverSoftware/src/DataProcessor.cpp +++ b/slsReceiverSoftware/src/DataProcessor.cpp @@ -115,8 +115,9 @@ void DataProcessor::SetFileFormat(const fileFormat f) { //remember the pointer values before they are destroyed int nd[MAX_DIMENSIONS];nd[0] = 0; nd[1] = 0; uint32_t* maxf = nullptr; - std::string* fname=nullptr; std::string* fpath=nullptr; uint64_t* findex=nullptr; - bool* owenable=nullptr; int* dindex=nullptr; int* nunits=nullptr; uint64_t* nf = nullptr; + std::string* fname=nullptr; std::string* fpath=nullptr; + uint64_t* findex=nullptr; bool* owenable=nullptr; int* dindex=nullptr; + int* nunits=nullptr; uint64_t* nf = nullptr; uint32_t* dr = nullptr; uint32_t* port = nullptr; file->GetMemberPointerValues(nd, maxf, fname, fpath, findex, owenable, dindex, nunits, nf, dr, port); diff --git a/slsReceiverSoftware/src/File.cpp b/slsReceiverSoftware/src/File.cpp index 83bfb0d59..ff04ae379 100755 --- a/slsReceiverSoftware/src/File.cpp +++ b/slsReceiverSoftware/src/File.cpp @@ -9,11 +9,12 @@ #include -File::File(int ind, uint32_t* maxf, - int* nd, std::string* fname, std::string* fpath, uint64_t* findex, bool* owenable, - int* dindex, int* nunits, uint64_t* nf, uint32_t* dr, uint32_t* portno, - bool* smode): +File::File(int ind, slsDetectorDefs::fileFormat type, uint32_t* maxf, + int* nd, std::string* fname, std::string* fpath, uint64_t* findex, + bool* owenable, int* dindex, int* nunits, uint64_t* nf, uint32_t* dr, + uint32_t* portno, bool* smode): index(ind), + formatType(type), maxFramesPerFile(maxf), numDetX(nd[0]), numDetY(nd[1]), @@ -29,11 +30,15 @@ File::File(int ind, uint32_t* maxf, silentMode(smode) { - master = index?false:true; + master = ((index == 0) && (*detIndex == 0)) ? true : false; } File::~File() {} +slsDetectorDefs::fileFormat File::GetFileType() { + return formatType; +} + std::string File::GetCurrentFileName() { return currentFileName; } diff --git a/slsReceiverSoftware/src/File.h b/slsReceiverSoftware/src/File.h index ea64aab33..5ef3b549c 100755 --- a/slsReceiverSoftware/src/File.h +++ b/slsReceiverSoftware/src/File.h @@ -22,6 +22,7 @@ class File : private virtual slsDetectorDefs { * Constructor * creates the File Writer * @param ind self index + * @param type file format type * @param maxf pointer to max frames per file * @param nd pointer to number of detectors in each dimension * @param fname pointer to file name prefix @@ -35,35 +36,17 @@ class File : private virtual slsDetectorDefs { * @param portno pointer to udp port number for logging * @param smode pointer to silent mode */ - File(int ind, uint32_t* maxf, - int* nd, std::string* fname, std::string* fpath, uint64_t* findex, bool* owenable, - int* dindex, int* nunits, uint64_t* nf, uint32_t* dr, uint32_t* portno, - bool* smode); + File(int ind, slsDetectorDefs::fileFormat type, uint32_t* maxf, + int* nd, std::string* fname, std::string* fpath, uint64_t* findex, + bool* owenable, int* dindex, int* nunits, uint64_t* nf, uint32_t* dr, + uint32_t* portno, bool* smode); - /** - * Destructor - */ virtual ~File(); - - /** - * Get Current File Name - * @returns current file name - */ + fileFormat GetFileType(); std::string GetCurrentFileName(); - - void resetSubFileIndex(); - /** - * Print all member values - */ virtual void PrintMembers(TLogLevel level = logDEBUG1); - /** - * Get Type - * @return type - */ - virtual fileFormat GetFileType() = 0; - /** * Get Member Pointer Values before the object is destroyed * @param nd pointer to number of detectors in each dimension @@ -78,23 +61,12 @@ class File : private virtual slsDetectorDefs { * @param dr pointer to dynamic range * @param portno pointer to dynamic range */ - void GetMemberPointerValues(int* nd, uint32_t*& maxf, std::string*& fname, std::string*& fpath, - uint64_t*& findex, bool*& owenable, - int*& dindex, int*& nunits, uint64_t*& nf, uint32_t*& dr, uint32_t*& portno); + void GetMemberPointerValues(int* nd, uint32_t*& maxf, std::string*& fname, + std::string*& fpath, uint64_t*& findex, bool*& owenable, int*& dindex, + int*& nunits, uint64_t*& nf, uint32_t*& dr, uint32_t*& portno); - /** - * Create file - */ virtual void CreateFile() = 0; - - /** - * Close Current File - */ virtual void CloseCurrentFile() = 0; - - /** - * Close Files - */ virtual void CloseAllFiles() = 0; /** @@ -123,7 +95,6 @@ class File : private virtual slsDetectorDefs { "should be overloaded by a derived class"; } - /** * End of Acquisition * @param anyPacketsCaught true if any packets are caught, else false @@ -135,60 +106,24 @@ class File : private virtual slsDetectorDefs { } protected: - - /** master file writer/reader */ bool master; - - /** Self Index */ int index; - - /** Maximum frames per file */ + slsDetectorDefs::fileFormat formatType; uint32_t* maxFramesPerFile; - - /** Master File Name */ std::string masterFileName; - - /** Current File Name */ std::string currentFileName; - - /** Number of Detectors in X dimension */ int numDetX; - - /** Number of Detectors in Y dimension */ int numDetY; - - /** File Name Prefix */ std::string* fileNamePrefix; - - /** File Path */ std::string* filePath; - - /** File Index */ uint64_t* fileIndex; - - /** Sub file index */ uint64_t subFileIndex{0}; - - /** Over write enable */ bool* overWriteEnable; - - /** Detector Index */ int* detIndex; - - /** Number of units per detector. Eg. Eiger has 2, others 1 */ int* numUnitsPerDetector; - - /** Number of images in acquisition */ uint64_t* numImages; - - /** Dynamic Range */ uint32_t* dynamicRange; - - /** UDP Port Number for logging */ uint32_t* udpPortNumber; - - /** Silent Mode */ bool* silentMode; - }; diff --git a/slsReceiverSoftware/src/HDF5File.cpp b/slsReceiverSoftware/src/HDF5File.cpp index ef9b29223..76c183f8b 100755 --- a/slsReceiverSoftware/src/HDF5File.cpp +++ b/slsReceiverSoftware/src/HDF5File.cpp @@ -13,9 +13,7 @@ #include -H5File* HDF5File::masterfd = 0; -hid_t HDF5File::virtualfd = 0; - +std::mutex HDF5File::hdf5Lib; HDF5File::HDF5File(int ind, uint32_t* maxf, @@ -24,7 +22,9 @@ HDF5File::HDF5File(int ind, uint32_t* maxf, uint32_t nx, uint32_t ny, bool* smode): - File(ind, maxf, nd, fname, fpath, findex, owenable, dindex, nunits, nf, dr, portno, smode), + File(ind, HDF5, maxf, nd, fname, fpath, findex, owenable, dindex, nunits, nf, dr, portno, smode), + masterfd(0), + virtualfd(0), filefd(0), dataspace(0), dataset(0), @@ -84,90 +84,65 @@ HDF5File::HDF5File(int ind, uint32_t* maxf, parameterNames.push_back("packets caught bit mask"); StrType strdatatype(PredType::C_S1, sizeof(bitset_storage)); parameterDataTypes.push_back(strdatatype); - } - HDF5File::~HDF5File() { CloseAllFiles(); } -void HDF5File::PrintMembers(TLogLevel level) { - File::PrintMembers(); - UpdateDataTypeFromDr(); - if (datatype == PredType::STD_U8LE) { - LOG(level) << "Data Type: 4 or 8"; - } else if (datatype == PredType::STD_U16LE) { - LOG(level) << "Data Type: 16"; - } else if (datatype == PredType::STD_U32LE) { - LOG(level) << "Data Type: 32"; - } else { - LOG(logERROR) << "unknown data type"; - } -} - - void HDF5File::SetNumberofPixels(uint32_t nx, uint32_t ny) { nPixelsX = nx; nPixelsY = ny; } - -slsDetectorDefs::fileFormat HDF5File::GetFileType() { - return HDF5; -} - - -void HDF5File::UpdateDataTypeFromDr() { - switch(*dynamicRange){ - case 16: datatype = PredType::STD_U16LE; break; - case 32: datatype = PredType::STD_U32LE; break; - default: datatype = PredType::STD_U8LE; break; - } -} - - void HDF5File::CreateFile() { numFilesinAcquisition++; numFramesInFile = 0; numActualPacketsInFile = 0; - std::ostringstream os; - os << *filePath << "/" << *fileNamePrefix << "_d" - << (*detIndex * (*numUnitsPerDetector) + index) << "_f" << subFileIndex << '_' - << *fileIndex << ".h5"; - currentFileName = os.str(); - //first time - if(subFileIndex == 0u) - UpdateDataTypeFromDr(); - - CreateDataFile(); - - if(!(*silentMode)) { - LOG(logINFO) << *udpPortNumber << ": HDF5 File created: " << currentFileName; + if(subFileIndex == 0u) { + switch(*dynamicRange){ + case 16: + datatype = PredType::STD_U16LE; + break; + case 32: + datatype = PredType::STD_U32LE; + break; + default: + datatype = PredType::STD_U8LE; + break; + } } + CreateDataFile(); } void HDF5File::CloseCurrentFile() { - CloseFile(filefd, nullptr, false); + CloseFile(filefd, false, false); for (unsigned int i = 0; i < dataset_para.size(); ++i) delete dataset_para[i]; dataset_para.clear(); - if(dataspace_para) {delete dataspace_para;dataspace_para=0;} - if(dataset) {delete dataset;dataset=0;} - if(dataspace) {delete dataspace;dataspace=0;} - if(filefd) {delete filefd;filefd=0;} + if(dataspace_para) { + delete dataspace_para; + dataspace_para = 0; + } + if(dataset) { + delete dataset; + dataset = 0; + } + if(dataspace) { + delete dataspace; + dataspace = 0; + } } - void HDF5File::CloseAllFiles() { numFilesinAcquisition = 0; { - CloseFile(filefd, nullptr, false); - if (master && (*detIndex==0)) { - CloseFile(masterfd, nullptr, true); - CloseFile(nullptr, &virtualfd, false); + CloseFile(filefd, false, false); + if (master) { + CloseFile(masterfd, false, true); + CloseFile(nullptr, true, false); } } for (unsigned int i = 0; i < dataset_para.size(); ++i) @@ -179,7 +154,6 @@ void HDF5File::CloseAllFiles() { if(filefd) delete filefd; } - void HDF5File::WriteToFile(char* buffer, int bufferSize, uint64_t currentFrameNumber, uint32_t numPacketsCaught) { // check if maxframesperfile = 0 for infinite @@ -200,7 +174,6 @@ void HDF5File::WriteToFile(char* buffer, int bufferSize, uint64_t currentFrameNu WriteParameterDatasets(currentFrameNumber, (sls_receiver_header*) (buffer)); } - void HDF5File::CreateMasterFile(bool masterFileWriteEnable, masterAttributes& masterFileAttributes) { //beginning of every acquisition @@ -208,62 +181,63 @@ void HDF5File::CreateMasterFile(bool masterFileWriteEnable, masterAttributes& ma numActualPacketsInFile = 0; extNumImages = *numImages; - if (masterFileWriteEnable && master && (*detIndex==0)) { + if (masterFileWriteEnable && master) { virtualfd = 0; - CreateMasterDataFile(); + CreateMasterDataFile(masterFileAttributes); } } - void HDF5File::EndofAcquisition(bool anyPacketsCaught, uint64_t numImagesCaught) { //not created before if (!virtualfd && anyPacketsCaught) { - // called only by the one maser receiver - if (master && (*detIndex==0)) { - + if (master && masterfd != nullptr) { //only one file and one sub image (link current file in master) if (((numFilesinAcquisition == 1) && (numDetY*numDetX) == 1)) { - LinkVirtualInMaster(); + //dataset name + std::ostringstream oss; + oss << "/data"; + if ((*numImages > 1)) { + oss << "_f" << std::setfill('0') << std::setw(12) << 0; + } + std::string dsetname = oss.str(); + + LinkVirtualInMaster(currentFileName, dsetname); } //create virutal file else{ - CreateVirtualFile(numImagesCaught);} + CreateVirtualDataFile( + // infinite images in 1 file, then maxfrperfile = numImagesCaught + ((*maxFramesPerFile == 0) ? + numImagesCaught+1 : *maxFramesPerFile), + numImagesCaught + 1); + } } } numFilesinAcquisition = 0; } - -// called only by the one maser receiver -void HDF5File::CreateVirtualFile(uint64_t numImagesCaught) { - - HDF5FileStatic::CreateVirtualDataFile( - // infinite images in 1 file, then maxfrperfile = numImagesCaught - ((*maxFramesPerFile == 0) ? numImagesCaught+1 : *maxFramesPerFile), - numImagesCaught + 1); -} - -void HDF5File::CloseFile(H5File* fd, hid_t* cfd, bool master) { - std::lock_guard lock(mutex); +void HDF5File::CloseFile(H5File* fd, bool virtualFile, bool masterFile) { + std::lock_guard lock(HDF5File::hdf5Lib); // c code due to only c implementation of H5Pset_virtual available - if (cfd != nullptr) { - if(*cfd) { - if (H5Fclose(*cfd) < 0 ) { - LOG(logERROR) << "Could not close virtual HDF5 handles"; - } - *cfd = 0; + if (virtualFile) { + if (virtualfd != 0) { + if (H5Fclose(virtualfd) < 0 ) { + LOG(logERROR) << "Could not close virtual HDF5 handles"; } + virtualfd = 0; + } } else { - Exception::dontPrint(); // to handle errors + try { + Exception::dontPrint(); // to handle errors if (fd) { delete fd; fd = 0; } } catch(const Exception& error) { LOG(logERROR) << "Could not close " - << master ? "master" : "data" + << (masterFile ? "master" : "data") << " HDF5 handles of index " << index; error.printErrorStack(); } @@ -271,7 +245,7 @@ void HDF5File::CloseFile(H5File* fd, hid_t* cfd, bool master) { } void HDF5File::WriteDataFile(uint64_t currentFrameNumber, char* buffer) { - std::lock_guard lock(mutex); + std::lock_guard lock(HDF5File::hdf5Lib); uint64_t nDimx = ((*maxFramesPerFile == 0) ? currentFrameNumber : currentFrameNumber % (*maxFramesPerFile)); @@ -292,14 +266,14 @@ void HDF5File::WriteDataFile(uint64_t currentFrameNumber, char* buffer) { catch(const Exception& error){ LOG(logERROR) << "Could not write to file in object " << index; error.printErrorStack(); - throw RuntimeError("Could not write to file in object " + + throw sls::RuntimeError("Could not write to file in object " + std::to_string(index)); } } void HDF5File::WriteParameterDatasets(uint64_t currentFrameNumber, sls_receiver_header* rheader) { - std::lock_guard lock(mutex); + std::lock_guard lock(HDF5File::hdf5Lib); uint64_t fnum = ((*maxFramesPerFile == 0) ? currentFrameNumber : currentFrameNumber % (*maxFramesPerFile)); @@ -360,13 +334,13 @@ void HDF5File::WriteParameterDatasets(uint64_t currentFrameNumber, } catch(const Exception& error){ error.printErrorStack(); - throw RuntimeError("Could not write parameters (index:" + + throw sls::RuntimeError("Could not write parameters (index:" + std::to_string(i) + ") to file in object " + std::to_string(index)); } } void HDF5File::ExtendDataset() { - std::lock_guard lock(mutex); + std::lock_guard lock(HDF5File::hdf5Lib); try{ Exception::dontPrint(); //to handle errors @@ -385,12 +359,12 @@ void HDF5File::ExtendDataset() { dataset_para[i]->extend(dims_para); delete dataspace_para; dataspace_para = 0; - dspace_dataspace_parapara = new DataSpace(dataset_para[0]->getSpace()); + dataspace_para = new DataSpace(dataset_para[0]->getSpace()); } catch(const Exception& error){ error.printErrorStack(); - throw RuntimeError("Could not extend dataset in object " + + throw sls::RuntimeError("Could not extend dataset in object " + std::to_string(index)); } if (!(*silentMode)) { @@ -400,8 +374,103 @@ void HDF5File::ExtendDataset() { extNumImages += *numImages; } -void HDF5File::CreateMasterDataFile() { - std::lock_guard lock(mutex); +void HDF5File::CreateDataFile() { + + std::ostringstream os; + os << *filePath << "/" << *fileNamePrefix << "_d" + << (*detIndex * (*numUnitsPerDetector) + index) << "_f" << subFileIndex << '_' + << *fileIndex << ".h5"; + currentFileName = os.str(); + + std::lock_guard lock(HDF5File::hdf5Lib); + + uint64_t framestosave = ((*maxFramesPerFile == 0) ? *numImages : // infinite images + (((extNumImages - subFileIndex) > (*maxFramesPerFile)) ? // save up to maximum at a time + (*maxFramesPerFile) : (extNumImages-subFileIndex))); + + uint64_t nDimx = framestosave; + uint32_t nDimy = nPixelsY; + uint32_t nDimz = ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX); + + try { + Exception::dontPrint(); //to handle errors + + //file + FileAccPropList fapl; + fapl.setFcloseDegree(H5F_CLOSE_STRONG); + filefd = 0; + if(!(*overWriteEnable)) + filefd = new H5File( currentFileName.c_str(), H5F_ACC_EXCL, + FileCreatPropList::DEFAULT, + fapl ); + else + filefd = new H5File( currentFileName.c_str(), H5F_ACC_TRUNC, + FileCreatPropList::DEFAULT, + fapl ); + + //attributes - version + double dValue = HDF5_WRITER_VERSION; + DataSpace dataspace_attr = DataSpace (H5S_SCALAR); + Attribute attribute = filefd->createAttribute("version", + PredType::NATIVE_DOUBLE, dataspace_attr); + attribute.write(PredType::NATIVE_DOUBLE, &dValue); + + //dataspace + hsize_t srcdims[3] = {nDimx, nDimy, nDimz}; + hsize_t srcdimsmax[3] = {H5S_UNLIMITED, nDimy, nDimz}; + dataspace = 0; + dataspace = new DataSpace (3,srcdims,srcdimsmax); + + + //dataset name + std::ostringstream osfn; + osfn << "/data"; + if (*numImages > 1) + osfn << "_f" << std::setfill('0') << std::setw(12) << subFileIndex; + std::string dsetname = osfn.str(); + + //dataset + //fill value + DSetCreatPropList plist; + int fill_value = -1; + plist.setFillValue(datatype, &fill_value); + // always create chunked dataset as unlimited is only + // supported with chunked layout + hsize_t chunk_dims[3] = {MAX_CHUNKED_IMAGES, nDimy, nDimz}; + plist.setChunk(3, chunk_dims); + dataset = 0; + dataset = new DataSet (filefd->createDataSet(dsetname.c_str(), datatype, + *dataspace, plist)); + + //create parameter datasets + hsize_t dims[1] = {nDimx}; + hsize_t dimsmax[1] = {H5S_UNLIMITED}; + dataspace_para = 0; + dataspace_para = new DataSpace (1,dims,dimsmax); + + // always create chunked dataset as unlimited is only + // supported with chunked layout + DSetCreatPropList paralist; + hsize_t chunkpara_dims[3] = {MAX_CHUNKED_IMAGES}; + paralist.setChunk(1, chunkpara_dims); + + for (unsigned int i = 0; i < parameterNames.size(); ++i){ + DataSet* ds = new DataSet(filefd->createDataSet(parameterNames[i], + parameterDataTypes[i], *dataspace_para, paralist)); + dataset_para.push_back(ds); + } + } + catch(const Exception& error){ + error.printErrorStack(); + if (filefd) filefd->close(); + throw sls::RuntimeError("Could not create HDF5 handles in object " + index); + } + if(!(*silentMode)) { + LOG(logINFO) << *udpPortNumber << ": HDF5 File created: " << currentFileName; + } +} + +void HDF5File::CreateMasterDataFile(masterAttributes& masterFileAttributes) { std::ostringstream os; os << *filePath << "/" << *fileNamePrefix << "_master" @@ -413,6 +482,8 @@ void HDF5File::CreateMasterDataFile() { } masterFileAttributes.version = HDF5_WRITER_VERSION; + std::lock_guard lock(HDF5File::hdf5Lib); + try { Exception::dontPrint(); //to handle errors @@ -454,7 +525,7 @@ void HDF5File::CreateMasterDataFile() { dataset = group5.createDataSet ( "dynamic range", PredType::NATIVE_INT, dataspace ); dataset.write ( &(masterFileAttributes.dynamicRange), PredType::NATIVE_INT); attribute = dataset.createAttribute("unit",strdatatype, dataspace); - attribute.write(strdatatype, std::string("bits")); + attribute.write(strdatatype, std::string("bits").c_str()); //Ten Giga iValue = masterFileAttributes.tenGiga; @@ -549,98 +620,11 @@ void HDF5File::CreateMasterDataFile() { } catch(const Exception& error) { error.printErrorStack(); if (masterfd) masterfd->close(); - throw RuntimeError("Could not create master HDF5 handles"); - } -} - -void HDF5File::CreateDataFile() { - std::lock_guard lock(mutex); - - uint64_t framestosave = ((*maxFramesPerFile == 0) ? *numImages : // infinite images - (((extNumImages - subFileIndex) > (*maxFramesPerFile)) ? // save up to maximum at a time - (*maxFramesPerFile) : (extNumImages-subFileIndex))); - - uint64_t nDimx = framestosave; - uint32_t nDimy = nPixelsY; - uint32_t nDimz = ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX); - - try { - Exception::dontPrint(); //to handle errors - - //file - FileAccPropList fapl; - fapl.setFcloseDegree(H5F_CLOSE_STRONG); - filefd = 0; - if(!(*overWriteEnable)) - filefd = new H5File( currentFileName.c_str(), H5F_ACC_EXCL, - FileCreatPropList::DEFAULT, - fapl ); - else - filefd = new H5File( currentFileName.c_str(), H5F_ACC_TRUNC, - FileCreatPropList::DEFAULT, - fapl ); - - //attributes - version - double dValue = HDF5_WRITER_VERSION; - DataSpace dataspace_attr = DataSpace (H5S_SCALAR); - Attribute attribute = filefd->createAttribute("version", - PredType::NATIVE_DOUBLE, dataspace_attr); - attribute.write(PredType::NATIVE_DOUBLE, &dValue); - - //dataspace - hsize_t srcdims[3] = {nDimx, nDimy, nDimz}; - hsize_t srcdimsmax[3] = {H5S_UNLIMITED, nDimy, nDimz}; - dataspace = 0; - dataspace = new DataSpace (3,srcdims,srcdimsmax); - - - //dataset name - std::ostringstream osfn; - osfn << "/data"; - if (*numImages > 1) - osfn << "_f" << std::setfill('0') << std::setw(12) << subFileIndex; - std::string dsetname = osfn.str(); - - //dataset - //fill value - DSetCreatPropList plist; - int fill_value = -1; - plist.setFillValue(datatype, &fill_value); - // always create chunked dataset as unlimited is only - // supported with chunked layout - hsize_t chunk_dims[3] = {MAX_CHUNKED_IMAGES, nDimy, nDimz}; - plist.setChunk(3, chunk_dims); - dataset = 0; - dataset = new DataSet (filefd->createDataSet(dsetname.c_str(), datatype, - *dataspace, plist)); - - //create parameter datasets - hsize_t dims[1] = {nDimx}; - hsize_t dimsmax[1] = {H5S_UNLIMITED}; - dataspace_para = 0; - dataspace_para = new DataSpace (1,dims,dimsmax); - - // always create chunked dataset as unlimited is only - // supported with chunked layout - DSetCreatPropList paralist; - hsize_t chunkpara_dims[3] = {MAX_CHUNKED_IMAGES}; - paralist.setChunk(1, chunkpara_dims); - - for (unsigned int i = 0; i < parameterNames.size(); ++i){ - DataSet* ds = new DataSet(filefd->createDataSet(parameterNames[i], - parameterDataTypes[i], *dataspace_para, paralist)); - dataset_para.push_back(ds); - } - } - catch(const Exception& error){ - error.printErrorStack(); - if (filefd) filefd->close(); - throw RuntimeError("Could not create HDF5 handles in object " + index); + throw sls::RuntimeError("Could not create master HDF5 handles"); } } void HDF5File::CreateVirtualDataFile(uint32_t maxFramesPerFile, uint64_t numf) { - std::lock_guard lock(mutex); std::ostringstream osfn; osfn << *filePath << "/" << *fileNamePrefix; @@ -657,56 +641,58 @@ void HDF5File::CreateVirtualDataFile(uint32_t maxFramesPerFile, uint64_t numf) { uint32_t nDimy = nPixelsY; uint32_t nDimz = ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX); + std::lock_guard lock(HDF5File::hdf5Lib); + try { //file hid_t dfal = H5Pcreate (H5P_FILE_ACCESS); if (dfal < 0) - throw RuntimeError("Could not create file access property for virtual file " + vname); + throw sls::RuntimeError("Could not create file access property for virtual file " + vname); if (H5Pset_fclose_degree (dfal, H5F_CLOSE_STRONG) < 0) - throw RuntimeError("Could not set strong file close degree for virtual file " + vname); + throw sls::RuntimeError("Could not set strong file close degree for virtual file " + vname); virtualfd = H5Fcreate( vname.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, dfal); if (virtualfd < 0) - throw RuntimeError("Could not create virtual file " + vname); + throw sls::RuntimeError("Could not create virtual file " + vname); //attributes - version hid_t dataspace_attr = H5Screate (H5S_SCALAR); if (dataspace_attr < 0) - throw RuntimeError("Could not create dataspace for attribute in virtual file " + vname); + throw sls::RuntimeError("Could not create dataspace for attribute in virtual file " + vname); hid_t attrid = H5Acreate2 (virtualfd, "version", H5T_NATIVE_DOUBLE, dataspace_attr, H5P_DEFAULT, H5P_DEFAULT); if (attrid < 0) - throw RuntimeError("Could not create attribute in virtual file " + vname); + throw sls::RuntimeError("Could not create attribute in virtual file " + vname); double attr_data = HDF5_WRITER_VERSION; if (H5Awrite (attrid, H5T_NATIVE_DOUBLE, &attr_data) < 0) - throw RuntimeError("Could not write attribute in virtual file " + vname); + throw sls::RuntimeError("Could not write attribute in virtual file " + vname); if (H5Aclose (attrid) < 0) - throw RuntimeError("Could not close attribute in virtual file " + vname); + throw sls::RuntimeError("Could not close attribute in virtual file " + vname); //virtual dataspace hsize_t vdsdims[3] = {numf, numDetY * nDimy, numDetz * nDimz}; hid_t vdsDataspace = H5Screate_simple(3, vdsdims ,NULL); if (vdsDataspace < 0) - throw RuntimeError("Could not create virtual dataspace in virtual file " + vname); + throw sls::RuntimeError("Could not create virtual dataspace in virtual file " + vname); hsize_t vdsdims_para[2] = {numf, (unsigned int) numDetY * numDetz}; hid_t vdsDataspace_para = H5Screate_simple(2, vdsdims_para, NULL); if (vdsDataspace_para < 0) - throw RuntimeError("Could not create virtual dataspace (parameters) in virtual file " + vname); + throw sls::RuntimeError("Could not create virtual dataspace (parameters) in virtual file " + vname); //fill values hid_t dcpl = H5Pcreate (H5P_DATASET_CREATE); if (dcpl < 0) - throw RuntimeError("Could not create file creation properties in virtual file " + vname); + throw sls::RuntimeError("Could not create file creation properties in virtual file " + vname); int fill_value = -1; if (H5Pset_fill_value (dcpl, GetDataTypeinC(datatype), &fill_value) < 0) - throw RuntimeError("Could not create fill value in virtual file " + vname); + throw sls::RuntimeError("Could not create fill value in virtual file " + vname); hid_t dcpl_para[parameterNames.size()]; for (unsigned int i = 0; i < parameterNames.size(); ++i) { dcpl_para[i] = H5Pcreate (H5P_DATASET_CREATE); if (dcpl_para[i] < 0) - throw RuntimeError("Could not create file creation properties (parameters) in virtual file " + vname); + throw sls::RuntimeError("Could not create file creation properties (parameters) in virtual file " + vname); if (H5Pset_fill_value (dcpl_para[i], GetDataTypeinC(parameterDataTypes[i]), &fill_value) < 0) - throw RuntimeError("Could not create fill value (parameters) in virtual file " + vname); + throw sls::RuntimeError("Could not create fill value (parameters) in virtual file " + vname); } //hyperslab @@ -726,11 +712,11 @@ void HDF5File::CreateVirtualDataFile(uint32_t maxFramesPerFile, uint64_t numf) { //setect hyperslabs if (H5Sselect_hyperslab (vdsDataspace, H5S_SELECT_SET, offset, NULL, count, NULL) < 0) { - throw RuntimeError("Could not select hyperslab"); + throw sls::RuntimeError("Could not select hyperslab"); } if (H5Sselect_hyperslab (vdsDataspace_para, H5S_SELECT_SET, offset_para, NULL, count_para, NULL) < 0) { - throw RuntimeError("Could not select hyperslab for parameters"); + throw sls::RuntimeError("Could not select hyperslab for parameters"); } //source file name @@ -761,23 +747,23 @@ void HDF5File::CreateVirtualDataFile(uint32_t maxFramesPerFile, uint64_t numf) { hsize_t srcdimsmax[3] = {H5S_UNLIMITED, nDimy, nDimz}; hid_t srcDataspace = H5Screate_simple(3, srcdims, srcdimsmax); if (srcDataspace < 0) - throw RuntimeError("Could not create source dataspace in virtual file " + vname); + throw sls::RuntimeError("Could not create source dataspace in virtual file " + vname); hsize_t srcdims_para[1] = {nDimx}; hsize_t srcdimsmax_para[1] = {H5S_UNLIMITED}; hid_t srcDataspace_para = H5Screate_simple(1, srcdims_para, srcdimsmax_para); if (srcDataspace_para < 0) - throw RuntimeError("Could not create source dataspace (parameters) in virtual file " + vname); + throw sls::RuntimeError("Could not create source dataspace (parameters) in virtual file " + vname); //mapping if (H5Pset_virtual(dcpl, vdsDataspace, relative_srcFileName.c_str(), srcDatasetName.c_str(), srcDataspace) < 0) { - throw RuntimeError("Could not set mapping for paramter 1"); + throw sls::RuntimeError("Could not set mapping for paramter 1"); } for (unsigned int k = 0; k < parameterNames.size(); ++k) { if (H5Pset_virtual(dcpl_para[k], vdsDataspace_para, relative_srcFileName.c_str(), parameterNames[k], srcDataspace_para) < 0) { - throw RuntimeError("Could not set mapping for paramter " + std::to_string(k)); + throw sls::RuntimeError("Could not set mapping for paramter " + std::to_string(k)); } } @@ -798,7 +784,7 @@ void HDF5File::CreateVirtualDataFile(uint32_t maxFramesPerFile, uint64_t numf) { hid_t vdsdataset = H5Dcreate2 (virtualfd, virtualDatasetName.c_str(), GetDataTypeinC(datatype), vdsDataspace, H5P_DEFAULT, dcpl, H5P_DEFAULT); if (vdsdataset < 0) - throw RuntimeError("Could not create virutal dataset in virtual file " + vname); + throw sls::RuntimeError("Could not create virutal dataset in virtual file " + vname); //virtual parameter dataset @@ -808,7 +794,7 @@ void HDF5File::CreateVirtualDataFile(uint32_t maxFramesPerFile, uint64_t numf) { GetDataTypeinC(parameterDataTypes[i]), vdsDataspace_para, H5P_DEFAULT, dcpl_para[i], H5P_DEFAULT); if (vdsdataset_para < 0) - throw RuntimeError("Could not create virutal dataset (parameters) in virtual file " + vname); + throw sls::RuntimeError("Could not create virutal dataset (parameters) in virtual file " + vname); } //close @@ -816,22 +802,16 @@ void HDF5File::CreateVirtualDataFile(uint32_t maxFramesPerFile, uint64_t numf) { virtualfd = 0; //link - LinkVirtualInMaster(masterFileName, vname, virtualDatasetName, parameterNames); - } catch (const RuntimeError &e) { + LinkVirtualInMaster(vname, virtualDatasetName); + } catch (const sls::RuntimeError &e) { if (virtualfd > 0) H5Fclose(virtualfd); virtualfd = 0; } } -void HDF5File::LinkVirtualInMaster() { - std::lock_guard lock(mutex); - - //dataset name - std::ostringstream osfn; - osfn << "/data"; - if ((*numImages > 1)) osfn << "_f" << std::setfill('0') << std::setw(12) << 0; - std::string dsetname = osfn.str(); +void HDF5File::LinkVirtualInMaster(std::string fname, std::string dsetname) { + std::lock_guard lock(HDF5File::hdf5Lib); char linkname[100]; hid_t vfd = 0; @@ -839,41 +819,41 @@ void HDF5File::LinkVirtualInMaster() { try { hid_t dfal = H5Pcreate (H5P_FILE_ACCESS); if (dfal < 0) - throw RuntimeError("Could not create file access property for link"); + throw sls::RuntimeError("Could not create file access property for link"); if (H5Pset_fclose_degree (dfal, H5F_CLOSE_STRONG) < 0) - throw RuntimeError("Could not set strong file close degree for link"); + throw sls::RuntimeError("Could not set strong file close degree for link"); //open master file hid_t mfd = H5Fopen( masterFileName.c_str(), H5F_ACC_RDWR, dfal); if (mfd < 0) - throw RuntimeError("Could not open master file"); + throw sls::RuntimeError("Could not open master file"); //open virtual file - vfd = H5Fopen( currentFileName.c_str(), H5F_ACC_RDWR, dfal); + vfd = H5Fopen( fname.c_str(), H5F_ACC_RDWR, dfal); if (vfd < 0) { H5Fclose(mfd); mfd = 0; - throw RuntimeError("Could not open virtual file"); + throw sls::RuntimeError("Could not open virtual file"); } // find relative path - std::string relative_virtualfname = currentFileName; + std::string relative_virtualfname = fname; { - size_t i = currentFileName.rfind('/', currentFileName.length()); + size_t i = fname.rfind('/', fname.length()); if (i != std::string::npos) - relative_virtualfname = (currentFileName.substr(i+1, currentFileName.length() - i)); + relative_virtualfname = (fname.substr(i+1, fname.length() - i)); } //**data dataset** hid_t vdset = H5Dopen2( vfd, dsetname.c_str(), H5P_DEFAULT); if (vdset < 0) { H5Fclose(mfd); - throw RuntimeError("Could not open virtual data dataset"); + throw sls::RuntimeError("Could not open virtual data dataset"); } sprintf(linkname, "/entry/data/%s",dsetname.c_str()); if(H5Lcreate_external( relative_virtualfname.c_str(), dsetname.c_str(), mfd, linkname, H5P_DEFAULT, H5P_DEFAULT) < 0) { H5Fclose(mfd); mfd = 0; - throw RuntimeError("Could not create link to data dataset"); + throw sls::RuntimeError("Could not create link to data dataset"); } H5Dclose(vdset); @@ -882,20 +862,20 @@ void HDF5File::LinkVirtualInMaster() { hid_t vdset_para = H5Dopen2( vfd, (std::string (parameterNames[i])).c_str(), H5P_DEFAULT); if (vdset_para < 0) { H5Fclose(mfd); mfd = 0; - throw RuntimeError("Could not open virtual parameter dataset to create link"); + throw sls::RuntimeError("Could not open virtual parameter dataset to create link"); } sprintf(linkname, "/entry/data/%s",(std::string (parameterNames[i])).c_str()); if(H5Lcreate_external( relative_virtualfname.c_str(), (std::string (parameterNames[i])).c_str(), mfd, linkname, H5P_DEFAULT, H5P_DEFAULT) < 0) { H5Fclose(mfd); mfd = 0; - throw RuntimeError("Could not create link to virtual parameter dataset"); + throw sls::RuntimeError("Could not create link to virtual parameter dataset"); } } H5Fclose(mfd); mfd = 0; H5Fclose(vfd); vfd = 0; - } catch () { + } catch (...) { if(vfd > 0) H5Fclose(vfd); vfd = 0; diff --git a/slsReceiverSoftware/src/HDF5File.h b/slsReceiverSoftware/src/HDF5File.h index c71cde775..0862ba6a2 100755 --- a/slsReceiverSoftware/src/HDF5File.h +++ b/slsReceiverSoftware/src/HDF5File.h @@ -47,7 +47,6 @@ class HDF5File : private virtual slsDetectorDefs, public File { uint32_t nx, uint32_t ny, bool* smode); ~HDF5File(); - void PrintMembers(TLogLevel level = logDEBUG1); void SetNumberofPixels(uint32_t nx, uint32_t ny); void CreateFile(); void CloseCurrentFile(); @@ -56,29 +55,24 @@ class HDF5File : private virtual slsDetectorDefs, public File { void CreateMasterFile(bool masterFileWriteEnable, masterAttributes& masterFileAttributes); void EndofAcquisition(bool anyPacketsCaught, uint64_t numImagesCaught); - private: - void CreateVirtualFile(uint64_t numImagesCaught); - fileFormat GetFileType(); - void UpdateDataTypeFromDr(); - void CloseFile(H5File* fd, hid_t* cfd, bool master); + void CloseFile(H5File* fd, bool virtualFile, bool masterFile); void WriteDataFile(uint64_t currentFrameNumber, char* buffer); void WriteParameterDatasets(uint64_t currentFrameNumber, sls_receiver_header* rheader); void ExtendDataset(); - void CreateMasterDataFile(); void CreateDataFile(); + void CreateMasterDataFile(masterAttributes& masterFileAttributes); void CreateVirtualDataFile(uint32_t maxFramesPerFile, uint64_t numf); - void LinkVirtualInMaster(); + void LinkVirtualInMaster(std::string fname, std::string dsetname); hid_t GetDataTypeinC(DataType dtype); + static std::mutex hdf5Lib; - static mutable std::mutex mutex; - - static H5File* masterfd; + H5File* masterfd; /** Virtual File handle ( only file name because code in C as H5Pset_virtual doesnt exist yet in C++) */ - static hid_t virtualfd; + hid_t virtualfd; H5File* filefd; DataSpace* dataspace; DataSet* dataset; diff --git a/slsReceiverSoftware/src/HDF5FileStatic.h b/slsReceiverSoftware/src/HDF5FileStatic.h deleted file mode 100755 index 3fd749c41..000000000 --- a/slsReceiverSoftware/src/HDF5FileStatic.h +++ /dev/null @@ -1,70 +0,0 @@ -#ifdef HDF5C -#pragma once -/************************************************ - * @file HDF5FileStatic.h - * @short creating, closing, writing and reading - * from HDF5 files - ***********************************************/ -/** - *@short creating, closing, writing and reading from HDF5 files - */ - -#include "H5Cpp.h" -#ifndef H5_NO_NAMESPACE -using namespace H5; -#endif -#include "sls_detector_defs.h" -#include "logger.h" - -#include -#include -#include -#include -#include //malloc -#include -#include //memset - -class HDF5FileStatic: public virtual slsDetectorDefs { - -public: - - - - - - - - - - -/** - * Create virtual file - * (in C because H5Pset_virtual doesnt exist yet in C++) - * @param virtualFileName virtual file name - * @param fd virtual file handle - * @param masterFileName master file name - * @param fpath file path - * @param fnameprefix file name prefix - * @param findex file index - * @param frindexenable frame index enable - * @param dindex readout index - * @param numunits number of units per readout. eg. eiger has 2 udp units per readout - * @param maxFramesPerFile maximum frames per file - * @param numf number of frames caught - * @param srcDataseName source dataset name - * @param dataType datatype of data dataset - * @param numDety number of readouts in Y dir - * @param numDetz number of readouts in Z dir - * @param nDimy number of objects in y dimension in source file (Number of pixels in y dir) - * @param nDimz number of objects in z dimension in source file (Number of pixels in x dir) - * @param version version of software for hdf5 writing - * @param parameterNames parameter names - * @param parameterDataTypes parameter datatypes - */ - - -}; - - - -#endif