From 90d1d0f8b8d276f7df786bca9aea4bc66c1147d3 Mon Sep 17 00:00:00 2001 From: Dhanya Thattil Date: Thu, 24 Mar 2022 17:15:23 +0100 Subject: [PATCH 01/12] wip, json master --- slsReceiverSoftware/src/BinaryMasterFile.cpp | 2 +- slsReceiverSoftware/src/MasterAttributes.cpp | 132 ++++++++++++++----- slsReceiverSoftware/src/MasterAttributes.h | 4 +- slsReceiverSoftware/src/receiver_defs.h | 2 +- 4 files changed, 105 insertions(+), 35 deletions(-) diff --git a/slsReceiverSoftware/src/BinaryMasterFile.cpp b/slsReceiverSoftware/src/BinaryMasterFile.cpp index eedf88879..7ebff6fd7 100644 --- a/slsReceiverSoftware/src/BinaryMasterFile.cpp +++ b/slsReceiverSoftware/src/BinaryMasterFile.cpp @@ -23,7 +23,7 @@ void BinaryMasterFile::CreateMasterFile(const std::string filePath, // create file name std::ostringstream os; os << filePath << "/" << fileNamePrefix << "_master" - << "_" << fileIndex << ".raw"; + << "_" << fileIndex << ".json"; fileName_ = os.str(); // create file diff --git a/slsReceiverSoftware/src/MasterAttributes.cpp b/slsReceiverSoftware/src/MasterAttributes.cpp index 447495fce..2503e51e1 100644 --- a/slsReceiverSoftware/src/MasterAttributes.cpp +++ b/slsReceiverSoftware/src/MasterAttributes.cpp @@ -2,29 +2,61 @@ // Copyright (C) 2021 Contributors to the SLS Detector Package #include "MasterAttributes.h" +#include + + void MasterAttributes::WriteMasterBinaryAttributes(FILE *fd) { LOG(logERROR) << "WriteMasterBinaryAttributes should have been called " "by a child class"; } -std::string MasterAttributes::GetBinaryMasterAttributes() { +void MasterAttributes::GetBinaryMasterAttributes(rapidjson::Writer* w) { time_t t = time(nullptr); - std::ostringstream oss; - oss << "Version : " << std::setprecision(2) - << BINARY_WRITER_VERSION << '\n' - << "TimeStamp : " << ctime(&t) << '\n' - << "Detector Type : " << sls::ToString(detType) << '\n' - << "Timing Mode : " << sls::ToString(timingMode) << '\n' - << "Geometry : " << sls::ToString(geometry) << '\n' - << "Image Size : " << imageSize << " bytes" << '\n' - << "Pixels : " << sls::ToString(nPixels) << '\n' - << "Max Frames Per File : " << maxFramesPerFile << '\n' - << "Frame Discard Policy : " << sls::ToString(frameDiscardMode) - << '\n' - << "Frame Padding : " << framePadding << '\n' - << "Scan Parameters : " << sls::ToString(scanParams) << '\n' - << "Total Frames : " << totalFrames << '\n'; - return oss.str(); + + w->Key("Version"); + w->SetMaxDecimalPlaces(2); + w->Double(BINARY_WRITER_VERSION); + + w->Key("Timestamp"); + w->String(ctime(&t)); + + w->Key("Detector Type"); + w->String(sls::ToString(detType).c_str()); + + w->Key("Timing Mode"); + w->String(sls::ToString(timingMode).c_str()); + + w->Key("Geometry"); + w->StartObject(); + w->Key("x"); + w->Uint(geometry.x); + w->Key("y"); + w->Uint(geometry.y); + w->EndObject(); + + w->Key("Image Size in bytes"); + w->Uint(imageSize); + + w->Key("Pixels"); + w->StartArray(); + w->Uint(nPixels.x); + w->Uint(nPixels.y); + w->EndArray(); + + w->Key("Max Frames Per File"); + w->Uint(maxFramesPerFile); + + w->Key("Frame Discard Policy"); + w->String(sls::ToString(frameDiscardMode).c_str()); + + w->Key("Frame Padding"); + w->Uint(framePadding); + + w->Key("Scan Parameters"); + w->String(sls::ToString(scanParams).c_str()); + + w->Key("Total Frames"); + w->Uint64(totalFrames); }; void MasterAttributes::WriteBinaryAttributes(FILE *fd, std::string message) { @@ -36,8 +68,30 @@ void MasterAttributes::WriteBinaryAttributes(FILE *fd, std::string message) { }; void MasterAttributes::WriteFinalBinaryAttributes(FILE *fd) { + /* + FILE* fp = fopen(json_file_name.c_str(), "r"); +char readBuffer[65536]; +FileReadStream is(fp, readBuffer, sizeof(readBuffer)); + +Document d, d2; +d.ParseStream(is); +assert(d.IsArray()); +fclose(fp); +d2.SetObject(); +Value json_objects(kObjectType); +json_objects.AddMember("three", 3, d2.GetAllocator()); +d.PushBack(json_objects, d2.GetAllocator()); + +FILE* outfile = fopen(json_file_name.c_str(), "w"); +char writeBuffer[65536]; +FileWriteStream os(outfile, writeBuffer, sizeof(writeBuffer)); + +Writer writer(os); +d.Accept (writer); +fclose(outfile); +*/ // adding few common parameters to the end - std::ostringstream oss; + /*std::ostringstream oss; if (!additionalJsonHeader.empty()) { oss << "Additional Json Header : " @@ -70,7 +124,7 @@ void MasterAttributes::WriteFinalBinaryAttributes(FILE *fd) { message.length()) { throw sls::RuntimeError( "Master binary file incorrect number of bytes written to file"); - } + }*/ }; #ifdef HDF5C @@ -270,7 +324,7 @@ void MasterAttributes::WriteHDF5Attributes(H5File *fd, Group *group) { void GotthardMasterAttributes::WriteMasterBinaryAttributes(FILE *fd) { std::ostringstream oss; - oss << MasterAttributes::GetBinaryMasterAttributes() + oss //<< MasterAttributes::GetBinaryMasterAttributes() << "Exptime : " << sls::ToString(exptime) << '\n' << "Period : " << sls::ToString(period) << '\n' << "Roi (xmin, xmax) : " << sls::ToString(roi) << '\n'; @@ -301,13 +355,27 @@ void MasterAttributes::WriteHDF5Attributes(H5File *fd, Group *group) { #endif void JungfrauMasterAttributes::WriteMasterBinaryAttributes(FILE *fd) { - std::ostringstream oss; - oss << MasterAttributes::GetBinaryMasterAttributes() - << "Exptime : " << sls::ToString(exptime) << '\n' - << "Period : " << sls::ToString(period) << '\n' - << "Number of UDP Interfaces : " << numUDPInterfaces << '\n' - << "Number of rows : " << readNRows << '\n'; - std::string message = oss.str(); + rapidjson::StringBuffer s; + rapidjson::Writer writer(s); + writer.StartObject(); + + MasterAttributes::GetBinaryMasterAttributes(&writer); + + writer.Key("Exptime"); + writer.String(sls::ToString(exptime).c_str()); + + writer.Key("Period"); + writer.String(sls::ToString(period).c_str()); + + writer.Key("Number of UDP Interfaces"); + writer.Uint(numUDPInterfaces); + + writer.Key("Number of rows"); + writer.Uint(readNRows); + + writer.EndObject(); + + std::string message = s.GetString(); MasterAttributes::WriteBinaryAttributes(fd, message); }; @@ -334,7 +402,7 @@ void MasterAttributes::WriteHDF5Attributes(H5File *fd, Group *group) { void EigerMasterAttributes::WriteMasterBinaryAttributes(FILE *fd) { std::ostringstream oss; - oss << MasterAttributes::GetBinaryMasterAttributes() + oss //<< MasterAttributes::GetBinaryMasterAttributes() << "Dynamic Range : " << dynamicRange << '\n' << "Ten Giga : " << tenGiga << '\n' << "Exptime : " << sls::ToString(exptime) << '\n' @@ -420,7 +488,7 @@ void MasterAttributes::WriteHDF5Attributes(H5File *fd, Group *group) { void Mythen3MasterAttributes::WriteMasterBinaryAttributes(FILE *fd) { std::ostringstream oss; - oss << MasterAttributes::GetBinaryMasterAttributes() + oss //<< MasterAttributes::GetBinaryMasterAttributes() << "Dynamic Range : " << dynamicRange << '\n' << "Ten Giga : " << tenGiga << '\n' << "Period : " << sls::ToString(period) << '\n' @@ -535,7 +603,7 @@ void MasterAttributes::WriteHDF5Attributes(H5File *fd, Group *group) { void Gotthard2MasterAttributes::WriteMasterBinaryAttributes(FILE *fd) { std::ostringstream oss; - oss << MasterAttributes::GetBinaryMasterAttributes() + oss //<< MasterAttributes::GetBinaryMasterAttributes() << "Exptime : " << sls::ToString(exptime) << '\n' << "Period : " << sls::ToString(period) << '\n' << "Burst Mode : " << sls::ToString(burstMode) @@ -565,7 +633,7 @@ void MasterAttributes::WriteHDF5Attributes(H5File *fd, Group *group) { void MoenchMasterAttributes::WriteMasterBinaryAttributes(FILE *fd) { std::ostringstream oss; - oss << MasterAttributes::GetBinaryMasterAttributes() + oss //<< MasterAttributes::GetBinaryMasterAttributes() << "Exptime : " << sls::ToString(exptime) << '\n' << "Period : " << sls::ToString(period) << '\n' << "Ten Giga : " << tenGiga << '\n' @@ -601,7 +669,7 @@ void MasterAttributes::WriteHDF5Attributes(H5File *fd, Group *group) { void CtbMasterAttributes::WriteMasterBinaryAttributes(FILE *fd) { std::ostringstream oss; - oss << MasterAttributes::GetBinaryMasterAttributes() + oss //<< MasterAttributes::GetBinaryMasterAttributes() << "Exptime : " << sls::ToString(exptime) << '\n' << "Period : " << sls::ToString(period) << '\n' << "Ten Giga : " << tenGiga << '\n' diff --git a/slsReceiverSoftware/src/MasterAttributes.h b/slsReceiverSoftware/src/MasterAttributes.h index 945ac7756..8c12daf89 100644 --- a/slsReceiverSoftware/src/MasterAttributes.h +++ b/slsReceiverSoftware/src/MasterAttributes.h @@ -7,6 +7,8 @@ #include "sls/logger.h" #include "sls/sls_detector_defs.h" +#include + #ifdef HDF5C #include "H5Cpp.h" #ifndef H5_NO_NAMESPACE @@ -68,7 +70,7 @@ class MasterAttributes { MasterAttributes() = default; virtual ~MasterAttributes() = default; virtual void WriteMasterBinaryAttributes(FILE *fd); - std::string GetBinaryMasterAttributes(); + void GetBinaryMasterAttributes(rapidjson::Writer* w); void WriteBinaryAttributes(FILE *fd, std::string message); void WriteFinalBinaryAttributes(FILE *fd); #ifdef HDF5C diff --git a/slsReceiverSoftware/src/receiver_defs.h b/slsReceiverSoftware/src/receiver_defs.h index 53cc31d2f..c2d85dceb 100644 --- a/slsReceiverSoftware/src/receiver_defs.h +++ b/slsReceiverSoftware/src/receiver_defs.h @@ -18,7 +18,7 @@ // versions #define HDF5_WRITER_VERSION (6.4) // 1 decimal places -#define BINARY_WRITER_VERSION (6.4) // 1 decimal places +#define BINARY_WRITER_VERSION (7.0) // 1 decimal places #define MAX_FRAMES_PER_FILE 20000 #define SHORT_MAX_FRAMES_PER_FILE 100000 From 0f02ffdc9ab57b807b23eb1d7f23cef390a061ad Mon Sep 17 00:00:00 2001 From: Dhanya Thattil Date: Fri, 25 Mar 2022 13:29:03 +0100 Subject: [PATCH 02/12] binary master json done, hdf5 left wip --- slsReceiverSoftware/src/BinaryMasterFile.cpp | 46 +++----- slsReceiverSoftware/src/BinaryMasterFile.h | 21 ++-- slsReceiverSoftware/src/DataProcessor.cpp | 91 ++++++---------- slsReceiverSoftware/src/DataProcessor.h | 13 +-- slsReceiverSoftware/src/File.h | 15 --- slsReceiverSoftware/src/HDF5MasterFile.h | 2 - slsReceiverSoftware/src/Implementation.cpp | 93 ++++++++-------- slsReceiverSoftware/src/Implementation.h | 1 + slsReceiverSoftware/src/MasterAttributes.cpp | 105 +++++++++---------- slsReceiverSoftware/src/MasterAttributes.h | 6 +- 10 files changed, 153 insertions(+), 240 deletions(-) diff --git a/slsReceiverSoftware/src/BinaryMasterFile.cpp b/slsReceiverSoftware/src/BinaryMasterFile.cpp index 7ebff6fd7..2ca87bd7c 100644 --- a/slsReceiverSoftware/src/BinaryMasterFile.cpp +++ b/slsReceiverSoftware/src/BinaryMasterFile.cpp @@ -3,17 +3,6 @@ #include "BinaryMasterFile.h" #include "MasterAttributes.h" -BinaryMasterFile::BinaryMasterFile() : File(BINARY) {} - -BinaryMasterFile::~BinaryMasterFile() { CloseFile(); } - -void BinaryMasterFile::CloseFile() { - if (fd_) { - fclose(fd_); - } - fd_ = nullptr; -} - void BinaryMasterFile::CreateMasterFile(const std::string filePath, const std::string fileNamePrefix, const uint64_t fileIndex, @@ -24,37 +13,26 @@ void BinaryMasterFile::CreateMasterFile(const std::string filePath, std::ostringstream os; os << filePath << "/" << fileNamePrefix << "_master" << "_" << fileIndex << ".json"; - fileName_ = os.str(); + std::string fileName = os.str(); // create file + FILE *fd{nullptr}; if (!overWriteEnable) { - if (nullptr == (fd_ = fopen((const char *)fileName_.c_str(), "wx"))) { - fd_ = nullptr; + if (nullptr == (fd = fopen((const char *)fileName.c_str(), "wx"))) { + fd = nullptr; throw sls::RuntimeError("Could not create binary master file " + - fileName_); + fileName); } - } else if (nullptr == (fd_ = fopen((const char *)fileName_.c_str(), "w"))) { - fd_ = nullptr; + } else if (nullptr == (fd = fopen((const char *)fileName.c_str(), "w"))) { + fd = nullptr; throw sls::RuntimeError( - "Could not create/overwrite binary master file " + fileName_); + "Could not create/overwrite binary master file " + fileName); } if (!silentMode) { - LOG(logINFO) << "Master File: " << fileName_; + LOG(logINFO) << "Master File: " << fileName; } - attr->WriteMasterBinaryAttributes(fd_); - CloseFile(); -} - -void BinaryMasterFile::UpdateMasterFile(MasterAttributes *attr, - bool silentMode) { - if (nullptr == (fd_ = fopen((const char *)fileName_.c_str(), "a"))) { - fd_ = nullptr; - throw sls::RuntimeError("Could not append binary master file " + - fileName_); - } - attr->WriteFinalBinaryAttributes(fd_); - CloseFile(); - if (!silentMode) { - LOG(logINFO) << "Updated Master File"; + attr->WriteMasterBinaryAttributes(fd); + if (fd) { + fclose(fd); } } diff --git a/slsReceiverSoftware/src/BinaryMasterFile.h b/slsReceiverSoftware/src/BinaryMasterFile.h index 909634c71..2ffe3e1d2 100644 --- a/slsReceiverSoftware/src/BinaryMasterFile.h +++ b/slsReceiverSoftware/src/BinaryMasterFile.h @@ -5,21 +5,12 @@ #include "File.h" #include "MasterAttributes.h" -class BinaryMasterFile : private virtual slsDetectorDefs, public File { +class BinaryMasterFile : private virtual slsDetectorDefs { public: - BinaryMasterFile(); - ~BinaryMasterFile(); - - void CloseFile() override; - void CreateMasterFile(const std::string filePath, - const std::string fileNamePrefix, - const uint64_t fileIndex, const bool overWriteEnable, - const bool silentMode, - MasterAttributes *attr) override; - void UpdateMasterFile(MasterAttributes *attr, bool silentMode) override; - - private: - FILE *fd_{nullptr}; - std::string fileName_; + static void CreateMasterFile(const std::string filePath, + const std::string fileNamePrefix, + const uint64_t fileIndex, + const bool overWriteEnable, + const bool silentMode, MasterAttributes *attr); }; \ No newline at end of file diff --git a/slsReceiverSoftware/src/DataProcessor.cpp b/slsReceiverSoftware/src/DataProcessor.cpp index a22abdfc1..dc1bc2d2d 100644 --- a/slsReceiverSoftware/src/DataProcessor.cpp +++ b/slsReceiverSoftware/src/DataProcessor.cpp @@ -82,8 +82,6 @@ void DataProcessor::SetGeneralData(GeneralData *generalData) { void DataProcessor::CloseFiles() { if (dataFile_) dataFile_->CloseFile(); - if (masterFile_) - masterFile_->CloseFile(); #ifdef HDF5C if (virtualFile_) virtualFile_->CloseFile(); @@ -96,10 +94,10 @@ void DataProcessor::DeleteFiles() { delete dataFile_; dataFile_ = nullptr; } - if (masterFile_) { - delete masterFile_; - masterFile_ = nullptr; - } + /* if (masterFile_) { + delete masterFile_; + masterFile_ = nullptr; + }*/ #ifdef HDF5C if (virtualFile_) { delete virtualFile_; @@ -108,27 +106,18 @@ void DataProcessor::DeleteFiles() { #endif } void DataProcessor::SetupFileWriter(const bool filewriteEnable, - const bool masterFilewriteEnable, const fileFormat fileFormatType, - const int modulePos, std::mutex *hdf5Lib) { + std::mutex *hdf5Lib) { DeleteFiles(); if (filewriteEnable) { switch (fileFormatType) { #ifdef HDF5C case HDF5: dataFile_ = new HDF5DataFile(index, hdf5Lib); - if (modulePos == 0 && index == 0) { - if (masterFilewriteEnable) { - masterFile_ = new HDF5MasterFile(hdf5Lib); - } - } break; #endif case BINARY: dataFile_ = new BinaryDataFile(index); - if (modulePos == 0 && index == 0 && masterFilewriteEnable) { - masterFile_ = new BinaryMasterFile(); - } break; default: throw sls::RuntimeError( @@ -138,23 +127,17 @@ void DataProcessor::SetupFileWriter(const bool filewriteEnable, } void DataProcessor::CreateFirstFiles( - MasterAttributes *attr, const std::string filePath, - const std::string fileNamePrefix, const uint64_t fileIndex, - const bool overWriteEnable, const bool silentMode, const int modulePos, - const int numUnitsPerReadout, const uint32_t udpPortNumber, - const uint32_t maxFramesPerFile, const uint64_t numImages, - const uint32_t dynamicRange, const bool detectorDataStream) { + const std::string filePath, const std::string fileNamePrefix, + const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode, + const int modulePos, const int numUnitsPerReadout, + const uint32_t udpPortNumber, const uint32_t maxFramesPerFile, + const uint64_t numImages, const uint32_t dynamicRange, + const bool detectorDataStream) { if (dataFile_ == nullptr) { throw sls::RuntimeError("file object not contstructed"); } CloseFiles(); - // master file write enabled - if (masterFile_) { - masterFile_->CreateMasterFile(filePath, fileNamePrefix, fileIndex, - overWriteEnable, silentMode, attr); - } - // deactivated (half module/ single port), dont write file if ((!*activated_) || (!detectorDataStream)) { return; @@ -238,38 +221,26 @@ void DataProcessor::LinkDataInMasterFile(const bool silentMode) { } #endif -void DataProcessor::UpdateMasterFile(bool silentMode) { - if (masterFile_) { - // final attributes - std::unique_ptr masterAttributes; - switch (detectorType_) { - case GOTTHARD: - masterAttributes = sls::make_unique(); - break; - case JUNGFRAU: - masterAttributes = sls::make_unique(); - break; - case EIGER: - masterAttributes = sls::make_unique(); - break; - case MYTHEN3: - masterAttributes = sls::make_unique(); - break; - case GOTTHARD2: - masterAttributes = sls::make_unique(); - break; - case MOENCH: - masterAttributes = sls::make_unique(); - break; - case CHIPTESTBOARD: - masterAttributes = sls::make_unique(); - break; - default: - throw sls::RuntimeError( - "Unknown detector type to set up master file attributes"); - } - masterAttributes->framesInFile = numFramesCaught_; - masterFile_->UpdateMasterFile(masterAttributes.get(), silentMode); +void DataProcessor::CreateMasterFile( + const std::string filePath, const std::string fileNamePrefix, + const uint64_t fileIndex, const bool overWriteEnable, bool silentMode, + const fileFormat fileFormatType, MasterAttributes *attr) { + + attr->framesInFile = numFramesCaught_; + + std::unique_ptr masterFile{nullptr}; + switch (fileFormatType) { +#ifdef HDF5C + case HDF5: + masterFile = sls::make_unique(hdf5Lib); + break; +#endif + case BINARY: + BinaryMasterFile::CreateMasterFile(filePath, fileNamePrefix, fileIndex, + overWriteEnable, silentMode, attr); + break; + default: + throw sls::RuntimeError("Unknown file format (compile with hdf5 flags"); } } diff --git a/slsReceiverSoftware/src/DataProcessor.h b/slsReceiverSoftware/src/DataProcessor.h index ebd5c4ffa..0bbeb3f40 100644 --- a/slsReceiverSoftware/src/DataProcessor.h +++ b/slsReceiverSoftware/src/DataProcessor.h @@ -45,11 +45,9 @@ class DataProcessor : private virtual slsDetectorDefs, public ThreadObject { void CloseFiles(); void DeleteFiles(); void SetupFileWriter(const bool filewriteEnable, - const bool masterFilewriteEnable, - const fileFormat fileFormatType, const int modulePos, - std::mutex *hdf5Lib); + const fileFormat fileFormatType, std::mutex *hdf5Lib); - void CreateFirstFiles(MasterAttributes *attr, const std::string filePath, + void CreateFirstFiles(const std::string filePath, const std::string fileNamePrefix, const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode, const int modulePos, @@ -71,7 +69,11 @@ class DataProcessor : private virtual slsDetectorDefs, public ThreadObject { const int numModY, std::mutex *hdf5Lib); void LinkDataInMasterFile(const bool silentMode); #endif - void UpdateMasterFile(bool silentMode); + void CreateMasterFile(const std::string filePath, + const std::string fileNamePrefix, + const uint64_t fileIndex, const bool overWriteEnable, + bool silentMode, const fileFormat fileFormatType, + MasterAttributes *attr); /** * Call back for raw data * args to raw data ready callback are @@ -179,7 +181,6 @@ class DataProcessor : private virtual slsDetectorDefs, public ThreadObject { bool firstStreamerFrame_{false}; File *dataFile_{nullptr}; - File *masterFile_{nullptr}; #ifdef HDF5C File *virtualFile_{nullptr}; #endif diff --git a/slsReceiverSoftware/src/File.h b/slsReceiverSoftware/src/File.h index da7af6ce9..8edb28052 100644 --- a/slsReceiverSoftware/src/File.h +++ b/slsReceiverSoftware/src/File.h @@ -103,16 +103,6 @@ class File : private virtual slsDetectorDefs { "should be overloaded by a derived class"; }; - virtual void CreateMasterFile(const std::string filePath, - const std::string fileNamePrefix, - const uint64_t fileIndex, - const bool overWriteEnable, - const bool silentMode, - MasterAttributes *attr) { - LOG(logERROR) << "This is a generic function CreateMasterFile that " - "should be overloaded by a derived class"; - }; - virtual void WriteToFile(char *buffer, const int buffersize, const uint64_t currentFrameNumber, const uint32_t numPacketsCaught) { @@ -120,11 +110,6 @@ class File : private virtual slsDetectorDefs { "should be overloaded by a derived class"; }; - virtual void UpdateMasterFile(MasterAttributes *attr, bool silentMode) { - LOG(logERROR) << "This is a generic function UpdateMasterFile that " - "should be overloaded by a derived class"; - }; - protected: slsDetectorDefs::fileFormat format_; }; diff --git a/slsReceiverSoftware/src/HDF5MasterFile.h b/slsReceiverSoftware/src/HDF5MasterFile.h index 5fb5dad26..350a2da90 100644 --- a/slsReceiverSoftware/src/HDF5MasterFile.h +++ b/slsReceiverSoftware/src/HDF5MasterFile.h @@ -12,7 +12,6 @@ class HDF5MasterFile : private virtual slsDetectorDefs, public File { HDF5MasterFile(std::mutex *hdf5Lib); ~HDF5MasterFile(); - void CloseFile() override; void LinkDataFile(std::string dataFilename, std::string dataSetname, const std::vector parameterNames, const bool silentMode) override; @@ -21,7 +20,6 @@ class HDF5MasterFile : private virtual slsDetectorDefs, public File { const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode, MasterAttributes *attr) override; - void UpdateMasterFile(MasterAttributes *attr, bool silentMode) override; private: std::mutex *hdf5Lib_; diff --git a/slsReceiverSoftware/src/Implementation.cpp b/slsReceiverSoftware/src/Implementation.cpp index 484f11988..64cff45ae 100644 --- a/slsReceiverSoftware/src/Implementation.cpp +++ b/slsReceiverSoftware/src/Implementation.cpp @@ -240,9 +240,6 @@ void Implementation::setModulePositionId(const int id) { xy portGeometry = GetPortGeometry(); streamingPort = DEFAULT_ZMQ_RX_PORTNO + modulePos * portGeometry.x; - for (const auto &it : dataProcessor) - it->SetupFileWriter(fileWriteEnable, masterFileWriteEnable, - fileFormatType, modulePos, &hdf5Lib); assert(numModules.y != 0); for (unsigned int i = 0; i < listener.size(); ++i) { uint16_t row = 0, col = 0; @@ -372,8 +369,7 @@ void Implementation::setFileFormat(const fileFormat f) { throw sls::RuntimeError("Unknown file format"); } for (const auto &it : dataProcessor) - it->SetupFileWriter(fileWriteEnable, masterFileWriteEnable, - fileFormatType, modulePos, &hdf5Lib); + it->SetupFileWriter(fileWriteEnable, fileFormatType, &hdf5Lib); } LOG(logINFO) << "File Format: " << sls::ToString(fileFormatType); @@ -409,8 +405,7 @@ void Implementation::setFileWriteEnable(const bool b) { if (fileWriteEnable != b) { fileWriteEnable = b; for (const auto &it : dataProcessor) - it->SetupFileWriter(fileWriteEnable, masterFileWriteEnable, - fileFormatType, modulePos, &hdf5Lib); + it->SetupFileWriter(fileWriteEnable, fileFormatType, &hdf5Lib); } LOG(logINFO) << "File Write Enable: " << (fileWriteEnable ? "enabled" : "disabled"); @@ -423,9 +418,6 @@ bool Implementation::getMasterFileWriteEnable() const { void Implementation::setMasterFileWriteEnable(const bool b) { if (masterFileWriteEnable != b) { masterFileWriteEnable = b; - for (const auto &it : dataProcessor) - it->SetupFileWriter(fileWriteEnable, masterFileWriteEnable, - fileFormatType, modulePos, &hdf5Lib); } LOG(logINFO) << "Master File Write Enable: " << (masterFileWriteEnable ? "enabled" : "disabled"); @@ -584,29 +576,9 @@ void Implementation::stopReceiver() { std::this_thread::sleep_for(std::chrono::milliseconds(5)); } -#ifdef HDF5C - if (fileWriteEnable && fileFormatType == HDF5) { - if (modulePos == 0) { - // more than 1 file, create virtual file - if (dataProcessor[0]->GetFilesInAcquisition() > 1 || - (numModules.x * numModules.y) > 1) { - dataProcessor[0]->CreateVirtualFile( - filePath, fileName, fileIndex, overwriteEnable, silentMode, - modulePos, numUDPInterfaces, framesPerFile, - numberOfTotalFrames, dynamicRange, numModules.x, - numModules.y, &hdf5Lib); - } - // link file in master - dataProcessor[0]->LinkDataInMasterFile(silentMode); - } - } -#endif - if (fileWriteEnable && masterFileWriteEnable && modulePos == 0) { - try { - dataProcessor[0]->UpdateMasterFile(silentMode); - } catch (...) { - ; // ignore it and just print it - } + if (fileWriteEnable && modulePos == 0) { + // master and virtual file (hdf5) + StartMasterWriter(); } // wait for the processes (dataStreamer) to be done @@ -753,9 +725,25 @@ void Implementation::CreateUDPSockets() { } void Implementation::SetupWriter() { + try { + for (unsigned int i = 0; i < dataProcessor.size(); ++i) { + dataProcessor[i]->CreateFirstFiles( + filePath, fileName, fileIndex, overwriteEnable, silentMode, + modulePos, numUDPInterfaces, udpPortNum[i], framesPerFile, + numberOfTotalFrames, dynamicRange, detectorDataStream[i]); + } + } catch (const sls::RuntimeError &e) { + shutDownUDPSockets(); + for (const auto &it : dataProcessor) + it->CloseFiles(); + throw sls::RuntimeError("Could not create first data file."); + } +} + +void Implementation::StartMasterWriter() { // master file - std::unique_ptr masterAttributes; - if (masterFileWriteEnable && modulePos == 0) { + if (masterFileWriteEnable) { + std::unique_ptr masterAttributes; switch (detType) { case GOTTHARD: masterAttributes = sls::make_unique(); @@ -838,22 +826,31 @@ void Implementation::SetupWriter() { masterAttributes->gateDelay3 = gateDelay3; masterAttributes->gates = numberOfGates; masterAttributes->additionalJsonHeader = additionalJsonHeader; - } - try { - for (unsigned int i = 0; i < dataProcessor.size(); ++i) { - dataProcessor[i]->CreateFirstFiles( - masterAttributes.get(), filePath, fileName, fileIndex, - overwriteEnable, silentMode, modulePos, numUDPInterfaces, - udpPortNum[i], framesPerFile, numberOfTotalFrames, dynamicRange, - detectorDataStream[i]); + try { + dataProcessor[0]->CreateMasterFile( + filePath, fileName, fileIndex, overwriteEnable, silentMode, + fileFormatType, masterAttributes.get()); + } catch (...) { + ; // ignore it and just print it } - } catch (const sls::RuntimeError &e) { - shutDownUDPSockets(); - for (const auto &it : dataProcessor) - it->CloseFiles(); - throw sls::RuntimeError("Could not create first data file."); } +#ifdef HDF5C + if (fileFormatType == HDF5) { + // virtual hdf5 file (if multiple files) + if (dataProcessor[0]->GetFilesInAcquisition() > 1 || + (numModules.x * numModules.y) > 1) { + dataProcessor[0]->CreateVirtualFile( + filePath, fileName, fileIndex, overwriteEnable, silentMode, + modulePos, numUDPInterfaces, framesPerFile, numberOfTotalFrames, + dynamicRange, numModules.x, numModules.y, &hdf5Lib); + } + // link file in master + if (masterFileWriteEnable) { + dataProcessor[0]->LinkDataInMasterFile(silentMode); + } + } +#endif } void Implementation::StartRunning() { diff --git a/slsReceiverSoftware/src/Implementation.h b/slsReceiverSoftware/src/Implementation.h index e798f1a20..fdd970139 100644 --- a/slsReceiverSoftware/src/Implementation.h +++ b/slsReceiverSoftware/src/Implementation.h @@ -274,6 +274,7 @@ class Implementation : private virtual slsDetectorDefs { void ResetParametersforNewAcquisition(); void CreateUDPSockets(); void SetupWriter(); + void StartMasterWriter(); void StartRunning(); /************************************************** diff --git a/slsReceiverSoftware/src/MasterAttributes.cpp b/slsReceiverSoftware/src/MasterAttributes.cpp index 2503e51e1..f4f4959a3 100644 --- a/slsReceiverSoftware/src/MasterAttributes.cpp +++ b/slsReceiverSoftware/src/MasterAttributes.cpp @@ -4,15 +4,15 @@ #include - void MasterAttributes::WriteMasterBinaryAttributes(FILE *fd) { LOG(logERROR) << "WriteMasterBinaryAttributes should have been called " "by a child class"; } -void MasterAttributes::GetBinaryMasterAttributes(rapidjson::Writer* w) { +void MasterAttributes::GetBinaryMasterAttributes( + rapidjson::Writer *w) { time_t t = time(nullptr); - + w->Key("Version"); w->SetMaxDecimalPlaces(2); w->Double(BINARY_WRITER_VERSION); @@ -38,10 +38,12 @@ void MasterAttributes::GetBinaryMasterAttributes(rapidjson::WriterUint(imageSize); w->Key("Pixels"); - w->StartArray(); + w->StartObject(); + w->Key("x"); w->Uint(nPixels.x); + w->Key("y"); w->Uint(nPixels.y); - w->EndArray(); + w->EndObject(); w->Key("Max Frames Per File"); w->Uint(maxFramesPerFile); @@ -67,64 +69,49 @@ void MasterAttributes::WriteBinaryAttributes(FILE *fd, std::string message) { } }; -void MasterAttributes::WriteFinalBinaryAttributes(FILE *fd) { - /* - FILE* fp = fopen(json_file_name.c_str(), "r"); -char readBuffer[65536]; -FileReadStream is(fp, readBuffer, sizeof(readBuffer)); - -Document d, d2; -d.ParseStream(is); -assert(d.IsArray()); -fclose(fp); -d2.SetObject(); -Value json_objects(kObjectType); -json_objects.AddMember("three", 3, d2.GetAllocator()); -d.PushBack(json_objects, d2.GetAllocator()); - -FILE* outfile = fopen(json_file_name.c_str(), "w"); -char writeBuffer[65536]; -FileWriteStream os(outfile, writeBuffer, sizeof(writeBuffer)); - -Writer writer(os); -d.Accept (writer); -fclose(outfile); -*/ +void MasterAttributes::WriteFinalBinaryAttributes( + rapidjson::Writer *w) { // adding few common parameters to the end - /*std::ostringstream oss; if (!additionalJsonHeader.empty()) { - oss << "Additional Json Header : " - << sls::ToString(additionalJsonHeader) << '\n'; + w->Key("Additional Json Header"); + w->String(sls::ToString(additionalJsonHeader).c_str()); } - oss << "Frames in File : " << framesInFile << '\n'; - // adding sls_receiver header format - oss << '\n' - << "#Frame Header" << '\n' - << "Frame Number : 8 bytes" << '\n' - << "SubFrame Number/ExpLength : 4 bytes" << '\n' - << "Packet Number : 4 bytes" << '\n' - << "Bunch ID : 8 bytes" << '\n' - << "Timestamp : 8 bytes" << '\n' - << "Module Id : 2 bytes" << '\n' - << "Row : 2 bytes" << '\n' - << "Column : 2 bytes" << '\n' - << "Reserved : 2 bytes" << '\n' - << "Debug : 4 bytes" << '\n' - << "Round Robin Number : 2 bytes" << '\n' - << "Detector Type : 1 byte" << '\n' - << "Header Version : 1 byte" << '\n' - << "Packets Caught Mask : 64 bytes" << '\n'; + w->Key("Frames in File"); + w->Uint64(framesInFile); - std::string message = oss.str(); - - // writing to file - if (fwrite((void *)message.c_str(), 1, message.length(), fd) != - message.length()) { - throw sls::RuntimeError( - "Master binary file incorrect number of bytes written to file"); - }*/ + w->Key("Frame Header Format"); + w->StartObject(); + w->Key("Frame Number"); + w->String("8 bytes"); + w->Key("SubFrame Number/ExpLength"); + w->String("4 bytes"); + w->Key("Packet Number"); + w->String("4 bytes"); + w->Key("Bunch ID"); + w->String("8 bytes"); + w->Key("Timestamp"); + w->String("8 bytes"); + w->Key("Module Id"); + w->String("2 bytes"); + w->Key("Row"); + w->String("2 bytes"); + w->Key("Column"); + w->String("2 bytes"); + w->Key("Reserved"); + w->String("2 bytes"); + w->Key("Debug"); + w->String("4 bytes"); + w->Key("Round Robin Number"); + w->String("2 bytes"); + w->Key("Detector Type"); + w->String("1 byte"); + w->Key("Header Version"); + w->String("1 byte"); + w->Key("Packets Caught Mask"); + w->String("64 bytes"); + w->EndObject(); }; #ifdef HDF5C @@ -373,8 +360,10 @@ void MasterAttributes::WriteHDF5Attributes(H5File *fd, Group *group) { writer.Key("Number of rows"); writer.Uint(readNRows); + MasterAttributes::WriteFinalBinaryAttributes(&writer); + writer.EndObject(); - + std::string message = s.GetString(); MasterAttributes::WriteBinaryAttributes(fd, message); }; diff --git a/slsReceiverSoftware/src/MasterAttributes.h b/slsReceiverSoftware/src/MasterAttributes.h index 8c12daf89..58a29e8a9 100644 --- a/slsReceiverSoftware/src/MasterAttributes.h +++ b/slsReceiverSoftware/src/MasterAttributes.h @@ -70,9 +70,11 @@ class MasterAttributes { MasterAttributes() = default; virtual ~MasterAttributes() = default; virtual void WriteMasterBinaryAttributes(FILE *fd); - void GetBinaryMasterAttributes(rapidjson::Writer* w); + void + GetBinaryMasterAttributes(rapidjson::Writer *w); void WriteBinaryAttributes(FILE *fd, std::string message); - void WriteFinalBinaryAttributes(FILE *fd); + void + WriteFinalBinaryAttributes(rapidjson::Writer *w); #ifdef HDF5C virtual void WriteMasterHDF5Attributes(H5File *fd, Group *group); void WriteHDF5Attributes(H5File *fd, Group *group); From cbed2e88c628ec8f144e14bcb96204275521ac10 Mon Sep 17 00:00:00 2001 From: Dhanya Thattil Date: Fri, 25 Mar 2022 15:34:39 +0100 Subject: [PATCH 03/12] wip, binary json master for other dets --- slsReceiverSoftware/src/MasterAttributes.cpp | 255 +++++++++++++------ 1 file changed, 177 insertions(+), 78 deletions(-) diff --git a/slsReceiverSoftware/src/MasterAttributes.cpp b/slsReceiverSoftware/src/MasterAttributes.cpp index f4f4959a3..e68de7dd7 100644 --- a/slsReceiverSoftware/src/MasterAttributes.cpp +++ b/slsReceiverSoftware/src/MasterAttributes.cpp @@ -310,12 +310,23 @@ void MasterAttributes::WriteHDF5Attributes(H5File *fd, Group *group) { #endif void GotthardMasterAttributes::WriteMasterBinaryAttributes(FILE *fd) { - std::ostringstream oss; - oss //<< MasterAttributes::GetBinaryMasterAttributes() - << "Exptime : " << sls::ToString(exptime) << '\n' - << "Period : " << sls::ToString(period) << '\n' - << "Roi (xmin, xmax) : " << sls::ToString(roi) << '\n'; - std::string message = oss.str(); + rapidjson::StringBuffer s; + rapidjson::Writer writer(s); + writer.StartObject(); + MasterAttributes::GetBinaryMasterAttributes(&writer); + + writer.Key("Exptime"); + writer.String(sls::ToString(exptime).c_str()); + + writer.Key("Period"); + writer.String(sls::ToString(period).c_str()); + + writer.Key("Roi (xmin, xmax)"); + writer.String(sls::ToString(roi).c_str()); + + MasterAttributes::WriteFinalBinaryAttributes(&writer); + writer.EndObject(); + std::string message = s.GetString(); MasterAttributes::WriteBinaryAttributes(fd, message); }; @@ -345,7 +356,6 @@ void MasterAttributes::WriteHDF5Attributes(H5File *fd, Group *group) { rapidjson::StringBuffer s; rapidjson::Writer writer(s); writer.StartObject(); - MasterAttributes::GetBinaryMasterAttributes(&writer); writer.Key("Exptime"); @@ -361,9 +371,7 @@ void MasterAttributes::WriteHDF5Attributes(H5File *fd, Group *group) { writer.Uint(readNRows); MasterAttributes::WriteFinalBinaryAttributes(&writer); - writer.EndObject(); - std::string message = s.GetString(); MasterAttributes::WriteBinaryAttributes(fd, message); }; @@ -390,22 +398,44 @@ void MasterAttributes::WriteHDF5Attributes(H5File *fd, Group *group) { #endif void EigerMasterAttributes::WriteMasterBinaryAttributes(FILE *fd) { - std::ostringstream oss; - oss //<< MasterAttributes::GetBinaryMasterAttributes() - << "Dynamic Range : " << dynamicRange << '\n' - << "Ten Giga : " << tenGiga << '\n' - << "Exptime : " << sls::ToString(exptime) << '\n' - << "Period : " << sls::ToString(period) << '\n' - << "Threshold Energy : " << thresholdEnergyeV << '\n' - << "SubExptime : " << sls::ToString(subExptime) - << '\n' - << "SubPeriod : " << sls::ToString(subPeriod) - << '\n' - << "Quad : " << quad << '\n' - << "Number of rows : " << readNRows << '\n' - << "Rate Corrections : " << sls::ToString(ratecorr) - << '\n'; - std::string message = oss.str(); + rapidjson::StringBuffer s; + rapidjson::Writer writer(s); + writer.StartObject(); + MasterAttributes::GetBinaryMasterAttributes(&writer); + + writer.Key("Dynamic Range"); + writer.Uint(dynamicRange); + + writer.Key("Ten Giga"); + writer.Uint(tenGiga); + + writer.Key("Exptime"); + writer.String(sls::ToString(exptime).c_str()); + + writer.Key("Period"); + writer.String(sls::ToString(period).c_str()); + + writer.Key("Threshold Energy"); + writer.Int(thresholdEnergyeV); + + writer.Key("Sub Exptime"); + writer.String(sls::ToString(subExptime).c_str()); + + writer.Key("Sub Period"); + writer.String(sls::ToString(subPeriod).c_str()); + + writer.Key("Quad"); + writer.Int(quad); + + writer.Key("Number of rows"); + writer.Int(readNRows); + + writer.Key("Rate Corrections"); + writer.String(sls::ToString(ratecorr).c_str()); + + MasterAttributes::WriteFinalBinaryAttributes(&writer); + writer.EndObject(); + std::string message = s.GetString(); MasterAttributes::WriteBinaryAttributes(fd, message); }; @@ -476,29 +506,50 @@ void MasterAttributes::WriteHDF5Attributes(H5File *fd, Group *group) { #endif void Mythen3MasterAttributes::WriteMasterBinaryAttributes(FILE *fd) { - std::ostringstream oss; - oss //<< MasterAttributes::GetBinaryMasterAttributes() - << "Dynamic Range : " << dynamicRange << '\n' - << "Ten Giga : " << tenGiga << '\n' - << "Period : " << sls::ToString(period) << '\n' - << "Counter Mask : " << sls::ToStringHex(counterMask) - << '\n' - << "Exptime1 : " << sls::ToString(exptime1) - << '\n' - << "Exptime2 : " << sls::ToString(exptime2) - << '\n' - << "Exptime3 : " << sls::ToString(exptime3) - << '\n' - << "GateDelay1 : " << sls::ToString(gateDelay1) - << '\n' - << "GateDelay2 : " << sls::ToString(gateDelay2) - << '\n' - << "GateDelay3 : " << sls::ToString(gateDelay3) - << '\n' - << "Gates : " << gates << '\n' - << "Threshold Energies : " - << sls::ToString(thresholdAllEnergyeV) << '\n'; - std::string message = oss.str(); + rapidjson::StringBuffer s; + rapidjson::Writer writer(s); + writer.StartObject(); + MasterAttributes::GetBinaryMasterAttributes(&writer); + + writer.Key("Dynamic Range"); + writer.Uint(dynamicRange); + + writer.Key("Ten Giga"); + writer.Uint(tenGiga); + + writer.Key("Period"); + writer.String(sls::ToString(period).c_str()); + + writer.Key("Counter Mask"); + writer.String(sls::ToStringHex(counterMask).c_str()); + + writer.Key("Exptime1"); + writer.String(sls::ToString(exptime1).c_str()); + + writer.Key("Exptime2"); + writer.String(sls::ToString(exptime2).c_str()); + + writer.Key("Exptime3"); + writer.String(sls::ToString(exptime3).c_str()); + + writer.Key("GateDelay1"); + writer.String(sls::ToString(gateDelay1).c_str()); + + writer.Key("GateDelay2"); + writer.String(sls::ToString(gateDelay2).c_str()); + + writer.Key("GateDelay3"); + writer.String(sls::ToString(gateDelay3).c_str()); + + writer.Key("Gates"); + writer.Uint(gates); + + writer.Key("Threshold Energies"); + writer.String(sls::ToString(thresholdAllEnergyeV).c_str()); + + MasterAttributes::WriteFinalBinaryAttributes(&writer); + writer.EndObject(); + std::string message = s.GetString(); MasterAttributes::WriteBinaryAttributes(fd, message); }; @@ -591,13 +642,23 @@ void MasterAttributes::WriteHDF5Attributes(H5File *fd, Group *group) { #endif void Gotthard2MasterAttributes::WriteMasterBinaryAttributes(FILE *fd) { - std::ostringstream oss; - oss //<< MasterAttributes::GetBinaryMasterAttributes() - << "Exptime : " << sls::ToString(exptime) << '\n' - << "Period : " << sls::ToString(period) << '\n' - << "Burst Mode : " << sls::ToString(burstMode) - << '\n'; - std::string message = oss.str(); + rapidjson::StringBuffer s; + rapidjson::Writer writer(s); + writer.StartObject(); + MasterAttributes::GetBinaryMasterAttributes(&writer); + + writer.Key("Exptime"); + writer.String(sls::ToString(exptime).c_str()); + + writer.Key("Period"); + writer.String(sls::ToString(period).c_str()); + + writer.Key("Burst Mode"); + writer.String(sls::ToString(burstMode).c_str()); + + MasterAttributes::WriteFinalBinaryAttributes(&writer); + writer.EndObject(); + std::string message = s.GetString(); MasterAttributes::WriteBinaryAttributes(fd, message); }; @@ -621,15 +682,29 @@ void MasterAttributes::WriteHDF5Attributes(H5File *fd, Group *group) { #endif void MoenchMasterAttributes::WriteMasterBinaryAttributes(FILE *fd) { - std::ostringstream oss; - oss //<< MasterAttributes::GetBinaryMasterAttributes() - << "Exptime : " << sls::ToString(exptime) << '\n' - << "Period : " << sls::ToString(period) << '\n' - << "Ten Giga : " << tenGiga << '\n' - << "ADC Mask : " << sls::ToStringHex(adcmask) - << '\n' - << "Analog Samples : " << analogSamples << '\n'; - std::string message = oss.str(); + rapidjson::StringBuffer s; + rapidjson::Writer writer(s); + writer.StartObject(); + MasterAttributes::GetBinaryMasterAttributes(&writer); + + writer.Key("Exptime"); + writer.String(sls::ToString(exptime).c_str()); + + writer.Key("Period"); + writer.String(sls::ToString(period).c_str()); + + writer.Key("Ten Giga"); + writer.Uint(tenGiga); + + writer.Key("ADC Mask"); + writer.String(sls::ToStringHex(adcmask).c_str()); + + writer.Key("Analog Samples"); + writer.Uint(analogSamples); + + MasterAttributes::WriteFinalBinaryAttributes(&writer); + writer.EndObject(); + std::string message = s.GetString(); MasterAttributes::WriteBinaryAttributes(fd, message); }; @@ -657,20 +732,44 @@ void MasterAttributes::WriteHDF5Attributes(H5File *fd, Group *group) { #endif void CtbMasterAttributes::WriteMasterBinaryAttributes(FILE *fd) { - std::ostringstream oss; - oss //<< MasterAttributes::GetBinaryMasterAttributes() - << "Exptime : " << sls::ToString(exptime) << '\n' - << "Period : " << sls::ToString(period) << '\n' - << "Ten Giga : " << tenGiga << '\n' - << "ADC Mask : " << sls::ToStringHex(adcmask) - << '\n' - << "Analog Flag : " << analog << '\n' - << "Analog Samples : " << analogSamples << '\n' - << "Digital Flag : " << digital << '\n' - << "Digital Samples : " << digitalSamples << '\n' - << "Dbit Offset : " << dbitoffset << '\n' - << "Dbit Bitset : " << dbitlist << '\n'; - std::string message = oss.str(); + rapidjson::StringBuffer s; + rapidjson::Writer writer(s); + writer.StartObject(); + MasterAttributes::GetBinaryMasterAttributes(&writer); + + writer.Key("Exptime"); + writer.String(sls::ToString(exptime).c_str()); + + writer.Key("Period"); + writer.String(sls::ToString(period).c_str()); + + writer.Key("Ten Giga"); + writer.Uint(tenGiga); + + writer.Key("ADC Mask"); + writer.String(sls::ToStringHex(adcmask).c_str()); + + writer.Key("Analog Flag"); + writer.Uint(analog); + + writer.Key("Analog Samples"); + writer.Uint(analogSamples); + + writer.Key("Digital Flag"); + writer.Uint(digital); + + writer.Key("Digital Samples"); + writer.Uint(digitalSamples); + + writer.Key("Dbit Offset"); + writer.Uint(dbitoffset); + + writer.Key("Dbit Bitset"); + writer.Uint64(dbitlist); + + MasterAttributes::WriteFinalBinaryAttributes(&writer); + writer.EndObject(); + std::string message = s.GetString(); MasterAttributes::WriteBinaryAttributes(fd, message); }; From f2be834d55f46b88dde05c0852d36930317aaf6e Mon Sep 17 00:00:00 2001 From: Dhanya Thattil Date: Mon, 28 Mar 2022 17:43:58 +0200 Subject: [PATCH 04/12] wip, binary file refactored for stringbuffer, hdf5 mid way, masterattributes to be de (inherited) :) --- slsReceiverSoftware/src/BinaryMasterFile.cpp | 38 ++- slsReceiverSoftware/src/BinaryMasterFile.h | 14 +- slsReceiverSoftware/src/DataProcessor.cpp | 39 +-- slsReceiverSoftware/src/DataProcessor.h | 25 +- slsReceiverSoftware/src/HDF5MasterFile.cpp | 100 ++----- slsReceiverSoftware/src/HDF5MasterFile.h | 27 +- slsReceiverSoftware/src/HDF5VirtualFile.cpp | 6 +- slsReceiverSoftware/src/HDF5VirtualFile.h | 4 +- slsReceiverSoftware/src/Implementation.cpp | 223 +++++++------- slsReceiverSoftware/src/Implementation.h | 3 +- slsReceiverSoftware/src/MasterAttributes.cpp | 298 +++++++------------ slsReceiverSoftware/src/MasterAttributes.h | 30 +- 12 files changed, 362 insertions(+), 445 deletions(-) diff --git a/slsReceiverSoftware/src/BinaryMasterFile.cpp b/slsReceiverSoftware/src/BinaryMasterFile.cpp index 2ca87bd7c..f6d0942fc 100644 --- a/slsReceiverSoftware/src/BinaryMasterFile.cpp +++ b/slsReceiverSoftware/src/BinaryMasterFile.cpp @@ -3,12 +3,12 @@ #include "BinaryMasterFile.h" #include "MasterAttributes.h" -void BinaryMasterFile::CreateMasterFile(const std::string filePath, - const std::string fileNamePrefix, - const uint64_t fileIndex, - const bool overWriteEnable, - const bool silentMode, - MasterAttributes *attr) { +std::string BinaryMasterFile::CreateMasterFile(const std::string filePath, + const std::string fileNamePrefix, + const uint64_t fileIndex, + const bool overWriteEnable, + const bool silentMode, + MasterAttributes *attr) { // create file name std::ostringstream os; os << filePath << "/" << fileNamePrefix << "_master" @@ -28,11 +28,31 @@ void BinaryMasterFile::CreateMasterFile(const std::string filePath, throw sls::RuntimeError( "Could not create/overwrite binary master file " + fileName); } - if (!silentMode) { - LOG(logINFO) << "Master File: " << fileName; + + std::string message = BinaryMasterFile::GetMasterAttributes(attr); + if (fwrite((void *)message.c_str(), 1, message.length(), fd) != + message.length()) { + throw sls::RuntimeError( + "Master binary file incorrect number of bytes written to file"); } - attr->WriteMasterBinaryAttributes(fd); if (fd) { fclose(fd); } + if (!silentMode) { + LOG(logINFO) << "Master File: " << fileName; + } + return fileName; +} + +std::string BinaryMasterFile::GetMasterAttributes(MasterAttributes *attr) { + rapidjson::StringBuffer s; + rapidjson::Writer writer(s); + writer.StartObject(); + + attr->GetCommonBinaryAttributes(&writer); + attr->GetSpecificBinaryAttributes(&writer); + attr->GetFinalBinaryAttributes(&writer); + + writer.EndObject(); + return s.GetString(); } diff --git a/slsReceiverSoftware/src/BinaryMasterFile.h b/slsReceiverSoftware/src/BinaryMasterFile.h index 2ffe3e1d2..aaac736ab 100644 --- a/slsReceiverSoftware/src/BinaryMasterFile.h +++ b/slsReceiverSoftware/src/BinaryMasterFile.h @@ -8,9 +8,13 @@ class BinaryMasterFile : private virtual slsDetectorDefs { public: - static void CreateMasterFile(const std::string filePath, - const std::string fileNamePrefix, - const uint64_t fileIndex, - const bool overWriteEnable, - const bool silentMode, MasterAttributes *attr); + static std::string CreateMasterFile(const std::string filePath, + const std::string fileNamePrefix, + const uint64_t fileIndex, + const bool overWriteEnable, + const bool silentMode, + MasterAttributes *attr); + + private: + static std::string GetMasterAttributes(MasterAttributes *attr); }; \ No newline at end of file diff --git a/slsReceiverSoftware/src/DataProcessor.cpp b/slsReceiverSoftware/src/DataProcessor.cpp index dc1bc2d2d..78985eaef 100644 --- a/slsReceiverSoftware/src/DataProcessor.cpp +++ b/slsReceiverSoftware/src/DataProcessor.cpp @@ -94,10 +94,6 @@ void DataProcessor::DeleteFiles() { delete dataFile_; dataFile_ = nullptr; } - /* if (masterFile_) { - delete masterFile_; - masterFile_ = nullptr; - }*/ #ifdef HDF5C if (virtualFile_) { delete virtualFile_; @@ -107,13 +103,13 @@ void DataProcessor::DeleteFiles() { } void DataProcessor::SetupFileWriter(const bool filewriteEnable, const fileFormat fileFormatType, - std::mutex *hdf5Lib) { + std::mutex *hdf5LibMutex) { DeleteFiles(); if (filewriteEnable) { switch (fileFormatType) { #ifdef HDF5C case HDF5: - dataFile_ = new HDF5DataFile(index, hdf5Lib); + dataFile_ = new HDF5DataFile(index, hdf5LibMutex); break; #endif case BINARY: @@ -178,7 +174,7 @@ void DataProcessor::CreateVirtualFile( const int modulePos, const int numUnitsPerReadout, const uint32_t maxFramesPerFile, const uint64_t numImages, const uint32_t dynamicRange, const int numModX, const int numModY, - std::mutex *hdf5Lib) { + std::mutex *hdf5LibMutex) { if (virtualFile_) { delete virtualFile_; @@ -186,7 +182,7 @@ void DataProcessor::CreateVirtualFile( bool gotthard25um = ((detectorType_ == GOTTHARD || detectorType_ == GOTTHARD2) && (numModX * numModY) == 2); - virtualFile_ = new HDF5VirtualFile(hdf5Lib, gotthard25um); + virtualFile_ = new HDF5VirtualFile(hdf5LibMutex, gotthard25um); // maxframesperfile = 0 for infinite files uint32_t framesPerFile = @@ -204,7 +200,9 @@ void DataProcessor::CreateVirtualFile( dataFile_->GetParameterNames(), dataFile_->GetParameterDataTypes()); } -void DataProcessor::LinkDataInMasterFile(const bool silentMode) { +void DataProcessor::LinkDataInMasterFile(const string &masterFileName, + const bool silentMode, + std::mutex *hdf5LibMutex) { std::string fname, datasetName; if (virtualFile_) { auto res = virtualFile_->GetFileAndDatasetName(); @@ -216,15 +214,17 @@ void DataProcessor::LinkDataInMasterFile(const bool silentMode) { datasetName = res[1]; } // link in master - masterFile_->LinkDataFile(fname, datasetName, - dataFile_->GetParameterNames(), silentMode); + HDF5MasterFile::LinkDataFile(masterFileName, fname, datasetName, + dataFile_->GetParameterNames(), silentMode, + &hdf5LibMutex); } #endif -void DataProcessor::CreateMasterFile( - const std::string filePath, const std::string fileNamePrefix, +std::string DataProcessor::CreateMasterFile( + const std::string &filePath, const std::string &fileNamePrefix, const uint64_t fileIndex, const bool overWriteEnable, bool silentMode, - const fileFormat fileFormatType, MasterAttributes *attr) { + const fileFormat fileFormatType, MasterAttributes *attr, + std::mutex *hdf5LibMutex) { attr->framesInFile = numFramesCaught_; @@ -232,13 +232,14 @@ void DataProcessor::CreateMasterFile( switch (fileFormatType) { #ifdef HDF5C case HDF5: - masterFile = sls::make_unique(hdf5Lib); - break; + return HDF5MasterFile::CreateMasterFile(filePath, fileNamePrefix, + fileIndex, overWriteEnable, + silentMode, attr, hdf5LibMutex); #endif case BINARY: - BinaryMasterFile::CreateMasterFile(filePath, fileNamePrefix, fileIndex, - overWriteEnable, silentMode, attr); - break; + return BinaryMasterFile::CreateMasterFile(filePath, fileNamePrefix, + fileIndex, overWriteEnable, + silentMode, attr); default: throw sls::RuntimeError("Unknown file format (compile with hdf5 flags"); } diff --git a/slsReceiverSoftware/src/DataProcessor.h b/slsReceiverSoftware/src/DataProcessor.h index 0bbeb3f40..2b8aacb9f 100644 --- a/slsReceiverSoftware/src/DataProcessor.h +++ b/slsReceiverSoftware/src/DataProcessor.h @@ -45,7 +45,8 @@ class DataProcessor : private virtual slsDetectorDefs, public ThreadObject { void CloseFiles(); void DeleteFiles(); void SetupFileWriter(const bool filewriteEnable, - const fileFormat fileFormatType, std::mutex *hdf5Lib); + const fileFormat fileFormatType, + std::mutex *hdf5LibMutex); void CreateFirstFiles(const std::string filePath, const std::string fileNamePrefix, @@ -58,22 +59,26 @@ class DataProcessor : private virtual slsDetectorDefs, public ThreadObject { const bool detectorDataStream); #ifdef HDF5C uint32_t GetFilesInAcquisition() const; - void CreateVirtualFile(const std::string filePath, - const std::string fileNamePrefix, + void CreateVirtualFile(const std::string &filePath, + const std::string &fileNamePrefix, const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode, const int modulePos, const int numUnitsPerReadout, const uint32_t maxFramesPerFile, const uint64_t numImages, const uint32_t dynamicRange, const int numModX, - const int numModY, std::mutex *hdf5Lib); - void LinkDataInMasterFile(const bool silentMode); + const int numModY, std::mutex *hdf5LibMutex); + void LinkDataInMasterFile(const string &masterFileName, + const bool silentMode, std::mutex *hdf5LibMutex); #endif - void CreateMasterFile(const std::string filePath, - const std::string fileNamePrefix, - const uint64_t fileIndex, const bool overWriteEnable, - bool silentMode, const fileFormat fileFormatType, - MasterAttributes *attr); + + std::string CreateMasterFile(const std::string &filePath, + const std::string &fileNamePrefix, + const uint64_t fileIndex, + const bool overWriteEnable, bool silentMode, + const fileFormat fileFormatType, + MasterAttributes *attr, + std::mutex *hdf5LibMutex); /** * Call back for raw data * args to raw data ready callback are diff --git a/slsReceiverSoftware/src/HDF5MasterFile.cpp b/slsReceiverSoftware/src/HDF5MasterFile.cpp index 2c3721501..ea425f14d 100644 --- a/slsReceiverSoftware/src/HDF5MasterFile.cpp +++ b/slsReceiverSoftware/src/HDF5MasterFile.cpp @@ -3,30 +3,12 @@ #include "HDF5MasterFile.h" #include "MasterAttributes.h" -HDF5MasterFile::HDF5MasterFile(std::mutex *hdf5Lib) - : File(HDF5), hdf5Lib_(hdf5Lib) {} - -HDF5MasterFile::~HDF5MasterFile() { CloseFile(); } - -void HDF5MasterFile::CloseFile() { - std::lock_guard lock(*hdf5Lib_); - try { - Exception::dontPrint(); // to handle errors - if (fd_) { - fd_->close(); - delete fd_; - fd_ = nullptr; - } - } catch (const Exception &error) { - LOG(logERROR) << "Could not close master HDF5 handles"; - error.printErrorStack(); - } -} - -void HDF5MasterFile::LinkDataFile(std::string dataFilename, - std::string dataSetname, +void HDF5MasterFile::LinkDataFile(const std::string &masterFileName, + const std::string &dataFilename, + const std::string &dataSetname, const std::vector parameterNames, - const bool silentMode) { + const bool silentMode, + std::mutex *hdf5LibMutex) { std::lock_guard lock(*hdf5Lib_); try { @@ -36,7 +18,7 @@ void HDF5MasterFile::LinkDataFile(std::string dataFilename, flist.setFcloseDegree(H5F_CLOSE_STRONG); // open master file - H5File masterfd(fileName_.c_str(), H5F_ACC_RDWR, + H5File masterfd(masterFileName.c_str(), H5F_ACC_RDWR, FileCreatPropList::DEFAULT, flist); // open data file @@ -69,7 +51,7 @@ void HDF5MasterFile::LinkDataFile(std::string dataFilename, masterfd.close(); } catch (const Exception &error) { error.printErrorStack(); - CloseFile(); + fd.close(); throw sls::RuntimeError("Could not link in master hdf5 file"); } if (!silentMode) { @@ -77,17 +59,15 @@ void HDF5MasterFile::LinkDataFile(std::string dataFilename, } } -void HDF5MasterFile::CreateMasterFile(const std::string filePath, - const std::string fileNamePrefix, - const uint64_t fileIndex, - const bool overWriteEnable, - const bool silentMode, - MasterAttributes *attr) { +std::string HDF5MasterFile::CreateMasterFile( + const std::string filePath, const std::string fileNamePrefix, + const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode, + MasterAttributes *attr, std::mutex *hdf5LibMutex) { std::ostringstream os; os << filePath << "/" << fileNamePrefix << "_master" << "_" << fileIndex << ".h5"; - fileName_ = os.str(); + std::string fileName = os.str(); std::lock_guard lock(*hdf5Lib_); @@ -96,67 +76,39 @@ void HDF5MasterFile::CreateMasterFile(const std::string filePath, FileAccPropList flist; flist.setFcloseDegree(H5F_CLOSE_STRONG); - fd_ = nullptr; - if (!(overWriteEnable)) - fd_ = new H5File(fileName_.c_str(), H5F_ACC_EXCL, - FileCreatPropList::DEFAULT, flist); - else - fd_ = new H5File(fileName_.c_str(), H5F_ACC_TRUNC, - FileCreatPropList::DEFAULT, flist); + + unsigned int createFlags = H5F_ACC_EXCL; + if (overWriteEnable) { + createFlags = H5F_ACC_TRUNC; + } + H5File fd(fileName.c_str(), createFlags, FileCreatPropList::DEFAULT, + flist); // attributes - version double dValue = HDF5_WRITER_VERSION; DataSpace dataspace_attr = DataSpace(H5S_SCALAR); - Attribute attribute = fd_->createAttribute( + Attribute attribute = fd.createAttribute( "version", PredType::NATIVE_DOUBLE, dataspace_attr); attribute.write(PredType::NATIVE_DOUBLE, &dValue); // Create a group in the file - Group group1(fd_->createGroup("entry")); + Group group1(fd.createGroup("entry")); Group group2(group1.createGroup("data")); Group group3(group1.createGroup("instrument")); Group group4(group3.createGroup("beam")); Group group5(group3.createGroup("detector")); Group group6(group1.createGroup("sample")); - // TODO find a way to get complete group link - attrGroupName_ = "/entry/instrument/detector"; - - attr->WriteMasterHDF5Attributes(fd_, &group5); - fd_->close(); - + attr->WriteMasterHDF5Attributes(&fd, &group5); + fd.close(); } catch (const Exception &error) { error.printErrorStack(); - CloseFile(); + fd.close(); throw sls::RuntimeError( "Could not create/overwrite master HDF5 handles"); } if (!silentMode) { - LOG(logINFO) << "Master File: " << fileName_; + LOG(logINFO) << "Master File: " << fileName; } + return fileName; } - -void HDF5MasterFile::UpdateMasterFile(MasterAttributes *attr, bool silentMode) { - std::lock_guard lock(*hdf5Lib_); - - try { - Exception::dontPrint(); // to handle errors - FileAccPropList flist; - flist.setFcloseDegree(H5F_CLOSE_STRONG); - fd_ = new H5File(fileName_.c_str(), H5F_ACC_RDWR, - FileCreatPropList::DEFAULT, flist); - - Group group = fd_->openGroup(attrGroupName_.c_str()); - attr->WriteFinalHDF5Attributes(fd_, &group); - fd_->close(); - - } catch (const Exception &error) { - error.printErrorStack(); - CloseFile(); - throw sls::RuntimeError( - "Could not create/overwrite master HDF5 handles"); - } - if (!silentMode) { - LOG(logINFO) << "Updated Master File"; - } -} \ No newline at end of file diff --git a/slsReceiverSoftware/src/HDF5MasterFile.h b/slsReceiverSoftware/src/HDF5MasterFile.h index 350a2da90..bb5431ab7 100644 --- a/slsReceiverSoftware/src/HDF5MasterFile.h +++ b/slsReceiverSoftware/src/HDF5MasterFile.h @@ -7,23 +7,16 @@ #include class HDF5MasterFile : private virtual slsDetectorDefs, public File { - public: - HDF5MasterFile(std::mutex *hdf5Lib); - ~HDF5MasterFile(); - - void LinkDataFile(std::string dataFilename, std::string dataSetname, + void LinkDataFile(const std::string &masterFileName, + const std::string &dataFilename, + const &std::string dataSetname, const std::vector parameterNames, - const bool silentMode) override; - void CreateMasterFile(const std::string filePath, - const std::string fileNamePrefix, - const uint64_t fileIndex, const bool overWriteEnable, - const bool silentMode, - MasterAttributes *attr) override; - - private: - std::mutex *hdf5Lib_; - H5File *fd_{nullptr}; - std::string fileName_; - std::string attrGroupName_; + const bool silentMode, std::mutex *hdf5LibMutex); + std::string CreateMasterFile(const std::string &filePath, + const std::string &fileNamePrefix, + const uint64_t fileIndex, + const bool overWriteEnable, + const bool silentMode, MasterAttributes *attr, + std::mutex *hdf5LibMutex); }; \ No newline at end of file diff --git a/slsReceiverSoftware/src/HDF5VirtualFile.cpp b/slsReceiverSoftware/src/HDF5VirtualFile.cpp index 015ad0ae3..fe1f0d4e3 100644 --- a/slsReceiverSoftware/src/HDF5VirtualFile.cpp +++ b/slsReceiverSoftware/src/HDF5VirtualFile.cpp @@ -5,8 +5,8 @@ #include -HDF5VirtualFile::HDF5VirtualFile(std::mutex *hdf5Lib, bool g25) - : File(HDF5), hdf5Lib_(hdf5Lib), gotthard25um(g25) {} +HDF5VirtualFile::HDF5VirtualFile(std::mutex *hdf5LibMutex, bool g25) + : File(HDF5), hdf5Lib_(hdf5LibMutex), gotthard25um(g25) {} HDF5VirtualFile::~HDF5VirtualFile() { CloseFile(); } @@ -30,7 +30,7 @@ void HDF5VirtualFile::CloseFile() { } void HDF5VirtualFile::CreateVirtualFile( - const std::string filePath, const std::string fileNamePrefix, + const &std::string filePath, const &std::string fileNamePrefix, const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode, const int modulePos, const int numUnitsPerReadout, const uint32_t maxFramesPerFile, const uint64_t numImages, diff --git a/slsReceiverSoftware/src/HDF5VirtualFile.h b/slsReceiverSoftware/src/HDF5VirtualFile.h index 89b1c0b60..eee01dad5 100644 --- a/slsReceiverSoftware/src/HDF5VirtualFile.h +++ b/slsReceiverSoftware/src/HDF5VirtualFile.h @@ -9,13 +9,13 @@ class HDF5VirtualFile : private virtual slsDetectorDefs, public File { public: - HDF5VirtualFile(std::mutex *hdf5Lib, bool g25); + HDF5VirtualFile(std::mutex *hdf5LibMutex, bool g25); ~HDF5VirtualFile(); std::array GetFileAndDatasetName() const override; void CloseFile() override; void CreateVirtualFile( - const std::string filePath, const std::string fileNamePrefix, + const std::string &filePath, const std::string &fileNamePrefix, const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode, const int modulePos, const int numUnitsPerReadout, const uint32_t maxFramesPerFile, diff --git a/slsReceiverSoftware/src/Implementation.cpp b/slsReceiverSoftware/src/Implementation.cpp index 64cff45ae..4d979f7da 100644 --- a/slsReceiverSoftware/src/Implementation.cpp +++ b/slsReceiverSoftware/src/Implementation.cpp @@ -369,7 +369,7 @@ void Implementation::setFileFormat(const fileFormat f) { throw sls::RuntimeError("Unknown file format"); } for (const auto &it : dataProcessor) - it->SetupFileWriter(fileWriteEnable, fileFormatType, &hdf5Lib); + it->SetupFileWriter(fileWriteEnable, fileFormatType, &hdf5LibMutex); } LOG(logINFO) << "File Format: " << sls::ToString(fileFormatType); @@ -405,7 +405,7 @@ void Implementation::setFileWriteEnable(const bool b) { if (fileWriteEnable != b) { fileWriteEnable = b; for (const auto &it : dataProcessor) - it->SetupFileWriter(fileWriteEnable, fileFormatType, &hdf5Lib); + it->SetupFileWriter(fileWriteEnable, fileFormatType, &hdf5LibMutex); } LOG(logINFO) << "File Write Enable: " << (fileWriteEnable ? "enabled" : "disabled"); @@ -741,116 +741,121 @@ void Implementation::SetupWriter() { } void Implementation::StartMasterWriter() { - // master file - if (masterFileWriteEnable) { - std::unique_ptr masterAttributes; - switch (detType) { - case GOTTHARD: - masterAttributes = sls::make_unique(); - break; - case JUNGFRAU: - masterAttributes = sls::make_unique(); - break; - case EIGER: - masterAttributes = sls::make_unique(); - break; - case MYTHEN3: - masterAttributes = sls::make_unique(); - break; - case GOTTHARD2: - masterAttributes = sls::make_unique(); - break; - case MOENCH: - masterAttributes = sls::make_unique(); - break; - case CHIPTESTBOARD: - masterAttributes = sls::make_unique(); - break; - default: - throw sls::RuntimeError( - "Unknown detector type to set up master file attributes"); - } - masterAttributes->detType = detType; - masterAttributes->timingMode = timingMode; - xy nm{numModules.x, numModules.y}; - if (quadEnable) { - nm.x = 1; - nm.y = 2; - } - masterAttributes->geometry = xy(nm.x, nm.y); - masterAttributes->imageSize = generalData->imageSize; - masterAttributes->nPixels = - xy(generalData->nPixelsX, generalData->nPixelsY); - masterAttributes->maxFramesPerFile = framesPerFile; - masterAttributes->frameDiscardMode = frameDiscardMode; - masterAttributes->framePadding = framePadding; - masterAttributes->scanParams = scanParams; - masterAttributes->totalFrames = numberOfTotalFrames; - masterAttributes->exptime = acquisitionTime; - masterAttributes->period = acquisitionPeriod; - masterAttributes->burstMode = burstMode; - masterAttributes->numUDPInterfaces = numUDPInterfaces; - masterAttributes->dynamicRange = dynamicRange; - masterAttributes->tenGiga = tengigaEnable; - masterAttributes->thresholdEnergyeV = thresholdEnergyeV; - masterAttributes->thresholdAllEnergyeV = thresholdAllEnergyeV; - masterAttributes->subExptime = subExpTime; - masterAttributes->subPeriod = subPeriod; - masterAttributes->quad = quadEnable; - masterAttributes->readNRows = readNRows; - masterAttributes->ratecorr = rateCorrections; - masterAttributes->adcmask = - tengigaEnable ? adcEnableMaskTenGiga : adcEnableMaskOneGiga; - masterAttributes->analog = - (readoutType == ANALOG_ONLY || readoutType == ANALOG_AND_DIGITAL) - ? 1 - : 0; - masterAttributes->analogSamples = numberOfAnalogSamples; - masterAttributes->digital = - (readoutType == DIGITAL_ONLY || readoutType == ANALOG_AND_DIGITAL) - ? 1 - : 0; - masterAttributes->digitalSamples = numberOfDigitalSamples; - masterAttributes->dbitoffset = ctbDbitOffset; - masterAttributes->dbitlist = 0; - for (auto &i : ctbDbitList) { - masterAttributes->dbitlist |= (1 << i); - } - masterAttributes->roi = roi; - masterAttributes->counterMask = counterMask; - masterAttributes->exptime1 = acquisitionTime1; - masterAttributes->exptime2 = acquisitionTime2; - masterAttributes->exptime3 = acquisitionTime3; - masterAttributes->gateDelay1 = gateDelay1; - masterAttributes->gateDelay2 = gateDelay2; - masterAttributes->gateDelay3 = gateDelay3; - masterAttributes->gates = numberOfGates; - masterAttributes->additionalJsonHeader = additionalJsonHeader; - - try { - dataProcessor[0]->CreateMasterFile( - filePath, fileName, fileIndex, overwriteEnable, silentMode, - fileFormatType, masterAttributes.get()); - } catch (...) { - ; // ignore it and just print it - } - } -#ifdef HDF5C - if (fileFormatType == HDF5) { - // virtual hdf5 file (if multiple files) - if (dataProcessor[0]->GetFilesInAcquisition() > 1 || - (numModules.x * numModules.y) > 1) { - dataProcessor[0]->CreateVirtualFile( - filePath, fileName, fileIndex, overwriteEnable, silentMode, - modulePos, numUDPInterfaces, framesPerFile, numberOfTotalFrames, - dynamicRange, numModules.x, numModules.y, &hdf5Lib); - } - // link file in master + try { + std::string masterFileName; + // master file if (masterFileWriteEnable) { - dataProcessor[0]->LinkDataInMasterFile(silentMode); + std::unique_ptr masterAttributes{nullptr}; + switch (detType) { + case GOTTHARD: + masterAttributes = sls::make_unique(); + break; + case JUNGFRAU: + masterAttributes = sls::make_unique(); + break; + case EIGER: + masterAttributes = sls::make_unique(); + break; + case MYTHEN3: + masterAttributes = sls::make_unique(); + break; + case GOTTHARD2: + masterAttributes = + sls::make_unique(); + break; + case MOENCH: + masterAttributes = sls::make_unique(); + break; + case CHIPTESTBOARD: + masterAttributes = sls::make_unique(); + break; + default: + throw sls::RuntimeError( + "Unknown detector type to set up master file attributes"); + } + masterAttributes->detType = detType; + masterAttributes->timingMode = timingMode; + xy nm{numModules.x, numModules.y}; + if (quadEnable) { + nm.x = 1; + nm.y = 2; + } + masterAttributes->geometry = xy(nm.x, nm.y); + masterAttributes->imageSize = generalData->imageSize; + masterAttributes->nPixels = + xy(generalData->nPixelsX, generalData->nPixelsY); + masterAttributes->maxFramesPerFile = framesPerFile; + masterAttributes->frameDiscardMode = frameDiscardMode; + masterAttributes->framePadding = framePadding; + masterAttributes->scanParams = scanParams; + masterAttributes->totalFrames = numberOfTotalFrames; + masterAttributes->exptime = acquisitionTime; + masterAttributes->period = acquisitionPeriod; + masterAttributes->burstMode = burstMode; + masterAttributes->numUDPInterfaces = numUDPInterfaces; + masterAttributes->dynamicRange = dynamicRange; + masterAttributes->tenGiga = tengigaEnable; + masterAttributes->thresholdEnergyeV = thresholdEnergyeV; + masterAttributes->thresholdAllEnergyeV = thresholdAllEnergyeV; + masterAttributes->subExptime = subExpTime; + masterAttributes->subPeriod = subPeriod; + masterAttributes->quad = quadEnable; + masterAttributes->readNRows = readNRows; + masterAttributes->ratecorr = rateCorrections; + masterAttributes->adcmask = + tengigaEnable ? adcEnableMaskTenGiga : adcEnableMaskOneGiga; + masterAttributes->analog = (readoutType == ANALOG_ONLY || + readoutType == ANALOG_AND_DIGITAL) + ? 1 + : 0; + masterAttributes->analogSamples = numberOfAnalogSamples; + masterAttributes->digital = (readoutType == DIGITAL_ONLY || + readoutType == ANALOG_AND_DIGITAL) + ? 1 + : 0; + masterAttributes->digitalSamples = numberOfDigitalSamples; + masterAttributes->dbitoffset = ctbDbitOffset; + masterAttributes->dbitlist = 0; + for (auto &i : ctbDbitList) { + masterAttributes->dbitlist |= (1 << i); + } + masterAttributes->roi = roi; + masterAttributes->counterMask = counterMask; + masterAttributes->exptime1 = acquisitionTime1; + masterAttributes->exptime2 = acquisitionTime2; + masterAttributes->exptime3 = acquisitionTime3; + masterAttributes->gateDelay1 = gateDelay1; + masterAttributes->gateDelay2 = gateDelay2; + masterAttributes->gateDelay3 = gateDelay3; + masterAttributes->gates = numberOfGates; + masterAttributes->additionalJsonHeader = additionalJsonHeader; + + // create master file + masterFileName = dataProcessor[0]->CreateMasterFile( + filePath, fileName, fileIndex, overwriteEnable, silentMode, + fileFormatType, masterAttributes.get(), &hdf5LibMutex); + } +#ifdef HDF5C + if (fileFormatType == HDF5) { + // create virtual hdf5 file (if multiple files) + if (dataProcessor[0]->GetFilesInAcquisition() > 1 || + (numModules.x * numModules.y) > 1) { + dataProcessor[0]->CreateVirtualFile( + filePath, fileName, fileIndex, overwriteEnable, silentMode, + modulePos, numUDPInterfaces, framesPerFile, + numberOfTotalFrames, dynamicRange, numModules.x, + numModules.y, &hdf5LibMutex); + } + // link file in master + if (masterFileWriteEnable) { + dataProcessor[0]->LinkDataInMasterFile( + masterFileName, silentMode, &hdf5LibMutex); + } } - } #endif + } catch (...) { + ; // ignore it and just print it + } } void Implementation::StartRunning() { diff --git a/slsReceiverSoftware/src/Implementation.h b/slsReceiverSoftware/src/Implementation.h index fdd970139..c792430e5 100644 --- a/slsReceiverSoftware/src/Implementation.h +++ b/slsReceiverSoftware/src/Implementation.h @@ -387,5 +387,6 @@ class Implementation : private virtual slsDetectorDefs { std::vector> fifo; Arping arping; - std::mutex hdf5Lib; + // mutex shared across all hdf5 virtual, master and data files + std::mutex hdf5LibMutex; }; diff --git a/slsReceiverSoftware/src/MasterAttributes.cpp b/slsReceiverSoftware/src/MasterAttributes.cpp index e68de7dd7..f57c8aab6 100644 --- a/slsReceiverSoftware/src/MasterAttributes.cpp +++ b/slsReceiverSoftware/src/MasterAttributes.cpp @@ -2,22 +2,21 @@ // Copyright (C) 2021 Contributors to the SLS Detector Package #include "MasterAttributes.h" -#include - -void MasterAttributes::WriteMasterBinaryAttributes(FILE *fd) { +void MasterAttributes::GetSpecificBinaryAttributes( + rapidjson::Writer *w) { LOG(logERROR) << "WriteMasterBinaryAttributes should have been called " "by a child class"; } -void MasterAttributes::GetBinaryMasterAttributes( +void MasterAttributes::GetCommonBinaryAttributes( rapidjson::Writer *w) { - time_t t = time(nullptr); w->Key("Version"); w->SetMaxDecimalPlaces(2); w->Double(BINARY_WRITER_VERSION); w->Key("Timestamp"); + time_t t = time(nullptr); w->String(ctime(&t)); w->Key("Detector Type"); @@ -61,15 +60,7 @@ void MasterAttributes::GetBinaryMasterAttributes( w->Uint64(totalFrames); }; -void MasterAttributes::WriteBinaryAttributes(FILE *fd, std::string message) { - if (fwrite((void *)message.c_str(), 1, message.length(), fd) != - message.length()) { - throw sls::RuntimeError( - "Master binary file incorrect number of bytes written to file"); - } -}; - -void MasterAttributes::WriteFinalBinaryAttributes( +void MasterAttributes::GetFinalBinaryAttributes( rapidjson::Writer *w) { // adding few common parameters to the end @@ -309,25 +300,15 @@ void MasterAttributes::WriteHDF5Attributes(H5File *fd, Group *group) { }; #endif - void GotthardMasterAttributes::WriteMasterBinaryAttributes(FILE *fd) { - rapidjson::StringBuffer s; - rapidjson::Writer writer(s); - writer.StartObject(); - MasterAttributes::GetBinaryMasterAttributes(&writer); - - writer.Key("Exptime"); - writer.String(sls::ToString(exptime).c_str()); + void GotthardMasterAttributes::GetSpecificBinaryAttributes(rapidjson::Writer *w) { + w->Key("Exptime"); + w->String(sls::ToString(exptime).c_str()); - writer.Key("Period"); - writer.String(sls::ToString(period).c_str()); + w->Key("Period"); + w->String(sls::ToString(period).c_str()); - writer.Key("Roi (xmin, xmax)"); - writer.String(sls::ToString(roi).c_str()); - - MasterAttributes::WriteFinalBinaryAttributes(&writer); - writer.EndObject(); - std::string message = s.GetString(); - MasterAttributes::WriteBinaryAttributes(fd, message); + w->Key("Roi (xmin, xmax)"); + w->String(sls::ToString(roi).c_str()); }; #ifdef HDF5C @@ -349,31 +330,22 @@ void MasterAttributes::WriteHDF5Attributes(H5File *fd, Group *group) { "roi xmax", PredType::NATIVE_INT, dataspace); dataset.write(&roi.xmax, PredType::NATIVE_INT); } + MasterAttributes::WriteFinalHDF5Attributes(fd, group); }; #endif - void JungfrauMasterAttributes::WriteMasterBinaryAttributes(FILE *fd) { - rapidjson::StringBuffer s; - rapidjson::Writer writer(s); - writer.StartObject(); - MasterAttributes::GetBinaryMasterAttributes(&writer); - - writer.Key("Exptime"); - writer.String(sls::ToString(exptime).c_str()); + void JungfrauMasterAttributes::GetSpecificBinaryAttributes(rapidjson::Writer *w) { + w->Key("Exptime"); + w->String(sls::ToString(exptime).c_str()); - writer.Key("Period"); - writer.String(sls::ToString(period).c_str()); + w->Key("Period"); + w->String(sls::ToString(period).c_str()); - writer.Key("Number of UDP Interfaces"); - writer.Uint(numUDPInterfaces); + w->Key("Number of UDP Interfaces"); + w->Uint(numUDPInterfaces); - writer.Key("Number of rows"); - writer.Uint(readNRows); - - MasterAttributes::WriteFinalBinaryAttributes(&writer); - writer.EndObject(); - std::string message = s.GetString(); - MasterAttributes::WriteBinaryAttributes(fd, message); + w->Key("Number of rows"); + w->Uint(readNRows); }; #ifdef HDF5C @@ -394,49 +366,40 @@ void MasterAttributes::WriteHDF5Attributes(H5File *fd, Group *group) { "Number of rows", PredType::NATIVE_INT, dataspace); dataset.write(&readNRows, PredType::NATIVE_INT); } + MasterAttributes::WriteFinalHDF5Attributes(fd, group); }; #endif - void EigerMasterAttributes::WriteMasterBinaryAttributes(FILE *fd) { - rapidjson::StringBuffer s; - rapidjson::Writer writer(s); - writer.StartObject(); - MasterAttributes::GetBinaryMasterAttributes(&writer); + void EigerMasterAttributes::GetSpecificBinaryAttributes(rapidjson::Writer *w) { + w->Key("Dynamic Range"); + w->Uint(dynamicRange); - writer.Key("Dynamic Range"); - writer.Uint(dynamicRange); + w->Key("Ten Giga"); + w->Uint(tenGiga); - writer.Key("Ten Giga"); - writer.Uint(tenGiga); - - writer.Key("Exptime"); - writer.String(sls::ToString(exptime).c_str()); + w->Key("Exptime"); + w->String(sls::ToString(exptime).c_str()); - writer.Key("Period"); - writer.String(sls::ToString(period).c_str()); + w->Key("Period"); + w->String(sls::ToString(period).c_str()); - writer.Key("Threshold Energy"); - writer.Int(thresholdEnergyeV); + w->Key("Threshold Energy"); + w->Int(thresholdEnergyeV); - writer.Key("Sub Exptime"); - writer.String(sls::ToString(subExptime).c_str()); + w->Key("Sub Exptime"); + w->String(sls::ToString(subExptime).c_str()); - writer.Key("Sub Period"); - writer.String(sls::ToString(subPeriod).c_str()); + w->Key("Sub Period"); + w->String(sls::ToString(subPeriod).c_str()); - writer.Key("Quad"); - writer.Int(quad); + w->Key("Quad"); + w->Int(quad); - writer.Key("Number of rows"); - writer.Int(readNRows); + w->Key("Number of rows"); + w->Int(readNRows); - writer.Key("Rate Corrections"); - writer.String(sls::ToString(ratecorr).c_str()); - - MasterAttributes::WriteFinalBinaryAttributes(&writer); - writer.EndObject(); - std::string message = s.GetString(); - MasterAttributes::WriteBinaryAttributes(fd, message); + w->Key("Rate Corrections"); + w->String(sls::ToString(ratecorr).c_str()); }; #ifdef HDF5C @@ -502,55 +465,46 @@ void MasterAttributes::WriteHDF5Attributes(H5File *fd, Group *group) { sls::strcpy_safe(c, sls::ToString(ratecorr)); dataset.write(c, strdatatype); } + MasterAttributes::WriteFinalHDF5Attributes(fd, group); }; #endif - void Mythen3MasterAttributes::WriteMasterBinaryAttributes(FILE *fd) { - rapidjson::StringBuffer s; - rapidjson::Writer writer(s); - writer.StartObject(); - MasterAttributes::GetBinaryMasterAttributes(&writer); + void Mythen3MasterAttributes::GetSpecificBinaryAttributes(rapidjson::Writer *w) { + w->Key("Dynamic Range"); + w->Uint(dynamicRange); - writer.Key("Dynamic Range"); - writer.Uint(dynamicRange); + w->Key("Ten Giga"); + w->Uint(tenGiga); - writer.Key("Ten Giga"); - writer.Uint(tenGiga); + w->Key("Period"); + w->String(sls::ToString(period).c_str()); - writer.Key("Period"); - writer.String(sls::ToString(period).c_str()); + w->Key("Counter Mask"); + w->String(sls::ToStringHex(counterMask).c_str()); - writer.Key("Counter Mask"); - writer.String(sls::ToStringHex(counterMask).c_str()); - - writer.Key("Exptime1"); - writer.String(sls::ToString(exptime1).c_str()); + w->Key("Exptime1"); + w->String(sls::ToString(exptime1).c_str()); - writer.Key("Exptime2"); - writer.String(sls::ToString(exptime2).c_str()); + w->Key("Exptime2"); + w->String(sls::ToString(exptime2).c_str()); - writer.Key("Exptime3"); - writer.String(sls::ToString(exptime3).c_str()); + w->Key("Exptime3"); + w->String(sls::ToString(exptime3).c_str()); - writer.Key("GateDelay1"); - writer.String(sls::ToString(gateDelay1).c_str()); + w->Key("GateDelay1"); + w->String(sls::ToString(gateDelay1).c_str()); - writer.Key("GateDelay2"); - writer.String(sls::ToString(gateDelay2).c_str()); + w->Key("GateDelay2"); + w->String(sls::ToString(gateDelay2).c_str()); - writer.Key("GateDelay3"); - writer.String(sls::ToString(gateDelay3).c_str()); + w->Key("GateDelay3"); + w->String(sls::ToString(gateDelay3).c_str()); - writer.Key("Gates"); - writer.Uint(gates); + w->Key("Gates"); + w->Uint(gates); - writer.Key("Threshold Energies"); - writer.String(sls::ToString(thresholdAllEnergyeV).c_str()); - - MasterAttributes::WriteFinalBinaryAttributes(&writer); - writer.EndObject(); - std::string message = s.GetString(); - MasterAttributes::WriteBinaryAttributes(fd, message); + w->Key("Threshold Energies"); + w->String(sls::ToString(thresholdAllEnergyeV).c_str()); }; #ifdef HDF5C @@ -638,28 +592,19 @@ void MasterAttributes::WriteHDF5Attributes(H5File *fd, Group *group) { sls::strcpy_safe(c, sls::ToString(thresholdAllEnergyeV)); dataset.write(c, strdatatype); } + MasterAttributes::WriteFinalHDF5Attributes(fd, group); }; #endif - void Gotthard2MasterAttributes::WriteMasterBinaryAttributes(FILE *fd) { - rapidjson::StringBuffer s; - rapidjson::Writer writer(s); - writer.StartObject(); - MasterAttributes::GetBinaryMasterAttributes(&writer); - - writer.Key("Exptime"); - writer.String(sls::ToString(exptime).c_str()); + void Gotthard2MasterAttributes::GetSpecificBinaryAttributes(rapidjson::Writer *w) { + w->Key("Exptime"); + w->String(sls::ToString(exptime).c_str()); - writer.Key("Period"); - writer.String(sls::ToString(period).c_str()); + w->Key("Period"); + w->String(sls::ToString(period).c_str()); - writer.Key("Burst Mode"); - writer.String(sls::ToString(burstMode).c_str()); - - MasterAttributes::WriteFinalBinaryAttributes(&writer); - writer.EndObject(); - std::string message = s.GetString(); - MasterAttributes::WriteBinaryAttributes(fd, message); + w->Key("Burst Mode"); + w->String(sls::ToString(burstMode).c_str()); }; #ifdef HDF5C @@ -678,34 +623,25 @@ void MasterAttributes::WriteHDF5Attributes(H5File *fd, Group *group) { sls::strcpy_safe(c, sls::ToString(burstMode)); dataset.write(c, strdatatype); } + MasterAttributes::WriteFinalHDF5Attributes(fd, group); }; #endif - void MoenchMasterAttributes::WriteMasterBinaryAttributes(FILE *fd) { - rapidjson::StringBuffer s; - rapidjson::Writer writer(s); - writer.StartObject(); - MasterAttributes::GetBinaryMasterAttributes(&writer); - - writer.Key("Exptime"); - writer.String(sls::ToString(exptime).c_str()); + void MoenchMasterAttributes::GetSpecificBinaryAttributes(rapidjson::Writer *w) { + w->Key("Exptime"); + w->String(sls::ToString(exptime).c_str()); - writer.Key("Period"); - writer.String(sls::ToString(period).c_str()); + w->Key("Period"); + w->String(sls::ToString(period).c_str()); - writer.Key("Ten Giga"); - writer.Uint(tenGiga); + w->Key("Ten Giga"); + w->Uint(tenGiga); - writer.Key("ADC Mask"); - writer.String(sls::ToStringHex(adcmask).c_str()); + w->Key("ADC Mask"); + w->String(sls::ToStringHex(adcmask).c_str()); - writer.Key("Analog Samples"); - writer.Uint(analogSamples); - - MasterAttributes::WriteFinalBinaryAttributes(&writer); - writer.EndObject(); - std::string message = s.GetString(); - MasterAttributes::WriteBinaryAttributes(fd, message); + w->Key("Analog Samples"); + w->Uint(analogSamples); }; #ifdef HDF5C @@ -728,49 +664,40 @@ void MasterAttributes::WriteHDF5Attributes(H5File *fd, Group *group) { "Analog Samples", PredType::NATIVE_INT, dataspace); dataset.write(&analogSamples, PredType::NATIVE_INT); } + MasterAttributes::WriteFinalHDF5Attributes(fd, group); }; #endif - void CtbMasterAttributes::WriteMasterBinaryAttributes(FILE *fd) { - rapidjson::StringBuffer s; - rapidjson::Writer writer(s); - writer.StartObject(); - MasterAttributes::GetBinaryMasterAttributes(&writer); - - writer.Key("Exptime"); - writer.String(sls::ToString(exptime).c_str()); + void CtbMasterAttributes::GetSpecificBinaryAttributes(rapidjson::Writer *w) { + w->Key("Exptime"); + w->String(sls::ToString(exptime).c_str()); - writer.Key("Period"); - writer.String(sls::ToString(period).c_str()); + w->Key("Period"); + w->String(sls::ToString(period).c_str()); - writer.Key("Ten Giga"); - writer.Uint(tenGiga); + w->Key("Ten Giga"); + w->Uint(tenGiga); - writer.Key("ADC Mask"); - writer.String(sls::ToStringHex(adcmask).c_str()); + w->Key("ADC Mask"); + w->String(sls::ToStringHex(adcmask).c_str()); - writer.Key("Analog Flag"); - writer.Uint(analog); + w->Key("Analog Flag"); + w->Uint(analog); - writer.Key("Analog Samples"); - writer.Uint(analogSamples); + w->Key("Analog Samples"); + w->Uint(analogSamples); - writer.Key("Digital Flag"); - writer.Uint(digital); + w->Key("Digital Flag"); + w->Uint(digital); - writer.Key("Digital Samples"); - writer.Uint(digitalSamples); + w->Key("Digital Samples"); + w->Uint(digitalSamples); - writer.Key("Dbit Offset"); - writer.Uint(dbitoffset); + w->Key("Dbit Offset"); + w->Uint(dbitoffset); - writer.Key("Dbit Bitset"); - writer.Uint64(dbitlist); - - MasterAttributes::WriteFinalBinaryAttributes(&writer); - writer.EndObject(); - std::string message = s.GetString(); - MasterAttributes::WriteBinaryAttributes(fd, message); + w->Key("Dbit Bitset"); + w->Uint64(dbitlist); }; #ifdef HDF5C @@ -828,5 +755,6 @@ void MasterAttributes::WriteHDF5Attributes(H5File *fd, Group *group) { "Dbit Bitset List", PredType::STD_U64LE, dataspace); dataset.write(&dbitlist, PredType::STD_U64LE); } + MasterAttributes::WriteFinalHDF5Attributes(fd, group); }; #endif diff --git a/slsReceiverSoftware/src/MasterAttributes.h b/slsReceiverSoftware/src/MasterAttributes.h index 58a29e8a9..27c7ba13b 100644 --- a/slsReceiverSoftware/src/MasterAttributes.h +++ b/slsReceiverSoftware/src/MasterAttributes.h @@ -7,6 +7,7 @@ #include "sls/logger.h" #include "sls/sls_detector_defs.h" +#include #include #ifdef HDF5C @@ -69,12 +70,12 @@ class MasterAttributes { MasterAttributes() = default; virtual ~MasterAttributes() = default; - virtual void WriteMasterBinaryAttributes(FILE *fd); + virtual void + GetSpecificBinaryAttributes(rapidjson::Writer *w); void - GetBinaryMasterAttributes(rapidjson::Writer *w); - void WriteBinaryAttributes(FILE *fd, std::string message); + GetCommonBinaryAttributes(rapidjson::Writer *w); void - WriteFinalBinaryAttributes(rapidjson::Writer *w); + GetFinalBinaryAttributes(rapidjson::Writer *w); #ifdef HDF5C virtual void WriteMasterHDF5Attributes(H5File *fd, Group *group); void WriteHDF5Attributes(H5File *fd, Group *group); @@ -89,7 +90,8 @@ class MasterAttributes { class GotthardMasterAttributes : public MasterAttributes { public: GotthardMasterAttributes() = default; - void WriteMasterBinaryAttributes(FILE *fd) override; + void GetSpecificBinaryAttributes( + rapidjson::Writer *w) override; #ifdef HDF5C void WriteMasterHDF5Attributes(H5File *fd, Group *group) override; #endif @@ -98,7 +100,8 @@ class GotthardMasterAttributes : public MasterAttributes { class JungfrauMasterAttributes : public MasterAttributes { public: JungfrauMasterAttributes() = default; - void WriteMasterBinaryAttributes(FILE *fd) override; + void GetSpecificBinaryAttributes( + rapidjson::Writer *w) override; #ifdef HDF5C void WriteMasterHDF5Attributes(H5File *fd, Group *group) override; #endif @@ -107,7 +110,8 @@ class JungfrauMasterAttributes : public MasterAttributes { class EigerMasterAttributes : public MasterAttributes { public: EigerMasterAttributes() = default; - void WriteMasterBinaryAttributes(FILE *fd) override; + void GetSpecificBinaryAttributes( + rapidjson::Writer *w) override; #ifdef HDF5C void WriteMasterHDF5Attributes(H5File *fd, Group *group) override; #endif @@ -116,7 +120,8 @@ class EigerMasterAttributes : public MasterAttributes { class Mythen3MasterAttributes : public MasterAttributes { public: Mythen3MasterAttributes() = default; - void WriteMasterBinaryAttributes(FILE *fd) override; + void GetSpecificBinaryAttributes( + rapidjson::Writer *w) override; #ifdef HDF5C void WriteMasterHDF5Attributes(H5File *fd, Group *group) override; #endif @@ -125,7 +130,8 @@ class Mythen3MasterAttributes : public MasterAttributes { class Gotthard2MasterAttributes : public MasterAttributes { public: Gotthard2MasterAttributes() = default; - void WriteMasterBinaryAttributes(FILE *fd) override; + void GetSpecificBinaryAttributes( + rapidjson::Writer *w) override; #ifdef HDF5C void WriteMasterHDF5Attributes(H5File *fd, Group *group) override; #endif @@ -134,7 +140,8 @@ class Gotthard2MasterAttributes : public MasterAttributes { class MoenchMasterAttributes : public MasterAttributes { public: MoenchMasterAttributes() = default; - void WriteMasterBinaryAttributes(FILE *fd) override; + void GetSpecificBinaryAttributes( + rapidjson::Writer *w) override; #ifdef HDF5C void WriteMasterHDF5Attributes(H5File *fd, Group *group) override; #endif @@ -143,7 +150,8 @@ class MoenchMasterAttributes : public MasterAttributes { class CtbMasterAttributes : public MasterAttributes { public: CtbMasterAttributes() = default; - void WriteMasterBinaryAttributes(FILE *fd) override; + void GetSpecificBinaryAttributes( + rapidjson::Writer *w) override; #ifdef HDF5C void WriteMasterHDF5Attributes(H5File *fd, Group *group) override; #endif From 6cd780ae9920e681b4ebaa77524116bbd902fecd Mon Sep 17 00:00:00 2001 From: Dhanya Thattil Date: Tue, 29 Mar 2022 11:49:30 +0200 Subject: [PATCH 05/12] wip --- slsReceiverSoftware/src/BinaryMasterFile.cpp | 8 +- slsReceiverSoftware/src/BinaryMasterFile.h | 1 - slsReceiverSoftware/src/DataProcessor.cpp | 2 +- slsReceiverSoftware/src/DataProcessor.h | 2 +- slsReceiverSoftware/src/HDF5MasterFile.cpp | 2 +- slsReceiverSoftware/src/HDF5MasterFile.h | 6 +- slsReceiverSoftware/src/Implementation.cpp | 119 +- slsReceiverSoftware/src/MasterAttributes.cpp | 1228 +++++++++--------- slsReceiverSoftware/src/MasterAttributes.h | 113 +- 9 files changed, 701 insertions(+), 780 deletions(-) diff --git a/slsReceiverSoftware/src/BinaryMasterFile.cpp b/slsReceiverSoftware/src/BinaryMasterFile.cpp index f6d0942fc..04fc0f2d7 100644 --- a/slsReceiverSoftware/src/BinaryMasterFile.cpp +++ b/slsReceiverSoftware/src/BinaryMasterFile.cpp @@ -47,12 +47,6 @@ std::string BinaryMasterFile::CreateMasterFile(const std::string filePath, std::string BinaryMasterFile::GetMasterAttributes(MasterAttributes *attr) { rapidjson::StringBuffer s; rapidjson::Writer writer(s); - writer.StartObject(); - - attr->GetCommonBinaryAttributes(&writer); - attr->GetSpecificBinaryAttributes(&writer); - attr->GetFinalBinaryAttributes(&writer); - - writer.EndObject(); + attr->GetBinaryAttributes(&writer); return s.GetString(); } diff --git a/slsReceiverSoftware/src/BinaryMasterFile.h b/slsReceiverSoftware/src/BinaryMasterFile.h index aaac736ab..9bfb4504a 100644 --- a/slsReceiverSoftware/src/BinaryMasterFile.h +++ b/slsReceiverSoftware/src/BinaryMasterFile.h @@ -2,7 +2,6 @@ // Copyright (C) 2021 Contributors to the SLS Detector Package #pragma once -#include "File.h" #include "MasterAttributes.h" class BinaryMasterFile : private virtual slsDetectorDefs { diff --git a/slsReceiverSoftware/src/DataProcessor.cpp b/slsReceiverSoftware/src/DataProcessor.cpp index 78985eaef..ed6788981 100644 --- a/slsReceiverSoftware/src/DataProcessor.cpp +++ b/slsReceiverSoftware/src/DataProcessor.cpp @@ -200,7 +200,7 @@ void DataProcessor::CreateVirtualFile( dataFile_->GetParameterNames(), dataFile_->GetParameterDataTypes()); } -void DataProcessor::LinkDataInMasterFile(const string &masterFileName, +void DataProcessor::LinkDataInMasterFile(const std::string &masterFileName, const bool silentMode, std::mutex *hdf5LibMutex) { std::string fname, datasetName; diff --git a/slsReceiverSoftware/src/DataProcessor.h b/slsReceiverSoftware/src/DataProcessor.h index 2b8aacb9f..2c8feed3d 100644 --- a/slsReceiverSoftware/src/DataProcessor.h +++ b/slsReceiverSoftware/src/DataProcessor.h @@ -68,7 +68,7 @@ class DataProcessor : private virtual slsDetectorDefs, public ThreadObject { const uint64_t numImages, const uint32_t dynamicRange, const int numModX, const int numModY, std::mutex *hdf5LibMutex); - void LinkDataInMasterFile(const string &masterFileName, + void LinkDataInMasterFile(const std::string &masterFileName, const bool silentMode, std::mutex *hdf5LibMutex); #endif diff --git a/slsReceiverSoftware/src/HDF5MasterFile.cpp b/slsReceiverSoftware/src/HDF5MasterFile.cpp index ea425f14d..35f15826e 100644 --- a/slsReceiverSoftware/src/HDF5MasterFile.cpp +++ b/slsReceiverSoftware/src/HDF5MasterFile.cpp @@ -99,7 +99,7 @@ std::string HDF5MasterFile::CreateMasterFile( Group group5(group3.createGroup("detector")); Group group6(group1.createGroup("sample")); - attr->WriteMasterHDF5Attributes(&fd, &group5); + attr->WriteHDF5Attributes(&fd, &group5); fd.close(); } catch (const Exception &error) { error.printErrorStack(); diff --git a/slsReceiverSoftware/src/HDF5MasterFile.h b/slsReceiverSoftware/src/HDF5MasterFile.h index bb5431ab7..4a9c3ae7c 100644 --- a/slsReceiverSoftware/src/HDF5MasterFile.h +++ b/slsReceiverSoftware/src/HDF5MasterFile.h @@ -2,15 +2,15 @@ // Copyright (C) 2021 Contributors to the SLS Detector Package #pragma once -#include "File.h" +#include "MasterAttributes.h" #include -class HDF5MasterFile : private virtual slsDetectorDefs, public File { +class HDF5MasterFile : private virtual slsDetectorDefs { public: void LinkDataFile(const std::string &masterFileName, const std::string &dataFilename, - const &std::string dataSetname, + const std::string &dataSetname, const std::vector parameterNames, const bool silentMode, std::mutex *hdf5LibMutex); std::string CreateMasterFile(const std::string &filePath, diff --git a/slsReceiverSoftware/src/Implementation.cpp b/slsReceiverSoftware/src/Implementation.cpp index 4d979f7da..ffa10e806 100644 --- a/slsReceiverSoftware/src/Implementation.cpp +++ b/slsReceiverSoftware/src/Implementation.cpp @@ -745,95 +745,68 @@ void Implementation::StartMasterWriter() { std::string masterFileName; // master file if (masterFileWriteEnable) { - std::unique_ptr masterAttributes{nullptr}; - switch (detType) { - case GOTTHARD: - masterAttributes = sls::make_unique(); - break; - case JUNGFRAU: - masterAttributes = sls::make_unique(); - break; - case EIGER: - masterAttributes = sls::make_unique(); - break; - case MYTHEN3: - masterAttributes = sls::make_unique(); - break; - case GOTTHARD2: - masterAttributes = - sls::make_unique(); - break; - case MOENCH: - masterAttributes = sls::make_unique(); - break; - case CHIPTESTBOARD: - masterAttributes = sls::make_unique(); - break; - default: - throw sls::RuntimeError( - "Unknown detector type to set up master file attributes"); - } - masterAttributes->detType = detType; - masterAttributes->timingMode = timingMode; + MasterAttributes masterAttributes; + masterAttributes.detType = detType; + masterAttributes.timingMode = timingMode; xy nm{numModules.x, numModules.y}; if (quadEnable) { nm.x = 1; nm.y = 2; } - masterAttributes->geometry = xy(nm.x, nm.y); - masterAttributes->imageSize = generalData->imageSize; - masterAttributes->nPixels = + masterAttributes.geometry = xy(nm.x, nm.y); + masterAttributes.imageSize = generalData->imageSize; + masterAttributes.nPixels = xy(generalData->nPixelsX, generalData->nPixelsY); - masterAttributes->maxFramesPerFile = framesPerFile; - masterAttributes->frameDiscardMode = frameDiscardMode; - masterAttributes->framePadding = framePadding; - masterAttributes->scanParams = scanParams; - masterAttributes->totalFrames = numberOfTotalFrames; - masterAttributes->exptime = acquisitionTime; - masterAttributes->period = acquisitionPeriod; - masterAttributes->burstMode = burstMode; - masterAttributes->numUDPInterfaces = numUDPInterfaces; - masterAttributes->dynamicRange = dynamicRange; - masterAttributes->tenGiga = tengigaEnable; - masterAttributes->thresholdEnergyeV = thresholdEnergyeV; - masterAttributes->thresholdAllEnergyeV = thresholdAllEnergyeV; - masterAttributes->subExptime = subExpTime; - masterAttributes->subPeriod = subPeriod; - masterAttributes->quad = quadEnable; - masterAttributes->readNRows = readNRows; - masterAttributes->ratecorr = rateCorrections; - masterAttributes->adcmask = + masterAttributes.maxFramesPerFile = framesPerFile; + masterAttributes.frameDiscardMode = frameDiscardMode; + masterAttributes.framePadding = framePadding; + masterAttributes.scanParams = scanParams; + masterAttributes.totalFrames = numberOfTotalFrames; + masterAttributes.exptime = acquisitionTime; + masterAttributes.period = acquisitionPeriod; + masterAttributes.burstMode = burstMode; + masterAttributes.numUDPInterfaces = numUDPInterfaces; + masterAttributes.dynamicRange = dynamicRange; + masterAttributes.tenGiga = tengigaEnable; + masterAttributes.thresholdEnergyeV = thresholdEnergyeV; + masterAttributes.thresholdAllEnergyeV = thresholdAllEnergyeV; + masterAttributes.subExptime = subExpTime; + masterAttributes.subPeriod = subPeriod; + masterAttributes.quad = quadEnable; + masterAttributes.readNRows = readNRows; + masterAttributes.ratecorr = rateCorrections; + masterAttributes.adcmask = tengigaEnable ? adcEnableMaskTenGiga : adcEnableMaskOneGiga; - masterAttributes->analog = (readoutType == ANALOG_ONLY || + masterAttributes.analog = (readoutType == ANALOG_ONLY || + readoutType == ANALOG_AND_DIGITAL) + ? 1 + : 0; + masterAttributes.analogSamples = numberOfAnalogSamples; + masterAttributes.digital = (readoutType == DIGITAL_ONLY || readoutType == ANALOG_AND_DIGITAL) ? 1 : 0; - masterAttributes->analogSamples = numberOfAnalogSamples; - masterAttributes->digital = (readoutType == DIGITAL_ONLY || - readoutType == ANALOG_AND_DIGITAL) - ? 1 - : 0; - masterAttributes->digitalSamples = numberOfDigitalSamples; - masterAttributes->dbitoffset = ctbDbitOffset; - masterAttributes->dbitlist = 0; + masterAttributes.digitalSamples = numberOfDigitalSamples; + masterAttributes.dbitoffset = ctbDbitOffset; + masterAttributes.dbitlist = 0; for (auto &i : ctbDbitList) { - masterAttributes->dbitlist |= (1 << i); + masterAttributes.dbitlist |= (1 << i); } - masterAttributes->roi = roi; - masterAttributes->counterMask = counterMask; - masterAttributes->exptime1 = acquisitionTime1; - masterAttributes->exptime2 = acquisitionTime2; - masterAttributes->exptime3 = acquisitionTime3; - masterAttributes->gateDelay1 = gateDelay1; - masterAttributes->gateDelay2 = gateDelay2; - masterAttributes->gateDelay3 = gateDelay3; - masterAttributes->gates = numberOfGates; - masterAttributes->additionalJsonHeader = additionalJsonHeader; + masterAttributes.roi = roi; + masterAttributes.counterMask = counterMask; + masterAttributes.exptimeArray[0] = acquisitionTime1; + masterAttributes.exptimeArray[1] = acquisitionTime2; + masterAttributes.exptimeArray[2] = acquisitionTime3; + masterAttributes.gateDelayArray[0] = gateDelay1; + masterAttributes.gateDelayArray[1] = gateDelay2; + masterAttributes.gateDelayArray[2] = gateDelay3; + masterAttributes.gates = numberOfGates; + masterAttributes.additionalJsonHeader = additionalJsonHeader; // create master file masterFileName = dataProcessor[0]->CreateMasterFile( filePath, fileName, fileIndex, overwriteEnable, silentMode, - fileFormatType, masterAttributes.get(), &hdf5LibMutex); + fileFormatType, &masterAttributes, &hdf5LibMutex); } #ifdef HDF5C if (fileFormatType == HDF5) { diff --git a/slsReceiverSoftware/src/MasterAttributes.cpp b/slsReceiverSoftware/src/MasterAttributes.cpp index f57c8aab6..250806cb6 100644 --- a/slsReceiverSoftware/src/MasterAttributes.cpp +++ b/slsReceiverSoftware/src/MasterAttributes.cpp @@ -2,29 +2,85 @@ // Copyright (C) 2021 Contributors to the SLS Detector Package #include "MasterAttributes.h" -void MasterAttributes::GetSpecificBinaryAttributes( +void MasterAttributes::GetBinaryAttributes( rapidjson::Writer *w) { - LOG(logERROR) << "WriteMasterBinaryAttributes should have been called " - "by a child class"; + w->StartObject(); + GetCommonBinaryAttributes(w); + switch (detType) { + case slsDetectorDefs::GOTTHARD: + GetGotthardBinaryAttributes(w); + break; + case slsDetectorDefs::JUNGFRAU: + GetJungfrauBinaryAttributes(w); + break; + case slsDetectorDefs::EIGER: + GetEigerBinaryAttributes(w); + break; + case slsDetectorDefs::MYTHEN3: + GetMythen3BinaryAttributes(w); + break; + case slsDetectorDefs::GOTTHARD2: + GetGotthard2BinaryAttributes(w); + break; + case slsDetectorDefs::MOENCH: + GetMoenchBinaryAttributes(w); + break; + case slsDetectorDefs::CHIPTESTBOARD: + GetCtbBinaryAttributes(w); + break; + default: + throw sls::RuntimeError( + "Unknown Detector type to get master attributes"); + } + GetFinalBinaryAttributes(w); + w->EndObject(); } +#ifdef HDF5C +void MasterAttributes::WriteHDF5Attributes(H5File *fd, Group *group) { + WriteCommonHDF5Attributes(fd, group); + switch (detType) { + case slsDetectorDefs::GOTTHARD: + WriteGotthardHDF5Attributes(fd, group); + break; + case slsDetectorDefs::JUNGFRAU: + WriteJungfrauHDF5Attributes(fd, group); + break; + case slsDetectorDefs::EIGER: + WriteEigerHDF5Attributes(fd, group); + break; + case slsDetectorDefs::MYTHEN3: + WriteMythen3HDF5Attributes(fd, group); + break; + case slsDetectorDefs::GOTTHARD2: + WriteGotthard2HDF5Attributes(fd, group); + break; + case slsDetectorDefs::MOENCH: + WriteMoenchHDF5Attributes(fd, group); + break; + case slsDetectorDefs::CHIPTESTBOARD: + WriteCtbHDF5Attributes(fd, group); + break; + default: + throw sls::RuntimeError( + "Unknown Detector type to get master attributes"); + } + WriteFinalHDF5Attributes(fd, group); +} +#endif + void MasterAttributes::GetCommonBinaryAttributes( rapidjson::Writer *w) { - w->Key("Version"); w->SetMaxDecimalPlaces(2); w->Double(BINARY_WRITER_VERSION); - w->Key("Timestamp"); time_t t = time(nullptr); w->String(ctime(&t)); - w->Key("Detector Type"); w->String(sls::ToString(detType).c_str()); - w->Key("Timing Mode"); w->String(sls::ToString(timingMode).c_str()); - w->Key("Geometry"); w->StartObject(); w->Key("x"); @@ -32,10 +88,8 @@ void MasterAttributes::GetCommonBinaryAttributes( w->Key("y"); w->Uint(geometry.y); w->EndObject(); - w->Key("Image Size in bytes"); w->Uint(imageSize); - w->Key("Pixels"); w->StartObject(); w->Key("x"); @@ -43,35 +97,27 @@ void MasterAttributes::GetCommonBinaryAttributes( w->Key("y"); w->Uint(nPixels.y); w->EndObject(); - w->Key("Max Frames Per File"); w->Uint(maxFramesPerFile); - w->Key("Frame Discard Policy"); w->String(sls::ToString(frameDiscardMode).c_str()); - w->Key("Frame Padding"); w->Uint(framePadding); - w->Key("Scan Parameters"); w->String(sls::ToString(scanParams).c_str()); - w->Key("Total Frames"); w->Uint64(totalFrames); -}; +} void MasterAttributes::GetFinalBinaryAttributes( rapidjson::Writer *w) { // adding few common parameters to the end - if (!additionalJsonHeader.empty()) { w->Key("Additional Json Header"); w->String(sls::ToString(additionalJsonHeader).c_str()); } - w->Key("Frames in File"); w->Uint64(framesInFile); - w->Key("Frame Header Format"); w->StartObject(); w->Key("Frame Number"); @@ -103,658 +149,578 @@ void MasterAttributes::GetFinalBinaryAttributes( w->Key("Packets Caught Mask"); w->String("64 bytes"); w->EndObject(); -}; +} #ifdef HDF5C -void MasterAttributes::WriteMasterHDF5Attributes(H5File *fd, Group *group) { - LOG(logERROR) << "WriteMasterHdf5Attributes should have been called " - "by a child class"; -}; - -void MasterAttributes::WriteHDF5Attributes(H5File *fd, Group *group) { +void MasterAttributes::WriteCommonHDF5Attributes(H5File *fd, Group *group) { char c[1024]; memset(c, 0, sizeof(c)); - // clang-format off - // version - { - double version = BINARY_WRITER_VERSION; - DataSpace dataspace = DataSpace(H5S_SCALAR); - Attribute attribute = fd->createAttribute( - "Version", PredType::NATIVE_DOUBLE, dataspace); - attribute.write(PredType::NATIVE_DOUBLE, &version); - } - // timestamp - { - time_t t = time(nullptr); - StrType strdatatype(PredType::C_S1, 256); - DataSpace dataspace = DataSpace(H5S_SCALAR); - DataSet dataset = + // version + { + double version = BINARY_WRITER_VERSION; + DataSpace dataspace = DataSpace(H5S_SCALAR); + Attribute attribute = + fd->createAttribute("Version", PredType::NATIVE_DOUBLE, dataspace); + attribute.write(PredType::NATIVE_DOUBLE, &version); + } + // timestamp + { + time_t t = time(nullptr); + StrType strdatatype(PredType::C_S1, 256); + DataSpace dataspace = DataSpace(H5S_SCALAR); + DataSet dataset = group->createDataSet("Timestamp", strdatatype, dataspace); - sls::strcpy_safe(c, std::string(ctime(&t))); - dataset.write(c, strdatatype); - } - // detector type - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - StrType strdatatype(PredType::C_S1, 256); - DataSet dataset = + sls::strcpy_safe(c, std::string(ctime(&t))); + dataset.write(c, strdatatype); + } + // detector type + { + DataSpace dataspace = DataSpace(H5S_SCALAR); + StrType strdatatype(PredType::C_S1, 256); + DataSet dataset = group->createDataSet("Detector Type", strdatatype, dataspace); - sls::strcpy_safe(c, sls::ToString(detType)); - dataset.write(c, strdatatype); - } - // timing mode - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - StrType strdatatype(PredType::C_S1, 256); - DataSet dataset = + sls::strcpy_safe(c, sls::ToString(detType)); + dataset.write(c, strdatatype); + } + // timing mode + { + DataSpace dataspace = DataSpace(H5S_SCALAR); + StrType strdatatype(PredType::C_S1, 256); + DataSet dataset = group->createDataSet("Timing Mode", strdatatype, dataspace); - sls::strcpy_safe(c, sls::ToString(timingMode)); - dataset.write(c, strdatatype); - } - //TODO: make this into an array? - // geometry x - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - DataSet dataset = group->createDataSet( - "Geometry in x axis", PredType::NATIVE_INT, dataspace); - dataset.write(&geometry.x, PredType::NATIVE_INT); - } - // geometry y - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - DataSet dataset = group->createDataSet( - "Geometry in y axis", PredType::NATIVE_INT, dataspace); - dataset.write(&geometry.y, PredType::NATIVE_INT); - } - // Image Size - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - DataSet dataset = group->createDataSet( - "Image Size", PredType::NATIVE_INT, dataspace); - dataset.write(&imageSize, PredType::NATIVE_INT); - DataSpace dataspaceAttr = DataSpace(H5S_SCALAR); - StrType strdatatype(PredType::C_S1, 256); - Attribute attribute = - dataset.createAttribute("Unit", strdatatype, dataspaceAttr); - sls::strcpy_safe(c, "bytes"); - attribute.write(strdatatype, c); - } - //TODO: make this into an array? - // npixels x - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - DataSet dataset = group->createDataSet( - "Number of pixels in x axis", PredType::NATIVE_INT, dataspace); - dataset.write(&nPixels.x, PredType::NATIVE_INT); - } - // npixels y - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - DataSet dataset = group->createDataSet( - "Number of pixels in y axis", PredType::NATIVE_INT, dataspace); - dataset.write(&nPixels.y, PredType::NATIVE_INT); - } - // Maximum frames per file - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - DataSet dataset = group->createDataSet( - "Maximum frames per file", PredType::NATIVE_INT, dataspace); - dataset.write(&maxFramesPerFile, PredType::NATIVE_INT); - } - // Frame Discard Policy - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - StrType strdatatype(PredType::C_S1, 256); - DataSet dataset = - group->createDataSet("Frame Discard Policy", strdatatype, dataspace); - sls::strcpy_safe(c, sls::ToString(frameDiscardMode)); - dataset.write(c, strdatatype); - } - // Frame Padding - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - DataSet dataset = group->createDataSet( - "Frame Padding", PredType::NATIVE_INT, dataspace); - dataset.write(&framePadding, PredType::NATIVE_INT); - } - // Scan Parameters - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - StrType strdatatype(PredType::C_S1, 256); - DataSet dataset = - group->createDataSet("Scan Parameters", strdatatype, dataspace); - sls::strcpy_safe(c, sls::ToString(scanParams)); - dataset.write(c, strdatatype); - } - // Total Frames - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - DataSet dataset = group->createDataSet( - "Total Frames", PredType::STD_U64LE, dataspace); - dataset.write(&totalFrames, PredType::STD_U64LE); - } - }; - - void MasterAttributes::WriteFinalHDF5Attributes(H5File *fd, Group *group) { - char c[1024]; - memset(c, 0, sizeof(c)); - // Total Frames in file - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - DataSet dataset = group->createDataSet( - "Frames in File", PredType::STD_U64LE, dataspace); - dataset.write(&framesInFile, PredType::STD_U64LE); - } - // additional json header - if (!additionalJsonHeader.empty()) { - std::string json = sls::ToString(additionalJsonHeader); - StrType strdatatype(PredType::C_S1, json.length()); - DataSpace dataspace = DataSpace(H5S_SCALAR); - DataSet dataset = - group->createDataSet("Additional JSON Header", strdatatype, dataspace); - sls::strcpy_safe(c, sls::ToString(additionalJsonHeader)); - dataset.write(c, strdatatype); - } - }; - - void MasterAttributes::WriteHDF5Exptime(H5File *fd, Group *group) { - DataSpace dataspace = DataSpace(H5S_SCALAR); - StrType strdatatype(PredType::C_S1, 256); - DataSet dataset = - group->createDataSet("Exposure Time", strdatatype, dataspace); - char c[1024]; - memset(c, 0, sizeof(c)); - sls::strcpy_safe(c, sls::ToString(exptime)); + sls::strcpy_safe(c, sls::ToString(timingMode)); dataset.write(c, strdatatype); - }; - - void MasterAttributes::WriteHDF5Period(H5File *fd, Group *group) { + } + // TODO: make this into an array? + // geometry x + { + DataSpace dataspace = DataSpace(H5S_SCALAR); + DataSet dataset = group->createDataSet("Geometry in x axis", + PredType::NATIVE_INT, dataspace); + dataset.write(&geometry.x, PredType::NATIVE_INT); + } + // geometry y + { + DataSpace dataspace = DataSpace(H5S_SCALAR); + DataSet dataset = group->createDataSet("Geometry in y axis", + PredType::NATIVE_INT, dataspace); + dataset.write(&geometry.y, PredType::NATIVE_INT); + } + // Image Size + { DataSpace dataspace = DataSpace(H5S_SCALAR); - StrType strdatatype(PredType::C_S1, 256); DataSet dataset = - group->createDataSet("Acquisition Period", strdatatype, dataspace); - char c[1024]; - memset(c, 0, sizeof(c)); - sls::strcpy_safe(c, sls::ToString(period)); - dataset.write(c, strdatatype); - }; - - void MasterAttributes::WriteHDF5DynamicRange(H5File *fd, Group *group) { - DataSpace dataspace = DataSpace(H5S_SCALAR); - DataSet dataset = group->createDataSet( - "Dynamic Range", PredType::NATIVE_INT, dataspace); - dataset.write(&dynamicRange, PredType::NATIVE_INT); + group->createDataSet("Image Size", PredType::NATIVE_INT, dataspace); + dataset.write(&imageSize, PredType::NATIVE_INT); DataSpace dataspaceAttr = DataSpace(H5S_SCALAR); StrType strdatatype(PredType::C_S1, 256); Attribute attribute = - dataset.createAttribute("Unit", strdatatype, dataspaceAttr); - char c[1024] = "bits"; - attribute.write( strdatatype, c); - }; - - void MasterAttributes::WriteHDF5TenGiga(H5File *fd, Group *group) { + dataset.createAttribute("Unit", strdatatype, dataspaceAttr); + sls::strcpy_safe(c, "bytes"); + attribute.write(strdatatype, c); + } + // TODO: make this into an array? + // npixels x + { DataSpace dataspace = DataSpace(H5S_SCALAR); - DataSet dataset = group->createDataSet( - "Ten Giga Enable", PredType::NATIVE_INT, dataspace); - dataset.write(&tenGiga, PredType::NATIVE_INT); - }; + DataSet dataset = group->createDataSet("Number of pixels in x axis", + PredType::NATIVE_INT, dataspace); + dataset.write(&nPixels.x, PredType::NATIVE_INT); + } + // npixels y + { + DataSpace dataspace = DataSpace(H5S_SCALAR); + DataSet dataset = group->createDataSet("Number of pixels in y axis", + PredType::NATIVE_INT, dataspace); + dataset.write(&nPixels.y, PredType::NATIVE_INT); + } + // Maximum frames per file + { + DataSpace dataspace = DataSpace(H5S_SCALAR); + DataSet dataset = group->createDataSet("Maximum frames per file", + PredType::NATIVE_INT, dataspace); + dataset.write(&maxFramesPerFile, PredType::NATIVE_INT); + } + // Frame Discard Policy + { + DataSpace dataspace = DataSpace(H5S_SCALAR); + StrType strdatatype(PredType::C_S1, 256); + DataSet dataset = group->createDataSet("Frame Discard Policy", + strdatatype, dataspace); + sls::strcpy_safe(c, sls::ToString(frameDiscardMode)); + dataset.write(c, strdatatype); + } + // Frame Padding + { + DataSpace dataspace = DataSpace(H5S_SCALAR); + DataSet dataset = group->createDataSet("Frame Padding", + PredType::NATIVE_INT, dataspace); + dataset.write(&framePadding, PredType::NATIVE_INT); + } + // Scan Parameters + { + DataSpace dataspace = DataSpace(H5S_SCALAR); + StrType strdatatype(PredType::C_S1, 256); + DataSet dataset = + group->createDataSet("Scan Parameters", strdatatype, dataspace); + sls::strcpy_safe(c, sls::ToString(scanParams)); + dataset.write(c, strdatatype); + } + // Total Frames + { + DataSpace dataspace = DataSpace(H5S_SCALAR); + DataSet dataset = group->createDataSet("Total Frames", + PredType::STD_U64LE, dataspace); + dataset.write(&totalFrames, PredType::STD_U64LE); + } +} + +void MasterAttributes::WriteFinalHDF5Attributes(H5File *fd, Group *group) { + char c[1024]; + memset(c, 0, sizeof(c)); + // Total Frames in file + { + DataSpace dataspace = DataSpace(H5S_SCALAR); + DataSet dataset = group->createDataSet("Frames in File", + PredType::STD_U64LE, dataspace); + dataset.write(&framesInFile, PredType::STD_U64LE); + } + // additional json header + if (!additionalJsonHeader.empty()) { + std::string json = sls::ToString(additionalJsonHeader); + StrType strdatatype(PredType::C_S1, json.length()); + DataSpace dataspace = DataSpace(H5S_SCALAR); + DataSet dataset = group->createDataSet("Additional JSON Header", + strdatatype, dataspace); + sls::strcpy_safe(c, sls::ToString(additionalJsonHeader)); + dataset.write(c, strdatatype); + } +} + +void MasterAttributes::WriteHDF5Exptime(H5File *fd, Group *group) { + DataSpace dataspace = DataSpace(H5S_SCALAR); + StrType strdatatype(PredType::C_S1, 256); + DataSet dataset = + group->createDataSet("Exposure Time", strdatatype, dataspace); + char c[1024]; + memset(c, 0, sizeof(c)); + sls::strcpy_safe(c, sls::ToString(exptime)); + dataset.write(c, strdatatype); +} + +void MasterAttributes::WriteHDF5Period(H5File *fd, Group *group) { + DataSpace dataspace = DataSpace(H5S_SCALAR); + StrType strdatatype(PredType::C_S1, 256); + DataSet dataset = + group->createDataSet("Acquisition Period", strdatatype, dataspace); + char c[1024]; + memset(c, 0, sizeof(c)); + sls::strcpy_safe(c, sls::ToString(period)); + dataset.write(c, strdatatype); +} + +void MasterAttributes::WriteHDF5DynamicRange(H5File *fd, Group *group) { + DataSpace dataspace = DataSpace(H5S_SCALAR); + DataSet dataset = + group->createDataSet("Dynamic Range", PredType::NATIVE_INT, dataspace); + dataset.write(&dynamicRange, PredType::NATIVE_INT); + DataSpace dataspaceAttr = DataSpace(H5S_SCALAR); + StrType strdatatype(PredType::C_S1, 256); + Attribute attribute = + dataset.createAttribute("Unit", strdatatype, dataspaceAttr); + char c[1024] = "bits"; + attribute.write(strdatatype, c); +} + +void MasterAttributes::WriteHDF5TenGiga(H5File *fd, Group *group) { + DataSpace dataspace = DataSpace(H5S_SCALAR); + DataSet dataset = group->createDataSet("Ten Giga Enable", + PredType::NATIVE_INT, dataspace); + dataset.write(&tenGiga, PredType::NATIVE_INT); +} + +void MasterAttributes::WriteHDF5ROI(H5File *fd, Group *group) { + // Roi xmin + { + DataSpace dataspace = DataSpace(H5S_SCALAR); + DataSet dataset = + group->createDataSet("roi xmin", PredType::NATIVE_INT, dataspace); + dataset.write(&roi.xmin, PredType::NATIVE_INT); + } + // Roi xmax + { + DataSpace dataspace = DataSpace(H5S_SCALAR); + DataSet dataset = + group->createDataSet("roi xmax", PredType::NATIVE_INT, dataspace); + dataset.write(&roi.xmax, PredType::NATIVE_INT); + } +} + +void MasterAttributes::WriteHDF5NumUDPInterfaces(H5File *fd, Group *group) { + DataSpace dataspace = DataSpace(H5S_SCALAR); + DataSet dataset = group->createDataSet("Number of UDP Interfaces", + PredType::NATIVE_INT, dataspace); + dataset.write(&numUDPInterfaces, PredType::NATIVE_INT); +} + +void MasterAttributes::WriteHDF5ReadNRows(H5File *fd, Group *group) { + DataSpace dataspace = DataSpace(H5S_SCALAR); + DataSet dataset = + group->createDataSet("Number of rows", PredType::NATIVE_INT, dataspace); + dataset.write(&readNRows, PredType::NATIVE_INT); +} + +void MasterAttributes::WriteHDF5ThresholdEnergy(H5File *fd, Group *group) { + char c[1024]{0}; + DataSpace dataspace = DataSpace(H5S_SCALAR); + DataSet dataset = group->createDataSet("Threshold Energy", + PredType::NATIVE_INT, dataspace); + dataset.write(&thresholdEnergyeV, PredType::NATIVE_INT); + DataSpace dataspaceAttr = DataSpace(H5S_SCALAR); + StrType strdatatype(PredType::C_S1, 256); + Attribute attribute = + dataset.createAttribute("Unit", strdatatype, dataspaceAttr); + sls::strcpy_safe(c, "eV"); + attribute.write(strdatatype, c); +} + +void MasterAttributes::WriteHDF5ThresholdEnergies(H5File *fd, Group *group) { + DataSpace dataspace = DataSpace(H5S_SCALAR); + StrType strdatatype(PredType::C_S1, 1024); + DataSet dataset = + group->createDataSet("Threshold Energies", strdatatype, dataspace); + sls::strcpy_safe(c, sls::ToString(thresholdAllEnergyeV)); + dataset.write(c, strdatatype); +} + +void MasterAttributes::WriteHDF5SubExpTime(H5File *fd, Group *group) { + char c[1024]{0}; + DataSpace dataspace = DataSpace(H5S_SCALAR); + StrType strdatatype(PredType::C_S1, 256); + DataSet dataset = + group->createDataSet("Sub Exposure Time", strdatatype, dataspace); + sls::strcpy_safe(c, sls::ToString(subExptime)); + dataset.write(c, strdatatype); +} + +void MasterAttributes::WriteHDF5SubPeriod(H5File *fd, Group *group) { + char c[1024]{0}; + DataSpace dataspace = DataSpace(H5S_SCALAR); + StrType strdatatype(PredType::C_S1, 256); + DataSet dataset = + group->createDataSet("Sub Period", strdatatype, dataspace); + sls::strcpy_safe(c, sls::ToString(subPeriod)); + dataset.write(c, strdatatype); +} + +void MasterAttributes::WriteHDF5SubQuad(H5File *fd, Group *group) { + DataSpace dataspace = DataSpace(H5S_SCALAR); + DataSet dataset = + group->createDataSet("Quad", PredType::NATIVE_INT, dataspace); + dataset.write(&quad, PredType::NATIVE_INT); +} + +void MasterAttributes::WriteHDF5RateCorrections(H5File *fd, Group *group) { + char c[1024]{0}; + DataSpace dataspace = DataSpace(H5S_SCALAR); + StrType strdatatype(PredType::C_S1, 1024); + DataSet dataset = + group->createDataSet("Rate Corrections", strdatatype, dataspace); + sls::strcpy_safe(c, sls::ToString(ratecorr)); + dataset.write(c, strdatatype); +} + +void MasterAttributes::WriteHDF5CounterMask(H5File *fd, Group *group) { + DataSpace dataspace = DataSpace(H5S_SCALAR); + DataSet dataset = + group->createDataSet("Counter Mask", PredType::STD_U32LE, dataspace); + dataset.write(&counterMask, PredType::STD_U32LE); +} + +void MasterAttributes::WriteHDF5ExptimeArray(H5File *fd, Group *group) { + for (int i = 0; i != 3; ++i) { + char c[1024]{0}; + DataSpace dataspace = DataSpace(H5S_SCALAR); + StrType strdatatype(PredType::C_S1, 256); + DataSet dataset = + group->createDataSet("Exposure Time1", strdatatype, dataspace); + sls::strcpy_safe(c, sls::ToString(exptimeArray[i])); + dataset.write(c, strdatatype); + } +} + +void MasterAttributes::WriteHDF5GateDelayArray(H5File *fd, Group *group) { + for (int i = 0; i != 3; ++i) { + char c[1024]{0}; + DataSpace dataspace = DataSpace(H5S_SCALAR); + StrType strdatatype(PredType::C_S1, 256); + DataSet dataset = + group->createDataSet("Gate Delay1", strdatatype, dataspace); + sls::strcpy_safe(c, sls::ToString(gateDelayArray[i])); + dataset.write(c, strdatatype); + } +} + +void MasterAttributes::WriteHDF5Gates(H5File *fd, Group *group) { + DataSpace dataspace = DataSpace(H5S_SCALAR); + DataSet dataset = + group->createDataSet("Gates", PredType::STD_U32LE, dataspace); + dataset.write(&gates, PredType::STD_U32LE); +} + +void MasterAttributes::WriteHDF5BurstMode(H5File *fd, Group *group) { + DataSpace dataspace = DataSpace(H5S_SCALAR); + StrType strdatatype(PredType::C_S1, 256); + DataSet dataset = + group->createDataSet("Burst Mode", strdatatype, dataspace); + char c[1024]; + memset(c, 0, sizeof(c)); + sls::strcpy_safe(c, sls::ToString(burstMode)); + dataset.write(c, strdatatype); +} + +void MasterAttributes::WriteHDF5AdcMask(H5File *fd, Group *group) { + DataSpace dataspace = DataSpace(H5S_SCALAR); + DataSet dataset = + group->createDataSet("ADC Mask", PredType::NATIVE_INT, dataspace); + dataset.write(&adcmask, PredType::NATIVE_INT); +} + +void MasterAttributes::WriteHDF5AnalogFlag(H5File *fd, Group *group) { + DataSpace dataspace = DataSpace(H5S_SCALAR); + DataSet dataset = + group->createDataSet("Analog Flag", PredType::NATIVE_INT, dataspace); + dataset.write(&analog, PredType::NATIVE_INT); +} + +void MasterAttributes::WriteHDF5AnalogSamples(H5File *fd, Group *group) { + DataSpace dataspace = DataSpace(H5S_SCALAR); + DataSet dataset = + group->createDataSet("Analog Samples", PredType::NATIVE_INT, dataspace); + dataset.write(&analogSamples, PredType::NATIVE_INT); +} + +void MasterAttributes::WriteHDF5DigitalFlag(H5File *fd, Group *group) { + DataSpace dataspace = DataSpace(H5S_SCALAR); + DataSet dataset = + group->createDataSet("Digital Flag", PredType::NATIVE_INT, dataspace); + dataset.write(&digital, PredType::NATIVE_INT); +} + +void MasterAttributes::WriteHDF5DigitalSamples(H5File *fd, Group *group) { + DataSpace dataspace = DataSpace(H5S_SCALAR); + DataSet dataset = group->createDataSet("Digital Samples", + PredType::NATIVE_INT, dataspace); + dataset.write(&digitalSamples, PredType::NATIVE_INT); +} + +void MasterAttributes::WriteHDF5DbitOffset(H5File *fd, Group *group) { + DataSpace dataspace = DataSpace(H5S_SCALAR); + DataSet dataset = + group->createDataSet("Dbit Offset", PredType::NATIVE_INT, dataspace); + dataset.write(&dbitoffset, PredType::NATIVE_INT); +} + +void MasterAttributes::WriteHDF5DbitList(H5File *fd, Group *group) { + DataSpace dataspace = DataSpace(H5S_SCALAR); + DataSet dataset = group->createDataSet("Dbit Bitset List", + PredType::STD_U64LE, dataspace); + dataset.write(&dbitlist, PredType::STD_U64LE); +} #endif - void GotthardMasterAttributes::GetSpecificBinaryAttributes(rapidjson::Writer *w) { - w->Key("Exptime"); - w->String(sls::ToString(exptime).c_str()); - - w->Key("Period"); - w->String(sls::ToString(period).c_str()); - - w->Key("Roi (xmin, xmax)"); - w->String(sls::ToString(roi).c_str()); - }; +void MasterAttributes::GetGotthardBinaryAttributes( + rapidjson::Writer *w) { + w->Key("Exptime"); + w->String(sls::ToString(exptime).c_str()); + w->Key("Period"); + w->String(sls::ToString(period).c_str()); + w->Key("Roi (xmin, xmax)"); + w->String(sls::ToString(roi).c_str()); +}; #ifdef HDF5C - void GotthardMasterAttributes::WriteMasterHDF5Attributes(H5File *fd, Group *group) { - MasterAttributes::WriteHDF5Attributes(fd, group); - MasterAttributes::WriteHDF5Exptime(fd, group); - MasterAttributes::WriteHDF5Period(fd, group); - // Roi xmin - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - DataSet dataset = group->createDataSet( - "roi xmin", PredType::NATIVE_INT, dataspace); - dataset.write(&roi.xmin, PredType::NATIVE_INT); - } - // Roi xmax - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - DataSet dataset = group->createDataSet( - "roi xmax", PredType::NATIVE_INT, dataspace); - dataset.write(&roi.xmax, PredType::NATIVE_INT); - } - MasterAttributes::WriteFinalHDF5Attributes(fd, group); - }; +void MasterAttributes::WriteGotthardHDF5Attributes(H5File *fd, Group *group) { + MasterAttributes::WriteHDF5Exptime(fd, group); + MasterAttributes::WriteHDF5Period(fd, group); + MasterAttributes::WriteHDF5ROI(fd, group); +} #endif - void JungfrauMasterAttributes::GetSpecificBinaryAttributes(rapidjson::Writer *w) { - w->Key("Exptime"); - w->String(sls::ToString(exptime).c_str()); - - w->Key("Period"); - w->String(sls::ToString(period).c_str()); - - w->Key("Number of UDP Interfaces"); - w->Uint(numUDPInterfaces); - - w->Key("Number of rows"); - w->Uint(readNRows); - }; +void MasterAttributes::GetJungfrauBinaryAttributes( + rapidjson::Writer *w) { + w->Key("Exptime"); + w->String(sls::ToString(exptime).c_str()); + w->Key("Period"); + w->String(sls::ToString(period).c_str()); + w->Key("Number of UDP Interfaces"); + w->Uint(numUDPInterfaces); + w->Key("Number of rows"); + w->Uint(readNRows); +} #ifdef HDF5C - void JungfrauMasterAttributes::WriteMasterHDF5Attributes(H5File *fd, Group *group) { - MasterAttributes::WriteHDF5Attributes(fd, group); - MasterAttributes::WriteHDF5Exptime(fd, group); - MasterAttributes::WriteHDF5Period(fd, group); - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - DataSet dataset = group->createDataSet( - "Number of UDP Interfaces", PredType::NATIVE_INT, dataspace); - dataset.write(&numUDPInterfaces, PredType::NATIVE_INT); - } - // readNRows - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - DataSet dataset = group->createDataSet( - "Number of rows", PredType::NATIVE_INT, dataspace); - dataset.write(&readNRows, PredType::NATIVE_INT); - } - MasterAttributes::WriteFinalHDF5Attributes(fd, group); - }; +void MasterAttributes::WriteJungfrauHDF5Attributes(H5File *fd, Group *group) { + MasterAttributes::WriteHDF5Exptime(fd, group); + MasterAttributes::WriteHDF5Period(fd, group); + MasterAttributes::WriteHDF5NumUDPInterfaces(fd, group); + MasterAttributes::WriteHDF5ReadNRows(fd, group); +} #endif - void EigerMasterAttributes::GetSpecificBinaryAttributes(rapidjson::Writer *w) { - w->Key("Dynamic Range"); - w->Uint(dynamicRange); - - w->Key("Ten Giga"); - w->Uint(tenGiga); - - w->Key("Exptime"); - w->String(sls::ToString(exptime).c_str()); - - w->Key("Period"); - w->String(sls::ToString(period).c_str()); - - w->Key("Threshold Energy"); - w->Int(thresholdEnergyeV); - - w->Key("Sub Exptime"); - w->String(sls::ToString(subExptime).c_str()); - - w->Key("Sub Period"); - w->String(sls::ToString(subPeriod).c_str()); - - w->Key("Quad"); - w->Int(quad); - - w->Key("Number of rows"); - w->Int(readNRows); - - w->Key("Rate Corrections"); - w->String(sls::ToString(ratecorr).c_str()); - }; +void MasterAttributes::GetEigerBinaryAttributes( + rapidjson::Writer *w) { + w->Key("Dynamic Range"); + w->Uint(dynamicRange); + w->Key("Ten Giga"); + w->Uint(tenGiga); + w->Key("Exptime"); + w->String(sls::ToString(exptime).c_str()); + w->Key("Period"); + w->String(sls::ToString(period).c_str()); + w->Key("Threshold Energy"); + w->Int(thresholdEnergyeV); + w->Key("Sub Exptime"); + w->String(sls::ToString(subExptime).c_str()); + w->Key("Sub Period"); + w->String(sls::ToString(subPeriod).c_str()); + w->Key("Quad"); + w->Int(quad); + w->Key("Number of rows"); + w->Int(readNRows); + w->Key("Rate Corrections"); + w->String(sls::ToString(ratecorr).c_str()); +} #ifdef HDF5C - void EigerMasterAttributes::WriteMasterHDF5Attributes(H5File *fd, Group *group) { - MasterAttributes::WriteHDF5Attributes(fd, group); - MasterAttributes::WriteHDF5DynamicRange(fd, group); - MasterAttributes::WriteHDF5TenGiga(fd, group); - MasterAttributes::WriteHDF5Exptime(fd, group); - MasterAttributes::WriteHDF5Period(fd, group); - char c[1024]; - memset(c, 0, sizeof(c)); - // threshold - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - DataSet dataset = group->createDataSet( - "Threshold Energy", PredType::NATIVE_INT, dataspace); - dataset.write(&thresholdEnergyeV, PredType::NATIVE_INT); - DataSpace dataspaceAttr = DataSpace(H5S_SCALAR); - StrType strdatatype(PredType::C_S1, 256); - Attribute attribute = - dataset.createAttribute("Unit", strdatatype, dataspaceAttr); - sls::strcpy_safe(c, "eV"); - attribute.write(strdatatype, c); - } - // SubExptime - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - StrType strdatatype(PredType::C_S1, 256); - DataSet dataset = group->createDataSet("Sub Exposure Time", - strdatatype, dataspace); - sls::strcpy_safe(c, sls::ToString(subExptime)); - dataset.write(c, strdatatype); - } - // SubPeriod - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - StrType strdatatype(PredType::C_S1, 256); - DataSet dataset = - group->createDataSet("Sub Period", strdatatype, dataspace); - sls::strcpy_safe(c, sls::ToString(subPeriod)); - dataset.write(c, strdatatype); - } - // Quad - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - DataSet dataset = - group->createDataSet("Quad", PredType::NATIVE_INT, dataspace); - dataset.write(&quad, PredType::NATIVE_INT); - } - // readNRows - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - DataSet dataset = group->createDataSet( - "Number of rows", PredType::NATIVE_INT, dataspace); - dataset.write(&readNRows, PredType::NATIVE_INT); - } - // Rate corrections - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - StrType strdatatype(PredType::C_S1, 1024); - DataSet dataset = group->createDataSet("Rate Corrections", - strdatatype, dataspace); - sls::strcpy_safe(c, sls::ToString(ratecorr)); - dataset.write(c, strdatatype); - } - MasterAttributes::WriteFinalHDF5Attributes(fd, group); - }; +void MasterAttributes::WriteEigerHDF5Attributes(H5File *fd, Group *group) { + MasterAttributes::WriteHDF5DynamicRange(fd, group); + MasterAttributes::WriteHDF5TenGiga(fd, group); + MasterAttributes::WriteHDF5Exptime(fd, group); + MasterAttributes::WriteHDF5Period(fd, group); + MasterAttributes::WriteHDF5ThresholdEnergy(fd, group); + MasterAttributes::WriteHDF5SubExpTime(fd, group); + MasterAttributes::WriteHDF5SubPeriod(fd, group); + MasterAttributes::WriteHDF5SubQuad(fd, group); + MasterAttributes::WriteHDF5ReadNRows(fd, group); + MasterAttributes::WriteHDF5RateCorrections(fd, group); +} #endif - void Mythen3MasterAttributes::GetSpecificBinaryAttributes(rapidjson::Writer *w) { - w->Key("Dynamic Range"); - w->Uint(dynamicRange); - - w->Key("Ten Giga"); - w->Uint(tenGiga); - - w->Key("Period"); - w->String(sls::ToString(period).c_str()); - - w->Key("Counter Mask"); - w->String(sls::ToStringHex(counterMask).c_str()); - - w->Key("Exptime1"); - w->String(sls::ToString(exptime1).c_str()); - - w->Key("Exptime2"); - w->String(sls::ToString(exptime2).c_str()); - - w->Key("Exptime3"); - w->String(sls::ToString(exptime3).c_str()); - - w->Key("GateDelay1"); - w->String(sls::ToString(gateDelay1).c_str()); - - w->Key("GateDelay2"); - w->String(sls::ToString(gateDelay2).c_str()); - - w->Key("GateDelay3"); - w->String(sls::ToString(gateDelay3).c_str()); - - w->Key("Gates"); - w->Uint(gates); - - w->Key("Threshold Energies"); - w->String(sls::ToString(thresholdAllEnergyeV).c_str()); - }; +void MasterAttributes::GetMythen3BinaryAttributes( + rapidjson::Writer *w) { + w->Key("Dynamic Range"); + w->Uint(dynamicRange); + w->Key("Ten Giga"); + w->Uint(tenGiga); + w->Key("Period"); + w->String(sls::ToString(period).c_str()); + w->Key("Counter Mask"); + w->String(sls::ToStringHex(counterMask).c_str()); + for (int i = 0; i != 3; ++i) { + w->Key((std::string("Exptime") + std::to_string(i + 1)).c_str()); + w->String(sls::ToString(exptimeArray[i]).c_str()); + } + for (int i = 0; i != 3; ++i) { + w->Key((std::string("GateDelay") + std::to_string(i + 1)).c_str()); + w->String(sls::ToString(gateDelayArray[i]).c_str()); + } + w->Key("Gates"); + w->Uint(gates); + w->Key("Threshold Energies"); + w->String(sls::ToString(thresholdAllEnergyeV).c_str()); +} #ifdef HDF5C - void Mythen3MasterAttributes::WriteMasterHDF5Attributes(H5File *fd, Group *group) { - MasterAttributes::WriteHDF5Attributes(fd, group); - MasterAttributes::WriteHDF5DynamicRange(fd, group); - MasterAttributes::WriteHDF5TenGiga(fd, group); - MasterAttributes::WriteHDF5Period(fd, group); - char c[1024]; - memset(c, 0, sizeof(c)); - // Counter Mask - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - DataSet dataset = group->createDataSet( - "Counter Mask", PredType::STD_U32LE, dataspace); - dataset.write(&counterMask, PredType::STD_U32LE); - } - // Exptime1 - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - StrType strdatatype(PredType::C_S1, 256); - DataSet dataset = - group->createDataSet("Exposure Time1", strdatatype, dataspace); - sls::strcpy_safe(c, sls::ToString(exptime1)); - dataset.write(c, strdatatype); - } - // Exptime2 - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - StrType strdatatype(PredType::C_S1, 256); - DataSet dataset = - group->createDataSet("Exposure Time2", strdatatype, dataspace); - sls::strcpy_safe(c, sls::ToString(exptime2)); - dataset.write(c, strdatatype); - } - // Exptime3 - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - StrType strdatatype(PredType::C_S1, 256); - DataSet dataset = - group->createDataSet("Exposure Time3", strdatatype, dataspace); - sls::strcpy_safe(c, sls::ToString(exptime3)); - dataset.write(c, strdatatype); - } - // GateDelay1 - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - StrType strdatatype(PredType::C_S1, 256); - DataSet dataset = - group->createDataSet("Gate Delay1", strdatatype, dataspace); - sls::strcpy_safe(c, sls::ToString(gateDelay1)); - dataset.write(c, strdatatype); - } - // GateDelay2 - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - StrType strdatatype(PredType::C_S1, 256); - DataSet dataset = - group->createDataSet("Gate Delay2", strdatatype, dataspace); - sls::strcpy_safe(c, sls::ToString(gateDelay2)); - dataset.write(c, strdatatype); - } - // GateDelay3 - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - StrType strdatatype(PredType::C_S1, 256); - DataSet dataset = - group->createDataSet("Gate Delay3", strdatatype, dataspace); - sls::strcpy_safe(c, sls::ToString(gateDelay3)); - dataset.write(c, strdatatype); - } - // Gates - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - DataSet dataset = - group->createDataSet("Gates", PredType::STD_U32LE, dataspace); - dataset.write(&gates, PredType::STD_U32LE); - } - // Threshold Energies - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - StrType strdatatype(PredType::C_S1, 1024); - DataSet dataset = group->createDataSet("Threshold Energies", - strdatatype, dataspace); - sls::strcpy_safe(c, sls::ToString(thresholdAllEnergyeV)); - dataset.write(c, strdatatype); - } - MasterAttributes::WriteFinalHDF5Attributes(fd, group); - }; +void MasterAttributes::WriteMythen3HDF5Attributes(H5File *fd, Group *group) { + MasterAttributes::WriteHDF5DynamicRange(fd, group); + MasterAttributes::WriteHDF5TenGiga(fd, group); + MasterAttributes::WriteHDF5Period(fd, group); + MasterAttributes::WriteHDF5CounterMask(fd, group); + MasterAttributes::WriteHDF5ExptimeArray(fd, group); + MasterAttributes::WriteHDF5GateDelayArray(fd, group); + MasterAttributes::WriteHDF5Gates(fd, group); + MasterAttributes::WriteHDF5ThresholdEnergies(fd, group); +} #endif - void Gotthard2MasterAttributes::GetSpecificBinaryAttributes(rapidjson::Writer *w) { - w->Key("Exptime"); - w->String(sls::ToString(exptime).c_str()); - - w->Key("Period"); - w->String(sls::ToString(period).c_str()); - - w->Key("Burst Mode"); - w->String(sls::ToString(burstMode).c_str()); - }; +void MasterAttributes::GetGotthard2BinaryAttributes( + rapidjson::Writer *w) { + w->Key("Exptime"); + w->String(sls::ToString(exptime).c_str()); + w->Key("Period"); + w->String(sls::ToString(period).c_str()); + w->Key("Burst Mode"); + w->String(sls::ToString(burstMode).c_str()); +} #ifdef HDF5C - void Gotthard2MasterAttributes::WriteMasterHDF5Attributes(H5File *fd, Group *group) { - MasterAttributes::WriteHDF5Attributes(fd, group); - MasterAttributes::WriteHDF5Exptime(fd, group); - MasterAttributes::WriteHDF5Period(fd, group); - // burst mode - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - StrType strdatatype(PredType::C_S1, 256); - DataSet dataset = - group->createDataSet("Burst Mode", strdatatype, dataspace); - char c[1024]; - memset(c, 0, sizeof(c)); - sls::strcpy_safe(c, sls::ToString(burstMode)); - dataset.write(c, strdatatype); - } - MasterAttributes::WriteFinalHDF5Attributes(fd, group); - }; +void MasterAttributes::WriteGotthard2HDF5Attributes(H5File *fd, Group *group) { + MasterAttributes::WriteHDF5Exptime(fd, group); + MasterAttributes::WriteHDF5Period(fd, group); + MasterAttributes::WriteHDF5BurstMode(fd, group); +} #endif - void MoenchMasterAttributes::GetSpecificBinaryAttributes(rapidjson::Writer *w) { - w->Key("Exptime"); - w->String(sls::ToString(exptime).c_str()); - - w->Key("Period"); - w->String(sls::ToString(period).c_str()); - - w->Key("Ten Giga"); - w->Uint(tenGiga); - - w->Key("ADC Mask"); - w->String(sls::ToStringHex(adcmask).c_str()); - - w->Key("Analog Samples"); - w->Uint(analogSamples); - }; +void MasterAttributes::GetMoenchBinaryAttributes( + rapidjson::Writer *w) { + w->Key("Exptime"); + w->String(sls::ToString(exptime).c_str()); + w->Key("Period"); + w->String(sls::ToString(period).c_str()); + w->Key("Ten Giga"); + w->Uint(tenGiga); + w->Key("ADC Mask"); + w->String(sls::ToStringHex(adcmask).c_str()); + w->Key("Analog Samples"); + w->Uint(analogSamples); +} #ifdef HDF5C - void MoenchMasterAttributes::WriteMasterHDF5Attributes(H5File *fd, Group *group) { - MasterAttributes::WriteHDF5Attributes(fd, group); - MasterAttributes::WriteHDF5Exptime(fd, group); - MasterAttributes::WriteHDF5Period(fd, group); - MasterAttributes::WriteHDF5TenGiga(fd, group); - // ADC Mask - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - DataSet dataset = group->createDataSet( - "ADC Mask", PredType::NATIVE_INT, dataspace); - dataset.write(&adcmask, PredType::NATIVE_INT); - } - // Analog Samples - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - DataSet dataset = group->createDataSet( - "Analog Samples", PredType::NATIVE_INT, dataspace); - dataset.write(&analogSamples, PredType::NATIVE_INT); - } - MasterAttributes::WriteFinalHDF5Attributes(fd, group); - }; +void MasterAttributes::WriteMoenchHDF5Attributes(H5File *fd, Group *group) { + MasterAttributes::WriteHDF5Exptime(fd, group); + MasterAttributes::WriteHDF5Period(fd, group); + MasterAttributes::WriteHDF5TenGiga(fd, group); + MasterAttributes::WriteHDF5AdcMask(fd, group); + MasterAttributes::WriteHDF5AnalogSamples(fd, group); +} #endif - void CtbMasterAttributes::GetSpecificBinaryAttributes(rapidjson::Writer *w) { - w->Key("Exptime"); - w->String(sls::ToString(exptime).c_str()); - - w->Key("Period"); - w->String(sls::ToString(period).c_str()); - - w->Key("Ten Giga"); - w->Uint(tenGiga); - - w->Key("ADC Mask"); - w->String(sls::ToStringHex(adcmask).c_str()); - - w->Key("Analog Flag"); - w->Uint(analog); - - w->Key("Analog Samples"); - w->Uint(analogSamples); - - w->Key("Digital Flag"); - w->Uint(digital); - - w->Key("Digital Samples"); - w->Uint(digitalSamples); - - w->Key("Dbit Offset"); - w->Uint(dbitoffset); - - w->Key("Dbit Bitset"); - w->Uint64(dbitlist); - }; +void MasterAttributes::GetCtbBinaryAttributes( + rapidjson::Writer *w) { + w->Key("Exptime"); + w->String(sls::ToString(exptime).c_str()); + w->Key("Period"); + w->String(sls::ToString(period).c_str()); + w->Key("Ten Giga"); + w->Uint(tenGiga); + w->Key("ADC Mask"); + w->String(sls::ToStringHex(adcmask).c_str()); + w->Key("Analog Flag"); + w->Uint(analog); + w->Key("Analog Samples"); + w->Uint(analogSamples); + w->Key("Digital Flag"); + w->Uint(digital); + w->Key("Digital Samples"); + w->Uint(digitalSamples); + w->Key("Dbit Offset"); + w->Uint(dbitoffset); + w->Key("Dbit Bitset"); + w->Uint64(dbitlist); +} #ifdef HDF5C - void CtbMasterAttributes::WriteMasterHDF5Attributes(H5File *fd, Group *group) { - MasterAttributes::WriteHDF5Attributes(fd, group); - MasterAttributes::WriteHDF5Exptime(fd, group); - MasterAttributes::WriteHDF5Period(fd, group); - MasterAttributes::WriteHDF5TenGiga(fd, group); - // ADC Mask - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - DataSet dataset = group->createDataSet( - "ADC mMsk", PredType::NATIVE_INT, dataspace); - dataset.write(&adcmask, PredType::NATIVE_INT); - } - // Analog Flag - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - DataSet dataset = group->createDataSet( - "Analog Flag", PredType::NATIVE_INT, dataspace); - dataset.write(&analog, PredType::NATIVE_INT); - } - // Analog Samples - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - DataSet dataset = group->createDataSet( - "Analog Samples", PredType::NATIVE_INT, dataspace); - dataset.write(&analogSamples, PredType::NATIVE_INT); - } - // Digital Flag - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - DataSet dataset = group->createDataSet( - "Digital Flag", PredType::NATIVE_INT, dataspace); - dataset.write(&digital, PredType::NATIVE_INT); - } - // Digital Samples - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - DataSet dataset = group->createDataSet( - "Digital Samples", PredType::NATIVE_INT, dataspace); - dataset.write(&digitalSamples, PredType::NATIVE_INT); - } - // Dbit Offset - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - DataSet dataset = group->createDataSet( - "Dbit Offset", PredType::NATIVE_INT, dataspace); - dataset.write(&dbitoffset, PredType::NATIVE_INT); - } - // Dbit List - { - DataSpace dataspace = DataSpace(H5S_SCALAR); - DataSet dataset = group->createDataSet( - "Dbit Bitset List", PredType::STD_U64LE, dataspace); - dataset.write(&dbitlist, PredType::STD_U64LE); - } - MasterAttributes::WriteFinalHDF5Attributes(fd, group); - }; +void MasterAttributes::WriteCtbHDF5Attributes(H5File *fd, Group *group) { + MasterAttributes::WriteHDF5Exptime(fd, group); + MasterAttributes::WriteHDF5Period(fd, group); + MasterAttributes::WriteHDF5TenGiga(fd, group); + MasterAttributes::WriteHDF5AdcMask(fd, group); + MasterAttributes::WriteHDF5AnalogFlag(fd, group); + MasterAttributes::WriteHDF5AnalogSamples(fd, group); + MasterAttributes::WriteHDF5DigitalFlag(fd, group); + MasterAttributes::WriteHDF5DigitalSamples(fd, group); + MasterAttributes::WriteHDF5DbitOffset(fd, group); + MasterAttributes::WriteHDF5DbitList(fd, group); +} #endif diff --git a/slsReceiverSoftware/src/MasterAttributes.h b/slsReceiverSoftware/src/MasterAttributes.h index 27c7ba13b..66ae25016 100644 --- a/slsReceiverSoftware/src/MasterAttributes.h +++ b/slsReceiverSoftware/src/MasterAttributes.h @@ -56,103 +56,92 @@ class MasterAttributes { uint64_t dbitlist{0}; slsDetectorDefs::ROI roi{}; uint32_t counterMask{0}; - ns exptime1{0}; - ns exptime2{0}; - ns exptime3{0}; - ns gateDelay1{0}; - ns gateDelay2{0}; - ns gateDelay3{0}; + std::array exptimeArray{}; + std::array gateDelayArray{}; uint32_t gates; std::map additionalJsonHeader; - - // Final Attributes (after acquisition) uint64_t framesInFile{0}; MasterAttributes() = default; - virtual ~MasterAttributes() = default; - virtual void - GetSpecificBinaryAttributes(rapidjson::Writer *w); + ~MasterAttributes() = default; + + void GetBinaryAttributes(rapidjson::Writer *w); +#ifdef HDF5C + void WriteHDF5Attributes(H5File *fd, Group *group); +#endif + void GetCommonBinaryAttributes(rapidjson::Writer *w); void GetFinalBinaryAttributes(rapidjson::Writer *w); #ifdef HDF5C - virtual void WriteMasterHDF5Attributes(H5File *fd, Group *group); - void WriteHDF5Attributes(H5File *fd, Group *group); + void WriteCommonHDF5Attributes(H5File *fd, Group *group); void WriteFinalHDF5Attributes(H5File *fd, Group *group); void WriteHDF5Exptime(H5File *fd, Group *group); void WriteHDF5Period(H5File *fd, Group *group); void WriteHDF5DynamicRange(H5File *fd, Group *group); void WriteHDF5TenGiga(H5File *fd, Group *group); + void WriteHDF5ROI(H5File *fd, Group *group); + void WriteHDF5NumUDPInterfaces(H5File *fd, Group *group); + void WriteHDF5ReadNRows(H5File *fd, Group *group); + void WriteHDF5ThresholdEnergy(H5File *fd, Group *group); + void WriteHDF5ThresholdEnergies(H5File *fd, Group *group); + void WriteHDF5SubExpTime(H5File *fd, Group *group); + void WriteHDF5SubPeriod(H5File *fd, Group *group); + void WriteHDF5SubQuad(H5File *fd, Group *group); + void WriteHDF5RateCorrections(H5File *fd, Group *group); + void WriteHDF5CounterMask(H5File *fd, Group *group); + void WriteHDF5ExptimeArray(H5File *fd, Group *group); + void WriteHDF5GateDelayArray(H5File *fd, Group *group); + void WriteHDF5Gates(H5File *fd, Group *group); + void WriteHDF5BurstMode(H5File *fd, Group *group); + void WriteHDF5AdcMask(H5File *fd, Group *group); + void WriteHDF5AnalogFlag(H5File *fd, Group *group); + void WriteHDF5AnalogSamples(H5File *fd, Group *group); + void WriteHDF5DigitalFlag(H5File *fd, Group *group); + void WriteHDF5DigitalSamples(H5File *fd, Group *group); + void WriteHDF5DbitOffset(H5File *fd, Group *group); + void WriteHDF5DbitList(H5File *fd, Group *group); #endif -}; -class GotthardMasterAttributes : public MasterAttributes { - public: - GotthardMasterAttributes() = default; - void GetSpecificBinaryAttributes( - rapidjson::Writer *w) override; + void + GetGotthardBinaryAttributes(rapidjson::Writer *w); #ifdef HDF5C - void WriteMasterHDF5Attributes(H5File *fd, Group *group) override; + void WriteGotthardHDF5Attributes(H5File *fd, Group *group); #endif -}; -class JungfrauMasterAttributes : public MasterAttributes { - public: - JungfrauMasterAttributes() = default; - void GetSpecificBinaryAttributes( - rapidjson::Writer *w) override; + void + GetJungfrauBinaryAttributes(rapidjson::Writer *w); #ifdef HDF5C - void WriteMasterHDF5Attributes(H5File *fd, Group *group) override; + void WriteJungfrauHDF5Attributes(H5File *fd, Group *group); #endif -}; -class EigerMasterAttributes : public MasterAttributes { - public: - EigerMasterAttributes() = default; - void GetSpecificBinaryAttributes( - rapidjson::Writer *w) override; + void + GetEigerBinaryAttributes(rapidjson::Writer *w); #ifdef HDF5C - void WriteMasterHDF5Attributes(H5File *fd, Group *group) override; + void WriteEigerHDF5Attributes(H5File *fd, Group *group); #endif -}; -class Mythen3MasterAttributes : public MasterAttributes { - public: - Mythen3MasterAttributes() = default; - void GetSpecificBinaryAttributes( - rapidjson::Writer *w) override; + void + GetMythen3BinaryAttributes(rapidjson::Writer *w); #ifdef HDF5C - void WriteMasterHDF5Attributes(H5File *fd, Group *group) override; + void WriteMythen3HDF5Attributes(H5File *fd, Group *group); #endif -}; -class Gotthard2MasterAttributes : public MasterAttributes { - public: - Gotthard2MasterAttributes() = default; - void GetSpecificBinaryAttributes( - rapidjson::Writer *w) override; + void + GetGotthard2BinaryAttributes(rapidjson::Writer *w); #ifdef HDF5C - void WriteMasterHDF5Attributes(H5File *fd, Group *group) override; + void WriteGotthard2HDF5Attributes(H5File *fd, Group *group); #endif -}; -class MoenchMasterAttributes : public MasterAttributes { - public: - MoenchMasterAttributes() = default; - void GetSpecificBinaryAttributes( - rapidjson::Writer *w) override; + void + GetMoenchBinaryAttributes(rapidjson::Writer *w); #ifdef HDF5C - void WriteMasterHDF5Attributes(H5File *fd, Group *group) override; + void WriteMoenchHDF5Attributes(H5File *fd, Group *group); #endif -}; -class CtbMasterAttributes : public MasterAttributes { - public: - CtbMasterAttributes() = default; - void GetSpecificBinaryAttributes( - rapidjson::Writer *w) override; + void GetCtbBinaryAttributes(rapidjson::Writer *w); #ifdef HDF5C - void WriteMasterHDF5Attributes(H5File *fd, Group *group) override; + void WriteCtbHDF5Attributes(H5File *fd, Group *group); #endif }; From b9aa0f46e4d0bfb0749760edd254a1bb378f12de Mon Sep 17 00:00:00 2001 From: Dhanya Thattil Date: Tue, 29 Mar 2022 13:02:57 +0200 Subject: [PATCH 06/12] wip, hdf5 refactored --- slsReceiverSoftware/CMakeLists.txt | 3 +- slsReceiverSoftware/src/DataProcessor.cpp | 58 ++---- slsReceiverSoftware/src/DataProcessor.h | 25 ++- slsReceiverSoftware/src/File.h | 21 -- slsReceiverSoftware/src/HDF5MasterFile.cpp | 114 ----------- slsReceiverSoftware/src/HDF5MasterFile.h | 22 --- .../{HDF5VirtualFile.cpp => HDF5Utility.cpp} | 183 ++++++++++++++---- slsReceiverSoftware/src/HDF5Utility.h | 43 ++++ slsReceiverSoftware/src/HDF5VirtualFile.h | 34 ---- slsReceiverSoftware/src/Implementation.cpp | 17 +- slsReceiverSoftware/src/MasterAttributes.cpp | 1 + 11 files changed, 229 insertions(+), 292 deletions(-) delete mode 100644 slsReceiverSoftware/src/HDF5MasterFile.cpp delete mode 100644 slsReceiverSoftware/src/HDF5MasterFile.h rename slsReceiverSoftware/src/{HDF5VirtualFile.cpp => HDF5Utility.cpp} (50%) create mode 100644 slsReceiverSoftware/src/HDF5Utility.h delete mode 100644 slsReceiverSoftware/src/HDF5VirtualFile.h diff --git a/slsReceiverSoftware/CMakeLists.txt b/slsReceiverSoftware/CMakeLists.txt index ef1bfb56d..8589e0314 100755 --- a/slsReceiverSoftware/CMakeLists.txt +++ b/slsReceiverSoftware/CMakeLists.txt @@ -28,8 +28,7 @@ if (SLS_USE_HDF5) ) list (APPEND SOURCES src/HDF5DataFile.cpp - src/HDF5MasterFile.cpp - src/HDF5VirtualFile.cpp + src/HDF5Utility.cpp ) endif (SLS_USE_HDF5) diff --git a/slsReceiverSoftware/src/DataProcessor.cpp b/slsReceiverSoftware/src/DataProcessor.cpp index ed6788981..927213509 100644 --- a/slsReceiverSoftware/src/DataProcessor.cpp +++ b/slsReceiverSoftware/src/DataProcessor.cpp @@ -15,8 +15,7 @@ #include "MasterAttributes.h" #ifdef HDF5C #include "HDF5DataFile.h" -#include "HDF5MasterFile.h" -#include "HDF5VirtualFile.h" +#include "HDF5Utility.h" #endif #include "DataStreamer.h" #include "sls/container_utils.h" @@ -82,10 +81,6 @@ void DataProcessor::SetGeneralData(GeneralData *generalData) { void DataProcessor::CloseFiles() { if (dataFile_) dataFile_->CloseFile(); -#ifdef HDF5C - if (virtualFile_) - virtualFile_->CloseFile(); -#endif } void DataProcessor::DeleteFiles() { @@ -94,12 +89,6 @@ void DataProcessor::DeleteFiles() { delete dataFile_; dataFile_ = nullptr; } -#ifdef HDF5C - if (virtualFile_) { - delete virtualFile_; - virtualFile_ = nullptr; - } -#endif } void DataProcessor::SetupFileWriter(const bool filewriteEnable, const fileFormat fileFormatType, @@ -168,21 +157,17 @@ uint32_t DataProcessor::GetFilesInAcquisition() const { return dataFile_->GetFilesInAcquisition(); } -void DataProcessor::CreateVirtualFile( - const std::string filePath, const std::string fileNamePrefix, +std::array DataProcessor::CreateVirtualFile( + const std::string &filePath, const std::string &fileNamePrefix, const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode, const int modulePos, const int numUnitsPerReadout, const uint32_t maxFramesPerFile, const uint64_t numImages, - const uint32_t dynamicRange, const int numModX, const int numModY, + const int numModX, const int numModY, const uint32_t dynamicRange, std::mutex *hdf5LibMutex) { - if (virtualFile_) { - delete virtualFile_; - } bool gotthard25um = ((detectorType_ == GOTTHARD || detectorType_ == GOTTHARD2) && (numModX * numModY) == 2); - virtualFile_ = new HDF5VirtualFile(hdf5LibMutex, gotthard25um); // maxframesperfile = 0 for infinite files uint32_t framesPerFile = @@ -192,31 +177,30 @@ void DataProcessor::CreateVirtualFile( // files (they exist anyway) assumption2: virtual file max frame index // is from R0 P0 (difference from others when missing frames or for a // stop acquisition) - virtualFile_->CreateVirtualFile( + return hdf5Utility::CreateVirtualFile( filePath, fileNamePrefix, fileIndex, overWriteEnable, silentMode, modulePos, numUnitsPerReadout, framesPerFile, numImages, generalData_->nPixelsX, generalData_->nPixelsY, dynamicRange, numFramesCaught_, numModX, numModY, dataFile_->GetPDataType(), - dataFile_->GetParameterNames(), dataFile_->GetParameterDataTypes()); + dataFile_->GetParameterNames(), dataFile_->GetParameterDataTypes(), + hdf5LibMutex, gotthard25um); } -void DataProcessor::LinkDataInMasterFile(const std::string &masterFileName, - const bool silentMode, - std::mutex *hdf5LibMutex) { - std::string fname, datasetName; - if (virtualFile_) { - auto res = virtualFile_->GetFileAndDatasetName(); - fname = res[0]; - datasetName = res[1]; - } else { +void DataProcessor::LinkFileInMaster(const std::string &masterFileName, + const std::string &virtualFileName, + const std::string &virtualDatasetName, + const bool silentMode, + std::mutex *hdf5LibMutex) { + std::string fname{virtualFileName}, datasetName{virtualDatasetName}; + // if no virtual file, link data file + if (virtualFileName.empty()) { auto res = dataFile_->GetFileAndDatasetName(); fname = res[0]; datasetName = res[1]; } - // link in master - HDF5MasterFile::LinkDataFile(masterFileName, fname, datasetName, - dataFile_->GetParameterNames(), silentMode, - &hdf5LibMutex); + hdf5Utility::LinkFileInMaster(masterFileName, fname, datasetName, + dataFile_->GetParameterNames(), silentMode, + hdf5LibMutex); } #endif @@ -232,9 +216,9 @@ std::string DataProcessor::CreateMasterFile( switch (fileFormatType) { #ifdef HDF5C case HDF5: - return HDF5MasterFile::CreateMasterFile(filePath, fileNamePrefix, - fileIndex, overWriteEnable, - silentMode, attr, hdf5LibMutex); + return hdf5Utility::CreateMasterFile(filePath, fileNamePrefix, + fileIndex, overWriteEnable, + silentMode, attr, hdf5LibMutex); #endif case BINARY: return BinaryMasterFile::CreateMasterFile(filePath, fileNamePrefix, diff --git a/slsReceiverSoftware/src/DataProcessor.h b/slsReceiverSoftware/src/DataProcessor.h index 2c8feed3d..adab81aac 100644 --- a/slsReceiverSoftware/src/DataProcessor.h +++ b/slsReceiverSoftware/src/DataProcessor.h @@ -59,17 +59,17 @@ class DataProcessor : private virtual slsDetectorDefs, public ThreadObject { const bool detectorDataStream); #ifdef HDF5C uint32_t GetFilesInAcquisition() const; - void CreateVirtualFile(const std::string &filePath, - const std::string &fileNamePrefix, - const uint64_t fileIndex, const bool overWriteEnable, - const bool silentMode, const int modulePos, - const int numUnitsPerReadout, - const uint32_t maxFramesPerFile, - const uint64_t numImages, - const uint32_t dynamicRange, const int numModX, - const int numModY, std::mutex *hdf5LibMutex); - void LinkDataInMasterFile(const std::string &masterFileName, - const bool silentMode, std::mutex *hdf5LibMutex); + std::array CreateVirtualFile( + const std::string &filePath, const std::string &fileNamePrefix, + const uint64_t fileIndex, const bool overWriteEnable, + const bool silentMode, const int modulePos, + const int numUnitsPerReadout, const uint32_t maxFramesPerFile, + const uint64_t numImages, const int numModX, const int numModY, + const uint32_t dynamicRange, std::mutex *hdf5LibMutex); + void LinkFileInMaster(const std::string &masterFileName, + const std::string &virtualFileName, + const std::string &virtualDatasetName, + const bool silentMode, std::mutex *hdf5LibMutex); #endif std::string CreateMasterFile(const std::string &filePath, @@ -186,9 +186,6 @@ class DataProcessor : private virtual slsDetectorDefs, public ThreadObject { bool firstStreamerFrame_{false}; File *dataFile_{nullptr}; -#ifdef HDF5C - File *virtualFile_{nullptr}; -#endif // call back /** diff --git a/slsReceiverSoftware/src/File.h b/slsReceiverSoftware/src/File.h index 8edb28052..57282c46a 100644 --- a/slsReceiverSoftware/src/File.h +++ b/slsReceiverSoftware/src/File.h @@ -60,20 +60,6 @@ class File : private virtual slsDetectorDefs { return std::vector{}; }; - virtual void CreateVirtualFile( - const std::string filePath, const std::string fileNamePrefix, - const uint64_t fileIndex, const bool overWriteEnable, - const bool silentMode, const int modulePos, - const int numUnitsPerReadout, const uint32_t maxFramesPerFile, - const uint64_t numImages, const uint32_t nPixelsX, - const uint32_t nPixelsY, const uint32_t dynamicRange, - const uint64_t numImagesCaught, const int numModX, const int numModY, - const DataType dataType, const std::vector parameterNames, - const std::vector parameterDataTypes) { - LOG(logERROR) << "This is a generic function CreateVirtualFile that " - "should be overloaded by a derived class"; - } - virtual void CreateFirstHDF5DataFile( const std::string filePath, const std::string fileNamePrefix, const uint64_t fileIndex, const bool overWriteEnable, @@ -85,13 +71,6 @@ class File : private virtual slsDetectorDefs { LOG(logERROR) << "This is a generic function CreateFirstDataFile that " "should be overloaded by a derived class"; }; - - virtual void LinkDataFile(std::string dataFilename, std::string dataSetname, - const std::vector parameterNames, - const bool silentMode) { - LOG(logERROR) << "This is a generic function LinkDataFile that " - "should be overloaded by a derived class"; - }; #endif virtual void CreateFirstBinaryDataFile( const std::string filePath, const std::string fileNamePrefix, diff --git a/slsReceiverSoftware/src/HDF5MasterFile.cpp b/slsReceiverSoftware/src/HDF5MasterFile.cpp deleted file mode 100644 index 35f15826e..000000000 --- a/slsReceiverSoftware/src/HDF5MasterFile.cpp +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-or-other -// Copyright (C) 2021 Contributors to the SLS Detector Package -#include "HDF5MasterFile.h" -#include "MasterAttributes.h" - -void HDF5MasterFile::LinkDataFile(const std::string &masterFileName, - const std::string &dataFilename, - const std::string &dataSetname, - const std::vector parameterNames, - const bool silentMode, - std::mutex *hdf5LibMutex) { - - std::lock_guard lock(*hdf5Lib_); - try { - Exception::dontPrint(); // to handle errors - - FileAccPropList flist; - flist.setFcloseDegree(H5F_CLOSE_STRONG); - - // open master file - H5File masterfd(masterFileName.c_str(), H5F_ACC_RDWR, - FileCreatPropList::DEFAULT, flist); - - // open data file - H5File fd(dataFilename.c_str(), H5F_ACC_RDONLY, - FileCreatPropList::DEFAULT, flist); - - // create link for data dataset - DataSet dset = fd.openDataSet(dataSetname.c_str()); - std::string linkname = std::string("/entry/data/") + dataSetname; - if (H5Lcreate_external(dataFilename.c_str(), dataSetname.c_str(), - masterfd.getLocId(), linkname.c_str(), - H5P_DEFAULT, H5P_DEFAULT) < 0) { - throw sls::RuntimeError( - "Could not create link to data dataset in master"); - } - - // create link for parameter datasets - for (unsigned int i = 0; i < parameterNames.size(); ++i) { - DataSet pDset = fd.openDataSet(parameterNames[i].c_str()); - linkname = std::string("/entry/data/") + parameterNames[i]; - if (H5Lcreate_external(dataFilename.c_str(), - parameterNames[i].c_str(), - masterfd.getLocId(), linkname.c_str(), - H5P_DEFAULT, H5P_DEFAULT) < 0) { - throw sls::RuntimeError( - "Could not create link to parameter dataset in master"); - } - } - fd.close(); - masterfd.close(); - } catch (const Exception &error) { - error.printErrorStack(); - fd.close(); - throw sls::RuntimeError("Could not link in master hdf5 file"); - } - if (!silentMode) { - LOG(logINFO) << "Linked in Master File: " << dataFilename; - } -} - -std::string HDF5MasterFile::CreateMasterFile( - const std::string filePath, const std::string fileNamePrefix, - const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode, - MasterAttributes *attr, std::mutex *hdf5LibMutex) { - - std::ostringstream os; - os << filePath << "/" << fileNamePrefix << "_master" - << "_" << fileIndex << ".h5"; - std::string fileName = os.str(); - - std::lock_guard lock(*hdf5Lib_); - - try { - Exception::dontPrint(); // to handle errors - - FileAccPropList flist; - flist.setFcloseDegree(H5F_CLOSE_STRONG); - - unsigned int createFlags = H5F_ACC_EXCL; - if (overWriteEnable) { - createFlags = H5F_ACC_TRUNC; - } - H5File fd(fileName.c_str(), createFlags, FileCreatPropList::DEFAULT, - flist); - - // attributes - version - double dValue = HDF5_WRITER_VERSION; - DataSpace dataspace_attr = DataSpace(H5S_SCALAR); - Attribute attribute = fd.createAttribute( - "version", PredType::NATIVE_DOUBLE, dataspace_attr); - attribute.write(PredType::NATIVE_DOUBLE, &dValue); - - // Create a group in the file - Group group1(fd.createGroup("entry")); - Group group2(group1.createGroup("data")); - Group group3(group1.createGroup("instrument")); - Group group4(group3.createGroup("beam")); - Group group5(group3.createGroup("detector")); - Group group6(group1.createGroup("sample")); - - attr->WriteHDF5Attributes(&fd, &group5); - fd.close(); - } catch (const Exception &error) { - error.printErrorStack(); - fd.close(); - throw sls::RuntimeError( - "Could not create/overwrite master HDF5 handles"); - } - if (!silentMode) { - LOG(logINFO) << "Master File: " << fileName; - } - return fileName; -} diff --git a/slsReceiverSoftware/src/HDF5MasterFile.h b/slsReceiverSoftware/src/HDF5MasterFile.h deleted file mode 100644 index 4a9c3ae7c..000000000 --- a/slsReceiverSoftware/src/HDF5MasterFile.h +++ /dev/null @@ -1,22 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-or-other -// Copyright (C) 2021 Contributors to the SLS Detector Package -#pragma once - -#include "MasterAttributes.h" - -#include - -class HDF5MasterFile : private virtual slsDetectorDefs { - public: - void LinkDataFile(const std::string &masterFileName, - const std::string &dataFilename, - const std::string &dataSetname, - const std::vector parameterNames, - const bool silentMode, std::mutex *hdf5LibMutex); - std::string CreateMasterFile(const std::string &filePath, - const std::string &fileNamePrefix, - const uint64_t fileIndex, - const bool overWriteEnable, - const bool silentMode, MasterAttributes *attr, - std::mutex *hdf5LibMutex); -}; \ No newline at end of file diff --git a/slsReceiverSoftware/src/HDF5VirtualFile.cpp b/slsReceiverSoftware/src/HDF5Utility.cpp similarity index 50% rename from slsReceiverSoftware/src/HDF5VirtualFile.cpp rename to slsReceiverSoftware/src/HDF5Utility.cpp index fe1f0d4e3..21891552b 100644 --- a/slsReceiverSoftware/src/HDF5VirtualFile.cpp +++ b/slsReceiverSoftware/src/HDF5Utility.cpp @@ -1,75 +1,172 @@ // SPDX-License-Identifier: LGPL-3.0-or-other // Copyright (C) 2021 Contributors to the SLS Detector Package -#include "HDF5VirtualFile.h" -#include "receiver_defs.h" +#include "HDF5Utility.h" +#include "sls/container_utils.h" #include -HDF5VirtualFile::HDF5VirtualFile(std::mutex *hdf5LibMutex, bool g25) - : File(HDF5), hdf5Lib_(hdf5LibMutex), gotthard25um(g25) {} +namespace hdf5Utility { -HDF5VirtualFile::~HDF5VirtualFile() { CloseFile(); } +void LinkFileInMaster(const std::string &masterFileName, + const std::string &dataFilename, + const std::string &dataSetname, + const std::vector parameterNames, + const bool silentMode, std::mutex *hdf5LibMutex) { -std::array HDF5VirtualFile::GetFileAndDatasetName() const { - return std::array{fileName_, dataSetName_}; -} - -void HDF5VirtualFile::CloseFile() { - std::lock_guard lock(*hdf5Lib_); + std::lock_guard lock(*hdf5LibMutex); + std::unique_ptr fd{nullptr}; try { Exception::dontPrint(); // to handle errors - if (fd_) { - fd_->close(); - delete fd_; - fd_ = nullptr; + + FileAccPropList flist; + flist.setFcloseDegree(H5F_CLOSE_STRONG); + + // open master file + H5File masterfd(masterFileName.c_str(), H5F_ACC_RDWR, + FileCreatPropList::DEFAULT, flist); + + // open data file + fd = sls::make_unique(dataFilename.c_str(), H5F_ACC_RDONLY, + FileCreatPropList::DEFAULT, flist); + + // create link for data dataset + DataSet dset = fd->openDataSet(dataSetname.c_str()); + std::string linkname = std::string("/entry/data/") + dataSetname; + if (H5Lcreate_external(dataFilename.c_str(), dataSetname.c_str(), + masterfd.getLocId(), linkname.c_str(), + H5P_DEFAULT, H5P_DEFAULT) < 0) { + throw sls::RuntimeError( + "Could not create link to data dataset in master"); } + + // create link for parameter datasets + for (unsigned int i = 0; i < parameterNames.size(); ++i) { + DataSet pDset = fd->openDataSet(parameterNames[i].c_str()); + linkname = std::string("/entry/data/") + parameterNames[i]; + if (H5Lcreate_external(dataFilename.c_str(), + parameterNames[i].c_str(), + masterfd.getLocId(), linkname.c_str(), + H5P_DEFAULT, H5P_DEFAULT) < 0) { + throw sls::RuntimeError( + "Could not create link to parameter dataset in master"); + } + } + fd->close(); + masterfd.close(); } catch (const Exception &error) { - LOG(logERROR) << "Could not close virtual HDF5 handles of index"; error.printErrorStack(); + if (fd != nullptr) + fd->close(); + throw sls::RuntimeError("Could not link in master hdf5 file"); + } + if (!silentMode) { + LOG(logINFO) << "Linked in Master File: " << dataFilename; } } -void HDF5VirtualFile::CreateVirtualFile( - const &std::string filePath, const &std::string fileNamePrefix, - const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode, - const int modulePos, const int numUnitsPerReadout, - const uint32_t maxFramesPerFile, const uint64_t numImages, - const uint32_t nPixelsX, const uint32_t nPixelsY, - const uint32_t dynamicRange, const uint64_t numImagesCaught, - const int numModX, const int numModY, const DataType dataType, - const std::vector parameterNames, - const std::vector parameterDataTypes) { +std::string CreateMasterFile(const std::string &filePath, + const std::string &fileNamePrefix, + const uint64_t fileIndex, + const bool overWriteEnable, const bool silentMode, + MasterAttributes *attr, std::mutex *hdf5LibMutex) { + + std::ostringstream os; + os << filePath << "/" << fileNamePrefix << "_master" + << "_" << fileIndex << ".h5"; + std::string fileName = os.str(); + + std::lock_guard lock(*hdf5LibMutex); + + std::unique_ptr fd{nullptr}; + try { + Exception::dontPrint(); // to handle errors + + FileAccPropList flist; + flist.setFcloseDegree(H5F_CLOSE_STRONG); + + unsigned int createFlags = H5F_ACC_EXCL; + if (overWriteEnable) { + createFlags = H5F_ACC_TRUNC; + } + fd = sls::make_unique(fileName.c_str(), createFlags, + FileCreatPropList::DEFAULT, flist); + + // attributes - version + double dValue = HDF5_WRITER_VERSION; + DataSpace dataspace_attr = DataSpace(H5S_SCALAR); + Attribute attribute = fd->createAttribute( + "version", PredType::NATIVE_DOUBLE, dataspace_attr); + attribute.write(PredType::NATIVE_DOUBLE, &dValue); + + // Create a group in the file + Group group1(fd->createGroup("entry")); + Group group2(group1.createGroup("data")); + Group group3(group1.createGroup("instrument")); + Group group4(group3.createGroup("beam")); + Group group5(group3.createGroup("detector")); + Group group6(group1.createGroup("sample")); + + attr->WriteHDF5Attributes(fd.get(), &group5); + fd->close(); + } catch (const Exception &error) { + error.printErrorStack(); + if (fd != nullptr) + fd->close(); + throw sls::RuntimeError( + "Could not create/overwrite master HDF5 handles"); + } + if (!silentMode) { + LOG(logINFO) << "Master File: " << fileName; + } + return fileName; +} + +std::array +CreateVirtualFile(const std::string &filePath, + const std::string &fileNamePrefix, const uint64_t fileIndex, + const bool overWriteEnable, const bool silentMode, + const int modulePos, const int numUnitsPerReadout, + const uint32_t maxFramesPerFile, const uint64_t numImages, + const uint32_t nPixelsX, const uint32_t nPixelsY, + const uint32_t dynamicRange, const uint64_t numImagesCaught, + const int numModX, const int numModY, const DataType dataType, + const std::vector parameterNames, + const std::vector parameterDataTypes, + std::mutex *hdf5LibMutex, bool gotthard25um) { + // virtual file name std::ostringstream osfn; osfn << filePath << "/" << fileNamePrefix << "_virtual" << "_" << fileIndex << ".h5"; - fileName_ = osfn.str(); + std::string fileName = osfn.str(); + + std::string dataSetName = "data"; unsigned int paraSize = parameterNames.size(); uint64_t numModZ = numModX; uint32_t nDimy = nPixelsY; uint32_t nDimz = ((dynamicRange == 4) ? (nPixelsX / 2) : nPixelsX); - std::lock_guard lock(*hdf5Lib_); + std::lock_guard lock(*hdf5LibMutex); + std::unique_ptr fd{nullptr}; try { Exception::dontPrint(); // to handle errors // file FileAccPropList fapl; fapl.setFcloseDegree(H5F_CLOSE_STRONG); - fd_ = nullptr; if (!overWriteEnable) - fd_ = new H5File(fileName_.c_str(), H5F_ACC_EXCL, - FileCreatPropList::DEFAULT, fapl); + fd = sls::make_unique(fileName.c_str(), H5F_ACC_EXCL, + FileCreatPropList::DEFAULT, fapl); else - fd_ = new H5File(fileName_.c_str(), H5F_ACC_TRUNC, - FileCreatPropList::DEFAULT, fapl); + fd = sls::make_unique(fileName.c_str(), H5F_ACC_TRUNC, + FileCreatPropList::DEFAULT, fapl); // attributes - version double dValue = HDF5_WRITER_VERSION; DataSpace dataspace_attr = DataSpace(H5S_SCALAR); - Attribute attribute = fd_->createAttribute( + Attribute attribute = fd->createAttribute( "version", PredType::NATIVE_DOUBLE, dataspace_attr); attribute.write(PredType::NATIVE_DOUBLE, &dValue); @@ -188,24 +285,28 @@ void HDF5VirtualFile::CreateVirtualFile( framesSaved += nDimx; } // datasets - dataSetName_ = "data"; - DataSet vdsDataSet(fd_->createDataSet(dataSetName_.c_str(), dataType, - vdsDataSpace, plist)); + DataSet vdsDataSet(fd->createDataSet(dataSetName.c_str(), dataType, + vdsDataSpace, plist)); for (unsigned int p = 0; p < paraSize; ++p) { - DataSet vdsDataSetPara(fd_->createDataSet( + DataSet vdsDataSetPara(fd->createDataSet( parameterNames[p].c_str(), parameterDataTypes[p], vdsDataSpacePara, plistPara[p])); } - fd_->close(); + fd->close(); } catch (const Exception &error) { error.printErrorStack(); - CloseFile(); + if (fd) { + fd->close(); + } throw sls::RuntimeError( "Could not create/overwrite virtual HDF5 handles"); } if (!silentMode) { - LOG(logINFO) << "Virtual File: " << fileName_; + LOG(logINFO) << "Virtual File: " << fileName; } + return std::array{fileName, dataSetName}; } + +} // namespace hdf5Utility diff --git a/slsReceiverSoftware/src/HDF5Utility.h b/slsReceiverSoftware/src/HDF5Utility.h new file mode 100644 index 000000000..c50e3129f --- /dev/null +++ b/slsReceiverSoftware/src/HDF5Utility.h @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: LGPL-3.0-or-other +// Copyright (C) 2021 Contributors to the SLS Detector Package +#pragma once + +#include "MasterAttributes.h" + +#ifdef HDF5C +#include "H5Cpp.h" +#ifndef H5_NO_NAMESPACE +using namespace H5; +#endif +#endif + +#include + +namespace hdf5Utility { + +void LinkFileInMaster(const std::string &masterFileName, + const std::string &dataFilename, + const std::string &dataSetname, + const std::vector parameterNames, + const bool silentMode, std::mutex *hdf5LibMutex); + +std::string CreateMasterFile(const std::string &filePath, + const std::string &fileNamePrefix, + const uint64_t fileIndex, + const bool overWriteEnable, const bool silentMode, + MasterAttributes *attr, std::mutex *hdf5LibMutex); + +std::array +CreateVirtualFile(const std::string &filePath, + const std::string &fileNamePrefix, const uint64_t fileIndex, + const bool overWriteEnable, const bool silentMode, + const int modulePos, const int numUnitsPerReadout, + const uint32_t maxFramesPerFile, const uint64_t numImages, + const uint32_t nPixelsX, const uint32_t nPixelsY, + const uint32_t dynamicRange, const uint64_t numImagesCaught, + const int numModX, const int numModY, const DataType dataType, + const std::vector parameterNames, + const std::vector parameterDataTypes, + std::mutex *hdf5LibMutex, bool gotthard25um); + +} // namespace hdf5Utility \ No newline at end of file diff --git a/slsReceiverSoftware/src/HDF5VirtualFile.h b/slsReceiverSoftware/src/HDF5VirtualFile.h deleted file mode 100644 index eee01dad5..000000000 --- a/slsReceiverSoftware/src/HDF5VirtualFile.h +++ /dev/null @@ -1,34 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-or-other -// Copyright (C) 2021 Contributors to the SLS Detector Package -#pragma once - -#include "File.h" - -#include - -class HDF5VirtualFile : private virtual slsDetectorDefs, public File { - - public: - HDF5VirtualFile(std::mutex *hdf5LibMutex, bool g25); - ~HDF5VirtualFile(); - - std::array GetFileAndDatasetName() const override; - void CloseFile() override; - void CreateVirtualFile( - const std::string &filePath, const std::string &fileNamePrefix, - const uint64_t fileIndex, const bool overWriteEnable, - const bool silentMode, const int modulePos, - const int numUnitsPerReadout, const uint32_t maxFramesPerFile, - const uint64_t numImages, const uint32_t nPixelsX, - const uint32_t nPixelsY, const uint32_t dynamicRange, - const uint64_t numImagesCaught, const int numModX, const int numModY, - const DataType dataType, const std::vector parameterNames, - const std::vector parameterDataTypes) override; - - private: - std::mutex *hdf5Lib_; - H5File *fd_{nullptr}; - std::string fileName_; - std::string dataSetName_; - bool gotthard25um; -}; \ No newline at end of file diff --git a/slsReceiverSoftware/src/Implementation.cpp b/slsReceiverSoftware/src/Implementation.cpp index ffa10e806..3ec1831d7 100644 --- a/slsReceiverSoftware/src/Implementation.cpp +++ b/slsReceiverSoftware/src/Implementation.cpp @@ -810,19 +810,22 @@ void Implementation::StartMasterWriter() { } #ifdef HDF5C if (fileFormatType == HDF5) { + std::array virtualFileAndDatasetNames; // create virtual hdf5 file (if multiple files) if (dataProcessor[0]->GetFilesInAcquisition() > 1 || (numModules.x * numModules.y) > 1) { - dataProcessor[0]->CreateVirtualFile( - filePath, fileName, fileIndex, overwriteEnable, silentMode, - modulePos, numUDPInterfaces, framesPerFile, - numberOfTotalFrames, dynamicRange, numModules.x, - numModules.y, &hdf5LibMutex); + virtualFileAndDatasetNames = + dataProcessor[0]->CreateVirtualFile( + filePath, fileName, fileIndex, overwriteEnable, + silentMode, modulePos, numUDPInterfaces, framesPerFile, + numberOfTotalFrames, numModules.x, numModules.y, + dynamicRange, &hdf5LibMutex); } // link file in master if (masterFileWriteEnable) { - dataProcessor[0]->LinkDataInMasterFile( - masterFileName, silentMode, &hdf5LibMutex); + dataProcessor[0]->LinkFileInMaster( + masterFileName, virtualFileAndDatasetNames[0], + virtualFileAndDatasetNames[1], silentMode, &hdf5LibMutex); } } #endif diff --git a/slsReceiverSoftware/src/MasterAttributes.cpp b/slsReceiverSoftware/src/MasterAttributes.cpp index 250806cb6..59de67ccd 100644 --- a/slsReceiverSoftware/src/MasterAttributes.cpp +++ b/slsReceiverSoftware/src/MasterAttributes.cpp @@ -385,6 +385,7 @@ void MasterAttributes::WriteHDF5ThresholdEnergy(H5File *fd, Group *group) { } void MasterAttributes::WriteHDF5ThresholdEnergies(H5File *fd, Group *group) { + char c[1024]{0}; DataSpace dataspace = DataSpace(H5S_SCALAR); StrType strdatatype(PredType::C_S1, 1024); DataSet dataset = From f5cca7a98fd1ac546a46db7e11fbc073773be322 Mon Sep 17 00:00:00 2001 From: Dhanya Thattil Date: Tue, 29 Mar 2022 13:30:06 +0200 Subject: [PATCH 07/12] removed binary master file as well --- slsReceiverSoftware/CMakeLists.txt | 3 +- slsReceiverSoftware/src/BinaryMasterFile.cpp | 52 ---------- slsReceiverSoftware/src/BinaryMasterFile.h | 19 ---- slsReceiverSoftware/src/DataProcessor.cpp | 22 ++--- slsReceiverSoftware/src/HDF5Utility.h | 43 --------- ...{HDF5Utility.cpp => MasterFileUtility.cpp} | 96 ++++++++++++++----- slsReceiverSoftware/src/MasterFileUtility.h | 50 ++++++++++ 7 files changed, 133 insertions(+), 152 deletions(-) delete mode 100644 slsReceiverSoftware/src/BinaryMasterFile.cpp delete mode 100644 slsReceiverSoftware/src/BinaryMasterFile.h delete mode 100644 slsReceiverSoftware/src/HDF5Utility.h rename slsReceiverSoftware/src/{HDF5Utility.cpp => MasterFileUtility.cpp} (77%) create mode 100644 slsReceiverSoftware/src/MasterFileUtility.h diff --git a/slsReceiverSoftware/CMakeLists.txt b/slsReceiverSoftware/CMakeLists.txt index 8589e0314..a72da71b7 100755 --- a/slsReceiverSoftware/CMakeLists.txt +++ b/slsReceiverSoftware/CMakeLists.txt @@ -6,7 +6,6 @@ set(SOURCES src/Receiver.cpp src/File.cpp src/BinaryDataFile.cpp - src/BinaryMasterFile.cpp src/ThreadObject.cpp src/Listener.cpp src/DataProcessor.cpp @@ -14,6 +13,7 @@ set(SOURCES src/Fifo.cpp src/Arping.cpp src/MasterAttributes.cpp + src/MasterFileUtility.cpp ) set(PUBLICHEADERS @@ -28,7 +28,6 @@ if (SLS_USE_HDF5) ) list (APPEND SOURCES src/HDF5DataFile.cpp - src/HDF5Utility.cpp ) endif (SLS_USE_HDF5) diff --git a/slsReceiverSoftware/src/BinaryMasterFile.cpp b/slsReceiverSoftware/src/BinaryMasterFile.cpp deleted file mode 100644 index 04fc0f2d7..000000000 --- a/slsReceiverSoftware/src/BinaryMasterFile.cpp +++ /dev/null @@ -1,52 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-or-other -// Copyright (C) 2021 Contributors to the SLS Detector Package -#include "BinaryMasterFile.h" -#include "MasterAttributes.h" - -std::string BinaryMasterFile::CreateMasterFile(const std::string filePath, - const std::string fileNamePrefix, - const uint64_t fileIndex, - const bool overWriteEnable, - const bool silentMode, - MasterAttributes *attr) { - // create file name - std::ostringstream os; - os << filePath << "/" << fileNamePrefix << "_master" - << "_" << fileIndex << ".json"; - std::string fileName = os.str(); - - // create file - FILE *fd{nullptr}; - if (!overWriteEnable) { - if (nullptr == (fd = fopen((const char *)fileName.c_str(), "wx"))) { - fd = nullptr; - throw sls::RuntimeError("Could not create binary master file " + - fileName); - } - } else if (nullptr == (fd = fopen((const char *)fileName.c_str(), "w"))) { - fd = nullptr; - throw sls::RuntimeError( - "Could not create/overwrite binary master file " + fileName); - } - - std::string message = BinaryMasterFile::GetMasterAttributes(attr); - if (fwrite((void *)message.c_str(), 1, message.length(), fd) != - message.length()) { - throw sls::RuntimeError( - "Master binary file incorrect number of bytes written to file"); - } - if (fd) { - fclose(fd); - } - if (!silentMode) { - LOG(logINFO) << "Master File: " << fileName; - } - return fileName; -} - -std::string BinaryMasterFile::GetMasterAttributes(MasterAttributes *attr) { - rapidjson::StringBuffer s; - rapidjson::Writer writer(s); - attr->GetBinaryAttributes(&writer); - return s.GetString(); -} diff --git a/slsReceiverSoftware/src/BinaryMasterFile.h b/slsReceiverSoftware/src/BinaryMasterFile.h deleted file mode 100644 index 9bfb4504a..000000000 --- a/slsReceiverSoftware/src/BinaryMasterFile.h +++ /dev/null @@ -1,19 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-or-other -// Copyright (C) 2021 Contributors to the SLS Detector Package -#pragma once - -#include "MasterAttributes.h" - -class BinaryMasterFile : private virtual slsDetectorDefs { - - public: - static std::string CreateMasterFile(const std::string filePath, - const std::string fileNamePrefix, - const uint64_t fileIndex, - const bool overWriteEnable, - const bool silentMode, - MasterAttributes *attr); - - private: - static std::string GetMasterAttributes(MasterAttributes *attr); -}; \ No newline at end of file diff --git a/slsReceiverSoftware/src/DataProcessor.cpp b/slsReceiverSoftware/src/DataProcessor.cpp index 927213509..4597d80ec 100644 --- a/slsReceiverSoftware/src/DataProcessor.cpp +++ b/slsReceiverSoftware/src/DataProcessor.cpp @@ -13,9 +13,9 @@ #include "Fifo.h" #include "GeneralData.h" #include "MasterAttributes.h" +#include "MasterFileUtility.h" #ifdef HDF5C #include "HDF5DataFile.h" -#include "HDF5Utility.h" #endif #include "DataStreamer.h" #include "sls/container_utils.h" @@ -177,7 +177,7 @@ std::array DataProcessor::CreateVirtualFile( // files (they exist anyway) assumption2: virtual file max frame index // is from R0 P0 (difference from others when missing frames or for a // stop acquisition) - return hdf5Utility::CreateVirtualFile( + return masterFileUtility::CreateVirtualHDF5File( filePath, fileNamePrefix, fileIndex, overWriteEnable, silentMode, modulePos, numUnitsPerReadout, framesPerFile, numImages, generalData_->nPixelsX, generalData_->nPixelsY, dynamicRange, @@ -198,9 +198,9 @@ void DataProcessor::LinkFileInMaster(const std::string &masterFileName, fname = res[0]; datasetName = res[1]; } - hdf5Utility::LinkFileInMaster(masterFileName, fname, datasetName, - dataFile_->GetParameterNames(), silentMode, - hdf5LibMutex); + masterFileUtility::LinkHDF5FileInMaster(masterFileName, fname, datasetName, + dataFile_->GetParameterNames(), + silentMode, hdf5LibMutex); } #endif @@ -216,14 +216,14 @@ std::string DataProcessor::CreateMasterFile( switch (fileFormatType) { #ifdef HDF5C case HDF5: - return hdf5Utility::CreateMasterFile(filePath, fileNamePrefix, - fileIndex, overWriteEnable, - silentMode, attr, hdf5LibMutex); + return masterFileUtility::CreateMasterHDF5File( + filePath, fileNamePrefix, fileIndex, overWriteEnable, silentMode, + attr, hdf5LibMutex); #endif case BINARY: - return BinaryMasterFile::CreateMasterFile(filePath, fileNamePrefix, - fileIndex, overWriteEnable, - silentMode, attr); + return masterFileUtility::CreateMasterBinaryFile( + filePath, fileNamePrefix, fileIndex, overWriteEnable, silentMode, + attr); default: throw sls::RuntimeError("Unknown file format (compile with hdf5 flags"); } diff --git a/slsReceiverSoftware/src/HDF5Utility.h b/slsReceiverSoftware/src/HDF5Utility.h deleted file mode 100644 index c50e3129f..000000000 --- a/slsReceiverSoftware/src/HDF5Utility.h +++ /dev/null @@ -1,43 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-or-other -// Copyright (C) 2021 Contributors to the SLS Detector Package -#pragma once - -#include "MasterAttributes.h" - -#ifdef HDF5C -#include "H5Cpp.h" -#ifndef H5_NO_NAMESPACE -using namespace H5; -#endif -#endif - -#include - -namespace hdf5Utility { - -void LinkFileInMaster(const std::string &masterFileName, - const std::string &dataFilename, - const std::string &dataSetname, - const std::vector parameterNames, - const bool silentMode, std::mutex *hdf5LibMutex); - -std::string CreateMasterFile(const std::string &filePath, - const std::string &fileNamePrefix, - const uint64_t fileIndex, - const bool overWriteEnable, const bool silentMode, - MasterAttributes *attr, std::mutex *hdf5LibMutex); - -std::array -CreateVirtualFile(const std::string &filePath, - const std::string &fileNamePrefix, const uint64_t fileIndex, - const bool overWriteEnable, const bool silentMode, - const int modulePos, const int numUnitsPerReadout, - const uint32_t maxFramesPerFile, const uint64_t numImages, - const uint32_t nPixelsX, const uint32_t nPixelsY, - const uint32_t dynamicRange, const uint64_t numImagesCaught, - const int numModX, const int numModY, const DataType dataType, - const std::vector parameterNames, - const std::vector parameterDataTypes, - std::mutex *hdf5LibMutex, bool gotthard25um); - -} // namespace hdf5Utility \ No newline at end of file diff --git a/slsReceiverSoftware/src/HDF5Utility.cpp b/slsReceiverSoftware/src/MasterFileUtility.cpp similarity index 77% rename from slsReceiverSoftware/src/HDF5Utility.cpp rename to slsReceiverSoftware/src/MasterFileUtility.cpp index 21891552b..0ae1dc57d 100644 --- a/slsReceiverSoftware/src/HDF5Utility.cpp +++ b/slsReceiverSoftware/src/MasterFileUtility.cpp @@ -1,17 +1,62 @@ // SPDX-License-Identifier: LGPL-3.0-or-other // Copyright (C) 2021 Contributors to the SLS Detector Package -#include "HDF5Utility.h" +#include "MasterFileUtility.h" #include "sls/container_utils.h" #include -namespace hdf5Utility { +namespace masterFileUtility { -void LinkFileInMaster(const std::string &masterFileName, - const std::string &dataFilename, - const std::string &dataSetname, - const std::vector parameterNames, - const bool silentMode, std::mutex *hdf5LibMutex) { +std::string CreateMasterBinaryFile(const std::string filePath, + const std::string fileNamePrefix, + const uint64_t fileIndex, + const bool overWriteEnable, + const bool silentMode, + MasterAttributes *attr) { + // create file name + std::ostringstream os; + os << filePath << "/" << fileNamePrefix << "_master" + << "_" << fileIndex << ".json"; + std::string fileName = os.str(); + + // create file + FILE *fd{nullptr}; + if (!overWriteEnable) { + if (nullptr == (fd = fopen((const char *)fileName.c_str(), "wx"))) { + fd = nullptr; + throw sls::RuntimeError("Could not create binary master file " + + fileName); + } + } else if (nullptr == (fd = fopen((const char *)fileName.c_str(), "w"))) { + fd = nullptr; + throw sls::RuntimeError( + "Could not create/overwrite binary master file " + fileName); + } + + rapidjson::StringBuffer s; + rapidjson::Writer writer(s); + attr->GetBinaryAttributes(&writer); + std::string message = s.GetString(); + if (fwrite((void *)message.c_str(), 1, message.length(), fd) != + message.length()) { + throw sls::RuntimeError( + "Master binary file incorrect number of bytes written to file"); + } + if (fd) { + fclose(fd); + } + if (!silentMode) { + LOG(logINFO) << "Master File: " << fileName; + } + return fileName; +} + +#ifdef HDF5C +void LinkHDF5FileInMaster(const std::string &masterFileName, + const std::string &dataFilename, + const std::string &dataSetname, + const std::vector parameterNames, + const bool silentMode, std::mutex *hdf5LibMutex) { std::lock_guard lock(*hdf5LibMutex); std::unique_ptr fd{nullptr}; @@ -64,11 +109,12 @@ void LinkFileInMaster(const std::string &masterFileName, } } -std::string CreateMasterFile(const std::string &filePath, - const std::string &fileNamePrefix, - const uint64_t fileIndex, - const bool overWriteEnable, const bool silentMode, - MasterAttributes *attr, std::mutex *hdf5LibMutex) { +std::string CreateMasterHDF5File(const std::string &filePath, + const std::string &fileNamePrefix, + const uint64_t fileIndex, + const bool overWriteEnable, + const bool silentMode, MasterAttributes *attr, + std::mutex *hdf5LibMutex) { std::ostringstream os; os << filePath << "/" << fileNamePrefix << "_master" @@ -121,18 +167,17 @@ std::string CreateMasterFile(const std::string &filePath, return fileName; } -std::array -CreateVirtualFile(const std::string &filePath, - const std::string &fileNamePrefix, const uint64_t fileIndex, - const bool overWriteEnable, const bool silentMode, - const int modulePos, const int numUnitsPerReadout, - const uint32_t maxFramesPerFile, const uint64_t numImages, - const uint32_t nPixelsX, const uint32_t nPixelsY, - const uint32_t dynamicRange, const uint64_t numImagesCaught, - const int numModX, const int numModY, const DataType dataType, - const std::vector parameterNames, - const std::vector parameterDataTypes, - std::mutex *hdf5LibMutex, bool gotthard25um) { +std::array CreateVirtualHDF5File( + const std::string &filePath, const std::string &fileNamePrefix, + const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode, + const int modulePos, const int numUnitsPerReadout, + const uint32_t maxFramesPerFile, const uint64_t numImages, + const uint32_t nPixelsX, const uint32_t nPixelsY, + const uint32_t dynamicRange, const uint64_t numImagesCaught, + const int numModX, const int numModY, const DataType dataType, + const std::vector parameterNames, + const std::vector parameterDataTypes, std::mutex *hdf5LibMutex, + bool gotthard25um) { // virtual file name std::ostringstream osfn; @@ -308,5 +353,6 @@ CreateVirtualFile(const std::string &filePath, } return std::array{fileName, dataSetName}; } +#endif -} // namespace hdf5Utility +} // namespace masterFileUtility diff --git a/slsReceiverSoftware/src/MasterFileUtility.h b/slsReceiverSoftware/src/MasterFileUtility.h new file mode 100644 index 000000000..d0a74b04f --- /dev/null +++ b/slsReceiverSoftware/src/MasterFileUtility.h @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: LGPL-3.0-or-other +// Copyright (C) 2021 Contributors to the SLS Detector Package +#pragma once + +#include "MasterAttributes.h" + +#ifdef HDF5C +#include "H5Cpp.h" +#include +#ifndef H5_NO_NAMESPACE +using namespace H5; +#endif +#endif + +namespace masterFileUtility { + +std::string CreateMasterBinaryFile(const std::string filePath, + const std::string fileNamePrefix, + const uint64_t fileIndex, + const bool overWriteEnable, + const bool silentMode, + MasterAttributes *attr); + +#ifdef HDF5C +void LinkHDF5FileInMaster(const std::string &masterFileName, + const std::string &dataFilename, + const std::string &dataSetname, + const std::vector parameterNames, + const bool silentMode, std::mutex *hdf5LibMutex); + +std::string CreateMasterHDF5File(const std::string &filePath, + const std::string &fileNamePrefix, + const uint64_t fileIndex, + const bool overWriteEnable, + const bool silentMode, MasterAttributes *attr, + std::mutex *hdf5LibMutex); + +std::array CreateVirtualHDF5File( + const std::string &filePath, const std::string &fileNamePrefix, + const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode, + const int modulePos, const int numUnitsPerReadout, + const uint32_t maxFramesPerFile, const uint64_t numImages, + const uint32_t nPixelsX, const uint32_t nPixelsY, + const uint32_t dynamicRange, const uint64_t numImagesCaught, + const int numModX, const int numModY, const DataType dataType, + const std::vector parameterNames, + const std::vector parameterDataTypes, std::mutex *hdf5LibMutex, + bool gotthard25um); +#endif +} // namespace masterFileUtility \ No newline at end of file From 8ce6868e466a3eeefe76676cea7ada4ba4e582a2 Mon Sep 17 00:00:00 2001 From: Dhanya Thattil Date: Tue, 29 Mar 2022 16:13:33 +0200 Subject: [PATCH 08/12] fixed compilation --- slsReceiverSoftware/src/DataProcessor.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/slsReceiverSoftware/src/DataProcessor.cpp b/slsReceiverSoftware/src/DataProcessor.cpp index 4597d80ec..a14e57509 100644 --- a/slsReceiverSoftware/src/DataProcessor.cpp +++ b/slsReceiverSoftware/src/DataProcessor.cpp @@ -9,7 +9,6 @@ #include "DataProcessor.h" #include "BinaryDataFile.h" -#include "BinaryMasterFile.h" #include "Fifo.h" #include "GeneralData.h" #include "MasterAttributes.h" From 74e325edb4fb3c1cf1604f010fc86fe98aab8ba6 Mon Sep 17 00:00:00 2001 From: Dhanya Thattil Date: Wed, 30 Mar 2022 09:06:09 +0200 Subject: [PATCH 09/12] removing \n in timestamp for json file --- slsReceiverSoftware/src/MasterAttributes.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/slsReceiverSoftware/src/MasterAttributes.cpp b/slsReceiverSoftware/src/MasterAttributes.cpp index 59de67ccd..743baeedb 100644 --- a/slsReceiverSoftware/src/MasterAttributes.cpp +++ b/slsReceiverSoftware/src/MasterAttributes.cpp @@ -76,7 +76,9 @@ void MasterAttributes::GetCommonBinaryAttributes( w->Double(BINARY_WRITER_VERSION); w->Key("Timestamp"); time_t t = time(nullptr); - w->String(ctime(&t)); + std::string sTime(ctime(&t)); + std::replace(sTime.begin(), sTime.end(), '\n', '\0'); + w->String(sTime.c_str()); w->Key("Detector Type"); w->String(sls::ToString(detType).c_str()); w->Key("Timing Mode"); From c57e528447aea603b8ebdf83bf704f75d53d8630 Mon Sep 17 00:00:00 2001 From: Dhanya Thattil Date: Wed, 30 Mar 2022 10:43:43 +0200 Subject: [PATCH 10/12] wip, prettywriter --- slsReceiverSoftware/src/MasterAttributes.cpp | 20 ++++++++--------- slsReceiverSoftware/src/MasterAttributes.h | 22 +++++++++---------- slsReceiverSoftware/src/MasterFileUtility.cpp | 2 +- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/slsReceiverSoftware/src/MasterAttributes.cpp b/slsReceiverSoftware/src/MasterAttributes.cpp index 743baeedb..ded5589da 100644 --- a/slsReceiverSoftware/src/MasterAttributes.cpp +++ b/slsReceiverSoftware/src/MasterAttributes.cpp @@ -3,7 +3,7 @@ #include "MasterAttributes.h" void MasterAttributes::GetBinaryAttributes( - rapidjson::Writer *w) { + rapidjson::PrettyWriter *w) { w->StartObject(); GetCommonBinaryAttributes(w); switch (detType) { @@ -70,7 +70,7 @@ void MasterAttributes::WriteHDF5Attributes(H5File *fd, Group *group) { #endif void MasterAttributes::GetCommonBinaryAttributes( - rapidjson::Writer *w) { + rapidjson::PrettyWriter *w) { w->Key("Version"); w->SetMaxDecimalPlaces(2); w->Double(BINARY_WRITER_VERSION); @@ -112,7 +112,7 @@ void MasterAttributes::GetCommonBinaryAttributes( } void MasterAttributes::GetFinalBinaryAttributes( - rapidjson::Writer *w) { + rapidjson::PrettyWriter *w) { // adding few common parameters to the end if (!additionalJsonHeader.empty()) { w->Key("Additional Json Header"); @@ -533,7 +533,7 @@ void MasterAttributes::WriteHDF5DbitList(H5File *fd, Group *group) { #endif void MasterAttributes::GetGotthardBinaryAttributes( - rapidjson::Writer *w) { + rapidjson::PrettyWriter *w) { w->Key("Exptime"); w->String(sls::ToString(exptime).c_str()); w->Key("Period"); @@ -551,7 +551,7 @@ void MasterAttributes::WriteGotthardHDF5Attributes(H5File *fd, Group *group) { #endif void MasterAttributes::GetJungfrauBinaryAttributes( - rapidjson::Writer *w) { + rapidjson::PrettyWriter *w) { w->Key("Exptime"); w->String(sls::ToString(exptime).c_str()); w->Key("Period"); @@ -572,7 +572,7 @@ void MasterAttributes::WriteJungfrauHDF5Attributes(H5File *fd, Group *group) { #endif void MasterAttributes::GetEigerBinaryAttributes( - rapidjson::Writer *w) { + rapidjson::PrettyWriter *w) { w->Key("Dynamic Range"); w->Uint(dynamicRange); w->Key("Ten Giga"); @@ -611,7 +611,7 @@ void MasterAttributes::WriteEigerHDF5Attributes(H5File *fd, Group *group) { #endif void MasterAttributes::GetMythen3BinaryAttributes( - rapidjson::Writer *w) { + rapidjson::PrettyWriter *w) { w->Key("Dynamic Range"); w->Uint(dynamicRange); w->Key("Ten Giga"); @@ -648,7 +648,7 @@ void MasterAttributes::WriteMythen3HDF5Attributes(H5File *fd, Group *group) { #endif void MasterAttributes::GetGotthard2BinaryAttributes( - rapidjson::Writer *w) { + rapidjson::PrettyWriter *w) { w->Key("Exptime"); w->String(sls::ToString(exptime).c_str()); w->Key("Period"); @@ -666,7 +666,7 @@ void MasterAttributes::WriteGotthard2HDF5Attributes(H5File *fd, Group *group) { #endif void MasterAttributes::GetMoenchBinaryAttributes( - rapidjson::Writer *w) { + rapidjson::PrettyWriter *w) { w->Key("Exptime"); w->String(sls::ToString(exptime).c_str()); w->Key("Period"); @@ -690,7 +690,7 @@ void MasterAttributes::WriteMoenchHDF5Attributes(H5File *fd, Group *group) { #endif void MasterAttributes::GetCtbBinaryAttributes( - rapidjson::Writer *w) { + rapidjson::PrettyWriter *w) { w->Key("Exptime"); w->String(sls::ToString(exptime).c_str()); w->Key("Period"); diff --git a/slsReceiverSoftware/src/MasterAttributes.h b/slsReceiverSoftware/src/MasterAttributes.h index 66ae25016..34479bfb0 100644 --- a/slsReceiverSoftware/src/MasterAttributes.h +++ b/slsReceiverSoftware/src/MasterAttributes.h @@ -8,7 +8,7 @@ #include "sls/sls_detector_defs.h" #include -#include +#include #ifdef HDF5C #include "H5Cpp.h" @@ -65,15 +65,15 @@ class MasterAttributes { MasterAttributes() = default; ~MasterAttributes() = default; - void GetBinaryAttributes(rapidjson::Writer *w); + void GetBinaryAttributes(rapidjson::PrettyWriter *w); #ifdef HDF5C void WriteHDF5Attributes(H5File *fd, Group *group); #endif void - GetCommonBinaryAttributes(rapidjson::Writer *w); + GetCommonBinaryAttributes(rapidjson::PrettyWriter *w); void - GetFinalBinaryAttributes(rapidjson::Writer *w); + GetFinalBinaryAttributes(rapidjson::PrettyWriter *w); #ifdef HDF5C void WriteCommonHDF5Attributes(H5File *fd, Group *group); void WriteFinalHDF5Attributes(H5File *fd, Group *group); @@ -105,42 +105,42 @@ class MasterAttributes { #endif void - GetGotthardBinaryAttributes(rapidjson::Writer *w); + GetGotthardBinaryAttributes(rapidjson::PrettyWriter *w); #ifdef HDF5C void WriteGotthardHDF5Attributes(H5File *fd, Group *group); #endif void - GetJungfrauBinaryAttributes(rapidjson::Writer *w); + GetJungfrauBinaryAttributes(rapidjson::PrettyWriter *w); #ifdef HDF5C void WriteJungfrauHDF5Attributes(H5File *fd, Group *group); #endif void - GetEigerBinaryAttributes(rapidjson::Writer *w); + GetEigerBinaryAttributes(rapidjson::PrettyWriter *w); #ifdef HDF5C void WriteEigerHDF5Attributes(H5File *fd, Group *group); #endif void - GetMythen3BinaryAttributes(rapidjson::Writer *w); + GetMythen3BinaryAttributes(rapidjson::PrettyWriter *w); #ifdef HDF5C void WriteMythen3HDF5Attributes(H5File *fd, Group *group); #endif void - GetGotthard2BinaryAttributes(rapidjson::Writer *w); + GetGotthard2BinaryAttributes(rapidjson::PrettyWriter *w); #ifdef HDF5C void WriteGotthard2HDF5Attributes(H5File *fd, Group *group); #endif void - GetMoenchBinaryAttributes(rapidjson::Writer *w); + GetMoenchBinaryAttributes(rapidjson::PrettyWriter *w); #ifdef HDF5C void WriteMoenchHDF5Attributes(H5File *fd, Group *group); #endif - void GetCtbBinaryAttributes(rapidjson::Writer *w); + void GetCtbBinaryAttributes(rapidjson::PrettyWriter *w); #ifdef HDF5C void WriteCtbHDF5Attributes(H5File *fd, Group *group); #endif diff --git a/slsReceiverSoftware/src/MasterFileUtility.cpp b/slsReceiverSoftware/src/MasterFileUtility.cpp index 0ae1dc57d..56f017fa2 100644 --- a/slsReceiverSoftware/src/MasterFileUtility.cpp +++ b/slsReceiverSoftware/src/MasterFileUtility.cpp @@ -34,7 +34,7 @@ std::string CreateMasterBinaryFile(const std::string filePath, } rapidjson::StringBuffer s; - rapidjson::Writer writer(s); + rapidjson::PrettyWriter writer(s); attr->GetBinaryAttributes(&writer); std::string message = s.GetString(); if (fwrite((void *)message.c_str(), 1, message.length(), fd) != From e1988bf08828ae667492679942c38ed2b7118dd5 Mon Sep 17 00:00:00 2001 From: Dhanya Thattil Date: Wed, 30 Mar 2022 16:47:43 +0200 Subject: [PATCH 11/12] fixes --- slsReceiverSoftware/src/DataProcessor.cpp | 2 +- slsReceiverSoftware/src/DataProcessor.h | 4 +-- slsReceiverSoftware/src/File.h | 5 +--- slsReceiverSoftware/src/MasterAttributes.cpp | 29 ++++++++----------- slsReceiverSoftware/src/MasterFileUtility.cpp | 27 +++++++---------- slsReceiverSoftware/src/MasterFileUtility.h | 4 +-- 6 files changed, 28 insertions(+), 43 deletions(-) diff --git a/slsReceiverSoftware/src/DataProcessor.cpp b/slsReceiverSoftware/src/DataProcessor.cpp index a14e57509..d6b801e3f 100644 --- a/slsReceiverSoftware/src/DataProcessor.cpp +++ b/slsReceiverSoftware/src/DataProcessor.cpp @@ -111,7 +111,7 @@ void DataProcessor::SetupFileWriter(const bool filewriteEnable, } void DataProcessor::CreateFirstFiles( - const std::string filePath, const std::string fileNamePrefix, + const std::string &filePath, const std::string &fileNamePrefix, const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode, const int modulePos, const int numUnitsPerReadout, const uint32_t udpPortNumber, const uint32_t maxFramesPerFile, diff --git a/slsReceiverSoftware/src/DataProcessor.h b/slsReceiverSoftware/src/DataProcessor.h index adab81aac..b62bb9d1f 100644 --- a/slsReceiverSoftware/src/DataProcessor.h +++ b/slsReceiverSoftware/src/DataProcessor.h @@ -48,8 +48,8 @@ class DataProcessor : private virtual slsDetectorDefs, public ThreadObject { const fileFormat fileFormatType, std::mutex *hdf5LibMutex); - void CreateFirstFiles(const std::string filePath, - const std::string fileNamePrefix, + void CreateFirstFiles(const std::string &filePath, + const std::string &fileNamePrefix, const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode, const int modulePos, const int numUnitsPerReadout, diff --git a/slsReceiverSoftware/src/File.h b/slsReceiverSoftware/src/File.h index 57282c46a..9b386a640 100644 --- a/slsReceiverSoftware/src/File.h +++ b/slsReceiverSoftware/src/File.h @@ -84,10 +84,7 @@ class File : private virtual slsDetectorDefs { virtual void WriteToFile(char *buffer, const int buffersize, const uint64_t currentFrameNumber, - const uint32_t numPacketsCaught) { - LOG(logERROR) << "This is a generic function WriteToFile that " - "should be overloaded by a derived class"; - }; + const uint32_t numPacketsCaught) = 0; protected: slsDetectorDefs::fileFormat format_; diff --git a/slsReceiverSoftware/src/MasterAttributes.cpp b/slsReceiverSoftware/src/MasterAttributes.cpp index ded5589da..b82cc0e42 100644 --- a/slsReceiverSoftware/src/MasterAttributes.cpp +++ b/slsReceiverSoftware/src/MasterAttributes.cpp @@ -155,8 +155,7 @@ void MasterAttributes::GetFinalBinaryAttributes( #ifdef HDF5C void MasterAttributes::WriteCommonHDF5Attributes(H5File *fd, Group *group) { - char c[1024]; - memset(c, 0, sizeof(c)); + char c[1024]{}; // version { double version = BINARY_WRITER_VERSION; @@ -278,8 +277,7 @@ void MasterAttributes::WriteCommonHDF5Attributes(H5File *fd, Group *group) { } void MasterAttributes::WriteFinalHDF5Attributes(H5File *fd, Group *group) { - char c[1024]; - memset(c, 0, sizeof(c)); + char c[1024]{}; // Total Frames in file { DataSpace dataspace = DataSpace(H5S_SCALAR); @@ -304,8 +302,7 @@ void MasterAttributes::WriteHDF5Exptime(H5File *fd, Group *group) { StrType strdatatype(PredType::C_S1, 256); DataSet dataset = group->createDataSet("Exposure Time", strdatatype, dataspace); - char c[1024]; - memset(c, 0, sizeof(c)); + char c[1024]{}; sls::strcpy_safe(c, sls::ToString(exptime)); dataset.write(c, strdatatype); } @@ -315,8 +312,7 @@ void MasterAttributes::WriteHDF5Period(H5File *fd, Group *group) { StrType strdatatype(PredType::C_S1, 256); DataSet dataset = group->createDataSet("Acquisition Period", strdatatype, dataspace); - char c[1024]; - memset(c, 0, sizeof(c)); + char c[1024]{}; sls::strcpy_safe(c, sls::ToString(period)); dataset.write(c, strdatatype); } @@ -373,7 +369,7 @@ void MasterAttributes::WriteHDF5ReadNRows(H5File *fd, Group *group) { } void MasterAttributes::WriteHDF5ThresholdEnergy(H5File *fd, Group *group) { - char c[1024]{0}; + char c[1024]{}; DataSpace dataspace = DataSpace(H5S_SCALAR); DataSet dataset = group->createDataSet("Threshold Energy", PredType::NATIVE_INT, dataspace); @@ -387,7 +383,7 @@ void MasterAttributes::WriteHDF5ThresholdEnergy(H5File *fd, Group *group) { } void MasterAttributes::WriteHDF5ThresholdEnergies(H5File *fd, Group *group) { - char c[1024]{0}; + char c[1024]{}; DataSpace dataspace = DataSpace(H5S_SCALAR); StrType strdatatype(PredType::C_S1, 1024); DataSet dataset = @@ -397,7 +393,7 @@ void MasterAttributes::WriteHDF5ThresholdEnergies(H5File *fd, Group *group) { } void MasterAttributes::WriteHDF5SubExpTime(H5File *fd, Group *group) { - char c[1024]{0}; + char c[1024]{}; DataSpace dataspace = DataSpace(H5S_SCALAR); StrType strdatatype(PredType::C_S1, 256); DataSet dataset = @@ -407,7 +403,7 @@ void MasterAttributes::WriteHDF5SubExpTime(H5File *fd, Group *group) { } void MasterAttributes::WriteHDF5SubPeriod(H5File *fd, Group *group) { - char c[1024]{0}; + char c[1024]{}; DataSpace dataspace = DataSpace(H5S_SCALAR); StrType strdatatype(PredType::C_S1, 256); DataSet dataset = @@ -424,7 +420,7 @@ void MasterAttributes::WriteHDF5SubQuad(H5File *fd, Group *group) { } void MasterAttributes::WriteHDF5RateCorrections(H5File *fd, Group *group) { - char c[1024]{0}; + char c[1024]{}; DataSpace dataspace = DataSpace(H5S_SCALAR); StrType strdatatype(PredType::C_S1, 1024); DataSet dataset = @@ -442,7 +438,7 @@ void MasterAttributes::WriteHDF5CounterMask(H5File *fd, Group *group) { void MasterAttributes::WriteHDF5ExptimeArray(H5File *fd, Group *group) { for (int i = 0; i != 3; ++i) { - char c[1024]{0}; + char c[1024]{}; DataSpace dataspace = DataSpace(H5S_SCALAR); StrType strdatatype(PredType::C_S1, 256); DataSet dataset = @@ -454,7 +450,7 @@ void MasterAttributes::WriteHDF5ExptimeArray(H5File *fd, Group *group) { void MasterAttributes::WriteHDF5GateDelayArray(H5File *fd, Group *group) { for (int i = 0; i != 3; ++i) { - char c[1024]{0}; + char c[1024]{}; DataSpace dataspace = DataSpace(H5S_SCALAR); StrType strdatatype(PredType::C_S1, 256); DataSet dataset = @@ -476,8 +472,7 @@ void MasterAttributes::WriteHDF5BurstMode(H5File *fd, Group *group) { StrType strdatatype(PredType::C_S1, 256); DataSet dataset = group->createDataSet("Burst Mode", strdatatype, dataspace); - char c[1024]; - memset(c, 0, sizeof(c)); + char c[1024]{}; sls::strcpy_safe(c, sls::ToString(burstMode)); dataset.write(c, strdatatype); } diff --git a/slsReceiverSoftware/src/MasterFileUtility.cpp b/slsReceiverSoftware/src/MasterFileUtility.cpp index 56f017fa2..506ba7b83 100644 --- a/slsReceiverSoftware/src/MasterFileUtility.cpp +++ b/slsReceiverSoftware/src/MasterFileUtility.cpp @@ -7,38 +7,31 @@ namespace masterFileUtility { -std::string CreateMasterBinaryFile(const std::string filePath, - const std::string fileNamePrefix, +std::string CreateMasterBinaryFile(const std::string &filePath, + const std::string &fileNamePrefix, const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode, MasterAttributes *attr) { - // create file name std::ostringstream os; os << filePath << "/" << fileNamePrefix << "_master" << "_" << fileIndex << ".json"; std::string fileName = os.str(); - // create file - FILE *fd{nullptr}; - if (!overWriteEnable) { - if (nullptr == (fd = fopen((const char *)fileName.c_str(), "wx"))) { - fd = nullptr; - throw sls::RuntimeError("Could not create binary master file " + + std::string mode = "w"; + if (!overWriteEnable) + mode = "wx"; + FILE *fd = fopen(fileName.c_str(), mode.c_str()); + if(!fd) { + throw sls::RuntimeError("Could not create/overwrite binary master file " + fileName); - } - } else if (nullptr == (fd = fopen((const char *)fileName.c_str(), "w"))) { - fd = nullptr; - throw sls::RuntimeError( - "Could not create/overwrite binary master file " + fileName); } rapidjson::StringBuffer s; rapidjson::PrettyWriter writer(s); attr->GetBinaryAttributes(&writer); - std::string message = s.GetString(); - if (fwrite((void *)message.c_str(), 1, message.length(), fd) != - message.length()) { + if (fwrite(s.GetString(), 1, strlen(s.GetString()), fd) != + strlen(s.GetString())) { throw sls::RuntimeError( "Master binary file incorrect number of bytes written to file"); } diff --git a/slsReceiverSoftware/src/MasterFileUtility.h b/slsReceiverSoftware/src/MasterFileUtility.h index d0a74b04f..ed90cc52e 100644 --- a/slsReceiverSoftware/src/MasterFileUtility.h +++ b/slsReceiverSoftware/src/MasterFileUtility.h @@ -14,8 +14,8 @@ using namespace H5; namespace masterFileUtility { -std::string CreateMasterBinaryFile(const std::string filePath, - const std::string fileNamePrefix, +std::string CreateMasterBinaryFile(const std::string &filePath, + const std::string &fileNamePrefix, const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode, From 76296507ffc678dc5ab4277c6d15a27382a9407e Mon Sep 17 00:00:00 2001 From: Dhanya Thattil Date: Thu, 31 Mar 2022 15:09:34 +0200 Subject: [PATCH 12/12] updated release --- RELEASE.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE.txt b/RELEASE.txt index 7eff077d3..01bbd7781 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -57,6 +57,7 @@ This document describes the differences between v7.0.0 and v6.x.x - ctb: can set names for all the dacs - fpga/kernel programming, checks if drive is a special file and not a normal file - gotthard 25 um image reconstructed in gui and virtual hdf5 (firmware updated for slave to reverse channels) +- master binary file in json format now 2. Resolved Issues ==================