detectors/slsDetectorPackage
25
0
Fork 0
mirror of https://github.com/slsdetectorgroup/slsDetectorPackage.git synced 2025-04-24 07:20:01 +02:00
Code Issues Actions Packages Releases Activity

binary master json done, hdf5 left wip

Browse Source
This commit is contained in:
maliakal_d 2022-03-25 13:29:03 +01:00
parent 90d1d0f8b8
commit 0f02ffdc9a
10 changed files with 153 additions and 240 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
slsReceiverSoftware/src/BinaryMasterFile.cpp
View File

@ -3,17 +3,6 @@
#include "BinaryMasterFile.h"
#include "MasterAttributes.h"
BinaryMasterFile::BinaryMasterFile() : File(BINARY) {}
BinaryMasterFile::~BinaryMasterFile() { CloseFile(); }
void BinaryMasterFile::CloseFile() {
if (fd_) {
fclose(fd_);
}
fd_ = nullptr;
}
void BinaryMasterFile::CreateMasterFile(const std::string filePath,
const std::string fileNamePrefix,
const uint64_t fileIndex,
@ -24,37 +13,26 @@ void BinaryMasterFile::CreateMasterFile(const std::string filePath,
std::ostringstream os;
os << filePath << "/" << fileNamePrefix << "_master"
<< "_" << fileIndex << ".json";
fileName_ = os.str();
std::string fileName = os.str();
// create file
FILE *fd{nullptr};
if (!overWriteEnable) {
if (nullptr == (fd_ = fopen((const char *)fileName_.c_str(), "wx"))) {
fd_ = nullptr;
if (nullptr == (fd = fopen((const char *)fileName.c_str(), "wx"))) {
fd = nullptr;
throw sls::RuntimeError("Could not create binary master file " +
fileName_);
fileName);
}
} else if (nullptr == (fd_ = fopen((const char *)fileName_.c_str(), "w"))) {
fd_ = nullptr;
} else if (nullptr == (fd = fopen((const char *)fileName.c_str(), "w"))) {
fd = nullptr;
throw sls::RuntimeError(
"Could not create/overwrite binary master file " + fileName_);
"Could not create/overwrite binary master file " + fileName);
}
if (!silentMode) {
LOG(logINFO) << "Master File: " << fileName_;
LOG(logINFO) << "Master File: " << fileName;
}
attr->WriteMasterBinaryAttributes(fd_);
CloseFile();
}
void BinaryMasterFile::UpdateMasterFile(MasterAttributes *attr,
bool silentMode) {
if (nullptr == (fd_ = fopen((const char *)fileName_.c_str(), "a"))) {
fd_ = nullptr;
throw sls::RuntimeError("Could not append binary master file " +
fileName_);
}
attr->WriteFinalBinaryAttributes(fd_);
CloseFile();
if (!silentMode) {
LOG(logINFO) << "Updated Master File";
attr->WriteMasterBinaryAttributes(fd);
if (fd) {
fclose(fd);
}
}

21
slsReceiverSoftware/src/BinaryMasterFile.h
View File

@ -5,21 +5,12 @@
#include "File.h"
#include "MasterAttributes.h"
class BinaryMasterFile : private virtual slsDetectorDefs, public File {
class BinaryMasterFile : private virtual slsDetectorDefs {
public:
BinaryMasterFile();
~BinaryMasterFile();
void CloseFile() override;
void CreateMasterFile(const std::string filePath,
const std::string fileNamePrefix,
const uint64_t fileIndex, const bool overWriteEnable,
const bool silentMode,
MasterAttributes *attr) override;
void UpdateMasterFile(MasterAttributes *attr, bool silentMode) override;
private:
FILE *fd_{nullptr};
std::string fileName_;
static void CreateMasterFile(const std::string filePath,
const std::string fileNamePrefix,
const uint64_t fileIndex,
const bool overWriteEnable,
const bool silentMode, MasterAttributes *attr);
};

91
slsReceiverSoftware/src/DataProcessor.cpp
View File

@ -82,8 +82,6 @@ void DataProcessor::SetGeneralData(GeneralData *generalData) {
void DataProcessor::CloseFiles() {
if (dataFile_)
dataFile_->CloseFile();
if (masterFile_)
masterFile_->CloseFile();
#ifdef HDF5C
if (virtualFile_)
virtualFile_->CloseFile();
@ -96,10 +94,10 @@ void DataProcessor::DeleteFiles() {
delete dataFile_;
dataFile_ = nullptr;
}
if (masterFile_) {
delete masterFile_;
masterFile_ = nullptr;
}
/* if (masterFile_) {
delete masterFile_;
masterFile_ = nullptr;
}*/
#ifdef HDF5C
if (virtualFile_) {
delete virtualFile_;
@ -108,27 +106,18 @@ void DataProcessor::DeleteFiles() {
#endif
}
void DataProcessor::SetupFileWriter(const bool filewriteEnable,
const bool masterFilewriteEnable,
const fileFormat fileFormatType,
const int modulePos, std::mutex *hdf5Lib) {
std::mutex *hdf5Lib) {
DeleteFiles();
if (filewriteEnable) {
switch (fileFormatType) {
#ifdef HDF5C
case HDF5:
dataFile_ = new HDF5DataFile(index, hdf5Lib);
if (modulePos == 0 && index == 0) {
if (masterFilewriteEnable) {
masterFile_ = new HDF5MasterFile(hdf5Lib);
}
}
break;
#endif
case BINARY:
dataFile_ = new BinaryDataFile(index);
if (modulePos == 0 && index == 0 && masterFilewriteEnable) {
masterFile_ = new BinaryMasterFile();
}
break;
default:
throw sls::RuntimeError(
@ -138,23 +127,17 @@ void DataProcessor::SetupFileWriter(const bool filewriteEnable,
}
void DataProcessor::CreateFirstFiles(
MasterAttributes *attr, const std::string filePath,
const std::string fileNamePrefix, const uint64_t fileIndex,
const bool overWriteEnable, const bool silentMode, const int modulePos,
const int numUnitsPerReadout, const uint32_t udpPortNumber,
const uint32_t maxFramesPerFile, const uint64_t numImages,
const uint32_t dynamicRange, const bool detectorDataStream) {
const std::string filePath, const std::string fileNamePrefix,
const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode,
const int modulePos, const int numUnitsPerReadout,
const uint32_t udpPortNumber, const uint32_t maxFramesPerFile,
const uint64_t numImages, const uint32_t dynamicRange,
const bool detectorDataStream) {
if (dataFile_ == nullptr) {
throw sls::RuntimeError("file object not contstructed");
}
CloseFiles();
// master file write enabled
if (masterFile_) {
masterFile_->CreateMasterFile(filePath, fileNamePrefix, fileIndex,
overWriteEnable, silentMode, attr);
}
// deactivated (half module/ single port), dont write file
if ((!*activated_) || (!detectorDataStream)) {
return;
@ -238,38 +221,26 @@ void DataProcessor::LinkDataInMasterFile(const bool silentMode) {
}
#endif
void DataProcessor::UpdateMasterFile(bool silentMode) {
if (masterFile_) {
// final attributes
std::unique_ptr<MasterAttributes> masterAttributes;
switch (detectorType_) {
case GOTTHARD:
masterAttributes = sls::make_unique<GotthardMasterAttributes>();
break;
case JUNGFRAU:
masterAttributes = sls::make_unique<JungfrauMasterAttributes>();
break;
case EIGER:
masterAttributes = sls::make_unique<EigerMasterAttributes>();
break;
case MYTHEN3:
masterAttributes = sls::make_unique<Mythen3MasterAttributes>();
break;
case GOTTHARD2:
masterAttributes = sls::make_unique<Gotthard2MasterAttributes>();
break;
case MOENCH:
masterAttributes = sls::make_unique<MoenchMasterAttributes>();
break;
case CHIPTESTBOARD:
masterAttributes = sls::make_unique<CtbMasterAttributes>();
break;
default:
throw sls::RuntimeError(
"Unknown detector type to set up master file attributes");
}
masterAttributes->framesInFile = numFramesCaught_;
masterFile_->UpdateMasterFile(masterAttributes.get(), silentMode);
void DataProcessor::CreateMasterFile(
const std::string filePath, const std::string fileNamePrefix,
const uint64_t fileIndex, const bool overWriteEnable, bool silentMode,
const fileFormat fileFormatType, MasterAttributes *attr) {
attr->framesInFile = numFramesCaught_;
std::unique_ptr<File> masterFile{nullptr};
switch (fileFormatType) {
#ifdef HDF5C
case HDF5:
masterFile = sls::make_unique<HDF5MasterFile>(hdf5Lib);
break;
#endif
case BINARY:
BinaryMasterFile::CreateMasterFile(filePath, fileNamePrefix, fileIndex,
overWriteEnable, silentMode, attr);
break;
default:
throw sls::RuntimeError("Unknown file format (compile with hdf5 flags");
}
}

13
slsReceiverSoftware/src/DataProcessor.h
View File

@ -45,11 +45,9 @@ class DataProcessor : private virtual slsDetectorDefs, public ThreadObject {
void CloseFiles();
void DeleteFiles();
void SetupFileWriter(const bool filewriteEnable,
const bool masterFilewriteEnable,
const fileFormat fileFormatType, const int modulePos,
std::mutex *hdf5Lib);
const fileFormat fileFormatType, std::mutex *hdf5Lib);
void CreateFirstFiles(MasterAttributes *attr, const std::string filePath,
void CreateFirstFiles(const std::string filePath,
const std::string fileNamePrefix,
const uint64_t fileIndex, const bool overWriteEnable,
const bool silentMode, const int modulePos,
@ -71,7 +69,11 @@ class DataProcessor : private virtual slsDetectorDefs, public ThreadObject {
const int numModY, std::mutex *hdf5Lib);
void LinkDataInMasterFile(const bool silentMode);
#endif
void UpdateMasterFile(bool silentMode);
void CreateMasterFile(const std::string filePath,
const std::string fileNamePrefix,
const uint64_t fileIndex, const bool overWriteEnable,
bool silentMode, const fileFormat fileFormatType,
MasterAttributes *attr);
/**
* Call back for raw data
* args to raw data ready callback are
@ -179,7 +181,6 @@ class DataProcessor : private virtual slsDetectorDefs, public ThreadObject {
bool firstStreamerFrame_{false};
File *dataFile_{nullptr};
File *masterFile_{nullptr};
#ifdef HDF5C
File *virtualFile_{nullptr};
#endif

15
slsReceiverSoftware/src/File.h
View File

@ -103,16 +103,6 @@ class File : private virtual slsDetectorDefs {
"should be overloaded by a derived class";
};
virtual void CreateMasterFile(const std::string filePath,
const std::string fileNamePrefix,
const uint64_t fileIndex,
const bool overWriteEnable,
const bool silentMode,
MasterAttributes *attr) {
LOG(logERROR) << "This is a generic function CreateMasterFile that "
"should be overloaded by a derived class";
};
virtual void WriteToFile(char *buffer, const int buffersize,
const uint64_t currentFrameNumber,
const uint32_t numPacketsCaught) {
@ -120,11 +110,6 @@ class File : private virtual slsDetectorDefs {
"should be overloaded by a derived class";
};
virtual void UpdateMasterFile(MasterAttributes *attr, bool silentMode) {
LOG(logERROR) << "This is a generic function UpdateMasterFile that "
"should be overloaded by a derived class";
};
protected:
slsDetectorDefs::fileFormat format_;
};

2
slsReceiverSoftware/src/HDF5MasterFile.h
View File

@ -12,7 +12,6 @@ class HDF5MasterFile : private virtual slsDetectorDefs, public File {
HDF5MasterFile(std::mutex *hdf5Lib);
~HDF5MasterFile();
void CloseFile() override;
void LinkDataFile(std::string dataFilename, std::string dataSetname,
const std::vector<std::string> parameterNames,
const bool silentMode) override;
@ -21,7 +20,6 @@ class HDF5MasterFile : private virtual slsDetectorDefs, public File {
const uint64_t fileIndex, const bool overWriteEnable,
const bool silentMode,
MasterAttributes *attr) override;
void UpdateMasterFile(MasterAttributes *attr, bool silentMode) override;
private:
std::mutex *hdf5Lib_;

93
slsReceiverSoftware/src/Implementation.cpp
View File

@ -240,9 +240,6 @@ void Implementation::setModulePositionId(const int id) {
xy portGeometry = GetPortGeometry();
streamingPort = DEFAULT_ZMQ_RX_PORTNO + modulePos * portGeometry.x;
for (const auto &it : dataProcessor)
it->SetupFileWriter(fileWriteEnable, masterFileWriteEnable,
fileFormatType, modulePos, &hdf5Lib);
assert(numModules.y != 0);
for (unsigned int i = 0; i < listener.size(); ++i) {
uint16_t row = 0, col = 0;
@ -372,8 +369,7 @@ void Implementation::setFileFormat(const fileFormat f) {
throw sls::RuntimeError("Unknown file format");
}
for (const auto &it : dataProcessor)
it->SetupFileWriter(fileWriteEnable, masterFileWriteEnable,
fileFormatType, modulePos, &hdf5Lib);
it->SetupFileWriter(fileWriteEnable, fileFormatType, &hdf5Lib);
}
LOG(logINFO) << "File Format: " << sls::ToString(fileFormatType);
@ -409,8 +405,7 @@ void Implementation::setFileWriteEnable(const bool b) {
if (fileWriteEnable != b) {
fileWriteEnable = b;
for (const auto &it : dataProcessor)
it->SetupFileWriter(fileWriteEnable, masterFileWriteEnable,
fileFormatType, modulePos, &hdf5Lib);
it->SetupFileWriter(fileWriteEnable, fileFormatType, &hdf5Lib);
}
LOG(logINFO) << "File Write Enable: "
<< (fileWriteEnable ? "enabled" : "disabled");
@ -423,9 +418,6 @@ bool Implementation::getMasterFileWriteEnable() const {
void Implementation::setMasterFileWriteEnable(const bool b) {
if (masterFileWriteEnable != b) {
masterFileWriteEnable = b;
for (const auto &it : dataProcessor)
it->SetupFileWriter(fileWriteEnable, masterFileWriteEnable,
fileFormatType, modulePos, &hdf5Lib);
}
LOG(logINFO) << "Master File Write Enable: "
<< (masterFileWriteEnable ? "enabled" : "disabled");
@ -584,29 +576,9 @@ void Implementation::stopReceiver() {
std::this_thread::sleep_for(std::chrono::milliseconds(5));
}
#ifdef HDF5C
if (fileWriteEnable && fileFormatType == HDF5) {
if (modulePos == 0) {
// more than 1 file, create virtual file
if (dataProcessor[0]->GetFilesInAcquisition() > 1 ||
(numModules.x * numModules.y) > 1) {
dataProcessor[0]->CreateVirtualFile(
filePath, fileName, fileIndex, overwriteEnable, silentMode,
modulePos, numUDPInterfaces, framesPerFile,
numberOfTotalFrames, dynamicRange, numModules.x,
numModules.y, &hdf5Lib);
}
// link file in master
dataProcessor[0]->LinkDataInMasterFile(silentMode);
}
}
#endif
if (fileWriteEnable && masterFileWriteEnable && modulePos == 0) {
try {
dataProcessor[0]->UpdateMasterFile(silentMode);
} catch (...) {
; // ignore it and just print it
}
if (fileWriteEnable && modulePos == 0) {
// master and virtual file (hdf5)
StartMasterWriter();
}
// wait for the processes (dataStreamer) to be done
@ -753,9 +725,25 @@ void Implementation::CreateUDPSockets() {
}
void Implementation::SetupWriter() {
try {
for (unsigned int i = 0; i < dataProcessor.size(); ++i) {
dataProcessor[i]->CreateFirstFiles(
filePath, fileName, fileIndex, overwriteEnable, silentMode,
modulePos, numUDPInterfaces, udpPortNum[i], framesPerFile,
numberOfTotalFrames, dynamicRange, detectorDataStream[i]);
}
} catch (const sls::RuntimeError &e) {
shutDownUDPSockets();
for (const auto &it : dataProcessor)
it->CloseFiles();
throw sls::RuntimeError("Could not create first data file.");
}
}
void Implementation::StartMasterWriter() {
// master file
std::unique_ptr<MasterAttributes> masterAttributes;
if (masterFileWriteEnable && modulePos == 0) {
if (masterFileWriteEnable) {
std::unique_ptr<MasterAttributes> masterAttributes;
switch (detType) {
case GOTTHARD:
masterAttributes = sls::make_unique<GotthardMasterAttributes>();
@ -838,22 +826,31 @@ void Implementation::SetupWriter() {
masterAttributes->gateDelay3 = gateDelay3;
masterAttributes->gates = numberOfGates;
masterAttributes->additionalJsonHeader = additionalJsonHeader;
}
try {
for (unsigned int i = 0; i < dataProcessor.size(); ++i) {
dataProcessor[i]->CreateFirstFiles(
masterAttributes.get(), filePath, fileName, fileIndex,
overwriteEnable, silentMode, modulePos, numUDPInterfaces,
udpPortNum[i], framesPerFile, numberOfTotalFrames, dynamicRange,
detectorDataStream[i]);
try {
dataProcessor[0]->CreateMasterFile(
filePath, fileName, fileIndex, overwriteEnable, silentMode,
fileFormatType, masterAttributes.get());
} catch (...) {
; // ignore it and just print it
}
} catch (const sls::RuntimeError &e) {
shutDownUDPSockets();
for (const auto &it : dataProcessor)
it->CloseFiles();
throw sls::RuntimeError("Could not create first data file.");
}
#ifdef HDF5C
if (fileFormatType == HDF5) {
// virtual hdf5 file (if multiple files)
if (dataProcessor[0]->GetFilesInAcquisition() > 1 ||
(numModules.x * numModules.y) > 1) {
dataProcessor[0]->CreateVirtualFile(
filePath, fileName, fileIndex, overwriteEnable, silentMode,
modulePos, numUDPInterfaces, framesPerFile, numberOfTotalFrames,
dynamicRange, numModules.x, numModules.y, &hdf5Lib);
}
// link file in master
if (masterFileWriteEnable) {
dataProcessor[0]->LinkDataInMasterFile(silentMode);
}
}
#endif
}
void Implementation::StartRunning() {

1
slsReceiverSoftware/src/Implementation.h
View File

@ -274,6 +274,7 @@ class Implementation : private virtual slsDetectorDefs {
void ResetParametersforNewAcquisition();
void CreateUDPSockets();
void SetupWriter();
void StartMasterWriter();
void StartRunning();
/**************************************************

101
slsReceiverSoftware/src/MasterAttributes.cpp
View File

@ -4,13 +4,13 @@
#include <rapidjson/stringbuffer.h>
void MasterAttributes::WriteMasterBinaryAttributes(FILE *fd) {
LOG(logERROR) << "WriteMasterBinaryAttributes should have been called "
"by a child class";
}
void MasterAttributes::GetBinaryMasterAttributes(rapidjson::Writer<rapidjson::StringBuffer>* w) {
void MasterAttributes::GetBinaryMasterAttributes(
rapidjson::Writer<rapidjson::StringBuffer> *w) {
time_t t = time(nullptr);
w->Key("Version");
@ -38,10 +38,12 @@ void MasterAttributes::GetBinaryMasterAttributes(rapidjson::Writer<rapidjson::St
w->Uint(imageSize);
w->Key("Pixels");
w->StartArray();
w->StartObject();
w->Key("x");
w->Uint(nPixels.x);
w->Key("y");
w->Uint(nPixels.y);
w->EndArray();
w->EndObject();
w->Key("Max Frames Per File");
w->Uint(maxFramesPerFile);
@ -67,64 +69,49 @@ void MasterAttributes::WriteBinaryAttributes(FILE *fd, std::string message) {
}
};
void MasterAttributes::WriteFinalBinaryAttributes(FILE *fd) {
/*
FILE* fp = fopen(json_file_name.c_str(), "r");
char readBuffer[65536];
FileReadStream is(fp, readBuffer, sizeof(readBuffer));
Document d, d2;
d.ParseStream(is);
assert(d.IsArray());
fclose(fp);
d2.SetObject();
Value json_objects(kObjectType);
json_objects.AddMember("three", 3, d2.GetAllocator());
d.PushBack(json_objects, d2.GetAllocator());
FILE* outfile = fopen(json_file_name.c_str(), "w");
char writeBuffer[65536];
FileWriteStream os(outfile, writeBuffer, sizeof(writeBuffer));
Writer writer(os);
d.Accept (writer);
fclose(outfile);
*/
void MasterAttributes::WriteFinalBinaryAttributes(
rapidjson::Writer<rapidjson::StringBuffer> *w) {
// adding few common parameters to the end
/*std::ostringstream oss;
if (!additionalJsonHeader.empty()) {
oss << "Additional Json Header : "
<< sls::ToString(additionalJsonHeader) << '\n';
w->Key("Additional Json Header");
w->String(sls::ToString(additionalJsonHeader).c_str());
}
oss << "Frames in File : " << framesInFile << '\n';
// adding sls_receiver header format
oss << '\n'
<< "#Frame Header" << '\n'
<< "Frame Number : 8 bytes" << '\n'
<< "SubFrame Number/ExpLength : 4 bytes" << '\n'
<< "Packet Number : 4 bytes" << '\n'
<< "Bunch ID : 8 bytes" << '\n'
<< "Timestamp : 8 bytes" << '\n'
<< "Module Id : 2 bytes" << '\n'
<< "Row : 2 bytes" << '\n'
<< "Column : 2 bytes" << '\n'
<< "Reserved : 2 bytes" << '\n'
<< "Debug : 4 bytes" << '\n'
<< "Round Robin Number : 2 bytes" << '\n'
<< "Detector Type : 1 byte" << '\n'
<< "Header Version : 1 byte" << '\n'
<< "Packets Caught Mask : 64 bytes" << '\n';
w->Key("Frames in File");
w->Uint64(framesInFile);
std::string message = oss.str();
// writing to file
if (fwrite((void *)message.c_str(), 1, message.length(), fd) !=
message.length()) {
throw sls::RuntimeError(
"Master binary file incorrect number of bytes written to file");
}*/
w->Key("Frame Header Format");
w->StartObject();
w->Key("Frame Number");
w->String("8 bytes");
w->Key("SubFrame Number/ExpLength");
w->String("4 bytes");
w->Key("Packet Number");
w->String("4 bytes");
w->Key("Bunch ID");
w->String("8 bytes");
w->Key("Timestamp");
w->String("8 bytes");
w->Key("Module Id");
w->String("2 bytes");
w->Key("Row");
w->String("2 bytes");
w->Key("Column");
w->String("2 bytes");
w->Key("Reserved");
w->String("2 bytes");
w->Key("Debug");
w->String("4 bytes");
w->Key("Round Robin Number");
w->String("2 bytes");
w->Key("Detector Type");
w->String("1 byte");
w->Key("Header Version");
w->String("1 byte");
w->Key("Packets Caught Mask");
w->String("64 bytes");
w->EndObject();
};
#ifdef HDF5C
@ -373,6 +360,8 @@ void MasterAttributes::WriteHDF5Attributes(H5File *fd, Group *group) {
writer.Key("Number of rows");
writer.Uint(readNRows);
MasterAttributes::WriteFinalBinaryAttributes(&writer);
writer.EndObject();
std::string message = s.GetString();

6
slsReceiverSoftware/src/MasterAttributes.h
View File

@ -70,9 +70,11 @@ class MasterAttributes {
MasterAttributes() = default;
virtual ~MasterAttributes() = default;
virtual void WriteMasterBinaryAttributes(FILE *fd);
void GetBinaryMasterAttributes(rapidjson::Writer<rapidjson::StringBuffer>* w);
void
GetBinaryMasterAttributes(rapidjson::Writer<rapidjson::StringBuffer> *w);
void WriteBinaryAttributes(FILE *fd, std::string message);
void WriteFinalBinaryAttributes(FILE *fd);
void
WriteFinalBinaryAttributes(rapidjson::Writer<rapidjson::StringBuffer> *w);
#ifdef HDF5C
virtual void WriteMasterHDF5Attributes(H5File *fd, Group *group);
void WriteHDF5Attributes(H5File *fd, Group *group);