detectors/slsDetectorPackage
25
0
Fork 0
mirror of https://github.com/slsdetectorgroup/slsDetectorPackage.git synced 2025-04-23 06:50:02 +02:00
Code Issues Actions Packages Releases Activity

wip

Browse Source
This commit is contained in:
maliakal_d 2021-06-30 13:03:31 +02:00
parent d7dc1912ac
commit fca9982b72
14 changed files with 314 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
slsReceiverSoftware/src/BinaryDataFile.cpp
View File

@ -1,7 +1,7 @@
#include "BinaryDataFile.h"
#include "sls/logger.h"
BinaryDataFile::BinaryDataFile(int index) : File(index, BINARY) {}
BinaryDataFile::BinaryDataFile(int index) : File(BINARY), index_(index) {}
BinaryDataFile::~BinaryDataFile() { CloseFile(); }
@ -11,3 +11,36 @@ void BinaryDataFile::CloseFile() {
}
fd_ = nullptr;
}
void BinaryDataFile::CreateDataFile(std::string filePath,
std::string fileNamePrefix,
uint64_t fileIndex, bool overWriteEnable,
bool silentMode, int detIndex,
int numUnitsPerDetector,
uint32_t udpPortNumber) {
numFramesInFile_ = 0;
std::ostringstream os;
os << filePath << "/" << fileNamePrefix << "_d"
<< (detIndex * numUnitsPerDetector + index_) << "_f" << subFileIndex_
<< '_' << fileIndex << ".raw";
fileName_ = os.str();
if (!overWriteEnable) {
if (nullptr == (fd_ = fopen((const char *)fileName_.c_str(), "wx"))) {
fd_ = nullptr;
throw sls::RuntimeError("Could not create/overwrite file " +
fileName_);
}
} else if (nullptr == (fd_ = fopen((const char *)fileName_.c_str(), "w"))) {
fd_ = nullptr;
throw sls::RuntimeError("Could not create file " + fileName_);
}
// setting to no file buffering
setvbuf(fd_, nullptr, _IONBF, 0);
if (!(silentMode)) {
LOG(logINFO) << "[" << udpPortNumber
<< "]: Binary File created: " << fileName_;
}
}

8
slsReceiverSoftware/src/BinaryDataFile.h
View File

@ -9,7 +9,15 @@ class BinaryDataFile : private virtual slsDetectorDefs, public File {
~BinaryDataFile();
void CloseFile() override;
void CreateDataFile(std::string filePath, std::string fileNamePrefix,
uint64_t fileIndex, bool overWriteEnable,
bool silentMode, int detIndex, int numUnitsPerDetector,
uint32_t udpPortNumber) override;
private:
uint32_t index_;
FILE *fd_{nullptr};
std::string fileName_;
uint32_t numFramesInFile_{0};
uint32_t subFileIndex_{0}; // to do reset in createfirstdatafile
};

13
slsReceiverSoftware/src/BinaryMasterFile.cpp
View File

@ -2,7 +2,7 @@
#include "MasterAttributes.h"
#include "sls/logger.h"
BinaryMasterFile::BinaryMasterFile(int index) : File(index, BINARY) {}
BinaryMasterFile::BinaryMasterFile() : File(BINARY) {}
BinaryMasterFile::~BinaryMasterFile() { CloseFile(); }
@ -13,11 +13,11 @@ void BinaryMasterFile::CloseFile() {
fd_ = nullptr;
}
void BinaryMasterFile::CreateMasterFile(MasterAttributes *attr,
std::string filePath,
void BinaryMasterFile::CreateMasterFile(std::string filePath,
std::string fileNamePrefix,
uint64_t fileIndex,
bool overWriteEnable, bool silentMode) {
bool overWriteEnable, bool silentMode,
MasterAttributes *attr) {
// create file name
std::ostringstream os;
os << filePath << "/" << fileNamePrefix << "_master"
@ -31,14 +31,13 @@ void BinaryMasterFile::CreateMasterFile(MasterAttributes *attr,
if (!(overWriteEnable)) {
if (nullptr == (fd_ = fopen((const char *)fileName_.c_str(), "wx"))) {
fd_ = nullptr;
throw sls::RuntimeError(
"Could not create binary master file (no overwrite) " +
throw sls::RuntimeError("Could not create binary master file " +
fileName_);
}
} else if (nullptr == (fd_ = fopen((const char *)fileName_.c_str(), "w"))) {
fd_ = nullptr;
throw sls::RuntimeError(
"Could not create binary master file (overwrite) " + fileName_);
"Could not create/overwrite binary master file " + fileName_);
}
attr->WriteMasterBinaryAttributes(fd_);

8
slsReceiverSoftware/src/BinaryMasterFile.h
View File

@ -6,13 +6,13 @@
class BinaryMasterFile : private virtual slsDetectorDefs, public File {
public:
BinaryMasterFile(int index);
BinaryMasterFile();
~BinaryMasterFile();
void CloseFile() override;
void CreateMasterFile(MasterAttributes *attr, std::string filePath,
std::string fileNamePrefix, uint64_t fileIndex,
bool overWriteEnable, bool silentMode) override;
void CreateMasterFile(std::string filePath, std::string fileNamePrefix,
uint64_t fileIndex, bool overWriteEnable,
bool silentMode, MasterAttributes *attr) override;
private:
FILE *fd_{nullptr};

3
slsReceiverSoftware/src/File.cpp
View File

@ -2,8 +2,7 @@
#include <iostream>
File::File(int index, slsDetectorDefs::fileFormat type)
: index_(index), type_(type) {}
File::File(slsDetectorDefs::fileFormat type) : type_(type) {}
File::~File() {}

24
slsReceiverSoftware/src/File.h
View File

@ -7,18 +7,32 @@ struct MasterAttributes;
class File : private virtual slsDetectorDefs {
public:
File(int index, slsDetectorDefs::fileFormat type);
File(slsDetectorDefs::fileFormat type);
virtual ~File();
fileFormat GetFileType();
virtual void CloseFile() = 0;
virtual void CreateMasterFile(MasterAttributes *attr, std::string filePath,
virtual void CreateMasterFile(std::string filePath,
std::string fileNamePrefix,
uint64_t fileIndex, bool overWriteEnable,
bool silentMode) = 0;
bool silentMode, MasterAttributes *attr) = 0;
// hdf5 only
virtual void
CreateFirstDataFile(std::string filePath, std::string fileNamePrefix,
uint64_t fileIndex, bool overWriteEnable,
bool silentMode, int detIndex, int numUnitsPerDetector,
uint32_t udpPortNumber, uint32_t maxFramesPerFile,
uint64_t numImages, uint32_t nPIxelsX,
uint32_t nPIxelsY, uint32_t dynamicRange) = 0;
virtual void CreateDataFile(std::string filePath,
std::string fileNamePrefix, uint64_t fileIndex,
bool overWriteEnable, bool silentMode,
int detIndex, int numUnitsPerDetector,
uint32_t udpPortNumber) = 0;
protected:
int index_;
slsDetectorDefs::fileFormat type_;
};

167
slsReceiverSoftware/src/HDF5DataFile.cpp
View File

@ -1,11 +1,41 @@
#include "HDF5DataFile.h"
#include "receiver_defs.h"
#include "sls/logger.h"
HDF5DataFile::HDF5DataFile(int index) : File(index, HDF5) {}
#include <iomanip>
HDF5DataFile::HDF5DataFile(int index, std::mutex *hdf5Lib)
: File(HDF5), index_(index), hdf5Lib_(hdf5Lib) {
parameterNames_ = std::vector<std::string>{
"frame number",
"exp length or sub exposure time",
"packets caught",
"bunch id",
"timestamp",
"mod id",
"row",
"column",
"reserved",
"debug",
"round robin number",
"detector type",
"detector header version",
"packets caught bit mask",
};
StrType strdatatype(PredType::C_S1, sizeof(bitset_storage));
parameterDataTypes_ = std::vector<DataType>{
PredType::STD_U64LE, PredType::STD_U32LE, PredType::STD_U32LE,
PredType::STD_U64LE, PredType::STD_U64LE, PredType::STD_U16LE,
PredType::STD_U16LE, PredType::STD_U16LE, PredType::STD_U16LE,
PredType::STD_U32LE, PredType::STD_U16LE, PredType::STD_U8LE,
PredType::STD_U8LE, strdatatype};
}
HDF5DataFile::~HDF5DataFile() { CloseFile(); }
void HDF5DataFile::CloseFile() {
std::lock_guard<std::mutex> lock(*hdf5Lib_);
try {
Exception::dontPrint(); // to handle errors
if (fd_) {
@ -18,4 +48,139 @@ void HDF5DataFile::CloseFile() {
<< index_;
error.printErrorStack();
}
if (dataSpace_) {
delete dataSpace_;
dataSpace_ = nullptr;
}
if (dataSet_) {
delete dataSet_;
dataSet_ = nullptr;
}
if (dataSpacePara_) {
delete dataSpacePara_;
dataSpacePara_ = nullptr;
}
for (auto it : dataSetPara_)
delete it;
dataSetPara_.clear();
}
void HDF5DataFile::CreateFirstDataFile(
std::string filePath, std::string fileNamePrefix, uint64_t fileIndex,
bool overWriteEnable, bool silentMode, int detIndex,
int numUnitsPerDetector, uint32_t udpPortNumber, uint32_t maxFramesPerFile,
uint64_t numImages, uint32_t nPIxelsX, uint32_t nPIxelsY,
uint32_t dynamicRange) {
CreateDataFile(filePath, fileNamePrefix, fileIndex, overWriteEnable,
silentMode, detIndex, numUnitsPerDetector, udpPortNumber);
maxFramesPerFile_ = maxFramesPerFile;
numImages_ = numImages;
nPixelsX_ = nPIxelsX;
nPixelsY_ = nPIxelsY;
dynamicRange_ = dynamicRange;
}
void HDF5DataFile::CreateDataFile(std::string filePath,
std::string fileNamePrefix,
uint64_t fileIndex, bool overWriteEnable,
bool silentMode, int detIndex,
int numUnitsPerDetector,
uint32_t udpPortNumber) {
std::ostringstream os;
os << filePath << "/" << fileNamePrefix << "_d"
<< (detIndex * numUnitsPerDetector + index_) << "_f" << subFileIndex_
<< '_' << fileIndex << ".h5";
fileName_ = os.str();
std::lock_guard<std::mutex> lock(*hdf5Lib_);
uint64_t framestosave =
((maxFramesPerFile_ == 0) ? numImages_ : // infinite images
(((extNumImages_ - subFileIndex_) > maxFramesPerFile_)
? // save up to maximum at a time
maxFramesPerFile_
: (extNumImages_ - subFileIndex_)));
uint64_t nDimx = framestosave;
uint32_t nDimy = nPixelsY_;
uint32_t nDimz = ((dynamicRange_ == 4) ? (nPixelsX_ / 2) : nPixelsX_);
try {
Exception::dontPrint(); // to handle errors
// file
FileAccPropList fapl;
fapl.setFcloseDegree(H5F_CLOSE_STRONG);
fd_ = nullptr;
if (!overWriteEnable)
fd_ = new H5File(fileName_.c_str(), H5F_ACC_EXCL,
FileCreatPropList::DEFAULT, fapl);
else
fd_ = new H5File(fileName_.c_str(), H5F_ACC_TRUNC,
FileCreatPropList::DEFAULT, fapl);
// attributes - version
double dValue = HDF5_WRITER_VERSION;
DataSpace dataspace_attr = DataSpace(H5S_SCALAR);
Attribute attribute = fd_->createAttribute(
"version", PredType::NATIVE_DOUBLE, dataspace_attr);
attribute.write(PredType::NATIVE_DOUBLE, &dValue);
// dataspace
hsize_t srcdims[3] = {nDimx, nDimy, nDimz};
hsize_t srcdimsmax[3] = {H5S_UNLIMITED, nDimy, nDimz};
dataSpace_ = nullptr;
dataSpace_ = new DataSpace(3, srcdims, srcdimsmax);
// dataset name
std::ostringstream osfn;
osfn << "/data";
if (numImages_ > 1)
osfn << "_f" << std::setfill('0') << std::setw(12) << subFileIndex_;
std::string dsetname = osfn.str();
// dataset
// fill value
DSetCreatPropList plist;
int fill_value = -1;
plist.setFillValue(dataType, &fill_value);
// always create chunked dataset as unlimited is only
// supported with chunked layout
hsize_t chunk_dims[3] = {MAX_CHUNKED_IMAGES, nDimy, nDimz};
plist.setChunk(3, chunk_dims);
dataSet_ = nullptr;
dataSet_ = new DataSet(
fd_->createDataSet(dsetname.c_str(), dataType, *dataSpace_, plist));
// create parameter datasets
hsize_t dims[1] = {nDimx};
hsize_t dimsmax[1] = {H5S_UNLIMITED};
dataSpacePara_ = nullptr;
dataSpacePara_ = new DataSpace(1, dims, dimsmax);
// always create chunked dataset as unlimited is only
// supported with chunked layout
DSetCreatPropList paralist;
hsize_t chunkpara_dims[3] = {MAX_CHUNKED_IMAGES};
paralist.setChunk(1, chunkpara_dims);
for (unsigned int i = 0; i < parameterNames_.size(); ++i) {
DataSet *ds = new DataSet(fd_->createDataSet(
parameterNames_[i].c_str(), parameterDataTypes_[i],
*dataSpacePara_, paralist));
dataSetPara_.push_back(ds);
}
} catch (const Exception &error) {
error.printErrorStack();
CloseFile();
throw sls::RuntimeError("Could not create HDF5 handles in object " +
index_);
}
if (!(silentMode)) {
LOG(logINFO) << "[" << udpPortNumber
<< "]: HDF5 File created: " << fileName_;
}
}

36
slsReceiverSoftware/src/HDF5DataFile.h
View File

@ -7,14 +7,48 @@
using namespace H5;
#endif
#include <mutex>
class HDF5DataFile : private virtual slsDetectorDefs, public File {
public:
HDF5DataFile(int index);
HDF5DataFile(int index, std::mutex *hdf5Lib);
~HDF5DataFile();
void CloseFile() override;
void CreateFirstDataFile(std::string filePath, std::string fileNamePrefix,
uint64_t fileIndex, bool overWriteEnable,
bool silentMode, int detIndex,
int numUnitsPerDetector, uint32_t udpPortNumber,
uint32_t maxFramesPerFile, uint64_t numImages,
uint32_t nPIxelsX, uint32_t nPIxelsY,
uint32_t dynamicRange) override;
void CreateDataFile(std::string filePath, std::string fileNamePrefix,
uint64_t fileIndex, bool overWriteEnable,
bool silentMode, int detIndex, int numUnitsPerDetector,
uint32_t udpPortNumber) override;
private:
int index_;
std::mutex *hdf5Lib_;
H5File *fd_{nullptr};
std::string fileName_;
DataSpace *dataSpace_{nullptr};
DataSet *dataSet_{nullptr};
DataType dataType{PredType::STD_U16LE};
DataSpace *dataSpacePara_{nullptr};
std::vector<DataSet *> dataSetPara_{nullptr};
std::vector<std::string> parameterNames_;
std::vector<DataType> parameterDataTypes_;
uint32_t subFileIndex_{0}; // to do reset in createfirstdatafile
uint32_t maxFramesPerFile_{0};
uint64_t numImages_{0};
uint64_t extNumImages_{0}; // TODO needs to be defined in startofacquisition
uint32_t nPixelsX_{0};
uint32_t nPixelsY_{0};
uint32_t dynamicRange_{0};
};

16
slsReceiverSoftware/src/HDF5MasterFile.cpp
View File

@ -2,12 +2,13 @@
#include "MasterAttributes.h"
#include "sls/logger.h"
HDF5MasterFile::HDF5MasterFile(int index, std::mutex *hdf5Lib)
: File(index, HDF5), hdf5Lib_(hdf5Lib) {}
HDF5MasterFile::HDF5MasterFile(std::mutex *hdf5Lib)
: File(HDF5), hdf5Lib_(hdf5Lib) {}
HDF5MasterFile::~HDF5MasterFile() { CloseFile(); }
void HDF5MasterFile::CloseFile() {
std::lock_guard<std::mutex> lock(*hdf5Lib_);
try {
Exception::dontPrint(); // to handle errors
if (fd_) {
@ -16,17 +17,15 @@ void HDF5MasterFile::CloseFile() {
fd_ = nullptr;
}
} catch (const Exception &error) {
LOG(logERROR) << "Could not close data HDF5 handles of index "
<< index_;
LOG(logERROR) << "Could not close master HDF5 handles";
error.printErrorStack();
}
}
void HDF5MasterFile::CreateMasterFile(MasterAttributes *attr,
std::string filePath,
void HDF5MasterFile::CreateMasterFile(std::string filePath,
std::string fileNamePrefix,
uint64_t fileIndex, bool overWriteEnable,
bool silentMode) {
bool silentMode, MasterAttributes *attr) {
std::ostringstream os;
os << filePath << "/" << fileNamePrefix << "_master"
@ -66,6 +65,7 @@ void HDF5MasterFile::CreateMasterFile(MasterAttributes *attr,
} catch (const Exception &error) {
error.printErrorStack();
CloseFile();
throw sls::RuntimeError("Could not create master HDF5 handles");
throw sls::RuntimeError(
"Could not create/overwrite master HDF5 handles");
}
}

8
slsReceiverSoftware/src/HDF5MasterFile.h
View File

@ -12,13 +12,13 @@ using namespace H5;
class HDF5MasterFile : private virtual slsDetectorDefs, public File {
public:
HDF5MasterFile(int index, std::mutex *hdf5Lib);
HDF5MasterFile(std::mutex *hdf5Lib);
~HDF5MasterFile();
void CloseFile() override;
void CreateMasterFile(MasterAttributes *attr, std::string filePath,
std::string fileNamePrefix, uint64_t fileIndex,
bool overWriteEnable, bool silentMode) override;
void CreateMasterFile(std::string filePath, std::string fileNamePrefix,
uint64_t fileIndex, bool overWriteEnable,
bool silentMode, MasterAttributes *attr) override;
private:
std::mutex *hdf5Lib_;

17
slsReceiverSoftware/src/HDF5VirtualFile.cpp
View File

@ -1,15 +1,28 @@
#include "HDF5VirtualFile.h"
#include "sls/logger.h"
HDF5VirtualFile::HDF5VirtualFile(int index) : File(index, HDF5), fd_(0) {}
HDF5VirtualFile::HDF5VirtualFile(std::mutex *hdf5Lib)
: File(HDF5), hdf5Lib_(hdf5Lib) {}
HDF5VirtualFile::~HDF5VirtualFile() { CloseFile(); }
void HDF5VirtualFile::CloseFile() {
if (fd_ != 0) {
std::lock_guard<std::mutex> lock(*hdf5Lib_);
/*if (fd_ != 0) {
if (H5Fclose(fd_) < 0) {
LOG(logERROR) << "Could not close virtual HDF5 handles";
}
fd_ = 0;
}*/
try {
Exception::dontPrint(); // to handle errors
if (fd_) {
fd_->close();
delete fd_;
fd_ = nullptr;
}
} catch (const Exception &error) {
LOG(logERROR) << "Could not close virtual HDF5 handles of index";
error.printErrorStack();
}
}

8
slsReceiverSoftware/src/HDF5VirtualFile.h
View File

@ -7,14 +7,18 @@
using namespace H5;
#endif
#include <mutex>
class HDF5VirtualFile : private virtual slsDetectorDefs, public File {
public:
HDF5VirtualFile(int index);
HDF5VirtualFile(std::mutex *hdf5Lib);
~HDF5VirtualFile();
void CloseFile() override;
private:
hid_t fd_;
std::mutex *hdf5Lib_;
// hid_t fd_;
H5File *fd_{nullptr};
};

7
slsReceiverSoftware/src/MasterAttributes.h
View File

@ -1,5 +1,6 @@
#pragma once
#include "receiver_defs.h"
#include "sls/ToString.h"
#include "sls/logger.h"
#include "sls/sls_detector_defs.h"
@ -14,10 +15,6 @@ using namespace H5;
#include <chrono>
using ns = std::chrono::nanoseconds;
// versions
#define HDF5_WRITER_VERSION (6.2) // 1 decimal places
#define BINARY_WRITER_VERSION (6.2) // 1 decimal places
struct MasterAttributes {
slsDetectorDefs::detectorType detType{slsDetectorDefs::GENERIC};
slsDetectorDefs::timingMode timingMode{slsDetectorDefs::AUTO_TIMING};
@ -36,7 +33,7 @@ struct MasterAttributes {
uint32_t dynamicRange{0};
uint32_t tenGiga{0};
int thresholdEnergyeV{0};
std::array<int, 3> thresholdAllEnergyeV={{0, 0, 0}};
std::array<int, 3> thresholdAllEnergyeV = {{0, 0, 0}};
ns subExptime{0};
ns subPeriod{0};
uint32_t quad{0};

5
slsReceiverSoftware/src/receiver_defs.h
View File

@ -13,6 +13,11 @@
#define MAX_SOCKET_INPUT_PACKET_QUEUE (250000)
// files
// versions
#define HDF5_WRITER_VERSION (6.2) // 1 decimal places
#define BINARY_WRITER_VERSION (6.2) // 1 decimal places
#define MAX_FRAMES_PER_FILE 20000
#define SHORT_MAX_FRAMES_PER_FILE 100000
#define MOENCH_MAX_FRAMES_PER_FILE 100000