mx/Jungfraujoch
0
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 37 Wiki Activity

File writer and spot finding improvements

Browse Source
This commit is contained in:
leonarski_f
2024-04-08 11:18:50 +02:00
parent 15d99c6162
commit c6d2b5eedf
72 changed files with 690 additions and 893 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
writer/HDF5DataFile.cpp
View File

@@ -4,6 +4,12 @@
#include "HDF5DataFile.h"
#include "../compression/JFJochCompressor.h"
#include "HDF5DataFilePluginAzInt.h"
#include "HDF5DataFilePluginMX.h"
#include "HDF5DataFilePluginXFEL.h"
#include "HDF5DataFilePluginJUNGFRAU.h"
#include "HDF5DataFilePluginResEstimation.h"
HDF5DataFile::HDF5DataFile(const std::string &in_filename,
const std::vector<float>& in_rad_int_bin_to_q,
size_t in_max_spots) {
@@ -16,25 +22,52 @@ HDF5DataFile::HDF5DataFile(const std::string &in_filename,
plugins.emplace_back(std::make_unique<HDF5DataFilePluginJUNGFRAU>());
plugins.emplace_back(std::make_unique<HDF5DataFilePluginAzInt>(in_rad_int_bin_to_q));
plugins.emplace_back(std::make_unique<HDF5DataFilePluginXFEL>());
plugins.emplace_back(std::make_unique<HDF5DataFilePluginResEstimation>());
plugins.emplace_back(std::make_unique<HDF5DataFilePluginMX>(in_max_spots));
}
HDF5DataFile::~HDF5DataFile() {
if (data_file) {
HDF5Group group_exp(*data_file, "/entry/detector");
group_exp.NXClass("NXcollection");
group_exp.SaveVector("timestamp", timestamp);
group_exp.SaveVector("exptime", exptime);
group_exp.SaveVector("number", number);
for (auto &p: plugins)
p->WriteFinal(*data_file);
data_file.reset();
std::string old_filename = filename + ".tmp";
std::rename(old_filename.c_str(), filename.c_str());
}
}
void HDF5DataFile::CreateFile(const DataMessage& msg) {
SetupSWMRFile(msg);
HDF5Dcpl dcpl;
HDF5DataType data_type(msg.image.pixel_depth_bytes, msg.image.pixel_is_signed);
xpixel = msg.image.xpixel;
ypixel = msg.image.ypixel;
dcpl.SetCompression(msg.image.algorithm, msg.image.pixel_depth_bytes, JFJochBitShuffleCompressor::DefaultBlockSize);
dcpl.SetChunking( {1, ypixel, xpixel});
if (msg.image.pixel_is_signed) {
if (msg.image.pixel_depth_bytes == 2)
dcpl.SetFillValue16(INT16_MIN);
else
dcpl.SetFillValue32(INT32_MIN);
}
data_file = std::make_unique<HDF5File>(filename + ".tmp");
chmod(filename.c_str(), S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); // default permissions
HDF5Group(*data_file, "/entry").NXClass("NXentry");
HDF5Group(*data_file, "/entry/data").NXClass("NXdata");
HDF5DataSpace data_space({1, ypixel, xpixel}, {H5S_UNLIMITED, ypixel, xpixel});
data_set = std::make_unique<HDF5DataSet>(*data_file, "/entry/data/data", data_type, data_space, dcpl);
data_file = std::make_unique<HDF5File>(filename, false, true);
data_set = std::make_unique<HDF5DataSet>(*data_file, "/entry/data/data");
for (auto &p: plugins)
p->OpenFile(*data_file, msg);
}
@@ -53,6 +86,7 @@ void HDF5DataFile::Write(const DataMessage &msg, uint64_t image_number) {
data_set->SetExtent({max_image_number+1, ypixel, xpixel});
timestamp.resize(max_image_number + 1);
exptime.resize(max_image_number + 1);
number.resize(max_image_number + 1);
}
nimages++;
@@ -63,17 +97,7 @@ void HDF5DataFile::Write(const DataMessage &msg, uint64_t image_number) {
timestamp[image_number] = msg.timestamp;
exptime[image_number] = msg.exptime;
auto now_time = std::chrono::system_clock::now();
auto time_diff = now_time - (last_flush + swmr_flush_period);
if (time_diff.count() >= 0) {
data_set->Flush();
for (auto &p: plugins)
p->Flush();
last_flush = now_time;
}
number[image_number] = msg.number;
}
HDF5DataFileStatistics HDF5DataFile::GetStatistics() const {
@@ -87,34 +111,3 @@ HDF5DataFileStatistics HDF5DataFile::GetStatistics() const {
size_t HDF5DataFile::GetNumImages() const {
return nimages;
}
void HDF5DataFile::SetupSWMRFile(const DataMessage& msg) {
HDF5Dcpl dcpl;
HDF5DataType data_type(msg.image.pixel_depth_bytes, msg.image.pixel_is_signed);
xpixel = msg.image.xpixel;
ypixel = msg.image.ypixel;
dcpl.SetCompression(msg.image.algorithm, msg.image.pixel_depth_bytes, JFJochBitShuffleCompressor::DefaultBlockSize);
dcpl.SetChunking( {1, ypixel, xpixel});
if (msg.image.pixel_is_signed) {
if (msg.image.pixel_depth_bytes == 2)
dcpl.SetFillValue16(INT16_MIN);
else
dcpl.SetFillValue32(INT32_MIN);
}
HDF5File data_file_local(filename, true, true);
chmod(filename.c_str(), S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); // default permissions
HDF5Group(data_file_local, "/entry").NXClass("NXentry");
HDF5Group(data_file_local, "/entry/data").NXClass("NXdata");
HDF5DataSpace data_space({1, ypixel, xpixel}, {H5S_UNLIMITED, ypixel, xpixel});
HDF5DataSet(data_file_local, "/entry/data/data", data_type, data_space, dcpl);
for (auto &p: plugins)
p->SetupSWMRFile(data_file_local, msg);
last_flush = std::chrono::system_clock::now();
}