mx/Jungfraujoch
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 42 Wiki Activity

Modifications in preparation to MAX IV experiment

Browse Source
This commit is contained in:
leonarski_f
2024-01-27 21:23:56 +01:00
parent 2446643489
commit f5f86d9ab6
250 changed files with 9363 additions and 3022 deletions
Download Patch File Download Diff File Expand all files Collapse all files

168
writer/HDF5DataFile.cpp
View File

@@ -21,77 +21,48 @@ HDF5DataFile::HDF5DataFile(const std::string &in_filename, int64_t width, int64_
dcpl.SetCompression(compression, pixel_depth_byte, JFJochBitShuffleCompressor::DefaultBlockSize);
dcpl.SetChunking( {1, ypixel, xpixel});
if (is_signed) {
if (pixel_depth_byte == 2)
dcpl.SetFillValue16(INT16_MIN);
else
dcpl.SetFillValue32(INT32_MIN);
}
}
HDF5DataFile::~HDF5DataFile() {
data_set.reset();
if (!spot_count.empty()) {
std::vector<hsize_t> dims = {spot_count.size(), max_spots};
result_group->SaveVector("nPeaks", spot_count);
result_group->SaveVector("indexingResult", indexing_result);
result_group->SaveVector("indexingLattice", indexing_lattice, {max_image_number + 1, 9});
if (!rad_int_bin_to_q.empty())
rad_int_group->SaveVector("bin_to_q", rad_int_bin_to_q);
if (!rad_int_file_avg.empty() && (rad_int_file_avg.size() == rad_int_bin_to_q.size()))
rad_int_group->SaveVector("file_avg", rad_int_file_avg);
if (data_file) {
HDF5Group group_exp(*data_file, "/entry/jungfrau");
group_exp.NXClass("NXcollection");
group_exp.SaveVector("bunch_id", bunch_id);
group_exp.SaveVector("info", jf_info);
group_exp.SaveVector("timestamp", timestamp);
group_exp.SaveVector("storage_cell", storage_cell);
group_exp.SaveVector("exptime", exptime);
group_exp.SaveVector("receiver_available_send_buffers", receiver_available_send_buffers);
group_exp.SaveVector("receiver_aq_dev_delay", receiver_aq_dev_delay);
}
rad_int_group.reset();
result_group.reset();
data_file.reset();
}
void HDF5DataFile::CreateFile() {
data_file = std::make_unique<HDF5File>(filename, true,false, false);
HDF5Group(*data_file, "/entry").NXClass("NXentry");
HDF5Group(*data_file, "/entry/data").NXClass("NXdata");
SetupSWMRFile();
result_group = std::make_unique<HDF5Group>(*data_file, "/entry/result");
result_group->NXClass("NXcollection");
data_file = std::make_unique<HDF5File>(filename, false, true);
data_set = std::make_unique<HDF5DataSet>(*data_file, "/entry/data/data");
data_set_spot_x = std::make_unique<HDF5DataSet>(*data_file, "/entry/result/peakXPosRaw");
data_set_spot_y = std::make_unique<HDF5DataSet>(*data_file, "/entry/result/peakYPosRaw");
data_set_spot_int = std::make_unique<HDF5DataSet>(*data_file, "/entry/result/peakTotalIntensity");
data_set_spot_indexed = std::make_unique<HDF5DataSet>(*data_file, "/entry/result/peakIndexed");
data_set_spot_npeaks = std::make_unique<HDF5DataSet>(*data_file, "/entry/result/nPeaks");
data_set_xfel_event_code = std::make_unique<HDF5DataSet>(*data_file, "/entry/xfel/eventCode");
data_set_xfel_pulseid = std::make_unique<HDF5DataSet>(*data_file, "/entry/xfel/pulseID");
data_set_indexed = std::make_unique<HDF5DataSet>(*data_file, "/entry/result/imageIndexed");
data_set_indexed_lattice = std::make_unique<HDF5DataSet>(*data_file, "/entry/result/latticeIndexed");
if (!rad_int_bin_to_q.empty())
data_set_az_int = std::make_unique<HDF5DataSet>(*data_file, "/entry/result/azimIntegration");
rad_int_group = std::make_unique<HDF5Group>(*data_file, "/entry/result/rad_int");
adu_histo_group = std::make_unique<HDF5Group>(*data_file, "/entry/result/adu_histogram");
HDF5DataSpace data_space({1, ypixel, xpixel},{H5S_UNLIMITED, ypixel, xpixel});
data_set = std::make_unique<HDF5DataSet>(*data_file, "/entry/data/data", data_type, data_space, dcpl);
HDF5DataSpace data_space_spots({1, max_spots},{H5S_UNLIMITED, max_spots});
HDF5Dcpl dcpl_spots;
dcpl_spots.SetChunking({1, max_spots});
data_set_spot_x = std::make_unique<HDF5DataSet>(*data_file, "/entry/result/peakXPosRaw",
HDF5DataType(0.0f), data_space_spots, dcpl_spots);
data_set_spot_y = std::make_unique<HDF5DataSet>(*data_file, "/entry/result/peakYPosRaw",
HDF5DataType(0.0f), data_space_spots, dcpl_spots);
data_set_spot_int = std::make_unique<HDF5DataSet>(*data_file, "/entry/result/peakTotalIntensity",
HDF5DataType(0.0f), data_space_spots, dcpl_spots);
data_set_spot_indexed = std::make_unique<HDF5DataSet>(*data_file, "/entry/result/peakIndexed",
HDF5DataType(1, false), data_space_spots, dcpl_spots);
spot_count.resize(1);
indexing_result.resize(1);
bunch_id.resize(1);
jf_info.resize(1);
receiver_available_send_buffers.resize(1);
receiver_aq_dev_delay.resize(1);
timestamp.resize(1);
storage_cell.resize(1);
exptime.resize(1);
indexing_lattice.resize(9);
}
void HDF5DataFile::Write(const DataMessage &msg, uint64_t image_number) {
@@ -106,25 +77,25 @@ void HDF5DataFile::Write(const DataMessage &msg, uint64_t image_number) {
data_set_spot_y->SetExtent({max_image_number+1, max_spots});
data_set_spot_int->SetExtent({max_image_number+1, max_spots});
data_set_spot_indexed->SetExtent({max_image_number+1, max_spots});
data_set_spot_npeaks->SetExtent({max_image_number+1});
data_set_xfel_pulseid->SetExtent({max_image_number+1});
data_set_xfel_event_code->SetExtent({max_image_number+1});
data_set_indexed->SetExtent({max_image_number + 1});
data_set_indexed_lattice->SetExtent({max_image_number + 1, 9});
if (!rad_int_bin_to_q.empty())
data_set_az_int->SetExtent({max_image_number + 1, rad_int_bin_to_q.size()});
spot_count.resize(max_image_number + 1);
indexing_result.resize(max_image_number + 1);
bunch_id.resize(max_image_number + 1);
jf_info.resize(max_image_number + 1);
receiver_available_send_buffers.resize(max_image_number + 1);
receiver_aq_dev_delay.resize(max_image_number + 1);
timestamp.resize(max_image_number + 1);
exptime.resize(max_image_number + 1);
storage_cell.resize(max_image_number + 1);
indexing_lattice.resize((max_image_number + 1) * 9);
}
nimages++;
data_set->WriteDirectChunk(msg.image.data, msg.image.size, {image_number, 0, 0});
size_t cnt = std::min(msg.spots.size(), max_spots);
spot_count[image_number] = cnt;
uint32_t cnt = std::min(msg.spots.size(), max_spots);
std::vector<float> spot_x(max_spots), spot_y(max_spots), spot_intensity(max_spots);
std::vector<uint8_t> spot_indexed(max_spots);
@@ -138,23 +109,42 @@ void HDF5DataFile::Write(const DataMessage &msg, uint64_t image_number) {
data_set_spot_y->WriteVec(spot_y, {image_number, 0}, {1, max_spots});
data_set_spot_int->WriteVec(spot_intensity, {image_number, 0}, {1, max_spots});
data_set_spot_indexed->WriteVec(spot_indexed, {image_number, 0}, {1, max_spots});
data_set_spot_npeaks->WriteScalar(cnt, {image_number});
data_set_xfel_event_code->WriteScalar(msg.xfel_event_code, {image_number});
data_set_xfel_pulseid->WriteScalar(msg.xfel_bunch_id, {image_number});
data_set_indexed->WriteScalar((uint8_t) msg.indexing_result, {image_number});
if (msg.indexing_lattice.size() == 9)
data_set_indexed_lattice->WriteVec(msg.indexing_lattice, {image_number, 0}, {1, 9});
indexing_result[image_number] = msg.indexing_result;
bunch_id[image_number] = msg.bunch_id;
jf_info[image_number] = msg.jf_info;
receiver_available_send_buffers[image_number] = msg.receiver_available_send_buffers;
receiver_aq_dev_delay[image_number] = msg.receiver_aq_dev_delay;
timestamp[image_number] = msg.timestamp;
storage_cell[image_number] = msg.storage_cell;
exptime[image_number] = msg.exptime;
if (msg.indexing_lattice.size() == 9) {
for (int i = 0; i < 9; i++)
indexing_lattice[image_number * 9 + i] = msg.indexing_lattice[i];
}
if (!msg.az_int_profile.empty() && (msg.az_int_profile.size() == rad_int_bin_to_q.size()))
data_set_az_int->WriteVec(msg.az_int_profile, {image_number, 0}, {1, rad_int_bin_to_q.size()});
if (!msg.rad_int_profile.empty() && (msg.rad_int_profile.size() == rad_int_bin_to_q.size()))
rad_int_group->SaveVector("img" + std::to_string(image_number), msg.rad_int_profile);
auto now_time = std::chrono::system_clock::now();
auto time_diff = now_time - (last_flush + swmr_flush_period);
if (time_diff.count() >= 0) {
data_set->Flush();
data_set_spot_x->Flush();
data_set_spot_y->Flush();
data_set_spot_int->Flush();
data_set_spot_indexed->Flush();
data_set_spot_npeaks->Flush();
data_set_xfel_event_code->Flush();
data_set_xfel_pulseid->Flush();
data_set_indexed->Flush();
data_set_indexed_lattice->Flush();
if (!rad_int_bin_to_q.empty())
data_set_az_int->Flush();
last_flush = now_time;
}
}
HDF5DataFileStatistics HDF5DataFile::GetStatistics() const {
@@ -168,3 +158,47 @@ HDF5DataFileStatistics HDF5DataFile::GetStatistics() const {
size_t HDF5DataFile::GetNumImages() const {
return nimages;
}
void HDF5DataFile::SetupSWMRFile() {
HDF5File data_file_local(filename, true, true);
HDF5Group(data_file_local, "/entry").NXClass("NXentry");
HDF5Group(data_file_local, "/entry/data").NXClass("NXdata");
HDF5Group(data_file_local, "/entry/xfel").NXClass("NXcollection");
HDF5Group(data_file_local, "/entry/result").NXClass("NXcollection");
HDF5DataSpace data_space({1, ypixel, xpixel}, {H5S_UNLIMITED, ypixel, xpixel});
HDF5DataSet(data_file_local, "/entry/data/data", data_type, data_space, dcpl);
HDF5DataSpace data_space_spots({1, max_spots}, {H5S_UNLIMITED, max_spots});
HDF5Dcpl dcpl_spots;
dcpl_spots.SetChunking({1, max_spots});
HDF5DataSet(data_file_local, "/entry/result/peakXPosRaw", HDF5DataType(0.0f), data_space_spots, dcpl_spots);
HDF5DataSet(data_file_local, "/entry/result/peakYPosRaw", HDF5DataType(0.0f), data_space_spots, dcpl_spots);
HDF5DataSet(data_file_local, "/entry/result/peakTotalIntensity", HDF5DataType(0.0f), data_space_spots, dcpl_spots);
HDF5DataSet(data_file_local, "/entry/result/peakIndexed", HDF5DataType(1, false), data_space_spots, dcpl_spots);
HDF5DataSpace data_space_linear({2}, {H5S_UNLIMITED});
HDF5Dcpl dcpl_linear;
dcpl_linear.SetChunking({1});
HDF5DataSet(data_file_local, "/entry/result/nPeaks", HDF5DataType(4, false), data_space_linear, dcpl_linear).SetExtent({1});
HDF5DataSet(data_file_local, "/entry/xfel/pulseID", HDF5DataType(8, false), data_space_linear, dcpl_linear).SetExtent({1});
HDF5DataSet(data_file_local, "/entry/xfel/eventCode", HDF5DataType(4, false), data_space_linear, dcpl_linear).SetExtent({1});
HDF5DataSet(data_file_local, "/entry/result/imageIndexed", HDF5DataType(1, false), data_space_linear, dcpl_linear).SetExtent({1});
HDF5DataSpace data_space_lattice({1, 9}, {H5S_UNLIMITED, 9});
HDF5Dcpl dcpl_lattice;
dcpl_lattice.SetChunking({1, 9});
HDF5DataSet(data_file_local, "/entry/result/latticeIndexed", HDF5DataType(0.0f), data_space_lattice, dcpl_lattice);
if (!rad_int_bin_to_q.empty()) {
data_file_local.SaveVector("/entry/result/azimIntegrationBinToQ", rad_int_bin_to_q);
HDF5DataSpace data_space_rad_int({1, rad_int_bin_to_q.size()}, {H5S_UNLIMITED, rad_int_bin_to_q.size()});
HDF5Dcpl dcpl_rad_int;
dcpl_rad_int.SetChunking({1, rad_int_bin_to_q.size()});
HDF5DataSet(data_file_local, "/entry/result/azimIntegration", HDF5DataType(0.0f), data_space_rad_int,
dcpl_rad_int);
}
last_flush = std::chrono::system_clock::now();
}