// Copyright (2019-2022) Paul Scherrer Institute // SPDX-License-Identifier: GPL-3.0-or-later #include "HDF5DataFile.h" #include "../compression/JFJochCompressor.h" HDF5DataFile::HDF5DataFile(const std::string &in_filename, int64_t width, int64_t height, int64_t pixel_depth_byte, bool is_signed, const std::vector& in_rad_int_bin_to_q, CompressionAlgorithm compression, size_t in_max_spots) : data_type(pixel_depth_byte, is_signed), max_spots(in_max_spots), rad_int_bin_to_q(in_rad_int_bin_to_q) { max_image_number = 0; nimages = 0; if (max_spots > 256) throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, "Max spot count must be less than 256 for performance reasons"); filename = in_filename; xpixel = width; ypixel = height; dcpl.SetCompression(compression, pixel_depth_byte, JFJochBitShuffleCompressor::DefaultBlockSize); dcpl.SetChunking( {1, ypixel, xpixel}); } HDF5DataFile::~HDF5DataFile() { data_set.reset(); if (!spot_count.empty()) { std::vector dims = {spot_count.size(), max_spots}; result_group->SaveVector("nPeaks", spot_count); result_group->SaveVector("indexingResult", indexing_result); result_group->SaveVector("indexingLattice", indexing_lattice, {max_image_number + 1, 9}); if (!rad_int_bin_to_q.empty()) rad_int_group->SaveVector("bin_to_q", rad_int_bin_to_q); if (!rad_int_file_avg.empty() && (rad_int_file_avg.size() == rad_int_bin_to_q.size())) rad_int_group->SaveVector("file_avg", rad_int_file_avg); HDF5Group group_exp(*data_file, "/entry/jungfrau"); group_exp.NXClass("NXcollection"); group_exp.SaveVector("bunch_id", bunch_id); group_exp.SaveVector("info", jf_info); group_exp.SaveVector("timestamp", timestamp); group_exp.SaveVector("storage_cell", storage_cell); group_exp.SaveVector("exptime", exptime); } rad_int_group.reset(); result_group.reset(); data_file.reset(); } void HDF5DataFile::CreateFile() { data_file = std::make_unique(filename, true,false, false); HDF5Group(*data_file, "/entry").NXClass("NXentry"); HDF5Group(*data_file, "/entry/data").NXClass("NXdata"); result_group = std::make_unique(*data_file, "/entry/result"); result_group->NXClass("NXcollection"); rad_int_group = std::make_unique(*data_file, "/entry/result/rad_int"); HDF5DataSpace data_space({1, ypixel, xpixel},{H5S_UNLIMITED, ypixel, xpixel}); data_set = std::make_unique(*data_file, "/entry/data/data", data_type, data_space, dcpl); HDF5DataSpace data_space_spots({1, max_spots},{H5S_UNLIMITED, max_spots}); HDF5Dcpl dcpl_spots; dcpl_spots.SetChunking({1, max_spots}); data_set_spot_x = std::make_unique(*data_file, "/entry/result/peakXPosRaw", HDF5DataType(0.0f), data_space_spots, dcpl_spots); data_set_spot_y = std::make_unique(*data_file, "/entry/result/peakYPosRaw", HDF5DataType(0.0f), data_space_spots, dcpl_spots); data_set_spot_int = std::make_unique(*data_file, "/entry/result/peakTotalIntensity", HDF5DataType(0.0f), data_space_spots, dcpl_spots); data_set_spot_indexed = std::make_unique(*data_file, "/entry/result/peakIndexed", HDF5DataType(1, false), data_space_spots, dcpl_spots); spot_count.resize(1); indexing_result.resize(1); bunch_id.resize(1); jf_info.resize(1); timestamp.resize(1); storage_cell.resize(1); exptime.resize(1); indexing_lattice.resize(9); } void HDF5DataFile::Write(const DataMessage &msg, uint64_t image_number) { std::lock_guard lock(hdf5_mutex); if (!data_file) CreateFile(); if (image_number > max_image_number) { max_image_number = image_number; data_set->SetExtent({max_image_number+1, ypixel, xpixel}); data_set_spot_x->SetExtent({max_image_number+1, max_spots}); data_set_spot_y->SetExtent({max_image_number+1, max_spots}); data_set_spot_int->SetExtent({max_image_number+1, max_spots}); data_set_spot_indexed->SetExtent({max_image_number+1, max_spots}); spot_count.resize(max_image_number + 1); indexing_result.resize(max_image_number + 1); bunch_id.resize(max_image_number + 1); jf_info.resize(max_image_number + 1); timestamp.resize(max_image_number + 1); exptime.resize(max_image_number + 1); storage_cell.resize(max_image_number + 1); indexing_lattice.resize((max_image_number + 1) * 9); } nimages++; data_set->WriteDirectChunk(msg.image.data, msg.image.size, {image_number, 0, 0}); size_t cnt = std::min(msg.spots.size(), max_spots); spot_count[image_number] = cnt; std::vector spot_x(max_spots), spot_y(max_spots), spot_intensity(max_spots); std::vector spot_indexed(max_spots); for (int i = 0; i < cnt; i++) { spot_x[i] = msg.spots[i].x; spot_y[i] = msg.spots[i].y; spot_intensity[i] = msg.spots[i].intensity; spot_indexed[i] = msg.spots[i].indexed; } data_set_spot_x->WriteVec(spot_x, {image_number, 0}, {1, max_spots}); data_set_spot_y->WriteVec(spot_y, {image_number, 0}, {1, max_spots}); data_set_spot_int->WriteVec(spot_intensity, {image_number, 0}, {1, max_spots}); data_set_spot_indexed->WriteVec(spot_indexed, {image_number, 0}, {1, max_spots}); indexing_result[image_number] = msg.indexing_result; bunch_id[image_number] = msg.bunch_id; jf_info[image_number] = msg.jf_info; timestamp[image_number] = msg.timestamp; storage_cell[image_number] = msg.storage_cell; exptime[image_number] = msg.exptime; if (msg.indexing_lattice.size() == 9) { for (int i = 0; i < 9; i++) indexing_lattice[image_number * 9 + i] = msg.indexing_lattice[i]; } if (!msg.rad_int_profile.empty() && (msg.rad_int_profile.size() == rad_int_bin_to_q.size())) rad_int_group->SaveVector("img" + std::to_string(image_number), msg.rad_int_profile); } HDF5DataFileStatistics HDF5DataFile::GetStatistics() const { HDF5DataFileStatistics ret; ret.max_image_number = max_image_number; ret.total_images = nimages; ret.filename = filename; return ret; } size_t HDF5DataFile::GetNumImages() const { return nimages; }