diff --git a/reader/JFJochHDF5Reader.cpp b/reader/JFJochHDF5Reader.cpp index 00ff82a4..b3979215 100644 --- a/reader/JFJochHDF5Reader.cpp +++ b/reader/JFJochHDF5Reader.cpp @@ -116,9 +116,18 @@ void JFJochHDF5Reader::ReadFile(const std::string& filename) { size_t image_size_x = 0; size_t image_size_y = 0; - if (master_file->Exists("/entry/data/data")) { - legacy_format = false; - auto dim = GetDimension(*master_file, "/entry/data/data"); + if (master_file->Exists("/entry/data/data")) + format = FileWriterFormat::NXmxVDS; + else if (master_file->Exists("/entry/data/data_000001")) { + if (master_file->IsExternalLink("/entry/data/data_000001")) + format = FileWriterFormat::NXmxLegacy; + else + format = FileWriterFormat::NXmxIntegrated; + } + + if (format == FileWriterFormat::NXmxVDS || format == FileWriterFormat::NXmxIntegrated) { + auto dim = GetDimension(*master_file, + (format == FileWriterFormat::NXmxIntegrated) ? "/entry/data/data_000001" : "/entry/data/data"); number_of_images = dim[0]; image_size_y = dim[1]; image_size_x = dim[2]; @@ -159,8 +168,7 @@ void JFJochHDF5Reader::ReadFile(const std::string& filename) { } if (master_file->Exists("/entry/image")) dataset->max_value = master_file->ReadOptVector("/entry/image/max_value"); - } else if (master_file->Exists("/entry/data/data_000001")) { - legacy_format = true; + } else if (format == FileWriterFormat::NXmxLegacy) { legacy_format_files.clear(); image_size_x = master_file->GetInt("/entry/instrument/detector/detectorSpecific/x_pixels_in_detector"); @@ -334,7 +342,7 @@ void JFJochHDF5Reader::ReadFile(const std::string& filename) { dataset->error_value = master_file->GetOptInt("/entry/instrument/detector/error_value"); - dataset->jfjoch_release = master_file->GetString("/entry/instrument/detector/jfjoch_release"); + dataset->jfjoch_release = master_file->GetString("/entry/instrument/detector/detectorSpecific/jfjoch_release"); InstrumentMetadata metadata; metadata.InstrumentName(master_file->GetString("/entry/instrument/name")); @@ -457,7 +465,9 @@ uint64_t JFJochHDF5Reader::GetNumberOfImages() const { CompressedImage JFJochHDF5Reader::LoadImageDataset(std::vector &tmp, HDF5Object &file, hsize_t number) { std::vector start = {static_cast(number), 0, 0}; - HDF5DataSet dataset(file, "/entry/data/data"); + const std::string dataset_name = (format == FileWriterFormat::NXmxIntegrated) ? "/entry/data/data_000001" : "/entry/data/data"; + + HDF5DataSet dataset(file, dataset_name); HDF5DataSpace dataspace(dataset); HDF5DataType datatype(dataset); HDF5Dcpl dcpl(dataset); @@ -508,7 +518,7 @@ bool JFJochHDF5Reader::LoadImage_i(std::shared_ptr &dataset uint32_t image_id; HDF5Object *source_file; - if (legacy_format) { + if (format == FileWriterFormat::NXmxLegacy) { uint32_t file_id = image_number / images_per_file; image_id = image_number % images_per_file; tmp_data_file = std::make_unique(legacy_format_files.at(file_id)); diff --git a/reader/JFJochHDF5Reader.h b/reader/JFJochHDF5Reader.h index 914050ab..a3455838 100644 --- a/reader/JFJochHDF5Reader.h +++ b/reader/JFJochHDF5Reader.h @@ -8,11 +8,12 @@ #include "../writer/HDF5Objects.h" class JFJochHDF5Reader : public JFJochReader { + FileWriterFormat format = FileWriterFormat::NoFile; + std::unique_ptr master_file; std::vector legacy_format_files; - bool legacy_format = false; size_t images_per_file = 1; size_t number_of_images = 0; diff --git a/tests/JFJochReaderTest.cpp b/tests/JFJochReaderTest.cpp index 0cbd082d..ec53eac9 100644 --- a/tests/JFJochReaderTest.cpp +++ b/tests/JFJochReaderTest.cpp @@ -1414,3 +1414,86 @@ TEST_CASE("JFJochReader_InstrumentMetadata_Sample_RingCurrent", "[HDF5][Full]") remove("test_meta_master.h5"); REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); } + +TEST_CASE("JFJochReader_NXmxIntegrated", "[HDF5][Full]") { + DiffractionExperiment x(DetJF(1)); + + x.FilePrefix("test_reader_integrated").ImagesPerTrigger(3).OverwriteExistingFiles(true); + x.BitDepthImage(16).PixelSigned(false).SetFileWriterFormat(FileWriterFormat::NXmxIntegrated); + x.Compression(CompressionAlgorithm::NO_COMPRESSION); + x.BeamX_pxl(100).BeamY_pxl(200).DetectorDistance_mm(150) + .IncidentEnergy_keV(WVL_1A_IN_KEV) + .FrameTime(std::chrono::microseconds(500), std::chrono::microseconds(10)); + + AzimuthalIntegrationSettings azint_settings; + azint_settings.AzimuthalBinCount(4); + x.ImportAzimuthalIntegrationSettings(azint_settings); + + std::vector image(x.GetPixelsNum(), 0); + image[0] = UINT16_MAX; + image[1] = 123; + image[5678] = 321; + + AzimuthalIntegration azint(x, PixelMask(x)); + RegisterHDF5Filter(); + + { + StartMessage start_message; + x.FillMessage(start_message); + start_message.az_int_bin_to_q = azint.GetBinToQ(); + start_message.az_int_bin_to_phi = azint.GetBinToPhi(); + start_message.az_int_q_bin_count = azint.GetQBinCount(); + start_message.az_int_phi_bin_count = azint.GetAzimuthalBinCount(); + + FileWriter file_set(start_message); + + for (int i = 0; i < x.GetImageNum(); i++) { + DataMessage message{}; + image[5678] = 321 + i; + message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum()); + message.number = i; + message.image_collection_efficiency = 0.9f + 0.01f * i; + message.az_int_profile = std::vector(azint_settings.GetBinCount(), static_cast(50 + i)); + + REQUIRE_NOTHROW(file_set.WriteHDF5(message)); + } + + EndMessage end_message; + end_message.max_image_number = x.GetImageNum(); + file_set.WriteHDF5(end_message); + file_set.Finalize(); + } + { + JFJochHDF5Reader reader; + REQUIRE_NOTHROW(reader.ReadFile("test_reader_integrated_master.h5")); + auto dataset = reader.GetDataset(); + + CHECK(dataset->experiment.GetImageNum() == 3); + REQUIRE(dataset->efficiency.size() == 3); + CHECK(dataset->efficiency[0] == Catch::Approx(0.90f)); + CHECK(dataset->efficiency[1] == Catch::Approx(0.91f)); + CHECK(dataset->efficiency[2] == Catch::Approx(0.92f)); + + CHECK(dataset->az_int_bin_to_q.size() == azint_settings.GetBinCount()); + CHECK(dataset->azimuthal_bins == azint_settings.GetAzimuthalBinCount()); + CHECK(dataset->q_bins == azint_settings.GetQBinCount()); + + std::shared_ptr reader_image; + REQUIRE_NOTHROW(reader_image = reader.LoadImage(1)); + REQUIRE(reader_image); + + CHECK(reader_image->Image()[0] == SATURATED_PXL_VALUE); + CHECK(reader_image->Image()[1] == 123); + CHECK(reader_image->Image()[5678] == 322); + + REQUIRE(reader_image->ImageData().image_collection_efficiency.has_value()); + CHECK(reader_image->ImageData().image_collection_efficiency.value() == Catch::Approx(0.91f)); + + REQUIRE(reader_image->ImageData().az_int_profile.size() == azint_settings.GetBinCount()); + CHECK(reader_image->ImageData().az_int_profile[0] == Catch::Approx(51.0f)); + CHECK(reader_image->ImageData().az_int_profile[23] == Catch::Approx(51.0f)); + } + remove("test_reader_integrated_master.h5"); + + REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); +} \ No newline at end of file diff --git a/writer/HDF5DataFile.cpp b/writer/HDF5DataFile.cpp index 563f3615..7e6a66d2 100644 --- a/writer/HDF5DataFile.cpp +++ b/writer/HDF5DataFile.cpp @@ -46,7 +46,7 @@ HDF5DataFile::HDF5DataFile(const StartMessage &msg, uint64_t in_file_number) { } tmp_filename = fmt::format("{}.{:08x}.tmp", filename, tmp_suffix); plugins.emplace_back(std::make_unique()); - plugins.emplace_back(std::make_unique()); + plugins.emplace_back(std::make_unique(msg)); plugins.emplace_back(std::make_unique(msg)); plugins.emplace_back(std::make_unique()); plugins.emplace_back(std::make_unique(msg)); @@ -102,7 +102,7 @@ HDF5DataFile::~HDF5DataFile() { } } -void HDF5DataFile::CreateFile(const DataMessage& msg, std::shared_ptr in_data_file) { +void HDF5DataFile::CreateFile(const DataMessage& msg, std::shared_ptr in_data_file, bool integrated) { HDF5Dcpl dcpl; HDF5DataType data_type(msg.image.GetMode()); @@ -135,8 +135,10 @@ void HDF5DataFile::CreateFile(const DataMessage& msg, std::shared_ptr HDF5Group(*data_file, "/entry").NXClass("NXentry"); HDF5Group(*data_file, "/entry/data").NXClass("NXdata"); + const std::string dataset_name = integrated ? "/entry/data/data_000001" : "/entry/data/data"; + HDF5DataSpace data_space({1, ypixel, xpixel}, {H5S_UNLIMITED, ypixel, xpixel}); - data_set = std::make_unique(*data_file, "/entry/data/data", data_type, data_space, dcpl); + data_set = std::make_unique(*data_file, dataset_name, data_type, data_space, dcpl); data_set->SetExtent({images_per_file, ypixel, xpixel}); for (auto &p: plugins) p->OpenFile(*data_file, msg, images_per_file); diff --git a/writer/HDF5DataFile.h b/writer/HDF5DataFile.h index 4a740bfe..1346a333 100644 --- a/writer/HDF5DataFile.h +++ b/writer/HDF5DataFile.h @@ -56,7 +56,7 @@ public: void Write(const DataMessage& msg, uint64_t image_number); size_t GetNumImages() const; - void CreateFile(const DataMessage& msg, std::shared_ptr data_file); + void CreateFile(const DataMessage& msg, std::shared_ptr data_file, bool integrated = false); }; #endif //HDF5DATAFILE_H diff --git a/writer/HDF5DataFilePluginDetector.cpp b/writer/HDF5DataFilePluginDetector.cpp index 8d50098a..56306ad4 100644 --- a/writer/HDF5DataFilePluginDetector.cpp +++ b/writer/HDF5DataFilePluginDetector.cpp @@ -3,6 +3,10 @@ #include "HDF5DataFilePluginDetector.h" +HDF5DataFilePluginDetector::HDF5DataFilePluginDetector(const StartMessage &msg) { + integrated_format = (msg.file_format == FileWriterFormat::NXmxIntegrated); +} + void HDF5DataFilePluginDetector::OpenFile(HDF5File &in_data_file, const DataMessage &msg, size_t images_per_file) { jf_info.reserve(images_per_file); storage_cell.reserve(images_per_file); @@ -42,22 +46,24 @@ void HDF5DataFilePluginDetector::Write(const DataMessage &msg, uint64_t image_nu } void HDF5DataFilePluginDetector::WriteFinal(HDF5File &data_file) { - if (!jf_info.empty()) - data_file.SaveVector("/entry/detector/det_info", jf_info.vec()); - if (!storage_cell.empty()) - data_file.SaveVector("/entry/detector/storage_cell_image", storage_cell.vec()); - if (!receiver_aq_dev_delay.empty()) - data_file.SaveVector("/entry/detector/rcv_delay", receiver_aq_dev_delay.vec()); - if (!receiver_free_buffers.empty()) - data_file.SaveVector("/entry/detector/rcv_free_send_buffers", receiver_free_buffers.vec()); - if (!packets_received.empty()) - data_file.SaveVector("/entry/detector/packets_received", packets_received.vec()); - if (!packets_expected.empty()) - data_file.SaveVector("/entry/detector/packets_expected", packets_expected.vec()); - if (!pixel_sum.empty()) - data_file.SaveVector("/entry/detector/pixel_sum", pixel_sum.vec()); - if (!processing_time.empty()) - data_file.SaveVector("/entry/detector/processing_time", processing_time.vec())->Units("s"); + const std::string prefix = integrated_format ? "/entry/instrument/detector/detectorSpecific" : "/entry/detector"; - data_file.SaveVector("/entry/detector/data_collection_efficiency_image", efficiency.vec()); + if (!jf_info.empty()) + data_file.SaveVector(prefix + "/det_info", jf_info.vec()); + if (!storage_cell.empty()) + data_file.SaveVector(prefix + "/storage_cell_image", storage_cell.vec()); + if (!receiver_aq_dev_delay.empty()) + data_file.SaveVector(prefix + "/rcv_delay", receiver_aq_dev_delay.vec()); + if (!receiver_free_buffers.empty()) + data_file.SaveVector(prefix + "/rcv_free_send_buffers", receiver_free_buffers.vec()); + if (!packets_received.empty()) + data_file.SaveVector(prefix + "/packets_received", packets_received.vec()); + if (!packets_expected.empty()) + data_file.SaveVector(prefix + "/packets_expected", packets_expected.vec()); + if (!pixel_sum.empty()) + data_file.SaveVector(prefix + "/pixel_sum", pixel_sum.vec()); + if (!processing_time.empty()) + data_file.SaveVector(prefix + "/processing_time", processing_time.vec())->Units("s"); + + data_file.SaveVector(prefix + "/data_collection_efficiency_image", efficiency.vec()); } diff --git a/writer/HDF5DataFilePluginDetector.h b/writer/HDF5DataFilePluginDetector.h index 6d530a48..d9662170 100644 --- a/writer/HDF5DataFilePluginDetector.h +++ b/writer/HDF5DataFilePluginDetector.h @@ -8,6 +8,7 @@ #include "../common/AutoIncrVector.h" class HDF5DataFilePluginDetector : public HDF5DataFilePlugin { + bool integrated_format = false; AutoIncrVector jf_info; AutoIncrVector storage_cell; AutoIncrVector receiver_aq_dev_delay; @@ -18,6 +19,7 @@ class HDF5DataFilePluginDetector : public HDF5DataFilePlugin { AutoIncrVector pixel_sum; AutoIncrVector processing_time; public: + HDF5DataFilePluginDetector(const StartMessage& msg); void OpenFile(HDF5File &data_file, const DataMessage& msg, size_t images_per_file) override; void Write(const DataMessage& msg, uint64_t image_number) override; void WriteFinal(HDF5File &data_file) override; diff --git a/writer/HDF5Objects.cpp b/writer/HDF5Objects.cpp index 7ed7a77d..b95425a5 100644 --- a/writer/HDF5Objects.cpp +++ b/writer/HDF5Objects.cpp @@ -942,6 +942,17 @@ std::vector HDF5Object::FindLeafs(const std::string &name) const { return ret; } +bool HDF5Object::IsExternalLink(const std::string& name) const { + H5L_info2_t link_info; + + // Get information about the link + if (H5Lget_info(id, name.c_str(), &link_info, H5P_DEFAULT) < 0) + throw JFJochException(JFJochExceptionCategory::HDF5, + "Failed to retrieve information about the link"); + + return (link_info.type == H5L_TYPE_EXTERNAL); +} + std::string HDF5Object::GetLinkedFileName(const std::string& name) const { H5L_info2_t link_info; diff --git a/writer/HDF5Objects.h b/writer/HDF5Objects.h index fb336455..09a0c48e 100644 --- a/writer/HDF5Objects.h +++ b/writer/HDF5Objects.h @@ -151,6 +151,7 @@ public: const std::vector& start, const std::vector& size); bool Exists(const std::string& name) const; + bool IsExternalLink(const std::string& name) const; std::string GetLinkedFileName(const std::string& name) const; std::vector FindLeafs(const std::string &name) const; std::vector GetDimension(const std::string &name);