diff --git a/.gitea/workflows/build_and_test.yml b/.gitea/workflows/build_and_test.yml index 4f682af8..99292e58 100644 --- a/.gitea/workflows/build_and_test.yml +++ b/.gitea/workflows/build_and_test.yml @@ -137,17 +137,30 @@ jobs: - name: Generate test data (with virtual data set and 4 linked image files) shell: bash run: | - set -euo pipefail - mkdir -p dials_test - cd dials_test - ../build/tools/jfjoch_hdf5_test ../tests/test_data/compression_benchmark.h5 -n100 -f25 -V - - name: Run DIALS processing + - name: Run DIALS processing on legacy format shell: bash run: | source /opt/dials-v3-27-0/dials_env.sh - set -euo pipefail + mkdir -p dials_test cd dials_test - xia2.ssx image=writing_test_master.h5 space_group=P43212 unit_cell=78.551,78.551,36.914,90.000,90.000,90.000 + ../build/tools/jfjoch_hdf5_test ../tests/test_data/compression_benchmark.h5 -n100 -o legacy + xia2.ssx image=legacy_master.h5 space_group=P43212 unit_cell=78.551,78.551,36.914,90.000,90.000,90.000 + - name: Run DIALS processing on VDS (master + 4 linked image files) + shell: bash + run: | + source /opt/dials-v3-27-0/dials_env.sh + mkdir -p dials_test + cd dials_test + ../build/tools/jfjoch_hdf5_test ../tests/test_data/compression_benchmark.h5 -n100 -f25 -V -o vds + xia2.ssx image=vds_master.h5 space_group=P43212 unit_cell=78.551,78.551,36.914,90.000,90.000,90.000 + - name: Run DIALS processing on single file format + shell: bash + run: | + source /opt/dials-v3-27-0/dials_env.sh + mkdir -p dials_test + cd dials_test + ../build/tools/jfjoch_hdf5_test ../tests/test_data/compression_benchmark.h5 -n100 -S -o single + xia2.ssx image=single_master.h5 space_group=P43212 unit_cell=78.551,78.551,36.914,90.000,90.000,90.000 python-client: name: Generate python client runs-on: jfjoch_rocky8 diff --git a/common/FileWriterSettings.cpp b/common/FileWriterSettings.cpp index 8f068dbf..46afddf0 100644 --- a/common/FileWriterSettings.cpp +++ b/common/FileWriterSettings.cpp @@ -14,6 +14,7 @@ FileWriterSettings &FileWriterSettings::HDF5MasterFormatVersion(FileWriterFormat case FileWriterFormat::DataOnly: case FileWriterFormat::NXmxLegacy: case FileWriterFormat::NXmxVDS: + case FileWriterFormat::NXmxIntegrated: case FileWriterFormat::TIFF: case FileWriterFormat::CBF: case FileWriterFormat::NoFile: diff --git a/frame_serialize/CBORStream2Deserializer.cpp b/frame_serialize/CBORStream2Deserializer.cpp index 98676334..90ddedb2 100644 --- a/frame_serialize/CBORStream2Deserializer.cpp +++ b/frame_serialize/CBORStream2Deserializer.cpp @@ -963,6 +963,7 @@ namespace { switch (tmp) { case FileWriterFormat::DataOnly: case FileWriterFormat::NXmxLegacy: + case FileWriterFormat::NXmxIntegrated: case FileWriterFormat::NXmxVDS: case FileWriterFormat::CBF: case FileWriterFormat::TIFF: @@ -1022,7 +1023,7 @@ namespace { if (j.contains("overwrite")) message.overwrite = j["overwrite"]; if (j.contains("xfel_pulse_id")) - message.overwrite = j["xfel_pulse_id"]; + message.xfel_pulse_id = j["xfel_pulse_id"]; if (j.contains("file_format")) message.file_format = ProcessHDF5Format(j["file_format"]); if (j.contains("poni_rot1")) diff --git a/tests/HDF5WritingTest.cpp b/tests/HDF5WritingTest.cpp index 23be0d97..251a234f 100644 --- a/tests/HDF5WritingTest.cpp +++ b/tests/HDF5WritingTest.cpp @@ -817,6 +817,274 @@ TEST_CASE("HDF5Writer_Link_VDS", "[HDF5][Full]") { REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); } +TEST_CASE("HDF5Writer_NXmxIntegrated_Basic", "[HDF5][Full]") { + DiffractionExperiment x(DetJF(1)); + + x.ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION) + .FilePrefix("integrated_basic"); + x.SetFileWriterFormat(FileWriterFormat::NXmxIntegrated).OverwriteExistingFiles(true); + + // NXmxIntegrated forces all images into one file + REQUIRE(x.GetImagesPerFile() == x.GetImageNum()); + + { + RegisterHDF5Filter(); + + StartMessage start_message; + x.FillMessage(start_message); + + REQUIRE(start_message.file_format == FileWriterFormat::NXmxIntegrated); + // images_per_file should equal total images for integrated + REQUIRE(start_message.images_per_file == x.GetImageNum()); + + EndMessage end_message; + end_message.max_image_number = x.GetImageNum(); + + FileWriter writer(start_message); + std::vector image(x.GetPixelsNum(), 42); + + for (int i = 0; i < x.GetImageNum(); i++) { + DataMessage message{}; + message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum()); + message.number = i; + REQUIRE_NOTHROW(writer.Write(message)); + } + + writer.WriteHDF5(end_message); + auto stats = writer.Finalize(); + + // All images in one file — only one stats entry + REQUIRE(stats.size() == 1); + REQUIRE(stats[0].total_images == x.GetImageNum()); + } + + // Single integrated file, no separate master or data files + REQUIRE(std::filesystem::exists("integrated_basic.h5")); + REQUIRE(!std::filesystem::exists("integrated_basic_master.h5")); + REQUIRE(!std::filesystem::exists("integrated_basic_data_000001.h5")); + + { + HDF5ReadOnlyFile file("integrated_basic.h5"); + + // Data should be directly in the file + std::unique_ptr dataset; + REQUIRE_NOTHROW(dataset = std::make_unique(file, "/entry/data/data")); + HDF5DataSpace file_space(*dataset); + REQUIRE(file_space.GetNumOfDimensions() == 3); + REQUIRE(file_space.GetDimensions()[0] == x.GetImageNum()); + REQUIRE(file_space.GetDimensions()[1] == x.GetYPixelsNum()); + REQUIRE(file_space.GetDimensions()[2] == x.GetXPixelsNum()); + + // Master metadata should also be present + REQUIRE_NOTHROW(dataset = std::make_unique(file, "/entry/instrument/detector/beam_center_x")); + REQUIRE(dataset->ReadScalar() == Catch::Approx(x.GetBeamX_pxl())); + + // No external links (unlike NXmxLegacy) + REQUIRE_THROWS(std::make_unique(file, "/entry/data/data_000001")); + } + + // No leftover HDF5 objects + REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); + remove("integrated_basic.h5"); +} + +TEST_CASE("HDF5Writer_NXmxIntegrated_WithSpots", "[HDF5][Full]") { + DiffractionExperiment x(DetJF(1)); + + x.ImagesPerTrigger(3).Compression(CompressionAlgorithm::NO_COMPRESSION) + .FilePrefix("integrated_spots"); + x.SetFileWriterFormat(FileWriterFormat::NXmxIntegrated).OverwriteExistingFiles(true); + + { + RegisterHDF5Filter(); + + StartMessage start_message; + x.FillMessage(start_message); + + EndMessage end_message; + end_message.max_image_number = x.GetImageNum(); + + FileWriter writer(start_message); + std::vector image(x.GetPixelsNum(), 10); + + std::vector spots; + spots.push_back({10.0f, 20.0f, 100.0f}); + spots.push_back({30.0f, 40.0f, 200.0f}); + + for (int i = 0; i < x.GetImageNum(); i++) { + DataMessage message{}; + message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum()); + message.spots = spots; + message.number = i; + message.image_collection_efficiency = 1.0f; + REQUIRE_NOTHROW(writer.Write(message)); + } + + writer.WriteHDF5(end_message); + auto stats = writer.Finalize(); + REQUIRE(stats.size() == 1); + } + + REQUIRE(std::filesystem::exists("integrated_spots.h5")); + { + HDF5ReadOnlyFile file("integrated_spots.h5"); + + // Detector plugin data should exist in the same file + REQUIRE(file.Exists("/entry/detector")); + + // Image data should exist + std::unique_ptr dataset; + REQUIRE_NOTHROW(dataset = std::make_unique(file, "/entry/data/data")); + } + + REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); + remove("integrated_spots.h5"); +} + +TEST_CASE("HDF5Writer_NXmxIntegrated_ZeroImages", "[HDF5][Full]") { + DiffractionExperiment x(DetJF(1)); + + x.ImagesPerTrigger(5).Compression(CompressionAlgorithm::NO_COMPRESSION) + .FilePrefix("integrated_zero"); + x.SetFileWriterFormat(FileWriterFormat::NXmxIntegrated).OverwriteExistingFiles(true); + + { + RegisterHDF5Filter(); + + StartMessage start_message; + x.FillMessage(start_message); + + EndMessage end_message; + end_message.max_image_number = 0; + + FileWriter writer(start_message); + // Write no images — just finalize + writer.WriteHDF5(end_message); + auto stats = writer.Finalize(); + + // No data files created + REQUIRE(stats.empty()); + } + + // Master file should still exist with metadata + REQUIRE(std::filesystem::exists("integrated_zero.h5")); + { + HDF5ReadOnlyFile file("integrated_zero.h5"); + REQUIRE(file.Exists("/entry")); + // No data dataset since no images written + REQUIRE_THROWS(std::make_unique(file, "/entry/data/data")); + } + + REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); + remove("integrated_zero.h5"); +} + +TEST_CASE("HDF5Writer_NXmxIntegrated_AzInt", "[HDF5][Full]") { + DiffractionExperiment x(DetJF(1)); + + x.DetectorDistance_mm(50).BeamX_pxl(500).BeamY_pxl(500); + x.QSpacingForAzimInt_recipA(0.1).QRangeForAzimInt_recipA(0.1, 4.0); + x.ImagesPerTrigger(3).Compression(CompressionAlgorithm::NO_COMPRESSION) + .FilePrefix("integrated_azint"); + x.SetFileWriterFormat(FileWriterFormat::NXmxIntegrated).OverwriteExistingFiles(true); + + PixelMask pixel_mask(x); + AzimuthalIntegration mapping(x, pixel_mask); + + { + RegisterHDF5Filter(); + + StartMessage start_message; + x.FillMessage(start_message); + start_message.az_int_bin_to_q = mapping.GetBinToQ(); + start_message.az_int_phi_bin_count = mapping.GetAzimuthalBinCount(); + start_message.az_int_q_bin_count = mapping.GetQBinCount(); + + EndMessage end_message; + end_message.max_image_number = x.GetImageNum(); + + FileWriter writer(start_message); + std::vector image(x.GetPixelsNum(), 5); + + for (int i = 0; i < x.GetImageNum(); i++) { + DataMessage message{}; + message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum()); + message.az_int_profile = std::vector(mapping.GetBinNumber(), static_cast(i)); + message.number = i; + REQUIRE_NOTHROW(writer.Write(message)); + } + + writer.WriteHDF5(end_message); + auto stats = writer.Finalize(); + REQUIRE(stats.size() == 1); + } + + REQUIRE(std::filesystem::exists("integrated_azint.h5")); + { + HDF5ReadOnlyFile file("integrated_azint.h5"); + + // Azimuthal integration bin mapping should exist (written by plugin) + std::unique_ptr dataset; + REQUIRE_NOTHROW(dataset = std::make_unique(file, "/entry/azint/bin_to_q")); + + // Per-image azint data should exist + REQUIRE_NOTHROW(dataset = std::make_unique(file, "/entry/azint/image")); + HDF5DataSpace space(*dataset); + REQUIRE(space.GetNumOfDimensions() == 3); + REQUIRE(space.GetDimensions()[0] == x.GetImageNum()); + } + + REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); + remove("integrated_azint.h5"); +} + +TEST_CASE("HDF5Writer_NXmxIntegrated_OutOfOrder", "[HDF5][Full]") { + // Test that out-of-order image delivery works with NXmxIntegrated + DiffractionExperiment x(DetJF(1)); + + x.ImagesPerTrigger(5).Compression(CompressionAlgorithm::NO_COMPRESSION) + .FilePrefix("integrated_ooo"); + x.SetFileWriterFormat(FileWriterFormat::NXmxIntegrated).OverwriteExistingFiles(true); + + { + RegisterHDF5Filter(); + + StartMessage start_message; + x.FillMessage(start_message); + + EndMessage end_message; + end_message.max_image_number = x.GetImageNum(); + + FileWriter writer(start_message); + std::vector image(x.GetPixelsNum(), 7); + + // Write images out of order + std::vector order = {3, 1, 4, 0, 2}; + for (int idx : order) { + DataMessage message{}; + message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum()); + message.number = idx; + REQUIRE_NOTHROW(writer.Write(message)); + } + + writer.WriteHDF5(end_message); + auto stats = writer.Finalize(); + REQUIRE(stats.size() == 1); + REQUIRE(stats[0].total_images == 5); + } + + REQUIRE(std::filesystem::exists("integrated_ooo.h5")); + { + HDF5ReadOnlyFile file("integrated_ooo.h5"); + std::unique_ptr dataset; + REQUIRE_NOTHROW(dataset = std::make_unique(file, "/entry/data/data")); + HDF5DataSpace file_space(*dataset); + REQUIRE(file_space.GetDimensions()[0] == 5); + } + + REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); + remove("integrated_ooo.h5"); +} TEST_CASE("HDF5Writer_NoMasterFile", "[HDF5][Full]") { DiffractionExperiment x(DetJF(1)); diff --git a/tests/JFJochReceiverProcessingTest.cpp b/tests/JFJochReceiverProcessingTest.cpp index 9156ad11..c9ff8de5 100644 --- a/tests/JFJochReceiverProcessingTest.cpp +++ b/tests/JFJochReceiverProcessingTest.cpp @@ -1749,4 +1749,87 @@ TEST_CASE("JFJochIntegrationTest_TCP_calibration", "[JFJochReceiver]") { REQUIRE(!service.GetProgress().has_value()); REQUIRE_NOTHROW(writer_future.get()); -} \ No newline at end of file +} + +TEST_CASE("JFJochIntegrationTest_TCP_lysozyme_spot_and_index_single_file", "[JFJochReceiver]") { + Logger logger(Catch::getResultCapture().getCurrentTestName()); + + RegisterHDF5Filter(); + + const uint16_t nthreads = 4; + + DiffractionExperiment experiment(DetJF4M()); + experiment.ImagesPerTrigger(5).NumTriggers(1).UseInternalPacketGenerator(true).ImagesPerFile(2) + .FilePrefix("lyso_test_tcp_single_file").JungfrauConvPhotonCnt(false).SetFileWriterFormat(FileWriterFormat::NXmxIntegrated).OverwriteExistingFiles(true) + .DetectorDistance_mm(75).BeamY_pxl(1136).BeamX_pxl(1090).IncidentEnergy_keV(12.4) + .SetUnitCell(UnitCell{.a = 36.9, .b = 78.95, .c = 78.95, .alpha =90, .beta = 90, .gamma = 90}); + experiment.SampleTemperature_K(123.0).RingCurrent_mA(115); + + PixelMask pixel_mask(experiment); + + // Load example image + HDF5ReadOnlyFile data("../../tests/test_data/compression_benchmark.h5"); + HDF5DataSet dataset(data, "/entry/data/data"); + HDF5DataSpace file_space(dataset); + + REQUIRE(file_space.GetDimensions()[2] == experiment.GetXPixelsNum()); + REQUIRE(file_space.GetDimensions()[1] == experiment.GetYPixelsNum()); + std::vector image_conv (file_space.GetDimensions()[1] * file_space.GetDimensions()[2]); + + std::vector start = {4,0,0}; + std::vector file_size = {1, file_space.GetDimensions()[1], file_space.GetDimensions()[2]}; + dataset.ReadVector(image_conv, start, file_size); + + std::vector image_raw_geom(experiment.GetModulesNum() * RAW_MODULE_SIZE); + ConvertedToRawGeometry(experiment, image_raw_geom.data(), image_conv.data()); + logger.Info("Loaded image"); + + // Setup acquisition device + AcquisitionDeviceGroup aq_devices; + std::unique_ptr test = std::make_unique(0, 64); + for (int m = 0; m < experiment.GetModulesNum(); m++) + test->SetInternalGeneratorFrame((uint16_t *) image_raw_geom.data() + m * RAW_MODULE_SIZE, m); + + aq_devices.Add(std::move(test)); + + TCPStreamPusher pusher("tcp://127.0.0.1:9121", 1); + + TCPImagePuller puller("tcp://127.0.0.1:9121"); + StreamWriter writer(logger, puller); + auto writer_future = std::async(std::launch::async, &StreamWriter::Run, &writer); + + JFJochReceiverService service(aq_devices, logger, pusher); + service.NumThreads(nthreads); + service.Indexing(experiment.GetIndexingSettings()); + + // No progress value at the start of measurement + REQUIRE(!service.GetProgress().has_value()); + + SpotFindingSettings settings = DiffractionExperiment::DefaultDataProcessingSettings(); + settings.signal_to_noise_threshold = 2.5; + settings.photon_count_threshold = 5; + settings.min_pix_per_spot = 1; + settings.max_pix_per_spot = 200; + settings.high_resolution_limit = 2.0; + settings.low_resolution_limit = 50.0; + service.SetSpotFindingSettings(settings); + + service.Start(experiment, pixel_mask, nullptr); + auto receiver_out = service.Stop(); + + CHECK(receiver_out.efficiency == 1.0); + REQUIRE(receiver_out.status.indexing_rate); + CHECK(receiver_out.status.indexing_rate.value() == 1.0); + CHECK(receiver_out.status.images_sent == experiment.GetImageNum()); + CHECK(receiver_out.writer_err.empty()); + CHECK(!receiver_out.status.cancelled); + + // No progress value at the end of measurement + REQUIRE(!service.GetProgress().has_value()); + + REQUIRE_NOTHROW(writer_future.get()); + + auto ack = pusher.GetImagesWritten(); + REQUIRE(ack.has_value()); + CHECK(ack == experiment.GetImageNum()); +} diff --git a/tools/jfjoch_hdf5_test.cpp b/tools/jfjoch_hdf5_test.cpp index e9874b7e..f9f4d2d7 100644 --- a/tools/jfjoch_hdf5_test.cpp +++ b/tools/jfjoch_hdf5_test.cpp @@ -26,7 +26,7 @@ int main(int argc, char **argv) { std::optional rotation; int opt; - while ((opt = getopt(argc, argv, "o:n:Vf:R:")) != -1) { + while ((opt = getopt(argc, argv, "o:n:Vf:R:S")) != -1) { switch (opt) { case 'o': prefix = optarg; @@ -37,6 +37,9 @@ int main(int argc, char **argv) { case 'V': format = FileWriterFormat::NXmxVDS; break; + case 'S': + format = FileWriterFormat::NXmxIntegrated; + break; case 'R': rotation = atof(optarg); break; diff --git a/writer/FileWriter.cpp b/writer/FileWriter.cpp index 213b060e..c7503786 100644 --- a/writer/FileWriter.cpp +++ b/writer/FileWriter.cpp @@ -25,6 +25,7 @@ FileWriter::FileWriter(const StartMessage &request) switch (format) { case FileWriterFormat::NXmxLegacy: case FileWriterFormat::NXmxVDS: + case FileWriterFormat::NXmxIntegrated: CreateHDF5MasterFile(request); break; case FileWriterFormat::CBF: @@ -42,6 +43,7 @@ void FileWriter::Write(const DataMessage &msg) { case FileWriterFormat::DataOnly: case FileWriterFormat::NXmxLegacy: case FileWriterFormat::NXmxVDS: + case FileWriterFormat::NXmxIntegrated: WriteHDF5(msg); break; case FileWriterFormat::CBF: @@ -70,8 +72,8 @@ void FileWriter::WriteHDF5(const DataMessage& msg) { if (msg.number < 0) throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, "No support for negative images"); - const uint64_t file_number = msg.number / start_message.images_per_file; - const uint64_t image_number = msg.number % start_message.images_per_file; + const uint64_t file_number = (start_message.images_per_file == 0) ? 0 : msg.number / start_message.images_per_file; + const uint64_t image_number = (start_message.images_per_file == 0) ? msg.number : msg.number % start_message.images_per_file; if (closed_files.contains(file_number)) return; @@ -79,9 +81,11 @@ void FileWriter::WriteHDF5(const DataMessage& msg) { if (files.size() <= file_number) files.resize(file_number + 1); - if (!files[file_number]) + if (!files[file_number]) { files[file_number] = std::make_unique(start_message, file_number); - + if (format == FileWriterFormat::NXmxIntegrated && master_file) + files[file_number]->CreateFile(msg, master_file->GetFile()); + } files[file_number]->Write(msg, image_number); if (files[file_number]->GetNumImages() == start_message.images_per_file) { @@ -121,13 +125,13 @@ void FileWriter::CloseOldFiles(uint64_t current_image_number) { std::vector FileWriter::Finalize() { std::lock_guard lock(hdf5_mutex); - if (master_file) - master_file.reset(); for (uint64_t f = 0; f < files.size(); ++f) { if (files[f] && !closed_files.contains(f)) CloseFile(f); } + if (master_file) + master_file.reset(); return stats; } diff --git a/writer/HDF5DataFile.cpp b/writer/HDF5DataFile.cpp index 5e99db29..563f3615 100644 --- a/writer/HDF5DataFile.cpp +++ b/writer/HDF5DataFile.cpp @@ -77,7 +77,7 @@ std::optional HDF5DataFile::Close() { } data_file.reset(); - if (!std::filesystem::exists(filename.c_str()) || overwrite) + if (manage_file && (!std::filesystem::exists(filename.c_str()) || overwrite)) std::rename(tmp_filename.c_str(), filename.c_str()); closed = true; @@ -102,7 +102,7 @@ HDF5DataFile::~HDF5DataFile() { } } -void HDF5DataFile::CreateFile(const DataMessage& msg) { +void HDF5DataFile::CreateFile(const DataMessage& msg, std::shared_ptr in_data_file) { HDF5Dcpl dcpl; HDF5DataType data_type(msg.image.GetMode()); @@ -130,7 +130,7 @@ void HDF5DataFile::CreateFile(const DataMessage& msg) { break; } - data_file = std::make_unique(tmp_filename); + data_file = in_data_file; HDF5Group(*data_file, "/entry").NXClass("NXentry"); HDF5Group(*data_file, "/entry/data").NXClass("NXdata"); @@ -149,11 +149,10 @@ void HDF5DataFile::Write(const DataMessage &msg, uint64_t image_number) { if (image_number >= images_per_file) throw JFJochException(JFJochExceptionCategory::FileWriteError, "Image number out of bounds"); - bool new_file = false; if (!data_file) { - CreateFile(msg); - new_file = true; + manage_file = true; + CreateFile(msg, std::make_shared(tmp_filename)); } if (new_file || (static_cast(image_number) > max_image_number)) { @@ -161,6 +160,7 @@ void HDF5DataFile::Write(const DataMessage &msg, uint64_t image_number) { timestamp.resize(max_image_number + 1); exptime.resize(max_image_number + 1); number.resize(max_image_number + 1); + new_file = false; } nimages++; diff --git a/writer/HDF5DataFile.h b/writer/HDF5DataFile.h index f2fe3bec..4a740bfe 100644 --- a/writer/HDF5DataFile.h +++ b/writer/HDF5DataFile.h @@ -25,7 +25,7 @@ class HDF5DataFile { std::string filename; std::string tmp_filename; - std::unique_ptr data_file = nullptr; + std::shared_ptr data_file = nullptr; std::unique_ptr data_set = nullptr; std::unique_ptr data_set_image_number = nullptr; std::vector> plugins; @@ -47,13 +47,16 @@ class HDF5DataFile { bool overwrite = false; int64_t file_number; - void CreateFile(const DataMessage& msg); + bool new_file = true; + bool manage_file = false; public: HDF5DataFile(const StartMessage &msg, uint64_t file_number); ~HDF5DataFile(); std::optional Close(); void Write(const DataMessage& msg, uint64_t image_number); size_t GetNumImages() const; + + void CreateFile(const DataMessage& msg, std::shared_ptr data_file); }; #endif //HDF5DATAFILE_H diff --git a/writer/HDF5DataFilePluginAzInt.cpp b/writer/HDF5DataFilePluginAzInt.cpp index 49847188..1f032e04 100644 --- a/writer/HDF5DataFilePluginAzInt.cpp +++ b/writer/HDF5DataFilePluginAzInt.cpp @@ -25,14 +25,13 @@ void HDF5DataFilePluginAzInt::OpenFile(HDF5File &data_file, const DataMessage &m data_file.SaveVector("/entry/azint/bin_to_phi", az_int_bin_to_phi, dim); az_int_image.reserve(images_per_file * azimuthal_bins * q_bins); - az_int_image.resize(msg.number * azimuthal_bins * q_bins); } void HDF5DataFilePluginAzInt::Write(const DataMessage &msg, uint64_t image_number) { if (az_int_bin_to_q.empty() || q_bins <= 0 || azimuthal_bins <= 0) return; - if (static_cast(image_number) >= max_image_number) { + if (image_number >= max_image_number || (max_image_number == 0)) { max_image_number = image_number; az_int_image.resize((max_image_number + 1) * azimuthal_bins * q_bins); } diff --git a/writer/HDF5NXmx.cpp b/writer/HDF5NXmx.cpp index 7889819b..adb18840 100644 --- a/writer/HDF5NXmx.cpp +++ b/writer/HDF5NXmx.cpp @@ -11,9 +11,17 @@ #include "../common/time_utc.h" #include "gemmi/symmetry.hpp" +namespace { + std::string GenFilename(const StartMessage &start) { + if (start.file_format == FileWriterFormat::NXmxIntegrated) + return fmt::format("{:s}.h5", start.file_prefix); + return fmt::format("{:s}_master.h5", start.file_prefix); + } +} + NXmx::NXmx(const StartMessage &start) : start_message(start), - filename(start.file_prefix + "_master.h5") { + filename(GenFilename(start)) { uint64_t tmp_suffix; try { if (!start.arm_date.empty()) @@ -31,7 +39,7 @@ NXmx::NXmx(const StartMessage &start) bool v1_10 = (start.file_format == FileWriterFormat::NXmxVDS); - hdf5_file = std::make_unique(tmp_filename, v1_10); + hdf5_file = std::make_shared(tmp_filename, v1_10); hdf5_file->Attr("file_name", filename); hdf5_file->Attr("HDF5_Version", hdf5_version()); HDF5Group(*hdf5_file, "/entry").NXClass("NXentry").SaveScalar("definition", "NXmx"); @@ -52,6 +60,8 @@ NXmx::~NXmx() { std::rename(tmp_filename.c_str(), filename.c_str()); } + + std::string HDF5Metadata::DataFileName(const StartMessage &msg, int64_t file_number) { if (file_number < 0) throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, @@ -184,7 +194,7 @@ void NXmx::LinkToData_VDS(const StartMessage &start, const EndMessage &end) { if (!start.az_int_bin_to_q.empty()) { size_t azimuthal_bins = start.az_int_phi_bin_count.value_or(1); size_t q_bins = start.az_int_q_bin_count.value_or(1); - if (q_bins > 0 & azimuthal_bins > 0) { + if (q_bins > 0 && azimuthal_bins > 0) { VDS(start, "/entry/azint/image", {total_images, azimuthal_bins, q_bins}, HDF5DataType(0.0f)); @@ -657,13 +667,17 @@ void NXmx::AzimuthalIntegration(const StartMessage &start, const EndMessage &end HDF5Group az_int_group(*hdf5_file, "/entry/azint"); az_int_group.NXClass("NXcollection"); - az_int_group.SaveVector("bin_to_q", start.az_int_bin_to_q, dim)->Units("reciprocal Angstrom"); - if (!start.az_int_bin_to_two_theta.empty()) - az_int_group.SaveVector("bin_to_two_theta", start.az_int_bin_to_two_theta, dim)->Units("degrees"); - if (!start.az_int_bin_to_phi.empty()) - az_int_group.SaveVector("bin_to_phi", start.az_int_bin_to_phi, dim)->Units("degrees"); - for (const auto &[x,y]: end.az_int_result) - az_int_group.SaveVector(x, y, dim); + if (start.file_format != FileWriterFormat::NXmxIntegrated) { + az_int_group.SaveVector("bin_to_q", start.az_int_bin_to_q, dim)->Units("reciprocal Angstrom"); + if (!start.az_int_bin_to_two_theta.empty()) + az_int_group.SaveVector("bin_to_two_theta", start.az_int_bin_to_two_theta, dim)->Units("degrees"); + if (!start.az_int_bin_to_phi.empty()) + az_int_group.SaveVector("bin_to_phi", start.az_int_bin_to_phi, dim)->Units("degrees"); + } + for (const auto &[x,y]: end.az_int_result) { + if (x != "image") + az_int_group.SaveVector(x, y, dim); + } } } @@ -693,10 +707,17 @@ void NXmx::Finalize(const EndMessage &end) { AzimuthalIntegration(start_message, end); ADUHistogram(end); - if (start_message.file_format == FileWriterFormat::NXmxVDS) - LinkToData_VDS(start_message, end); - else if (start_message.file_format == FileWriterFormat::NXmxLegacy) - LinkToData(start_message, end); + switch (start_message.file_format.value_or(FileWriterFormat::NXmxLegacy)) { + case FileWriterFormat::NXmxLegacy: + LinkToData(start_message, end); + break; + case FileWriterFormat::NXmxVDS: + LinkToData_VDS(start_message, end); + break; + case FileWriterFormat::NXmxIntegrated: + default: + break; + } if (end.rotation_lattice) SaveVector(*hdf5_file, "/entry/MX/rotationLatticeIndexed", end.rotation_lattice->GetVector()) @@ -732,6 +753,6 @@ void NXmx::UserData(const StartMessage &start) { } } -HDF5File *NXmx::GetFile() { - return hdf5_file.get(); +std::shared_ptr NXmx::GetFile() { + return hdf5_file; } diff --git a/writer/HDF5NXmx.h b/writer/HDF5NXmx.h index ec2bc389..a02bfe2b 100644 --- a/writer/HDF5NXmx.h +++ b/writer/HDF5NXmx.h @@ -13,7 +13,7 @@ namespace HDF5Metadata { } class NXmx { - std::unique_ptr hdf5_file; + std::shared_ptr hdf5_file; const StartMessage start_message; const std::string filename; std::string tmp_filename; @@ -59,7 +59,7 @@ public: void Finalize(const EndMessage &end); void WriteCalibration(const CompressedImage &image); - HDF5File *GetFile(); + std::shared_ptr GetFile(); }; #endif //JUNGFRAUJOCH_HDF5NXMX_H