From 6fa1797f8e8aaa162238f3d8a554e13c20fd860b Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Fri, 20 Nov 2026 02:41:11 +0100 Subject: [PATCH] jfjoch_viewer: keep showing the images of a raw sweep after processing it A processing job on a CBF, marCCD, SMV or SFRM sweep writes a _process.h5 that holds the per-image analysis but no images: there is nothing in a directory of frames for it to link to. The viewer showed that run by reading the file on its own, so after processing the image area stayed empty. JFJochHDF5Reader::ReadFile now takes the reader of the sweep the file was made from; for a file without pixels each image is read from that sweep (frame /entry/detector/number[i]) and the file's analysis is laid over it. The viewer passes its open raw reader when it shows such a run. HDF5 inputs are unchanged: their runs already link the original images. Tests: JFJochHDF5Reader_ProcessFileOfRawSweep_PixelsFromSweep (a run over one frame of a two-frame miniCBF sweep shows that frame's pixels under the run's analysis) and OpenableFilePatterns (the browser/dialog wildcards). The viewer documentation said the process file carried the images; corrected. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HVeDQrPqZ52Tp6rzzZGxf3 --- docs/JFJOCH_VIEWER.md | 13 +++-- reader/JFJochHDF5Reader.cpp | 28 ++++++++-- reader/JFJochHDF5Reader.h | 8 ++- reader/JFJochReader.h | 4 +- tests/JFJochReaderTest.cpp | 86 +++++++++++++++++++++++++++++ viewer/JFJochImageReadingWorker.cpp | 2 +- viewer/JFJochImageReadingWorker.h | 7 ++- 7 files changed, 130 insertions(+), 18 deletions(-) diff --git a/docs/JFJOCH_VIEWER.md b/docs/JFJOCH_VIEWER.md index 832f98dfc..701a83db9 100644 --- a/docs/JFJOCH_VIEWER.md +++ b/docs/JFJOCH_VIEWER.md @@ -25,12 +25,14 @@ newer on an Apple Silicon Mac** (M1 and newer; Intel Macs are not supported) and - Opens HDF5 files written by [`jfjoch_writer`](JFJOCH_WRITER.md) (`*_master.h5`) and the `*_process.h5` files produced by [`rugnux`](RUGNUX.md). It also opens NXmx files written by DECTRIS detectors, though that path has had only limited testing. -- Opens PILATUS miniCBF, marCCD and SMV rotation sweeps (the formats listed under +- Opens PILATUS miniCBF, marCCD, SMV and Bruker SFRM rotation sweeps (the formats listed under [Input formats](RUGNUX_FORMATS.md)). These store one frame per file, so naming any frame opens - the whole sweep it belongs to. A raw frame carries the images and the geometry but no analysis + the whole sweep it belongs to; frames numbered in their extension (`name.001`, `name.0001`) are + listed like any other. A raw frame carries the images and the geometry but no analysis results, so the spot, reflection and per-image plot panels stay empty until something is computed. - A processing job on such a sweep writes the images into its `_process.h5`, and its run is shown from - that file like a run of an HDF5 dataset; *Original* goes back to the sweep. + A processing job on such a sweep writes its per-image analysis, without the images, into its + `_process.h5`; its run shows the sweep's images with that analysis over them, and *Original* goes + back to the sweep alone. - Runs an **embedded data-processing pipeline** — the same analysis code as the rest of Jungfraujoch — performing spot finding, indexing and integration on the displayed image, with the result drawn over it. This interactive analysis is not written anywhere. @@ -239,7 +241,8 @@ keyboard without them `Home` / `End` are `Fn`+`←` / `Fn`+`→` and `Page Up` / ## Opening data -- **File ▸ Open** (`Ctrl+O`) — open a local HDF5 file, or any frame of a miniCBF, marCCD or SMV sweep. +- **File ▸ Open** (`Ctrl+O`) — open a local HDF5 file, or any frame of a miniCBF, marCCD, SMV or SFRM + sweep. The *All supported* filter, and the file browser panel, list every file the viewer opens. - **File ▸ Open HTTP** (`Ctrl+H`) — connect to a `jfjoch_broker` HTTP endpoint to follow a live collection. The dialog defaults to host `localhost` and port `8080`; these defaults can be overridden with the environment variables `JUNGFRAUJOCH_HTTP_HOST` and `JUNGFRAUJOCH_HTTP_PORT`. diff --git a/reader/JFJochHDF5Reader.cpp b/reader/JFJochHDF5Reader.cpp index 22ffdb7ea..ef0bcab6b 100644 --- a/reader/JFJochHDF5Reader.cpp +++ b/reader/JFJochHDF5Reader.cpp @@ -4,7 +4,7 @@ #include "JFJochHDF5Reader.h" #include "../common/JFJochException.h" -void JFJochHDF5Reader::ReadFile(const std::string &filename) { +void JFJochHDF5Reader::ReadFile(const std::string &filename, JFJochReader *pixels) { std::unique_lock ul(hdf5_mutex); image_source_.Clear(); snapshots_.clear(); @@ -16,6 +16,7 @@ void JFJochHDF5Reader::ReadFile(const std::string &filename) { auto open_result = metadata->Open(filename, default_experiment); has_pixels = open_result.image_layout.format != FileWriterFormat::NoFile; + pixel_reader_ = has_pixels ? nullptr : pixels; image_source_.Configure(std::move(open_result.image_layout)); // Original file: per-image metadata is co-located with the pixels. metadata->UseImageSourceForMetadata(&image_source_); @@ -44,6 +45,7 @@ uint64_t JFJochHDF5Reader::GetNumberOfImages() const { void JFJochHDF5Reader::Close() { std::unique_lock ul(hdf5_mutex); image_source_.Clear(); + pixel_reader_ = nullptr; snapshots_.clear(); active_metadata_.reset(); active_snapshot_.clear(); @@ -88,20 +90,34 @@ bool JFJochHDF5Reader::LoadImage_i(std::shared_ptr &dataset std::vector &buffer, int64_t image_number, bool update_dataset) { - std::unique_lock ul(hdf5_mutex); (void) update_dataset; - if (!dataset || !has_pixels) + if (!dataset) return false; + + // A file without images takes its pixels from the sweep it was processed from. That reader is asked + // outside hdf5_mutex, which it may take itself. + int64_t metadata_number = image_number; + if (!has_pixels) { + if (!pixel_reader_) + return false; + if (!dataset->source_image_number.empty()) + metadata_number = dataset->source_image_number.at(image_number); + auto pixel_dataset = std::const_pointer_cast(pixel_reader_->GetDataset()); + if (!pixel_reader_->LoadImage_i(pixel_dataset, message, buffer, metadata_number, false)) + return false; + } + + std::unique_lock ul(hdf5_mutex); if (!active_metadata_) throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Cannot load image if file not loaded"); // Pixels from the shared image source, per-image metadata from the active snapshot. - auto loc = GetImageLocation(image_number); - message.image = image_source_.ReadImageAt(buffer, loc); + if (has_pixels) + message.image = image_source_.ReadImageAt(buffer, GetImageLocation(image_number)); message.number = image_number; - active_metadata_->FillPerImage(message, image_number, dataset); + active_metadata_->FillPerImage(message, metadata_number, dataset); return true; } diff --git a/reader/JFJochHDF5Reader.h b/reader/JFJochHDF5Reader.h index 053213873..4b659e13b 100644 --- a/reader/JFJochHDF5Reader.h +++ b/reader/JFJochHDF5Reader.h @@ -32,8 +32,9 @@ class JFJochHDF5Reader : public JFJochReader { std::string active_snapshot_; size_t number_of_images = 0; // False for a file of per-image analysis without the images (a _process.h5 of an input it could - // not link to): its images have metadata, but no pixels to load. + // not link to): its images have metadata, and their pixels come from pixel_reader_ where one is given. bool has_pixels = false; + JFJochReader *pixel_reader_ = nullptr; bool LoadImage_i(std::shared_ptr &dataset, DataMessage &message, @@ -46,7 +47,10 @@ class JFJochHDF5Reader : public JFJochReader { public: ~JFJochHDF5Reader() override = default; - void ReadFile(const std::string &filename); + // pixels is the reader of the sweep the file was processed from, for a file that holds no images of + // its own - the _process.h5 of a CBF, marCCD, SMV or SFRM sweep: image i of the file is then frame + // /entry/detector/number[i] of that sweep. The caller keeps it open while this file is. + void ReadFile(const std::string &filename, JFJochReader *pixels = nullptr); uint64_t GetNumberOfImages() const override; diff --git a/reader/JFJochReader.h b/reader/JFJochReader.h index 700b2fe32..20cbf5ece 100644 --- a/reader/JFJochReader.h +++ b/reader/JFJochReader.h @@ -17,8 +17,10 @@ #include "JFJochReaderSpots.h" class JFJochReader { - // Reads each of its inputs' frames through them (JFJochMultiReader.h). + // Read each of their inputs' frames through them (JFJochMultiReader.h), or the frames of the sweep a + // file without pixels was made from (JFJochHDF5Reader.h). friend class JFJochMultiReader; + friend class JFJochHDF5Reader; mutable std::mutex m; std::shared_ptr dataset; diff --git a/tests/JFJochReaderTest.cpp b/tests/JFJochReaderTest.cpp index 70406c4b5..9517b96fc 100644 --- a/tests/JFJochReaderTest.cpp +++ b/tests/JFJochReaderTest.cpp @@ -13,6 +13,7 @@ #include "../reader/JFJochSMVReader.h" #include "../reader/JFJochXTCReader.h" #include "../reader/MiniCBF.h" +#include "../reader/FileNamePatterns.h" #include "../rugnux/RugnuxDefaults.h" #include "../compression/JFJochCompressor.h" #include "../frame_serialize/CBORStream2Serializer.h" @@ -26,6 +27,7 @@ #include #include #include +#include #include TEST_CASE("HDF5DataType_Sign","[HDF5]") { @@ -5082,6 +5084,90 @@ TEST_CASE("JFJochCBFReader_GzippedSweep", "[HDF5][Full]") { } } +// The _process.h5 of a raw sweep holds the per-image analysis but not the images: there is nothing +// in a directory of CBFs for it to link to. Opened over the sweep's reader, it shows the sweep's +// pixels under its own analysis, image i being the sweep frame it records as /entry/detector/number[i]. +TEST_CASE("JFJochHDF5Reader_ProcessFileOfRawSweep_PixelsFromSweep", "[HDF5][Full][portable]") { + RegisterHDF5Filter(); + const int64_t nx = 24, ny = 16; + WriteMiniCBFSweep("cbfproc", "# Oscillation_axis OMEGA\n", nx, ny); + JFJochCBFReader sweep; + REQUIRE_NOTHROW(sweep.ReadFiles("cbfproc_0001.cbf")); + REQUIRE(sweep.GetNumberOfImages() == 2); + + // A run over the second frame only, as rugnux writes one for a non-HDF5 input: no images. + { + DiffractionExperiment x(sweep.GetDataset()->experiment); + x.FilePrefix("cbfproc_run").ImagesPerTrigger(1).OverwriteExistingFiles(true) + .SetFileWriterFormat(FileWriterFormat::NXmxIntegrated); + StartMessage start_message; + x.FillMessage(start_message); + start_message.master_suffix = "process"; + start_message.write_images = false; + FileWriter file_set(start_message); + ScanResultGenerator generator(x); + const std::vector unused(nx * ny, 0); + DataMessage message{}; + message.image = CompressedImage(unused, nx, ny); + message.number = 0; + message.original_number = 1; + message.indexing_result = true; + message.bkg_estimate = 42.0; + REQUIRE_NOTHROW(file_set.WriteHDF5(message)); + generator.Add(message); + EndMessage end_message; + end_message.max_image_number = 1; + generator.FillEndMessage(end_message); + file_set.WriteHDF5(end_message); + file_set.Finalize(); + } + + { + JFJochHDF5Reader alone; + REQUIRE_NOTHROW(alone.ReadFile("cbfproc_run_process.h5")); + REQUIRE(alone.GetNumberOfImages() == 1); + CHECK(!alone.LoadImage(0)); // no pixels of its own + + JFJochHDF5Reader run; + REQUIRE_NOTHROW(run.ReadFile("cbfproc_run_process.h5", &sweep)); + REQUIRE(run.GetNumberOfImages() == 1); + const auto image = run.LoadImage(0); + REQUIRE(image); + CHECK(image->Image().size() == static_cast(nx * ny)); + CHECK(image->ImageData().number == 0); + REQUIRE(image->ImageData().bkg_estimate.has_value()); + CHECK(image->ImageData().bkg_estimate.value() == Catch::Approx(42.0)); + REQUIRE(image->ImageData().indexing_result.has_value()); + CHECK(image->ImageData().indexing_result.value()); + } + sweep.Close(); + RemoveMiniCBFSweep("cbfproc"); + remove("cbfproc_run_process.h5"); + REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); +} + +// The viewer lists a file when its name matches one of OPENABLE_FILE_PATTERNS (QDir wildcards, +// matched without regard to case, as QFileSystemModel does by default). +TEST_CASE("OpenableFilePatterns", "[portable]") { + auto listed = [](const std::string &name) { + for (const auto &p: OPENABLE_FILE_PATTERNS) { + std::string re; + for (char c: p) + re += (c == '*') ? std::string(".*") : (c == '.') ? std::string("\\.") : std::string(1, c); + if (std::regex_match(name, std::regex(re, std::regex::icase))) + return true; + } + return false; + }; + for (const char *name: {"run_master.h5", "run_process.h5", "run_0001.cbf", "run_0001.cbf.gz", + "formal.001", "frame.0001", "frame.00001", "x_001.img", "x_001.mccd", + "x_001.osc", "x_001.smv", "x_0001.sfrm", "X_0001.CBF", "r0001.xtc", + "run.cxi"}) + CHECK(listed(name)); + for (const char *name: {"run_data_000001.h5", "XDS.INP", "notes.txt", "run.mtz", "file.1"}) + CHECK_FALSE(listed(name)); +} + // The DECTRIS "hdf5 nexus v2024.2 nxmx" format: /entry/data/data is 4D, [image, channel, y, x], a // VDS over the data files, with the pixel mask kept per channel. Only the first channel is read. TEST_CASE("JFJochReader_DECTRISMultichannel", "[HDF5][Full]") { diff --git a/viewer/JFJochImageReadingWorker.cpp b/viewer/JFJochImageReadingWorker.cpp index 9f9ae9c8d..0399deb5c 100644 --- a/viewer/JFJochImageReadingWorker.cpp +++ b/viewer/JFJochImageReadingWorker.cpp @@ -1266,7 +1266,7 @@ void JFJochImageReadingWorker::ActivateSnapshot_i(const QString &name) { if (id == "Original") { file_reader = raw_reader_; } else { - hdf5_reader.ReadFile(raw_runs_.at(id).path); + hdf5_reader.ReadFile(raw_runs_.at(id).path, raw_reader_); // analysis from the run, pixels from the sweep raw_runs_.at(id).dataset = hdf5_reader.GetDataset(); file_reader = &hdf5_reader; } diff --git a/viewer/JFJochImageReadingWorker.h b/viewer/JFJochImageReadingWorker.h index e0a76c053..2cf374503 100644 --- a/viewer/JFJochImageReadingWorker.h +++ b/viewer/JFJochImageReadingWorker.h @@ -100,9 +100,10 @@ private: bool cbf_mode = false; // "a raw sweep is open", whichever of the three formats it is // The runs of a raw sweep. A run of an HDF5 file is a snapshot - its metadata read back over the - // file's own images - but a raw sweep has no HDF5 to hang one on. Its run's _process.h5 carries - // the images as well, so showing that run means reading that file (with hdf5_reader), and showing - // "Original" means reading the sweep again (raw_reader_). + // file's own images - but a raw sweep has no HDF5 to hang one on. Its run's _process.h5 holds the + // per-image analysis without the images, so showing that run means reading that file (with + // hdf5_reader) over the sweep's pixels (raw_reader_, kept open), and showing "Original" means + // reading the sweep alone again. struct RawRun { std::string path; std::shared_ptr dataset;