diff --git a/reader/CMakeLists.txt b/reader/CMakeLists.txt index d2488fa27..06f304776 100644 --- a/reader/CMakeLists.txt +++ b/reader/CMakeLists.txt @@ -1,7 +1,5 @@ ADD_LIBRARY(JFJochReader STATIC JFJochReader.cpp JFJochReader.h - ReadAhead.cpp - ReadAhead.h JFJochHDF5Reader.cpp JFJochHDF5Reader.h SweepLayout.cpp diff --git a/reader/HDF5ImageLocator.cpp b/reader/HDF5ImageLocator.cpp index eed666c85..05d6f908e 100644 --- a/reader/HDF5ImageLocator.cpp +++ b/reader/HDF5ImageLocator.cpp @@ -4,8 +4,6 @@ #include "HDF5ImageLocator.h" #include "../common/JFJochException.h" -#include - namespace { // Coalesce consecutive single-image mappings into one contiguous range when the source and // virtual images stay contiguous in the same file/dataset. @@ -103,22 +101,6 @@ HDF5ImageLocator::Location HDF5ImageLocator::Resolve(int64_t global_image) const return {layout_.master_file, static_cast(global_image), layout_.master_filename}; } -std::vector HDF5ImageLocator::DataFiles() const { - std::vector ret; - if (layout_.format == FileWriterFormat::NXmxLegacy) { - for (const auto &f: layout_.legacy_files) - ret.push_back(f.path); - } else if (layout_.format == FileWriterFormat::NXmxVDS - && layout_.data_layout == HDF5DataSetLayout::VIRTUAL) { - for (const auto &mapping: layout_.vds_mappings) - if (CoversFirstChannel(mapping) - && std::find(ret.begin(), ret.end(), mapping.filename) == ret.end()) - ret.push_back(mapping.filename); - } else if (!layout_.master_filename.empty()) - ret.push_back(layout_.master_filename); - return ret; -} - std::vector HDF5ImageLocator::GetSourceMapping(uint64_t first_image, std::optional image_count, uint64_t total_images, diff --git a/reader/HDF5ImageLocator.h b/reader/HDF5ImageLocator.h index c08377e7a..bc5bf6579 100644 --- a/reader/HDF5ImageLocator.h +++ b/reader/HDF5ImageLocator.h @@ -64,10 +64,6 @@ public: // by the layout. Does not bounds-check against the total image count - the caller does that. Location Resolve(int64_t global_image) const; - // The files that hold the pixels, in image order: the data files of a legacy or VDS dataset, the - // master itself when the images are in it. - std::vector DataFiles() const; - // Source mapping for re-writing a derived file (e.g. _process.h5) so it links back to the // original pixel sources rather than to a master. total_images is supplied by the caller. // stride is the step between consecutive images of the derived file in the SOURCE: image i of the diff --git a/reader/HDF5ImageSource.h b/reader/HDF5ImageSource.h index fe53c17f1..b3bc77dff 100644 --- a/reader/HDF5ImageSource.h +++ b/reader/HDF5ImageSource.h @@ -70,9 +70,6 @@ public: // file that holds a legacy/VDS image's per-image metadata. HDF5ImageLocator::Location Resolve(int64_t global) const; - // The files holding the pixels, in image order. - std::vector DataFiles() const { return locator_.DataFiles(); } - // Read the pixels at a resolved location into a CompressedImage backed by `buffer`. Templated on // the allocator so a caller can hand over a buffer that does not zero what it is about to // overwrite (RawByteBuffer). diff --git a/reader/JFJochCBFReader.cpp b/reader/JFJochCBFReader.cpp index 117bea505..8c06459d3 100644 --- a/reader/JFJochCBFReader.cpp +++ b/reader/JFJochCBFReader.cpp @@ -395,7 +395,6 @@ bool JFJochCBFReader::ReadRawImage(int64_t image_number, JFJochReaderRawImage &i if (!HasImage(image_number)) return false; image.image = DecodeInto(image_number, image.image_buffer, image.read_buffer); - NoteImageRead(); return true; } diff --git a/reader/JFJochCBFReader.h b/reader/JFJochCBFReader.h index 0511fad47..1ee73ab89 100644 --- a/reader/JFJochCBFReader.h +++ b/reader/JFJochCBFReader.h @@ -37,8 +37,6 @@ class JFJochCBFReader : public JFJochReader { template CompressedImage DecodeInto(int64_t image_number, Buffer &buffer, std::vector &scratch) const; - std::vector DataFiles() const override { return files_; } - public: ~JFJochCBFReader() override = default; diff --git a/reader/JFJochHDF5Reader.cpp b/reader/JFJochHDF5Reader.cpp index bffbc0296..4009a8cdf 100644 --- a/reader/JFJochHDF5Reader.cpp +++ b/reader/JFJochHDF5Reader.cpp @@ -72,21 +72,14 @@ bool JFJochHDF5Reader::ReadRawImage(int64_t image_number, JFJochReaderRawImage & chunk = image_source_.PrepareDirectRead(loc); if (!chunk) { ret.image = image_source_.ReadImageAt(ret.image_buffer, loc); - NoteImageRead(); return true; } } ret.image = HDF5ImageSource::ReadDirect(ret.image_buffer, *chunk); - NoteImageRead(); return true; } -std::vector JFJochHDF5Reader::DataFiles() const { - std::unique_lock ul(hdf5_mutex); - return image_source_.DataFiles(); -} - bool JFJochHDF5Reader::LoadImage_i(std::shared_ptr &dataset, DataMessage &message, std::vector &buffer, diff --git a/reader/JFJochHDF5Reader.h b/reader/JFJochHDF5Reader.h index 6f2650464..175fedd89 100644 --- a/reader/JFJochHDF5Reader.h +++ b/reader/JFJochHDF5Reader.h @@ -40,8 +40,6 @@ class JFJochHDF5Reader : public JFJochReader { HDF5ImageLocator::Location GetImageLocation(int64_t image_number) const; - std::vector DataFiles() const override; - public: ~JFJochHDF5Reader() override = default; diff --git a/reader/JFJochMarCCDReader.cpp b/reader/JFJochMarCCDReader.cpp index 552b67aa1..317bd5e7b 100644 --- a/reader/JFJochMarCCDReader.cpp +++ b/reader/JFJochMarCCDReader.cpp @@ -204,7 +204,6 @@ bool JFJochMarCCDReader::ReadRawImage(int64_t image_number, JFJochReaderRawImage if (!HasImage(image_number)) return false; image.image = DecodeInto(image_number, image.image_buffer, image.read_buffer); - NoteImageRead(); return true; } diff --git a/reader/JFJochMarCCDReader.h b/reader/JFJochMarCCDReader.h index 74aa7be0c..886009162 100644 --- a/reader/JFJochMarCCDReader.h +++ b/reader/JFJochMarCCDReader.h @@ -38,8 +38,6 @@ class JFJochMarCCDReader : public JFJochReader { template CompressedImage DecodeInto(int64_t image_number, Buffer &buffer, std::vector &scratch) const; - std::vector DataFiles() const override { return files_; } - public: ~JFJochMarCCDReader() override = default; diff --git a/reader/JFJochReader.cpp b/reader/JFJochReader.cpp index f7a17a073..e8912ce0a 100644 --- a/reader/JFJochReader.cpp +++ b/reader/JFJochReader.cpp @@ -64,13 +64,6 @@ std::shared_ptr JFJochReader::GetRawImage(int64_t image_nu return ret; } -void JFJochReader::StartReadAhead() { - read_ahead_.reset(); - const auto files = DataFiles(); - if (!files.empty() && GetNumberOfImages() > 0) - read_ahead_ = std::make_unique(files, GetNumberOfImages()); -} - void JFJochReader::SetStartMessage(const std::shared_ptr &val) { std::unique_lock ul(m); dataset = val; diff --git a/reader/JFJochReader.h b/reader/JFJochReader.h index d59075ecf..31a5b59d8 100644 --- a/reader/JFJochReader.h +++ b/reader/JFJochReader.h @@ -6,7 +6,6 @@ #include #include -#include #include #include "../common/JFJochMessages.h" @@ -16,7 +15,6 @@ #include "JFJochReaderDataset.h" #include "JFJochReaderImage.h" #include "JFJochReaderSpots.h" -#include "ReadAhead.h" class JFJochReader { mutable std::mutex m; @@ -33,17 +31,8 @@ class JFJochReader { int64_t n_image, int64_t image_jump, JFJochReaderImage &image); - - std::unique_ptr read_ahead_; protected: void SetStartMessage(const std::shared_ptr &val); - - // The files that hold the frame data, in the order the images sit in them: what StartReadAhead() - // streams. Empty for a reader with no files of its own. - virtual std::vector DataFiles() const { return {}; } - // Every ReadRawImage() that read an image reports it here, which is what moves the read-ahead on. - void NoteImageRead() { if (read_ahead_) read_ahead_->ImageRead(); } - DiffractionExperiment default_experiment; public: virtual ~JFJochReader() = default; @@ -55,11 +44,6 @@ public: virtual void Close() = 0; - // Start streaming the frame data into the page cache ahead of the image reads (see ReadAhead). - // For a batch run over the whole dataset; call it once the dataset is open, before any image is - // read. It changes only when the bytes arrive, never what is read. - void StartReadAhead(); - std::shared_ptr LoadImage(int64_t image_number, int64_t summation_factor = 1); void UpdateGeomMetadata(const DiffractionExperiment& experiment); diff --git a/reader/JFJochSMVReader.cpp b/reader/JFJochSMVReader.cpp index 548a746e6..65d47b0d3 100644 --- a/reader/JFJochSMVReader.cpp +++ b/reader/JFJochSMVReader.cpp @@ -204,7 +204,6 @@ bool JFJochSMVReader::ReadRawImage(int64_t image_number, JFJochReaderRawImage &i if (!HasImage(image_number)) return false; image.image = DecodeInto(image_number, image.image_buffer, image.read_buffer); - NoteImageRead(); return true; } diff --git a/reader/JFJochSMVReader.h b/reader/JFJochSMVReader.h index 815a08e60..ec67a8187 100644 --- a/reader/JFJochSMVReader.h +++ b/reader/JFJochSMVReader.h @@ -38,8 +38,6 @@ class JFJochSMVReader : public JFJochReader { template CompressedImage DecodeInto(int64_t image_number, Buffer &buffer, std::vector &scratch) const; - std::vector DataFiles() const override { return files_; } - public: ~JFJochSMVReader() override = default; diff --git a/reader/ReadAhead.cpp b/reader/ReadAhead.cpp deleted file mode 100644 index 3f87bd654..000000000 --- a/reader/ReadAhead.cpp +++ /dev/null @@ -1,81 +0,0 @@ -// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute -// SPDX-License-Identifier: GPL-3.0-only - -#include "ReadAhead.h" - -#include -#include -#include -#include - -#ifdef _WIN32 -#include -#endif - -namespace { - -constexpr uint64_t PIECE_BYTES = 4 << 20; -constexpr size_t THREADS = 8; - -// Memory the system can hand out without swapping. Where there is no such figure (macOS), 4 GiB. -uint64_t AvailableMemory() { -#ifdef _WIN32 - MEMORYSTATUSEX status{}; - status.dwLength = sizeof(status); - if (GlobalMemoryStatusEx(&status)) - return status.ullAvailPhys; -#else - std::ifstream meminfo("/proc/meminfo"); - std::string key; - uint64_t kb = 0; - while (meminfo >> key >> kb) { - if (key == "MemAvailable:") - return kb * 1024; - meminfo.ignore(std::numeric_limits::max(), '\n'); - } -#endif - return 4ULL << 30; -} - -} // namespace - -ReadAhead::ReadAhead(std::vector files, uint64_t n_images) : files_(std::move(files)) { - uint64_t total = 0; - for (size_t i = 0; i < files_.size(); i++) { - std::error_code ec; - const uint64_t size = std::filesystem::file_size(files_[i], ec); // a gap in the sweep is "" - if (ec) - continue; - for (uint64_t offset = 0; offset < size; offset += PIECE_BYTES) - pieces_.push_back({i, offset, total + offset}); - total += size; - } - bytes_per_image_ = static_cast(total) / static_cast(n_images); - window_ = static_cast(AvailableMemory() / 4); - - for (size_t t = 0; t < THREADS; t++) - threads_.emplace_back(&ReadAhead::Run, this); -} - -ReadAhead::~ReadAhead() { - stop_ = true; - for (auto &t: threads_) - t.join(); -} - -void ReadAhead::Run() { - std::vector buffer(PIECE_BYTES); - for (size_t i = next_piece_++; i < pieces_.size(); i = next_piece_++) { - const auto &piece = pieces_[i]; - while (static_cast(piece.position) > window_ + static_cast(images_read_) * bytes_per_image_) { - if (stop_) - return; - std::this_thread::sleep_for(std::chrono::milliseconds(20)); - } - if (stop_) - return; - std::ifstream in(files_[piece.file], std::ios::binary); - in.seekg(static_cast(piece.offset)); - in.read(buffer.data(), static_cast(buffer.size())); - } -} diff --git a/reader/ReadAhead.h b/reader/ReadAhead.h deleted file mode 100644 index 07cae9fac..000000000 --- a/reader/ReadAhead.h +++ /dev/null @@ -1,58 +0,0 @@ -// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute -// SPDX-License-Identifier: GPL-3.0-only - -#pragma once - -#include -#include -#include -#include -#include - -// Streams the files that hold a dataset's frames into the page cache, front to back, ahead of the -// image loops that read them. -// -// On a spinning disk a cold run otherwise waits on the disk: the header scan, the pre-scan and the -// first-pass indexing touch a few hundred frames and leave the disk idle for tens of seconds, and -// then the first image loop reads everything at whatever rate its scattered requests get. Streaming -// from the moment the dataset is open keeps the disk busy through all of that, at its sequential -// rate, so the loops find their frames already in memory. -// -// The files are read in 4 MB pieces by a few threads that take the pieces strictly in order: on the -// disk measured here one thread streams at 125 MB/s, eight such threads at 190 MB/s (and 32 at -// 157 MB/s), so eight keep the stream ahead of an image loop that is itself reading from disk. -// -// It changes only WHEN bytes arrive, never what is read: the loops still read every image themselves, -// and a frame the read-ahead has not reached yet is read cold, as it always was. -// -// It never runs more than a window - a quarter of the memory available when it starts - ahead of -// what the loops have read. A dataset larger than that is not streamed in only to push out its own -// beginning before the first loop gets there; and a page the read-ahead brought in is then read a -// second time by the loop, which is what keeps it in the page cache for the loops after. -class ReadAhead { - struct Piece { - size_t file; - uint64_t offset; // in the file - uint64_t position; // in the whole stream - }; - - std::vector files_; - std::vector pieces_; - double window_ = 0; - double bytes_per_image_ = 0; - std::atomic next_piece_{0}; - std::atomic images_read_{0}; - std::atomic stop_{false}; - std::vector threads_; - - void Run(); -public: - // files: in the order the images sit in them; n_images: how many images they hold between them. - ReadAhead(std::vector files, uint64_t n_images); - ~ReadAhead(); - ReadAhead(const ReadAhead &) = delete; - ReadAhead &operator=(const ReadAhead &) = delete; - - // An image loop has read one image; the window moves on by one image's worth of bytes. - void ImageRead() { images_read_++; } -}; diff --git a/rugnux/rugnux_cli.cpp b/rugnux/rugnux_cli.cpp index 80f21fbe5..8eb64c648 100644 --- a/rugnux/rugnux_cli.cpp +++ b/rugnux/rugnux_cli.cpp @@ -1456,10 +1456,6 @@ static int RunRugnux(int argc, char **argv) { exit(EXIT_FAILURE); } JFJochReader &reader = *reader_ptr; - // Stream the frames in from disk from now on, so a cold or slow disk works through the setup and - // the pre-scan instead of sitting idle until the first image loop. --mode scale reads no images. - if (mode != RugnuxMode::Scale) - reader.StartReadAhead(); const auto dataset = reader.GetDataset(); if (!dataset) {