// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute // SPDX-License-Identifier: GPL-3.0-only #include "ReadAhead.h" #include #include #include #include #ifdef _WIN32 #include #endif namespace { constexpr uint64_t PIECE_BYTES = 4 << 20; constexpr size_t THREADS = 8; // Memory the system can hand out without swapping. Where there is no such figure (macOS), 4 GiB. uint64_t AvailableMemory() { #ifdef _WIN32 MEMORYSTATUSEX status{}; status.dwLength = sizeof(status); if (GlobalMemoryStatusEx(&status)) return status.ullAvailPhys; #else std::ifstream meminfo("/proc/meminfo"); std::string key; uint64_t kb = 0; while (meminfo >> key >> kb) { if (key == "MemAvailable:") return kb * 1024; meminfo.ignore(std::numeric_limits::max(), '\n'); } #endif return 4ULL << 30; } } // namespace ReadAhead::ReadAhead(std::vector files, uint64_t n_images) : files_(std::move(files)) { uint64_t total = 0; for (size_t i = 0; i < files_.size(); i++) { std::error_code ec; const uint64_t size = std::filesystem::file_size(files_[i], ec); // a gap in the sweep is "" if (ec) continue; for (uint64_t offset = 0; offset < size; offset += PIECE_BYTES) pieces_.push_back({i, offset, total + offset}); total += size; } bytes_per_image_ = static_cast(total) / static_cast(n_images); window_ = static_cast(AvailableMemory() / 4); for (size_t t = 0; t < THREADS; t++) threads_.emplace_back(&ReadAhead::Run, this); } ReadAhead::~ReadAhead() { stop_ = true; for (auto &t: threads_) t.join(); } void ReadAhead::Run() { std::vector buffer(PIECE_BYTES); for (size_t i = next_piece_++; i < pieces_.size(); i = next_piece_++) { const auto &piece = pieces_[i]; while (static_cast(piece.position) > window_ + static_cast(images_read_) * bytes_per_image_) { if (stop_) return; std::this_thread::sleep_for(std::chrono::milliseconds(20)); } if (stop_) return; std::ifstream in(files_[piece.file], std::ios::binary); in.seekg(static_cast(piece.offset)); in.read(buffer.data(), static_cast(buffer.size())); } }