diff --git a/reader/JFJochXTCReader.cpp b/reader/JFJochXTCReader.cpp index 772f16b28..5d979c2c7 100644 --- a/reader/JFJochXTCReader.cpp +++ b/reader/JFJochXTCReader.cpp @@ -4,9 +4,12 @@ #include "JFJochXTCReader.h" #include +#include #include +#include #include #include +#include #include "../common/JFJochException.h" #include "../common/Logger.h" @@ -42,8 +45,24 @@ void JFJochXTCReader::ReadFiles(const std::vector &streams) { std::string model; uint32_t binning = 0; size_t damaged = 0; - for (const auto &stream: streams) { - auto index = xtc::IndexFile(stream); + // The streams are indexed in parallel: each is a walk of small reads from one place in the file to the + // next, which on a run not in the page cache waits on the disk one read at a time. + std::vector indices(streams.size()); + { + std::atomic next = 0; + const size_t nthreads = std::min(streams.size(), std::max(1u, std::thread::hardware_concurrency())); + std::vector> workers; + for (size_t t = 0; t < nthreads; t++) + workers.push_back(std::async(std::launch::async, [&] { + for (size_t i = next.fetch_add(1); i < streams.size(); i = next.fetch_add(1)) + indices[i] = xtc::IndexFile(streams[i]); + })); + for (auto &w: workers) + w.get(); + } + for (size_t s = 0; s < streams.size(); s++) { + const auto &stream = streams[s]; + const auto &index = indices[s]; all.insert(all.end(), index.frames.begin(), index.frames.end()); energies.insert(energies.end(), index.photon_energy_eV.begin(), index.photon_energy_eV.end()); if (!index.rayonix_model.empty()) { @@ -142,11 +161,17 @@ CompressedImage JFJochXTCReader::DecodeInto(int64_t image_number, Buffer &buffer if (image_number < 0 || static_cast(image_number) >= frames_.size()) throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Image number out of range"); const auto &f = frames_[image_number]; - buffer.resize(static_cast(f.width) * f.height * sizeof(int32_t)); + const size_t npixel = static_cast(f.width) * f.height; + buffer.resize(npixel * sizeof(int32_t)); + scratch.resize(npixel * sizeof(uint16_t)); + xtc::ReadPixels(f, reinterpret_cast(scratch.data())); + const auto in = reinterpret_cast(scratch.data()); auto out = reinterpret_cast(buffer.data()); - xtc::ReadFrame(f, out, scratch); + const auto offset = static_cast(f.pixel_offset); + for (size_t i = 0; i < npixel; i++) + out[i] = static_cast(in[i]) - offset; if (adu_per_photon_ != 1.0f) - for (size_t i = 0; i < static_cast(f.width) * f.height; i++) + for (size_t i = 0; i < npixel; i++) out[i] = static_cast(std::lround(static_cast(out[i]) / adu_per_photon_)); return CompressedImage(buffer.data(), buffer.size(), f.width, f.height, CompressedImageMode::Int32, CompressionAlgorithm::NO_COMPRESSION); @@ -169,6 +194,16 @@ bool JFJochXTCReader::LoadImage_i(std::shared_ptr &dataset, bool JFJochXTCReader::ReadRawImage(int64_t image_number, JFJochReaderRawImage &image) { if (image_number < 0 || static_cast(image_number) >= frames_.size()) return false; + const auto &f = frames_[image_number]; + if (f.pixel_offset == 0 && adu_per_photon_ == 1.0f) { + // The pixels as the file stores them, two bytes each, where there is nothing to take off or divide + // by: the consumer widens them, on the device where there is one. + image.image_buffer.resize(static_cast(f.width) * f.height * sizeof(uint16_t)); + xtc::ReadPixels(f, reinterpret_cast(image.image_buffer.data())); + image.image = CompressedImage(image.image_buffer.data(), image.image_buffer.size(), f.width, f.height, + CompressedImageMode::Int32, CompressionAlgorithm::UINT16_WIDENED); + return true; + } image.image = DecodeInto(image_number, image.image_buffer, image.read_buffer); return true; } diff --git a/reader/XTC.cpp b/reader/XTC.cpp index 8429114e8..46e48cc34 100644 --- a/reader/XTC.cpp +++ b/reader/XTC.cpp @@ -146,18 +146,11 @@ Index IndexFile(const std::string &file) { return index; } -void ReadFrame(const Frame &frame, int32_t *out, std::vector &scratch) { - const size_t npixel = static_cast(frame.width) * frame.height; - scratch.resize(2 * npixel); +void ReadPixels(const Frame &frame, uint16_t *out) { std::ifstream f = Open(frame.file); - if (!ReadAt(f, frame.offset, scratch.data(), scratch.size())) + if (!ReadAt(f, frame.offset, out, 2 * static_cast(frame.width) * frame.height)) throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, frame.file + ": an image ends past the end of the file"); - for (size_t i = 0; i < npixel; i++) { - uint16_t v; - std::memcpy(&v, scratch.data() + 2 * i, 2); - out[i] = static_cast(v) - static_cast(frame.pixel_offset); - } } } // namespace xtc diff --git a/reader/XTC.h b/reader/XTC.h index 156db6d44..bc41e6be4 100644 --- a/reader/XTC.h +++ b/reader/XTC.h @@ -57,7 +57,8 @@ bool CanRead(const std::string &path); Index IndexFile(const std::string &file); -// Reads one image as 32-bit pixels into out (width*height values), through scratch. -void ReadFrame(const Frame &frame, int32_t *out, std::vector &scratch); +// Reads one image's pixels as stored - 16 bits each, the camera's offset still on - into out (width*height +// values). +void ReadPixels(const Frame &frame, uint16_t *out); } // namespace xtc