Read raw images from several threads at once in a test
The three GetRawImage cases are single-threaded, so none of them enters the path the change is for: the chunk address is taken under the HDF5 lock and the bytes are read outside it, which only means anything when several workers are inside the reader at once - which is how rugnux drives it. Eight of them pulling every image and comparing against the bytes handed to the writer. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01VfYvJT5Nb71suJCowRBn5z
This commit is contained in:
@@ -9,6 +9,8 @@
|
||||
#include "../reader/JFJochHDF5Reader.h"
|
||||
#include "../compression/JFJochCompressor.h"
|
||||
|
||||
#include <future>
|
||||
|
||||
TEST_CASE("HDF5DataType_Sign","[HDF5]") {
|
||||
HDF5DataType type_u8((uint8_t)0), type_fl(0.0f), type_i32((int32_t) 0), type_u32((uint32_t) 0);
|
||||
|
||||
@@ -1736,6 +1738,67 @@ TEST_CASE("JFJochReader_GetRawImage_NXmxLegacy", "[HDF5][Full]") {
|
||||
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
||||
}
|
||||
|
||||
// GetRawImage takes the chunk address under the HDF5 lock, then reads the bytes outside it, so this
|
||||
// is the one path where several workers are inside the reader at once - which is how rugnux uses it.
|
||||
// The per-image cases above are all single-threaded and would not notice the file being pulled from
|
||||
// under a read, nor a cache entry racing its own creation.
|
||||
TEST_CASE("JFJochReader_GetRawImage_Concurrent", "[HDF5][Full]") {
|
||||
DiffractionExperiment x(DetJF(1));
|
||||
x.FilePrefix("test_raw_concurrent").ImagesPerTrigger(16).OverwriteExistingFiles(true);
|
||||
x.BitDepthImage(16).ImagesPerFile(4).SetFileWriterFormat(FileWriterFormat::NXmxVDS).PixelSigned(true);
|
||||
x.Compression(CompressionAlgorithm::BSHUF_ZSTD);
|
||||
|
||||
std::vector<int16_t> image(x.GetPixelsNum());
|
||||
for (size_t i = 0; i < image.size(); i++)
|
||||
image[i] = static_cast<int16_t>((i * 11 + 5) % UINT16_MAX);
|
||||
|
||||
RegisterHDF5Filter();
|
||||
JFJochBitShuffleCompressor compressor(CompressionAlgorithm::BSHUF_ZSTD);
|
||||
const auto compressed_image = compressor.Compress(image);
|
||||
{
|
||||
StartMessage start_message;
|
||||
x.FillMessage(start_message);
|
||||
FileWriter file_set(start_message);
|
||||
for (int i = 0; i < x.GetImageNum(); i++) {
|
||||
DataMessage message{};
|
||||
message.image = CompressedImage(compressed_image, x.GetXPixelsNum(), x.GetYPixelsNum(),
|
||||
CompressedImageMode::Int16, CompressionAlgorithm::BSHUF_ZSTD);
|
||||
message.number = i;
|
||||
REQUIRE_NOTHROW(file_set.WriteHDF5(message));
|
||||
}
|
||||
EndMessage end_message;
|
||||
end_message.max_image_number = x.GetImageNum();
|
||||
file_set.WriteHDF5(end_message);
|
||||
file_set.Finalize();
|
||||
}
|
||||
{
|
||||
JFJochHDF5Reader reader;
|
||||
REQUIRE_NOTHROW(reader.ReadFile("test_raw_concurrent_master.h5"));
|
||||
|
||||
std::vector<std::future<bool>> workers;
|
||||
workers.reserve(8);
|
||||
for (int w = 0; w < 8; w++) {
|
||||
workers.push_back(std::async(std::launch::async, [&reader, &compressed_image, &x]() {
|
||||
for (int i = 0; i < x.GetImageNum(); i++) {
|
||||
auto raw = reader.GetRawImage(i);
|
||||
if (raw->image_buffer.size() != compressed_image.size())
|
||||
return false;
|
||||
if (memcmp(raw->image_buffer.data(), compressed_image.data(),
|
||||
compressed_image.size()) != 0)
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}));
|
||||
}
|
||||
for (auto &worker: workers)
|
||||
CHECK(worker.get());
|
||||
}
|
||||
remove("test_raw_concurrent_master.h5");
|
||||
for (int f = 1; f <= 4; f++)
|
||||
remove(("test_raw_concurrent_data_00000" + std::to_string(f) + ".h5").c_str());
|
||||
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
||||
}
|
||||
|
||||
TEST_CASE("JFJochReader_GetRawImage_VDS", "[HDF5][Full]") {
|
||||
DiffractionExperiment x(DetJF(1));
|
||||
|
||||
|
||||
Reference in New Issue
Block a user