detectors/aare
25
0
Fork 0
mirror of https://github.com/slsdetectorgroup/aare.git synced 2025-06-11 06:47:14 +02:00
Code Issues Actions Packages Releases Activity

read subfiles with unordered and missing frames

Browse Source
This commit is contained in:
Bechir Braham
2024-04-22 18:15:02 +02:00
parent a4850892e0
commit 1177fd129d
6 changed files with 71 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
file_io/include/aare/file_io/FileInterface.hpp
View File

@ -76,7 +76,8 @@ class FileInterface {
virtual void read_into(std::byte *image_buf, size_t n_frames) = 0;
/**
* @brief get the frame number at the given frame index
* @brief get the frame number stored in the file at the given frame_index
* @note throws when subfiles have different frame numbers at the same index (note only relevant for RawFile)
* @param frame_index index of the frame
* @return frame number
*/

9
file_io/include/aare/file_io/RawFile.hpp
View File

@ -25,9 +25,13 @@ class RawFile : public FileInterface {
* @param frame frame to write
*/
void write([[maybe_unused]] Frame &frame) override { throw std::runtime_error("Not implemented"); };
Frame read() override { return get_frame(this->current_frame++); };
Frame read() override {
return get_frame(frame_number(this->current_frame++));
};
std::vector<Frame> read(size_t n_frames) override;
void read_into(std::byte *image_buf) override { return get_frame_into(this->current_frame++, image_buf); };
void read_into(std::byte *image_buf) override {
return get_frame_into(frame_number(this->current_frame++), image_buf);
};
void read_into(std::byte *image_buf, size_t n_frames) override;
size_t frame_number(size_t frame_index) override;
@ -156,6 +160,7 @@ class RawFile : public FileInterface {
RawFileConfig cfg{0, 0};
TimingMode timing_mode{};
bool quad{false};
size_t m_starting_frame{};
};
} // namespace aare

9
file_io/include/aare/file_io/SubFile.hpp
View File

@ -45,7 +45,7 @@ class SubFile {
* @param bitdepth bitdepth of the subfile
* @throws std::invalid_argument if the detector,type pair is not supported
*/
SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth);
SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth,int subfile_id);
/**
* @brief read the subfile into a buffer
@ -75,7 +75,9 @@ class SubFile {
* @return number of bytes read
*/
size_t get_part(std::byte *buffer, size_t frame_number);
size_t frame_number(size_t frame_index);
size_t frame_number_in_file(size_t frame_index);
size_t correct_frame_number(size_t frame_number);
~SubFile() noexcept;
// TODO: define the inlines as variables and assign them in constructor
inline size_t bytes_per_part() const { return (m_bitdepth / 8) * m_rows * m_cols; }
@ -88,7 +90,8 @@ class SubFile {
size_t m_rows{};
size_t m_cols{};
size_t n_frames{};
int m_sub_file_index_{};
size_t cached_offset{};
int m_subfile_id{};
};
} // namespace aare

23
file_io/src/RawFile.cpp
View File

@ -24,13 +24,14 @@ RawFile::RawFile(const std::filesystem::path &fname, const std::string &mode, co
} else {
throw std::runtime_error(LOCATION + "Unsupported mode");
}
m_starting_frame = this->frame_number(0);
}
void RawFile::open_subfiles() {
for (size_t i = 0; i != n_subfiles; ++i) {
auto v = std::vector<SubFile *>(n_subfile_parts);
for (size_t j = 0; j != n_subfile_parts; ++j) {
v[j] = new SubFile(data_fname(i, j), m_type, subfile_rows, subfile_cols, m_bitdepth);
v[j] = new SubFile(data_fname(i, j), m_type, subfile_rows, subfile_cols, m_bitdepth, i);
}
subfiles.push_back(v);
}
@ -200,15 +201,15 @@ void RawFile::get_frame_into(size_t frame_number, std::byte *frame_buffer) {
if (frame_number > this->m_total_frames) {
throw std::runtime_error(LOCATION + "Frame number out of range");
}
size_t const subfile_id = frame_number / this->max_frames_per_file;
size_t const subfile_id = (frame_number - m_starting_frame) / this->max_frames_per_file;
// create frame and get its buffer
if (this->geometry.col == 1) {
// get the part from each subfile and copy it to the frame
for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) {
auto part_offset = this->subfiles[subfile_id][part_idx]->bytes_per_part();
this->subfiles[subfile_id][part_idx]->get_part(frame_buffer + part_idx * part_offset,
frame_number % this->max_frames_per_file);
this->subfiles[subfile_id][part_idx]->get_part(frame_buffer + part_idx * part_offset, frame_number);
}
} else {
@ -217,7 +218,7 @@ void RawFile::get_frame_into(size_t frame_number, std::byte *frame_buffer) {
auto *part_buffer = new std::byte[bytes_per_part];
for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) {
this->subfiles[subfile_id][part_idx]->get_part(part_buffer, frame_number % this->max_frames_per_file);
this->subfiles[subfile_id][part_idx]->get_part(part_buffer, frame_number);
for (size_t cur_row = 0; cur_row < (this->subfile_rows); cur_row++) {
auto irow = cur_row + (part_idx / this->geometry.col) * this->subfile_rows;
auto icol = (part_idx % this->geometry.col) * this->subfile_cols;
@ -253,7 +254,17 @@ size_t RawFile::frame_number(size_t frame_index) {
throw std::runtime_error(LOCATION + "Frame number out of range");
}
size_t const subfile_id = frame_index / this->max_frames_per_file;
return this->subfiles[subfile_id][0]->frame_number(frame_index % this->max_frames_per_file);
size_t prev_frame_nbr{};
bool first_time_in_loop = true;
for (auto &subfile_parts : this->subfiles[subfile_id]) {
auto cur_frame_nbr = subfile_parts->frame_number_in_file(frame_index % this->max_frames_per_file);
if ((not first_time_in_loop) && cur_frame_nbr != prev_frame_nbr) {
throw std::runtime_error(LOCATION + "Frame number different in subfiles");
}
prev_frame_nbr = cur_frame_nbr;
first_time_in_loop = false;
}
return prev_frame_nbr;
}
RawFile::~RawFile() {

59
file_io/src/SubFile.cpp
View File

@ -7,9 +7,11 @@
namespace aare {
SubFile::SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth)
SubFile::SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth,
int subfile_id)
: m_bitdepth(bitdepth), m_fname(fname), m_rows(rows), m_cols(cols),
n_frames(std::filesystem::file_size(fname) / (sizeof(sls_detector_header) + rows * cols * bitdepth / 8)) {
n_frames(std::filesystem::file_size(fname) / (sizeof(sls_detector_header) + rows * cols * bitdepth / 8)),
m_subfile_id(subfile_id) {
if (read_impl_map.find({detector, bitdepth}) == read_impl_map.end()) {
auto error_msg = LOCATION + "No read_impl function found for detector: " + toString(detector) +
@ -17,23 +19,36 @@ SubFile::SubFile(const std::filesystem::path &fname, DetectorType detector, size
throw std::invalid_argument(error_msg);
}
this->read_impl = read_impl_map.at({detector, bitdepth});
if (!(fp = fopen(fname.c_str(), "rb"))) {
throw std::runtime_error(fmt::format(LOCATION + "Could not open: {} for reading", fname.c_str()));
}
}
size_t SubFile::correct_frame_number(size_t frame_number) {
sls_detector_header h{};
for (size_t offset = cached_offset; offset < n_frames; offset++) {
size_t ret = (frame_number + offset) % n_frames;
fseek(fp, (sizeof(sls_detector_header) + bytes_per_part()) * ret, SEEK_SET); // NOLINT
size_t const rc = fread(reinterpret_cast<char *>(&h), sizeof(h), 1, fp);
if (rc != 1)
throw std::runtime_error(LOCATION + "Could not read header from file");
if (h.frameNumber == frame_number) {
cached_offset = offset;
return ret;
}
}
aare::logger::error("m_subfile_id:", m_subfile_id);
aare::logger::error(LOCATION, ": frame:", h.frameNumber, "frame_number:", frame_number, "toto",
(h.frameNumber - (m_subfile_id * n_frames)));
throw std::runtime_error(fmt::format(LOCATION + "Could not find frame {} in file", frame_number));
}
size_t SubFile::get_part(std::byte *buffer, size_t frame_number) {
if (frame_number >= n_frames) {
throw std::runtime_error("Frame number out of range");
}
// TODO: find a way to avoid opening and closing the file for each frame
aare::logger::debug(LOCATION, "frame:", frame_number, "file:", m_fname.c_str());
fp = fopen(m_fname.c_str(), "rb");
if (!fp) {
throw std::runtime_error(fmt::format("Could not open: {} for reading", m_fname.c_str()));
}
fseek(fp, sizeof(sls_detector_header) + (sizeof(sls_detector_header) + bytes_per_part()) * frame_number, // NOLINT
SEEK_SET);
size_t tmp = correct_frame_number(frame_number);
fseek(fp, sizeof(sls_detector_header) + (sizeof(sls_detector_header) + bytes_per_part()) * tmp, SEEK_SET); // NOLINT
auto ret = (this->*read_impl)(buffer);
if (fclose(fp))
throw std::runtime_error(LOCATION + "Could not close file");
return ret;
}
@ -89,20 +104,20 @@ template <typename DataType> size_t SubFile::read_impl_flip(std::byte *buffer) {
return rc;
};
size_t SubFile::frame_number(size_t frame_index) {
size_t SubFile::frame_number_in_file(size_t frame_index) {
sls_detector_header h{};
fp = fopen(this->m_fname.c_str(), "r");
if (!fp)
throw std::runtime_error(LOCATION + fmt::format("Could not open: {} for reading", m_fname.c_str()));
fseek(fp, (sizeof(sls_detector_header) + bytes_per_part()) * frame_index, SEEK_SET); // NOLINT
size_t const rc = fread(reinterpret_cast<char *>(&h), sizeof(h), 1, fp);
if (rc != 1)
throw std::runtime_error(LOCATION + "Could not read header from file");
if (fclose(fp)) {
throw std::runtime_error(LOCATION + "Could not close file");
}
return h.frameNumber;
}
SubFile::~SubFile() noexcept {
if (fp) {
if (fclose(fp))
aare::logger::error(LOCATION, "Could not close file");
}
}
} // namespace aare

3
file_io/test/RawFile.test.cpp
View File

@ -32,7 +32,7 @@ TEST_CASE("Read frame numbers from a jungfrau raw file") {
}
}
TEST_CASE("Read data from a jungfrau 500k single port raw file") {
TEST_CASE("Read data from a jungfrau 500k single port raw file", "[debug]") {
auto fpath = test_data_path() / "jungfrau" / "jungfrau_single_master_0.json";
REQUIRE(std::filesystem::exists(fpath));
@ -45,6 +45,7 @@ TEST_CASE("Read data from a jungfrau 500k single port raw file") {
CHECK(frame.rows() == 512);
CHECK(frame.cols() == 1024);
CHECK(frame.view<uint16_t>()(0, 0) == pixel_0_0[i]);
// frame.view<uint16_t>();
}
}