detectors/aare
25
0
Fork 0
mirror of https://github.com/slsdetectorgroup/aare.git synced 2025-06-16 09:17:12 +02:00
Code Issues Actions Packages Releases Activity

only files within the ROI are opened & geometry always read in RawMasterFile

Browse Source
This commit is contained in:
mazzol_a
2025-06-04 16:34:40 +02:00
parent 69964e08d5
commit 9c6e629298
5 changed files with 208 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
include/aare/RawFile.hpp
View File

@ -1,11 +1,10 @@
#pragma once
#include "aare/FileInterface.hpp"
#include "aare/RawMasterFile.hpp"
#include "aare/Frame.hpp"
#include "aare/NDArray.hpp" //for pixel map
#include "aare/RawMasterFile.hpp"
#include "aare/RawSubFile.hpp"
#include <optional>
namespace aare {
@ -30,12 +29,15 @@ struct ModuleConfig {
* Consider using that unless you need raw file specific functionality.
*/
class RawFile : public FileInterface {
friend class RawMasterFile;
std::vector<std::unique_ptr<RawSubFile>> m_subfiles;
ModuleConfig cfg{0, 0};
RawMasterFile m_master;
size_t m_current_frame{};
size_t m_current_subfile{};
DetectorGeometry m_geometry;
std::vector<ssize_t> m_modules_in_roi{};
public:
/**
@ -55,8 +57,8 @@ class RawFile : public FileInterface {
// TODO! do we need to adapt the API?
void read_into(std::byte *image_buf, DetectorHeader *header);
void read_into(std::byte *image_buf, size_t n_frames, DetectorHeader *header);
void read_into(std::byte *image_buf, size_t n_frames,
DetectorHeader *header);
size_t frame_number(size_t frame_index) override;
size_t bytes_per_frame() override;
@ -70,23 +72,21 @@ class RawFile : public FileInterface {
size_t bitdepth() const override;
xy geometry();
size_t n_modules() const;
size_t n_modules_in_roi() const;
RawMasterFile master() const;
DetectorType detector_type() const override;
private:
/**
* @brief read the frame at the given frame index into the image buffer
* @param frame_number frame number to read
* @param image_buf buffer to store the frame
*/
void get_frame_into(size_t frame_index, std::byte *frame_buffer, DetectorHeader *header = nullptr);
void get_frame_into(size_t frame_index, std::byte *frame_buffer,
DetectorHeader *header = nullptr);
/**
* @brief get the frame at the given frame index
@ -95,8 +95,6 @@ class RawFile : public FileInterface {
*/
Frame get_frame(size_t frame_index);
/**
* @brief read the header of the file
* @param fname path to the data subfile
@ -108,5 +106,4 @@ class RawFile : public FileInterface {
void find_geometry();
};
} // namespace aare

7
include/aare/RawMasterFile.hpp
View File

@ -1,5 +1,7 @@
#pragma once
// #include "aare/RawFile.hpp"
#include "aare/defs.hpp"
#include <algorithm>
#include <filesystem>
#include <fmt/format.h>
#include <fstream>
@ -61,6 +63,7 @@ class ScanParameters {
void increment_stop();
};
class RawFile; // forward declaration
/**
* @brief Class for parsing a master file either in our .json format or the old
@ -101,7 +104,6 @@ class RawMasterFile {
std::optional<ROI> m_roi;
public:
RawMasterFile(const std::filesystem::path &fpath);
@ -129,15 +131,14 @@ class RawMasterFile {
std::optional<size_t> number_of_rows() const;
std::optional<uint8_t> quad() const;
std::optional<ROI> roi() const;
ScanParameters scan_parameters() const;
private:
void parse_json(const std::filesystem::path &fpath);
void parse_raw(const std::filesystem::path &fpath);
void retrieve_geometry();
};
} // namespace aare

128
src/RawFile.cpp
View File

@ -1,9 +1,9 @@
#include "aare/RawFile.hpp"
#include "aare/algorithm.hpp"
#include "aare/PixelMap.hpp"
#include "aare/algorithm.hpp"
#include "aare/defs.hpp"
#include "aare/logger.hpp"
#include "aare/geo_helpers.hpp"
#include "aare/logger.hpp"
#include <fmt/format.h>
#include <nlohmann/json.hpp>
@ -15,10 +15,22 @@ namespace aare {
RawFile::RawFile(const std::filesystem::path &fname, const std::string &mode)
: m_master(fname) {
m_mode = mode;
if (mode == "r") {
find_geometry();
if (m_master.roi()) {
m_geometry = update_geometry_with_roi(m_geometry, m_master.roi().value());
m_geometry =
update_geometry_with_roi(m_geometry, m_master.roi().value());
m_modules_in_roi.reserve(n_modules());
for (size_t module_index = 0; module_index < n_modules();
++module_index) {
if (m_geometry.module_pixel_0[module_index].width == 0 &&
m_geometry.module_pixel_0[module_index].height == 0)
m_modules_in_roi.push_back(module_index);
}
} else {
m_modules_in_roi.resize(n_modules());
std::iota(m_modules_in_roi.begin(), m_modules_in_roi.end(), 0);
}
open_subfiles();
} else {
@ -47,13 +59,13 @@ void RawFile::read_into(std::byte *image_buf) {
return get_frame_into(m_current_frame++, image_buf);
}
void RawFile::read_into(std::byte *image_buf, DetectorHeader *header) {
return get_frame_into(m_current_frame++, image_buf, header);
}
void RawFile::read_into(std::byte *image_buf, size_t n_frames, DetectorHeader *header) {
void RawFile::read_into(std::byte *image_buf, size_t n_frames,
DetectorHeader *header) {
// return get_frame_into(m_current_frame++, image_buf, header);
for (size_t i = 0; i < n_frames; i++) {
@ -62,14 +74,15 @@ void RawFile::read_into(std::byte *image_buf, size_t n_frames, DetectorHeader *h
if (header)
header += n_modules();
}
}
size_t RawFile::n_modules() const { return m_master.n_modules(); }
size_t RawFile::n_modules_in_roi() const { return m_modules_in_roi.size(); }
size_t RawFile::bytes_per_frame() {
return m_geometry.pixels_x * m_geometry.pixels_y * m_master.bitdepth() / bits_per_byte;
return m_geometry.pixels_x * m_geometry.pixels_y * m_master.bitdepth() /
bits_per_byte;
}
size_t RawFile::pixels_per_frame() {
// return m_rows * m_cols;
@ -99,7 +112,7 @@ xy RawFile::geometry() { return m_master.geometry(); }
void RawFile::open_subfiles() {
if (m_mode == "r")
for (size_t i = 0; i != n_modules(); ++i) {
for (size_t i : m_modules_in_roi) {
auto pos = m_geometry.module_pixel_0[i];
m_subfiles.emplace_back(std::make_unique<RawSubFile>(
m_master.data_fname(i, 0), m_master.detector_type(), pos.height,
@ -128,76 +141,59 @@ DetectorHeader RawFile::read_header(const std::filesystem::path &fname) {
return h;
}
RawMasterFile RawFile::master() const { return m_master; }
/**
* @brief Find the geometry of the detector by opening all the subfiles and
* reading the headers.
* @brief Find the geometry of the detector
*/
void RawFile::find_geometry() {
//Hold the maximal row and column number found
//Later used for calculating the total number of rows and columns
uint16_t r{};
uint16_t c{};
for (size_t i = 0; i < n_modules(); i++) {
auto h = read_header(m_master.data_fname(i, 0));
r = std::max(r, h.row);
c = std::max(c, h.column);
// positions.push_back({h.row, h.column});
// TODO potentially update for Eiger
for (size_t col = 0; col < m_master.geometry().col; ++col)
for (size_t row = 0; row < m_master.geometry().row; ++row) {
ModuleGeometry g;
g.origin_x = h.column * m_master.pixels_x();
g.origin_y = h.row * m_master.pixels_y();
g.row_index = h.row;
g.col_index = h.column;
g.origin_x = col * m_master.pixels_x();
g.origin_y = row * m_master.pixels_y();
g.row_index = row;
g.col_index = col;
g.width = m_master.pixels_x();
g.height = m_master.pixels_y();
m_geometry.module_pixel_0.push_back(g);
}
r++;
c++;
m_geometry.pixels_y = (r * m_master.pixels_y());
m_geometry.pixels_x = (c * m_master.pixels_x());
m_geometry.modules_x = c;
m_geometry.modules_y = r;
m_geometry.pixels_y += static_cast<size_t>((r - 1) * cfg.module_gap_row);
m_geometry.pixels_y = (m_master.geometry().row * m_master.pixels_y());
m_geometry.pixels_x = (m_master.geometry().col * m_master.pixels_x());
m_geometry.modules_x = m_master.geometry().col;
m_geometry.modules_y = m_master.geometry().row;
m_geometry.pixels_y +=
static_cast<size_t>((m_master.geometry().row - 1) * cfg.module_gap_row);
}
Frame RawFile::get_frame(size_t frame_index) {
auto f = Frame(m_geometry.pixels_y, m_geometry.pixels_x, Dtype::from_bitdepth(m_master.bitdepth()));
auto f = Frame(m_geometry.pixels_y, m_geometry.pixels_x,
Dtype::from_bitdepth(m_master.bitdepth()));
std::byte *frame_buffer = f.data();
get_frame_into(frame_index, frame_buffer);
return f;
}
size_t RawFile::bytes_per_pixel() const { return m_master.bitdepth() / 8; }
size_t RawFile::bytes_per_pixel() const {
return m_master.bitdepth() / 8;
}
void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer, DetectorHeader *header) {
void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer,
DetectorHeader *header) {
LOG(logDEBUG) << "RawFile::get_frame_into(" << frame_index << ")";
if (frame_index >= total_frames()) {
throw std::runtime_error(LOCATION + "Frame number out of range");
}
std::vector<size_t> frame_numbers(n_modules());
std::vector<size_t> frame_indices(n_modules(), frame_index);
std::vector<size_t> frame_numbers(n_modules_in_roi());
std::vector<size_t> frame_indices(n_modules_in_roi(), frame_index);
// sync the frame numbers
if (n_modules() != 1) { // if we have more than one module
for (size_t part_idx = 0; part_idx != n_modules(); ++part_idx) {
frame_numbers[part_idx] = m_subfiles[part_idx]->frame_number(frame_index);
for (size_t part_idx = 0; part_idx != n_modules_in_roi(); ++part_idx) {
frame_numbers[part_idx] =
m_subfiles[part_idx]->frame_number(frame_index);
}
// 1. if frame number vector is the same break
@ -218,22 +214,31 @@ void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer, Detect
}
frame_numbers[min_frame_idx] =
m_subfiles[min_frame_idx]->frame_number(frame_indices[min_frame_idx]);
m_subfiles[min_frame_idx]->frame_number(
frame_indices[min_frame_idx]);
}
}
if (m_master.geometry().col == 1) {
// get the part from each subfile and copy it to the frame
for (size_t part_idx = 0; part_idx != n_modules(); ++part_idx) {
for (size_t part_idx = 0; part_idx != n_modules_in_roi(); ++part_idx) {
auto corrected_idx = frame_indices[part_idx];
// This is where we start writing
auto offset = (m_geometry.module_pixel_0[part_idx].origin_y * m_geometry.pixels_x +
m_geometry.module_pixel_0[part_idx].origin_x)*m_master.bitdepth()/8;
if (m_geometry.module_pixel_0[part_idx].origin_x!=0)
throw std::runtime_error(LOCATION + " Implementation error. x pos not 0.");
auto offset = (m_geometry.module_pixel_0[m_modules_in_roi[part_idx]]
.origin_y *
m_geometry.pixels_x +
m_geometry.module_pixel_0[m_modules_in_roi[part_idx]]
.origin_x) *
m_master.bitdepth() / 8;
if (m_geometry.module_pixel_0[m_modules_in_roi[part_idx]]
.origin_x != 0)
throw std::runtime_error(
LOCATION +
" Implementation error. x pos not 0."); // TODO: origin can
// still change if
// roi changes
// TODO! What if the files don't match?
m_subfiles[part_idx]->seek(corrected_idx);
m_subfiles[part_idx]->read_into(frame_buffer + offset, header);
@ -254,8 +259,8 @@ void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer, Detect
// TODO! if we have many submodules we should reorder them on the module
// level
for (size_t part_idx = 0; part_idx != n_modules(); ++part_idx) {
auto pos = m_geometry.module_pixel_0[part_idx];
for (size_t part_idx = 0; part_idx != n_modules_in_roi(); ++part_idx) {
auto pos = m_geometry.module_pixel_0[m_modules_in_roi[part_idx]];
auto corrected_idx = frame_indices[part_idx];
m_subfiles[part_idx]->seek(corrected_idx);
@ -271,15 +276,13 @@ void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer, Detect
auto dest = (irow * this->m_geometry.pixels_x + icol);
dest = dest * m_master.bitdepth() / 8;
memcpy(frame_buffer + dest,
part_buffer + cur_row * pos.width *
m_master.bitdepth() / 8,
part_buffer +
cur_row * pos.width * m_master.bitdepth() / 8,
pos.width * m_master.bitdepth() / 8);
}
}
delete[] part_buffer;
}
}
std::vector<Frame> RawFile::read_n(size_t n_frames) {
@ -299,5 +302,4 @@ size_t RawFile::frame_number(size_t frame_index) {
return m_subfiles[0]->frame_number(frame_index);
}
} // namespace aare

59
src/RawMasterFile.cpp
View File

@ -1,4 +1,5 @@
#include "aare/RawMasterFile.hpp"
#include "aare/RawFile.hpp"
#include <sstream>
namespace aare {
@ -37,18 +38,15 @@ std::filesystem::path RawFileNameComponents::master_fname() const {
}
std::filesystem::path RawFileNameComponents::data_fname(size_t mod_id,
size_t file_id
) const {
size_t file_id) const {
std::string fmt = "{}_d{}_f{}_{}.raw";
// Before version X we used to name the data files f000000000000
if (m_old_scheme) {
fmt = "{}_d{}_f{:012}_{}.raw";
}
return m_base_path / fmt::format(fmt, m_base_name, mod_id,
file_id, m_file_index);
return m_base_path /
fmt::format(fmt, m_base_name, mod_id, file_id, m_file_index);
}
void RawFileNameComponents::set_old_scheme(bool old_scheme) {
@ -85,14 +83,11 @@ ScanParameters::ScanParameters(const std::string& par){
int ScanParameters::start() const { return m_start; }
int ScanParameters::stop() const { return m_stop; }
void ScanParameters::increment_stop(){
m_stop += 1;
}
void ScanParameters::increment_stop() { m_stop += 1; }
int ScanParameters::step() const { return m_step; }
const std::string &ScanParameters::dac() const { return m_dac; }
bool ScanParameters::enabled() const { return m_enabled; }
RawMasterFile::RawMasterFile(const std::filesystem::path &fpath)
: m_fnc(fpath) {
if (!std::filesystem::exists(fpath)) {
@ -163,10 +158,8 @@ ScanParameters RawMasterFile::scan_parameters() const {
return m_scan_parameters;
}
std::optional<ROI> RawMasterFile::roi() const { return m_roi; }
void RawMasterFile::parse_json(const std::filesystem::path &fpath) {
std::ifstream ifs(fpath);
json j;
@ -177,7 +170,10 @@ void RawMasterFile::parse_json(const std::filesystem::path &fpath) {
m_type = StringTo<DetectorType>(j["Detector Type"].get<std::string>());
m_timing_mode = StringTo<TimingMode>(j["Timing Mode"].get<std::string>());
m_geometry = {j["Geometry"]["y"], j["Geometry"]["x"]};
m_geometry = {
j["Geometry"]["y"],
j["Geometry"]["x"]}; // TODO: isnt it only available for version > 7.1?
// - try block default should be 1x1
m_image_size_in_bytes = j["Image Size in bytes"];
m_frames_in_file = j["Frames in File"];
@ -205,7 +201,6 @@ void RawMasterFile::parse_json(const std::filesystem::path &fpath) {
// keep the optional empty
}
// ----------------------------------------------------------------
// Special treatment of analog flag because of Moench03
try {
@ -261,13 +256,13 @@ void RawMasterFile::parse_json(const std::filesystem::path &fpath) {
std::string scan_parameters = j.at("Scan Parameters");
m_scan_parameters = ScanParameters(scan_parameters);
if (v < 7.21) {
m_scan_parameters.increment_stop(); //adjust for endpoint being included
m_scan_parameters
.increment_stop(); // adjust for endpoint being included
}
} catch (const json::out_of_range &e) {
// not a scan
}
try {
ROI tmp_roi;
auto obj = j.at("Receiver Roi");
@ -288,19 +283,14 @@ void RawMasterFile::parse_json(const std::filesystem::path &fpath) {
m_roi = tmp_roi;
}
} catch (const json::out_of_range &e) {
// leave the optional empty
}
// if we have an roi we need to update the geometry for the subfiles
if (m_roi) {
}
// Update detector type for Moench
// TODO! How does this work with old .raw master files?
#ifdef AARE_VERBOSE
@ -410,15 +400,34 @@ void RawMasterFile::parse_raw(const std::filesystem::path &fpath) {
m_type = DetectorType::Moench03_old;
}
//TODO! Look for d0, d1...dn and update geometry
if (m_geometry.col == 0 && m_geometry.row == 0) {
m_geometry = {1,1};
fmt::print("Warning: No geometry found in master file. Assuming 1x1\n");
retrieve_geometry();
fmt::print("Warning: No geometry found in master file. Retrieved "
"geometry of {}x{}\n",
m_geometry.row, m_geometry.col);
}
// TODO! Read files and find actual frames
if (m_frames_in_file == 0)
m_frames_in_file = m_total_frames_expected;
}
void RawMasterFile::retrieve_geometry() {
uint32_t module_index = 0;
uint16_t rows = 0;
uint16_t cols = 0;
// TODO use case for Eiger
while (std::filesystem::exists(data_fname(module_index, 0))) {
auto header = RawFile::read_header(data_fname(module_index, 0));
rows = std::max(rows, header.row);
cols = std::max(cols, header.column);
++rows;
++cols;
++module_index;
}
m_geometry = {rows, cols};
}
} // namespace aare

43
src/RawSubFile.cpp
View File

@ -1,27 +1,22 @@
#include "aare/RawSubFile.hpp"
#include "aare/PixelMap.hpp"
#include "aare/algorithm.hpp"
#include "aare/utils/ifstream_helpers.hpp"
#include "aare/logger.hpp"
#include "aare/utils/ifstream_helpers.hpp"
#include <cstring> // memcpy
#include <fmt/core.h>
#include <iostream>
#include <regex>
namespace aare {
RawSubFile::RawSubFile(const std::filesystem::path &fname,
DetectorType detector, size_t rows, size_t cols,
size_t bitdepth, uint32_t pos_row, uint32_t pos_col)
: m_detector_type(detector), m_bitdepth(bitdepth),
m_rows(rows), m_cols(cols),
m_bytes_per_frame((m_bitdepth / 8) * m_rows * m_cols), m_pos_row(pos_row),
m_pos_col(pos_col) {
: m_detector_type(detector), m_bitdepth(bitdepth), m_rows(rows),
m_cols(cols), m_bytes_per_frame((m_bitdepth / 8) * m_rows * m_cols),
m_pos_row(pos_row), m_pos_col(pos_col) {
LOG(logDEBUG) << "RawSubFile::RawSubFile()";
if (m_detector_type == DetectorType::Moench03_old) {
@ -30,7 +25,6 @@ RawSubFile::RawSubFile(const std::filesystem::path &fname,
m_pixel_map = GenerateEigerFlipRowsPixelMap();
}
parse_fname(fname);
scan_files();
open_file(m_current_file_index); // open the first file
@ -51,7 +45,8 @@ void RawSubFile::seek(size_t frame_index) {
auto frame_offset = (file_index)
? frame_index - m_last_frame_in_file[file_index - 1]
: frame_index;
auto byte_offset = frame_offset * (m_bytes_per_frame + sizeof(DetectorHeader));
auto byte_offset =
frame_offset * (m_bytes_per_frame + sizeof(DetectorHeader));
m_file.seekg(byte_offset);
}
@ -85,7 +80,8 @@ void RawSubFile::read_into(std::byte *image_buf, DetectorHeader *header) {
} else if (m_bitdepth == 32) {
read_with_map<uint32_t>(image_buf);
} else {
throw std::runtime_error("Unsupported bitdepth for read with pixel map");
throw std::runtime_error(
"Unsupported bitdepth for read with pixel map");
}
} else {
@ -105,7 +101,8 @@ void RawSubFile::read_into(std::byte *image_buf, DetectorHeader *header) {
}
}
void RawSubFile::read_into(std::byte *image_buf, size_t n_frames, DetectorHeader *header) {
void RawSubFile::read_into(std::byte *image_buf, size_t n_frames,
DetectorHeader *header) {
for (size_t i = 0; i < n_frames; i++) {
read_into(image_buf, header);
image_buf += bytes_per_frame();
@ -115,10 +112,7 @@ void RawSubFile::read_into(std::byte *image_buf, size_t n_frames, DetectorHeader
}
}
template <typename T>
void RawSubFile::read_with_map(std::byte *image_buf) {
template <typename T> void RawSubFile::read_with_map(std::byte *image_buf) {
auto part_buffer = new std::byte[bytes_per_frame()];
m_file.read(reinterpret_cast<char *>(part_buffer), bytes_per_frame());
auto *data = reinterpret_cast<T *>(image_buf);
@ -157,14 +151,17 @@ void RawSubFile::parse_fname(const std::filesystem::path &fname) {
std::smatch match;
if (std::regex_match(m_base_name, match, pattern)) {
m_offset = std::stoi(match[4].str()); // find the first file index in case of a truncated series
m_base_name = match[1].str() + match[2].str() + match[3].str() + "{}" + match[5].str();
m_offset = std::stoi(match[4].str()); // find the first file index in
// case of a truncated series
m_base_name = match[1].str() + match[2].str() + match[3].str() + "{}" +
match[5].str();
LOG(logDEBUG) << "Base name: " << m_base_name;
LOG(logDEBUG) << "Offset: " << m_offset;
LOG(logDEBUG) << "Path: " << m_path.string();
} else {
throw std::runtime_error(
LOCATION + fmt::format("Could not parse file name {}", fname.string()));
LOCATION +
fmt::format("Could not parse file name {}", fname.string()));
}
}
@ -180,7 +177,8 @@ void RawSubFile::open_file(size_t file_index) {
m_file.open(fname, std::ios::binary);
if (!m_file.is_open()) {
throw std::runtime_error(
LOCATION + fmt::format("Could not open file {}", fpath(file_index).string()));
LOCATION +
fmt::format("Could not open file {}", fpath(file_index).string()));
}
m_current_file_index = file_index;
}
@ -195,7 +193,8 @@ void RawSubFile::scan_files() {
auto n_frames = std::filesystem::file_size(fpath(file_index)) /
(m_bytes_per_frame + sizeof(DetectorHeader));
m_last_frame_in_file.push_back(n_frames);
LOG(logDEBUG) << "Found: " << n_frames << " frames in file: " << fpath(file_index).string();
LOG(logDEBUG) << "Found: " << n_frames
<< " frames in file: " << fpath(file_index).string();
++file_index;
}