mirror of
https://github.com/slsdetectorgroup/aare.git
synced 2025-06-15 00:37:13 +02:00
RawFile is now using RawMasterFile
This commit is contained in:
Erik Fröjdh
350
src/RawFile.cpp
350
src/RawFile.cpp
@ -1,7 +1,7 @@
|
||||
#include "aare/RawFile.hpp"
|
||||
#include "aare/PixelMap.hpp"
|
||||
#include "aare/defs.hpp"
|
||||
#include "aare/json.hpp"
|
||||
#include "aare/PixelMap.hpp"
|
||||
|
||||
#include <fmt/format.h>
|
||||
#include <nlohmann/json.hpp>
|
||||
@ -10,38 +10,80 @@ using json = nlohmann::json;
|
||||
|
||||
namespace aare {
|
||||
|
||||
RawFile::RawFile(const std::filesystem::path &fname, const std::string &mode, const FileConfig &config) {
|
||||
RawFile::RawFile(const std::filesystem::path &fname, const std::string &mode)
|
||||
: m_master(fname) {
|
||||
m_mode = mode;
|
||||
m_fname = fname;
|
||||
if (mode == "r") {
|
||||
if (config != FileConfig()) {
|
||||
// aare::logger::warn(
|
||||
// "In read mode it is not necessary to provide a config, the provided config will be ignored");
|
||||
}
|
||||
parse_fname();
|
||||
parse_metadata();
|
||||
find_number_of_subfiles();
|
||||
n_subfile_parts = m_master.geometry().col * m_master.geometry().row;
|
||||
find_geometry();
|
||||
open_subfiles();
|
||||
} else {
|
||||
throw std::runtime_error(LOCATION + "Unsupported mode. Can only read RawFiles.");
|
||||
throw std::runtime_error(LOCATION +
|
||||
"Unsupported mode. Can only read RawFiles.");
|
||||
}
|
||||
}
|
||||
|
||||
Frame RawFile::read_frame() { return get_frame(m_current_frame++); };
|
||||
|
||||
Frame RawFile::read_frame(size_t frame_number) {
|
||||
seek(frame_number);
|
||||
return read_frame();
|
||||
}
|
||||
|
||||
void RawFile::read_into(std::byte *image_buf, size_t n_frames) {
|
||||
// TODO: implement this in a more efficient way
|
||||
|
||||
for (size_t i = 0; i < n_frames; i++) {
|
||||
this->get_frame_into(m_current_frame++, image_buf);
|
||||
image_buf += bytes_per_frame();
|
||||
}
|
||||
}
|
||||
void RawFile::read_into(std::byte *image_buf) {
|
||||
return get_frame_into(m_current_frame++, image_buf);
|
||||
};
|
||||
|
||||
size_t RawFile::bytes_per_frame() {
|
||||
return m_rows * m_cols * m_master.bitdepth() / 8;
|
||||
}
|
||||
size_t RawFile::pixels_per_frame() { return m_rows * m_cols; }
|
||||
|
||||
DetectorType RawFile::detector_type() const { return m_master.detector_type(); }
|
||||
|
||||
void RawFile::seek(size_t frame_index) {
|
||||
// check if the frame number is greater than the total frames
|
||||
// if frame_number == total_frames, then the next read will throw an error
|
||||
if (frame_index > total_frames()) {
|
||||
throw std::runtime_error(
|
||||
fmt::format("frame number {} is greater than total frames {}",
|
||||
frame_index, total_frames()));
|
||||
}
|
||||
m_current_frame = frame_index;
|
||||
};
|
||||
|
||||
size_t RawFile::tell() { return m_current_frame; };
|
||||
|
||||
size_t RawFile::total_frames() const { return m_master.frames_in_file(); }
|
||||
size_t RawFile::rows() const { return m_rows; }
|
||||
size_t RawFile::cols() const { return m_cols; }
|
||||
size_t RawFile::bitdepth() const { return m_master.bitdepth(); }
|
||||
xy RawFile::geometry() { return m_master.geometry(); }
|
||||
|
||||
void RawFile::open_subfiles() {
|
||||
if (m_mode == "r")
|
||||
for (size_t i = 0; i != n_subfiles; ++i) {
|
||||
auto v = std::vector<SubFile *>(n_subfile_parts);
|
||||
for (size_t j = 0; j != n_subfile_parts; ++j) {
|
||||
v[j] = new SubFile(data_fname(i, j), m_type, subfile_rows, subfile_cols, m_bitdepth);
|
||||
v[j] =
|
||||
new SubFile(m_master.data_fname(j, i),
|
||||
m_master.detector_type(), m_master.pixels_y(),
|
||||
m_master.pixels_x(), m_master.bitdepth());
|
||||
}
|
||||
subfiles.push_back(v);
|
||||
}
|
||||
else {
|
||||
auto v = std::vector<SubFile *>(n_subfile_parts); // only one subfile is implemented
|
||||
v[0] = new SubFile(data_fname(0, 0), m_type, m_rows, m_cols, m_bitdepth, "w");
|
||||
subfiles.push_back(v);
|
||||
throw std::runtime_error(LOCATION +
|
||||
"Unsupported mode. Can only read RawFiles.");
|
||||
}
|
||||
}
|
||||
|
||||
@ -49,7 +91,8 @@ DetectorHeader RawFile::read_header(const std::filesystem::path &fname) {
|
||||
DetectorHeader h{};
|
||||
FILE *fp = fopen(fname.string().c_str(), "r");
|
||||
if (!fp)
|
||||
throw std::runtime_error(fmt::format("Could not open: {} for reading", fname.string()));
|
||||
throw std::runtime_error(
|
||||
fmt::format("Could not open: {} for reading", fname.string()));
|
||||
|
||||
size_t const rc = fread(reinterpret_cast<char *>(&h), sizeof(h), 1, fp);
|
||||
if (rc != 1)
|
||||
@ -67,16 +110,12 @@ bool RawFile::is_master_file(const std::filesystem::path &fpath) {
|
||||
|
||||
void RawFile::find_number_of_subfiles() {
|
||||
int n_mod = 0;
|
||||
while (std::filesystem::exists(data_fname(++n_mod, 0)))
|
||||
while (std::filesystem::exists(m_master.data_fname(0, ++n_mod)))
|
||||
;
|
||||
n_subfiles = n_mod;
|
||||
}
|
||||
std::filesystem::path RawFile::data_fname(size_t mod_id, size_t file_id) {
|
||||
return this->m_base_path / fmt::format("{}_d{}_f{}_{}.raw", this->m_base_name, file_id, mod_id, this->m_findex);
|
||||
}
|
||||
|
||||
std::filesystem::path RawFile::master_fname() {
|
||||
return this->m_base_path / fmt::format("{}_master_{}{}", this->m_base_name, this->m_findex, this->m_ext);
|
||||
#ifdef AARE_VERBOSE
|
||||
fmt::print("Found: {} subfiles\n", n_subfiles);
|
||||
#endif
|
||||
}
|
||||
|
||||
void RawFile::find_geometry() {
|
||||
@ -84,7 +123,7 @@ void RawFile::find_geometry() {
|
||||
uint16_t c{};
|
||||
for (size_t i = 0; i < n_subfile_parts; i++) {
|
||||
for (size_t j = 0; j != n_subfiles; ++j) {
|
||||
auto h = this->read_header(data_fname(j, i));
|
||||
auto h = this->read_header(m_master.data_fname(i, j));
|
||||
r = std::max(r, h.row);
|
||||
c = std::max(c, h.column);
|
||||
|
||||
@ -95,268 +134,129 @@ void RawFile::find_geometry() {
|
||||
r++;
|
||||
c++;
|
||||
|
||||
m_rows = (r * subfile_rows);
|
||||
m_cols = (c * subfile_cols);
|
||||
m_rows = (r * m_master.pixels_y());
|
||||
m_cols = (c * m_master.pixels_x());
|
||||
|
||||
m_rows += static_cast<size_t>((r - 1) * cfg.module_gap_row);
|
||||
}
|
||||
|
||||
void RawFile::parse_metadata() {
|
||||
if (m_ext == ".raw") {
|
||||
parse_raw_metadata();
|
||||
if (m_bitdepth == 0) {
|
||||
switch (m_type) {
|
||||
case DetectorType::Eiger:
|
||||
m_bitdepth = 32;
|
||||
break;
|
||||
default:
|
||||
m_bitdepth = 16;
|
||||
}
|
||||
}
|
||||
} else if (m_ext == ".json") {
|
||||
parse_json_metadata();
|
||||
} else {
|
||||
throw std::runtime_error(LOCATION + "Unsupported file type");
|
||||
}
|
||||
n_subfile_parts = static_cast<size_t>(m_geometry.row) * m_geometry.col;
|
||||
}
|
||||
|
||||
void RawFile::parse_json_metadata() {
|
||||
std::ifstream ifs(master_fname());
|
||||
json j;
|
||||
ifs >> j;
|
||||
double v = j["Version"];
|
||||
version = fmt::format("{:.1f}", v);
|
||||
m_type = StringTo<DetectorType>(j["Detector Type"].get<std::string>());
|
||||
timing_mode = StringTo<TimingMode>(j["Timing Mode"].get<std::string>());
|
||||
m_total_frames = j["Frames in File"];
|
||||
subfile_rows = j["Pixels"]["y"];
|
||||
subfile_cols = j["Pixels"]["x"];
|
||||
max_frames_per_file = j["Max Frames Per File"];
|
||||
try {
|
||||
m_bitdepth = j.at("Dynamic Range");
|
||||
} catch (const json::out_of_range &e) {
|
||||
m_bitdepth = 16;
|
||||
}
|
||||
|
||||
try {
|
||||
m_analog_samples = j.at("Analog Samples");
|
||||
}catch (const json::out_of_range &e) {
|
||||
m_analog_samples = 0;
|
||||
}
|
||||
try{
|
||||
std::string adc_mask = j.at("ADC Mask");
|
||||
m_adc_mask = std::stoul(adc_mask, nullptr, 16);
|
||||
// fmt::print("ADC Mask: {}, n_set: {}\n", m_adc_mask, __builtin_popcount(m_adc_mask));
|
||||
}catch (const json::out_of_range &e) {
|
||||
m_adc_mask = 0;
|
||||
}
|
||||
|
||||
|
||||
try {
|
||||
m_digital_samples = j.at("Digital Samples");
|
||||
}catch (const json::out_of_range &e) {
|
||||
m_digital_samples = 0;
|
||||
}
|
||||
|
||||
//Update detector type for Moench
|
||||
//TODO! How does this work with old .raw master files?
|
||||
if (m_type == DetectorType::Moench && m_analog_samples == 0 && subfile_rows == 400) {
|
||||
m_type = DetectorType::Moench03;
|
||||
}else if (m_type == DetectorType::Moench && subfile_rows == 400 && m_analog_samples == 5000) {
|
||||
m_type = DetectorType::Moench03_old;
|
||||
}
|
||||
|
||||
//Here we know we have a ChipTestBoard file update the geometry?
|
||||
//TODO! Carry on information about digtial, and transceivers
|
||||
if (m_type == DetectorType::ChipTestBoard) {
|
||||
subfile_rows = 1;
|
||||
subfile_cols = m_analog_samples*__builtin_popcount(m_adc_mask);
|
||||
}
|
||||
|
||||
|
||||
|
||||
// only Eiger had quad
|
||||
if (m_type == DetectorType::Eiger) {
|
||||
quad = (j["Quad"] == 1);
|
||||
}
|
||||
|
||||
m_geometry = {j["Geometry"]["y"], j["Geometry"]["x"]};
|
||||
}
|
||||
void RawFile::parse_raw_metadata() {
|
||||
std::ifstream ifs(master_fname());
|
||||
for (std::string line; std::getline(ifs, line);) {
|
||||
if (line == "#Frame Header")
|
||||
break;
|
||||
auto pos = line.find(':');
|
||||
auto key_pos = pos;
|
||||
while (key_pos != std::string::npos && std::isspace(line[--key_pos]))
|
||||
;
|
||||
if (key_pos != std::string::npos) {
|
||||
auto key = line.substr(0, key_pos + 1);
|
||||
auto value = line.substr(pos + 2);
|
||||
// do the actual parsing
|
||||
if (key == "Version") {
|
||||
version = value;
|
||||
} else if (key == "TimeStamp") {
|
||||
|
||||
} else if (key == "Detector Type") {
|
||||
m_type = StringTo<DetectorType>(value);
|
||||
} else if (key == "Timing Mode") {
|
||||
timing_mode = StringTo<TimingMode>(value);
|
||||
} else if (key == "Pixels") {
|
||||
// Total number of pixels cannot be found yet looking at
|
||||
// submodule
|
||||
pos = value.find(',');
|
||||
subfile_cols = std::stoi(value.substr(1, pos));
|
||||
subfile_rows = std::stoi(value.substr(pos + 1));
|
||||
} else if (key == "Total Frames") {
|
||||
m_total_frames = std::stoi(value);
|
||||
} else if (key == "Dynamic Range") {
|
||||
m_bitdepth = std::stoi(value);
|
||||
} else if (key == "Quad") {
|
||||
quad = (value == "1");
|
||||
} else if (key == "Max Frames Per File") {
|
||||
max_frames_per_file = std::stoi(value);
|
||||
} else if (key == "Geometry") {
|
||||
pos = value.find(',');
|
||||
m_geometry = {static_cast<uint32_t>(std::stoi(value.substr(1, pos))),
|
||||
static_cast<uint32_t>(std::stoi(value.substr(pos + 1)))};
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void RawFile::parse_fname() {
|
||||
bool wrong_format = false;
|
||||
m_base_path = m_fname.parent_path().string();
|
||||
m_base_name = m_fname.stem().string();
|
||||
m_ext = m_fname.extension().string();
|
||||
try {
|
||||
auto pos = m_base_name.rfind('_');
|
||||
m_findex = std::stoi(m_base_name.substr(pos + 1));
|
||||
} catch (const std::invalid_argument &e) {
|
||||
m_findex = 0;
|
||||
wrong_format = true;
|
||||
}
|
||||
auto pos = m_base_name.find("_master_");
|
||||
if (pos != std::string::npos) {
|
||||
m_base_name.erase(pos);
|
||||
wrong_format = true;
|
||||
}
|
||||
if (wrong_format && (m_mode == "w+" || m_mode == "w")) {
|
||||
// aare::logger::warn("Master Filename", m_fname, "is not in the correct format");
|
||||
// aare::logger::warn("using", master_fname(), "as the master file");
|
||||
}
|
||||
}
|
||||
|
||||
Frame RawFile::get_frame(size_t frame_index) {
|
||||
auto f = Frame(this->m_rows, this->m_cols, Dtype::from_bitdepth(this->m_bitdepth));
|
||||
auto f = Frame(m_rows, m_cols, Dtype::from_bitdepth(m_master.bitdepth()));
|
||||
std::byte *frame_buffer = f.data();
|
||||
get_frame_into(frame_index, frame_buffer);
|
||||
return f;
|
||||
}
|
||||
|
||||
void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer) {
|
||||
if (frame_index > this->m_total_frames) {
|
||||
if (frame_index > total_frames()) {
|
||||
throw std::runtime_error(LOCATION + "Frame number out of range");
|
||||
}
|
||||
std::vector<size_t> frame_numbers(this->n_subfile_parts);
|
||||
std::vector<size_t> frame_indices(this->n_subfile_parts, frame_index);
|
||||
std::vector<size_t> frame_numbers(n_subfile_parts);
|
||||
std::vector<size_t> frame_indices(n_subfile_parts, frame_index);
|
||||
|
||||
if (n_subfile_parts != 1) {
|
||||
for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) {
|
||||
auto subfile_id = frame_index / this->max_frames_per_file;
|
||||
for (size_t part_idx = 0; part_idx != n_subfile_parts; ++part_idx) {
|
||||
auto subfile_id = frame_index / m_master.max_frames_per_file();
|
||||
frame_numbers[part_idx] =
|
||||
this->subfiles[subfile_id][part_idx]->frame_number(frame_index % this->max_frames_per_file);
|
||||
subfiles[subfile_id][part_idx]->frame_number(
|
||||
frame_index % m_master.max_frames_per_file());
|
||||
}
|
||||
// 1. if frame number vector is the same break
|
||||
while (std::adjacent_find(frame_numbers.begin(), frame_numbers.end(), std::not_equal_to<>()) !=
|
||||
while (std::adjacent_find(frame_numbers.begin(), frame_numbers.end(),
|
||||
std::not_equal_to<>()) !=
|
||||
frame_numbers.end()) {
|
||||
// 2. find the index of the minimum frame number,
|
||||
auto min_frame_idx =
|
||||
std::distance(frame_numbers.begin(), std::min_element(frame_numbers.begin(), frame_numbers.end()));
|
||||
auto min_frame_idx = std::distance(
|
||||
frame_numbers.begin(),
|
||||
std::min_element(frame_numbers.begin(), frame_numbers.end()));
|
||||
// 3. increase its index and update its respective frame number
|
||||
frame_indices[min_frame_idx]++;
|
||||
// 4. if we can't increase its index => throw error
|
||||
if (frame_indices[min_frame_idx] >= this->m_total_frames) {
|
||||
throw std::runtime_error(LOCATION + "Frame number out of range");
|
||||
if (frame_indices[min_frame_idx] >= total_frames()) {
|
||||
throw std::runtime_error(LOCATION +
|
||||
"Frame number out of range");
|
||||
}
|
||||
auto subfile_id = frame_indices[min_frame_idx] / this->max_frames_per_file;
|
||||
frame_numbers[min_frame_idx] = this->subfiles[subfile_id][min_frame_idx]->frame_number(
|
||||
frame_indices[min_frame_idx] % this->max_frames_per_file);
|
||||
auto subfile_id =
|
||||
frame_indices[min_frame_idx] / m_master.max_frames_per_file();
|
||||
frame_numbers[min_frame_idx] =
|
||||
subfiles[subfile_id][min_frame_idx]->frame_number(
|
||||
frame_indices[min_frame_idx] %
|
||||
m_master.max_frames_per_file());
|
||||
}
|
||||
}
|
||||
|
||||
if (this->m_geometry.col == 1) {
|
||||
if (m_master.geometry().col == 1) {
|
||||
// get the part from each subfile and copy it to the frame
|
||||
for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) {
|
||||
for (size_t part_idx = 0; part_idx != n_subfile_parts; ++part_idx) {
|
||||
auto corrected_idx = frame_indices[part_idx];
|
||||
auto subfile_id = corrected_idx / this->max_frames_per_file;
|
||||
auto part_offset = this->subfiles[subfile_id][part_idx]->bytes_per_part();
|
||||
this->subfiles[subfile_id][part_idx]->get_part(frame_buffer + part_idx * part_offset,
|
||||
corrected_idx % this->max_frames_per_file);
|
||||
auto subfile_id = corrected_idx / m_master.max_frames_per_file();
|
||||
auto part_offset = subfiles[subfile_id][part_idx]->bytes_per_part();
|
||||
subfiles[subfile_id][part_idx]->get_part(
|
||||
frame_buffer + part_idx * part_offset,
|
||||
corrected_idx % m_master.max_frames_per_file());
|
||||
}
|
||||
|
||||
} else {
|
||||
|
||||
// create a buffer that will hold a the frame part
|
||||
auto bytes_per_part = this->subfile_rows * this->subfile_cols * this->m_bitdepth / 8;
|
||||
auto bytes_per_part = m_master.pixels_y() * m_master.pixels_x() *
|
||||
m_master.bitdepth() /
|
||||
8; // TODO! replace with image_size_in_bytes
|
||||
auto *part_buffer = new std::byte[bytes_per_part];
|
||||
|
||||
//TODO! if we have many submodules we should reorder them on the module level
|
||||
// TODO! if we have many submodules we should reorder them on the module
|
||||
// level
|
||||
|
||||
for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) {
|
||||
for (size_t part_idx = 0; part_idx != n_subfile_parts; ++part_idx) {
|
||||
auto corrected_idx = frame_indices[part_idx];
|
||||
auto subfile_id = corrected_idx / this->max_frames_per_file;
|
||||
auto subfile_id = corrected_idx / m_master.max_frames_per_file();
|
||||
|
||||
this->subfiles[subfile_id][part_idx]->get_part(part_buffer, corrected_idx % this->max_frames_per_file);
|
||||
for (size_t cur_row = 0; cur_row < (this->subfile_rows); cur_row++) {
|
||||
auto irow = cur_row + (part_idx / this->m_geometry.col) * this->subfile_rows;
|
||||
auto icol = (part_idx % this->m_geometry.col) * this->subfile_cols;
|
||||
subfiles[subfile_id][part_idx]->get_part(
|
||||
part_buffer, corrected_idx % m_master.max_frames_per_file());
|
||||
for (size_t cur_row = 0; cur_row < (m_master.pixels_y());
|
||||
cur_row++) {
|
||||
auto irow = cur_row + (part_idx / m_master.geometry().col) *
|
||||
m_master.pixels_y();
|
||||
auto icol =
|
||||
(part_idx % m_master.geometry().col) * m_master.pixels_x();
|
||||
auto dest = (irow * this->m_cols + icol);
|
||||
dest = dest * this->m_bitdepth / 8;
|
||||
memcpy(frame_buffer + dest, part_buffer + cur_row * this->subfile_cols * this->m_bitdepth / 8,
|
||||
this->subfile_cols * this->m_bitdepth / 8);
|
||||
dest = dest * m_master.bitdepth() / 8;
|
||||
memcpy(frame_buffer + dest,
|
||||
part_buffer + cur_row * m_master.pixels_x() *
|
||||
m_master.bitdepth() / 8,
|
||||
m_master.pixels_x() * m_master.bitdepth() / 8);
|
||||
}
|
||||
}
|
||||
delete[] part_buffer;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
std::vector<Frame> RawFile::read_n(size_t n_frames) {
|
||||
// TODO: implement this in a more efficient way
|
||||
std::vector<Frame> frames;
|
||||
for (size_t i = 0; i < n_frames; i++) {
|
||||
frames.push_back(this->get_frame(this->current_frame));
|
||||
this->current_frame++;
|
||||
frames.push_back(this->get_frame(m_current_frame));
|
||||
m_current_frame++;
|
||||
}
|
||||
return frames;
|
||||
}
|
||||
void RawFile::read_into(std::byte *image_buf, size_t n_frames) {
|
||||
// TODO: implement this in a more efficient way
|
||||
|
||||
for (size_t i = 0; i < n_frames; i++) {
|
||||
this->get_frame_into(this->current_frame++, image_buf);
|
||||
image_buf += this->bytes_per_frame();
|
||||
}
|
||||
}
|
||||
|
||||
size_t RawFile::frame_number(size_t frame_index) {
|
||||
if (frame_index >= this->m_total_frames) {
|
||||
if (frame_index >= m_master.frames_in_file()) {
|
||||
throw std::runtime_error(LOCATION + " Frame number out of range");
|
||||
}
|
||||
size_t subfile_id = frame_index / this->max_frames_per_file;
|
||||
if(subfile_id >= this->subfiles.size()){
|
||||
throw std::runtime_error(LOCATION + " Subfile out of range. Possible missing data.");
|
||||
size_t subfile_id = frame_index / m_master.max_frames_per_file();
|
||||
if (subfile_id >= subfiles.size()) {
|
||||
throw std::runtime_error(
|
||||
LOCATION + " Subfile out of range. Possible missing data.");
|
||||
}
|
||||
return this->subfiles[subfile_id][0]->frame_number(frame_index % this->max_frames_per_file);
|
||||
return subfiles[subfile_id][0]->frame_number(
|
||||
frame_index % m_master.max_frames_per_file());
|
||||
}
|
||||
|
||||
RawFile::~RawFile() noexcept {
|
||||
RawFile::~RawFile() {
|
||||
|
||||
//TODO! Fix this, for file closing
|
||||
// TODO! Fix this, for file closing
|
||||
for (auto &vec : subfiles) {
|
||||
for (auto *subfile : vec) {
|
||||
delete subfile;
|
||||
|
||||
Reference in New Issue
Block a user