RawFile is now using RawMasterFile

This commit is contained in:
Erik Fröjdh 2024-11-06 09:10:09 +01:00
parent 654c1db3f4
commit 25812cb291
20 changed files with 511 additions and 408 deletions

View File

@ -32,6 +32,7 @@ option(AARE_TESTS "Build tests" OFF)
option(AARE_EXAMPLES "Build examples" OFF)
option(AARE_IN_GITHUB_ACTIONS "Running in Github Actions" OFF)
option(AARE_DOCS "Build documentation" OFF)
option(AARE_VERBOSE "Verbose output" OFF)
# Configure which of the dependencies to use FetchContent for
option(AARE_FETCH_FMT "Use FetchContent to download fmt" ON)
@ -52,6 +53,10 @@ if(AARE_SYSTEM_LIBRARIES)
set(AARE_FETCH_ZMQ OFF CACHE BOOL "Disabled FetchContent for libzmq" FORCE)
endif()
if(AARE_VERBOSE)
add_compile_definitions(AARE_VERBOSE)
endif()
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)

View File

@ -19,11 +19,13 @@ set(SPHINX_SOURCE_FILES
src/Dtype.rst
src/ClusterFinder.rst
src/Pedestal.rst
src/RawFile.rst
src/RawMasterFile.rst
src/VarClusterFinder.rst
src/pyVarClusterFinder.rst
src/pyFile.rst
src/pyCtbRawFile.rst
src/pyRawMasterFile.rst
)
foreach(filename ${SPHINX_SOURCE_FILES})

8
docs/src/RawFile.rst Normal file
View File

@ -0,0 +1,8 @@
RawFile
===============
.. doxygenclass:: aare::RawFile
:members:
:undoc-members:
:private-members:

View File

@ -3,6 +3,12 @@ RawMasterFile
.. doxygenclass:: aare::RawMasterFile
:members:
:undoc-members:
:private-members:
.. doxygenclass:: aare::RawFileNameComponents
:members:
:undoc-members:
:private-members:

View File

@ -16,6 +16,7 @@ AARE
Dtype
ClusterFinder
Pedestal
RawFile
RawMasterFile
VarClusterFinder
@ -25,5 +26,6 @@ AARE
pyFile
pyCtbRawFile
pyRawMasterFile
pyVarClusterFinder

View File

@ -0,0 +1,10 @@
RawMasterFile
===================
.. py:currentmodule:: aare
.. autoclass:: RawMasterFile
:members:
:undoc-members:
:show-inheritance:
:inherited-members:

View File

@ -135,7 +135,7 @@ class FileInterface {
virtual size_t bitdepth() const = 0;
DetectorType detector_type() const { return m_type; }
virtual DetectorType detector_type() const = 0;
// function to query the data type of the file
/*virtual DataType dtype = 0; */
@ -144,18 +144,18 @@ class FileInterface {
protected:
std::string m_mode{};
std::filesystem::path m_fname{};
std::filesystem::path m_base_path{};
std::string m_base_name{}, m_ext{};
int m_findex{};
size_t m_total_frames{};
size_t max_frames_per_file{};
std::string version{};
DetectorType m_type{DetectorType::Unknown};
size_t m_rows{};
size_t m_cols{};
size_t m_bitdepth{};
size_t current_frame{};
// std::filesystem::path m_fname{};
// std::filesystem::path m_base_path{};
// std::string m_base_name{}, m_ext{};
// int m_findex{};
// size_t m_total_frames{};
// size_t max_frames_per_file{};
// std::string version{};
// DetectorType m_type{DetectorType::Unknown};
// size_t m_rows{};
// size_t m_cols{};
// size_t m_bitdepth{};
// size_t current_frame{};
};
} // namespace aare

View File

@ -47,6 +47,8 @@ class NumpyFile : public FileInterface {
size_t cols() const override { return m_header.shape[2]; }
size_t bitdepth() const override { return m_header.dtype.bitdepth(); }
DetectorType detector_type() const override { return DetectorType::Unknown; }
/**
* @brief get the data type of the numpy file
* @return DType
@ -104,6 +106,10 @@ class NumpyFile : public FileInterface {
size_t m_bytes_per_frame{};
size_t m_pixels_per_frame{};
size_t m_cols;
size_t m_rows;
size_t m_bitdepth;
void load_metadata();
void get_frame_into(size_t /*frame_number*/, std::byte * /*image_buf*/);
Frame get_frame(size_t frame_number);

View File

@ -1,7 +1,8 @@
#pragma once
#include "aare/FileInterface.hpp"
#include "aare/RawMasterFile.hpp"
#include "aare/Frame.hpp"
#include "aare/NDArray.hpp" //for pixel map
#include "aare/FileInterface.hpp"
#include "aare/SubFile.hpp"
#include <optional>
@ -22,122 +23,61 @@ struct ModuleConfig {
};
/**
* @brief RawFile class to read .raw and .json files
* @note derived from FileInterface
* @note documentation can also be found in the FileInterface class
* @brief Class to read .raw files. The class will parse the master file
* to find the correct geometry for the frames.
* @note A more generic interface is available in the aare::File class.
* Consider using that unless you need raw file specific functionality.
*/
class RawFile : public FileInterface {
size_t n_subfiles{};
size_t n_subfile_parts{};
std::vector<std::vector<SubFile *>> subfiles;
size_t subfile_rows{}, subfile_cols{};
xy m_geometry{};
std::vector<xy> positions;
ModuleConfig cfg{0, 0};
TimingMode timing_mode{};
bool quad{false};
//Stuff that we might need with Ctb files
uint32_t m_analog_samples{};
uint32_t m_digital_samples{};
uint32_t m_adc_mask{};
RawMasterFile m_master;
size_t m_current_frame{};
size_t m_rows{};
size_t m_cols{};
public:
/**
* @brief RawFile constructor
* @param fname path to the file
* @param mode file mode (r, w)
* @param cfg file configuration
* @param fname path to the master file (.json)
* @param mode file mode (only "r" is supported at the moment)
*/
RawFile(const std::filesystem::path &fname, const std::string &mode = "r",
const FileConfig &config = FileConfig{});
RawFile(const std::filesystem::path &fname, const std::string &mode = "r");
virtual ~RawFile() override;
Frame read_frame() override { return get_frame(this->current_frame++); };
Frame read_frame(size_t frame_number) override{
seek(frame_number);
return read_frame();
}
Frame read_frame() override;
Frame read_frame(size_t frame_number) override;
std::vector<Frame> read_n(size_t n_frames) override;
void read_into(std::byte *image_buf) override { return get_frame_into(this->current_frame++, image_buf); };
void read_into(std::byte *image_buf) override;
void read_into(std::byte *image_buf, size_t n_frames) override;
size_t frame_number(size_t frame_index) override;
size_t bytes_per_frame() override;
size_t pixels_per_frame() override;
void seek(size_t frame_index) override;
size_t tell() override;
size_t total_frames() const override;
size_t rows() const override;
size_t cols() const override;
size_t bitdepth() const override;
xy geometry();
/**
* @brief get the number of bytess per frame
* @return size of one frame in bytes
*/
size_t bytes_per_frame() override { return m_rows * m_cols * m_bitdepth / 8; }
DetectorType detector_type() const override;
/**
* @brief get the number of pixels in the frame
* @return number of pixels
*/
size_t pixels_per_frame() override { return m_rows * m_cols; }
// goto frame index
void seek(size_t frame_index) override {
// check if the frame number is greater than the total frames
// if frame_number == total_frames, then the next read will throw an error
if (frame_index > this->total_frames()) {
throw std::runtime_error(
fmt::format("frame number {} is greater than total frames {}", frame_index, m_total_frames));
}
this->current_frame = frame_index;
};
// return the position of the file pointer (in number of frames)
size_t tell() override { return this->current_frame; };
/**
private:
/**
* @brief check if the file is a master file
* @param fpath path to the file
*/
static bool is_master_file(const std::filesystem::path &fpath);
/**
* @brief set the module gap row and column
* @param row gap between rows
* @param col gap between columns
*/
inline void set_config(int row, int col) {
cfg.module_gap_row = row;
cfg.module_gap_col = col;
}
// TODO! Deal with fast quad and missing files
// TODO! Deal with fast quad and missing files
/**
* @brief get the number of subfiles for the RawFile
* @return number of subfiles
*/
void find_number_of_subfiles();
/**
* @brief get the master file name path for the RawFile
* @return path to the master file
*/
inline std::filesystem::path master_fname();
/**
* @brief get the data file name path for the RawFile with the given module id and file id
* @param mod_id module id
* @param file_id file id
* @return path to the data file
*/
inline std::filesystem::path data_fname(size_t mod_id, size_t file_id);
/**
* @brief destructor: will delete the subfiles
*/
~RawFile() noexcept override;
size_t total_frames() const override { return m_total_frames; }
size_t rows() const override { return m_rows; }
size_t cols() const override { return m_cols; }
size_t bitdepth() const override { return m_bitdepth; }
xy geometry() { return m_geometry; }
private:
/**
* @brief read the frame at the given frame index into the image buffer
* @param frame_number frame number to read
@ -152,30 +92,7 @@ class RawFile : public FileInterface {
*/
Frame get_frame(size_t frame_index);
/**
* @brief parse the file name to get the extension, base name and index
*/
void parse_fname();
/**
* @brief parse the metadata from the file
*/
void parse_metadata();
/**
* @brief parse the metadata of a .raw file
*/
void parse_raw_metadata();
/**
* @brief parse the metadata of a .json file
*/
void parse_json_metadata();
/**
* @brief finds the geometry of the file
*/
void find_geometry();
/**
* @brief read the header of the file
@ -184,11 +101,10 @@ class RawFile : public FileInterface {
*/
static DetectorHeader read_header(const std::filesystem::path &fname);
/**
* @brief open the subfiles
*/
void open_subfiles();
void find_number_of_subfiles();
void open_subfiles();
void find_geometry();
};
} // namespace aare

View File

@ -10,6 +10,9 @@ using json = nlohmann::json;
namespace aare {
/**
* @brief Implementation used in RawMasterFile to parse the file name
*/
class RawFileNameComponents {
std::filesystem::path m_base_path{};
std::string m_base_name{};
@ -18,20 +21,21 @@ class RawFileNameComponents {
public:
RawFileNameComponents(const std::filesystem::path &fname);
std::filesystem::path master_fname() const {
return m_base_path /
fmt::format("{}_master_{}{}", m_base_name, m_file_index, m_ext);
}
std::filesystem::path data_fname(size_t mod_id, size_t file_id) const{
return m_base_path / fmt::format("{}_d{}_f{}_{}.raw", m_base_name,
mod_id, file_id, m_file_index);
}
/// @brief Get the filename including path of the master file.
/// (i.e. what was passed in to the constructor))
std::filesystem::path master_fname() const;
const std::filesystem::path &base_path() const { return m_base_path; }
const std::string &base_name() const { return m_base_name; }
const std::string &ext() const { return m_ext; }
int file_index() const { return m_file_index; }
/// @brief Get the filename including path of the data file.
/// @param mod_id module id run_d[module_id]_f0_0
/// @param file_id file id run_d0_f[file_id]_0
std::filesystem::path data_fname(size_t mod_id, size_t file_id) const;
const std::filesystem::path &base_path() const;
const std::string &base_name() const;
const std::string &ext() const;
int file_index() const;
};
@ -44,19 +48,28 @@ class RawMasterFile {
DetectorType m_type;
TimingMode m_timing_mode;
size_t m_image_size_in_bytes;
size_t m_frames_in_file;
size_t m_pixels_y;
size_t m_pixels_x;
size_t m_bitdepth;
size_t m_image_size_in_bytes{};
size_t m_frames_in_file{};
size_t m_total_frames_expected{};
size_t m_pixels_y{};
size_t m_pixels_x{};
size_t m_bitdepth{};
size_t m_max_frames_per_file;
uint32_t m_adc_mask;
FrameDiscardPolicy m_frame_discard_policy;
size_t m_frame_padding;
xy m_geometry;
size_t m_max_frames_per_file{};
uint32_t m_adc_mask{};
FrameDiscardPolicy m_frame_discard_policy{};
size_t m_frame_padding{};
//TODO! should these be bool?
uint8_t m_analog_flag{};
uint8_t m_digital_flag{};
std::optional<size_t> m_analog_samples;
std::optional<size_t> m_digital_samples;
std::optional<size_t> m_number_of_rows;
std::optional<uint8_t> m_quad;
public:
RawMasterFile(const std::filesystem::path &fpath);
@ -75,11 +88,13 @@ class RawMasterFile {
size_t frame_padding() const;
const FrameDiscardPolicy &frame_discard_policy() const;
size_t total_frames_expected() const;
xy geometry() const;
std::optional<size_t> analog_samples() const;
std::optional<size_t> digital_samples() const;
std::optional<size_t> number_of_rows() const;
std::optional<uint8_t> quad() const;
private:
void parse_json(const std::filesystem::path &fpath);
void parse_raw(const std::filesystem::path &fpath);

View File

@ -1,5 +1,5 @@
# Make the compiled classes that live in _aare available from aare.
from . import _aare
from ._aare import VarClusterFinder, File
from ._aare import VarClusterFinder, File, RawMasterFile
from .CtbRawFile import CtbRawFile

View File

@ -26,7 +26,8 @@ fpath = Path('/Users/erik/data/Moench05/moench05_multifile_master_0.json')
with CtbRawFile(fpath, transform = transform.moench05) as f:
for header, image in f:
print(f'Frame number: {header["frameNumber"]}')
# with CtbRawFile(fpath, transform = transform.moench05) as f:
# for header, image in f:
# print(f'Frame number: {header["frameNumber"]}')
f = aare.RawMasterFile(fpath)

View File

@ -1,4 +1,5 @@
#include "aare/CtbRawFile.hpp"
#include "aare/RawMasterFile.hpp"
#include "aare/File.hpp"
#include "aare/Frame.hpp"
#include "aare/defs.hpp"
@ -149,6 +150,25 @@ void define_file_io_bindings(py::module &m) {
return "<FileConfig: " + a.to_string() + ">";
});
py::class_<RawMasterFile>(m, "RawMasterFile")
.def(py::init<const std::filesystem::path &>())
.def("data_fname", &RawMasterFile::data_fname)
.def_property_readonly("version", &RawMasterFile::version)
.def_property_readonly("detector_type", &RawMasterFile::detector_type)
.def_property_readonly("timing_mode", &RawMasterFile::timing_mode)
.def_property_readonly("image_size_in_bytes",
&RawMasterFile::image_size_in_bytes)
.def_property_readonly("frames_in_file", &RawMasterFile::frames_in_file)
.def_property_readonly("pixels_y", &RawMasterFile::pixels_y)
.def_property_readonly("pixels_x", &RawMasterFile::pixels_x)
.def_property_readonly("max_frames_per_file",
&RawMasterFile::max_frames_per_file)
.def_property_readonly("bitdepth", &RawMasterFile::bitdepth)
.def_property_readonly("analog_samples", &RawMasterFile::analog_samples)
.def_property_readonly("digital_samples",
&RawMasterFile::digital_samples);
// py::class_<ClusterHeader>(m, "ClusterHeader")
// .def(py::init<>())
// .def_readwrite("frame_number", &ClusterHeader::frame_number)

View File

@ -22,7 +22,7 @@ File::File(const std::filesystem::path &fname, const std::string &mode,
// TODO! How do we read raw files directly?
if (fname.extension() == ".raw" || fname.extension() == ".json") {
// file_impl = new RawFile(fname, mode, cfg);
file_impl = std::make_unique<RawFile>(fname, mode, cfg);
file_impl = std::make_unique<RawFile>(fname, mode);
}
else if (fname.extension() == ".npy") {
// file_impl = new NumpyFile(fname, mode, cfg);

View File

@ -8,12 +8,12 @@ namespace aare {
NumpyFile::NumpyFile(const std::filesystem::path &fname, const std::string &mode, FileConfig cfg) {
// TODO! add opts to constructor
m_fname = fname;
m_mode = mode;
if (mode == "r") {
fp = fopen(m_fname.string().c_str(), "rb");
fp = fopen(fname.string().c_str(), "rb");
if (!fp) {
throw std::runtime_error(fmt::format("Could not open: {} for reading", m_fname.string()));
throw std::runtime_error(fmt::format("Could not open: {} for reading", fname.string()));
}
load_metadata();
} else if (mode == "w") {
@ -22,11 +22,11 @@ NumpyFile::NumpyFile(const std::filesystem::path &fname, const std::string &mode
m_cols = cfg.cols;
m_header = {cfg.dtype, false, {cfg.rows, cfg.cols}};
m_header.shape = {0, cfg.rows, cfg.cols};
fp = fopen(m_fname.string().c_str(), "wb");
fp = fopen(fname.string().c_str(), "wb");
if (!fp) {
throw std::runtime_error(fmt::format("Could not open: {} for reading", m_fname.string()));
throw std::runtime_error(fmt::format("Could not open: {} for reading", fname.string()));
}
initial_header_len = aare::NumpyHelpers::write_header(std::filesystem::path(m_fname.c_str()), m_header);
initial_header_len = aare::NumpyHelpers::write_header(std::filesystem::path(fname.c_str()), m_header);
}
m_pixels_per_frame = std::accumulate(m_header.shape.begin() + 1, m_header.shape.end(), 1, std::multiplies<>());

View File

@ -1,7 +1,7 @@
#include "aare/RawFile.hpp"
#include "aare/PixelMap.hpp"
#include "aare/defs.hpp"
#include "aare/json.hpp"
#include "aare/PixelMap.hpp"
#include <fmt/format.h>
#include <nlohmann/json.hpp>
@ -10,38 +10,80 @@ using json = nlohmann::json;
namespace aare {
RawFile::RawFile(const std::filesystem::path &fname, const std::string &mode, const FileConfig &config) {
RawFile::RawFile(const std::filesystem::path &fname, const std::string &mode)
: m_master(fname) {
m_mode = mode;
m_fname = fname;
if (mode == "r") {
if (config != FileConfig()) {
// aare::logger::warn(
// "In read mode it is not necessary to provide a config, the provided config will be ignored");
}
parse_fname();
parse_metadata();
find_number_of_subfiles();
n_subfile_parts = m_master.geometry().col * m_master.geometry().row;
find_geometry();
open_subfiles();
} else {
throw std::runtime_error(LOCATION + "Unsupported mode. Can only read RawFiles.");
throw std::runtime_error(LOCATION +
"Unsupported mode. Can only read RawFiles.");
}
}
Frame RawFile::read_frame() { return get_frame(m_current_frame++); };
Frame RawFile::read_frame(size_t frame_number) {
seek(frame_number);
return read_frame();
}
void RawFile::read_into(std::byte *image_buf, size_t n_frames) {
// TODO: implement this in a more efficient way
for (size_t i = 0; i < n_frames; i++) {
this->get_frame_into(m_current_frame++, image_buf);
image_buf += bytes_per_frame();
}
}
void RawFile::read_into(std::byte *image_buf) {
return get_frame_into(m_current_frame++, image_buf);
};
size_t RawFile::bytes_per_frame() {
return m_rows * m_cols * m_master.bitdepth() / 8;
}
size_t RawFile::pixels_per_frame() { return m_rows * m_cols; }
DetectorType RawFile::detector_type() const { return m_master.detector_type(); }
void RawFile::seek(size_t frame_index) {
// check if the frame number is greater than the total frames
// if frame_number == total_frames, then the next read will throw an error
if (frame_index > total_frames()) {
throw std::runtime_error(
fmt::format("frame number {} is greater than total frames {}",
frame_index, total_frames()));
}
m_current_frame = frame_index;
};
size_t RawFile::tell() { return m_current_frame; };
size_t RawFile::total_frames() const { return m_master.frames_in_file(); }
size_t RawFile::rows() const { return m_rows; }
size_t RawFile::cols() const { return m_cols; }
size_t RawFile::bitdepth() const { return m_master.bitdepth(); }
xy RawFile::geometry() { return m_master.geometry(); }
void RawFile::open_subfiles() {
if (m_mode == "r")
for (size_t i = 0; i != n_subfiles; ++i) {
auto v = std::vector<SubFile *>(n_subfile_parts);
for (size_t j = 0; j != n_subfile_parts; ++j) {
v[j] = new SubFile(data_fname(i, j), m_type, subfile_rows, subfile_cols, m_bitdepth);
v[j] =
new SubFile(m_master.data_fname(j, i),
m_master.detector_type(), m_master.pixels_y(),
m_master.pixels_x(), m_master.bitdepth());
}
subfiles.push_back(v);
}
else {
auto v = std::vector<SubFile *>(n_subfile_parts); // only one subfile is implemented
v[0] = new SubFile(data_fname(0, 0), m_type, m_rows, m_cols, m_bitdepth, "w");
subfiles.push_back(v);
throw std::runtime_error(LOCATION +
"Unsupported mode. Can only read RawFiles.");
}
}
@ -49,7 +91,8 @@ DetectorHeader RawFile::read_header(const std::filesystem::path &fname) {
DetectorHeader h{};
FILE *fp = fopen(fname.string().c_str(), "r");
if (!fp)
throw std::runtime_error(fmt::format("Could not open: {} for reading", fname.string()));
throw std::runtime_error(
fmt::format("Could not open: {} for reading", fname.string()));
size_t const rc = fread(reinterpret_cast<char *>(&h), sizeof(h), 1, fp);
if (rc != 1)
@ -67,16 +110,12 @@ bool RawFile::is_master_file(const std::filesystem::path &fpath) {
void RawFile::find_number_of_subfiles() {
int n_mod = 0;
while (std::filesystem::exists(data_fname(++n_mod, 0)))
while (std::filesystem::exists(m_master.data_fname(0, ++n_mod)))
;
n_subfiles = n_mod;
}
std::filesystem::path RawFile::data_fname(size_t mod_id, size_t file_id) {
return this->m_base_path / fmt::format("{}_d{}_f{}_{}.raw", this->m_base_name, file_id, mod_id, this->m_findex);
}
std::filesystem::path RawFile::master_fname() {
return this->m_base_path / fmt::format("{}_master_{}{}", this->m_base_name, this->m_findex, this->m_ext);
#ifdef AARE_VERBOSE
fmt::print("Found: {} subfiles\n", n_subfiles);
#endif
}
void RawFile::find_geometry() {
@ -84,7 +123,7 @@ void RawFile::find_geometry() {
uint16_t c{};
for (size_t i = 0; i < n_subfile_parts; i++) {
for (size_t j = 0; j != n_subfiles; ++j) {
auto h = this->read_header(data_fname(j, i));
auto h = this->read_header(m_master.data_fname(i, j));
r = std::max(r, h.row);
c = std::max(c, h.column);
@ -95,268 +134,129 @@ void RawFile::find_geometry() {
r++;
c++;
m_rows = (r * subfile_rows);
m_cols = (c * subfile_cols);
m_rows = (r * m_master.pixels_y());
m_cols = (c * m_master.pixels_x());
m_rows += static_cast<size_t>((r - 1) * cfg.module_gap_row);
}
void RawFile::parse_metadata() {
if (m_ext == ".raw") {
parse_raw_metadata();
if (m_bitdepth == 0) {
switch (m_type) {
case DetectorType::Eiger:
m_bitdepth = 32;
break;
default:
m_bitdepth = 16;
}
}
} else if (m_ext == ".json") {
parse_json_metadata();
} else {
throw std::runtime_error(LOCATION + "Unsupported file type");
}
n_subfile_parts = static_cast<size_t>(m_geometry.row) * m_geometry.col;
}
void RawFile::parse_json_metadata() {
std::ifstream ifs(master_fname());
json j;
ifs >> j;
double v = j["Version"];
version = fmt::format("{:.1f}", v);
m_type = StringTo<DetectorType>(j["Detector Type"].get<std::string>());
timing_mode = StringTo<TimingMode>(j["Timing Mode"].get<std::string>());
m_total_frames = j["Frames in File"];
subfile_rows = j["Pixels"]["y"];
subfile_cols = j["Pixels"]["x"];
max_frames_per_file = j["Max Frames Per File"];
try {
m_bitdepth = j.at("Dynamic Range");
} catch (const json::out_of_range &e) {
m_bitdepth = 16;
}
try {
m_analog_samples = j.at("Analog Samples");
}catch (const json::out_of_range &e) {
m_analog_samples = 0;
}
try{
std::string adc_mask = j.at("ADC Mask");
m_adc_mask = std::stoul(adc_mask, nullptr, 16);
// fmt::print("ADC Mask: {}, n_set: {}\n", m_adc_mask, __builtin_popcount(m_adc_mask));
}catch (const json::out_of_range &e) {
m_adc_mask = 0;
}
try {
m_digital_samples = j.at("Digital Samples");
}catch (const json::out_of_range &e) {
m_digital_samples = 0;
}
//Update detector type for Moench
//TODO! How does this work with old .raw master files?
if (m_type == DetectorType::Moench && m_analog_samples == 0 && subfile_rows == 400) {
m_type = DetectorType::Moench03;
}else if (m_type == DetectorType::Moench && subfile_rows == 400 && m_analog_samples == 5000) {
m_type = DetectorType::Moench03_old;
}
//Here we know we have a ChipTestBoard file update the geometry?
//TODO! Carry on information about digtial, and transceivers
if (m_type == DetectorType::ChipTestBoard) {
subfile_rows = 1;
subfile_cols = m_analog_samples*__builtin_popcount(m_adc_mask);
}
// only Eiger had quad
if (m_type == DetectorType::Eiger) {
quad = (j["Quad"] == 1);
}
m_geometry = {j["Geometry"]["y"], j["Geometry"]["x"]};
}
void RawFile::parse_raw_metadata() {
std::ifstream ifs(master_fname());
for (std::string line; std::getline(ifs, line);) {
if (line == "#Frame Header")
break;
auto pos = line.find(':');
auto key_pos = pos;
while (key_pos != std::string::npos && std::isspace(line[--key_pos]))
;
if (key_pos != std::string::npos) {
auto key = line.substr(0, key_pos + 1);
auto value = line.substr(pos + 2);
// do the actual parsing
if (key == "Version") {
version = value;
} else if (key == "TimeStamp") {
} else if (key == "Detector Type") {
m_type = StringTo<DetectorType>(value);
} else if (key == "Timing Mode") {
timing_mode = StringTo<TimingMode>(value);
} else if (key == "Pixels") {
// Total number of pixels cannot be found yet looking at
// submodule
pos = value.find(',');
subfile_cols = std::stoi(value.substr(1, pos));
subfile_rows = std::stoi(value.substr(pos + 1));
} else if (key == "Total Frames") {
m_total_frames = std::stoi(value);
} else if (key == "Dynamic Range") {
m_bitdepth = std::stoi(value);
} else if (key == "Quad") {
quad = (value == "1");
} else if (key == "Max Frames Per File") {
max_frames_per_file = std::stoi(value);
} else if (key == "Geometry") {
pos = value.find(',');
m_geometry = {static_cast<uint32_t>(std::stoi(value.substr(1, pos))),
static_cast<uint32_t>(std::stoi(value.substr(pos + 1)))};
}
}
}
}
void RawFile::parse_fname() {
bool wrong_format = false;
m_base_path = m_fname.parent_path().string();
m_base_name = m_fname.stem().string();
m_ext = m_fname.extension().string();
try {
auto pos = m_base_name.rfind('_');
m_findex = std::stoi(m_base_name.substr(pos + 1));
} catch (const std::invalid_argument &e) {
m_findex = 0;
wrong_format = true;
}
auto pos = m_base_name.find("_master_");
if (pos != std::string::npos) {
m_base_name.erase(pos);
wrong_format = true;
}
if (wrong_format && (m_mode == "w+" || m_mode == "w")) {
// aare::logger::warn("Master Filename", m_fname, "is not in the correct format");
// aare::logger::warn("using", master_fname(), "as the master file");
}
}
Frame RawFile::get_frame(size_t frame_index) {
auto f = Frame(this->m_rows, this->m_cols, Dtype::from_bitdepth(this->m_bitdepth));
auto f = Frame(m_rows, m_cols, Dtype::from_bitdepth(m_master.bitdepth()));
std::byte *frame_buffer = f.data();
get_frame_into(frame_index, frame_buffer);
return f;
}
void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer) {
if (frame_index > this->m_total_frames) {
if (frame_index > total_frames()) {
throw std::runtime_error(LOCATION + "Frame number out of range");
}
std::vector<size_t> frame_numbers(this->n_subfile_parts);
std::vector<size_t> frame_indices(this->n_subfile_parts, frame_index);
std::vector<size_t> frame_numbers(n_subfile_parts);
std::vector<size_t> frame_indices(n_subfile_parts, frame_index);
if (n_subfile_parts != 1) {
for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) {
auto subfile_id = frame_index / this->max_frames_per_file;
for (size_t part_idx = 0; part_idx != n_subfile_parts; ++part_idx) {
auto subfile_id = frame_index / m_master.max_frames_per_file();
frame_numbers[part_idx] =
this->subfiles[subfile_id][part_idx]->frame_number(frame_index % this->max_frames_per_file);
subfiles[subfile_id][part_idx]->frame_number(
frame_index % m_master.max_frames_per_file());
}
// 1. if frame number vector is the same break
while (std::adjacent_find(frame_numbers.begin(), frame_numbers.end(), std::not_equal_to<>()) !=
while (std::adjacent_find(frame_numbers.begin(), frame_numbers.end(),
std::not_equal_to<>()) !=
frame_numbers.end()) {
// 2. find the index of the minimum frame number,
auto min_frame_idx =
std::distance(frame_numbers.begin(), std::min_element(frame_numbers.begin(), frame_numbers.end()));
auto min_frame_idx = std::distance(
frame_numbers.begin(),
std::min_element(frame_numbers.begin(), frame_numbers.end()));
// 3. increase its index and update its respective frame number
frame_indices[min_frame_idx]++;
// 4. if we can't increase its index => throw error
if (frame_indices[min_frame_idx] >= this->m_total_frames) {
throw std::runtime_error(LOCATION + "Frame number out of range");
if (frame_indices[min_frame_idx] >= total_frames()) {
throw std::runtime_error(LOCATION +
"Frame number out of range");
}
auto subfile_id = frame_indices[min_frame_idx] / this->max_frames_per_file;
frame_numbers[min_frame_idx] = this->subfiles[subfile_id][min_frame_idx]->frame_number(
frame_indices[min_frame_idx] % this->max_frames_per_file);
auto subfile_id =
frame_indices[min_frame_idx] / m_master.max_frames_per_file();
frame_numbers[min_frame_idx] =
subfiles[subfile_id][min_frame_idx]->frame_number(
frame_indices[min_frame_idx] %
m_master.max_frames_per_file());
}
}
if (this->m_geometry.col == 1) {
if (m_master.geometry().col == 1) {
// get the part from each subfile and copy it to the frame
for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) {
for (size_t part_idx = 0; part_idx != n_subfile_parts; ++part_idx) {
auto corrected_idx = frame_indices[part_idx];
auto subfile_id = corrected_idx / this->max_frames_per_file;
auto part_offset = this->subfiles[subfile_id][part_idx]->bytes_per_part();
this->subfiles[subfile_id][part_idx]->get_part(frame_buffer + part_idx * part_offset,
corrected_idx % this->max_frames_per_file);
auto subfile_id = corrected_idx / m_master.max_frames_per_file();
auto part_offset = subfiles[subfile_id][part_idx]->bytes_per_part();
subfiles[subfile_id][part_idx]->get_part(
frame_buffer + part_idx * part_offset,
corrected_idx % m_master.max_frames_per_file());
}
} else {
// create a buffer that will hold a the frame part
auto bytes_per_part = this->subfile_rows * this->subfile_cols * this->m_bitdepth / 8;
auto bytes_per_part = m_master.pixels_y() * m_master.pixels_x() *
m_master.bitdepth() /
8; // TODO! replace with image_size_in_bytes
auto *part_buffer = new std::byte[bytes_per_part];
//TODO! if we have many submodules we should reorder them on the module level
// TODO! if we have many submodules we should reorder them on the module
// level
for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) {
for (size_t part_idx = 0; part_idx != n_subfile_parts; ++part_idx) {
auto corrected_idx = frame_indices[part_idx];
auto subfile_id = corrected_idx / this->max_frames_per_file;
auto subfile_id = corrected_idx / m_master.max_frames_per_file();
this->subfiles[subfile_id][part_idx]->get_part(part_buffer, corrected_idx % this->max_frames_per_file);
for (size_t cur_row = 0; cur_row < (this->subfile_rows); cur_row++) {
auto irow = cur_row + (part_idx / this->m_geometry.col) * this->subfile_rows;
auto icol = (part_idx % this->m_geometry.col) * this->subfile_cols;
subfiles[subfile_id][part_idx]->get_part(
part_buffer, corrected_idx % m_master.max_frames_per_file());
for (size_t cur_row = 0; cur_row < (m_master.pixels_y());
cur_row++) {
auto irow = cur_row + (part_idx / m_master.geometry().col) *
m_master.pixels_y();
auto icol =
(part_idx % m_master.geometry().col) * m_master.pixels_x();
auto dest = (irow * this->m_cols + icol);
dest = dest * this->m_bitdepth / 8;
memcpy(frame_buffer + dest, part_buffer + cur_row * this->subfile_cols * this->m_bitdepth / 8,
this->subfile_cols * this->m_bitdepth / 8);
dest = dest * m_master.bitdepth() / 8;
memcpy(frame_buffer + dest,
part_buffer + cur_row * m_master.pixels_x() *
m_master.bitdepth() / 8,
m_master.pixels_x() * m_master.bitdepth() / 8);
}
}
delete[] part_buffer;
}
}
std::vector<Frame> RawFile::read_n(size_t n_frames) {
// TODO: implement this in a more efficient way
std::vector<Frame> frames;
for (size_t i = 0; i < n_frames; i++) {
frames.push_back(this->get_frame(this->current_frame));
this->current_frame++;
frames.push_back(this->get_frame(m_current_frame));
m_current_frame++;
}
return frames;
}
void RawFile::read_into(std::byte *image_buf, size_t n_frames) {
// TODO: implement this in a more efficient way
for (size_t i = 0; i < n_frames; i++) {
this->get_frame_into(this->current_frame++, image_buf);
image_buf += this->bytes_per_frame();
}
}
size_t RawFile::frame_number(size_t frame_index) {
if (frame_index >= this->m_total_frames) {
if (frame_index >= m_master.frames_in_file()) {
throw std::runtime_error(LOCATION + " Frame number out of range");
}
size_t subfile_id = frame_index / this->max_frames_per_file;
if(subfile_id >= this->subfiles.size()){
throw std::runtime_error(LOCATION + " Subfile out of range. Possible missing data.");
size_t subfile_id = frame_index / m_master.max_frames_per_file();
if (subfile_id >= subfiles.size()) {
throw std::runtime_error(
LOCATION + " Subfile out of range. Possible missing data.");
}
return this->subfiles[subfile_id][0]->frame_number(frame_index % this->max_frames_per_file);
return subfiles[subfile_id][0]->frame_number(
frame_index % m_master.max_frames_per_file());
}
RawFile::~RawFile() noexcept {
RawFile::~RawFile() {
//TODO! Fix this, for file closing
// TODO! Fix this, for file closing
for (auto &vec : subfiles) {
for (auto *subfile : vec) {
delete subfile;

View File

@ -63,7 +63,7 @@ TEST_CASE("Read a frame numbers where the subfile is missing throws") {
REQUIRE_THROWS(f.frame_number(4));
REQUIRE_THROWS(f.frame_number(7));
REQUIRE_THROWS(f.frame_number(937));
// REQUIRE_THROWS(f.frame_number(10));
REQUIRE_THROWS(f.frame_number(10));
}
@ -142,8 +142,9 @@ TEST_CASE("Read multipart files") {
}
TEST_CASE("Read file with unordered frames") {
//TODO! Better explanation and error message
auto fpath = test_data_path() / "mythen" / "scan242_master_3.raw";
REQUIRE(std::filesystem::exists(fpath));
File f(fpath, "r");
File f(fpath);
REQUIRE_THROWS((f.read_frame()));
}

View File

@ -8,6 +8,11 @@ RawFileNameComponents::RawFileNameComponents(
m_base_name = fname.stem();
m_ext = fname.extension();
if (m_ext != ".json" && m_ext != ".raw") {
throw std::runtime_error(LOCATION +
"Unsupported file type. (only .json or .raw)");
}
// parse file index
try {
auto pos = m_base_name.rfind('_');
@ -26,6 +31,22 @@ RawFileNameComponents::RawFileNameComponents(
}
}
std::filesystem::path RawFileNameComponents::master_fname() const {
return m_base_path /
fmt::format("{}_master_{}{}", m_base_name, m_file_index, m_ext);
}
std::filesystem::path RawFileNameComponents::data_fname(size_t mod_id, size_t file_id) const{
return m_base_path / fmt::format("{}_d{}_f{}_{}.raw", m_base_name,
mod_id, file_id, m_file_index);
}
const std::filesystem::path& RawFileNameComponents::base_path() const { return m_base_path; }
const std::string& RawFileNameComponents::base_name() const { return m_base_name; }
const std::string& RawFileNameComponents::ext() const { return m_ext; }
int RawFileNameComponents::file_index() const { return m_file_index; }
RawMasterFile::RawMasterFile(const std::filesystem::path &fpath)
: m_fnc(fpath) {
if (!std::filesystem::exists(fpath)) {
@ -63,6 +84,18 @@ const FrameDiscardPolicy &RawMasterFile::frame_discard_policy() const {
return m_frame_discard_policy;
}
size_t RawMasterFile::total_frames_expected() const {
return m_total_frames_expected;
}
std::optional<size_t> RawMasterFile::number_of_rows() const {
return m_number_of_rows;
}
xy RawMasterFile::geometry() const { return m_geometry; }
std::optional<uint8_t> RawMasterFile::quad() const { return m_quad; }
// optional values, these may or may not be present in the master file
// and are therefore modeled as std::optional
std::optional<size_t> RawMasterFile::analog_samples() const {
@ -82,6 +115,8 @@ void RawMasterFile::parse_json(const std::filesystem::path &fpath) {
m_type = StringTo<DetectorType>(j["Detector Type"].get<std::string>());
m_timing_mode = StringTo<TimingMode>(j["Timing Mode"].get<std::string>());
m_geometry = {j["Geometry"]["y"], j["Geometry"]["x"]};
m_image_size_in_bytes = j["Image Size in bytes"];
m_frames_in_file = j["Frames in File"];
m_pixels_y = j["Pixels"]["y"];
@ -96,15 +131,32 @@ void RawMasterFile::parse_json(const std::filesystem::path &fpath) {
} catch (const json::out_of_range &e) {
m_bitdepth = 16;
}
m_total_frames_expected = j["Total Frames"];
m_frame_padding = j["Frame Padding"];
m_frame_discard_policy = StringTo<FrameDiscardPolicy>(
j["Frame Discard Policy"].get<std::string>());
try {
m_analog_samples = j.at("Analog Samples");
m_number_of_rows = j.at("Number of rows");
} catch (const json::out_of_range &e) {
// m_analog_samples = 0;
// keep the optional empty
}
try {
int analog_flag = j.at("Analog Flag");
if (analog_flag) {
m_analog_samples = j.at("Analog Samples");
}
} catch (const json::out_of_range &e) {
// keep the optional empty
}
try{
m_quad = j.at("Quad");
}catch (const json::out_of_range &e) {
// keep the optional empty
}
// try{
// std::string adc_mask = j.at("ADC Mask");
@ -114,9 +166,12 @@ void RawMasterFile::parse_json(const std::filesystem::path &fpath) {
// }
try {
m_digital_samples = j.at("Digital Samples");
int digital_flag = j.at("Digital Flag");
if (digital_flag) {
m_digital_samples = j.at("Digital Samples");
}
} catch (const json::out_of_range &e) {
// m_digital_samples = 0;
// keep the optional empty
}
// //Update detector type for Moench
@ -144,6 +199,73 @@ void RawMasterFile::parse_json(const std::filesystem::path &fpath) {
// m_geometry = {j["Geometry"]["y"], j["Geometry"]["x"]};
}
void RawMasterFile::parse_raw(const std::filesystem::path &fpath) {
throw std::runtime_error("Not implemented");
std::ifstream ifs(fpath);
for (std::string line; std::getline(ifs, line);) {
if (line == "#Frame Header")
break;
auto pos = line.find(':');
auto key_pos = pos;
while (key_pos != std::string::npos && std::isspace(line[--key_pos]))
;
if (key_pos != std::string::npos) {
auto key = line.substr(0, key_pos + 1);
auto value = line.substr(pos + 2);
// do the actual parsing
if (key == "Version") {
m_version = value;
} else if (key == "TimeStamp") {
} else if (key == "Detector Type") {
m_type = StringTo<DetectorType>(value);
} else if (key == "Timing Mode") {
m_timing_mode = StringTo<TimingMode>(value);
} else if (key == "Image Size") {
m_image_size_in_bytes = std::stoi(value);
} else if (key == "Frame Padding"){
m_frame_padding = std::stoi(value);
// } else if (key == "Frame Discard Policy"){
// m_frame_discard_policy = StringTo<FrameDiscardPolicy>(value);
// } else if (key == "Number of rows"){
// m_number_of_rows = std::stoi(value);
} else if (key == "Analog Flag") {
m_analog_flag = std::stoi(value);
} else if (key == "Digital Flag") {
m_digital_flag = std::stoi(value);
} else if (key == "Analog Samples") {
if (m_analog_flag == 1) {
m_analog_samples = std::stoi(value);
}
} else if (key == "Digital Samples") {
if (m_digital_flag == 1) {
m_digital_samples = std::stoi(value);
}
} else if (key == "Frames in File") {
m_frames_in_file = std::stoi(value);
// } else if (key == "ADC Mask") {
// m_adc_mask = std::stoi(value, nullptr, 16);
} else if (key == "Pixels") {
// Total number of pixels cannot be found yet looking at
// submodule
pos = value.find(',');
m_pixels_x = std::stoi(value.substr(1, pos));
m_pixels_y = std::stoi(value.substr(pos + 1));
} else if (key == "Total Frames") {
m_total_frames_expected = std::stoi(value);
} else if (key == "Dynamic Range") {
m_bitdepth = std::stoi(value);
} else if (key == "Quad") {
m_quad = std::stoi(value);
} else if (key == "Max Frames Per File") {
m_max_frames_per_file = std::stoi(value);
} else if (key == "Geometry") {
pos = value.find(',');
m_geometry = {
static_cast<uint32_t>(std::stoi(value.substr(1, pos))),
static_cast<uint32_t>(std::stoi(value.substr(pos + 1)))};
}
}
}
}
} // namespace aare

View File

@ -11,6 +11,15 @@ TEST_CASE("Parse a master file fname"){
REQUIRE(m.base_name() == "test");
REQUIRE(m.ext() == ".json");
REQUIRE(m.file_index() == 1);
REQUIRE(m.base_path() == "");
}
TEST_CASE("Extraction of base path works"){
RawFileNameComponents m("some/path/test_master_73.json");
REQUIRE(m.base_name() == "test");
REQUIRE(m.ext() == ".json");
REQUIRE(m.file_index() == 73);
REQUIRE(m.base_path() == "some/path");
}
TEST_CASE("Construction of master file name and data files"){
@ -22,8 +31,14 @@ TEST_CASE("Construction of master file name and data files"){
REQUIRE(m.data_fname(1, 1) == "test_d1_f1_1.raw");
}
TEST_CASE("Master file name does not fit pattern"){
REQUIRE_THROWS(RawFileNameComponents("somefile.json"));
REQUIRE_THROWS(RawFileNameComponents("another_test_d0_f0_1.raw"));
REQUIRE_THROWS(RawFileNameComponents("test_master_1.txt"));
}
TEST_CASE("Parse a master file"){
TEST_CASE("Parse a master file in .json format"){
auto fpath = test_data_path() / "jungfrau" / "jungfrau_single_master_0.json";
REQUIRE(std::filesystem::exists(fpath));
RawMasterFile f(fpath);
@ -35,10 +50,13 @@ TEST_CASE("Parse a master file"){
REQUIRE(f.detector_type() == DetectorType::Jungfrau);
// "Timing Mode": "auto",
REQUIRE(f.timing_mode() == TimingMode::Auto);
// "Geometry": {
// "x": 1,
// "y": 1
// },
REQUIRE(f.geometry().col == 1);
REQUIRE(f.geometry().row == 1);
// "Image Size in bytes": 1048576,
REQUIRE(f.image_size_in_bytes() == 1048576);
@ -54,10 +72,15 @@ TEST_CASE("Parse a master file"){
//Jungfrau doesn't write but it is 16
REQUIRE(f.bitdepth() == 16);
// "Frame Discard Policy": "nodiscard",
// "Frame Padding": 1,
REQUIRE(f.frame_padding() == 1);
// "Scan Parameters": "[disabled]",
// "Total Frames": 10,
REQUIRE(f.total_frames_expected() == 10);
// "Receiver Roi": {
// "xmin": 4294967295,
// "xmax": 4294967295,
@ -68,7 +91,11 @@ TEST_CASE("Parse a master file"){
// "Period": "1ms",
// "Number of UDP Interfaces": 1,
// "Number of rows": 512,
REQUIRE(f.number_of_rows() == 512);
// "Frames in File": 10,
REQUIRE(f.frames_in_file() == 10);
//TODO! Should we parse this?
// "Frame Header Format": {
// "Frame Number": "8 bytes",
// "SubFrame Number/ExpLength": "4 bytes",
@ -92,6 +119,70 @@ TEST_CASE("Parse a master file"){
}
TEST_CASE("Parse a master file in .raw format"){
auto fpath = test_data_path() / "moench/moench04_noise_200V_sto_both_100us_no_light_thresh_900_master_0.raw";
REQUIRE(std::filesystem::exists(fpath));
RawMasterFile f(fpath);
// Version : 6.4
REQUIRE(f.version() == "6.4");
// TimeStamp : Wed Aug 31 09:08:49 2022
// Detector Type : ChipTestBoard
REQUIRE(f.detector_type() == DetectorType::ChipTestBoard);
// Timing Mode : auto
REQUIRE(f.timing_mode() == TimingMode::Auto);
// Geometry : [1, 1]
REQUIRE(f.geometry().col == 1);
REQUIRE(f.geometry().row == 1);
// Image Size : 360000 bytes
REQUIRE(f.image_size_in_bytes() == 360000);
// Pixels : [96, 1]
REQUIRE(f.pixels_x() == 96);
REQUIRE(f.pixels_y() == 1);
// Max Frames Per File : 20000
REQUIRE(f.max_frames_per_file() == 20000);
// Frame Discard Policy : nodiscard
// Frame Padding : 1
REQUIRE(f.frame_padding() == 1);
// Scan Parameters : [disabled]
// Total Frames : 100
REQUIRE(f.total_frames_expected() == 100);
// Exptime : 100us
// Period : 4ms
// Ten Giga : 1
// ADC Mask : 0xffffffff
// Analog Flag : 1
// Analog Samples : 5000
REQUIRE(f.analog_samples() == 5000);
// Digital Flag : 1
// Digital Samples : 5000
REQUIRE(f.digital_samples() == 5000);
// Dbit Offset : 0
// Dbit Bitset : 0
// Frames in File : 100
REQUIRE(f.frames_in_file() == 100);
// #Frame Header
// Frame Number : 8 bytes
// SubFrame Number/ExpLength : 4 bytes
// Packet Number : 4 bytes
// Bunch ID : 8 bytes
// Timestamp : 8 bytes
// Module Id : 2 bytes
// Row : 2 bytes
// Column : 2 bytes
// Reserved : 2 bytes
// Debug : 4 bytes
// Round Robin Number : 2 bytes
// Detector Type : 1 byte
// Header Version : 1 byte
// Packets Caught Mask : 64 bytes
}
TEST_CASE("Read eiger master file"){
auto fpath = test_data_path() / "eiger" / "eiger_500k_32bit_master_0.json";

View File

@ -25,17 +25,15 @@ SubFile::SubFile(const std::filesystem::path &fname, DetectorType detector, size
if (mode == "r") {
fp = fopen(m_fname.string().c_str(), "rb");
} else {
// if file exists, open in read/write mode (without truncating the file)
// if file does not exist, open in write mode
if (std::filesystem::exists(fname)) {
fp = fopen(m_fname.string().c_str(), "r+b");
} else {
fp = fopen(m_fname.string().c_str(), "wb");
}
throw std::runtime_error(LOCATION + "Unsupported mode. Can only read RawFiles.");
}
if (fp == nullptr) {
throw std::runtime_error(LOCATION + "Could not open file for writing");
throw std::runtime_error(LOCATION + fmt::format("Could not open file {}", m_fname.string()));
}
#ifdef AARE_VERBOSE
fmt::print("Opened file: {} with {} frames\n", m_fname.string(), n_frames);
fmt::print("m_rows: {}, m_cols: {}, m_bitdepth: {}\n", m_rows, m_cols, m_bitdepth);
#endif
}
size_t SubFile::get_part(std::byte *buffer, size_t frame_index) {