detectors/aare
25
0
Fork 0
mirror of https://github.com/slsdetectorgroup/aare.git synced 2025-06-12 23:37:13 +02:00
Code Issues Actions Packages Releases Activity

wip at fixing hdf5 master file
Some checks failed
Build on RHEL9 / build (push) Failing after 1m27s
Details
Build on RHEL8 / build (push) Failing after 1m36s
Details

Browse Source
This commit is contained in:
maliakal_d
2025-06-06 16:36:40 +02:00
parent d7242671b2
commit 480e28c927
8 changed files with 148 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
include/aare/Hdf5File.hpp
View File

@ -8,7 +8,7 @@
namespace aare {
struct H5Handles {
class H5Handles {
std::string file_name;
std::string dataset_name;
H5::H5File file;
@ -20,6 +20,7 @@ struct H5Handles {
std::vector<hsize_t> count;
std::vector<hsize_t> offset;
public:
H5Handles(const std::string& fname, const std::string& dname):
file_name(fname),
dataset_name(dname),
@ -31,6 +32,10 @@ struct H5Handles {
initialize_memspace();
}
std::vector<hsize_t> get_dims() const {
return dims;
}
void seek(size_t frame_index) {
if (frame_index >= dims[0]) {
throw std::runtime_error(LOCATION + "Invalid frame number");

57
include/aare/Hdf5MasterFile.hpp
View File

@ -6,18 +6,6 @@
#include <fstream>
#include <optional>
namespace fmt {
template <typename T> struct formatter<std::optional<T>> : formatter<T> {
template <typename FormatContext>
auto format(const std::optional<T> &opt, FormatContext &ctx) {
if (opt) {
return formatter<T>::format(*opt, ctx);
} else {
return format_to(ctx.out(), "nullopt");
}
}
};
} // namespace fmt
namespace aare {
@ -29,7 +17,7 @@ class Hdf5FileNameComponents {
std::filesystem::path m_base_path{};
std::string m_base_name{};
std::string m_ext{};
int m_file_index{}; // TODO! is this measurement_index?
int m_file_index{};
public:
Hdf5FileNameComponents(const std::filesystem::path &fname);
@ -50,39 +38,6 @@ class Hdf5FileNameComponents {
void set_old_scheme(bool old_scheme);
};
/*
class ScanParameters {
bool m_enabled = false;
std::string m_dac;
int m_start = 0;
int m_stop = 0;
int m_step = 0;
// TODO! add settleTime, requires string to time conversion
public:
ScanParameters(const std::string &par);
ScanParameters() = default;
ScanParameters(const ScanParameters &) = default;
ScanParameters &operator=(const ScanParameters &) = default;
ScanParameters(ScanParameters &&) = default;
int start() const;
int stop() const;
int step() const;
const std::string &dac() const;
bool enabled() const;
void increment_stop();
};
struct ROI {
int64_t xmin{};
int64_t xmax{};
int64_t ymin{};
int64_t ymax{};
int64_t height() const { return ymax - ymin; }
int64_t width() const { return xmax - xmin; }
};
*/
/**
* @brief Class for parsing a master file either in our .json format or the old
@ -104,7 +59,7 @@ class Hdf5MasterFile {
xy m_geometry{};
size_t m_max_frames_per_file{};
// uint32_t m_adc_mask{}; // TODO! implement reading
uint32_t m_adc_mask{}; // TODO! implement reading
FrameDiscardPolicy m_frame_discard_policy{};
size_t m_frame_padding{};
@ -113,7 +68,7 @@ class Hdf5MasterFile {
uint8_t m_digital_flag{};
uint8_t m_transceiver_flag{};
// ScanParameters m_scan_parameters;
ScanParameters m_scan_parameters;
std::optional<size_t> m_analog_samples;
std::optional<size_t> m_digital_samples;
@ -121,7 +76,7 @@ class Hdf5MasterFile {
std::optional<size_t> m_number_of_rows;
std::optional<uint8_t> m_quad;
// std::optional<ROI> m_roi;
std::optional<ROI> m_roi;
public:
Hdf5MasterFile(const std::filesystem::path &fpath);
@ -151,9 +106,9 @@ class Hdf5MasterFile {
std::optional<size_t> number_of_rows() const;
std::optional<uint8_t> quad() const;
// std::optional<ROI> roi() const;
std::optional<ROI> roi() const;
// ScanParameters scan_parameters() const;
ScanParameters scan_parameters() const;
private:
static const std::string metadata_group_name;

23
include/aare/RawMasterFile.hpp
View File

@ -39,28 +39,6 @@ class RawFileNameComponents {
void set_old_scheme(bool old_scheme);
};
class ScanParameters {
bool m_enabled = false;
std::string m_dac;
int m_start = 0;
int m_stop = 0;
int m_step = 0;
//TODO! add settleTime, requires string to time conversion
public:
ScanParameters(const std::string &par);
ScanParameters() = default;
ScanParameters(const ScanParameters &) = default;
ScanParameters &operator=(const ScanParameters &) = default;
ScanParameters(ScanParameters &&) = default;
int start() const;
int stop() const;
int step() const;
const std::string &dac() const;
bool enabled() const;
void increment_stop();
};
/**
* @brief Class for parsing a master file either in our .json format or the old
@ -132,7 +110,6 @@ class RawMasterFile {
std::optional<ROI> roi() const;
ScanParameters scan_parameters() const;
private:

54
include/aare/defs.hpp
View File

@ -12,7 +12,8 @@
#include <variant>
#include <vector>
#include <iostream>
#include <sstream>
#include <optional>
/**
* @brief LOCATION macro to get the current location in the code
@ -225,6 +226,46 @@ struct ROI{
using dynamic_shape = std::vector<ssize_t>;
class ScanParameters {
bool m_enabled = false;
std::string m_dac;
int m_start = 0;
int m_stop = 0;
int m_step = 0;
//TODO! add settleTime, requires string to time conversion
public:
// "[enabled\ndac dac 4\nstart 500\nstop 2200\nstep 5\nsettleTime 100us\n]"
ScanParameters(const std::string &par) {
std::istringstream iss(par.substr(1, par.size()-2));
std::string line;
while(std::getline(iss, line)){
if(line == "enabled"){
m_enabled = true;
}else if(line.find("dac") != std::string::npos){
m_dac = line.substr(4);
}else if(line.find("start") != std::string::npos){
m_start = std::stoi(line.substr(6));
}else if(line.find("stop") != std::string::npos){
m_stop = std::stoi(line.substr(5));
}else if(line.find("step") != std::string::npos){
m_step = std::stoi(line.substr(5));
}
}
};
ScanParameters() = default;
ScanParameters(const ScanParameters &) = default;
ScanParameters &operator=(const ScanParameters &) = default;
ScanParameters(ScanParameters &&) = default;
int start() const { return m_start; };
int stop() const { return m_stop; };
int step() const { return m_step; };
const std::string &dac() const { return m_dac; };
bool enabled() const { return m_enabled; };
void increment_stop() { m_stop += 1; };
};
//TODO! Can we uniform enums between the libraries?
/**
@ -266,6 +307,8 @@ template <> std::string ToString(TimingMode arg);
template <> FrameDiscardPolicy StringTo(const std::string & /*mode*/);
template <> std::string ToString(FrameDiscardPolicy arg);
template <> std::string ToString(ScanParameters arg);
using DataTypeVariants = std::variant<uint16_t, uint32_t>;
template <typename T>
@ -280,5 +323,14 @@ std::ostream &operator<<(std::ostream &os, const std::vector<T> &vec) {
return os;
}
template <typename T>
std::ostream &operator<<(std::ostream &os, const std::optional<T> &opt) {
if (opt)
os << *opt;
else
os << "nullopt";
return os;
}
} // namespace aare

6
src/Hdf5File.cpp
View File

@ -193,9 +193,9 @@ void Hdf5File::open_data_file() {
try {
m_data_file = std::make_unique<H5Handles>(m_master.master_fname().string(), metadata_group_name + "/data");
m_total_frames = m_data_file->dims[0];
m_rows = m_data_file->dims[1];
m_cols = m_data_file->dims[2];
m_total_frames = m_data_file->get_dims()[0];
m_rows = m_data_file->get_dims()[1];
m_cols = m_data_file->get_dims()[2];
//fmt::print("Data Dataset dimensions: frames = {}, rows = {}, cols = {}\n",
// m_total_frames, m_rows, m_cols);
} catch (const H5::Exception &e) {

136
src/Hdf5MasterFile.cpp
View File

@ -1,4 +1,5 @@
#include "aare/Hdf5MasterFile.hpp"
#include "aare/logger.hpp"
#include <sstream>
#include <iomanip>
namespace aare {
@ -62,32 +63,6 @@ const std::string &Hdf5FileNameComponents::base_name() const {
const std::string &Hdf5FileNameComponents::ext() const { return m_ext; }
int Hdf5FileNameComponents::file_index() const { return m_file_index; }
// "[enabled\ndac dac 4\nstart 500\nstop 2200\nstep 5\nsettleTime 100us\n]"
/*ScanParameters::ScanParameters(const std::string &par) {
std::istringstream iss(par.substr(1, par.size() - 2));
std::string line;
while (std::getline(iss, line)) {
if (line == "enabled") {
m_enabled = true;
} else if (line.find("dac") != std::string::npos) {
m_dac = line.substr(4);
} else if (line.find("start") != std::string::npos) {
m_start = std::stoi(line.substr(6));
} else if (line.find("stop") != std::string::npos) {
m_stop = std::stoi(line.substr(5));
} else if (line.find("step") != std::string::npos) {
m_step = std::stoi(line.substr(5));
}
}
}
int ScanParameters::start() const { return m_start; }
int ScanParameters::stop() const { return m_stop; }
void ScanParameters::increment_stop() { m_stop += 1; };
//int ScanParameters::step() const { return m_step; }
const std::string &ScanParameters::dac() const { return m_dac; }
bool ScanParameters::enabled() const { return m_enabled; }
*/
Hdf5MasterFile::Hdf5MasterFile(const std::filesystem::path &fpath)
: m_fnc(fpath) {
if (!std::filesystem::exists(fpath)) {
@ -131,18 +106,12 @@ size_t Hdf5MasterFile::total_frames_expected() const {
return m_total_frames_expected;
}
std::optional<size_t> Hdf5MasterFile::number_of_rows() const {
return m_number_of_rows;
}
xy Hdf5MasterFile::geometry() const { return m_geometry; }
size_t Hdf5MasterFile::n_modules() const {
return m_geometry.row * m_geometry.col;
}
std::optional<uint8_t> Hdf5MasterFile::quad() const { return m_quad; }
// optional values, these may or may not be present in the master file
// and are therefore modeled as std::optional
std::optional<size_t> Hdf5MasterFile::analog_samples() const {
@ -156,13 +125,17 @@ std::optional<size_t> Hdf5MasterFile::transceiver_samples() const {
return m_transceiver_samples;
}
/*
std::optional<size_t> Hdf5MasterFile::number_of_rows() const {
return m_number_of_rows;
}
std::optional<uint8_t> Hdf5MasterFile::quad() const { return m_quad; }
std::optional<ROI> Hdf5MasterFile::roi() const { return m_roi; }
ScanParameters Hdf5MasterFile::scan_parameters() const {
return m_scan_parameters;
}
*/
// std::optional<ROI> Hdf5MasterFile::roi() const { return m_roi; }
const std::string Hdf5MasterFile::metadata_group_name =
"/entry/instrument/detector/";
@ -202,59 +175,59 @@ void Hdf5MasterFile::parse_acquisition_metadata(
H5::H5File file(fpath, H5F_ACC_RDONLY);
// Attribute - version
double dVersion{0.0};
{
H5::Attribute attr = file.openAttribute("version");
H5::DataType attr_type = attr.getDataType();
double value{0.0};
attr.read(attr_type, &value);
attr.read(attr_type, &dVersion);
std::ostringstream oss;
oss << std::fixed << std::setprecision(1) << value;
oss << std::fixed << std::setprecision(1) << dVersion;
m_version = oss.str();
// fmt::print("Version: {}\n", m_version);
LOG(logDEBUG) << "Version: " << m_version;
}
// Scalar Dataset
// Detector Type
m_type = StringTo<DetectorType>(h5_get_scalar_dataset<std::string>(
file, std::string(metadata_group_name + "Detector Type")));
// fmt::print("Detector Type: {}\n", (ToString(m_type)));
LOG(logDEBUG) << "Detector Type: " << ToString(m_type);
// Timing Mode
m_timing_mode = StringTo<TimingMode>(h5_get_scalar_dataset<std::string>(
file, std::string(metadata_group_name + "Timing Mode")));
// fmt::print("Timing Mode: {}\n", (ToString(m_timing_mode)));
LOG(logDEBUG) << "Timing Mode: " << ToString(m_timing_mode);
// Geometry
m_geometry.row = h5_get_scalar_dataset<int>(
file, std::string(metadata_group_name + "Geometry in y axis"));
m_geometry.col = h5_get_scalar_dataset<int>(
file, std::string(metadata_group_name + "Geometry in x axis"));
// fmt::print("Geometry: {}\n", m_geometry.to_string());
LOG(logDEBUG) << "Geometry: " << m_geometry.to_string();
// Image Size
m_image_size_in_bytes = h5_get_scalar_dataset<int>(
file, std::string(metadata_group_name + "Image Size"));
// fmt::print("Image size: {}\n", m_image_size_in_bytes);
LOG(logDEBUG) << "Image size: {}\n" << m_image_size_in_bytes;
// Frames in File
m_frames_in_file = h5_get_scalar_dataset<uint64_t>(
file, std::string(metadata_group_name + "Frames in File"));
// fmt::print("Frames in File: {}\n", m_frames_in_file);
LOG(logDEBUG) << "Frames in File: " << m_frames_in_file;
// Pixels
m_pixels_y = h5_get_scalar_dataset<int>(
file,
std::string(metadata_group_name + "Number of pixels in y axis"));
// fmt::print("Pixels in y: {}\n", m_pixels_y);
LOG(logDEBUG) << "Pixels in y: " << m_pixels_y;
m_pixels_x = h5_get_scalar_dataset<int>(
file,
std::string(metadata_group_name + "Number of pixels in x axis"));
// fmt::print("Pixels in x: {}\n", m_pixels_x);
LOG(logDEBUG) << "Pixels in x: " << m_pixels_x;
// Max Frames per File
m_max_frames_per_file = h5_get_scalar_dataset<int>(
file, std::string(metadata_group_name + "Maximum frames per file"));
// fmt::print("Max frames per File: {}\n", m_max_frames_per_file);
LOG(logDEBUG) << "Max frames per File: " << m_max_frames_per_file;
// Bit Depth
// Not all detectors write the bitdepth but in case
@ -266,27 +239,26 @@ void Hdf5MasterFile::parse_acquisition_metadata(
} catch (H5::FileIException &e) {
m_bitdepth = 16;
}
// fmt::print("Bit Depth: {}\n", m_bitdepth);
LOG(logDEBUG) << "Bit Depth: " << m_bitdepth;
H5Eset_auto(H5E_DEFAULT, reinterpret_cast<H5E_auto2_t>(H5Eprint2),
stderr);
// Total Frames
m_total_frames_expected = h5_get_scalar_dataset<uint64_t>(
file, std::string(metadata_group_name + "Total Frames"));
// fmt::print("Total Frames: {}\n", m_total_frames_expected);
LOG(logDEBUG) << "Total Frames: " << m_total_frames_expected;
// Frame Padding
m_frame_padding = h5_get_scalar_dataset<int>(
file, std::string(metadata_group_name + "Frame Padding"));
// fmt::print("Frame Padding: {}\n", m_frame_padding);
LOG(logDEBUG) << "Frame Padding: " << m_frame_padding;
// Frame Discard Policy
m_frame_discard_policy =
StringTo<FrameDiscardPolicy>(h5_get_scalar_dataset<std::string>(
file,
std::string(metadata_group_name + "Frame Discard Policy")));
// fmt::print("Frame Discard Policy: {}\n",
// (ToString(m_frame_discard_policy)));
LOG(logDEBUG) << "Frame Discard Policy: " << ToString(m_frame_discard_policy);
// Number of rows
// Not all detectors write the Number of rows but in case
@ -297,7 +269,7 @@ void Hdf5MasterFile::parse_acquisition_metadata(
} catch (H5::FileIException &e) {
// keep the optional empty
}
// fmt::print("Number of rows: {}\n", m_number_of_rows);
LOG(logDEBUG) << "Number of rows: " << m_number_of_rows;
H5Eset_auto(H5E_DEFAULT, reinterpret_cast<H5E_auto2_t>(H5Eprint2),
stderr);
@ -313,7 +285,7 @@ void Hdf5MasterFile::parse_acquisition_metadata(
// to try to decode analog samples (Old Moench03)
m_analog_flag = 1;
}
// fmt::print("Analog Flag: {}\n", m_analog_flag);
LOG(logDEBUG) << "Analog Flag: " << m_analog_flag;
H5Eset_auto(H5E_DEFAULT, reinterpret_cast<H5E_auto2_t>(H5Eprint2),
stderr);
@ -331,7 +303,7 @@ void Hdf5MasterFile::parse_acquisition_metadata(
}
H5Eset_auto(H5E_DEFAULT, reinterpret_cast<H5E_auto2_t>(H5Eprint2),
stderr);
// fmt::print("Analog Samples: {}\n", m_analog_samples);
LOG(logDEBUG) << "Analog Samples: " << m_analog_samples;
//-----------------------------------------------------------------
// Quad
@ -342,23 +314,23 @@ void Hdf5MasterFile::parse_acquisition_metadata(
} catch (H5::FileIException &e) {
// keep the optional empty
}
// fmt::print("Quad: {}\n", m_quad);
LOG(logDEBUG) << "Quad: " << m_quad;
H5Eset_auto(H5E_DEFAULT, reinterpret_cast<H5E_auto2_t>(H5Eprint2),
stderr);
// ADC Mask
// H5::Exception::dontPrint();
// try {
// m_adc_mask = h5_get_scalar_dataset<int>(
// file, std::string(metadata_group_name + "ADC
// Mask"));
// } catch (H5::FileIException &e) {
// m_adc_mask = 0;
// }
// fmt::print("ADC Mask: {}\n", m_adc_mask);
// H5Eset_auto(H5E_DEFAULT,
// reinterpret_cast<H5E_auto2_t>(H5Eprint2),
// stderr);
H5::Exception::dontPrint();
try {
m_adc_mask = h5_get_scalar_dataset<int>(
file, std::string(metadata_group_name + "ADC
Mask"));
} catch (H5::FileIException &e) {
m_adc_mask = 0;
}
LOG(logDEBUG) << "ADC Mask: " << m_adc_mask;
H5Eset_auto(H5E_DEFAULT, reinterpret_cast<H5E_auto2_t>(H5Eprint2),
stderr);
// Digital Flag, Digital Samples
H5::Exception::dontPrint();
@ -372,8 +344,8 @@ void Hdf5MasterFile::parse_acquisition_metadata(
} catch (H5::FileIException &e) {
// keep the optional empty
}
// fmt::print("Digital Flag: {}\n", m_digital_flag);
// fmt::print("Digital Samples: {}\n", m_digital_samples);
LOG(logDEBUG) << "Digital Flag: " << m_digital_flag;
LOG(logDEBUG) << "Digital Samples: " << m_digital_samples;
H5Eset_auto(H5E_DEFAULT, reinterpret_cast<H5E_auto2_t>(H5Eprint2),
stderr);
@ -390,23 +362,23 @@ void Hdf5MasterFile::parse_acquisition_metadata(
} catch (H5::FileIException &e) {
// keep the optional empty
}
// fmt::print("Transceiver Flag: {}\n", m_transceiver_flag);
// fmt::print("Transceiver Samples: {}\n",
// m_transceiver_samples);
LOG(logDEBUG) << "Transceiver Flag: " << m_transceiver_flag;
LOG(logDEBUG) << "Transceiver Samples: ",m_transceiver_samples;
H5Eset_auto(H5E_DEFAULT, reinterpret_cast<H5E_auto2_t>(H5Eprint2),
stderr);
// scan parameters
/*try{
std::string scan_parameters = j.at("Scan Parameters");
try {
std::string scan_parameters = h5_get_scalar_dataset<std::string>(
file, std::string(metadata_group_name + "Scan Parameters"));
m_scan_parameters = ScanParameters(scan_parameters);
if(v<7.21){
m_scan_parameters.increment_stop(); //adjust for
endpoint being included
}
}catch (const json::out_of_range &e) {
// not a scan
if (dVersion < 7.21){
m_scan_parameters.increment_stop(); //adjust for endpoint being included
}
} catch (H5::FileIException &e) {
// keep the optional empty
}
LOG(logDEBUG) << "Scan Parameters: " << ToString(m_scan_parameters);
try{
ROI tmp_roi;

28
src/RawMasterFile.cpp
View File

@ -64,34 +64,6 @@ const std::string &RawFileNameComponents::base_name() const {
const std::string &RawFileNameComponents::ext() const { return m_ext; }
int RawFileNameComponents::file_index() const { return m_file_index; }
// "[enabled\ndac dac 4\nstart 500\nstop 2200\nstep 5\nsettleTime 100us\n]"
ScanParameters::ScanParameters(const std::string& par){
std::istringstream iss(par.substr(1, par.size()-2));
std::string line;
while(std::getline(iss, line)){
if(line == "enabled"){
m_enabled = true;
}else if(line.find("dac") != std::string::npos){
m_dac = line.substr(4);
}else if(line.find("start") != std::string::npos){
m_start = std::stoi(line.substr(6));
}else if(line.find("stop") != std::string::npos){
m_stop = std::stoi(line.substr(5));
}else if(line.find("step") != std::string::npos){
m_step = std::stoi(line.substr(5));
}
}
}
int ScanParameters::start() const { return m_start; }
int ScanParameters::stop() const { return m_stop; }
void ScanParameters::increment_stop(){
m_stop += 1;
}
int ScanParameters::step() const { return m_step; }
const std::string &ScanParameters::dac() const { return m_dac; }
bool ScanParameters::enabled() const { return m_enabled; }
RawMasterFile::RawMasterFile(const std::filesystem::path &fpath)
: m_fnc(fpath) {

26
src/defs.cpp
View File

@ -1,6 +1,7 @@
#include "aare/defs.hpp"
#include <stdexcept>
#include <string>
#include <chrono>
#include <fmt/core.h>
namespace aare {
@ -153,6 +154,31 @@ template <> FrameDiscardPolicy StringTo(const std::string &arg) {
throw std::runtime_error("Could not decode frame discard policy from: \"" + arg + "\"");
}
/**
* @brief Convert a ScanParameters to a string
* @param type ScanParameters
* @return string representation of the ScanParameters
*/
template <> std::string ToString(ScanParameters arg) {
std::ostringstream oss;
oss << '[';
if (arg.enable) {
oss << "enabled" << std::endl
<< "dac " << ToString(arg.dacInd) << std::endl
<< "start " << arg.startOffset << std::endl
<< "stop " << arg.stopOffset << std::endl
<< "step " << arg.stepSize << std::endl
<< "settleTime "
<< ToString(std::chrono::nanoseconds{arg.dacSettleTime_ns})
<< std::endl;
} else {
oss << "disabled";
}
oss << ']';
return oss.str();
}
// template <> TimingMode StringTo<TimingMode>(std::string mode);
} // namespace aare