From 9c6e6292981b79bbf93e2f49c0f9d404a5dcead1 Mon Sep 17 00:00:00 2001 From: Alice Date: Wed, 4 Jun 2025 16:34:40 +0200 Subject: [PATCH 01/20] only files within the ROI are opened & geometry always read in RawMasterFile --- include/aare/RawFile.hpp | 25 +++--- include/aare/RawMasterFile.hpp | 9 +- src/RawFile.cpp | 160 +++++++++++++++++---------------- src/RawMasterFile.cpp | 149 +++++++++++++++--------------- src/RawSubFile.cpp | 65 +++++++------- 5 files changed, 208 insertions(+), 200 deletions(-) diff --git a/include/aare/RawFile.hpp b/include/aare/RawFile.hpp index 1cca1fd..5c6e74b 100644 --- a/include/aare/RawFile.hpp +++ b/include/aare/RawFile.hpp @@ -1,11 +1,10 @@ #pragma once #include "aare/FileInterface.hpp" -#include "aare/RawMasterFile.hpp" #include "aare/Frame.hpp" #include "aare/NDArray.hpp" //for pixel map +#include "aare/RawMasterFile.hpp" #include "aare/RawSubFile.hpp" - #include namespace aare { @@ -30,12 +29,15 @@ struct ModuleConfig { * Consider using that unless you need raw file specific functionality. */ class RawFile : public FileInterface { + + friend class RawMasterFile; std::vector> m_subfiles; ModuleConfig cfg{0, 0}; RawMasterFile m_master; size_t m_current_frame{}; size_t m_current_subfile{}; DetectorGeometry m_geometry; + std::vector m_modules_in_roi{}; public: /** @@ -53,10 +55,10 @@ class RawFile : public FileInterface { void read_into(std::byte *image_buf) override; void read_into(std::byte *image_buf, size_t n_frames) override; - //TODO! do we need to adapt the API? + // TODO! do we need to adapt the API? void read_into(std::byte *image_buf, DetectorHeader *header); - void read_into(std::byte *image_buf, size_t n_frames, DetectorHeader *header); - + void read_into(std::byte *image_buf, size_t n_frames, + DetectorHeader *header); size_t frame_number(size_t frame_index) override; size_t bytes_per_frame() override; @@ -70,23 +72,21 @@ class RawFile : public FileInterface { size_t bitdepth() const override; xy geometry(); size_t n_modules() const; - + size_t n_modules_in_roi() const; + RawMasterFile master() const; - - DetectorType detector_type() const override; private: - /** * @brief read the frame at the given frame index into the image buffer * @param frame_number frame number to read * @param image_buf buffer to store the frame */ - void get_frame_into(size_t frame_index, std::byte *frame_buffer, DetectorHeader *header = nullptr); - + void get_frame_into(size_t frame_index, std::byte *frame_buffer, + DetectorHeader *header = nullptr); /** * @brief get the frame at the given frame index @@ -95,8 +95,6 @@ class RawFile : public FileInterface { */ Frame get_frame(size_t frame_index); - - /** * @brief read the header of the file * @param fname path to the data subfile @@ -108,5 +106,4 @@ class RawFile : public FileInterface { void find_geometry(); }; - } // namespace aare \ No newline at end of file diff --git a/include/aare/RawMasterFile.hpp b/include/aare/RawMasterFile.hpp index 4d143a6..9d65081 100644 --- a/include/aare/RawMasterFile.hpp +++ b/include/aare/RawMasterFile.hpp @@ -1,5 +1,7 @@ #pragma once +// #include "aare/RawFile.hpp" #include "aare/defs.hpp" +#include #include #include #include @@ -45,7 +47,7 @@ class ScanParameters { int m_start = 0; int m_stop = 0; int m_step = 0; - //TODO! add settleTime, requires string to time conversion + // TODO! add settleTime, requires string to time conversion public: ScanParameters(const std::string &par); @@ -61,6 +63,7 @@ class ScanParameters { void increment_stop(); }; +class RawFile; // forward declaration /** * @brief Class for parsing a master file either in our .json format or the old @@ -101,7 +104,6 @@ class RawMasterFile { std::optional m_roi; - public: RawMasterFile(const std::filesystem::path &fpath); @@ -129,15 +131,14 @@ class RawMasterFile { std::optional number_of_rows() const; std::optional quad() const; - std::optional roi() const; - ScanParameters scan_parameters() const; private: void parse_json(const std::filesystem::path &fpath); void parse_raw(const std::filesystem::path &fpath); + void retrieve_geometry(); }; } // namespace aare \ No newline at end of file diff --git a/src/RawFile.cpp b/src/RawFile.cpp index 122cf96..fbf1dad 100644 --- a/src/RawFile.cpp +++ b/src/RawFile.cpp @@ -1,9 +1,9 @@ #include "aare/RawFile.hpp" -#include "aare/algorithm.hpp" #include "aare/PixelMap.hpp" +#include "aare/algorithm.hpp" #include "aare/defs.hpp" -#include "aare/logger.hpp" #include "aare/geo_helpers.hpp" +#include "aare/logger.hpp" #include #include @@ -15,10 +15,22 @@ namespace aare { RawFile::RawFile(const std::filesystem::path &fname, const std::string &mode) : m_master(fname) { m_mode = mode; + if (mode == "r") { find_geometry(); - if (m_master.roi()){ - m_geometry = update_geometry_with_roi(m_geometry, m_master.roi().value()); + if (m_master.roi()) { + m_geometry = + update_geometry_with_roi(m_geometry, m_master.roi().value()); + m_modules_in_roi.reserve(n_modules()); + for (size_t module_index = 0; module_index < n_modules(); + ++module_index) { + if (m_geometry.module_pixel_0[module_index].width == 0 && + m_geometry.module_pixel_0[module_index].height == 0) + m_modules_in_roi.push_back(module_index); + } + } else { + m_modules_in_roi.resize(n_modules()); + std::iota(m_modules_in_roi.begin(), m_modules_in_roi.end(), 0); } open_subfiles(); } else { @@ -47,32 +59,33 @@ void RawFile::read_into(std::byte *image_buf) { return get_frame_into(m_current_frame++, image_buf); } - void RawFile::read_into(std::byte *image_buf, DetectorHeader *header) { return get_frame_into(m_current_frame++, image_buf, header); } -void RawFile::read_into(std::byte *image_buf, size_t n_frames, DetectorHeader *header) { +void RawFile::read_into(std::byte *image_buf, size_t n_frames, + DetectorHeader *header) { // return get_frame_into(m_current_frame++, image_buf, header); for (size_t i = 0; i < n_frames; i++) { this->get_frame_into(m_current_frame++, image_buf, header); image_buf += bytes_per_frame(); - if(header) - header+=n_modules(); + if (header) + header += n_modules(); } - } size_t RawFile::n_modules() const { return m_master.n_modules(); } +size_t RawFile::n_modules_in_roi() const { return m_modules_in_roi.size(); } size_t RawFile::bytes_per_frame() { - return m_geometry.pixels_x * m_geometry.pixels_y * m_master.bitdepth() / bits_per_byte; + return m_geometry.pixels_x * m_geometry.pixels_y * m_master.bitdepth() / + bits_per_byte; } -size_t RawFile::pixels_per_frame() { - // return m_rows * m_cols; +size_t RawFile::pixels_per_frame() { + // return m_rows * m_cols; return m_geometry.pixels_x * m_geometry.pixels_y; } @@ -99,7 +112,7 @@ xy RawFile::geometry() { return m_master.geometry(); } void RawFile::open_subfiles() { if (m_mode == "r") - for (size_t i = 0; i != n_modules(); ++i) { + for (size_t i : m_modules_in_roi) { auto pos = m_geometry.module_pixel_0[i]; m_subfiles.emplace_back(std::make_unique( m_master.data_fname(i, 0), m_master.detector_type(), pos.height, @@ -128,76 +141,59 @@ DetectorHeader RawFile::read_header(const std::filesystem::path &fname) { return h; } - RawMasterFile RawFile::master() const { return m_master; } /** - * @brief Find the geometry of the detector by opening all the subfiles and - * reading the headers. + * @brief Find the geometry of the detector */ void RawFile::find_geometry() { - - //Hold the maximal row and column number found - //Later used for calculating the total number of rows and columns - uint16_t r{}; - uint16_t c{}; + // TODO potentially update for Eiger + for (size_t col = 0; col < m_master.geometry().col; ++col) + for (size_t row = 0; row < m_master.geometry().row; ++row) { + ModuleGeometry g; + g.origin_x = col * m_master.pixels_x(); + g.origin_y = row * m_master.pixels_y(); + g.row_index = row; + g.col_index = col; + g.width = m_master.pixels_x(); + g.height = m_master.pixels_y(); + m_geometry.module_pixel_0.push_back(g); + } - for (size_t i = 0; i < n_modules(); i++) { - auto h = read_header(m_master.data_fname(i, 0)); - r = std::max(r, h.row); - c = std::max(c, h.column); - // positions.push_back({h.row, h.column}); - - ModuleGeometry g; - g.origin_x = h.column * m_master.pixels_x(); - g.origin_y = h.row * m_master.pixels_y(); - g.row_index = h.row; - g.col_index = h.column; - g.width = m_master.pixels_x(); - g.height = m_master.pixels_y(); - m_geometry.module_pixel_0.push_back(g); - - } - - r++; - c++; - - m_geometry.pixels_y = (r * m_master.pixels_y()); - m_geometry.pixels_x = (c * m_master.pixels_x()); - m_geometry.modules_x = c; - m_geometry.modules_y = r; - m_geometry.pixels_y += static_cast((r - 1) * cfg.module_gap_row); - + m_geometry.pixels_y = (m_master.geometry().row * m_master.pixels_y()); + m_geometry.pixels_x = (m_master.geometry().col * m_master.pixels_x()); + m_geometry.modules_x = m_master.geometry().col; + m_geometry.modules_y = m_master.geometry().row; + m_geometry.pixels_y += + static_cast((m_master.geometry().row - 1) * cfg.module_gap_row); } - Frame RawFile::get_frame(size_t frame_index) { - auto f = Frame(m_geometry.pixels_y, m_geometry.pixels_x, Dtype::from_bitdepth(m_master.bitdepth())); + auto f = Frame(m_geometry.pixels_y, m_geometry.pixels_x, + Dtype::from_bitdepth(m_master.bitdepth())); std::byte *frame_buffer = f.data(); get_frame_into(frame_index, frame_buffer); return f; } +size_t RawFile::bytes_per_pixel() const { return m_master.bitdepth() / 8; } -size_t RawFile::bytes_per_pixel() const { - return m_master.bitdepth() / 8; -} - -void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer, DetectorHeader *header) { +void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer, + DetectorHeader *header) { LOG(logDEBUG) << "RawFile::get_frame_into(" << frame_index << ")"; if (frame_index >= total_frames()) { throw std::runtime_error(LOCATION + "Frame number out of range"); } - std::vector frame_numbers(n_modules()); - std::vector frame_indices(n_modules(), frame_index); - + std::vector frame_numbers(n_modules_in_roi()); + std::vector frame_indices(n_modules_in_roi(), frame_index); // sync the frame numbers - if (n_modules() != 1) { //if we have more than one module - for (size_t part_idx = 0; part_idx != n_modules(); ++part_idx) { - frame_numbers[part_idx] = m_subfiles[part_idx]->frame_number(frame_index); + if (n_modules() != 1) { // if we have more than one module + for (size_t part_idx = 0; part_idx != n_modules_in_roi(); ++part_idx) { + frame_numbers[part_idx] = + m_subfiles[part_idx]->frame_number(frame_index); } // 1. if frame number vector is the same break @@ -218,23 +214,32 @@ void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer, Detect } frame_numbers[min_frame_idx] = - m_subfiles[min_frame_idx]->frame_number(frame_indices[min_frame_idx]); + m_subfiles[min_frame_idx]->frame_number( + frame_indices[min_frame_idx]); } } if (m_master.geometry().col == 1) { // get the part from each subfile and copy it to the frame - for (size_t part_idx = 0; part_idx != n_modules(); ++part_idx) { + for (size_t part_idx = 0; part_idx != n_modules_in_roi(); ++part_idx) { auto corrected_idx = frame_indices[part_idx]; - - // This is where we start writing - auto offset = (m_geometry.module_pixel_0[part_idx].origin_y * m_geometry.pixels_x + - m_geometry.module_pixel_0[part_idx].origin_x)*m_master.bitdepth()/8; - if (m_geometry.module_pixel_0[part_idx].origin_x!=0) - throw std::runtime_error(LOCATION + " Implementation error. x pos not 0."); - - //TODO! What if the files don't match? + // This is where we start writing + auto offset = (m_geometry.module_pixel_0[m_modules_in_roi[part_idx]] + .origin_y * + m_geometry.pixels_x + + m_geometry.module_pixel_0[m_modules_in_roi[part_idx]] + .origin_x) * + m_master.bitdepth() / 8; + + if (m_geometry.module_pixel_0[m_modules_in_roi[part_idx]] + .origin_x != 0) + throw std::runtime_error( + LOCATION + + " Implementation error. x pos not 0."); // TODO: origin can + // still change if + // roi changes + // TODO! What if the files don't match? m_subfiles[part_idx]->seek(corrected_idx); m_subfiles[part_idx]->read_into(frame_buffer + offset, header); if (header) @@ -242,7 +247,7 @@ void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer, Detect } } else { - //TODO! should we read row by row? + // TODO! should we read row by row? // create a buffer large enough to hold a full module auto bytes_per_part = m_master.pixels_y() * m_master.pixels_x() * @@ -254,13 +259,13 @@ void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer, Detect // TODO! if we have many submodules we should reorder them on the module // level - for (size_t part_idx = 0; part_idx != n_modules(); ++part_idx) { - auto pos = m_geometry.module_pixel_0[part_idx]; + for (size_t part_idx = 0; part_idx != n_modules_in_roi(); ++part_idx) { + auto pos = m_geometry.module_pixel_0[m_modules_in_roi[part_idx]]; auto corrected_idx = frame_indices[part_idx]; m_subfiles[part_idx]->seek(corrected_idx); m_subfiles[part_idx]->read_into(part_buffer, header); - if(header) + if (header) ++header; for (size_t cur_row = 0; cur_row < static_cast(pos.height); @@ -271,15 +276,13 @@ void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer, Detect auto dest = (irow * this->m_geometry.pixels_x + icol); dest = dest * m_master.bitdepth() / 8; memcpy(frame_buffer + dest, - part_buffer + cur_row * pos.width * - m_master.bitdepth() / 8, + part_buffer + + cur_row * pos.width * m_master.bitdepth() / 8, pos.width * m_master.bitdepth() / 8); - } } delete[] part_buffer; } - } std::vector RawFile::read_n(size_t n_frames) { @@ -299,5 +302,4 @@ size_t RawFile::frame_number(size_t frame_index) { return m_subfiles[0]->frame_number(frame_index); } - } // namespace aare diff --git a/src/RawMasterFile.cpp b/src/RawMasterFile.cpp index 8a2db87..d4adf2c 100644 --- a/src/RawMasterFile.cpp +++ b/src/RawMasterFile.cpp @@ -1,5 +1,6 @@ #include "aare/RawMasterFile.hpp" -#include +#include "aare/RawFile.hpp" +#include namespace aare { RawFileNameComponents::RawFileNameComponents( @@ -37,18 +38,15 @@ std::filesystem::path RawFileNameComponents::master_fname() const { } std::filesystem::path RawFileNameComponents::data_fname(size_t mod_id, - size_t file_id - ) const { - - - + size_t file_id) const { + std::string fmt = "{}_d{}_f{}_{}.raw"; - //Before version X we used to name the data files f000000000000 + // Before version X we used to name the data files f000000000000 if (m_old_scheme) { fmt = "{}_d{}_f{:012}_{}.raw"; } - return m_base_path / fmt::format(fmt, m_base_name, mod_id, - file_id, m_file_index); + return m_base_path / + fmt::format(fmt, m_base_name, mod_id, file_id, m_file_index); } void RawFileNameComponents::set_old_scheme(bool old_scheme) { @@ -65,19 +63,19 @@ const std::string &RawFileNameComponents::ext() const { return m_ext; } int RawFileNameComponents::file_index() const { return m_file_index; } // "[enabled\ndac dac 4\nstart 500\nstop 2200\nstep 5\nsettleTime 100us\n]" -ScanParameters::ScanParameters(const std::string& par){ - std::istringstream iss(par.substr(1, par.size()-2)); +ScanParameters::ScanParameters(const std::string &par) { + std::istringstream iss(par.substr(1, par.size() - 2)); std::string line; - while(std::getline(iss, line)){ - if(line == "enabled"){ + while (std::getline(iss, line)) { + if (line == "enabled") { m_enabled = true; - }else if(line.find("dac") != std::string::npos){ + } else if (line.find("dac") != std::string::npos) { m_dac = line.substr(4); - }else if(line.find("start") != std::string::npos){ + } else if (line.find("start") != std::string::npos) { m_start = std::stoi(line.substr(6)); - }else if(line.find("stop") != std::string::npos){ + } else if (line.find("stop") != std::string::npos) { m_stop = std::stoi(line.substr(5)); - }else if(line.find("step") != std::string::npos){ + } else if (line.find("step") != std::string::npos) { m_step = std::stoi(line.substr(5)); } } @@ -85,14 +83,11 @@ ScanParameters::ScanParameters(const std::string& par){ int ScanParameters::start() const { return m_start; } int ScanParameters::stop() const { return m_stop; } -void ScanParameters::increment_stop(){ - m_stop += 1; -} +void ScanParameters::increment_stop() { m_stop += 1; } int ScanParameters::step() const { return m_step; } const std::string &ScanParameters::dac() const { return m_dac; } bool ScanParameters::enabled() const { return m_enabled; } - RawMasterFile::RawMasterFile(const std::filesystem::path &fpath) : m_fnc(fpath) { if (!std::filesystem::exists(fpath)) { @@ -163,10 +158,8 @@ ScanParameters RawMasterFile::scan_parameters() const { return m_scan_parameters; } - std::optional RawMasterFile::roi() const { return m_roi; } - void RawMasterFile::parse_json(const std::filesystem::path &fpath) { std::ifstream ifs(fpath); json j; @@ -177,7 +170,10 @@ void RawMasterFile::parse_json(const std::filesystem::path &fpath) { m_type = StringTo(j["Detector Type"].get()); m_timing_mode = StringTo(j["Timing Mode"].get()); - m_geometry = {j["Geometry"]["y"], j["Geometry"]["x"]}; + m_geometry = { + j["Geometry"]["y"], + j["Geometry"]["x"]}; // TODO: isnt it only available for version > 7.1? + // - try block default should be 1x1 m_image_size_in_bytes = j["Image Size in bytes"]; m_frames_in_file = j["Frames in File"]; @@ -205,17 +201,16 @@ void RawMasterFile::parse_json(const std::filesystem::path &fpath) { // keep the optional empty } - // ---------------------------------------------------------------- // Special treatment of analog flag because of Moench03 - try{ + try { m_analog_flag = j.at("Analog Flag"); - }catch (const json::out_of_range &e) { + } catch (const json::out_of_range &e) { // if it doesn't work still set it to one // to try to decode analog samples (Old Moench03) m_analog_flag = 1; } - try { + try { if (m_analog_flag) { m_analog_samples = j.at("Analog Samples"); } @@ -248,27 +243,27 @@ void RawMasterFile::parse_json(const std::filesystem::path &fpath) { // keep the optional empty } - try{ + try { m_transceiver_flag = j.at("Transceiver Flag"); - if(m_transceiver_flag){ + if (m_transceiver_flag) { m_transceiver_samples = j.at("Transceiver Samples"); } - }catch (const json::out_of_range &e) { + } catch (const json::out_of_range &e) { // keep the optional empty } - try{ + try { std::string scan_parameters = j.at("Scan Parameters"); m_scan_parameters = ScanParameters(scan_parameters); - if(v<7.21){ - m_scan_parameters.increment_stop(); //adjust for endpoint being included - } - }catch (const json::out_of_range &e) { + if (v < 7.21) { + m_scan_parameters + .increment_stop(); // adjust for endpoint being included + } + } catch (const json::out_of_range &e) { // not a scan } - - try{ + try { ROI tmp_roi; auto obj = j.at("Receiver Roi"); tmp_roi.xmin = obj.at("xmin"); @@ -276,37 +271,32 @@ void RawMasterFile::parse_json(const std::filesystem::path &fpath) { tmp_roi.ymin = obj.at("ymin"); tmp_roi.ymax = obj.at("ymax"); - //if any of the values are set update the roi + // if any of the values are set update the roi if (tmp_roi.xmin != 4294967295 || tmp_roi.xmax != 4294967295 || tmp_roi.ymin != 4294967295 || tmp_roi.ymax != 4294967295) { - - if(v<7.21){ + + if (v < 7.21) { tmp_roi.xmax++; tmp_roi.ymax++; } - + m_roi = tmp_roi; } - - }catch (const json::out_of_range &e) { + } catch (const json::out_of_range &e) { // leave the optional empty } - //if we have an roi we need to update the geometry for the subfiles - if (m_roi){ - + // if we have an roi we need to update the geometry for the subfiles + if (m_roi) { } - - - - // Update detector type for Moench - // TODO! How does this work with old .raw master files? - #ifdef AARE_VERBOSE +// Update detector type for Moench +// TODO! How does this work with old .raw master files? +#ifdef AARE_VERBOSE fmt::print("Detecting Moench03: m_pixels_y: {}, m_analog_samples: {}\n", m_pixels_y, m_analog_samples.value_or(0)); - #endif +#endif if (m_type == DetectorType::Moench && !m_analog_samples && m_pixels_y == 400) { m_type = DetectorType::Moench03; @@ -332,19 +322,19 @@ void RawMasterFile::parse_raw(const std::filesystem::path &fpath) { if (key == "Version") { m_version = value; - //TODO!: How old versions can we handle? + // TODO!: How old versions can we handle? auto v = std::stod(value); - //TODO! figure out exactly when we did the change - //This enables padding of f to 12 digits - if (v<4.0) + // TODO! figure out exactly when we did the change + // This enables padding of f to 12 digits + if (v < 4.0) m_fnc.set_old_scheme(true); } else if (key == "TimeStamp") { } else if (key == "Detector Type") { m_type = StringTo(value); - if(m_type==DetectorType::Moench){ + if (m_type == DetectorType::Moench) { m_type = DetectorType::Moench03_old; } } else if (key == "Timing Mode") { @@ -381,10 +371,10 @@ void RawMasterFile::parse_raw(const std::filesystem::path &fpath) { pos = value.find(','); m_pixels_x = std::stoi(value.substr(1, pos)); m_pixels_y = std::stoi(value.substr(pos + 1)); - }else if(key == "row"){ + } else if (key == "row") { pos = value.find('p'); m_pixels_y = std::stoi(value.substr(0, pos)); - }else if(key == "col"){ + } else if (key == "col") { pos = value.find('p'); m_pixels_x = std::stoi(value.substr(0, pos)); } else if (key == "Total Frames") { @@ -395,8 +385,8 @@ void RawMasterFile::parse_raw(const std::filesystem::path &fpath) { m_quad = std::stoi(value); } else if (key == "Max Frames Per File") { m_max_frames_per_file = std::stoi(value); - }else if(key == "Max. Frames Per File"){ - //Version 3.0 way of writing it + } else if (key == "Max. Frames Per File") { + // Version 3.0 way of writing it m_max_frames_per_file = std::stoi(value); } else if (key == "Geometry") { pos = value.find(','); @@ -410,15 +400,34 @@ void RawMasterFile::parse_raw(const std::filesystem::path &fpath) { m_type = DetectorType::Moench03_old; } - - //TODO! Look for d0, d1...dn and update geometry - if(m_geometry.col == 0 && m_geometry.row == 0){ - m_geometry = {1,1}; - fmt::print("Warning: No geometry found in master file. Assuming 1x1\n"); + if (m_geometry.col == 0 && m_geometry.row == 0) { + retrieve_geometry(); + fmt::print("Warning: No geometry found in master file. Retrieved " + "geometry of {}x{}\n", + m_geometry.row, m_geometry.col); } - //TODO! Read files and find actual frames - if(m_frames_in_file==0) + // TODO! Read files and find actual frames + if (m_frames_in_file == 0) m_frames_in_file = m_total_frames_expected; } + +void RawMasterFile::retrieve_geometry() { + uint32_t module_index = 0; + uint16_t rows = 0; + uint16_t cols = 0; + // TODO use case for Eiger + while (std::filesystem::exists(data_fname(module_index, 0))) { + auto header = RawFile::read_header(data_fname(module_index, 0)); + + rows = std::max(rows, header.row); + cols = std::max(cols, header.column); + ++rows; + ++cols; + ++module_index; + } + + m_geometry = {rows, cols}; +} + } // namespace aare diff --git a/src/RawSubFile.cpp b/src/RawSubFile.cpp index a8d29ce..3ed2c6f 100644 --- a/src/RawSubFile.cpp +++ b/src/RawSubFile.cpp @@ -1,36 +1,30 @@ #include "aare/RawSubFile.hpp" #include "aare/PixelMap.hpp" #include "aare/algorithm.hpp" -#include "aare/utils/ifstream_helpers.hpp" #include "aare/logger.hpp" - +#include "aare/utils/ifstream_helpers.hpp" #include // memcpy #include #include #include - - - namespace aare { RawSubFile::RawSubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth, uint32_t pos_row, uint32_t pos_col) - : m_detector_type(detector), m_bitdepth(bitdepth), - m_rows(rows), m_cols(cols), - m_bytes_per_frame((m_bitdepth / 8) * m_rows * m_cols), m_pos_row(pos_row), - m_pos_col(pos_col) { + : m_detector_type(detector), m_bitdepth(bitdepth), m_rows(rows), + m_cols(cols), m_bytes_per_frame((m_bitdepth / 8) * m_rows * m_cols), + m_pos_row(pos_row), m_pos_col(pos_col) { - LOG(logDEBUG) << "RawSubFile::RawSubFile()"; + LOG(logDEBUG) << "RawSubFile::RawSubFile()"; if (m_detector_type == DetectorType::Moench03_old) { m_pixel_map = GenerateMoench03PixelMap(); } else if (m_detector_type == DetectorType::Eiger && m_pos_row % 2 == 0) { m_pixel_map = GenerateEigerFlipRowsPixelMap(); } - parse_fname(fname); scan_files(); open_file(m_current_file_index); // open the first file @@ -51,7 +45,8 @@ void RawSubFile::seek(size_t frame_index) { auto frame_offset = (file_index) ? frame_index - m_last_frame_in_file[file_index - 1] : frame_index; - auto byte_offset = frame_offset * (m_bytes_per_frame + sizeof(DetectorHeader)); + auto byte_offset = + frame_offset * (m_bytes_per_frame + sizeof(DetectorHeader)); m_file.seekg(byte_offset); } @@ -69,7 +64,7 @@ void RawSubFile::read_into(std::byte *image_buf, DetectorHeader *header) { m_file.seekg(sizeof(DetectorHeader), std::ios::cur); } - if (m_file.fail()){ + if (m_file.fail()) { throw std::runtime_error(LOCATION + ifstream_error_msg(m_file)); } @@ -78,14 +73,15 @@ void RawSubFile::read_into(std::byte *image_buf, DetectorHeader *header) { // read into a temporary buffer and then copy the data to the buffer // in the correct order // TODO! add 4 bit support - if(m_bitdepth == 8){ + if (m_bitdepth == 8) { read_with_map(image_buf); - }else if (m_bitdepth == 16) { + } else if (m_bitdepth == 16) { read_with_map(image_buf); } else if (m_bitdepth == 32) { read_with_map(image_buf); - }else{ - throw std::runtime_error("Unsupported bitdepth for read with pixel map"); + } else { + throw std::runtime_error( + "Unsupported bitdepth for read with pixel map"); } } else { @@ -93,11 +89,11 @@ void RawSubFile::read_into(std::byte *image_buf, DetectorHeader *header) { m_file.read(reinterpret_cast(image_buf), bytes_per_frame()); } - if (m_file.fail()){ + if (m_file.fail()) { throw std::runtime_error(LOCATION + ifstream_error_msg(m_file)); } - ++ m_current_frame_index; + ++m_current_frame_index; if (m_current_frame_index >= m_last_frame_in_file[m_current_file_index] && (m_current_frame_index < m_total_frames)) { ++m_current_file_index; @@ -105,7 +101,8 @@ void RawSubFile::read_into(std::byte *image_buf, DetectorHeader *header) { } } -void RawSubFile::read_into(std::byte *image_buf, size_t n_frames, DetectorHeader *header) { +void RawSubFile::read_into(std::byte *image_buf, size_t n_frames, + DetectorHeader *header) { for (size_t i = 0; i < n_frames; i++) { read_into(image_buf, header); image_buf += bytes_per_frame(); @@ -115,10 +112,7 @@ void RawSubFile::read_into(std::byte *image_buf, size_t n_frames, DetectorHeader } } - - -template -void RawSubFile::read_with_map(std::byte *image_buf) { +template void RawSubFile::read_with_map(std::byte *image_buf) { auto part_buffer = new std::byte[bytes_per_frame()]; m_file.read(reinterpret_cast(part_buffer), bytes_per_frame()); auto *data = reinterpret_cast(image_buf); @@ -157,14 +151,17 @@ void RawSubFile::parse_fname(const std::filesystem::path &fname) { std::smatch match; if (std::regex_match(m_base_name, match, pattern)) { - m_offset = std::stoi(match[4].str()); // find the first file index in case of a truncated series - m_base_name = match[1].str() + match[2].str() + match[3].str() + "{}" + match[5].str(); + m_offset = std::stoi(match[4].str()); // find the first file index in + // case of a truncated series + m_base_name = match[1].str() + match[2].str() + match[3].str() + "{}" + + match[5].str(); LOG(logDEBUG) << "Base name: " << m_base_name; LOG(logDEBUG) << "Offset: " << m_offset; LOG(logDEBUG) << "Path: " << m_path.string(); } else { throw std::runtime_error( - LOCATION + fmt::format("Could not parse file name {}", fname.string())); + LOCATION + + fmt::format("Could not parse file name {}", fname.string())); } } @@ -175,12 +172,13 @@ std::filesystem::path RawSubFile::fpath(size_t file_index) const { void RawSubFile::open_file(size_t file_index) { m_file.close(); - auto fname = fpath(file_index+m_offset); + auto fname = fpath(file_index + m_offset); LOG(logDEBUG) << "RawSubFile::open_file(): " << fname.string(); m_file.open(fname, std::ios::binary); if (!m_file.is_open()) { throw std::runtime_error( - LOCATION + fmt::format("Could not open file {}", fpath(file_index).string())); + LOCATION + + fmt::format("Could not open file {}", fpath(file_index).string())); } m_current_file_index = file_index; } @@ -190,20 +188,21 @@ void RawSubFile::scan_files() { // find how many files we have and the number of frames in each file m_last_frame_in_file.clear(); size_t file_index = m_offset; - + while (std::filesystem::exists(fpath(file_index))) { auto n_frames = std::filesystem::file_size(fpath(file_index)) / (m_bytes_per_frame + sizeof(DetectorHeader)); m_last_frame_in_file.push_back(n_frames); - LOG(logDEBUG) << "Found: " << n_frames << " frames in file: " << fpath(file_index).string(); + LOG(logDEBUG) << "Found: " << n_frames + << " frames in file: " << fpath(file_index).string(); ++file_index; } // find where we need to open the next file and total number of frames m_last_frame_in_file = cumsum(m_last_frame_in_file); - if(m_last_frame_in_file.empty()){ + if (m_last_frame_in_file.empty()) { m_total_frames = 0; - }else{ + } else { m_total_frames = m_last_frame_in_file.back(); } } From 87d8682b1e02e5b48a356279a0fd8642ea2f30a3 Mon Sep 17 00:00:00 2001 From: Alice Date: Fri, 6 Jun 2025 11:31:39 +0200 Subject: [PATCH 02/20] added test cases --- include/aare/RawFile.hpp | 13 +- include/aare/RawMasterFile.hpp | 2 + src/RawFile.cpp | 27 ++-- src/RawFile.test.cpp | 156 ++++++++++++++++++---- src/RawMasterFile.cpp | 26 +++- src/RawMasterFile.test.cpp | 229 +++++++++++++++++---------------- 6 files changed, 298 insertions(+), 155 deletions(-) diff --git a/include/aare/RawFile.hpp b/include/aare/RawFile.hpp index 5c6e74b..78ebf4c 100644 --- a/include/aare/RawFile.hpp +++ b/include/aare/RawFile.hpp @@ -36,9 +36,12 @@ class RawFile : public FileInterface { RawMasterFile m_master; size_t m_current_frame{}; size_t m_current_subfile{}; - DetectorGeometry m_geometry; + std::vector m_modules_in_roi{}; + protected: + DetectorGeometry m_geometry; + public: /** * @brief RawFile constructor @@ -95,15 +98,17 @@ class RawFile : public FileInterface { */ Frame get_frame(size_t frame_index); + void open_subfiles(); + + protected: + void find_geometry(); + /** * @brief read the header of the file * @param fname path to the data subfile * @return DetectorHeader */ static DetectorHeader read_header(const std::filesystem::path &fname); - - void open_subfiles(); - void find_geometry(); }; } // namespace aare \ No newline at end of file diff --git a/include/aare/RawMasterFile.hpp b/include/aare/RawMasterFile.hpp index 9d65081..671fa36 100644 --- a/include/aare/RawMasterFile.hpp +++ b/include/aare/RawMasterFile.hpp @@ -81,6 +81,7 @@ class RawMasterFile { size_t m_pixels_y{}; size_t m_pixels_x{}; size_t m_bitdepth{}; + size_t m_num_udp_interfaces_per_module = 1; xy m_geometry{}; @@ -119,6 +120,7 @@ class RawMasterFile { size_t max_frames_per_file() const; size_t bitdepth() const; size_t frame_padding() const; + size_t num_udp_interfaces_per_module() const; const FrameDiscardPolicy &frame_discard_policy() const; size_t total_frames_expected() const; diff --git a/src/RawFile.cpp b/src/RawFile.cpp index fbf1dad..2a60de1 100644 --- a/src/RawFile.cpp +++ b/src/RawFile.cpp @@ -24,8 +24,8 @@ RawFile::RawFile(const std::filesystem::path &fname, const std::string &mode) m_modules_in_roi.reserve(n_modules()); for (size_t module_index = 0; module_index < n_modules(); ++module_index) { - if (m_geometry.module_pixel_0[module_index].width == 0 && - m_geometry.module_pixel_0[module_index].height == 0) + if (m_geometry.module_pixel_0[module_index].width != 0 && + m_geometry.module_pixel_0[module_index].height != 0) m_modules_in_roi.push_back(module_index); } } else { @@ -148,17 +148,20 @@ RawMasterFile RawFile::master() const { return m_master; } */ void RawFile::find_geometry() { - // TODO potentially update for Eiger - for (size_t col = 0; col < m_master.geometry().col; ++col) + for (size_t col = 0; col < m_master.geometry().col; + col += m_master.num_udp_interfaces_per_module()) for (size_t row = 0; row < m_master.geometry().row; ++row) { - ModuleGeometry g; - g.origin_x = col * m_master.pixels_x(); - g.origin_y = row * m_master.pixels_y(); - g.row_index = row; - g.col_index = col; - g.width = m_master.pixels_x(); - g.height = m_master.pixels_y(); - m_geometry.module_pixel_0.push_back(g); + for (size_t port = 0; + port < m_master.num_udp_interfaces_per_module(); ++port) { + ModuleGeometry g; + g.origin_x = (col + port) * m_master.pixels_x(); + g.origin_y = row * m_master.pixels_y(); + g.row_index = row; + g.col_index = col + port; + g.width = m_master.pixels_x(); + g.height = m_master.pixels_y(); + m_geometry.module_pixel_0.push_back(g); + } } m_geometry.pixels_y = (m_master.geometry().row * m_master.pixels_y()); diff --git a/src/RawFile.test.cpp b/src/RawFile.test.cpp index 9109985..239bfb2 100644 --- a/src/RawFile.test.cpp +++ b/src/RawFile.test.cpp @@ -1,18 +1,21 @@ +#include "aare/RawFile.hpp" #include "aare/File.hpp" #include "aare/RawMasterFile.hpp" //needed for ROI -#include "aare/RawFile.hpp" #include +#include #include #include "test_config.hpp" - using aare::File; +using aare::RawFile; +using namespace aare; TEST_CASE("Read number of frames from a jungfrau raw file", "[.integration]") { - auto fpath = test_data_path() / "jungfrau" / "jungfrau_single_master_0.json"; + auto fpath = + test_data_path() / "jungfrau" / "jungfrau_single_master_0.json"; REQUIRE(std::filesystem::exists(fpath)); File f(fpath, "r"); @@ -20,7 +23,8 @@ TEST_CASE("Read number of frames from a jungfrau raw file", "[.integration]") { } TEST_CASE("Read frame numbers from a jungfrau raw file", "[.integration]") { - auto fpath = test_data_path() / "jungfrau" / "jungfrau_single_master_0.json"; + auto fpath = + test_data_path() / "jungfrau" / "jungfrau_single_master_0.json"; REQUIRE(std::filesystem::exists(fpath)); File f(fpath, "r"); @@ -36,7 +40,8 @@ TEST_CASE("Read frame numbers from a jungfrau raw file", "[.integration]") { } TEST_CASE("Read a frame number too high throws", "[.integration]") { - auto fpath = test_data_path() / "jungfrau" / "jungfrau_single_master_0.json"; + auto fpath = + test_data_path() / "jungfrau" / "jungfrau_single_master_0.json"; REQUIRE(std::filesystem::exists(fpath)); File f(fpath, "r"); @@ -49,8 +54,10 @@ TEST_CASE("Read a frame number too high throws", "[.integration]") { REQUIRE_THROWS(f.frame_number(10)); } -TEST_CASE("Read a frame numbers where the subfile is missing throws", "[.integration]") { - auto fpath = test_data_path() / "jungfrau" / "jungfrau_missing_subfile_master_0.json"; +TEST_CASE("Read a frame numbers where the subfile is missing throws", + "[.integration]") { + auto fpath = test_data_path() / "jungfrau" / + "jungfrau_missing_subfile_master_0.json"; REQUIRE(std::filesystem::exists(fpath)); File f(fpath, "r"); @@ -58,7 +65,7 @@ TEST_CASE("Read a frame numbers where the subfile is missing throws", "[.integra // we know this file has 10 frames with frame numbers 1 to 10 // f0 1,2,3 // f1 4,5,6 - but files f1-f3 are missing - // f2 7,8,9 - gone + // f2 7,8,9 - gone // f3 10 - gone REQUIRE(f.frame_number(0) == 1); REQUIRE(f.frame_number(1) == 2); @@ -69,15 +76,18 @@ TEST_CASE("Read a frame numbers where the subfile is missing throws", "[.integra REQUIRE_THROWS(f.frame_number(10)); } - -TEST_CASE("Read data from a jungfrau 500k single port raw file", "[.integration]") { - auto fpath = test_data_path() / "jungfrau" / "jungfrau_single_master_0.json"; +TEST_CASE("Read data from a jungfrau 500k single port raw file", + "[.integration]") { + auto fpath = + test_data_path() / "jungfrau" / "jungfrau_single_master_0.json"; REQUIRE(std::filesystem::exists(fpath)); File f(fpath, "r"); - // we know this file has 10 frames with pixel 0,0 being: 2123, 2051, 2109, 2117, 2089, 2095, 2072, 2126, 2097, 2102 - std::vector pixel_0_0 = {2123, 2051, 2109, 2117, 2089, 2095, 2072, 2126, 2097, 2102}; + // we know this file has 10 frames with pixel 0,0 being: 2123, 2051, 2109, + // 2117, 2089, 2095, 2072, 2126, 2097, 2102 + std::vector pixel_0_0 = {2123, 2051, 2109, 2117, 2089, + 2095, 2072, 2126, 2097, 2102}; for (size_t i = 0; i < 10; i++) { auto frame = f.read_frame(); CHECK(frame.rows() == 512); @@ -100,10 +110,12 @@ TEST_CASE("Read frame numbers from a raw file", "[.integration]") { } TEST_CASE("Compare reading from a numpy file with a raw file", "[.files]") { - auto fpath_raw = test_data_path() / "raw/jungfrau" / "jungfrau_single_master_0.json"; + auto fpath_raw = + test_data_path() / "raw/jungfrau" / "jungfrau_single_master_0.json"; REQUIRE(std::filesystem::exists(fpath_raw)); - auto fpath_npy = test_data_path() / "raw/jungfrau" / "jungfrau_single_0.npy"; + auto fpath_npy = + test_data_path() / "raw/jungfrau" / "jungfrau_single_0.npy"; REQUIRE(std::filesystem::exists(fpath_npy)); File raw(fpath_raw, "r"); @@ -121,17 +133,23 @@ TEST_CASE("Compare reading from a numpy file with a raw file", "[.files]") { } TEST_CASE("Read multipart files", "[.integration]") { - auto fpath = test_data_path() / "jungfrau" / "jungfrau_double_master_0.json"; + auto fpath = + test_data_path() / "jungfrau" / "jungfrau_double_master_0.json"; REQUIRE(std::filesystem::exists(fpath)); File f(fpath, "r"); // we know this file has 10 frames check read_multiport.py for the values - std::vector pixel_0_0 = {2099, 2121, 2108, 2084, 2084, 2118, 2066, 2108, 2112, 2116}; - std::vector pixel_0_1 = {2842, 2796, 2865, 2798, 2805, 2817, 2852, 2789, 2792, 2833}; - std::vector pixel_255_1023 = {2149, 2037, 2115, 2102, 2118, 2090, 2036, 2071, 2073, 2142}; - std::vector pixel_511_1023 = {3231, 3169, 3167, 3162, 3168, 3160, 3171, 3171, 3169, 3171}; - std::vector pixel_1_0 = {2748, 2614, 2665, 2629, 2618, 2630, 2631, 2634, 2577, 2598}; + std::vector pixel_0_0 = {2099, 2121, 2108, 2084, 2084, + 2118, 2066, 2108, 2112, 2116}; + std::vector pixel_0_1 = {2842, 2796, 2865, 2798, 2805, + 2817, 2852, 2789, 2792, 2833}; + std::vector pixel_255_1023 = {2149, 2037, 2115, 2102, 2118, + 2090, 2036, 2071, 2073, 2142}; + std::vector pixel_511_1023 = {3231, 3169, 3167, 3162, 3168, + 3160, 3171, 3171, 3169, 3171}; + std::vector pixel_1_0 = {2748, 2614, 2665, 2629, 2618, + 2630, 2631, 2634, 2577, 2598}; for (size_t i = 0; i < 10; i++) { auto frame = f.read_frame(); @@ -145,12 +163,102 @@ TEST_CASE("Read multipart files", "[.integration]") { } } +// dummy test class for easy member access +class RawFileTestWrapper : public RawFile { + public: + RawFileTestWrapper(const std::filesystem::path &fname, + const std::string &mode = "r") + : RawFile(fname, mode) {} + void find_geometry() { RawFile::find_geometry(); } + + DetectorGeometry get_geometry() { return this->m_geometry; } + + static DetectorHeader read_header(const std::filesystem::path &fname) { + return RawFile::read_header(fname); + } +}; + +struct TestParameters { + const std::string master_filename{}; + const uint8_t num_ports{}; + const DetectorGeometry geometry{}; +}; + +TEST_CASE("check find_geometry", "[.integration][.files][.rawfile]") { + + auto test_parameters = GENERATE( + TestParameters{ + "raw/jungfrau_2modules_version6.1.2/run_master_0.raw", 2, + DetectorGeometry{1, 2, 1024, 1024, 0, 0, + std::vector{ + ModuleGeometry{0, 0, 512, 1024, 0, 0}, + ModuleGeometry{0, 512, 512, 1024, 0, 1}}}}, + TestParameters{ + "raw/eiger_1_module_version7.0.0/eiger_1mod_master_7.json", 4, + DetectorGeometry{2, 2, 1024, 512, 0, 0, + std::vector{ + ModuleGeometry{0, 0, 256, 512, 0, 0}, + ModuleGeometry{512, 0, 256, 512, 0, 1}, + ModuleGeometry{0, 256, 256, 512, 1, 0}, + ModuleGeometry{512, 256, 256, 512, 1, 1}}}}); + + auto fpath = test_data_path() / test_parameters.master_filename; + REQUIRE(std::filesystem::exists(fpath)); + + RawFileTestWrapper f(fpath, "r"); + + auto geometry = f.get_geometry(); + + CHECK(geometry.modules_x == test_parameters.geometry.modules_x); + CHECK(geometry.modules_y == test_parameters.geometry.modules_y); + CHECK(geometry.pixels_x == test_parameters.geometry.pixels_x); + CHECK(geometry.pixels_y == test_parameters.geometry.pixels_y); + + REQUIRE(geometry.module_pixel_0.size() == test_parameters.num_ports); + + // compare to data stored in header + for (size_t i = 0; i < test_parameters.num_ports; ++i) { + + auto subfile1_path = f.master().data_fname(i, 0); + REQUIRE(std::filesystem::exists(subfile1_path)); + + auto header = RawFileTestWrapper::read_header(subfile1_path); + + CHECK(header.column == geometry.module_pixel_0[i].col_index); + CHECK(header.row == geometry.module_pixel_0[i].row_index); + + CHECK(geometry.module_pixel_0[i].height == + test_parameters.geometry.module_pixel_0[i].height); + CHECK(geometry.module_pixel_0[i].width == + test_parameters.geometry.module_pixel_0[i].width); + + CHECK(geometry.module_pixel_0[i].origin_x == + test_parameters.geometry.module_pixel_0[i].origin_x); + CHECK(geometry.module_pixel_0[i].origin_y == + test_parameters.geometry.module_pixel_0[i].origin_y); + } +} + +TEST_CASE("Open multi module file with ROI", + "[.integration][.files][.rawfile]") { + + auto fpath = test_data_path() / "SingleChipROI/Data_master_0.json"; + REQUIRE(std::filesystem::exists(fpath)); + + RawFile f(fpath, "r"); + + REQUIRE(f.master().roi().value().width() == 256); + REQUIRE(f.master().roi().value().height() == 256); + + CHECK(f.n_modules() == 2); + + CHECK(f.n_modules_in_roi() == 1); +} + TEST_CASE("Read file with unordered frames", "[.integration]") { - //TODO! Better explanation and error message + // TODO! Better explanation and error message auto fpath = test_data_path() / "mythen" / "scan242_master_3.raw"; REQUIRE(std::filesystem::exists(fpath)); File f(fpath); REQUIRE_THROWS((f.read_frame())); } - - diff --git a/src/RawMasterFile.cpp b/src/RawMasterFile.cpp index d4adf2c..f9f277a 100644 --- a/src/RawMasterFile.cpp +++ b/src/RawMasterFile.cpp @@ -139,6 +139,10 @@ size_t RawMasterFile::n_modules() const { return m_geometry.row * m_geometry.col; } +size_t RawMasterFile::num_udp_interfaces_per_module() const { + return m_num_udp_interfaces_per_module; +} + std::optional RawMasterFile::quad() const { return m_quad; } // optional values, these may or may not be present in the master file @@ -262,6 +266,12 @@ void RawMasterFile::parse_json(const std::filesystem::path &fpath) { } catch (const json::out_of_range &e) { // not a scan } + try { + m_num_udp_interfaces_per_module = j.at("Number of UDP Interfaces"); + } catch (const json::out_of_range &e) { + if (m_type == DetectorType::Eiger) + m_num_udp_interfaces_per_module = 2; + } try { ROI tmp_roi; @@ -276,14 +286,14 @@ void RawMasterFile::parse_json(const std::filesystem::path &fpath) { tmp_roi.ymin != 4294967295 || tmp_roi.ymax != 4294967295) { if (v < 7.21) { - tmp_roi.xmax++; + tmp_roi.xmax++; // why is it updated tmp_roi.ymax++; } - m_roi = tmp_roi; } } catch (const json::out_of_range &e) { + std::cout << e.what() << std::endl; // leave the optional empty } @@ -337,6 +347,9 @@ void RawMasterFile::parse_raw(const std::filesystem::path &fpath) { if (m_type == DetectorType::Moench) { m_type = DetectorType::Moench03_old; } + if (m_type == DetectorType::Eiger) { + m_num_udp_interfaces_per_module = 2; + } } else if (key == "Timing Mode") { m_timing_mode = StringTo(value); } else if (key == "Image Size") { @@ -393,6 +406,8 @@ void RawMasterFile::parse_raw(const std::filesystem::path &fpath) { m_geometry = { static_cast(std::stoi(value.substr(1, pos))), static_cast(std::stoi(value.substr(pos + 1)))}; + } else if (key == "Number of UDP Interfaces") { + m_num_udp_interfaces_per_module = std::stoi(value); } } } @@ -417,15 +432,18 @@ void RawMasterFile::retrieve_geometry() { uint16_t rows = 0; uint16_t cols = 0; // TODO use case for Eiger + while (std::filesystem::exists(data_fname(module_index, 0))) { + auto header = RawFile::read_header(data_fname(module_index, 0)); rows = std::max(rows, header.row); cols = std::max(cols, header.column); - ++rows; - ++cols; + ++module_index; } + ++rows; + ++cols; m_geometry = {rows, cols}; } diff --git a/src/RawMasterFile.test.cpp b/src/RawMasterFile.test.cpp index 560217f..0342604 100644 --- a/src/RawMasterFile.test.cpp +++ b/src/RawMasterFile.test.cpp @@ -1,12 +1,12 @@ #include "aare/RawMasterFile.hpp" -#include #include "test_config.hpp" +#include +#include using namespace aare; - -TEST_CASE("Parse a master file fname"){ +TEST_CASE("Parse a master file fname") { RawFileNameComponents m("test_master_1.json"); REQUIRE(m.base_name() == "test"); REQUIRE(m.ext() == ".json"); @@ -14,7 +14,7 @@ TEST_CASE("Parse a master file fname"){ REQUIRE(m.base_path() == ""); } -TEST_CASE("Extraction of base path works"){ +TEST_CASE("Extraction of base path works") { RawFileNameComponents m("some/path/test_master_73.json"); REQUIRE(m.base_name() == "test"); REQUIRE(m.ext() == ".json"); @@ -22,7 +22,7 @@ TEST_CASE("Extraction of base path works"){ REQUIRE(m.base_path() == "some/path"); } -TEST_CASE("Construction of master file name and data files"){ +TEST_CASE("Construction of master file name and data files") { RawFileNameComponents m("test_master_1.json"); REQUIRE(m.master_fname() == "test_master_1.json"); REQUIRE(m.data_fname(0, 0) == "test_d0_f0_1.raw"); @@ -31,7 +31,7 @@ TEST_CASE("Construction of master file name and data files"){ REQUIRE(m.data_fname(1, 1) == "test_d1_f1_1.raw"); } -TEST_CASE("Construction of master file name and data files using old scheme"){ +TEST_CASE("Construction of master file name and data files using old scheme") { RawFileNameComponents m("test_master_1.raw"); m.set_old_scheme(true); REQUIRE(m.master_fname() == "test_master_1.raw"); @@ -41,16 +41,15 @@ TEST_CASE("Construction of master file name and data files using old scheme"){ REQUIRE(m.data_fname(1, 1) == "test_d1_f000000000001_1.raw"); } -TEST_CASE("Master file name does not fit pattern"){ +TEST_CASE("Master file name does not fit pattern") { REQUIRE_THROWS(RawFileNameComponents("somefile.json")); REQUIRE_THROWS(RawFileNameComponents("another_test_d0_f0_1.raw")); REQUIRE_THROWS(RawFileNameComponents("test_master_1.txt")); } - - -TEST_CASE("Parse scan parameters"){ - ScanParameters s("[enabled\ndac dac 4\nstart 500\nstop 2200\nstep 5\nsettleTime 100us\n]"); +TEST_CASE("Parse scan parameters") { + ScanParameters s("[enabled\ndac dac 4\nstart 500\nstop 2200\nstep " + "5\nsettleTime 100us\n]"); REQUIRE(s.enabled()); REQUIRE(s.dac() == "dac 4"); REQUIRE(s.start() == 500); @@ -58,7 +57,7 @@ TEST_CASE("Parse scan parameters"){ REQUIRE(s.step() == 5); } -TEST_CASE("A disabled scan"){ +TEST_CASE("A disabled scan") { ScanParameters s("[disabled]"); REQUIRE_FALSE(s.enabled()); REQUIRE(s.dac() == ""); @@ -67,9 +66,9 @@ TEST_CASE("A disabled scan"){ REQUIRE(s.step() == 0); } - -TEST_CASE("Parse a master file in .json format", "[.integration]"){ - auto fpath = test_data_path() / "jungfrau" / "jungfrau_single_master_0.json"; +TEST_CASE("Parse a master file in .json format", "[.integration]") { + auto fpath = + test_data_path() / "jungfrau" / "jungfrau_single_master_0.json"; REQUIRE(std::filesystem::exists(fpath)); RawMasterFile f(fpath); @@ -80,7 +79,7 @@ TEST_CASE("Parse a master file in .json format", "[.integration]"){ REQUIRE(f.detector_type() == DetectorType::Jungfrau); // "Timing Mode": "auto", REQUIRE(f.timing_mode() == TimingMode::Auto); - + // "Geometry": { // "x": 1, // "y": 1 @@ -100,10 +99,9 @@ TEST_CASE("Parse a master file in .json format", "[.integration]"){ // "Max Frames Per File": 3, REQUIRE(f.max_frames_per_file() == 3); - //Jungfrau doesn't write but it is 16 + // Jungfrau doesn't write but it is 16 REQUIRE(f.bitdepth() == 16); - // "Frame Discard Policy": "nodiscard", // "Frame Padding": 1, @@ -125,33 +123,48 @@ TEST_CASE("Parse a master file in .json format", "[.integration]"){ // "Frames in File": 10, REQUIRE(f.frames_in_file() == 10); - //TODO! Should we parse this? - // "Frame Header Format": { - // "Frame Number": "8 bytes", - // "SubFrame Number/ExpLength": "4 bytes", - // "Packet Number": "4 bytes", - // "Bunch ID": "8 bytes", - // "Timestamp": "8 bytes", - // "Module Id": "2 bytes", - // "Row": "2 bytes", - // "Column": "2 bytes", - // "Reserved": "2 bytes", - // "Debug": "4 bytes", - // "Round Robin Number": "2 bytes", - // "Detector Type": "1 byte", - // "Header Version": "1 byte", - // "Packets Caught Mask": "64 bytes" - // } - // } + // TODO! Should we parse this? + // "Frame Header Format": { + // "Frame Number": "8 bytes", + // "SubFrame Number/ExpLength": "4 bytes", + // "Packet Number": "4 bytes", + // "Bunch ID": "8 bytes", + // "Timestamp": "8 bytes", + // "Module Id": "2 bytes", + // "Row": "2 bytes", + // "Column": "2 bytes", + // "Reserved": "2 bytes", + // "Debug": "4 bytes", + // "Round Robin Number": "2 bytes", + // "Detector Type": "1 byte", + // "Header Version": "1 byte", + // "Packets Caught Mask": "64 bytes" + // } + // } REQUIRE_FALSE(f.analog_samples()); - REQUIRE_FALSE(f.digital_samples()); - + REQUIRE_FALSE(f.digital_samples()); } -TEST_CASE("Parse a master file in .raw format", "[.integration]"){ - - auto fpath = test_data_path() / "moench/moench04_noise_200V_sto_both_100us_no_light_thresh_900_master_0.raw"; +TEST_CASE("Parse a master file in old .raw format", + "[.integration][.files][.rawmasterfile]") { + auto fpath = test_data_path() / + "raw/jungfrau_2modules_version6.1.2/run_master_0.raw"; + REQUIRE(std::filesystem::exists(fpath)); + RawMasterFile f(fpath); + + CHECK(f.num_udp_interfaces_per_module() == 1); + CHECK(f.n_modules() == 2); + CHECK(f.geometry().row == 2); + CHECK(f.geometry().col == 1); +} + +TEST_CASE("Parse a master file in .raw format", "[.integration]") { + + auto fpath = + test_data_path() / + "moench/" + "moench04_noise_200V_sto_both_100us_no_light_thresh_900_master_0.raw"; REQUIRE(std::filesystem::exists(fpath)); RawMasterFile f(fpath); @@ -209,80 +222,74 @@ TEST_CASE("Parse a master file in .raw format", "[.integration]"){ // Detector Type : 1 byte // Header Version : 1 byte // Packets Caught Mask : 64 bytes - - } - -TEST_CASE("Read eiger master file", "[.integration]"){ -auto fpath = test_data_path() / "eiger" / "eiger_500k_32bit_master_0.json"; +TEST_CASE("Read eiger master file", "[.integration]") { + auto fpath = test_data_path() / "eiger" / "eiger_500k_32bit_master_0.json"; REQUIRE(std::filesystem::exists(fpath)); RawMasterFile f(fpath); - -// { -// "Version": 7.2, -REQUIRE(f.version() == "7.2"); -// "Timestamp": "Tue Mar 26 17:24:34 2024", -// "Detector Type": "Eiger", -REQUIRE(f.detector_type() == DetectorType::Eiger); -// "Timing Mode": "auto", -REQUIRE(f.timing_mode() == TimingMode::Auto); -// "Geometry": { -// "x": 2, -// "y": 2 -// }, -// "Image Size in bytes": 524288, -REQUIRE(f.image_size_in_bytes() == 524288); -// "Pixels": { -// "x": 512, -REQUIRE(f.pixels_x() == 512); -// "y": 256 -REQUIRE(f.pixels_y() == 256); -// }, -// "Max Frames Per File": 10000, -REQUIRE(f.max_frames_per_file() == 10000); -// "Frame Discard Policy": "nodiscard", -REQUIRE(f.frame_discard_policy() == FrameDiscardPolicy::NoDiscard); -// "Frame Padding": 1, -REQUIRE(f.frame_padding() == 1); - -// "Scan Parameters": "[disabled]", -// "Total Frames": 3, -// "Receiver Roi": { -// "xmin": 4294967295, -// "xmax": 4294967295, -// "ymin": 4294967295, -// "ymax": 4294967295 -// }, -// "Dynamic Range": 32, -// "Ten Giga": 0, -// "Exptime": "5s", -// "Period": "1s", -// "Threshold Energy": -1, -// "Sub Exptime": "2.62144ms", -// "Sub Period": "2.62144ms", -// "Quad": 0, -// "Number of rows": 256, -// "Rate Corrections": "[0, 0]", -// "Frames in File": 3, -// "Frame Header Format": { -// "Frame Number": "8 bytes", -// "SubFrame Number/ExpLength": "4 bytes", -// "Packet Number": "4 bytes", -// "Bunch ID": "8 bytes", -// "Timestamp": "8 bytes", -// "Module Id": "2 bytes", -// "Row": "2 bytes", -// "Column": "2 bytes", -// "Reserved": "2 bytes", -// "Debug": "4 bytes", -// "Round Robin Number": "2 bytes", -// "Detector Type": "1 byte", -// "Header Version": "1 byte", -// "Packets Caught Mask": "64 bytes" -// } -// } - + // { + // "Version": 7.2, + REQUIRE(f.version() == "7.2"); + // "Timestamp": "Tue Mar 26 17:24:34 2024", + // "Detector Type": "Eiger", + REQUIRE(f.detector_type() == DetectorType::Eiger); + // "Timing Mode": "auto", + REQUIRE(f.timing_mode() == TimingMode::Auto); + // "Geometry": { + // "x": 2, + // "y": 2 + // }, + // "Image Size in bytes": 524288, + REQUIRE(f.image_size_in_bytes() == 524288); + // "Pixels": { + // "x": 512, + REQUIRE(f.pixels_x() == 512); + // "y": 256 + REQUIRE(f.pixels_y() == 256); + // }, + // "Max Frames Per File": 10000, + REQUIRE(f.max_frames_per_file() == 10000); + // "Frame Discard Policy": "nodiscard", + REQUIRE(f.frame_discard_policy() == FrameDiscardPolicy::NoDiscard); + // "Frame Padding": 1, + REQUIRE(f.frame_padding() == 1); + // "Scan Parameters": "[disabled]", + // "Total Frames": 3, + // "Receiver Roi": { + // "xmin": 4294967295, + // "xmax": 4294967295, + // "ymin": 4294967295, + // "ymax": 4294967295 + // }, + // "Dynamic Range": 32, + // "Ten Giga": 0, + // "Exptime": "5s", + // "Period": "1s", + // "Threshold Energy": -1, + // "Sub Exptime": "2.62144ms", + // "Sub Period": "2.62144ms", + // "Quad": 0, + // "Number of rows": 256, + // "Rate Corrections": "[0, 0]", + // "Frames in File": 3, + // "Frame Header Format": { + // "Frame Number": "8 bytes", + // "SubFrame Number/ExpLength": "4 bytes", + // "Packet Number": "4 bytes", + // "Bunch ID": "8 bytes", + // "Timestamp": "8 bytes", + // "Module Id": "2 bytes", + // "Row": "2 bytes", + // "Column": "2 bytes", + // "Reserved": "2 bytes", + // "Debug": "4 bytes", + // "Round Robin Number": "2 bytes", + // "Detector Type": "1 byte", + // "Header Version": "1 byte", + // "Packets Caught Mask": "64 bytes" + // } + // } } \ No newline at end of file From ad7525cd02e64dc2b422def0aed29515a4214d92 Mon Sep 17 00:00:00 2001 From: Alice Date: Tue, 10 Jun 2025 11:35:15 +0200 Subject: [PATCH 03/20] considered num_udp_interafces for jungfrau and quad structure for eiger --- include/aare/RawMasterFile.hpp | 7 +++-- include/aare/defs.hpp | 57 ++++++++++++++++------------------ src/RawFile.cpp | 34 ++++++++++++-------- src/RawFile.test.cpp | 22 +++++++++++-- src/RawMasterFile.cpp | 28 +++++++++++------ src/RawMasterFile.test.cpp | 2 +- 6 files changed, 90 insertions(+), 60 deletions(-) diff --git a/include/aare/RawMasterFile.hpp b/include/aare/RawMasterFile.hpp index 671fa36..ffcae66 100644 --- a/include/aare/RawMasterFile.hpp +++ b/include/aare/RawMasterFile.hpp @@ -82,8 +82,10 @@ class RawMasterFile { size_t m_pixels_x{}; size_t m_bitdepth{}; size_t m_num_udp_interfaces_per_module = 1; + uint8_t m_quad = 0; xy m_geometry{}; + xy m_udp_interfaces_per_module{1, 1}; size_t m_max_frames_per_file{}; // uint32_t m_adc_mask{}; // TODO! implement reading @@ -101,7 +103,6 @@ class RawMasterFile { std::optional m_digital_samples; std::optional m_transceiver_samples; std::optional m_number_of_rows; - std::optional m_quad; std::optional m_roi; @@ -120,18 +121,18 @@ class RawMasterFile { size_t max_frames_per_file() const; size_t bitdepth() const; size_t frame_padding() const; - size_t num_udp_interfaces_per_module() const; + xy udp_interfaces_per_module() const; const FrameDiscardPolicy &frame_discard_policy() const; size_t total_frames_expected() const; xy geometry() const; size_t n_modules() const; + uint8_t quad() const; std::optional analog_samples() const; std::optional digital_samples() const; std::optional transceiver_samples() const; std::optional number_of_rows() const; - std::optional quad() const; std::optional roi() const; diff --git a/include/aare/defs.hpp b/include/aare/defs.hpp index ccf07a5..71d8c49 100644 --- a/include/aare/defs.hpp +++ b/include/aare/defs.hpp @@ -3,16 +3,15 @@ #include "aare/Dtype.hpp" #include -#include #include #include #include +#include #include #include #include #include - /** * @brief LOCATION macro to get the current location in the code */ @@ -20,28 +19,24 @@ std::string(__FILE__) + std::string(":") + std::to_string(__LINE__) + \ ":" + std::string(__func__) + ":" - - #ifdef AARE_CUSTOM_ASSERT -#define AARE_ASSERT(expr)\ - if (expr)\ - {}\ - else\ +#define AARE_ASSERT(expr) \ + if (expr) { \ + } else \ aare::assert_failed(LOCATION + " Assertion failed: " + #expr + "\n"); #else -#define AARE_ASSERT(cond)\ - do { (void)sizeof(cond); } while(0) +#define AARE_ASSERT(cond) \ + do { \ + (void)sizeof(cond); \ + } while (0) #endif - namespace aare { inline constexpr size_t bits_per_byte = 8; void assert_failed(const std::string &msg); - - class DynamicCluster { public: int cluster_sizeX; @@ -55,7 +50,7 @@ class DynamicCluster { public: DynamicCluster(int cluster_sizeX_, int cluster_sizeY_, - Dtype dt_ = Dtype(typeid(int32_t))) + Dtype dt_ = Dtype(typeid(int32_t))) : cluster_sizeX(cluster_sizeX_), cluster_sizeY(cluster_sizeY_), dt(dt_) { m_data = new std::byte[cluster_sizeX * cluster_sizeY * dt.bytes()]{}; @@ -179,24 +174,24 @@ template struct t_xy { }; using xy = t_xy; - /** - * @brief Class to hold the geometry of a module. Where pixel 0 is located and the size of the module + * @brief Class to hold the geometry of a module. Where pixel 0 is located and + * the size of the module */ -struct ModuleGeometry{ +struct ModuleGeometry { int origin_x{}; int origin_y{}; int height{}; int width{}; int row_index{}; - int col_index{}; + int col_index{}; }; /** - * @brief Class to hold the geometry of a detector. Number of modules, their size and where pixel 0 - * for each module is located + * @brief Class to hold the geometry of a detector. Number of modules, their + * size and where pixel 0 for each module is located */ -struct DetectorGeometry{ +struct DetectorGeometry { int modules_x{}; int modules_y{}; int pixels_x{}; @@ -204,35 +199,34 @@ struct DetectorGeometry{ int module_gap_row{}; int module_gap_col{}; std::vector module_pixel_0; - + auto size() const { return module_pixel_0.size(); } }; -struct ROI{ +struct ROI { ssize_t xmin{}; ssize_t xmax{}; ssize_t ymin{}; ssize_t ymax{}; - + ssize_t height() const { return ymax - ymin; } ssize_t width() const { return xmax - xmin; } bool contains(ssize_t x, ssize_t y) const { return x >= xmin && x < xmax && y >= ymin && y < ymax; } - }; - +}; using dynamic_shape = std::vector; -//TODO! Can we uniform enums between the libraries? +// TODO! Can we uniform enums between the libraries? /** - * @brief Enum class to identify different detectors. + * @brief Enum class to identify different detectors. * The values are the same as in slsDetectorPackage * Different spelling to avoid confusion with the slsDetectorPackage */ enum class DetectorType { - //Standard detectors match the enum values from slsDetectorPackage + // Standard detectors match the enum values from slsDetectorPackage Generic, Eiger, Gotthard, @@ -243,8 +237,9 @@ enum class DetectorType { Gotthard2, Xilinx_ChipTestBoard, - //Additional detectors used for defining processing. Variants of the standard ones. - Moench03=100, + // Additional detectors used for defining processing. Variants of the + // standard ones. + Moench03 = 100, Moench03_old, Unknown }; diff --git a/src/RawFile.cpp b/src/RawFile.cpp index 2a60de1..9e78a7f 100644 --- a/src/RawFile.cpp +++ b/src/RawFile.cpp @@ -149,19 +149,27 @@ RawMasterFile RawFile::master() const { return m_master; } void RawFile::find_geometry() { for (size_t col = 0; col < m_master.geometry().col; - col += m_master.num_udp_interfaces_per_module()) - for (size_t row = 0; row < m_master.geometry().row; ++row) { - for (size_t port = 0; - port < m_master.num_udp_interfaces_per_module(); ++port) { - ModuleGeometry g; - g.origin_x = (col + port) * m_master.pixels_x(); - g.origin_y = row * m_master.pixels_y(); - g.row_index = row; - g.col_index = col + port; - g.width = m_master.pixels_x(); - g.height = m_master.pixels_y(); - m_geometry.module_pixel_0.push_back(g); - } + col += m_master.udp_interfaces_per_module().col) + for (size_t row = 0; row < m_master.geometry().row; + row += m_master.udp_interfaces_per_module().row) { + for (size_t port_row = 0; + port_row < m_master.udp_interfaces_per_module().row; + ++port_row) + for (size_t port_col = 0; + port_col < m_master.udp_interfaces_per_module().col; + ++port_col) { + ModuleGeometry g; + g.origin_x = (col + port_col) * m_master.pixels_x(); + g.origin_y = (row + port_row) * + m_master.pixels_y(); // TODO: quad doesnt seem + // to have an effect + g.row_index = m_master.quad() ? (row + port_row + 1) % 2 + : (row + port_row); + g.col_index = col + port_col; + g.width = m_master.pixels_x(); + g.height = m_master.pixels_y(); + m_geometry.module_pixel_0.push_back(g); + } } m_geometry.pixels_y = (m_master.geometry().row * m_master.pixels_y()); diff --git a/src/RawFile.test.cpp b/src/RawFile.test.cpp index 239bfb2..03641bf 100644 --- a/src/RawFile.test.cpp +++ b/src/RawFile.test.cpp @@ -200,9 +200,27 @@ TEST_CASE("check find_geometry", "[.integration][.files][.rawfile]") { ModuleGeometry{0, 0, 256, 512, 0, 0}, ModuleGeometry{512, 0, 256, 512, 0, 1}, ModuleGeometry{0, 256, 256, 512, 1, 0}, - ModuleGeometry{512, 256, 256, 512, 1, 1}}}}); + ModuleGeometry{512, 256, 256, 512, 1, 1}}}}, + + TestParameters{ + "raw/jungfrau_2modules_2interfaces/run_master_0.json", 4, + DetectorGeometry{1, 4, 1024, 1024, 0, 0, + std::vector{ + ModuleGeometry{0, 0, 256, 1024, 0, 0}, + ModuleGeometry{0, 256, 256, 1024, 1, 0}, + ModuleGeometry{0, 512, 256, 1024, 2, 0}, + ModuleGeometry{0, 768, 256, 1024, 3, 0}}}}, + TestParameters{ + "raw/W13_230320/" + "W13_vthreshscan_m21C_300V_800eV_vrpre3400_master_0.json", + 2, + DetectorGeometry{1, 2, 512, 512, 0, 0, + std::vector{ + ModuleGeometry{0, 0, 256, 512, 0, 0}, + ModuleGeometry{0, 256, 256, 512, 1, 0}}}}); auto fpath = test_data_path() / test_parameters.master_filename; + REQUIRE(std::filesystem::exists(fpath)); RawFileTestWrapper f(fpath, "r"); @@ -242,7 +260,7 @@ TEST_CASE("check find_geometry", "[.integration][.files][.rawfile]") { TEST_CASE("Open multi module file with ROI", "[.integration][.files][.rawfile]") { - auto fpath = test_data_path() / "SingleChipROI/Data_master_0.json"; + auto fpath = test_data_path() / "raw/SingleChipROI/Data_master_0.json"; REQUIRE(std::filesystem::exists(fpath)); RawFile f(fpath, "r"); diff --git a/src/RawMasterFile.cpp b/src/RawMasterFile.cpp index f9f277a..f6ef268 100644 --- a/src/RawMasterFile.cpp +++ b/src/RawMasterFile.cpp @@ -139,11 +139,11 @@ size_t RawMasterFile::n_modules() const { return m_geometry.row * m_geometry.col; } -size_t RawMasterFile::num_udp_interfaces_per_module() const { - return m_num_udp_interfaces_per_module; +xy RawMasterFile::udp_interfaces_per_module() const { + return m_udp_interfaces_per_module; } -std::optional RawMasterFile::quad() const { return m_quad; } +uint8_t RawMasterFile::quad() const { return m_quad; } // optional values, these may or may not be present in the master file // and are therefore modeled as std::optional @@ -267,10 +267,13 @@ void RawMasterFile::parse_json(const std::filesystem::path &fpath) { // not a scan } try { - m_num_udp_interfaces_per_module = j.at("Number of UDP Interfaces"); + m_udp_interfaces_per_module = {j.at("Number of UDP Interfaces"), 1}; } catch (const json::out_of_range &e) { - if (m_type == DetectorType::Eiger) - m_num_udp_interfaces_per_module = 2; + if (m_type == DetectorType::Eiger && m_quad == 1) + m_udp_interfaces_per_module = {2, 1}; + else if (m_type == DetectorType::Eiger) { + m_udp_interfaces_per_module = {1, 2}; + } } try { @@ -347,9 +350,6 @@ void RawMasterFile::parse_raw(const std::filesystem::path &fpath) { if (m_type == DetectorType::Moench) { m_type = DetectorType::Moench03_old; } - if (m_type == DetectorType::Eiger) { - m_num_udp_interfaces_per_module = 2; - } } else if (key == "Timing Mode") { m_timing_mode = StringTo(value); } else if (key == "Image Size") { @@ -407,10 +407,18 @@ void RawMasterFile::parse_raw(const std::filesystem::path &fpath) { static_cast(std::stoi(value.substr(1, pos))), static_cast(std::stoi(value.substr(pos + 1)))}; } else if (key == "Number of UDP Interfaces") { - m_num_udp_interfaces_per_module = std::stoi(value); + m_udp_interfaces_per_module = { + static_cast(std::stoi(value)), 1}; } } } + + if (m_type == DetectorType::Eiger && m_quad == 1) { + m_udp_interfaces_per_module = {2, 1}; + } else if (m_type == DetectorType::Eiger) { + m_udp_interfaces_per_module = {1, 2}; + } + if (m_pixels_x == 400 && m_pixels_y == 400) { m_type = DetectorType::Moench03_old; } diff --git a/src/RawMasterFile.test.cpp b/src/RawMasterFile.test.cpp index 0342604..c623935 100644 --- a/src/RawMasterFile.test.cpp +++ b/src/RawMasterFile.test.cpp @@ -153,7 +153,7 @@ TEST_CASE("Parse a master file in old .raw format", REQUIRE(std::filesystem::exists(fpath)); RawMasterFile f(fpath); - CHECK(f.num_udp_interfaces_per_module() == 1); + CHECK(f.udp_interfaces_per_module() == xy{1, 1}); CHECK(f.n_modules() == 2); CHECK(f.geometry().row == 2); CHECK(f.geometry().col == 1); From 6438a4bef136f7d4bbd366e49b86e1a8c2a0f359 Mon Sep 17 00:00:00 2001 From: Alice Date: Tue, 10 Jun 2025 12:00:07 +0200 Subject: [PATCH 04/20] updated python bindings --- python/src/raw_file.hpp | 5 +++-- python/src/raw_master_file.hpp | 7 +++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/python/src/raw_file.hpp b/python/src/raw_file.hpp index 8d72220..689b84e 100644 --- a/python/src/raw_file.hpp +++ b/python/src/raw_file.hpp @@ -58,13 +58,14 @@ void define_raw_file_io_bindings(py::module &m) { throw std::runtime_error("No frames left in file"); } std::vector shape{n_frames, self.rows(), self.cols()}; - + // return headers from all subfiles py::array_t header; if (self.n_modules() == 1) { header = py::array_t(n_frames); } else { - header = py::array_t({self.n_modules(), n_frames}); + header = py::array_t( + {self.n_modules(), n_frames}); } // py::array_t header({self.n_mod(), n_frames}); diff --git a/python/src/raw_master_file.hpp b/python/src/raw_master_file.hpp index 943437f..34db4d3 100644 --- a/python/src/raw_master_file.hpp +++ b/python/src/raw_master_file.hpp @@ -57,7 +57,10 @@ void define_raw_master_file_bindings(py::module &m) { .def_property_readonly("total_frames_expected", &RawMasterFile::total_frames_expected) .def_property_readonly("geometry", &RawMasterFile::geometry) - .def_property_readonly("analog_samples", &RawMasterFile::analog_samples, R"( + .def_property_readonly("udp_interfaces_per_module", + &RawMasterFile::udp_interfaces_per_module) + .def_property_readonly("analog_samples", &RawMasterFile::analog_samples, + R"( Number of analog samples Returns @@ -66,7 +69,7 @@ void define_raw_master_file_bindings(py::module &m) { The number of analog samples in the file (or None if not enabled) )") .def_property_readonly("digital_samples", - &RawMasterFile::digital_samples, R"( + &RawMasterFile::digital_samples, R"( Number of digital samples Returns From cfe7c31fe4aaab5d72f01c36c5330d8892bb2c8e Mon Sep 17 00:00:00 2001 From: Alice Date: Thu, 12 Jun 2025 09:52:43 +0200 Subject: [PATCH 05/20] changed data path of test data --- src/RawFile.test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/RawFile.test.cpp b/src/RawFile.test.cpp index 03641bf..c6b6623 100644 --- a/src/RawFile.test.cpp +++ b/src/RawFile.test.cpp @@ -211,7 +211,7 @@ TEST_CASE("check find_geometry", "[.integration][.files][.rawfile]") { ModuleGeometry{0, 512, 256, 1024, 2, 0}, ModuleGeometry{0, 768, 256, 1024, 3, 0}}}}, TestParameters{ - "raw/W13_230320/" + "raw/eiger_quad_data/" "W13_vthreshscan_m21C_300V_800eV_vrpre3400_master_0.json", 2, DetectorGeometry{1, 2, 512, 512, 0, 0, From 75f63607fc003ace9b3c73399c8827cb07d9c88f Mon Sep 17 00:00:00 2001 From: Alice Date: Thu, 12 Jun 2025 17:46:10 +0200 Subject: [PATCH 06/20] friend_test macro --- CMakeLists.txt | 9 +++++---- include/aare/RawFile.hpp | 25 +++++++++++++++++-------- include/aare/RawMasterFile.hpp | 3 +-- src/RawFile.cpp | 1 + src/RawFile.test.cpp | 12 +++++++++--- tests/CMakeLists.txt | 5 ++++- tests/friend_test.hpp | 4 ++++ tests/test_macros.hpp | 20 ++++++++++++++++++++ 8 files changed, 61 insertions(+), 18 deletions(-) create mode 100644 tests/friend_test.hpp create mode 100644 tests/test_macros.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index fa9838e..f0bd5d7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -335,13 +335,10 @@ if(AARE_ASAN) ) endif() - - - - if(AARE_TESTS) enable_testing() add_subdirectory(tests) + target_compile_definitions(tests PRIVATE AARE_TESTS) endif() ###------------------------------------------------------------------------------MAIN LIBRARY @@ -424,6 +421,10 @@ target_link_libraries( ) +if(AARE_TESTS) + target_compile_definitions(aare_core PRIVATE AARE_TESTS) +endif() + set_target_properties(aare_core PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR} PUBLIC_HEADER "${PUBLICHEADERS}" diff --git a/include/aare/RawFile.hpp b/include/aare/RawFile.hpp index 78ebf4c..6d90931 100644 --- a/include/aare/RawFile.hpp +++ b/include/aare/RawFile.hpp @@ -5,6 +5,10 @@ #include "aare/RawMasterFile.hpp" #include "aare/RawSubFile.hpp" +#ifdef AARE_TESTS +#include "../tests/friend_test.hpp" +#endif + #include namespace aare { @@ -21,6 +25,9 @@ struct ModuleConfig { return true; } }; +#ifdef AARE_TESTS +TEST_CASE_PRIVATE_FWD(check_find_geometry) // forward declaration +#endif /** * @brief Class to read .raw files. The class will parse the master file @@ -30,7 +37,9 @@ struct ModuleConfig { */ class RawFile : public FileInterface { - friend class RawMasterFile; +#ifdef AARE_TESTS + FRIEND_TEST(check_find_geometry) +#endif std::vector> m_subfiles; ModuleConfig cfg{0, 0}; RawMasterFile m_master; @@ -81,6 +90,13 @@ class RawFile : public FileInterface { DetectorType detector_type() const override; + /** + * @brief read the header of the file + * @param fname path to the data subfile + * @return DetectorHeader + */ + static DetectorHeader read_header(const std::filesystem::path &fname); + private: /** * @brief read the frame at the given frame index into the image buffer @@ -102,13 +118,6 @@ class RawFile : public FileInterface { protected: void find_geometry(); - - /** - * @brief read the header of the file - * @param fname path to the data subfile - * @return DetectorHeader - */ - static DetectorHeader read_header(const std::filesystem::path &fname); }; } // namespace aare \ No newline at end of file diff --git a/include/aare/RawMasterFile.hpp b/include/aare/RawMasterFile.hpp index ffcae66..ba052dc 100644 --- a/include/aare/RawMasterFile.hpp +++ b/include/aare/RawMasterFile.hpp @@ -1,5 +1,4 @@ #pragma once -// #include "aare/RawFile.hpp" #include "aare/defs.hpp" #include #include @@ -63,7 +62,7 @@ class ScanParameters { void increment_stop(); }; -class RawFile; // forward declaration +// class RawFile; // forward declaration /** * @brief Class for parsing a master file either in our .json format or the old diff --git a/src/RawFile.cpp b/src/RawFile.cpp index 9e78a7f..1ddbebf 100644 --- a/src/RawFile.cpp +++ b/src/RawFile.cpp @@ -148,6 +148,7 @@ RawMasterFile RawFile::master() const { return m_master; } */ void RawFile::find_geometry() { + m_geometry.reserve(n_modules()); for (size_t col = 0; col < m_master.geometry().col; col += m_master.udp_interfaces_per_module().col) for (size_t row = 0; row < m_master.geometry().row; diff --git a/src/RawFile.test.cpp b/src/RawFile.test.cpp index c6b6623..aaf84d8 100644 --- a/src/RawFile.test.cpp +++ b/src/RawFile.test.cpp @@ -7,6 +7,7 @@ #include #include "test_config.hpp" +#include "test_macros.hpp" using aare::File; using aare::RawFile; @@ -184,7 +185,8 @@ struct TestParameters { const DetectorGeometry geometry{}; }; -TEST_CASE("check find_geometry", "[.integration][.files][.rawfile]") { +TEST_CASE_PRIVATE(aare, check_find_geometry, "check find_geometry", + "[.integration][.files][.rawfile]") { auto test_parameters = GENERATE( TestParameters{ @@ -223,9 +225,11 @@ TEST_CASE("check find_geometry", "[.integration][.files][.rawfile]") { REQUIRE(std::filesystem::exists(fpath)); - RawFileTestWrapper f(fpath, "r"); + RawFile f(fpath, "r"); - auto geometry = f.get_geometry(); + f.find_geometry(); + + auto geometry = f.m_geometry; CHECK(geometry.modules_x == test_parameters.geometry.modules_x); CHECK(geometry.modules_y == test_parameters.geometry.modules_y); @@ -257,6 +261,8 @@ TEST_CASE("check find_geometry", "[.integration][.files][.rawfile]") { } } +} // close the namespace + TEST_CASE("Open multi module file with ROI", "[.integration][.files][.rawfile]") { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 1906508..994dc47 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -40,5 +40,8 @@ target_sources(tests PRIVATE ${TestSources} ) #configure a header to pass test file paths get_filename_component(TEST_FILE_PATH ${PROJECT_SOURCE_DIR}/data ABSOLUTE) configure_file(test_config.hpp.in test_config.hpp) -target_include_directories(tests PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) +target_include_directories(tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) + + + diff --git a/tests/friend_test.hpp b/tests/friend_test.hpp new file mode 100644 index 0000000..7d16000 --- /dev/null +++ b/tests/friend_test.hpp @@ -0,0 +1,4 @@ +#define FRIEND_TEST(test_name) friend void test_name##_impl(); + +#define TEST_CASE_PRIVATE_FWD(test_name) \ + void test_name##_impl(); // foward declaration diff --git a/tests/test_macros.hpp b/tests/test_macros.hpp new file mode 100644 index 0000000..4e89923 --- /dev/null +++ b/tests/test_macros.hpp @@ -0,0 +1,20 @@ +#include + +#include +#include +#include + +#define TEST_CASE_PRIVATE(namespace_name, test_name, test_name_str, \ + test_tags_str) \ + namespace namespace_name { \ + void test_name##_impl(); \ + namespace { \ + struct test_name##_Invoker : Catch::ITestInvoker { \ + void invoke() const override { test_name##_impl(); } \ + }; \ + Catch::AutoReg \ + autoReg_##test_name(Catch::Detail::make_unique(), \ + Catch::SourceLineInfo(__FILE__, __LINE__), "", \ + Catch::NameAndTags{test_name_str, test_tags_str}); \ + } \ + void test_name##_impl() From bd7870e75a9c21ef42e6367662c8ad3c3cd14b38 Mon Sep 17 00:00:00 2001 From: Alice Date: Thu, 12 Jun 2025 18:14:48 +0200 Subject: [PATCH 07/20] review comments --- include/aare/RawFile.hpp | 7 ++++--- src/RawFile.cpp | 2 +- src/RawFile.test.cpp | 24 ++++++------------------ 3 files changed, 11 insertions(+), 22 deletions(-) diff --git a/include/aare/RawFile.hpp b/include/aare/RawFile.hpp index 6d90931..d48dc9b 100644 --- a/include/aare/RawFile.hpp +++ b/include/aare/RawFile.hpp @@ -27,6 +27,7 @@ struct ModuleConfig { }; #ifdef AARE_TESTS TEST_CASE_PRIVATE_FWD(check_find_geometry) // forward declaration +TEST_CASE_PRIVATE_FWD(open_multi_module_file_with_roi) #endif /** @@ -39,6 +40,7 @@ class RawFile : public FileInterface { #ifdef AARE_TESTS FRIEND_TEST(check_find_geometry) + FRIEND_TEST(open_multi_module_file_with_roi) #endif std::vector> m_subfiles; ModuleConfig cfg{0, 0}; @@ -48,7 +50,6 @@ class RawFile : public FileInterface { std::vector m_modules_in_roi{}; - protected: DetectorGeometry m_geometry; public: @@ -84,7 +85,6 @@ class RawFile : public FileInterface { size_t bitdepth() const override; xy geometry(); size_t n_modules() const; - size_t n_modules_in_roi() const; RawMasterFile master() const; @@ -116,7 +116,8 @@ class RawFile : public FileInterface { void open_subfiles(); - protected: + size_t n_modules_in_roi() const; + void find_geometry(); }; diff --git a/src/RawFile.cpp b/src/RawFile.cpp index 1ddbebf..23d653b 100644 --- a/src/RawFile.cpp +++ b/src/RawFile.cpp @@ -148,7 +148,7 @@ RawMasterFile RawFile::master() const { return m_master; } */ void RawFile::find_geometry() { - m_geometry.reserve(n_modules()); + m_geometry.module_pixel_0.reserve(n_modules()); for (size_t col = 0; col < m_master.geometry().col; col += m_master.udp_interfaces_per_module().col) for (size_t row = 0; row < m_master.geometry().row; diff --git a/src/RawFile.test.cpp b/src/RawFile.test.cpp index aaf84d8..b0409de 100644 --- a/src/RawFile.test.cpp +++ b/src/RawFile.test.cpp @@ -164,21 +164,6 @@ TEST_CASE("Read multipart files", "[.integration]") { } } -// dummy test class for easy member access -class RawFileTestWrapper : public RawFile { - public: - RawFileTestWrapper(const std::filesystem::path &fname, - const std::string &mode = "r") - : RawFile(fname, mode) {} - void find_geometry() { RawFile::find_geometry(); } - - DetectorGeometry get_geometry() { return this->m_geometry; } - - static DetectorHeader read_header(const std::filesystem::path &fname) { - return RawFile::read_header(fname); - } -}; - struct TestParameters { const std::string master_filename{}; const uint8_t num_ports{}; @@ -227,6 +212,7 @@ TEST_CASE_PRIVATE(aare, check_find_geometry, "check find_geometry", RawFile f(fpath, "r"); + f.m_geometry.module_pixel_0.clear(); f.find_geometry(); auto geometry = f.m_geometry; @@ -244,7 +230,7 @@ TEST_CASE_PRIVATE(aare, check_find_geometry, "check find_geometry", auto subfile1_path = f.master().data_fname(i, 0); REQUIRE(std::filesystem::exists(subfile1_path)); - auto header = RawFileTestWrapper::read_header(subfile1_path); + auto header = RawFile::read_header(subfile1_path); CHECK(header.column == geometry.module_pixel_0[i].col_index); CHECK(header.row == geometry.module_pixel_0[i].row_index); @@ -263,8 +249,9 @@ TEST_CASE_PRIVATE(aare, check_find_geometry, "check find_geometry", } // close the namespace -TEST_CASE("Open multi module file with ROI", - "[.integration][.files][.rawfile]") { +TEST_CASE_PRIVATE(aare, open_multi_module_file_with_roi, + "Open multi module file with ROI", + "[.integration][.files][.rawfile]") { auto fpath = test_data_path() / "raw/SingleChipROI/Data_master_0.json"; REQUIRE(std::filesystem::exists(fpath)); @@ -278,6 +265,7 @@ TEST_CASE("Open multi module file with ROI", CHECK(f.n_modules_in_roi() == 1); } +} // close namespace aare TEST_CASE("Read file with unordered frames", "[.integration]") { // TODO! Better explanation and error message From be67bbab6baab2bd48d447acd81aaf9126909164 Mon Sep 17 00:00:00 2001 From: Alice Date: Fri, 13 Jun 2025 16:16:23 +0200 Subject: [PATCH 08/20] extended DetectorGeometry class with find_geometry, update_geometry (refactoring) --- CMakeLists.txt | 6 +- include/aare/DetectorGeometry.hpp | 81 +++++++++++ include/aare/RawFile.hpp | 22 +-- include/aare/defs.hpp | 29 ---- include/aare/geo_helpers.hpp | 15 -- src/DetectorGeometry.cpp | 152 ++++++++++++++++++++ src/DetectorGeometry.test.cpp | 194 +++++++++++++++++++++++++ src/RawFile.cpp | 130 +++++++---------- src/RawFile.test.cpp | 93 ++++++------ src/geo_helpers.cpp | 72 ---------- src/geo_helpers.test.cpp | 228 ------------------------------ 11 files changed, 526 insertions(+), 496 deletions(-) create mode 100644 include/aare/DetectorGeometry.hpp delete mode 100644 include/aare/geo_helpers.hpp create mode 100644 src/DetectorGeometry.cpp create mode 100644 src/DetectorGeometry.test.cpp delete mode 100644 src/geo_helpers.cpp delete mode 100644 src/geo_helpers.test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index f0bd5d7..e08768b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -361,7 +361,7 @@ set(PUBLICHEADERS include/aare/FilePtr.hpp include/aare/Frame.hpp include/aare/GainMap.hpp - include/aare/geo_helpers.hpp + include/aare/DetectorGeometry.hpp include/aare/JungfrauDataFile.hpp include/aare/NDArray.hpp include/aare/NDView.hpp @@ -386,7 +386,7 @@ set(SourceFiles ${CMAKE_CURRENT_SOURCE_DIR}/src/File.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/FilePtr.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Fit.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/src/geo_helpers.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/DetectorGeometry.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/JungfrauDataFile.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyFile.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyHelpers.cpp @@ -441,7 +441,7 @@ if(AARE_TESTS) ${CMAKE_CURRENT_SOURCE_DIR}/src/decode.test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Dtype.test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Frame.test.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/src/geo_helpers.test.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/DetectorGeometry.test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/RawMasterFile.test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/NDArray.test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/NDView.test.cpp diff --git a/include/aare/DetectorGeometry.hpp b/include/aare/DetectorGeometry.hpp new file mode 100644 index 0000000..0511319 --- /dev/null +++ b/include/aare/DetectorGeometry.hpp @@ -0,0 +1,81 @@ +#pragma once +#include "aare/RawMasterFile.hpp" //ROI refactor away +#include "aare/defs.hpp" +namespace aare { + +struct ModuleConfig { + int module_gap_row{}; + int module_gap_col{}; + + bool operator==(const ModuleConfig &other) const { + if (module_gap_col != other.module_gap_col) + return false; + if (module_gap_row != other.module_gap_row) + return false; + return true; + } +}; + +/** + * @brief Class to hold the geometry of a module. Where pixel 0 is located and + * the size of the module + */ +struct ModuleGeometry { + int origin_x{}; + int origin_y{}; + int height{}; + int width{}; + int row_index{}; + int col_index{}; +}; + +/** + * @brief Class to hold the geometry of a detector. Number of modules, their + * size and where pixel 0 for each module is located + */ +class DetectorGeometry { + public: + DetectorGeometry(const xy &geometry, const ssize_t module_pixels_x, + const ssize_t module_pixels_y, + const xy udp_interfaces_per_module = xy{1, 1}, + const bool quad = false); + + ~DetectorGeometry() = default; + + /** + * @brief Update the detector geometry given a region of interest + * + * @param roi + * @return DetectorGeometry + */ + void update_geometry_with_roi(ROI roi); + + size_t n_modules() const; + + size_t n_modules_in_roi() const; + + size_t pixels_x() const; + size_t pixels_y() const; + + size_t modules_x() const; + size_t modules_y() const; + + const std::vector &get_modules_in_roi() const; + + ssize_t get_modules_in_roi(const size_t index) const; + + const std::vector &get_module_geometries() const; + + const ModuleGeometry &get_module_geometries(const size_t index) const; + + private: + size_t m_modules_x{}; + size_t m_modules_y{}; + size_t m_pixels_x{}; + size_t m_pixels_y{}; + static constexpr ModuleConfig cfg{0, 0}; + std::vector module_geometries{}; + std::vector modules_in_roi{}; +}; + +} // namespace aare \ No newline at end of file diff --git a/include/aare/RawFile.hpp b/include/aare/RawFile.hpp index d48dc9b..67390fa 100644 --- a/include/aare/RawFile.hpp +++ b/include/aare/RawFile.hpp @@ -1,4 +1,5 @@ #pragma once +#include "aare/DetectorGeometry.hpp" #include "aare/FileInterface.hpp" #include "aare/Frame.hpp" #include "aare/NDArray.hpp" //for pixel map @@ -13,18 +14,6 @@ namespace aare { -struct ModuleConfig { - int module_gap_row{}; - int module_gap_col{}; - - bool operator==(const ModuleConfig &other) const { - if (module_gap_col != other.module_gap_col) - return false; - if (module_gap_row != other.module_gap_row) - return false; - return true; - } -}; #ifdef AARE_TESTS TEST_CASE_PRIVATE_FWD(check_find_geometry) // forward declaration TEST_CASE_PRIVATE_FWD(open_multi_module_file_with_roi) @@ -43,13 +32,11 @@ class RawFile : public FileInterface { FRIEND_TEST(open_multi_module_file_with_roi) #endif std::vector> m_subfiles; - ModuleConfig cfg{0, 0}; + RawMasterFile m_master; size_t m_current_frame{}; size_t m_current_subfile{}; - std::vector m_modules_in_roi{}; - DetectorGeometry m_geometry; public: @@ -84,7 +71,6 @@ class RawFile : public FileInterface { size_t cols() const override; size_t bitdepth() const override; xy geometry(); - size_t n_modules() const; RawMasterFile master() const; @@ -115,10 +101,6 @@ class RawFile : public FileInterface { Frame get_frame(size_t frame_index); void open_subfiles(); - - size_t n_modules_in_roi() const; - - void find_geometry(); }; } // namespace aare \ No newline at end of file diff --git a/include/aare/defs.hpp b/include/aare/defs.hpp index 71d8c49..9bb8cfa 100644 --- a/include/aare/defs.hpp +++ b/include/aare/defs.hpp @@ -174,35 +174,6 @@ template struct t_xy { }; using xy = t_xy; -/** - * @brief Class to hold the geometry of a module. Where pixel 0 is located and - * the size of the module - */ -struct ModuleGeometry { - int origin_x{}; - int origin_y{}; - int height{}; - int width{}; - int row_index{}; - int col_index{}; -}; - -/** - * @brief Class to hold the geometry of a detector. Number of modules, their - * size and where pixel 0 for each module is located - */ -struct DetectorGeometry { - int modules_x{}; - int modules_y{}; - int pixels_x{}; - int pixels_y{}; - int module_gap_row{}; - int module_gap_col{}; - std::vector module_pixel_0; - - auto size() const { return module_pixel_0.size(); } -}; - struct ROI { ssize_t xmin{}; ssize_t xmax{}; diff --git a/include/aare/geo_helpers.hpp b/include/aare/geo_helpers.hpp deleted file mode 100644 index c6454a5..0000000 --- a/include/aare/geo_helpers.hpp +++ /dev/null @@ -1,15 +0,0 @@ -#pragma once -#include "aare/RawMasterFile.hpp" //ROI refactor away -#include "aare/defs.hpp" -namespace aare { - -/** - * @brief Update the detector geometry given a region of interest - * - * @param geo - * @param roi - * @return DetectorGeometry - */ -DetectorGeometry update_geometry_with_roi(DetectorGeometry geo, ROI roi); - -} // namespace aare \ No newline at end of file diff --git a/src/DetectorGeometry.cpp b/src/DetectorGeometry.cpp new file mode 100644 index 0000000..c7f0125 --- /dev/null +++ b/src/DetectorGeometry.cpp @@ -0,0 +1,152 @@ + +#include "aare/DetectorGeometry.hpp" +#include "fmt/core.h" +#include +#include + +namespace aare { + +DetectorGeometry::DetectorGeometry(const xy &geometry, + const ssize_t module_pixels_x, + const ssize_t module_pixels_y, + const xy udp_interfaces_per_module, + const bool quad) { + + size_t num_modules = geometry.col * geometry.row; + module_geometries.reserve(num_modules); + for (size_t col = 0; col < geometry.col; + col += udp_interfaces_per_module.col) + for (size_t row = 0; row < geometry.row; + row += udp_interfaces_per_module.row) { + for (size_t port_row = 0; port_row < udp_interfaces_per_module.row; + ++port_row) + for (size_t port_col = 0; + port_col < udp_interfaces_per_module.col; ++port_col) { + ModuleGeometry g; + g.origin_x = (col + port_col) * module_pixels_x; + g.origin_y = (row + port_row) * + module_pixels_y; // TODO: quad doesnt seem + // to have an effect + g.row_index = + quad ? (row + port_row + 1) % 2 : (row + port_row); + g.col_index = col + port_col; + g.width = module_pixels_x; + g.height = module_pixels_y; + module_geometries.push_back(g); + } + } + + m_pixels_y = (geometry.row * module_pixels_y); + m_pixels_x = (geometry.col * module_pixels_x); + m_modules_x = geometry.col; + m_modules_y = geometry.row; + m_pixels_y += static_cast((geometry.row - 1) * cfg.module_gap_row); + + modules_in_roi.resize(num_modules); + std::iota(modules_in_roi.begin(), modules_in_roi.end(), 0); +} + +size_t DetectorGeometry::n_modules() const { return m_modules_x * m_modules_y; } + +size_t DetectorGeometry::n_modules_in_roi() const { + return modules_in_roi.size(); +}; + +size_t DetectorGeometry::pixels_x() const { return m_pixels_x; } +size_t DetectorGeometry::pixels_y() const { return m_pixels_y; } + +size_t DetectorGeometry::modules_x() const { return m_modules_x; }; +size_t DetectorGeometry::modules_y() const { return m_modules_y; }; + +const std::vector &DetectorGeometry::get_modules_in_roi() const { + return modules_in_roi; +} + +ssize_t DetectorGeometry::get_modules_in_roi(const size_t index) const { + return modules_in_roi[index]; +} + +const std::vector & +DetectorGeometry::get_module_geometries() const { + return module_geometries; +} + +const ModuleGeometry & +DetectorGeometry::get_module_geometries(const size_t index) const { + return module_geometries[index]; +} + +void DetectorGeometry::update_geometry_with_roi(ROI roi) { +#ifdef AARE_VERBOSE + fmt::println("update_geometry_with_roi() called with ROI: {} {} {} {}", + roi.xmin, roi.xmax, roi.ymin, roi.ymax); + fmt::println("Geometry: {} {} {} {} {} {}", m_modules_x, m_modules_y, + m_pixels_x, m_pixels_y, cfg.module_gap_row, + cfg.module_gap_col); + +#endif + + modules_in_roi.clear(); + modules_in_roi.reserve(m_modules_x * m_modules_y); + int pos_y = 0; + int pos_y_increment = 0; + for (size_t row = 0; row < m_modules_y; row++) { + int pos_x = 0; + for (size_t col = 0; col < m_modules_x; col++) { + auto &m = module_geometries[row * m_modules_x + col]; + + auto original_height = m.height; + auto original_width = m.width; + + // module is to the left of the roi + if (m.origin_x + m.width < roi.xmin) { + m.width = 0; + + // roi is in module + } else { + // here we only arrive when the roi is in or to the left of + // the module + if (roi.xmin > m.origin_x) { + m.width -= roi.xmin - m.origin_x; + } + if (roi.xmax < m.origin_x + original_width) { + m.width -= m.origin_x + original_width - roi.xmax; + } + m.origin_x = pos_x; + pos_x += m.width; + } + + if (m.origin_y + m.height < roi.ymin) { + m.height = 0; + } else { + if ((roi.ymin > m.origin_y) && + (roi.ymin < m.origin_y + m.height)) { + m.height -= roi.ymin - m.origin_y; + } + if (roi.ymax < m.origin_y + original_height) { + m.height -= m.origin_y + original_height - roi.ymax; + } + m.origin_y = pos_y; + pos_y_increment = m.height; + } + + if (m.height != 0 && m.width != 0) { + modules_in_roi.push_back(row * m_modules_x + col); + } + +#ifdef AARE_VERBOSE + fmt::println("Module {} {} {} {}", m.origin_x, m.origin_y, m.width, + m.height); +#endif + } + // increment pos_y + pos_y += pos_y_increment; + } + + // m_rows = roi.height(); + // m_cols = roi.width(); + m_pixels_x = roi.width(); + m_pixels_y = roi.height(); +} + +} // namespace aare \ No newline at end of file diff --git a/src/DetectorGeometry.test.cpp b/src/DetectorGeometry.test.cpp new file mode 100644 index 0000000..1424f0a --- /dev/null +++ b/src/DetectorGeometry.test.cpp @@ -0,0 +1,194 @@ +#include "aare/File.hpp" +#include "aare/RawFile.hpp" +#include "aare/RawMasterFile.hpp" //needed for ROI + +#include +#include + +#include "aare/DetectorGeometry.hpp" +#include "test_config.hpp" + +TEST_CASE("Simple ROIs on one module") { + // DetectorGeometry update_geometry_with_roi(DetectorGeometry geo, aare::ROI + // roi) + aare::DetectorGeometry geo(aare::xy{1, 1}, 1024, 512); + + REQUIRE(geo.get_module_geometries(0).origin_x == 0); + REQUIRE(geo.get_module_geometries(0).origin_y == 0); + REQUIRE(geo.get_module_geometries(0).width == 1024); + REQUIRE(geo.get_module_geometries(0).height == 512); + + SECTION("ROI is the whole module") { + aare::ROI roi; + roi.xmin = 0; + roi.xmax = 1024; + roi.ymin = 0; + roi.ymax = 512; + geo.update_geometry_with_roi(roi); + + REQUIRE(geo.pixels_x() == 1024); + REQUIRE(geo.pixels_y() == 512); + REQUIRE(geo.modules_x() == 1); + REQUIRE(geo.modules_y() == 1); + REQUIRE(geo.get_module_geometries(0).height == 512); + REQUIRE(geo.get_module_geometries(0).width == 1024); + } + SECTION("ROI is the top left corner of the module") { + aare::ROI roi; + roi.xmin = 100; + roi.xmax = 200; + roi.ymin = 150; + roi.ymax = 200; + geo.update_geometry_with_roi(roi); + + REQUIRE(geo.pixels_x() == 100); + REQUIRE(geo.pixels_y() == 50); + REQUIRE(geo.modules_x() == 1); + REQUIRE(geo.modules_y() == 1); + REQUIRE(geo.get_module_geometries(0).height == 50); + REQUIRE(geo.get_module_geometries(0).width == 100); + } + + SECTION("ROI is a small square") { + aare::ROI roi; + roi.xmin = 1000; + roi.xmax = 1010; + roi.ymin = 500; + roi.ymax = 510; + geo.update_geometry_with_roi(roi); + + REQUIRE(geo.pixels_x() == 10); + REQUIRE(geo.pixels_y() == 10); + REQUIRE(geo.modules_x() == 1); + REQUIRE(geo.modules_y() == 1); + REQUIRE(geo.get_module_geometries(0).height == 10); + REQUIRE(geo.get_module_geometries(0).width == 10); + } + SECTION("ROI is a few columns") { + aare::ROI roi; + roi.xmin = 750; + roi.xmax = 800; + roi.ymin = 0; + roi.ymax = 512; + geo.update_geometry_with_roi(roi); + + REQUIRE(geo.pixels_x() == 50); + REQUIRE(geo.pixels_y() == 512); + REQUIRE(geo.modules_x() == 1); + REQUIRE(geo.modules_y() == 1); + REQUIRE(geo.get_module_geometries(0).height == 512); + REQUIRE(geo.get_module_geometries(0).width == 50); + } +} + +TEST_CASE("Two modules side by side") { + // DetectorGeometry update_geometry_with_roi(DetectorGeometry geo, aare::ROI + // roi) + aare::DetectorGeometry geo(aare::xy{1, 2}, 1024, 512); + + REQUIRE(geo.get_module_geometries(0).origin_x == 0); + REQUIRE(geo.get_module_geometries(0).origin_y == 0); + REQUIRE(geo.get_module_geometries(0).width == 1024); + REQUIRE(geo.get_module_geometries(0).height == 512); + REQUIRE(geo.get_module_geometries(1).origin_x == 1024); + REQUIRE(geo.get_module_geometries(1).origin_y == 0); + + SECTION("ROI is the whole image") { + aare::ROI roi; + roi.xmin = 0; + roi.xmax = 2048; + roi.ymin = 0; + roi.ymax = 512; + geo.update_geometry_with_roi(roi); + + REQUIRE(geo.pixels_x() == 2048); + REQUIRE(geo.pixels_y() == 512); + REQUIRE(geo.modules_x() == 2); + REQUIRE(geo.modules_y() == 1); + } + SECTION("rectangle on both modules") { + aare::ROI roi; + roi.xmin = 800; + roi.xmax = 1300; + roi.ymin = 200; + roi.ymax = 499; + geo.update_geometry_with_roi(roi); + + REQUIRE(geo.pixels_x() == 500); + REQUIRE(geo.pixels_y() == 299); + REQUIRE(geo.modules_x() == 2); + REQUIRE(geo.modules_y() == 1); + REQUIRE(geo.get_module_geometries(0).height == 299); + REQUIRE(geo.get_module_geometries(0).width == 224); + REQUIRE(geo.get_module_geometries(1).height == 299); + REQUIRE(geo.get_module_geometries(1).width == 276); + } +} + +TEST_CASE("Three modules side by side") { + // DetectorGeometry update_geometry_with_roi(DetectorGeometry geo, aare::ROI + // roi) + aare::DetectorGeometry geo(aare::xy{1, 3}, 1024, 512); + aare::ROI roi; + roi.xmin = 700; + roi.xmax = 2500; + roi.ymin = 0; + roi.ymax = 123; + + REQUIRE(geo.get_module_geometries(0).origin_x == 0); + REQUIRE(geo.get_module_geometries(0).origin_y == 0); + REQUIRE(geo.get_module_geometries(0).width == 1024); + REQUIRE(geo.get_module_geometries(0).height == 512); + REQUIRE(geo.get_module_geometries(1).origin_x == 1024); + REQUIRE(geo.get_module_geometries(2).origin_x == 2048); + + geo.update_geometry_with_roi(roi); + + REQUIRE(geo.pixels_x() == 1800); + REQUIRE(geo.pixels_y() == 123); + REQUIRE(geo.modules_x() == 3); + REQUIRE(geo.modules_y() == 1); + REQUIRE(geo.get_module_geometries(0).height == 123); + REQUIRE(geo.get_module_geometries(0).width == 324); + REQUIRE(geo.get_module_geometries(1).height == 123); + REQUIRE(geo.get_module_geometries(1).width == 1024); + REQUIRE(geo.get_module_geometries(2).height == 123); + REQUIRE(geo.get_module_geometries(2).width == 452); +} + +TEST_CASE("Four modules as a square") { + // DetectorGeometry update_geometry_with_roi(DetectorGeometry geo, aare::ROI + // roi) + aare::DetectorGeometry geo(aare::xy{2, 2}, 1024, 512, aare::xy{1, 2}); + aare::ROI roi; + roi.xmin = 500; + roi.xmax = 2000; + roi.ymin = 500; + roi.ymax = 600; + + REQUIRE(geo.get_module_geometries(0).origin_x == 0); + REQUIRE(geo.get_module_geometries(0).origin_y == 0); + REQUIRE(geo.get_module_geometries(0).width == 1024); + REQUIRE(geo.get_module_geometries(0).height == 512); + REQUIRE(geo.get_module_geometries(1).origin_x == 1024); + REQUIRE(geo.get_module_geometries(1).origin_y == 0); + REQUIRE(geo.get_module_geometries(2).origin_x == 0); + REQUIRE(geo.get_module_geometries(2).origin_y == 512); + REQUIRE(geo.get_module_geometries(3).origin_x == 1024); + REQUIRE(geo.get_module_geometries(3).origin_y == 512); + + geo.update_geometry_with_roi(roi); + + REQUIRE(geo.pixels_x() == 1500); + REQUIRE(geo.pixels_y() == 100); + REQUIRE(geo.modules_x() == 2); + REQUIRE(geo.modules_y() == 2); + REQUIRE(geo.get_module_geometries(0).height == 12); + REQUIRE(geo.get_module_geometries(0).width == 524); + REQUIRE(geo.get_module_geometries(1).height == 12); + REQUIRE(geo.get_module_geometries(1).width == 976); + REQUIRE(geo.get_module_geometries(2).height == 88); + REQUIRE(geo.get_module_geometries(2).width == 524); + REQUIRE(geo.get_module_geometries(3).height == 88); + REQUIRE(geo.get_module_geometries(3).width == 976); +} \ No newline at end of file diff --git a/src/RawFile.cpp b/src/RawFile.cpp index 23d653b..2c46a90 100644 --- a/src/RawFile.cpp +++ b/src/RawFile.cpp @@ -1,8 +1,8 @@ #include "aare/RawFile.hpp" +#include "aare/DetectorGeometry.hpp" #include "aare/PixelMap.hpp" #include "aare/algorithm.hpp" #include "aare/defs.hpp" -#include "aare/geo_helpers.hpp" #include "aare/logger.hpp" #include @@ -13,26 +13,16 @@ using json = nlohmann::json; namespace aare { RawFile::RawFile(const std::filesystem::path &fname, const std::string &mode) - : m_master(fname) { + : m_master(fname), + m_geometry(m_master.geometry(), m_master.pixels_x(), m_master.pixels_y(), + m_master.udp_interfaces_per_module(), m_master.quad()) { m_mode = mode; if (mode == "r") { - find_geometry(); if (m_master.roi()) { - m_geometry = - update_geometry_with_roi(m_geometry, m_master.roi().value()); - m_modules_in_roi.reserve(n_modules()); - for (size_t module_index = 0; module_index < n_modules(); - ++module_index) { - if (m_geometry.module_pixel_0[module_index].width != 0 && - m_geometry.module_pixel_0[module_index].height != 0) - m_modules_in_roi.push_back(module_index); - } - } else { - m_modules_in_roi.resize(n_modules()); - std::iota(m_modules_in_roi.begin(), m_modules_in_roi.end(), 0); + m_geometry.update_geometry_with_roi(m_master.roi().value()); + open_subfiles(); } - open_subfiles(); } else { throw std::runtime_error(LOCATION + " Unsupported mode. Can only read RawFiles."); @@ -72,28 +62,30 @@ void RawFile::read_into(std::byte *image_buf, size_t n_frames, this->get_frame_into(m_current_frame++, image_buf, header); image_buf += bytes_per_frame(); if (header) - header += n_modules(); + header += m_geometry.n_modules(); } } -size_t RawFile::n_modules() const { return m_master.n_modules(); } +// size_t RawFile::n_modules() const { return m_master.n_modules(); } -size_t RawFile::n_modules_in_roi() const { return m_modules_in_roi.size(); } +// size_t RawFile::n_modules_in_roi() const { return +// m_modules_in_roi.size(); } size_t RawFile::bytes_per_frame() { - return m_geometry.pixels_x * m_geometry.pixels_y * m_master.bitdepth() / + return m_geometry.pixels_x() * m_geometry.pixels_y() * m_master.bitdepth() / bits_per_byte; } size_t RawFile::pixels_per_frame() { // return m_rows * m_cols; - return m_geometry.pixels_x * m_geometry.pixels_y; + return m_geometry.pixels_x() * m_geometry.pixels_y(); } DetectorType RawFile::detector_type() const { return m_master.detector_type(); } void RawFile::seek(size_t frame_index) { // check if the frame number is greater than the total frames - // if frame_number == total_frames, then the next read will throw an error + // if frame_number == total_frames, then the next read will throw an + // error if (frame_index > total_frames()) { throw std::runtime_error( fmt::format("frame number {} is greater than total frames {}", @@ -105,15 +97,15 @@ void RawFile::seek(size_t frame_index) { size_t RawFile::tell() { return m_current_frame; } size_t RawFile::total_frames() const { return m_master.frames_in_file(); } -size_t RawFile::rows() const { return m_geometry.pixels_y; } -size_t RawFile::cols() const { return m_geometry.pixels_x; } +size_t RawFile::rows() const { return m_geometry.pixels_y(); } +size_t RawFile::cols() const { return m_geometry.pixels_x(); } size_t RawFile::bitdepth() const { return m_master.bitdepth(); } xy RawFile::geometry() { return m_master.geometry(); } void RawFile::open_subfiles() { if (m_mode == "r") - for (size_t i : m_modules_in_roi) { - auto pos = m_geometry.module_pixel_0[i]; + for (size_t i : m_geometry.get_modules_in_roi()) { + auto pos = m_geometry.get_module_geometries(i); m_subfiles.emplace_back(std::make_unique( m_master.data_fname(i, 0), m_master.detector_type(), pos.height, pos.width, m_master.bitdepth(), pos.row_index, pos.col_index)); @@ -143,46 +135,8 @@ DetectorHeader RawFile::read_header(const std::filesystem::path &fname) { RawMasterFile RawFile::master() const { return m_master; } -/** - * @brief Find the geometry of the detector - */ -void RawFile::find_geometry() { - - m_geometry.module_pixel_0.reserve(n_modules()); - for (size_t col = 0; col < m_master.geometry().col; - col += m_master.udp_interfaces_per_module().col) - for (size_t row = 0; row < m_master.geometry().row; - row += m_master.udp_interfaces_per_module().row) { - for (size_t port_row = 0; - port_row < m_master.udp_interfaces_per_module().row; - ++port_row) - for (size_t port_col = 0; - port_col < m_master.udp_interfaces_per_module().col; - ++port_col) { - ModuleGeometry g; - g.origin_x = (col + port_col) * m_master.pixels_x(); - g.origin_y = (row + port_row) * - m_master.pixels_y(); // TODO: quad doesnt seem - // to have an effect - g.row_index = m_master.quad() ? (row + port_row + 1) % 2 - : (row + port_row); - g.col_index = col + port_col; - g.width = m_master.pixels_x(); - g.height = m_master.pixels_y(); - m_geometry.module_pixel_0.push_back(g); - } - } - - m_geometry.pixels_y = (m_master.geometry().row * m_master.pixels_y()); - m_geometry.pixels_x = (m_master.geometry().col * m_master.pixels_x()); - m_geometry.modules_x = m_master.geometry().col; - m_geometry.modules_y = m_master.geometry().row; - m_geometry.pixels_y += - static_cast((m_master.geometry().row - 1) * cfg.module_gap_row); -} - Frame RawFile::get_frame(size_t frame_index) { - auto f = Frame(m_geometry.pixels_y, m_geometry.pixels_x, + auto f = Frame(m_geometry.pixels_y(), m_geometry.pixels_x(), Dtype::from_bitdepth(m_master.bitdepth())); std::byte *frame_buffer = f.data(); get_frame_into(frame_index, frame_buffer); @@ -197,13 +151,15 @@ void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer, if (frame_index >= total_frames()) { throw std::runtime_error(LOCATION + "Frame number out of range"); } - std::vector frame_numbers(n_modules_in_roi()); - std::vector frame_indices(n_modules_in_roi(), frame_index); + std::vector frame_numbers(m_geometry.n_modules_in_roi()); + std::vector frame_indices(m_geometry.n_modules_in_roi(), + frame_index); // sync the frame numbers - if (n_modules() != 1) { // if we have more than one module - for (size_t part_idx = 0; part_idx != n_modules_in_roi(); ++part_idx) { + if (m_geometry.n_modules() != 1) { // if we have more than one module + for (size_t part_idx = 0; part_idx != m_geometry.n_modules_in_roi(); + ++part_idx) { frame_numbers[part_idx] = m_subfiles[part_idx]->frame_number(frame_index); } @@ -233,24 +189,32 @@ void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer, if (m_master.geometry().col == 1) { // get the part from each subfile and copy it to the frame - for (size_t part_idx = 0; part_idx != n_modules_in_roi(); ++part_idx) { + for (size_t part_idx = 0; part_idx != m_geometry.n_modules_in_roi(); + ++part_idx) { auto corrected_idx = frame_indices[part_idx]; // This is where we start writing - auto offset = (m_geometry.module_pixel_0[m_modules_in_roi[part_idx]] + auto offset = (m_geometry + .get_module_geometries( + m_geometry.get_modules_in_roi(part_idx)) .origin_y * - m_geometry.pixels_x + - m_geometry.module_pixel_0[m_modules_in_roi[part_idx]] + m_geometry.pixels_x() + + m_geometry + .get_module_geometries( + m_geometry.get_modules_in_roi(part_idx)) .origin_x) * m_master.bitdepth() / 8; - if (m_geometry.module_pixel_0[m_modules_in_roi[part_idx]] + if (m_geometry + .get_module_geometries( + m_geometry.get_modules_in_roi(part_idx)) .origin_x != 0) throw std::runtime_error( LOCATION + - " Implementation error. x pos not 0."); // TODO: origin can - // still change if - // roi changes + " Implementation error. x pos not 0."); // TODO: origin + // can still + // change if roi + // changes // TODO! What if the files don't match? m_subfiles[part_idx]->seek(corrected_idx); m_subfiles[part_idx]->read_into(frame_buffer + offset, header); @@ -268,11 +232,13 @@ void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer, auto *part_buffer = new std::byte[bytes_per_part]; - // TODO! if we have many submodules we should reorder them on the module - // level + // TODO! if we have many submodules we should reorder them on the + // module level - for (size_t part_idx = 0; part_idx != n_modules_in_roi(); ++part_idx) { - auto pos = m_geometry.module_pixel_0[m_modules_in_roi[part_idx]]; + for (size_t part_idx = 0; part_idx != m_geometry.n_modules_in_roi(); + ++part_idx) { + auto pos = m_geometry.get_module_geometries( + m_geometry.get_modules_in_roi(part_idx)); auto corrected_idx = frame_indices[part_idx]; m_subfiles[part_idx]->seek(corrected_idx); @@ -285,7 +251,7 @@ void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer, auto irow = (pos.origin_y + cur_row); auto icol = pos.origin_x; - auto dest = (irow * this->m_geometry.pixels_x + icol); + auto dest = (irow * this->m_geometry.pixels_x() + icol); dest = dest * m_master.bitdepth() / 8; memcpy(frame_buffer + dest, part_buffer + diff --git a/src/RawFile.test.cpp b/src/RawFile.test.cpp index b0409de..978d011 100644 --- a/src/RawFile.test.cpp +++ b/src/RawFile.test.cpp @@ -167,44 +167,45 @@ TEST_CASE("Read multipart files", "[.integration]") { struct TestParameters { const std::string master_filename{}; const uint8_t num_ports{}; - const DetectorGeometry geometry{}; + const size_t modules_x{}; + const size_t modules_y{}; + const size_t pixels_x{}; + const size_t pixels_y{}; + std::vector module_geometries{}; }; TEST_CASE_PRIVATE(aare, check_find_geometry, "check find_geometry", "[.integration][.files][.rawfile]") { auto test_parameters = GENERATE( + TestParameters{"raw/jungfrau_2modules_version6.1.2/run_master_0.raw", 2, + 1, 2, 1024, 1024, + std::vector{ + ModuleGeometry{0, 0, 512, 1024, 0, 0}, + ModuleGeometry{0, 512, 512, 1024, 0, 1}}}, TestParameters{ - "raw/jungfrau_2modules_version6.1.2/run_master_0.raw", 2, - DetectorGeometry{1, 2, 1024, 1024, 0, 0, - std::vector{ - ModuleGeometry{0, 0, 512, 1024, 0, 0}, - ModuleGeometry{0, 512, 512, 1024, 0, 1}}}}, - TestParameters{ - "raw/eiger_1_module_version7.0.0/eiger_1mod_master_7.json", 4, - DetectorGeometry{2, 2, 1024, 512, 0, 0, - std::vector{ - ModuleGeometry{0, 0, 256, 512, 0, 0}, - ModuleGeometry{512, 0, 256, 512, 0, 1}, - ModuleGeometry{0, 256, 256, 512, 1, 0}, - ModuleGeometry{512, 256, 256, 512, 1, 1}}}}, + "raw/eiger_1_module_version7.0.0/eiger_1mod_master_7.json", 4, 2, 2, + 1024, 512, + std::vector{ + ModuleGeometry{0, 0, 256, 512, 0, 0}, + ModuleGeometry{512, 0, 256, 512, 0, 1}, + ModuleGeometry{0, 256, 256, 512, 1, 0}, + ModuleGeometry{512, 256, 256, 512, 1, 1}}}, - TestParameters{ - "raw/jungfrau_2modules_2interfaces/run_master_0.json", 4, - DetectorGeometry{1, 4, 1024, 1024, 0, 0, - std::vector{ - ModuleGeometry{0, 0, 256, 1024, 0, 0}, - ModuleGeometry{0, 256, 256, 1024, 1, 0}, - ModuleGeometry{0, 512, 256, 1024, 2, 0}, - ModuleGeometry{0, 768, 256, 1024, 3, 0}}}}, + TestParameters{"raw/jungfrau_2modules_2interfaces/run_master_0.json", 4, + 1, 4, 1024, 1024, + std::vector{ + ModuleGeometry{0, 0, 256, 1024, 0, 0}, + ModuleGeometry{0, 256, 256, 1024, 1, 0}, + ModuleGeometry{0, 512, 256, 1024, 2, 0}, + ModuleGeometry{0, 768, 256, 1024, 3, 0}}}, TestParameters{ "raw/eiger_quad_data/" "W13_vthreshscan_m21C_300V_800eV_vrpre3400_master_0.json", - 2, - DetectorGeometry{1, 2, 512, 512, 0, 0, - std::vector{ - ModuleGeometry{0, 0, 256, 512, 0, 0}, - ModuleGeometry{0, 256, 256, 512, 1, 0}}}}); + 2, 1, 2, 512, 512, + std::vector{ + ModuleGeometry{0, 0, 256, 512, 0, 0}, + ModuleGeometry{0, 256, 256, 512, 1, 0}}}); auto fpath = test_data_path() / test_parameters.master_filename; @@ -212,17 +213,15 @@ TEST_CASE_PRIVATE(aare, check_find_geometry, "check find_geometry", RawFile f(fpath, "r"); - f.m_geometry.module_pixel_0.clear(); - f.find_geometry(); - auto geometry = f.m_geometry; - CHECK(geometry.modules_x == test_parameters.geometry.modules_x); - CHECK(geometry.modules_y == test_parameters.geometry.modules_y); - CHECK(geometry.pixels_x == test_parameters.geometry.pixels_x); - CHECK(geometry.pixels_y == test_parameters.geometry.pixels_y); + CHECK(geometry.modules_x() == test_parameters.modules_x); + CHECK(geometry.modules_y() == test_parameters.modules_y); + CHECK(geometry.pixels_x() == test_parameters.pixels_x); + CHECK(geometry.pixels_y() == test_parameters.pixels_y); - REQUIRE(geometry.module_pixel_0.size() == test_parameters.num_ports); + REQUIRE(geometry.get_module_geometries().size() == + test_parameters.num_ports); // compare to data stored in header for (size_t i = 0; i < test_parameters.num_ports; ++i) { @@ -232,18 +231,18 @@ TEST_CASE_PRIVATE(aare, check_find_geometry, "check find_geometry", auto header = RawFile::read_header(subfile1_path); - CHECK(header.column == geometry.module_pixel_0[i].col_index); - CHECK(header.row == geometry.module_pixel_0[i].row_index); + CHECK(header.column == geometry.get_module_geometries(i).col_index); + CHECK(header.row == geometry.get_module_geometries(i).row_index); - CHECK(geometry.module_pixel_0[i].height == - test_parameters.geometry.module_pixel_0[i].height); - CHECK(geometry.module_pixel_0[i].width == - test_parameters.geometry.module_pixel_0[i].width); + CHECK(geometry.get_module_geometries(i).height == + test_parameters.module_geometries[i].height); + CHECK(geometry.get_module_geometries(i).width == + test_parameters.module_geometries[i].width); - CHECK(geometry.module_pixel_0[i].origin_x == - test_parameters.geometry.module_pixel_0[i].origin_x); - CHECK(geometry.module_pixel_0[i].origin_y == - test_parameters.geometry.module_pixel_0[i].origin_y); + CHECK(geometry.get_module_geometries(i).origin_x == + test_parameters.module_geometries[i].origin_x); + CHECK(geometry.get_module_geometries(i).origin_y == + test_parameters.module_geometries[i].origin_y); } } @@ -261,9 +260,9 @@ TEST_CASE_PRIVATE(aare, open_multi_module_file_with_roi, REQUIRE(f.master().roi().value().width() == 256); REQUIRE(f.master().roi().value().height() == 256); - CHECK(f.n_modules() == 2); + CHECK(f.m_geometry.n_modules() == 2); - CHECK(f.n_modules_in_roi() == 1); + CHECK(f.m_geometry.n_modules_in_roi() == 1); } } // close namespace aare diff --git a/src/geo_helpers.cpp b/src/geo_helpers.cpp deleted file mode 100644 index 96a9056..0000000 --- a/src/geo_helpers.cpp +++ /dev/null @@ -1,72 +0,0 @@ - -#include "aare/geo_helpers.hpp" -#include "fmt/core.h" - -namespace aare { - -DetectorGeometry update_geometry_with_roi(DetectorGeometry geo, aare::ROI roi) { -#ifdef AARE_VERBOSE - fmt::println("update_geometry_with_roi() called with ROI: {} {} {} {}", - roi.xmin, roi.xmax, roi.ymin, roi.ymax); - fmt::println("Geometry: {} {} {} {} {} {}", geo.modules_x, geo.modules_y, - geo.pixels_x, geo.pixels_y, geo.module_gap_row, - geo.module_gap_col); -#endif - int pos_y = 0; - int pos_y_increment = 0; - for (int row = 0; row < geo.modules_y; row++) { - int pos_x = 0; - for (int col = 0; col < geo.modules_x; col++) { - auto &m = geo.module_pixel_0[row * geo.modules_x + col]; - auto original_height = m.height; - auto original_width = m.width; - - // module is to the left of the roi - if (m.origin_x + m.width < roi.xmin) { - m.width = 0; - - // roi is in module - } else { - // here we only arrive when the roi is in or to the left of - // the module - if (roi.xmin > m.origin_x) { - m.width -= roi.xmin - m.origin_x; - } - if (roi.xmax < m.origin_x + original_width) { - m.width -= m.origin_x + original_width - roi.xmax; - } - m.origin_x = pos_x; - pos_x += m.width; - } - - if (m.origin_y + m.height < roi.ymin) { - m.height = 0; - } else { - if ((roi.ymin > m.origin_y) && - (roi.ymin < m.origin_y + m.height)) { - m.height -= roi.ymin - m.origin_y; - } - if (roi.ymax < m.origin_y + original_height) { - m.height -= m.origin_y + original_height - roi.ymax; - } - m.origin_y = pos_y; - pos_y_increment = m.height; - } -#ifdef AARE_VERBOSE - fmt::println("Module {} {} {} {}", m.origin_x, m.origin_y, m.width, - m.height); -#endif - } - // increment pos_y - pos_y += pos_y_increment; - } - - // m_rows = roi.height(); - // m_cols = roi.width(); - geo.pixels_x = roi.width(); - geo.pixels_y = roi.height(); - - return geo; -} - -} // namespace aare \ No newline at end of file diff --git a/src/geo_helpers.test.cpp b/src/geo_helpers.test.cpp deleted file mode 100644 index 48ae9cf..0000000 --- a/src/geo_helpers.test.cpp +++ /dev/null @@ -1,228 +0,0 @@ -#include "aare/File.hpp" -#include "aare/RawFile.hpp" -#include "aare/RawMasterFile.hpp" //needed for ROI - -#include -#include - -#include "aare/geo_helpers.hpp" -#include "test_config.hpp" - -TEST_CASE("Simple ROIs on one module") { - // DetectorGeometry update_geometry_with_roi(DetectorGeometry geo, aare::ROI - // roi) - aare::DetectorGeometry geo; - - aare::ModuleGeometry mod; - mod.origin_x = 0; - mod.origin_y = 0; - mod.width = 1024; - mod.height = 512; - - geo.pixels_x = 1024; - geo.pixels_y = 512; - geo.modules_x = 1; - geo.modules_y = 1; - geo.module_pixel_0.push_back(mod); - - SECTION("ROI is the whole module") { - aare::ROI roi; - roi.xmin = 0; - roi.xmax = 1024; - roi.ymin = 0; - roi.ymax = 512; - auto updated_geo = aare::update_geometry_with_roi(geo, roi); - - REQUIRE(updated_geo.pixels_x == 1024); - REQUIRE(updated_geo.pixels_y == 512); - REQUIRE(updated_geo.modules_x == 1); - REQUIRE(updated_geo.modules_y == 1); - REQUIRE(updated_geo.module_pixel_0[0].height == 512); - REQUIRE(updated_geo.module_pixel_0[0].width == 1024); - } - SECTION("ROI is the top left corner of the module") { - aare::ROI roi; - roi.xmin = 100; - roi.xmax = 200; - roi.ymin = 150; - roi.ymax = 200; - auto updated_geo = aare::update_geometry_with_roi(geo, roi); - - REQUIRE(updated_geo.pixels_x == 100); - REQUIRE(updated_geo.pixels_y == 50); - REQUIRE(updated_geo.modules_x == 1); - REQUIRE(updated_geo.modules_y == 1); - REQUIRE(updated_geo.module_pixel_0[0].height == 50); - REQUIRE(updated_geo.module_pixel_0[0].width == 100); - } - - SECTION("ROI is a small square") { - aare::ROI roi; - roi.xmin = 1000; - roi.xmax = 1010; - roi.ymin = 500; - roi.ymax = 510; - auto updated_geo = aare::update_geometry_with_roi(geo, roi); - - REQUIRE(updated_geo.pixels_x == 10); - REQUIRE(updated_geo.pixels_y == 10); - REQUIRE(updated_geo.modules_x == 1); - REQUIRE(updated_geo.modules_y == 1); - REQUIRE(updated_geo.module_pixel_0[0].height == 10); - REQUIRE(updated_geo.module_pixel_0[0].width == 10); - } - SECTION("ROI is a few columns") { - aare::ROI roi; - roi.xmin = 750; - roi.xmax = 800; - roi.ymin = 0; - roi.ymax = 512; - auto updated_geo = aare::update_geometry_with_roi(geo, roi); - - REQUIRE(updated_geo.pixels_x == 50); - REQUIRE(updated_geo.pixels_y == 512); - REQUIRE(updated_geo.modules_x == 1); - REQUIRE(updated_geo.modules_y == 1); - REQUIRE(updated_geo.module_pixel_0[0].height == 512); - REQUIRE(updated_geo.module_pixel_0[0].width == 50); - } -} - -TEST_CASE("Two modules side by side") { - // DetectorGeometry update_geometry_with_roi(DetectorGeometry geo, aare::ROI - // roi) - aare::DetectorGeometry geo; - - aare::ModuleGeometry mod; - mod.origin_x = 0; - mod.origin_y = 0; - mod.width = 1024; - mod.height = 512; - - geo.pixels_x = 2048; - geo.pixels_y = 512; - geo.modules_x = 2; - geo.modules_y = 1; - - geo.module_pixel_0.push_back(mod); - mod.origin_x = 1024; - geo.module_pixel_0.push_back(mod); - - SECTION("ROI is the whole image") { - aare::ROI roi; - roi.xmin = 0; - roi.xmax = 2048; - roi.ymin = 0; - roi.ymax = 512; - auto updated_geo = aare::update_geometry_with_roi(geo, roi); - - REQUIRE(updated_geo.pixels_x == 2048); - REQUIRE(updated_geo.pixels_y == 512); - REQUIRE(updated_geo.modules_x == 2); - REQUIRE(updated_geo.modules_y == 1); - } - SECTION("rectangle on both modules") { - aare::ROI roi; - roi.xmin = 800; - roi.xmax = 1300; - roi.ymin = 200; - roi.ymax = 499; - auto updated_geo = aare::update_geometry_with_roi(geo, roi); - - REQUIRE(updated_geo.pixels_x == 500); - REQUIRE(updated_geo.pixels_y == 299); - REQUIRE(updated_geo.modules_x == 2); - REQUIRE(updated_geo.modules_y == 1); - REQUIRE(updated_geo.module_pixel_0[0].height == 299); - REQUIRE(updated_geo.module_pixel_0[0].width == 224); - REQUIRE(updated_geo.module_pixel_0[1].height == 299); - REQUIRE(updated_geo.module_pixel_0[1].width == 276); - } -} - -TEST_CASE("Three modules side by side") { - // DetectorGeometry update_geometry_with_roi(DetectorGeometry geo, aare::ROI - // roi) - aare::DetectorGeometry geo; - aare::ROI roi; - roi.xmin = 700; - roi.xmax = 2500; - roi.ymin = 0; - roi.ymax = 123; - - aare::ModuleGeometry mod; - mod.origin_x = 0; - mod.origin_y = 0; - mod.width = 1024; - mod.height = 512; - - geo.pixels_x = 3072; - geo.pixels_y = 512; - geo.modules_x = 3; - geo.modules_y = 1; - - geo.module_pixel_0.push_back(mod); - mod.origin_x = 1024; - geo.module_pixel_0.push_back(mod); - mod.origin_x = 2048; - geo.module_pixel_0.push_back(mod); - - auto updated_geo = aare::update_geometry_with_roi(geo, roi); - - REQUIRE(updated_geo.pixels_x == 1800); - REQUIRE(updated_geo.pixels_y == 123); - REQUIRE(updated_geo.modules_x == 3); - REQUIRE(updated_geo.modules_y == 1); - REQUIRE(updated_geo.module_pixel_0[0].height == 123); - REQUIRE(updated_geo.module_pixel_0[0].width == 324); - REQUIRE(updated_geo.module_pixel_0[1].height == 123); - REQUIRE(updated_geo.module_pixel_0[1].width == 1024); - REQUIRE(updated_geo.module_pixel_0[2].height == 123); - REQUIRE(updated_geo.module_pixel_0[2].width == 452); -} - -TEST_CASE("Four modules as a square") { - // DetectorGeometry update_geometry_with_roi(DetectorGeometry geo, aare::ROI - // roi) - aare::DetectorGeometry geo; - aare::ROI roi; - roi.xmin = 500; - roi.xmax = 2000; - roi.ymin = 500; - roi.ymax = 600; - - aare::ModuleGeometry mod; - mod.origin_x = 0; - mod.origin_y = 0; - mod.width = 1024; - mod.height = 512; - - geo.pixels_x = 2048; - geo.pixels_y = 1024; - geo.modules_x = 2; - geo.modules_y = 2; - - geo.module_pixel_0.push_back(mod); - mod.origin_x = 1024; - geo.module_pixel_0.push_back(mod); - mod.origin_x = 0; - mod.origin_y = 512; - geo.module_pixel_0.push_back(mod); - mod.origin_x = 1024; - geo.module_pixel_0.push_back(mod); - - auto updated_geo = aare::update_geometry_with_roi(geo, roi); - - REQUIRE(updated_geo.pixels_x == 1500); - REQUIRE(updated_geo.pixels_y == 100); - REQUIRE(updated_geo.modules_x == 2); - REQUIRE(updated_geo.modules_y == 2); - REQUIRE(updated_geo.module_pixel_0[0].height == 12); - REQUIRE(updated_geo.module_pixel_0[0].width == 524); - REQUIRE(updated_geo.module_pixel_0[1].height == 12); - REQUIRE(updated_geo.module_pixel_0[1].width == 976); - REQUIRE(updated_geo.module_pixel_0[2].height == 88); - REQUIRE(updated_geo.module_pixel_0[2].width == 524); - REQUIRE(updated_geo.module_pixel_0[3].height == 88); - REQUIRE(updated_geo.module_pixel_0[3].width == 976); -} \ No newline at end of file From 0f56846e3d12684fcbfcdcf0714232465ec64523 Mon Sep 17 00:00:00 2001 From: Alice Date: Fri, 13 Jun 2025 16:33:25 +0200 Subject: [PATCH 09/20] deleted some commented lines --- include/aare/RawMasterFile.hpp | 2 -- src/RawFile.cpp | 5 ----- 2 files changed, 7 deletions(-) diff --git a/include/aare/RawMasterFile.hpp b/include/aare/RawMasterFile.hpp index ba052dc..7a2af6f 100644 --- a/include/aare/RawMasterFile.hpp +++ b/include/aare/RawMasterFile.hpp @@ -62,8 +62,6 @@ class ScanParameters { void increment_stop(); }; -// class RawFile; // forward declaration - /** * @brief Class for parsing a master file either in our .json format or the old * .raw format diff --git a/src/RawFile.cpp b/src/RawFile.cpp index 2c46a90..cdcc0e4 100644 --- a/src/RawFile.cpp +++ b/src/RawFile.cpp @@ -66,11 +66,6 @@ void RawFile::read_into(std::byte *image_buf, size_t n_frames, } } -// size_t RawFile::n_modules() const { return m_master.n_modules(); } - -// size_t RawFile::n_modules_in_roi() const { return -// m_modules_in_roi.size(); } - size_t RawFile::bytes_per_frame() { return m_geometry.pixels_x() * m_geometry.pixels_y() * m_master.bitdepth() / bits_per_byte; From 05828baa54de5fc5e4eae227cf3f7e68739a2b58 Mon Sep 17 00:00:00 2001 From: Alice Date: Fri, 13 Jun 2025 16:38:46 +0200 Subject: [PATCH 10/20] removed n_modules from python bindings --- python/src/raw_file.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/python/src/raw_file.hpp b/python/src/raw_file.hpp index 689b84e..7cfb9c9 100644 --- a/python/src/raw_file.hpp +++ b/python/src/raw_file.hpp @@ -101,7 +101,6 @@ void define_raw_file_io_bindings(py::module &m) { .def_property_readonly("cols", &RawFile::cols) .def_property_readonly("bitdepth", &RawFile::bitdepth) .def_property_readonly("geometry", &RawFile::geometry) - .def_property_readonly("n_modules", &RawFile::n_modules) .def_property_readonly("detector_type", &RawFile::detector_type) .def_property_readonly("master", &RawFile::master); } \ No newline at end of file From b13f864b2b897e27473c16c09948908da72537d3 Mon Sep 17 00:00:00 2001 From: Alice Date: Fri, 13 Jun 2025 17:01:13 +0200 Subject: [PATCH 11/20] need n_modules --- include/aare/RawFile.hpp | 1 + python/src/raw_file.hpp | 3 ++- src/RawFile.cpp | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/include/aare/RawFile.hpp b/include/aare/RawFile.hpp index 67390fa..82106bc 100644 --- a/include/aare/RawFile.hpp +++ b/include/aare/RawFile.hpp @@ -71,6 +71,7 @@ class RawFile : public FileInterface { size_t cols() const override; size_t bitdepth() const override; xy geometry(); + size_t n_modules() const; RawMasterFile master() const; diff --git a/python/src/raw_file.hpp b/python/src/raw_file.hpp index 7cfb9c9..8b8a5a6 100644 --- a/python/src/raw_file.hpp +++ b/python/src/raw_file.hpp @@ -102,5 +102,6 @@ void define_raw_file_io_bindings(py::module &m) { .def_property_readonly("bitdepth", &RawFile::bitdepth) .def_property_readonly("geometry", &RawFile::geometry) .def_property_readonly("detector_type", &RawFile::detector_type) - .def_property_readonly("master", &RawFile::master); + .def_property_readonly("master", &RawFile::master) + .def_property_readonly("n_modules", &RawFile::n_modules); } \ No newline at end of file diff --git a/src/RawFile.cpp b/src/RawFile.cpp index cdcc0e4..dc6c907 100644 --- a/src/RawFile.cpp +++ b/src/RawFile.cpp @@ -97,6 +97,8 @@ size_t RawFile::cols() const { return m_geometry.pixels_x(); } size_t RawFile::bitdepth() const { return m_master.bitdepth(); } xy RawFile::geometry() { return m_master.geometry(); } +size_t RawFile::n_modules() const { return m_geometry.n_modules(); }; + void RawFile::open_subfiles() { if (m_mode == "r") for (size_t i : m_geometry.get_modules_in_roi()) { From 35114cde9d815e0f0be7eeb270da5ddb78c928e1 Mon Sep 17 00:00:00 2001 From: Alice Date: Fri, 13 Jun 2025 18:12:47 +0200 Subject: [PATCH 12/20] fatal error did not open any subfiles --- src/ClusterFinderMT.test.cpp | 6 ++++-- src/RawFile.cpp | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/ClusterFinderMT.test.cpp b/src/ClusterFinderMT.test.cpp index 9289592..f4abd58 100644 --- a/src/ClusterFinderMT.test.cpp +++ b/src/ClusterFinderMT.test.cpp @@ -58,8 +58,10 @@ class ClusterFinderMTWrapper }; TEST_CASE("multithreaded cluster finder", "[.files][.ClusterFinder]") { - auto fpath = "/mnt/sls_det_storage/matterhorn_data/aare_test_data/" - "Moench03new/cu_half_speed_master_4.json"; + auto fpath = + test_data_path() / "clust/Moench03new/cu_half_speed_master_4.json"; + + REQUIRE(std::filesystem::exists(fpath)); File file(fpath); diff --git a/src/RawFile.cpp b/src/RawFile.cpp index dc6c907..788d012 100644 --- a/src/RawFile.cpp +++ b/src/RawFile.cpp @@ -21,8 +21,8 @@ RawFile::RawFile(const std::filesystem::path &fname, const std::string &mode) if (mode == "r") { if (m_master.roi()) { m_geometry.update_geometry_with_roi(m_master.roi().value()); - open_subfiles(); } + open_subfiles(); } else { throw std::runtime_error(LOCATION + " Unsupported mode. Can only read RawFiles."); From 9f8eee5d08c8a23ec416193eef7a212ad3812972 Mon Sep 17 00:00:00 2001 From: Alice Date: Mon, 16 Jun 2025 11:07:00 +0200 Subject: [PATCH 13/20] fixed python bindings - only read headers of modules that are in the roi --- include/aare/RawFile.hpp | 1 + python/src/raw_file.hpp | 5 +++-- python/tests/test_RawFile.py | 14 ++++++++++++++ src/RawFile.cpp | 5 ++++- src/RawFile.test.cpp | 17 +++++++++++++---- 5 files changed, 35 insertions(+), 7 deletions(-) create mode 100644 python/tests/test_RawFile.py diff --git a/include/aare/RawFile.hpp b/include/aare/RawFile.hpp index 82106bc..0209733 100644 --- a/include/aare/RawFile.hpp +++ b/include/aare/RawFile.hpp @@ -72,6 +72,7 @@ class RawFile : public FileInterface { size_t bitdepth() const override; xy geometry(); size_t n_modules() const; + size_t n_modules_in_roi() const; RawMasterFile master() const; diff --git a/python/src/raw_file.hpp b/python/src/raw_file.hpp index 8b8a5a6..83f1110 100644 --- a/python/src/raw_file.hpp +++ b/python/src/raw_file.hpp @@ -65,7 +65,7 @@ void define_raw_file_io_bindings(py::module &m) { header = py::array_t(n_frames); } else { header = py::array_t( - {self.n_modules(), n_frames}); + {self.n_modules_in_roi(), n_frames}); } // py::array_t header({self.n_mod(), n_frames}); @@ -103,5 +103,6 @@ void define_raw_file_io_bindings(py::module &m) { .def_property_readonly("geometry", &RawFile::geometry) .def_property_readonly("detector_type", &RawFile::detector_type) .def_property_readonly("master", &RawFile::master) - .def_property_readonly("n_modules", &RawFile::n_modules); + .def_property_readonly("n_modules", &RawFile::n_modules) + .def_property_readonly("n_modules_in_roi", &RawFile::n_modules_in_roi); } \ No newline at end of file diff --git a/python/tests/test_RawFile.py b/python/tests/test_RawFile.py new file mode 100644 index 0000000..769b4e6 --- /dev/null +++ b/python/tests/test_RawFile.py @@ -0,0 +1,14 @@ +import pytest +from aare import RawFile + +@pytest.mark.files +def test_read_rawfile_with_roi(test_data_path): + + # Starting with f1 there is now 7 frames left in the series of files + print(test_data_path) + with RawFile(test_data_path / "raw/SingleChipROI/Data_master_0.json") as f: + headers, frames = f.read() + + assert headers.size == 10100 + assert frames.shape == (10100, 256, 256) + \ No newline at end of file diff --git a/src/RawFile.cpp b/src/RawFile.cpp index 788d012..f03ec89 100644 --- a/src/RawFile.cpp +++ b/src/RawFile.cpp @@ -62,7 +62,7 @@ void RawFile::read_into(std::byte *image_buf, size_t n_frames, this->get_frame_into(m_current_frame++, image_buf, header); image_buf += bytes_per_frame(); if (header) - header += m_geometry.n_modules(); + header += m_geometry.n_modules_in_roi(); } } @@ -98,6 +98,9 @@ size_t RawFile::bitdepth() const { return m_master.bitdepth(); } xy RawFile::geometry() { return m_master.geometry(); } size_t RawFile::n_modules() const { return m_geometry.n_modules(); }; +size_t RawFile::n_modules_in_roi() const { + return m_geometry.n_modules_in_roi(); +}; void RawFile::open_subfiles() { if (m_mode == "r") diff --git a/src/RawFile.test.cpp b/src/RawFile.test.cpp index 978d011..7eb3707 100644 --- a/src/RawFile.test.cpp +++ b/src/RawFile.test.cpp @@ -257,12 +257,21 @@ TEST_CASE_PRIVATE(aare, open_multi_module_file_with_roi, RawFile f(fpath, "r"); - REQUIRE(f.master().roi().value().width() == 256); - REQUIRE(f.master().roi().value().height() == 256); + SECTION("read 2 frames") { + REQUIRE(f.master().roi().value().width() == 256); + REQUIRE(f.master().roi().value().height() == 256); - CHECK(f.m_geometry.n_modules() == 2); + CHECK(f.m_geometry.n_modules() == 2); - CHECK(f.m_geometry.n_modules_in_roi() == 1); + CHECK(f.m_geometry.n_modules_in_roi() == 1); + + auto frames = f.read_n(2); + + CHECK(frames.size() == 2); + + CHECK(frames[0].rows() == 256); + CHECK(frames[1].cols() == 256); + } } } // close namespace aare From c6990dabad47ce4f7f86d4f0941edd6a21023083 Mon Sep 17 00:00:00 2001 From: Alice Date: Mon, 23 Jun 2025 11:44:06 +0200 Subject: [PATCH 14/20] deleted unused variables --- include/aare/RawFile.hpp | 1 - include/aare/RawMasterFile.hpp | 1 - tests/test_macros.hpp | 4 ++-- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/include/aare/RawFile.hpp b/include/aare/RawFile.hpp index 0209733..aa75d5c 100644 --- a/include/aare/RawFile.hpp +++ b/include/aare/RawFile.hpp @@ -35,7 +35,6 @@ class RawFile : public FileInterface { RawMasterFile m_master; size_t m_current_frame{}; - size_t m_current_subfile{}; DetectorGeometry m_geometry; diff --git a/include/aare/RawMasterFile.hpp b/include/aare/RawMasterFile.hpp index 7a2af6f..57189f7 100644 --- a/include/aare/RawMasterFile.hpp +++ b/include/aare/RawMasterFile.hpp @@ -78,7 +78,6 @@ class RawMasterFile { size_t m_pixels_y{}; size_t m_pixels_x{}; size_t m_bitdepth{}; - size_t m_num_udp_interfaces_per_module = 1; uint8_t m_quad = 0; xy m_geometry{}; diff --git a/tests/test_macros.hpp b/tests/test_macros.hpp index 4e89923..0d9f59d 100644 --- a/tests/test_macros.hpp +++ b/tests/test_macros.hpp @@ -8,7 +8,7 @@ test_tags_str) \ namespace namespace_name { \ void test_name##_impl(); \ - namespace { \ + \ struct test_name##_Invoker : Catch::ITestInvoker { \ void invoke() const override { test_name##_impl(); } \ }; \ @@ -16,5 +16,5 @@ autoReg_##test_name(Catch::Detail::make_unique(), \ Catch::SourceLineInfo(__FILE__, __LINE__), "", \ Catch::NameAndTags{test_name_str, test_tags_str}); \ - } \ + \ void test_name##_impl() From 318e640639ad1be45a3db8d7758eebd838aaea03 Mon Sep 17 00:00:00 2001 From: Alice Date: Mon, 23 Jun 2025 12:16:41 +0200 Subject: [PATCH 15/20] only test over the public interface --- include/aare/RawFile.hpp | 11 +---------- src/RawFile.cpp | 5 ++++- src/RawFile.test.cpp | 22 ++++++++++------------ 3 files changed, 15 insertions(+), 23 deletions(-) diff --git a/include/aare/RawFile.hpp b/include/aare/RawFile.hpp index aa75d5c..b63095a 100644 --- a/include/aare/RawFile.hpp +++ b/include/aare/RawFile.hpp @@ -14,11 +14,6 @@ namespace aare { -#ifdef AARE_TESTS -TEST_CASE_PRIVATE_FWD(check_find_geometry) // forward declaration -TEST_CASE_PRIVATE_FWD(open_multi_module_file_with_roi) -#endif - /** * @brief Class to read .raw files. The class will parse the master file * to find the correct geometry for the frames. @@ -27,10 +22,6 @@ TEST_CASE_PRIVATE_FWD(open_multi_module_file_with_roi) */ class RawFile : public FileInterface { -#ifdef AARE_TESTS - FRIEND_TEST(check_find_geometry) - FRIEND_TEST(open_multi_module_file_with_roi) -#endif std::vector> m_subfiles; RawMasterFile m_master; @@ -69,9 +60,9 @@ class RawFile : public FileInterface { size_t rows() const override; size_t cols() const override; size_t bitdepth() const override; - xy geometry(); size_t n_modules() const; size_t n_modules_in_roi() const; + xy geometry() const; RawMasterFile master() const; diff --git a/src/RawFile.cpp b/src/RawFile.cpp index f03ec89..fab8150 100644 --- a/src/RawFile.cpp +++ b/src/RawFile.cpp @@ -95,7 +95,10 @@ size_t RawFile::total_frames() const { return m_master.frames_in_file(); } size_t RawFile::rows() const { return m_geometry.pixels_y(); } size_t RawFile::cols() const { return m_geometry.pixels_x(); } size_t RawFile::bitdepth() const { return m_master.bitdepth(); } -xy RawFile::geometry() { return m_master.geometry(); } +xy RawFile::geometry() const { + return xy{static_cast(m_geometry.modules_y()), + static_cast(m_geometry.modules_x())}; +} size_t RawFile::n_modules() const { return m_geometry.n_modules(); }; size_t RawFile::n_modules_in_roi() const { diff --git a/src/RawFile.test.cpp b/src/RawFile.test.cpp index 7eb3707..a34bde0 100644 --- a/src/RawFile.test.cpp +++ b/src/RawFile.test.cpp @@ -174,8 +174,7 @@ struct TestParameters { std::vector module_geometries{}; }; -TEST_CASE_PRIVATE(aare, check_find_geometry, "check find_geometry", - "[.integration][.files][.rawfile]") { +TEST_CASE("check find_geometry", "[.integration][.files][.rawfile]") { auto test_parameters = GENERATE( TestParameters{"raw/jungfrau_2modules_version6.1.2/run_master_0.raw", 2, @@ -211,9 +210,11 @@ TEST_CASE_PRIVATE(aare, check_find_geometry, "check find_geometry", REQUIRE(std::filesystem::exists(fpath)); - RawFile f(fpath, "r"); + RawMasterFile master_file(fpath); - auto geometry = f.m_geometry; + auto geometry = DetectorGeometry( + master_file.geometry(), master_file.pixels_x(), master_file.pixels_y(), + master_file.udp_interfaces_per_module(), master_file.quad()); CHECK(geometry.modules_x() == test_parameters.modules_x); CHECK(geometry.modules_y() == test_parameters.modules_y); @@ -224,6 +225,7 @@ TEST_CASE_PRIVATE(aare, check_find_geometry, "check find_geometry", test_parameters.num_ports); // compare to data stored in header + RawFile f(fpath, "r"); for (size_t i = 0; i < test_parameters.num_ports; ++i) { auto subfile1_path = f.master().data_fname(i, 0); @@ -246,11 +248,8 @@ TEST_CASE_PRIVATE(aare, check_find_geometry, "check find_geometry", } } -} // close the namespace - -TEST_CASE_PRIVATE(aare, open_multi_module_file_with_roi, - "Open multi module file with ROI", - "[.integration][.files][.rawfile]") { +TEST_CASE("Open multi module file with ROI", + "[.integration][.files][.rawfile]") { auto fpath = test_data_path() / "raw/SingleChipROI/Data_master_0.json"; REQUIRE(std::filesystem::exists(fpath)); @@ -261,9 +260,9 @@ TEST_CASE_PRIVATE(aare, open_multi_module_file_with_roi, REQUIRE(f.master().roi().value().width() == 256); REQUIRE(f.master().roi().value().height() == 256); - CHECK(f.m_geometry.n_modules() == 2); + CHECK(f.n_modules() == 2); - CHECK(f.m_geometry.n_modules_in_roi() == 1); + CHECK(f.n_modules_in_roi() == 1); auto frames = f.read_n(2); @@ -273,7 +272,6 @@ TEST_CASE_PRIVATE(aare, open_multi_module_file_with_roi, CHECK(frames[1].cols() == 256); } } -} // close namespace aare TEST_CASE("Read file with unordered frames", "[.integration]") { // TODO! Better explanation and error message From 664055de9234625d39e616c3864bf2d63d6a3b75 Mon Sep 17 00:00:00 2001 From: Alice Date: Mon, 23 Jun 2025 17:26:59 +0200 Subject: [PATCH 16/20] fixed quad structure --- src/DetectorGeometry.cpp | 7 +++---- src/RawFile.test.cpp | 5 ++--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/DetectorGeometry.cpp b/src/DetectorGeometry.cpp index c7f0125..9658aad 100644 --- a/src/DetectorGeometry.cpp +++ b/src/DetectorGeometry.cpp @@ -23,13 +23,12 @@ DetectorGeometry::DetectorGeometry(const xy &geometry, for (size_t port_col = 0; port_col < udp_interfaces_per_module.col; ++port_col) { ModuleGeometry g; - g.origin_x = (col + port_col) * module_pixels_x; - g.origin_y = (row + port_row) * - module_pixels_y; // TODO: quad doesnt seem - // to have an effect g.row_index = quad ? (row + port_row + 1) % 2 : (row + port_row); g.col_index = col + port_col; + g.origin_x = g.col_index * module_pixels_x; + g.origin_y = g.row_index * module_pixels_y; + g.width = module_pixels_x; g.height = module_pixels_y; module_geometries.push_back(g); diff --git a/src/RawFile.test.cpp b/src/RawFile.test.cpp index a34bde0..4fff715 100644 --- a/src/RawFile.test.cpp +++ b/src/RawFile.test.cpp @@ -202,9 +202,8 @@ TEST_CASE("check find_geometry", "[.integration][.files][.rawfile]") { "raw/eiger_quad_data/" "W13_vthreshscan_m21C_300V_800eV_vrpre3400_master_0.json", 2, 1, 2, 512, 512, - std::vector{ - ModuleGeometry{0, 0, 256, 512, 0, 0}, - ModuleGeometry{0, 256, 256, 512, 1, 0}}}); + std::vector{ModuleGeometry{0, 256, 256, 512, 1, 0}, + ModuleGeometry{0, 0, 256, 512, 0, 0}}}); auto fpath = test_data_path() / test_parameters.master_filename; From c92be4bca2da579f67e2e45df184cb930f20df31 Mon Sep 17 00:00:00 2001 From: Alice Date: Tue, 24 Jun 2025 11:29:25 +0200 Subject: [PATCH 17/20] added eiger quad test --- python/tests/test_RawFile.py | 64 ++++++++++++++++++++++++++++++++-- src/NumpyFile.cpp | 4 +-- src/RawFile.test.cpp | 66 +++++++++++++++++++++++++++++------- 3 files changed, 117 insertions(+), 17 deletions(-) diff --git a/python/tests/test_RawFile.py b/python/tests/test_RawFile.py index 769b4e6..69e4d46 100644 --- a/python/tests/test_RawFile.py +++ b/python/tests/test_RawFile.py @@ -1,14 +1,74 @@ import pytest from aare import RawFile +import numpy as np @pytest.mark.files def test_read_rawfile_with_roi(test_data_path): # Starting with f1 there is now 7 frames left in the series of files - print(test_data_path) with RawFile(test_data_path / "raw/SingleChipROI/Data_master_0.json") as f: headers, frames = f.read() assert headers.size == 10100 assert frames.shape == (10100, 256, 256) - \ No newline at end of file + +@pytest.mark.files +def test_read_rawfile_quad_eiger_and_compare_to_numpy(test_data_path): + + d0 = test_data_path/'raw/eiger_quad_data/W13_vrpreampscan_m21C_300V_800eV_vthre2000_d0_f0_0.raw' + d1 = test_data_path/'raw/eiger_quad_data/W13_vrpreampscan_m21C_300V_800eV_vthre2000_d1_f0_0.raw' + + image = np.zeros((512,512), dtype=np.uint32) + + with open(d0) as f: + raw = np.fromfile(f, dtype=np.uint32, count = 256*512, offset = 20*256*512*4 + 112*21).reshape(256,512) + + image[256:,:] = raw + + with open(d1) as f: + raw = np.fromfile(f, dtype=np.uint32, count = 256*512, offset = 20*256*512*4 + 112*21).reshape(256,512) + + image[0:256,:] = raw[::-1,:] + + with RawFile(test_data_path/'raw/eiger_quad_data/W13_vrpreampscan_m21C_300V_800eV_vthre2000_master_0.json') as f: + f.seek(20) + header, image1 = f.read_frame() + + assert (image == image1).all() + + +@pytest.mark.files +def test_read_rawfile_eiger_and_compare_to_numpy(test_data_path): + d0 = test_data_path/'raw/eiger/Lab6_20500eV_2deg_20240629_d0_f0_7.raw' + d1 = test_data_path/'raw/eiger/Lab6_20500eV_2deg_20240629_d1_f0_7.raw' + d2 = test_data_path/'raw/eiger/Lab6_20500eV_2deg_20240629_d2_f0_7.raw' + d3 = test_data_path/'raw/eiger/Lab6_20500eV_2deg_20240629_d3_f0_7.raw' + + image = np.zeros((512,1024), dtype=np.uint32) + + #TODO why is there no header offset? + with open(d0) as f: + raw = np.fromfile(f, dtype=np.uint32, count = 256*512, offset=112).reshape(256,512) + + image[0:256,0:512] = raw[::-1] + + with open(d1) as f: + raw = np.fromfile(f, dtype=np.uint32, count = 256*512, offset=112).reshape(256,512) + + image[0:256,512:] = raw[::-1] + + with open(d2) as f: + raw = np.fromfile(f, dtype=np.uint32, count = 256*512, offset=112).reshape(256,512) + + image[256:,0:512] = raw + + with open(d3) as f: + raw = np.fromfile(f, dtype=np.uint32, count = 256*512, offset=112).reshape(256,512) + + image[256:,512:] = raw + + + with RawFile(test_data_path/'raw/eiger/Lab6_20500eV_2deg_20240629_master_7.json') as f: + header, image1 = f.read_frame() + + assert (image == image1).all() diff --git a/src/NumpyFile.cpp b/src/NumpyFile.cpp index 4e0c215..5fd4d15 100644 --- a/src/NumpyFile.cpp +++ b/src/NumpyFile.cpp @@ -56,7 +56,7 @@ void NumpyFile::write_impl(void *data, uint64_t size) { } Frame NumpyFile::get_frame(size_t frame_number) { - Frame frame(m_header.shape[1], m_header.shape[2], m_header.dtype); + Frame frame(m_header.shape[0], m_header.shape[1], m_header.dtype); get_frame_into(frame_number, frame.data()); return frame; } @@ -67,7 +67,7 @@ void NumpyFile::get_frame_into(size_t frame_number, std::byte *image_buf) { if (frame_number > m_header.shape[0]) { throw std::invalid_argument("Frame number out of range"); } - if (fseek(fp, header_size + frame_number * m_bytes_per_frame, + if (fseek(fp, frame_number * m_bytes_per_frame, SEEK_SET)) // NOLINT throw std::runtime_error("Could not seek to frame"); diff --git a/src/RawFile.test.cpp b/src/RawFile.test.cpp index 4fff715..4e09ead 100644 --- a/src/RawFile.test.cpp +++ b/src/RawFile.test.cpp @@ -111,25 +111,65 @@ TEST_CASE("Read frame numbers from a raw file", "[.integration]") { } TEST_CASE("Compare reading from a numpy file with a raw file", "[.files]") { - auto fpath_raw = - test_data_path() / "raw/jungfrau" / "jungfrau_single_master_0.json"; - REQUIRE(std::filesystem::exists(fpath_raw)); - auto fpath_npy = - test_data_path() / "raw/jungfrau" / "jungfrau_single_0.npy"; - REQUIRE(std::filesystem::exists(fpath_npy)); + SECTION("jungfrau data") { + auto fpath_raw = + test_data_path() / "raw/jungfrau" / "jungfrau_single_master_0.json"; + REQUIRE(std::filesystem::exists(fpath_raw)); - File raw(fpath_raw, "r"); - File npy(fpath_npy, "r"); + auto fpath_npy = + test_data_path() / "raw/jungfrau" / "jungfrau_single_0.npy"; + REQUIRE(std::filesystem::exists(fpath_npy)); - CHECK(raw.total_frames() == 10); - CHECK(npy.total_frames() == 10); + File raw(fpath_raw, "r"); + File npy(fpath_npy, "r"); - for (size_t i = 0; i < 10; ++i) { - CHECK(raw.tell() == i); + CHECK(raw.total_frames() == 10); + CHECK(npy.total_frames() == 10); + + for (size_t i = 0; i < 10; ++i) { + CHECK(raw.tell() == i); + auto raw_frame = raw.read_frame(); + auto npy_frame = npy.read_frame(); + CHECK((raw_frame.view() == npy_frame.view())); + } + } + + SECTION("eiger quad data") { + auto fpath_raw = + test_data_path() / "raw/eiger_quad_data" / + "W13_vrpreampscan_m21C_300V_800eV_vthre2000_master_0.json"; + REQUIRE(std::filesystem::exists(fpath_raw)); + + auto fpath_npy = test_data_path() / "raw/eiger_quad_data" / + "W13_vrpreampscan_m21C_300V_800eV_vthre2000.npy"; + REQUIRE(std::filesystem::exists(fpath_npy)); + + File raw(fpath_raw, "r"); + File npy(fpath_npy, "r"); + + raw.seek(20); auto raw_frame = raw.read_frame(); + auto npy_frame = npy.read_frame(); - CHECK((raw_frame.view() == npy_frame.view())); + CHECK((raw_frame.view() == npy_frame.view())); + } + SECTION("eiger data") { + auto fpath_raw = test_data_path() / "raw/eiger" / + "Lab6_20500eV_2deg_20240629_master_7.json"; + REQUIRE(std::filesystem::exists(fpath_raw)); + + auto fpath_npy = + test_data_path() / "raw/eiger" / "Lab6_20500eV_2deg_20240629_7.npy"; + REQUIRE(std::filesystem::exists(fpath_npy)); + + File raw(fpath_raw, "r"); + File npy(fpath_npy, "r"); + + auto raw_frame = raw.read_frame(); + + auto npy_frame = npy.read_frame(); + CHECK((raw_frame.view() == npy_frame.view())); } } From df4dbb8fd035d2984e8c6394f5520995ca7123d7 Mon Sep 17 00:00:00 2001 From: Alice Date: Tue, 24 Jun 2025 13:39:32 +0200 Subject: [PATCH 18/20] fixed numpy test --- src/NumpyFile.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/NumpyFile.cpp b/src/NumpyFile.cpp index 5fd4d15..4e0c215 100644 --- a/src/NumpyFile.cpp +++ b/src/NumpyFile.cpp @@ -56,7 +56,7 @@ void NumpyFile::write_impl(void *data, uint64_t size) { } Frame NumpyFile::get_frame(size_t frame_number) { - Frame frame(m_header.shape[0], m_header.shape[1], m_header.dtype); + Frame frame(m_header.shape[1], m_header.shape[2], m_header.dtype); get_frame_into(frame_number, frame.data()); return frame; } @@ -67,7 +67,7 @@ void NumpyFile::get_frame_into(size_t frame_number, std::byte *image_buf) { if (frame_number > m_header.shape[0]) { throw std::invalid_argument("Frame number out of range"); } - if (fseek(fp, frame_number * m_bytes_per_frame, + if (fseek(fp, header_size + frame_number * m_bytes_per_frame, SEEK_SET)) // NOLINT throw std::runtime_error("Could not seek to frame"); From 8e7c9eadff2bdf07f9f8ba716f5e8531d3887457 Mon Sep 17 00:00:00 2001 From: Alice Date: Tue, 24 Jun 2025 13:49:05 +0200 Subject: [PATCH 19/20] fixed cmake merge --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3e3c820..6215482 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -413,6 +413,7 @@ target_link_libraries( if(AARE_TESTS) target_compile_definitions(aare_core PRIVATE AARE_TESTS) +endif() if(AARE_VERBOSE) target_compile_definitions(aare_core PUBLIC AARE_VERBOSE) target_compile_definitions(aare_core PUBLIC AARE_LOG_LEVEL=aare::logDEBUG5) From ff7312f45deeb1f58e5cec4a9b0ce73ef0066405 Mon Sep 17 00:00:00 2001 From: Alice Date: Tue, 24 Jun 2025 16:24:25 +0200 Subject: [PATCH 20/20] replaced fmt with LOG --- python/tests/test_RawFile.py | 5 ++--- src/RawMasterFile.cpp | 8 +++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/python/tests/test_RawFile.py b/python/tests/test_RawFile.py index 69e4d46..5edfa84 100644 --- a/python/tests/test_RawFile.py +++ b/python/tests/test_RawFile.py @@ -5,9 +5,8 @@ import numpy as np @pytest.mark.files def test_read_rawfile_with_roi(test_data_path): - # Starting with f1 there is now 7 frames left in the series of files - with RawFile(test_data_path / "raw/SingleChipROI/Data_master_0.json") as f: - headers, frames = f.read() + with RawFile(test_data_path / "raw/SingleChipROI/Data_master_0.json") as f: + headers, frames = f.read() assert headers.size == 10100 assert frames.shape == (10100, 256, 256) diff --git a/src/RawMasterFile.cpp b/src/RawMasterFile.cpp index f6ef268..f12b7e0 100644 --- a/src/RawMasterFile.cpp +++ b/src/RawMasterFile.cpp @@ -1,6 +1,8 @@ #include "aare/RawMasterFile.hpp" #include "aare/RawFile.hpp" +#include "aare/logger.hpp" #include + namespace aare { RawFileNameComponents::RawFileNameComponents( @@ -425,9 +427,9 @@ void RawMasterFile::parse_raw(const std::filesystem::path &fpath) { if (m_geometry.col == 0 && m_geometry.row == 0) { retrieve_geometry(); - fmt::print("Warning: No geometry found in master file. Retrieved " - "geometry of {}x{}\n", - m_geometry.row, m_geometry.col); + LOG(TLogLevel::logWARNING) + << "No geometry found in master file. Retrieved geometry of " + << m_geometry.row << " x " << m_geometry.col << "\n "; } // TODO! Read files and find actual frames