detectors/aare
25
0
Fork 0
mirror of https://github.com/slsdetectorgroup/aare.git synced 2025-04-19 13:20:03 +02:00
Code Issues Actions Packages Releases Activity

WIP

Browse Source
This commit is contained in:
froejdh_e 2024-11-11 17:13:48 +01:00
parent b172c7aa0a
commit ecf1b2a90b
18 changed files with 537 additions and 348 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.clang-format Normal file
View File

@ -0,0 +1,7 @@
BasedOnStyle: LLVM
IndentWidth: 4
UseTab: Never
ColumnLimit: 80
AlignConsecutiveAssignments: false
AlignConsecutiveMacros: true

12
CMakeLists.txt
View File

@ -33,6 +33,7 @@ option(AARE_EXAMPLES "Build examples" OFF)
option(AARE_IN_GITHUB_ACTIONS "Running in Github Actions" OFF)
option(AARE_DOCS "Build documentation" OFF)
option(AARE_VERBOSE "Verbose output" OFF)
option(AARE_CUSTOM_ASSERT "Use custom assert" OFF)
# Configure which of the dependencies to use FetchContent for
option(AARE_FETCH_FMT "Use FetchContent to download fmt" ON)
@ -57,6 +58,9 @@ if(AARE_VERBOSE)
add_compile_definitions(AARE_VERBOSE)
endif()
if(AARE_CUSTOM_ASSERT)
add_compile_definitions(AARE_CUSTOM_ASSERT)
endif()
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
@ -242,7 +246,6 @@ set(PUBLICHEADERS
include/aare/Dtype.hpp
include/aare/File.hpp
include/aare/FileInterface.hpp
include/aare/RawMasterFile.hpp
include/aare/Frame.hpp
include/aare/NDArray.hpp
include/aare/NDView.hpp
@ -251,7 +254,8 @@ set(PUBLICHEADERS
include/aare/Pedestal.hpp
include/aare/PixelMap.hpp
include/aare/RawFile.hpp
include/aare/SubFile.hpp
include/aare/RawSubFile.hpp
include/aare/RawMasterFile.hpp
include/aare/VarClusterFinder.hpp
)
@ -263,11 +267,11 @@ set(SourceFiles
${CMAKE_CURRENT_SOURCE_DIR}/src/Dtype.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/Frame.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/File.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/RawMasterFile.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyFile.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/PixelMap.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/RawFile.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/SubFile.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/RawSubFile.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/RawMasterFile.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyHelpers.cpp
)

1
docs/CMakeLists.txt
View File

@ -20,6 +20,7 @@ set(SPHINX_SOURCE_FILES
src/ClusterFinder.rst
src/Pedestal.rst
src/RawFile.rst
src/RawSubFile.rst
src/RawMasterFile.rst
src/VarClusterFinder.rst
src/pyVarClusterFinder.rst

8
docs/src/RawSubFile.rst Normal file
View File

@ -0,0 +1,8 @@
RawSubFile
===============
.. doxygenclass:: aare::RawSubFile
:members:
:undoc-members:
:private-members:

20
docs/src/index.rst
View File

@ -5,6 +5,16 @@ AARE
Hello
.. toctree::
:caption: Python API
:maxdepth: 1
pyFile
pyCtbRawFile
pyRawMasterFile
pyVarClusterFinder
.. toctree::
:caption: C++ API
:maxdepth: 1
@ -17,15 +27,9 @@ AARE
ClusterFinder
Pedestal
RawFile
RawSubFile
RawMasterFile
VarClusterFinder
.. toctree::
:caption: Python API
:maxdepth: 1
pyFile
pyCtbRawFile
pyRawMasterFile
pyVarClusterFinder

15
include/aare/RawFile.hpp
View File

@ -3,7 +3,7 @@
#include "aare/RawMasterFile.hpp"
#include "aare/Frame.hpp"
#include "aare/NDArray.hpp" //for pixel map
#include "aare/SubFile.hpp"
#include "aare/RawSubFile.hpp"
#include <optional>
@ -32,8 +32,9 @@ class RawFile : public FileInterface {
size_t n_subfiles{}; //f0,f1...fn
size_t n_subfile_parts{}; // d0,d1...dn
//TODO! move to vector of SubFile instead of pointers
std::vector<std::vector<SubFile *>> subfiles; //subfiles[f0,f1...fn][d0,d1...dn]
std::vector<std::vector<RawSubFile *>> subfiles; //subfiles[f0,f1...fn][d0,d1...dn]
std::vector<xy> positions;
std::vector<ModuleGeometry> m_module_pixel_0;
ModuleConfig cfg{0, 0};
RawMasterFile m_master;
@ -57,6 +58,10 @@ class RawFile : public FileInterface {
std::vector<Frame> read_n(size_t n_frames) override;
void read_into(std::byte *image_buf) override;
void read_into(std::byte *image_buf, size_t n_frames) override;
//TODO! do we need to adapt the API?
void read_into(std::byte *image_buf, DetectorHeader *header);
size_t frame_number(size_t frame_index) override;
size_t bytes_per_frame() override;
size_t pixels_per_frame() override;
@ -66,7 +71,9 @@ class RawFile : public FileInterface {
size_t rows() const override;
size_t cols() const override;
size_t bitdepth() const override;
size_t bytes_per_pixel() const;
xy geometry();
size_t n_mod() const;
DetectorType detector_type() const override;
@ -84,7 +91,7 @@ class RawFile : public FileInterface {
* @param frame_number frame number to read
* @param image_buf buffer to store the frame
*/
void get_frame_into(size_t frame_index, std::byte *frame_buffer);
void get_frame_into(size_t frame_index, std::byte *frame_buffer, DetectorHeader *header = nullptr);
/**
* @brief get the frame at the given frame index
@ -102,7 +109,7 @@ class RawFile : public FileInterface {
*/
static DetectorHeader read_header(const std::filesystem::path &fname);
void update_geometry_with_roi();
int find_number_of_subfiles();
void open_subfiles();
void find_geometry();

3
include/aare/RawMasterFile.hpp
View File

@ -63,6 +63,9 @@ struct ROI{
size_t xmax{};
size_t ymin{};
size_t ymax{};
size_t height() const { return ymax - ymin; }
size_t width() const { return xmax - xmin; }
};
/**

69
include/aare/RawSubFile.hpp Normal file
View File

@ -0,0 +1,69 @@
#pragma once
#include "aare/Frame.hpp"
#include "aare/defs.hpp"
#include <cstdint>
#include <filesystem>
#include <map>
#include <optional>
namespace aare {
/**
* @brief Class to read a singe subfile written in .raw format. Used from RawFile to read
* the entire detector. Can be used directly to read part of the image.
*/
class RawSubFile {
protected:
std::ifstream m_file;
size_t m_bitdepth;
std::filesystem::path m_fname;
size_t m_rows{};
size_t m_cols{};
size_t m_bytes_per_frame{};
size_t n_frames{};
DetectorType m_detector_type;
std::optional<NDArray<ssize_t, 2>> pixel_map;
public:
/**
* @brief SubFile constructor
* @param fname path to the subfile
* @param detector detector type
* @param rows number of rows in the subfile
* @param cols number of columns in the subfile
* @param bitdepth bitdepth of the subfile
* @throws std::invalid_argument if the detector,type pair is not supported
*/
RawSubFile(const std::filesystem::path &fname, DetectorType detector,
size_t rows, size_t cols, size_t bitdepth);
~RawSubFile() = default;
/**
* @brief Seek to the given frame number
* @note Puts the file pointer at the start of the header, not the start of the data
* @param frame_index frame position in file to seek to
* @throws std::runtime_error if the frame number is out of range
*/
void seek(size_t frame_index);
size_t tell();
void read_into(std::byte *image_buf, DetectorHeader *header = nullptr);
void get_part(std::byte *buffer, size_t frame_index);
void read_header(DetectorHeader *header);
size_t rows() const;
size_t cols() const;
size_t frame_number(size_t frame_index);
size_t bytes_per_frame() const { return m_bytes_per_frame; }
size_t pixels_per_frame() const { return m_rows * m_cols; }
size_t bytes_per_pixel() const { return m_bitdepth / 8; }
};
} // namespace aare

82
include/aare/SubFile.hpp
View File

@ -1,82 +0,0 @@
#pragma once
#include "aare/Frame.hpp"
#include "aare/defs.hpp"
#include <cstdint>
#include <filesystem>
#include <map>
#include <optional>
namespace aare {
/**
* @brief Class to read a subfile from a RawFile
*/
class SubFile {
protected:
FILE *fp = nullptr;
size_t m_bitdepth;
std::filesystem::path m_fname;
size_t m_rows{};
size_t m_cols{};
std::string m_mode;
size_t n_frames{};
int m_sub_file_index_{};
DetectorType m_detector_type;
std::optional<NDArray<ssize_t, 2>> pixel_map;
public:
/**
* @brief SubFile constructor
* @param fname path to the subfile
* @param detector detector type
* @param rows number of rows in the subfile
* @param cols number of columns in the subfile
* @param bitdepth bitdepth of the subfile
* @throws std::invalid_argument if the detector,type pair is not supported
*/
SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth,
const std::string &mode = "r");
/**
* @brief read the subfile into a buffer
* @param buffer pointer to the buffer to read the data into
* @return number of bytes read
*/
size_t read_impl_normal(std::byte *buffer);
/**
* @brief read the subfile into a buffer with the bytes flipped
* @param buffer pointer to the buffer to read the data into
* @return number of bytes read
*/
template <typename DataType> size_t read_impl_flip(std::byte *buffer);
/**
* @brief read the subfile into a buffer with the bytes reordered
* @param buffer pointer to the buffer to read the data into
* @return number of bytes read
*/
template <typename DataType> size_t read_impl_reorder(std::byte *buffer);
/**
* @brief read the subfile into a buffer with the bytes reordered and flipped
* @param buffer pointer to the buffer to read the data into
* @param frame_number frame number to read
* @return number of bytes read
*/
size_t get_part(std::byte *buffer, size_t frame_index);
size_t frame_number(size_t frame_index);
size_t bytes_per_part() const { return (m_bitdepth / 8) * m_rows * m_cols; }
size_t pixels_per_part() const { return m_rows * m_cols; }
~SubFile();
};
} // namespace aare

28
include/aare/defs.hpp
View File

@ -14,6 +14,11 @@
#include <variant>
#include <vector>
/**
* @brief LOCATION macro to get the current location in the code
*/
@ -21,8 +26,23 @@
std::string(__FILE__) + std::string(":") + std::to_string(__LINE__) + \
":" + std::string(__func__) + ":"
#ifdef AARE_CUSTOM_ASSERT
#define AARE_ASSERT(expr)\
if (expr)\
{}\
else\
aare::assert_failed(LOCATION + " Assertion failed: " + #expr + "\n");
#else
#define AARE_ASSERT(cond)\
do { (void)sizeof(cond); } while(0)
#endif
namespace aare {
void assert_failed(const std::string &msg);
class Cluster {
public:
int cluster_sizeX;
@ -164,6 +184,14 @@ template <typename T> struct t_xy {
};
using xy = t_xy<uint32_t>;
struct ModuleGeometry{
int x{};
int y{};
int height{};
int width{};
};
using dynamic_shape = std::vector<int64_t>;
//TODO! Can we uniform enums between the libraries?

5
python/aare/__init__.py
View File

@ -1,7 +1,8 @@
# Make the compiled classes that live in _aare available from aare.
from . import _aare
from ._aare import VarClusterFinder, File, RawMasterFile
from ._aare import Pedestal, ClusterFinder
from ._aare import File, RawFile, RawMasterFile, RawSubFile
from ._aare import Pedestal, ClusterFinder, VarClusterFinder
from ._aare import DetectorType
from .CtbRawFile import CtbRawFile
from .ScanParameters import ScanParameters

98
python/examples/play.py
View File

@ -52,88 +52,36 @@ def get_Mh02_frames(fname):
# [frame, counter, row, col]
# plt.imshow(data[0,0])
# p = Path('/Users/erik/data/aare_test_data/jungfrau/jungfrau_single_master_0.json')
base = Path('/mnt/sls_det_storage/matterhorn_data/aare_test_data/ci/aare_test_data')
# p = Path(base / 'jungfrau/jungfrau_single_master_0.json')
# f = aare.File(p)
# frame = f.read_frame()
# fig, ax = plt.subplots()
# im = ax.imshow(frame, cmap='viridis')
# fpath = Path('/Users/erik/data/Moench03old/test_034_irradiated_noise_g4_hg_exptime_2000us_master_0.json')
# # fpath = Path('/Users/erik/data/Moench05/moench05_multifile_master_0.json')
# # f = aare.CtbRawFile(fpath, transform = transform.moench05)
# # with CtbRawFile(fpath, transform = transform.moench05) as f:
# # for header, image in f:
# # print(f'Frame number: {header["frameNumber"]}')
# # m = aare.RawMasterFile(fpath)
# f = aare.File(fpath)
# for i in range(10):
# frame = f.read_frame()
# cf = aare.ClusterFinder((400,400),(3,3))
# for i in range(100):
# cf.push_pedestal_frame(f.read_frame())
# f.seek(0)
# pd = f.read_n(100).mean(axis=0)
# clusters = cf.find_clusters_without_threshold(f.read_frame())
# # f2 = aare.CtbRawFile(fpath, transform=transform.matterhorn02)
# # header, data = f2.read()
# # plt.plot(data[:,0,20,20])
# from aare import RawMasterFile, File, RawSubFile, DetectorType, RawFile
# base = Path('/mnt/sls_det_storage/matterhorn_data/aare_test_data/Jungfrau10/Jungfrau_DoubleModule_1UDP_ROI/SideBySide/')
# fpath = Path('241019_JF_12keV_Si_FF_GaAs_FF_7p88mmFilter_PedestalStart_ZPos_5.5_master_0.json')
# raw = Path('241019_JF_12keV_Si_FF_GaAs_FF_7p88mmFilter_PedestalStart_ZPos_5.5_d0_f0_0.raw')
# base = Path('/Users/erik/data/matterhorn/raw')
# fpath = Path(base / 'scan_15keV_vrf700_vrsh700_th0_master_0.json')
# f = aare.CtbRawFile(fpath, transform=transform.matterhorn02)
# f.seek(100)
# header1, image1 = f.read_frame()
# m = RawMasterFile(base / fpath)
# # roi = m.roi
# # rows = roi.ymax-roi.ymin+1
# # cols = 1024-roi.xmin
# # sf = RawSubFile(base / raw, DetectorType.Jungfrau, rows, cols, 16)
# fpath = Path(base / 'scan_all15keV_vrf500_vrsh700_th0_master_0.json')
from aare import RawFile
# f = aare.CtbRawFile(fpath, transform=transform.matterhorn02)
# f.seek(100)
# header4, image4 = f.read_frame()
# n_counters = image.shape[1] / 48**2 / 2
# for i in range(100):
# header, image = f.read_frame()
# print(header['frameNumber'])
#Data come in "blocks" of 4 pixels/receiver
# data = get_Mh02_frames(fpath.as_posix())
# rawi = np.zeros(48*48*4+56, dtype = np.uint16)
# for i,v in enumerate(rawi[56:]):
# rawi[i+56] = i
# raw = image.view(np.uint16)
# pixel_map = decode(1, rawi)
# # img = np.take(raw, pixel_map)
# pm = np.zeros((4, 48,48), dtype = np.int64)
# for counter in range(4):
# for row in range(48):
# for col in range(48):
# pm[counter, row, col] = row*48 + col+counter*48*48
# f2 = aare.CtbRawFile(fpath, transform=transform.matterhorn02)
# header, data = f2.read()
# plt.plot(data[:,0,20,20])
from aare import RawMasterFile, File
fpath = Path('/mnt/sls_det_storage/matterhorn_data/aare_test_data/Jungfrau10/Jungfrau_DoubleModule_1UDP_ROI/SideBySide/241019_JF_12keV_Si_FF_GaAs_FF_7p88mmFilter_PedestalStart_ZPos_5.5_master_0.json')
m = RawMasterFile(fpath)
f = File(fpath)
base = Path('/mnt/sls_det_storage/matterhorn_data/aare_test_data/Jungfrau10/Jungfrau_DoubleModule_1UDP_ROI/SideBySide/')
fname = base / Path('241019_JF_12keV_Si_FF_GaAs_FF_7p88mmFilter_PedestalStart_ZPos_5.5_master_0.json')
# fname = Path(base / 'jungfrau/jungfrau_single_master_0.json')
f = RawFile(fname)
h,img = f.read_frame()
print(f'{h["frameNumber"]}')

152
python/src/file.hpp
View File

@ -1,7 +1,9 @@
#include "aare/CtbRawFile.hpp"
#include "aare/RawMasterFile.hpp"
#include "aare/File.hpp"
#include "aare/Frame.hpp"
#include "aare/RawFile.hpp"
#include "aare/RawMasterFile.hpp"
#include "aare/RawSubFile.hpp"
#include "aare/defs.hpp"
// #include "aare/fClusterFileV2.hpp"
@ -17,8 +19,17 @@
namespace py = pybind11;
using namespace ::aare;
void define_file_io_bindings(py::module &m)
{
void define_file_io_bindings(py::module &m) {
py::enum_<DetectorType>(m, "DetectorType")
.value("Jungfrau", DetectorType::Jungfrau)
.value("Eiger", DetectorType::Eiger)
.value("Mythen3", DetectorType::Mythen3)
.value("Moench", DetectorType::Moench)
.value("Moench03", DetectorType::Moench03)
.value("Moench03_old", DetectorType::Moench03_old)
.value("ChipTestBoard", DetectorType::ChipTestBoard)
.value("Unknown", DetectorType::Unknown);
PYBIND11_NUMPY_DTYPE(DetectorHeader, frameNumber, expLength, packetNumber,
bunchId, timestamp, modId, row, column, reserved,
@ -27,8 +38,7 @@ void define_file_io_bindings(py::module &m)
py::class_<CtbRawFile>(m, "CtbRawFile")
.def(py::init<const std::filesystem::path &>())
.def("read_frame",
[](CtbRawFile &self)
{
[](CtbRawFile &self) {
size_t image_size = self.image_size_in_bytes();
py::array image;
std::vector<ssize_t> shape;
@ -50,15 +60,16 @@ void define_file_io_bindings(py::module &m)
.def("seek", &CtbRawFile::seek)
.def("tell", &CtbRawFile::tell)
.def("master", &CtbRawFile::master)
.def_property_readonly("image_size_in_bytes", &CtbRawFile::image_size_in_bytes)
.def_property_readonly("image_size_in_bytes",
&CtbRawFile::image_size_in_bytes)
.def_property_readonly("frames_in_file", &CtbRawFile::frames_in_file);
py::class_<File>(m, "File")
.def(py::init([](const std::filesystem::path &fname)
{ return File(fname, "r", {}); }))
.def(py::init([](const std::filesystem::path &fname) {
return File(fname, "r", {});
}))
.def(py::init(
[](const std::filesystem::path &fname, const std::string &mode)
{
[](const std::filesystem::path &fname, const std::string &mode) {
return File(fname, mode, {});
}))
.def(py::init<const std::filesystem::path &, const std::string &,
@ -76,27 +87,20 @@ void define_file_io_bindings(py::module &m)
.def_property_readonly("bytes_per_pixel", &File::bytes_per_pixel)
.def_property_readonly(
"detector_type",
[](File &self)
{ return ToString(self.detector_type()); })
[](File &self) { return ToString(self.detector_type()); })
.def("read_frame",
[](File &self)
{
[](File &self) {
const uint8_t item_size = self.bytes_per_pixel();
py::array image;
std::vector<ssize_t> shape;
shape.reserve(2);
shape.push_back(self.rows());
shape.push_back(self.cols());
if (item_size == 1)
{
if (item_size == 1) {
image = py::array_t<uint8_t>(shape);
}
else if (item_size == 2)
{
} else if (item_size == 2) {
image = py::array_t<uint16_t>(shape);
}
else if (item_size == 4)
{
} else if (item_size == 4) {
image = py::array_t<uint32_t>(shape);
}
self.read_into(
@ -104,8 +108,7 @@ void define_file_io_bindings(py::module &m)
return image;
})
.def("read_frame",
[](File &self, size_t frame_number)
{
[](File &self, size_t frame_number) {
self.seek(frame_number);
const uint8_t item_size = self.bytes_per_pixel();
py::array image;
@ -113,24 +116,18 @@ void define_file_io_bindings(py::module &m)
shape.reserve(2);
shape.push_back(self.rows());
shape.push_back(self.cols());
if (item_size == 1)
{
if (item_size == 1) {
image = py::array_t<uint8_t>(shape);
}
else if (item_size == 2)
{
} else if (item_size == 2) {
image = py::array_t<uint16_t>(shape);
}
else if (item_size == 4)
{
} else if (item_size == 4) {
image = py::array_t<uint32_t>(shape);
}
self.read_into(
reinterpret_cast<std::byte *>(image.mutable_data()));
return image;
})
.def("read_n", [](File &self, size_t n_frames)
{
.def("read_n", [](File &self, size_t n_frames) {
const uint8_t item_size = self.bytes_per_pixel();
py::array image;
std::vector<ssize_t> shape;
@ -147,7 +144,8 @@ void define_file_io_bindings(py::module &m)
}
self.read_into(reinterpret_cast<std::byte *>(image.mutable_data()),
n_frames);
return image; });
return image;
});
py::class_<FileConfig>(m, "FileConfig")
.def(py::init<>())
@ -161,8 +159,9 @@ void define_file_io_bindings(py::module &m)
.def_readwrite("dtype", &FileConfig::dtype)
.def("__eq__", &FileConfig::operator==)
.def("__ne__", &FileConfig::operator!=)
.def("__repr__", [](const FileConfig &a)
{ return "<FileConfig: " + a.to_string() + ">"; });
.def("__repr__", [](const FileConfig &a) {
return "<FileConfig: " + a.to_string() + ">";
});
py::class_<RawMasterFile>(m, "RawMasterFile")
.def(py::init<const std::filesystem::path &>())
@ -181,15 +180,18 @@ void define_file_io_bindings(py::module &m)
.def_property_readonly("frame_padding", &RawMasterFile::frame_padding)
.def_property_readonly("frame_discard_policy",
&RawMasterFile::frame_discard_policy)
.def_property_readonly("total_frames_expected", &RawMasterFile::total_frames_expected)
.def_property_readonly("total_frames_expected",
&RawMasterFile::total_frames_expected)
.def_property_readonly("geometry", &RawMasterFile::geometry)
.def_property_readonly("analog_samples", &RawMasterFile::analog_samples)
.def_property_readonly("digital_samples",
&RawMasterFile::digital_samples)
.def_property_readonly("transceiver_samples", &RawMasterFile::transceiver_samples)
.def_property_readonly("transceiver_samples",
&RawMasterFile::transceiver_samples)
.def_property_readonly("number_of_rows", &RawMasterFile::number_of_rows)
.def_property_readonly("quad", &RawMasterFile::quad)
.def_property_readonly("scan_parameters", &RawMasterFile::scan_parameters)
.def_property_readonly("scan_parameters",
&RawMasterFile::scan_parameters)
.def_property_readonly("roi", &RawMasterFile::roi);
py::class_<ScanParameters>(m, "ScanParameters")
@ -206,7 +208,75 @@ void define_file_io_bindings(py::module &m)
.def_readwrite("xmin", &ROI::xmin)
.def_readwrite("xmax", &ROI::xmax)
.def_readwrite("ymin", &ROI::ymin)
.def_readwrite("ymax", &ROI::ymax);
.def_readwrite("ymax", &ROI::ymax)
.def("__str__", [](const ROI& self){
return fmt::format("ROI: xmin: {} xmax: {} ymin: {} ymax: {}", self.xmin, self.xmax, self.ymin, self.ymax);
});
py::class_<RawFile>(m, "RawFile")
.def(py::init<const std::filesystem::path &>())
.def("read_frame",
[](RawFile &self) {
size_t image_size = self.bytes_per_frame();
const uint8_t item_size = self.bytes_per_pixel();
fmt::print("Image size in bytes: {}\n", image_size);
py::array image;
std::vector<ssize_t> shape;
shape.reserve(2);
shape.push_back(self.rows());
shape.push_back(self.cols());
//return headers from all subfiles
py::array_t<DetectorHeader> header(self.n_mod());
if (item_size == 1) {
image = py::array_t<uint8_t>(shape);
} else if (item_size == 2) {
image = py::array_t<uint16_t>(shape);
} else if (item_size == 4) {
image = py::array_t<uint32_t>(shape);
}
fmt::print("item_size: {} rows: {} cols: {}\n", item_size, self.rows(), self.cols());
self.read_into(
reinterpret_cast<std::byte *>(image.mutable_data()),
header.mutable_data());
return py::make_tuple(header, image);
});
py::class_<RawSubFile>(m, "RawSubFile")
.def(py::init<const std::filesystem::path &, DetectorType, size_t,
size_t, size_t>())
.def_property_readonly("bytes_per_frame", &RawSubFile::bytes_per_frame)
.def_property_readonly("pixels_per_frame",
&RawSubFile::pixels_per_frame)
.def("seek", &RawSubFile::seek)
.def("tell", &RawSubFile::tell)
.def_property_readonly("rows", &RawSubFile::rows)
.def_property_readonly("cols", &RawSubFile::cols)
.def("read_frame",
[](RawSubFile &self) {
const uint8_t item_size = self.bytes_per_pixel();
py::array image;
std::vector<ssize_t> shape;
shape.reserve(2);
shape.push_back(self.rows());
shape.push_back(self.cols());
if (item_size == 1) {
image = py::array_t<uint8_t>(shape);
} else if (item_size == 2) {
image = py::array_t<uint16_t>(shape);
} else if (item_size == 4) {
image = py::array_t<uint32_t>(shape);
}
fmt::print("item_size: {} rows: {} cols: {}\n", item_size, self.rows(), self.cols());
self.read_into(
reinterpret_cast<std::byte *>(image.mutable_data()));
return image;
});
// py::class_<ClusterHeader>(m, "ClusterHeader")
// .def(py::init<>())
// .def_readwrite("frame_number", &ClusterHeader::frame_number)

186
src/RawFile.cpp
View File

@ -13,10 +13,11 @@ RawFile::RawFile(const std::filesystem::path &fname, const std::string &mode)
: m_master(fname) {
m_mode = mode;
if (mode == "r") {
n_subfiles = find_number_of_subfiles(); //f0,f1...fn
n_subfile_parts = m_master.geometry().col * m_master.geometry().row; // d0,d1...dn
n_subfiles = find_number_of_subfiles(); // f0,f1...fn
n_subfile_parts =
m_master.geometry().col * m_master.geometry().row; // d0,d1...dn
find_geometry();
//update_geometry_from_roi();
update_geometry_with_roi();
open_subfiles();
} else {
throw std::runtime_error(LOCATION +
@ -39,10 +40,19 @@ void RawFile::read_into(std::byte *image_buf, size_t n_frames) {
image_buf += bytes_per_frame();
}
}
void RawFile::read_into(std::byte *image_buf) {
return get_frame_into(m_current_frame++, image_buf);
};
void RawFile::read_into(std::byte *image_buf, DetectorHeader *header) {
return get_frame_into(m_current_frame++, image_buf, header);
};
size_t RawFile::n_mod() const { return n_subfile_parts; }
size_t RawFile::bytes_per_frame() {
return m_rows * m_cols * m_master.bitdepth() / 8;
}
@ -72,12 +82,12 @@ xy RawFile::geometry() { return m_master.geometry(); }
void RawFile::open_subfiles() {
if (m_mode == "r")
for (size_t i = 0; i != n_subfiles; ++i) {
auto v = std::vector<SubFile *>(n_subfile_parts);
auto v = std::vector<RawSubFile *>(n_subfile_parts);
for (size_t j = 0; j != n_subfile_parts; ++j) {
v[j] =
new SubFile(m_master.data_fname(j, i),
m_master.detector_type(), m_master.pixels_y(),
m_master.pixels_x(), m_master.bitdepth());
auto pos = m_module_pixel_0[j];
v[j] = new RawSubFile(m_master.data_fname(j, i),
m_master.detector_type(), pos.height,
pos.width, m_master.bitdepth());
}
subfiles.push_back(v);
}
@ -111,38 +121,30 @@ bool RawFile::is_master_file(const std::filesystem::path &fpath) {
int RawFile::find_number_of_subfiles() {
int n_files = 0;
// f0,f1...fn How many files is the data split into?
while (std::filesystem::exists(m_master.data_fname(0, ++n_files)))
;
while (std::filesystem::exists(m_master.data_fname(0, n_files)))
n_files++; // increment after test
#ifdef AARE_VERBOSE
fmt::print("Found: {} subfiles\n", n_subfiles);
#endif
#ifdef AARE_VERBOSE
fmt::print("Found: {} subfiles\n", n_files);
#endif
return n_files;
}
void RawFile::find_geometry() {
uint16_t r{};
uint16_t c{};
// for (size_t i = 0; i < n_subfile_parts; i++) {
// for (size_t j = 0; j != n_subfiles; ++j) {
// auto h = this->read_header(m_master.data_fname(i, j));
// r = std::max(r, h.row);
// c = std::max(c, h.column);
// positions.push_back({h.row, h.column});
// }
// }
std::vector<xy> module_position;
for (size_t i = 0; i < n_subfile_parts; i++) {
auto h = this->read_header(m_master.data_fname(i, 0));
r = std::max(r, h.row);
c = std::max(c, h.column);
positions.push_back({h.row, h.column});
xy pos = {h.row * m_master.pixels_y(), h.column* m_master.pixels_x()};
module_position.emplace_back(pos);
auto h = this->read_header(m_master.data_fname(i, 0));
r = std::max(r, h.row);
c = std::max(c, h.column);
positions.push_back({h.row, h.column});
ModuleGeometry g;
g.x = h.column * m_master.pixels_x();
g.y = h.row * m_master.pixels_y();
g.width = m_master.pixels_x();
g.height = m_master.pixels_y();
m_module_pixel_0.push_back(g);
}
r++;
@ -152,10 +154,79 @@ void RawFile::find_geometry() {
m_cols = (c * m_master.pixels_x());
m_rows += static_cast<size_t>((r - 1) * cfg.module_gap_row);
for (size_t i=0; i < module_position.size(); i++){
fmt::print("Module {} at position: ({},{})\n", i, module_position[i].row, module_position[i].col);
#ifdef AARE_VERBOSE
fmt::print("\nRawFile::find_geometry()\n");
for (size_t i = 0; i < m_module_pixel_0.size(); i++) {
fmt::print("Module {} at position: (r:{},c:{})\n", i,
m_module_pixel_0[i].y, m_module_pixel_0[i].x);
}
fmt::print("Image size: {}x{}\n\n", m_rows, m_cols);
#endif
}
void RawFile::update_geometry_with_roi() {
// TODO! implement this
if (m_master.roi()) {
auto roi = m_master.roi().value();
// TODO! can we do this cleaner?
int pos_y = 0;
int pos_y_increment = 0;
for (size_t row = 0; row < m_master.geometry().row; row++) {
int pos_x = 0;
for (size_t col = 0; col < m_master.geometry().col; col++) {
auto &m = m_module_pixel_0[row * m_master.geometry().col + col];
auto original_height = m.height;
auto original_width = m.width;
// module is to the left of the roi
if (m.x + m.width < roi.xmin) {
m.width = 0;
// roi is in module
} else {
// here we only arrive when the roi is in or to the left of
// the module
if (roi.xmin > m.x) {
m.width -= roi.xmin - m.x;
}
if (roi.xmax < m.x + m.width) {
m.width -= m.x + original_width - roi.xmax;
}
m.x = pos_x;
pos_x += m.width;
}
if (m.y + m.height < roi.ymin) {
m.height = 0;
} else {
if (roi.ymin < m.y + m.height) {
m.height -= roi.ymin;
}
if (roi.ymax < m.y + m.height) {
m.height -= m.y + original_height - roi.ymax;
}
m.y = pos_y;
pos_y_increment = m.height;
}
}
// increment pos_y
pos_y += pos_y_increment;
}
m_rows = roi.height();
m_cols = roi.width();
}
#ifdef AARE_VERBOSE
fmt::print("RawFile::update_geometry_with_roi()\n");
for (const auto &m : m_module_pixel_0) {
fmt::print("Module at position: (r:{}, c:{}, h:{}, w:{})\n", m.y, m.x,
m.height, m.width);
}
fmt::print("Updated image size: {}x{}\n\n", m_rows, m_cols);
fmt::print("\n");
#endif
}
Frame RawFile::get_frame(size_t frame_index) {
@ -165,13 +236,18 @@ Frame RawFile::get_frame(size_t frame_index) {
return f;
}
void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer) {
size_t RawFile::bytes_per_pixel() const {
return m_master.bitdepth() / 8;
}
void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer, DetectorHeader *header) {
if (frame_index > total_frames()) {
throw std::runtime_error(LOCATION + "Frame number out of range");
}
std::vector<size_t> frame_numbers(n_subfile_parts);
std::vector<size_t> frame_indices(n_subfile_parts, frame_index);
// sync the frame numbers
if (n_subfile_parts != 1) {
for (size_t part_idx = 0; part_idx != n_subfile_parts; ++part_idx) {
auto subfile_id = frame_index / m_master.max_frames_per_file();
@ -208,15 +284,24 @@ void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer) {
for (size_t part_idx = 0; part_idx != n_subfile_parts; ++part_idx) {
auto corrected_idx = frame_indices[part_idx];
auto subfile_id = corrected_idx / m_master.max_frames_per_file();
auto part_offset = subfiles[subfile_id][part_idx]->bytes_per_part();
subfiles[subfile_id][part_idx]->get_part(
frame_buffer + part_idx * part_offset,
corrected_idx % m_master.max_frames_per_file());
// This is where we start writing
auto offset = (m_module_pixel_0[part_idx].y * m_cols +
m_module_pixel_0[part_idx].x)*m_master.bitdepth()/8;
if (m_module_pixel_0[part_idx].x!=0)
throw std::runtime_error(LOCATION + "Implementation error. x pos not 0.");
subfiles[subfile_id][part_idx]->seek(corrected_idx % m_master.max_frames_per_file());
subfiles[subfile_id][part_idx]->read_into(frame_buffer + offset, header);
if (header)
++header;
}
} else {
//TODO! should we read row by row?
// create a buffer that will hold a the frame part
// create a buffer large enough to hold a full module
auto bytes_per_part = m_master.pixels_y() * m_master.pixels_x() *
m_master.bitdepth() /
8; // TODO! replace with image_size_in_bytes
@ -226,23 +311,26 @@ void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer) {
// level
for (size_t part_idx = 0; part_idx != n_subfile_parts; ++part_idx) {
auto pos = m_module_pixel_0[part_idx];
auto corrected_idx = frame_indices[part_idx];
auto subfile_id = corrected_idx / m_master.max_frames_per_file();
subfiles[subfile_id][part_idx]->get_part(
part_buffer, corrected_idx % m_master.max_frames_per_file());
for (size_t cur_row = 0; cur_row < (m_master.pixels_y());
subfiles[subfile_id][part_idx]->seek(corrected_idx % m_master.max_frames_per_file());
subfiles[subfile_id][part_idx]->read_into(part_buffer, header);
if(header)
++header;
for (size_t cur_row = 0; cur_row < (pos.height);
cur_row++) {
auto irow = cur_row + (part_idx / m_master.geometry().col) *
m_master.pixels_y();
auto icol =
(part_idx % m_master.geometry().col) * m_master.pixels_x();
auto irow = (pos.y + cur_row);
auto icol = pos.x;
auto dest = (irow * this->m_cols + icol);
dest = dest * m_master.bitdepth() / 8;
memcpy(frame_buffer + dest,
part_buffer + cur_row * m_master.pixels_x() *
part_buffer + cur_row * pos.width *
m_master.bitdepth() / 8,
m_master.pixels_x() * m_master.bitdepth() / 8);
pos.width * m_master.bitdepth() / 8);
}
}
delete[] part_buffer;

8
src/RawMasterFile.cpp
View File

@ -8,6 +8,8 @@ RawFileNameComponents::RawFileNameComponents(
m_base_name = fname.stem();
m_ext = fname.extension();
AARE_ASSERT(false);
if (m_ext != ".json" && m_ext != ".raw") {
throw std::runtime_error(LOCATION +
"Unsupported file type. (only .json or .raw)");
@ -254,6 +256,12 @@ void RawMasterFile::parse_json(const std::filesystem::path &fpath) {
//if any of the values are set update the roi
if (tmp_roi.xmin != 4294967295 || tmp_roi.xmax != 4294967295 ||
tmp_roi.ymin != 4294967295 || tmp_roi.ymax != 4294967295) {
if(v<7.21){
tmp_roi.xmax++;
tmp_roi.ymax++;
}
m_roi = tmp_roi;
}

99
src/RawSubFile.cpp Normal file
View File

@ -0,0 +1,99 @@
#include "aare/RawSubFile.hpp"
#include "aare/PixelMap.hpp"
#include <cstring> // memcpy
#include <fmt/core.h>
#include <iostream>
namespace aare {
RawSubFile::RawSubFile(const std::filesystem::path &fname,
DetectorType detector, size_t rows, size_t cols,
size_t bitdepth)
: m_bitdepth(bitdepth), m_fname(fname), m_rows(rows), m_cols(cols),
m_detector_type(detector),
m_bytes_per_frame((m_bitdepth / 8) * m_rows * m_cols) {
if (m_detector_type == DetectorType::Moench03_old) {
pixel_map = GenerateMoench03PixelMap();
}
if (std::filesystem::exists(fname)) {
n_frames = std::filesystem::file_size(fname) /
(sizeof(DetectorHeader) + rows * cols * bitdepth / 8);
} else {
throw std::runtime_error(
LOCATION + fmt::format("File {} does not exist", m_fname.string()));
}
// fp = fopen(m_fname.string().c_str(), "rb");
m_file.open(m_fname, std::ios::binary);
if (!m_file.is_open()) {
throw std::runtime_error(
LOCATION + fmt::format("Could not open file {}", m_fname.string()));
}
#ifdef AARE_VERBOSE
fmt::print("Opened file: {} with {} frames\n", m_fname.string(), n_frames);
fmt::print("m_rows: {}, m_cols: {}, m_bitdepth: {}\n", m_rows, m_cols,
m_bitdepth);
fmt::print("file size: {}\n", std::filesystem::file_size(fname));
#endif
}
void RawSubFile::seek(size_t frame_index) {
if (frame_index >= n_frames) {
throw std::runtime_error("Frame number out of range");
}
m_file.seekg((sizeof(DetectorHeader) + bytes_per_frame()) * frame_index);
}
size_t RawSubFile::tell() {
return m_file.tellg() / (sizeof(DetectorHeader) + bytes_per_frame());
}
void RawSubFile::read_into(std::byte *image_buf, DetectorHeader *header) {
if(header){
fmt::print("Reading header\n");
m_file.read(reinterpret_cast<char *>(header), sizeof(DetectorHeader));
} else {
m_file.seekg(sizeof(DetectorHeader), std::ios::cur);
fmt::print("Skipping heading header\n");
}
//TODO! expand support for different bitdepths
if(pixel_map){
// read into a temporary buffer and then copy the data to the buffer
// in the correct order
// currently this only supports 16 bit data!
auto part_buffer = new std::byte[bytes_per_frame()];
m_file.read(reinterpret_cast<char *>(part_buffer), bytes_per_frame());
auto *data = reinterpret_cast<uint16_t *>(image_buf);
auto *part_data = reinterpret_cast<uint16_t *>(part_buffer);
for (size_t i = 0; i < pixels_per_frame(); i++) {
data[i] = part_data[(*pixel_map)(i)];
}
delete[] part_buffer;
} else {
// read directly into the buffer
m_file.read(reinterpret_cast<char *>(image_buf), bytes_per_frame());
}
}
size_t RawSubFile::rows() const { return m_rows; }
size_t RawSubFile::cols() const { return m_cols; }
void RawSubFile::get_part(std::byte *buffer, size_t frame_index) {
seek(frame_index);
read_into(buffer, nullptr);
}
size_t RawSubFile::frame_number(size_t frame_index) {
seek(frame_index);
DetectorHeader h{};
m_file.read(reinterpret_cast<char *>(&h), sizeof(DetectorHeader));
return h.frameNumber;
}
} // namespace aare

82
src/SubFile.cpp
View File

@ -1,82 +0,0 @@
#include "aare/SubFile.hpp"
#include "aare/PixelMap.hpp"
#include <cstring> // memcpy
#include <fmt/core.h>
#include <iostream>
namespace aare {
SubFile::SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth,
const std::string &mode)
: m_bitdepth(bitdepth), m_fname(fname), m_rows(rows), m_cols(cols), m_mode(mode), m_detector_type(detector) {
if (m_detector_type == DetectorType::Moench03_old) {
pixel_map = GenerateMoench03PixelMap();
}
if (std::filesystem::exists(fname)) {
n_frames = std::filesystem::file_size(fname) / (sizeof(DetectorHeader) + rows * cols * bitdepth / 8);
} else {
n_frames = 0;
}
if (mode == "r") {
fp = fopen(m_fname.string().c_str(), "rb");
} else {
throw std::runtime_error(LOCATION + "Unsupported mode. Can only read RawFiles.");
}
if (fp == nullptr) {
throw std::runtime_error(LOCATION + fmt::format("Could not open file {}", m_fname.string()));
}
#ifdef AARE_VERBOSE
fmt::print("Opened file: {} with {} frames\n", m_fname.string(), n_frames);
fmt::print("m_rows: {}, m_cols: {}, m_bitdepth: {}\n", m_rows, m_cols, m_bitdepth);
#endif
}
size_t SubFile::get_part(std::byte *buffer, size_t frame_index) {
if (frame_index >= n_frames) {
throw std::runtime_error("Frame number out of range");
}
fseek(fp, sizeof(DetectorHeader) + (sizeof(DetectorHeader) + bytes_per_part()) * frame_index, // NOLINT
SEEK_SET);
if (pixel_map){
// read into a temporary buffer and then copy the data to the buffer
// in the correct order
auto part_buffer = new std::byte[bytes_per_part()];
auto wc = fread(part_buffer, bytes_per_part(), 1, fp);
auto *data = reinterpret_cast<uint16_t *>(buffer);
auto *part_data = reinterpret_cast<uint16_t *>(part_buffer);
for (size_t i = 0; i < pixels_per_part(); i++) {
data[i] = part_data[(*pixel_map)(i)];
}
delete[] part_buffer;
return wc;
}else{
// read directly into the buffer
return fread(buffer, this->bytes_per_part(), 1, this->fp);
}
}
size_t SubFile::frame_number(size_t frame_index) {
DetectorHeader h{};
fseek(fp, (sizeof(DetectorHeader) + bytes_per_part()) * frame_index, SEEK_SET); // NOLINT
size_t const rc = fread(reinterpret_cast<char *>(&h), sizeof(h), 1, fp);
if (rc != 1)
throw std::runtime_error(LOCATION + "Could not read header from file");
return h.frameNumber;
}
SubFile::~SubFile() {
if (fp) {
fclose(fp);
}
}
} // namespace aare

10
src/defs.cpp
View File

@ -1,9 +1,17 @@
#include "aare/defs.hpp"
#include <stdexcept>
#include <string>
#include <fmt/core.h>
namespace aare {
void assert_failed(const std::string &msg)
{
fmt::print(msg);
exit(1);
}
/**
* @brief Convert a DetectorType to a string
* @param type DetectorType