detectors/aare
25
0
Fork 0
mirror of https://github.com/slsdetectorgroup/aare.git synced 2025-06-06 12:50:40 +02:00
Code Issues Actions Packages Releases Activity

added CtbRawFile

Browse Source
This commit is contained in:
Erik Fröjdh 2024-11-05 14:36:18 +01:00
parent b8a4498379
commit 80a39415de
24 changed files with 687 additions and 262 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
CMakeLists.txt
View File

@ -237,7 +237,7 @@ set(PUBLICHEADERS
include/aare/Dtype.hpp
include/aare/File.hpp
include/aare/FileInterface.hpp
include/aare/file_utils.hpp
include/aare/RawMasterFile.hpp
include/aare/Frame.hpp
include/aare/json.hpp
include/aare/NDArray.hpp
@ -259,7 +259,7 @@ set(SourceFiles
${CMAKE_CURRENT_SOURCE_DIR}/src/Dtype.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/Frame.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/File.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/file_utils.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/RawMasterFile.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyFile.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/PixelMap.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/RawFile.cpp
@ -290,7 +290,7 @@ if(AARE_TESTS)
${CMAKE_CURRENT_SOURCE_DIR}/src/defs.test.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/Dtype.test.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/Frame.test.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/file_utils.test.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/RawMasterFile.test.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/NDArray.test.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/NDView.test.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/ClusterFinder.test.cpp

1
docs/CMakeLists.txt
View File

@ -22,6 +22,7 @@ set(SPHINX_SOURCE_FILES
src/VarClusterFinder.rst
src/pyVarClusterFinder.rst
src/pyFile.rst
src/pyCtbRawFile.rst
)
foreach(filename ${SPHINX_SOURCE_FILES})

2
docs/src/index.rst
View File

@ -23,4 +23,6 @@ AARE
:maxdepth: 1
pyFile
pyCtbRawFile
pyVarClusterFinder

11
docs/src/pyCtbRawFile.rst Normal file
View File

@ -0,0 +1,11 @@
CtbRawFile
============
.. py:currentmodule:: aare
.. autoclass:: CtbRawFile
:members:
:undoc-members:
:show-inheritance:
:inherited-members:

22
include/aare/CtbRawFile.hpp
View File

@ -1,19 +1,37 @@
#pragma once
#include "aare/FileInterface.hpp"
#include "aare/file_utils.hpp"
#include "aare/RawMasterFile.hpp"
#include "aare/Frame.hpp"
#include <filesystem>
#include <fstream>
namespace aare{
class CtbRawFile{
FileNameComponents m_fnc{};
RawMasterFile m_master;
std::ifstream m_file;
size_t m_current_frame{0};
size_t m_current_subfile{0};
size_t m_num_subfiles{0};
public:
CtbRawFile(const std::filesystem::path &fname);
void read_into(std::byte *image_buf, DetectorHeader* header = nullptr);
void seek(size_t frame_index); //!< seek to the given frame index
size_t tell() const; //!< get the frame index of the file pointer
// in the specific class we can expose more functionality
size_t image_size_in_bytes() const { return m_master.image_size_in_bytes(); }
private:
void find_subfiles();
size_t sub_file_index(size_t frame_index) const {
return frame_index / m_master.max_frames_per_file();
}
void open_data_file(size_t subfile_index);
};

4
include/aare/RawFile.hpp
View File

@ -180,9 +180,9 @@ class RawFile : public FileInterface {
/**
* @brief read the header of the file
* @param fname path to the data subfile
* @return sls_detector_header
* @return DetectorHeader
*/
static sls_detector_header read_header(const std::filesystem::path &fname);
static DetectorHeader read_header(const std::filesystem::path &fname);
/**
* @brief open the subfiles

96
include/aare/RawMasterFile.hpp Normal file
View File

@ -0,0 +1,96 @@
#pragma once
#include "aare/defs.hpp"
#include <filesystem>
#include <fmt/format.h>
#include <fstream>
#include <nlohmann/json.hpp>
using json = nlohmann::json;
namespace aare {
class RawFileNameComponents {
std::filesystem::path m_base_path{};
std::string m_base_name{};
std::string m_ext{};
int m_file_index{}; // TODO! is this measurement_index?
public:
RawFileNameComponents(const std::filesystem::path &fname);
std::filesystem::path master_fname() const {
return m_base_path /
fmt::format("{}_master_{}{}", m_base_name, m_file_index, m_ext);
}
std::filesystem::path data_fname(size_t mod_id, size_t file_id) {
return m_base_path / fmt::format("{}_d{}_f{}_{}.raw", m_base_name,
mod_id, file_id, m_file_index);
}
const std::filesystem::path &base_path() const { return m_base_path; }
const std::string &base_name() const { return m_base_name; }
const std::string &ext() const { return m_ext; }
int file_index() const { return m_file_index; }
};
class RawMasterFile {
RawFileNameComponents m_fnc;
std::string m_version;
DetectorType m_type;
TimingMode m_timing_mode;
size_t m_image_size_in_bytes;
size_t m_frames_in_file;
size_t m_pixels_y;
size_t m_pixels_x;
size_t m_bitdepth;
size_t m_max_frames_per_file;
uint32_t m_adc_mask;
FrameDiscardPolicy m_frame_discard_policy;
size_t m_frame_padding;
std::optional<size_t> m_analog_samples;
std::optional<size_t> m_digital_samples;
public:
RawMasterFile(const std::filesystem::path &fpath) : m_fnc(fpath) {
if (!std::filesystem::exists(fpath)) {
throw std::runtime_error(LOCATION + " File does not exist");
}
if (m_fnc.ext() == ".json") {
parse_json(fpath);
} else if (m_fnc.ext() == ".raw") {
parse_raw(fpath);
} else {
throw std::runtime_error(LOCATION + "Unsupported file type");
}
}
const std::string &version() const { return m_version; }
const DetectorType &detector_type() const { return m_type; }
const TimingMode &timing_mode() const { return m_timing_mode; }
size_t image_size_in_bytes() const { return m_image_size_in_bytes; }
size_t frames_in_file() const { return m_frames_in_file; }
size_t pixels_y() const { return m_pixels_y; }
size_t pixels_x() const { return m_pixels_x; }
size_t max_frames_per_file() const { return m_max_frames_per_file; }
size_t bitdepth() const { return m_bitdepth; }
size_t frame_padding() const { return m_frame_padding; }
const FrameDiscardPolicy &frame_discard_policy() const {
return m_frame_discard_policy;
}
std::optional<size_t> analog_samples() const { return m_analog_samples; }
std::optional<size_t> digital_samples() const { return m_digital_samples; }
std::filesystem::path data_fname(size_t mod_id, size_t file_id) {
return m_fnc.data_fname(mod_id, file_id);
}
private:
void parse_json(const std::filesystem::path &fpath);
void parse_raw(const std::filesystem::path &fpath);
};
} // namespace aare

2
include/aare/SubFile.hpp
View File

@ -15,7 +15,7 @@ namespace aare {
*/
class SubFile {
public:
size_t write_part(std::byte *buffer, sls_detector_header header, size_t frame_index);
size_t write_part(std::byte *buffer, DetectorHeader header, size_t frame_index);
/**
* @brief SubFile constructor
* @param fname path to the subfile

6
include/aare/defs.hpp
View File

@ -111,7 +111,7 @@ class Cluster {
/**
* @brief header contained in parts of frames
*/
struct sls_detector_header {
struct DetectorHeader {
uint64_t frameNumber;
uint32_t expLength;
uint32_t packetNumber;
@ -166,6 +166,7 @@ using xy = t_xy<uint32_t>;
using dynamic_shape = std::vector<int64_t>;
//TODO! Can we uniform enums between the libraries?
enum class DetectorType {
Jungfrau,
Eiger,
@ -178,6 +179,7 @@ enum class DetectorType {
};
enum class TimingMode { Auto, Trigger };
enum class FrameDiscardPolicy { NoDiscard, Discard };
template <class T> T StringTo(const std::string &arg) { return T(arg); }
@ -188,6 +190,8 @@ template <> std::string ToString(DetectorType arg);
template <> TimingMode StringTo(const std::string & /*mode*/);
template <> FrameDiscardPolicy StringTo(const std::string & /*mode*/);
using DataTypeVariants = std::variant<uint16_t, uint32_t>;
} // namespace aare

119
include/aare/file_utils.hpp
View File

@ -1,119 +0,0 @@
#pragma once
#include "aare/defs.hpp"
#include <filesystem>
#include <fmt/format.h>
#include <fstream>
#include <nlohmann/json.hpp>
using json = nlohmann::json;
namespace aare {
bool is_master_file(const std::filesystem::path &fpath);
struct FileNameComponents {
std::filesystem::path base_path{};
std::string base_name{};
std::string ext{};
int findex{};
bool valid{false}; // TODO! how do we do error handling?
std::filesystem::path master_fname() const {
return base_path /
fmt::format("{}_master_{}{}", base_name, findex, ext);
}
std::filesystem::path data_fname(size_t mod_id, size_t file_id) {
return base_path / fmt::format("{}_d{}_f{}_{}.raw", base_name, file_id,
mod_id, findex);
}
};
FileNameComponents parse_fname(const std::filesystem::path &fname);
class MasterFile {
FileNameComponents m_fnc;
std::string m_version;
DetectorType m_type;
TimingMode m_timing_mode;
size_t m_total_frames;
size_t m_subfile_rows;
size_t m_subfile_cols;
size_t m_bitdepth;
size_t m_analog_samples;
size_t m_digital_samples;
size_t m_max_frames_per_file;
uint32_t m_adc_mask;
public:
MasterFile(const std::filesystem::path &fpath) {
m_fnc = parse_fname(fpath);
std::ifstream ifs(fpath);
json j;
ifs >> j;
double v = j["Version"];
m_version = fmt::format("{:.1f}", v);
m_type = StringTo<DetectorType>(j["Detector// Type"].get<std::string>());
m_timing_mode =
StringTo<TimingMode>(j["Timing Mode"].get<std::string>());
m_total_frames = j["Frames in File"];
m_subfile_rows = j["Pixels"]["y"];
m_subfile_cols = j["Pixels"]["x"];
m_max_frames_per_file = j["Max Frames Per File"];
try {
m_bitdepth = j.at("Dynamic Range");
} catch (const json::out_of_range &e) {
m_bitdepth = 16;
}
try {
m_analog_samples = j.at("Analog Samples");
}catch (const json::out_of_range &e) {
m_analog_samples = 0;
}
try{
std::string adc_mask = j.at("ADC Mask");
m_adc_mask = std::stoul(adc_mask, nullptr, 16);
}catch (const json::out_of_range &e) {
m_adc_mask = 0;
}
try {
m_digital_samples = j.at("Digital Samples");
}catch (const json::out_of_range &e) {
m_digital_samples = 0;
}
//Update detector type for Moench
//TODO! How does this work with old .raw master files?
if (m_type == DetectorType::Moench && m_analog_samples == 0 &&
m_subfile_rows == 400) {
m_type = DetectorType::Moench03;
}else if (m_type == DetectorType::Moench && m_subfile_rows == 400 &&
m_analog_samples == 5000) {
m_type = DetectorType::Moench03_old;
}
// //Here we know we have a ChipTestBoard file update the geometry?
// //TODO! Carry on information about digtial, and transceivers
// if (m_type == DetectorType::ChipTestBoard) {
// subfile_rows = 1;
// subfile_cols = m_analog_samples*__builtin_popcount(m_adc_mask);
// }
// // only Eiger had quad
// if (m_type == DetectorType::Eiger) {
// quad = (j["Quad"] == 1);
// }
// m_geometry = {j["Geometry"]["y"], j["Geometry"]["x"]};
}
};
} // namespace aare

2
python/CMakeLists.txt
View File

@ -27,6 +27,8 @@ target_link_libraries(_aare PRIVATE aare_core aare_compiler_flags)
# List of python files to be copied to the build directory
set( PYTHON_FILES
aare/__init__.py
aare/CtbRawFile.py
aare/transform.py
)
# Copy the python files to the build directory

69
python/aare/CtbRawFile.py Normal file
View File

@ -0,0 +1,69 @@
from . import _aare
import numpy as np
class CtbRawFile(_aare.CtbRawFile):
def __init__(self, fname, transform = None):
super().__init__(fname)
self.transform = transform
def read_frame(self, frame_index = None):
"""Read one frame from the file.
Args:
frame_index (int): If not None, seek to this frame before reading.
Returns:
tuple: header, data
"""
if frame_index is not None:
self.seek(frame_index)
header, data = super().read_frame()
if header.shape == (1,):
header = header[0]
if self.transform:
res = self.transform(data)
if isinstance(res, tuple):
return header, *res
else:
return header, res
def read_n(self, n_frames):
# Do the first read to figure out what we have
tmp_header, tmp_data = self.read_frame()
# Allocate arrays for
header = np.zeros(n_frames, dtype = tmp_header.dtype)
data = np.zeros((n_frames, *tmp_data.shape), dtype = tmp_data.dtype)
# Copy the first frame
header[0] = tmp_header
data[0] = tmp_data
# Do the rest of the reading
for i in range(1, n_frames):
header[i], data[i] = self.read_frame()
return header, data
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
pass
def __iter__(self):
return self
def __next__(self):
try:
return self.read_frame()
except RuntimeError:
# TODO! find a good way to check that we actually have the right exception
raise StopIteration

7
python/aare/__init__.py
View File

@ -1,4 +1,5 @@
# Make the compiled classes that live in _aare available from aare.
from ._aare import File
from ._aare import VarClusterFinder
from ._aare import GenerateMoench03PixelMap, GenerateMoench05PixelMap
from . import _aare
from ._aare import VarClusterFinder, File
from .CtbRawFile import CtbRawFile

15
python/aare/transform.py Normal file
View File

@ -0,0 +1,15 @@
import numpy as np
from . import _aare
class Moench05Transform:
#Could be moved to C++ without changing the interface
def __init__(self):
print('map created')
self.pixel_map = _aare.GenerateMoench05PixelMap()
def __call__(self, data):
return np.take(data.view(np.uint16), self.pixel_map)
moench05 = Moench05Transform()

19
python/examples/play.py
View File

@ -3,6 +3,10 @@ import numpy as np
plt.ion()
import aare
from aare import CtbRawFile
print('aare imported')
from aare import transform
print('transform imported')
from pathlib import Path
# p = Path('/Users/erik/data/aare_test_data/jungfrau/jungfrau_single_master_0.json')
@ -15,11 +19,14 @@ from pathlib import Path
# fpath = Path('/Users/erik/data/Moench03old/test_034_irradiated_noise_g4_hg_exptime_2000us_master_0.json')
fpath = Path('/Users/erik/data/Moench05/moench05_master_0.json')
f = aare.File(fpath)
f.seek(437)
frame = f.read_frame()
fpath = Path('/Users/erik/data/Moench05/moench05_multifile_master_0.json')
m = aare.GenerateMoench05PixelMap()
img = np.take(frame, m.astype(np.int64))
# f = aare.CtbRawFile(fpath, transform = transform.moench05)
with CtbRawFile(fpath, transform = transform.moench05) as f:
for header, image in f:
print(f'Frame number: {header["frameNumber"]}')

91
python/src/file.hpp
View File

@ -1,32 +1,63 @@
#include "aare/Frame.hpp"
#include "aare/File.hpp"
#include "aare/CtbRawFile.hpp"
#include "aare/File.hpp"
#include "aare/Frame.hpp"
#include "aare/defs.hpp"
// #include "aare/fClusterFileV2.hpp"
#include <cstdint>
#include <filesystem>
#include <pybind11/numpy.h>
#include <pybind11/iostream.h>
#include <pybind11/numpy.h>
#include <pybind11/pybind11.h>
#include <pybind11/stl.h>
#include <pybind11/stl/filesystem.h>
#include <string>
namespace py = pybind11;
using namespace::aare;
using namespace ::aare;
void define_file_io_bindings(py::module &m) {
py::class_<CtbRawFile>(m, "CtbRawFile")
.def(py::init<const std::filesystem::path &>());
PYBIND11_NUMPY_DTYPE(DetectorHeader, frameNumber, expLength, packetNumber,
bunchId, timestamp, modId, row, column, reserved,
debug, roundRNumber, detType, version, packetMask);
py::class_<CtbRawFile>(m, "CtbRawFile")
.def(py::init<const std::filesystem::path &>())
.def("read_frame",
[](CtbRawFile &self) {
size_t image_size = self.image_size_in_bytes();
py::array image;
std::vector<ssize_t> shape;
shape.reserve(2);
shape.push_back(1);
shape.push_back(image_size);
py::array_t<DetectorHeader> header(1);
// always read bytes
image = py::array_t<uint8_t>(shape);
self.read_into(
reinterpret_cast<std::byte *>(image.mutable_data()),
header.mutable_data());
return py::make_tuple(header, image);
})
.def("seek", &CtbRawFile::seek)
.def("tell", &CtbRawFile::tell);
py::class_<File>(m, "File")
.def(py::init([](const std::filesystem::path &fname) { return File(fname, "r", {}); }))
.def(
py::init([](const std::filesystem::path &fname, const std::string &mode) { return File(fname, mode, {}); }))
.def(py::init<const std::filesystem::path &, const std::string &, const FileConfig &>())
.def(py::init([](const std::filesystem::path &fname) {
return File(fname, "r", {});
}))
.def(py::init(
[](const std::filesystem::path &fname, const std::string &mode) {
return File(fname, mode, {});
}))
.def(py::init<const std::filesystem::path &, const std::string &,
const FileConfig &>())
.def("frame_number", &File::frame_number)
.def_property_readonly("bytes_per_frame", &File::bytes_per_frame)
@ -38,10 +69,11 @@ void define_file_io_bindings(py::module &m) {
.def_property_readonly("cols", &File::cols)
.def_property_readonly("bitdepth", &File::bitdepth)
.def_property_readonly("bytes_per_pixel", &File::bytes_per_pixel)
.def_property_readonly("detector_type", [](File &self){
return ToString(self.detector_type());
})
.def("read_frame", [](File &self) {
.def_property_readonly(
"detector_type",
[](File &self) { return ToString(self.detector_type()); })
.def("read_frame",
[](File &self) {
const uint8_t item_size = self.bytes_per_pixel();
py::array image;
std::vector<ssize_t> shape;
@ -55,10 +87,12 @@ void define_file_io_bindings(py::module &m) {
} else if (item_size == 4) {
image = py::array_t<uint32_t>(shape);
}
self.read_into(reinterpret_cast<std::byte *>(image.mutable_data()));
self.read_into(
reinterpret_cast<std::byte *>(image.mutable_data()));
return image;
})
.def("read_frame", [](File &self, size_t frame_number) {
.def("read_frame",
[](File &self, size_t frame_number) {
self.seek(frame_number);
const uint8_t item_size = self.bytes_per_pixel();
py::array image;
@ -73,7 +107,8 @@ void define_file_io_bindings(py::module &m) {
} else if (item_size == 4) {
image = py::array_t<uint32_t>(shape);
}
self.read_into(reinterpret_cast<std::byte *>(image.mutable_data()));
self.read_into(
reinterpret_cast<std::byte *>(image.mutable_data()));
return image;
})
.def("read_n", [](File &self, size_t n_frames) {
@ -91,7 +126,8 @@ void define_file_io_bindings(py::module &m) {
} else if (item_size == 4) {
image = py::array_t<uint32_t>(shape);
}
self.read_into(reinterpret_cast<std::byte *>(image.mutable_data()), n_frames);
self.read_into(reinterpret_cast<std::byte *>(image.mutable_data()),
n_frames);
return image;
});
@ -107,37 +143,44 @@ void define_file_io_bindings(py::module &m) {
.def_readwrite("dtype", &FileConfig::dtype)
.def("__eq__", &FileConfig::operator==)
.def("__ne__", &FileConfig::operator!=)
.def("__repr__", [](const FileConfig &a) { return "<FileConfig: " + a.to_string() + ">"; });
.def("__repr__", [](const FileConfig &a) {
return "<FileConfig: " + a.to_string() + ">";
});
// py::class_<ClusterHeader>(m, "ClusterHeader")
// .def(py::init<>())
// .def_readwrite("frame_number", &ClusterHeader::frame_number)
// .def_readwrite("n_clusters", &ClusterHeader::n_clusters)
// .def("__repr__", [](const ClusterHeader &a) { return "<ClusterHeader: " + a.to_string() + ">"; });
// .def("__repr__", [](const ClusterHeader &a) { return "<ClusterHeader:
// " + a.to_string() + ">"; });
// py::class_<ClusterV2_>(m, "ClusterV2_")
// .def(py::init<>())
// .def_readwrite("x", &ClusterV2_::x)
// .def_readwrite("y", &ClusterV2_::y)
// .def_readwrite("data", &ClusterV2_::data)
// .def("__repr__", [](const ClusterV2_ &a) { return "<ClusterV2_: " + a.to_string(false) + ">"; });
// .def("__repr__", [](const ClusterV2_ &a) { return "<ClusterV2_: " +
// a.to_string(false) + ">"; });
// py::class_<ClusterV2>(m, "ClusterV2")
// .def(py::init<>())
// .def_readwrite("cluster", &ClusterV2::cluster)
// .def_readwrite("frame_number", &ClusterV2::frame_number)
// .def("__repr__", [](const ClusterV2 &a) { return "<ClusterV2: " + a.to_string() + ">"; });
// .def("__repr__", [](const ClusterV2 &a) { return "<ClusterV2: " +
// a.to_string() + ">"; });
// py::class_<ClusterFileV2>(m, "ClusterFileV2")
// .def(py::init<const std::filesystem::path &, const std::string &>())
// .def("read", py::overload_cast<>(&ClusterFileV2::read))
// .def("read", py::overload_cast<int>(&ClusterFileV2::read))
// .def("frame_number", &ClusterFileV2::frame_number)
// .def("write", py::overload_cast<std::vector<ClusterV2> const &>(&ClusterFileV2::write))
// .def("write", py::overload_cast<std::vector<ClusterV2> const
// &>(&ClusterFileV2::write))
// .def("close", &ClusterFileV2::close);
// m.def("to_clustV2", [](std::vector<Cluster> &clusters, const int frame_number) {
// m.def("to_clustV2", [](std::vector<Cluster> &clusters, const int
// frame_number) {
// std::vector<ClusterV2> clusters_;
// for (auto &c : clusters) {
// ClusterV2 cluster;

68
src/CtbRawFile.cpp
View File

@ -1,22 +1,72 @@
#include "aare/CtbRawFile.hpp"
#include <fmt/format.h>
namespace aare {
CtbRawFile::CtbRawFile(const std::filesystem::path &fname) : m_master(fname) {
namespace aare{
CtbRawFile::CtbRawFile(const std::filesystem::path &fname){
if(!std::filesystem::exists(fname)){
throw std::runtime_error(LOCATION + "File does not exist");
if (m_master.detector_type() != DetectorType::ChipTestBoard) {
throw std::runtime_error(LOCATION + "Not a Ctb file");
}
m_fnc = parse_fname(fname);
if(!m_fnc.valid){
throw std::runtime_error(LOCATION + "Could not parse master file name");
find_subfiles();
// open the first subfile
m_file.open(m_master.data_fname(0, 0), std::ios::binary);
}
size_t CtbRawFile::tell() const { return m_current_frame; }
void CtbRawFile::seek(size_t frame_number) {
if (auto index = sub_file_index(frame_number); index != m_current_subfile) {
open_data_file(index);
}
size_t frame_number_in_file = frame_number % m_master.max_frames_per_file();
m_file.seekg((sizeof(DetectorHeader)+m_master.image_size_in_bytes()) * frame_number_in_file);
m_current_frame = frame_number;
}
void CtbRawFile::find_subfiles() {
// we can semi safely assume that there is only one module for CTB
while (std::filesystem::exists(m_master.data_fname(0, m_num_subfiles)))
m_num_subfiles++;
fmt::print("Found {} subfiles\n", m_num_subfiles);
}
void CtbRawFile::open_data_file(size_t subfile_index) {
if (subfile_index >= m_num_subfiles) {
throw std::runtime_error(LOCATION + "Subfile index out of range");
}
m_current_subfile = subfile_index;
m_file = std::ifstream(m_master.data_fname(0, subfile_index), std::ios::binary); // only one module for CTB
if (!m_file.is_open()) {
throw std::runtime_error(LOCATION + "Could not open data file");
}
}
void CtbRawFile::read_into(std::byte *image_buf, DetectorHeader* header) {
if(m_current_frame >= m_master.frames_in_file()){
throw std::runtime_error(LOCATION + "End of file reached");
}
if(m_current_frame != 0 && m_current_frame % m_master.max_frames_per_file() == 0){
open_data_file(m_current_subfile+1);
}
if(header){
m_file.read(reinterpret_cast<char *>(header), sizeof(DetectorHeader));
}else{
m_file.seekg(sizeof(DetectorHeader), std::ios::cur);
}
m_file.read(reinterpret_cast<char *>(image_buf), m_master.image_size_in_bytes());
m_current_frame++;
}
} // namespace aare

4
src/RawFile.cpp
View File

@ -45,8 +45,8 @@ void RawFile::open_subfiles() {
}
}
sls_detector_header RawFile::read_header(const std::filesystem::path &fname) {
sls_detector_header h{};
DetectorHeader RawFile::read_header(const std::filesystem::path &fname) {
DetectorHeader h{};
FILE *fp = fopen(fname.string().c_str(), "r");
if (!fp)
throw std::runtime_error(fmt::format("Could not open: {} for reading", fname.string()));

103
src/RawMasterFile.cpp Normal file
View File

@ -0,0 +1,103 @@
#include "aare/RawMasterFile.hpp"
namespace aare {
RawFileNameComponents::RawFileNameComponents(const std::filesystem::path &fname) {
m_base_path = fname.parent_path();
m_base_name = fname.stem();
m_ext = fname.extension();
//parse file index
try {
auto pos = m_base_name.rfind('_');
m_file_index = std::stoi(m_base_name.substr(pos + 1));
} catch (const std::invalid_argument &e) {
throw std::runtime_error(LOCATION + "Could not parse file index");
}
//remove master from base name
auto pos = m_base_name.find("_master_");
if (pos != std::string::npos) {
m_base_name.erase(pos);
}else{
throw std::runtime_error(LOCATION + "Could not find _master_ in file name");
}
}
void RawMasterFile::parse_json(const std::filesystem::path &fpath) {
std::ifstream ifs(fpath);
json j;
ifs >> j;
double v = j["Version"];
m_version = fmt::format("{:.1f}", v);
m_type = StringTo<DetectorType>(j["Detector Type"].get<std::string>());
m_timing_mode =
StringTo<TimingMode>(j["Timing Mode"].get<std::string>());
m_image_size_in_bytes = j["Image Size in bytes"];
m_frames_in_file = j["Frames in File"];
m_pixels_y = j["Pixels"]["y"];
m_pixels_x = j["Pixels"]["x"];
m_max_frames_per_file = j["Max Frames Per File"];
//Not all detectors write the bitdepth but in case
//its not there it is 16
try {
m_bitdepth = j.at("Dynamic Range");
} catch (const json::out_of_range &e) {
m_bitdepth = 16;
}
m_frame_padding = j["Frame Padding"];
m_frame_discard_policy = StringTo<FrameDiscardPolicy>(
j["Frame Discard Policy"].get<std::string>());
try {
m_analog_samples = j.at("Analog Samples");
}catch (const json::out_of_range &e) {
// m_analog_samples = 0;
}
// try{
// std::string adc_mask = j.at("ADC Mask");
// m_adc_mask = std::stoul(adc_mask, nullptr, 16);
// }catch (const json::out_of_range &e) {
// m_adc_mask = 0;
// }
try {
m_digital_samples = j.at("Digital Samples");
}catch (const json::out_of_range &e) {
// m_digital_samples = 0;
}
// //Update detector type for Moench
// //TODO! How does this work with old .raw master files?
// if (m_type == DetectorType::Moench && m_analog_samples == 0 &&
// m_subfile_rows == 400) {
// m_type = DetectorType::Moench03;
// }else if (m_type == DetectorType::Moench && m_subfile_rows == 400 &&
// m_analog_samples == 5000) {
// m_type = DetectorType::Moench03_old;
// }
// //Here we know we have a ChipTestBoard file update the geometry?
// //TODO! Carry on information about digtial, and transceivers
// if (m_type == DetectorType::ChipTestBoard) {
// subfile_rows = 1;
// subfile_cols = m_analog_samples*__builtin_popcount(m_adc_mask);
// }
// // only Eiger had quad
// if (m_type == DetectorType::Eiger) {
// quad = (j["Quad"] == 1);
// }
// m_geometry = {j["Geometry"]["y"], j["Geometry"]["x"]};
}
void RawMasterFile::parse_raw(const std::filesystem::path &fpath) {
throw std::runtime_error("Not implemented");
}
} // namespace aare

167
src/RawMasterFile.test.cpp Normal file
View File

@ -0,0 +1,167 @@
#include "aare/RawMasterFile.hpp"
#include <catch2/catch_test_macros.hpp>
#include "test_config.hpp"
using namespace aare;
TEST_CASE("Parse a master file fname"){
RawFileNameComponents m("test_master_1.json");
REQUIRE(m.base_name() == "test");
REQUIRE(m.ext() == ".json");
REQUIRE(m.file_index() == 1);
}
TEST_CASE("Construction of master file name and data files"){
RawFileNameComponents m("test_master_1.json");
REQUIRE(m.master_fname() == "test_master_1.json");
REQUIRE(m.data_fname(0, 0) == "test_d0_f0_1.raw");
REQUIRE(m.data_fname(1, 0) == "test_d1_f0_1.raw");
REQUIRE(m.data_fname(0, 1) == "test_d0_f1_1.raw");
REQUIRE(m.data_fname(1, 1) == "test_d1_f1_1.raw");
}
TEST_CASE("Parse a master file"){
auto fpath = test_data_path() / "jungfrau" / "jungfrau_single_master_0.json";
REQUIRE(std::filesystem::exists(fpath));
RawMasterFile f(fpath);
// "Version": 7.2,
REQUIRE(f.version() == "7.2");
// "Timestamp": "Tue Feb 20 08:28:24 2024",
// "Detector Type": "Jungfrau",
REQUIRE(f.detector_type() == DetectorType::Jungfrau);
// "Timing Mode": "auto",
REQUIRE(f.timing_mode() == TimingMode::Auto);
// "Geometry": {
// "x": 1,
// "y": 1
// },
// "Image Size in bytes": 1048576,
REQUIRE(f.image_size_in_bytes() == 1048576);
// "Pixels": {
// "x": 1024,
REQUIRE(f.pixels_x() == 1024);
// "y": 512
REQUIRE(f.pixels_y() == 512);
// },
// "Max Frames Per File": 3,
REQUIRE(f.max_frames_per_file() == 3);
//Jungfrau doesn't write but it is 16
REQUIRE(f.bitdepth() == 16);
// "Frame Discard Policy": "nodiscard",
// "Frame Padding": 1,
// "Scan Parameters": "[disabled]",
// "Total Frames": 10,
// "Receiver Roi": {
// "xmin": 4294967295,
// "xmax": 4294967295,
// "ymin": 4294967295,
// "ymax": 4294967295
// },
// "Exptime": "10us",
// "Period": "1ms",
// "Number of UDP Interfaces": 1,
// "Number of rows": 512,
// "Frames in File": 10,
// "Frame Header Format": {
// "Frame Number": "8 bytes",
// "SubFrame Number/ExpLength": "4 bytes",
// "Packet Number": "4 bytes",
// "Bunch ID": "8 bytes",
// "Timestamp": "8 bytes",
// "Module Id": "2 bytes",
// "Row": "2 bytes",
// "Column": "2 bytes",
// "Reserved": "2 bytes",
// "Debug": "4 bytes",
// "Round Robin Number": "2 bytes",
// "Detector Type": "1 byte",
// "Header Version": "1 byte",
// "Packets Caught Mask": "64 bytes"
// }
// }
REQUIRE_FALSE(f.analog_samples());
REQUIRE_FALSE(f.digital_samples());
}
TEST_CASE("Read eiger master file"){
auto fpath = test_data_path() / "eiger" / "eiger_500k_32bit_master_0.json";
REQUIRE(std::filesystem::exists(fpath));
RawMasterFile f(fpath);
// {
// "Version": 7.2,
REQUIRE(f.version() == "7.2");
// "Timestamp": "Tue Mar 26 17:24:34 2024",
// "Detector Type": "Eiger",
REQUIRE(f.detector_type() == DetectorType::Eiger);
// "Timing Mode": "auto",
REQUIRE(f.timing_mode() == TimingMode::Auto);
// "Geometry": {
// "x": 2,
// "y": 2
// },
// "Image Size in bytes": 524288,
REQUIRE(f.image_size_in_bytes() == 524288);
// "Pixels": {
// "x": 512,
REQUIRE(f.pixels_x() == 512);
// "y": 256
REQUIRE(f.pixels_y() == 256);
// },
// "Max Frames Per File": 10000,
REQUIRE(f.max_frames_per_file() == 10000);
// "Frame Discard Policy": "nodiscard",
REQUIRE(f.frame_discard_policy() == FrameDiscardPolicy::NoDiscard);
// "Frame Padding": 1,
REQUIRE(f.frame_padding() == 1);
// "Scan Parameters": "[disabled]",
// "Total Frames": 3,
// "Receiver Roi": {
// "xmin": 4294967295,
// "xmax": 4294967295,
// "ymin": 4294967295,
// "ymax": 4294967295
// },
// "Dynamic Range": 32,
// "Ten Giga": 0,
// "Exptime": "5s",
// "Period": "1s",
// "Threshold Energy": -1,
// "Sub Exptime": "2.62144ms",
// "Sub Period": "2.62144ms",
// "Quad": 0,
// "Number of rows": 256,
// "Rate Corrections": "[0, 0]",
// "Frames in File": 3,
// "Frame Header Format": {
// "Frame Number": "8 bytes",
// "SubFrame Number/ExpLength": "4 bytes",
// "Packet Number": "4 bytes",
// "Bunch ID": "8 bytes",
// "Timestamp": "8 bytes",
// "Module Id": "2 bytes",
// "Row": "2 bytes",
// "Column": "2 bytes",
// "Reserved": "2 bytes",
// "Debug": "4 bytes",
// "Round Robin Number": "2 bytes",
// "Detector Type": "1 byte",
// "Header Version": "1 byte",
// "Packets Caught Mask": "64 bytes"
// }
// }
}

12
src/SubFile.cpp
View File

@ -17,7 +17,7 @@ SubFile::SubFile(const std::filesystem::path &fname, DetectorType detector, size
}
if (std::filesystem::exists(fname)) {
n_frames = std::filesystem::file_size(fname) / (sizeof(sls_detector_header) + rows * cols * bitdepth / 8);
n_frames = std::filesystem::file_size(fname) / (sizeof(DetectorHeader) + rows * cols * bitdepth / 8);
} else {
n_frames = 0;
}
@ -42,7 +42,7 @@ size_t SubFile::get_part(std::byte *buffer, size_t frame_index) {
if (frame_index >= n_frames) {
throw std::runtime_error("Frame number out of range");
}
fseek(fp, sizeof(sls_detector_header) + (sizeof(sls_detector_header) + bytes_per_part()) * frame_index, // NOLINT
fseek(fp, sizeof(DetectorHeader) + (sizeof(DetectorHeader) + bytes_per_part()) * frame_index, // NOLINT
SEEK_SET);
if (pixel_map){
@ -63,11 +63,11 @@ size_t SubFile::get_part(std::byte *buffer, size_t frame_index) {
}
}
size_t SubFile::write_part(std::byte *buffer, sls_detector_header header, size_t frame_index) {
size_t SubFile::write_part(std::byte *buffer, DetectorHeader header, size_t frame_index) {
if (frame_index > n_frames) {
throw std::runtime_error("Frame number out of range");
}
fseek(fp, static_cast<int64_t>((sizeof(sls_detector_header) + bytes_per_part()) * frame_index), SEEK_SET);
fseek(fp, static_cast<int64_t>((sizeof(DetectorHeader) + bytes_per_part()) * frame_index), SEEK_SET);
auto wc = fwrite(reinterpret_cast<char *>(&header), sizeof(header), 1, fp);
wc += fwrite(buffer, bytes_per_part(), 1, fp);
@ -75,8 +75,8 @@ size_t SubFile::write_part(std::byte *buffer, sls_detector_header header, size_t
}
size_t SubFile::frame_number(size_t frame_index) {
sls_detector_header h{};
fseek(fp, (sizeof(sls_detector_header) + bytes_per_part()) * frame_index, SEEK_SET); // NOLINT
DetectorHeader h{};
fseek(fp, (sizeof(DetectorHeader) + bytes_per_part()) * frame_index, SEEK_SET); // NOLINT
size_t const rc = fread(reinterpret_cast<char *>(&h), sizeof(h), 1, fp);
if (rc != 1)
throw std::runtime_error(LOCATION + "Could not read header from file");

8
src/defs.cpp
View File

@ -68,6 +68,14 @@ template <> TimingMode StringTo(const std::string &arg) {
throw std::runtime_error("Could not decode timing mode from: \"" + arg + "\"");
}
template <> FrameDiscardPolicy StringTo(const std::string &arg) {
if (arg == "nodiscard")
return FrameDiscardPolicy::NoDiscard;
if (arg == "discard")
return FrameDiscardPolicy::Discard;
throw std::runtime_error("Could not decode frame discard policy from: \"" + arg + "\"");
}
// template <> TimingMode StringTo<TimingMode>(std::string mode);
} // namespace aare

33
src/file_utils.cpp
View File

@ -1,33 +0,0 @@
#include "aare/file_utils.hpp"
namespace aare {
bool is_master_file(const std::filesystem::path &fpath) {
std::string const stem = fpath.stem().string();
return stem.find("_master_") != std::string::npos;
}
FileNameComponents parse_fname(const std::filesystem::path &fname) {
FileNameComponents fnc;
fnc.base_path = fname.parent_path();
fnc.base_name = fname.stem();
fnc.ext = fname.extension();
try {
auto pos = fnc.base_name.rfind('_');
fnc.findex = std::stoi(fnc.base_name.substr(pos + 1));
} catch (const std::invalid_argument &e) {
fnc.valid = false;
}
auto pos = fnc.base_name.find("_master_");
if (pos != std::string::npos) {
fnc.base_name.erase(pos);
}else{
fnc.valid = false;
}
fnc.valid = true;
return fnc;
}
} // namespace aare

20
src/file_utils.test.cpp
View File

@ -1,20 +0,0 @@
#include "aare/file_utils.hpp"
#include <catch2/catch_test_macros.hpp>
using namespace aare;
TEST_CASE("Use filename to determine if it is a master file") {
REQUIRE(is_master_file("test_master_1.json"));
}
TEST_CASE("Parse a master file fname"){
auto fnc = parse_fname("test_master_1.json");
REQUIRE(fnc.base_name == "test");
REQUIRE(fnc.ext == ".json");
REQUIRE(fnc.findex == 1);
REQUIRE(fnc.master_fname() == "test_master_1.json");
REQUIRE(fnc.data_fname(1, 2) == "test_d2_f1_1.raw");
}