read jungfrau simple file. leak errors

This commit is contained in:
Bechir Braham 2024-02-22 15:14:16 +01:00
parent a6ff0fc794
commit 315bf6eb5d
No known key found for this signature in database
GPG Key ID: 7F511B55FD8E9671
19 changed files with 359 additions and 100 deletions

View File

@ -85,7 +85,10 @@
"__verbose_abort": "cpp",
"execution": "cpp",
"ios": "cpp",
"locale": "cpp"
"locale": "cpp",
"__tree": "cpp",
"queue": "cpp",
"stack": "cpp"
},
"C_Cpp.errorSquiggles": "enabled"
}

1
data/.gitignore vendored
View File

@ -1 +1,2 @@
*.raw

View File

@ -0,0 +1,47 @@
{
"Version": 7.2,
"Timestamp": "Tue Feb 20 08:29:19 2024",
"Detector Type": "Jungfrau",
"Timing Mode": "auto",
"Geometry": {
"x": 1,
"y": 2
},
"Image Size in bytes": 524288,
"Pixels": {
"x": 1024,
"y": 256
},
"Max Frames Per File": 3,
"Frame Discard Policy": "nodiscard",
"Frame Padding": 1,
"Scan Parameters": "[disabled]",
"Total Frames": 10,
"Receiver Roi": {
"xmin": 4294967295,
"xmax": 4294967295,
"ymin": 4294967295,
"ymax": 4294967295
},
"Exptime": "10us",
"Period": "1ms",
"Number of UDP Interfaces": 2,
"Number of rows": 512,
"Frames in File": 10,
"Frame Header Format": {
"Frame Number": "8 bytes",
"SubFrame Number/ExpLength": "4 bytes",
"Packet Number": "4 bytes",
"Bunch ID": "8 bytes",
"Timestamp": "8 bytes",
"Module Id": "2 bytes",
"Row": "2 bytes",
"Column": "2 bytes",
"Reserved": "2 bytes",
"Debug": "4 bytes",
"Round Robin Number": "2 bytes",
"Detector Type": "1 byte",
"Header Version": "1 byte",
"Packets Caught Mask": "64 bytes"
}
}

View File

@ -0,0 +1,47 @@
{
"Version": 7.2,
"Timestamp": "Tue Feb 20 08:28:24 2024",
"Detector Type": "Jungfrau",
"Timing Mode": "auto",
"Geometry": {
"x": 1,
"y": 1
},
"Image Size in bytes": 1048576,
"Pixels": {
"x": 1024,
"y": 512
},
"Max Frames Per File": 3,
"Frame Discard Policy": "nodiscard",
"Frame Padding": 1,
"Scan Parameters": "[disabled]",
"Total Frames": 10,
"Receiver Roi": {
"xmin": 4294967295,
"xmax": 4294967295,
"ymin": 4294967295,
"ymax": 4294967295
},
"Exptime": "10us",
"Period": "1ms",
"Number of UDP Interfaces": 1,
"Number of rows": 512,
"Frames in File": 10,
"Frame Header Format": {
"Frame Number": "8 bytes",
"SubFrame Number/ExpLength": "4 bytes",
"Packet Number": "4 bytes",
"Bunch ID": "8 bytes",
"Timestamp": "8 bytes",
"Module Id": "2 bytes",
"Row": "2 bytes",
"Column": "2 bytes",
"Reserved": "2 bytes",
"Debug": "4 bytes",
"Round Robin Number": "2 bytes",
"Detector Type": "1 byte",
"Header Version": "1 byte",
"Packets Caught Mask": "64 bytes"
}
}

48
data/read_frame.py Normal file
View File

@ -0,0 +1,48 @@
import numpy as np
import matplotlib.pyplot as plt
plt.ion()
header_dt = np.dtype(
[
("Frame Number", "u8"),
("SubFrame Number/ExpLength", "u4"),
("Packet Number", "u4"),
("Bunch ID", "u8"),
("Timestamp", "u8"),
("Module Id", "u2"),
("Row", "u2"),
("Column", "u2"),
("Reserved", "u2"),
("Debug", "u4"),
("Round Robin Number", "u2"),
("Detector Type", "u1"),
("Header Version", "u1"),
("Packets caught mask", "8u8")
]
)
# Read three frames from a jungfrau file with a single interface
rows = 512
cols = 1024
frames = 3
data = np.zeros((frames,rows,cols), dtype = np.uint16)
header = np.zeros(frames, dtype = header_dt)
for file_id in range(4):
file_name = 'jungfrau_single_d0_f{}_0.raw'.format(file_id)
print("Reading file:", file_name)
with open(file_name) as f:
for i in range(frames if file_id != 3 else 1):
header[i] = np.fromfile(f, dtype=header_dt, count = 1)
data[i] = np.fromfile(f, dtype=np.uint16,count = rows*cols).reshape(rows,cols)
for i in range(frames if file_id != 3 else 1 ):
print("frame:",i)
print(data[i][0,0],data[i][0,1],data[i][1,0],data[i][rows-1,cols-1])
print("")
#fig, ax = plt.subplots()
#im = ax.imshow(data[0])
#im.set_clim(2000,4000)

View File

@ -6,6 +6,7 @@
#include <string_view>
#include <stdexcept>
#include <fmt/format.h>
#include <variant>
typedef struct {
uint64_t frameNumber;
@ -29,7 +30,7 @@ struct xy {
int col;
};
using image_shape = std::array<ssize_t, 2>;
// using image_shape = std::array<ssize_t, 2>;
using dynamic_shape = std::vector<ssize_t>;
enum class DetectorType { Jungfrau, Eiger, Mythen3, Moench };
@ -45,3 +46,4 @@ template <> DetectorType StringTo(std::string);
template <> TimingMode StringTo(std::string);
using DataTypeVariants = std::variant<uint16_t, uint32_t>;

View File

@ -1 +1,3 @@
target_include_directories(aare PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
target_include_directories(aare PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
target_sources(aare PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/Frame.cpp")

View File

@ -1,3 +1,26 @@
#include "Frame.hpp"
#include <iostream>
ssize_t Frame::cols() const { return ssize_t(); }
template <class DataType>
Frame<DataType>::Frame(std::byte* bytes, ssize_t rows, ssize_t cols)
{
this->rows = rows;
this->cols = cols;
data = new DataType[rows * cols];
std::memcpy(data, bytes, sizeof(DataType) * rows * cols);
}
template <class DataType>
DataType Frame<DataType>::get(int row, int col) {
if (row < 0 || row >= rows || col < 0 || col >= cols) {
std::cerr << "Invalid row or column index" << std::endl;
return 0;
}
return data[row * cols + col];
}
template class Frame<uint16_t>;
template class Frame<uint32_t>;

View File

@ -1,3 +1,4 @@
#pragma once
#include <cstddef>
#include <sys/types.h>
#include <cstdint>
@ -5,20 +6,19 @@
#include <vector>
#include "defs.hpp"
class Frame{
ssize_t nrows{};
ssize_t ncols{};
uint8_t bitdepth_{};
std::unique_ptr<std::byte[]> data_{nullptr};
public:
ssize_t rows() const;
ssize_t cols() const;
image_shape shape() const;
uint8_t bits_per_pixel() const;
uint8_t bytes_per_pixel() const;
size_t total_bytes() const;
/**
* @brief Frame class to represent a single frame of data
* model class
* should be able to work with streams coming from files or network
*/
template <class DataType>
class Frame {
DataType* data{nullptr};
ssize_t rows{};
ssize_t cols{};
public:
Frame(std::byte* fp, ssize_t rows, ssize_t cols);
DataType get(int row, int col);
};

View File

@ -4,6 +4,11 @@
#include <filesystem>
#include <fmt/core.h>
#include "SubFile.hpp"
#include <iostream>
#include "Frame.hpp"
struct RawFileConfig {
int module_gap_row{};
@ -24,12 +29,13 @@ class File {
private:
using config = RawFileConfig;
public:
std::vector<SubFileVariants> subfiles;
std::vector<SubFileBase*> subfiles;
std::filesystem::path fname;
std::filesystem::path base_path;
std::string base_name, ext;
int findex, n_subfiles;
size_t total_frames{};
size_t max_frames_per_file{};
std::string version;
DetectorType type;
@ -40,6 +46,7 @@ class File {
ssize_t rows{};
ssize_t cols{};
uint8_t bitdepth{};
using data_type = uint16_t;
std::vector<xy> positions;
config cfg{0,0};
@ -54,12 +61,11 @@ class File {
}
// TODO! Deal with fast quad and missing files
inline void find_number_of_subfiles() {
void find_number_of_subfiles() {
int n_mod = 0;
while (std::filesystem::exists(data_fname(n_mod, 0))) {
n_mod++;
}
while (std::filesystem::exists(data_fname(++n_mod, 0)));
n_subfiles = n_mod;
}
inline std::filesystem::path master_fname() {
@ -67,8 +73,11 @@ class File {
fmt::format("{}_master_{}{}", base_name, findex, ext);
}
inline std::filesystem::path data_fname(int mod_id, int file_id) {
return base_path / fmt::format("{}_d{}_f{}_{}.raw", base_name, mod_id,
file_id, findex);
return base_path / fmt::format("{}_d{}_f{}_{}.raw", base_name, file_id,
mod_id, findex);
}
virtual Frame<uint16_t> get_frame(int frame_number)=0;
// size_t total_frames();
};

View File

@ -0,0 +1,19 @@
#include "JsonFile.hpp"
#include <typeinfo>
Frame<uint16_t> JsonFile::get_frame(int frame_number){
int subfile_id=frame_number/max_frames_per_file;
std::byte* buffer;
size_t frame_size = subfiles[subfile_id]->bytes_per_frame();
buffer = new std::byte[frame_size];
subfiles[subfile_id]->get_frame(buffer, frame_number%max_frames_per_file);
return Frame<uint16_t>(buffer, rows, cols);
}

View File

@ -1,6 +1,10 @@
#pragma once
#include "Frame.hpp"
#include "defs.hpp"
#include "File.hpp"
class JsonFile: public File
{
Frame<uint16_t> get_frame(int frame_number);
};

View File

@ -8,7 +8,10 @@
*
*/
template <class Header, class DataType>
SubFile<Header, DataType>::SubFile(std::filesystem::path fname, ssize_t rows, ssize_t cols) : rows(rows), cols(cols) {
SubFile<Header, DataType>::SubFile(std::filesystem::path fname, ssize_t rows, ssize_t cols) {
this->rows = rows;
this->cols = cols;
this->fname = fname;
fp = fopen(fname.c_str(), "rb");
if (fp == nullptr) {
throw std::runtime_error("Could not open file " + fname.string());
@ -16,8 +19,18 @@ SubFile<Header, DataType>::SubFile(std::filesystem::path fname, ssize_t rows, ss
std::cout<<"File opened"<<std::endl;
n_frames = std::filesystem::file_size(fname) / (sizeof(Header) + rows * cols * sizeof(DataType));
std::cout<<"Number of frames: "<<n_frames<<std::endl;
}
template <class Header, class DataType>
size_t SubFile<Header, DataType>::get_frame(std::byte *buffer, int frame_number) {
if (frame_number >= n_frames or frame_number < 0) {
throw std::runtime_error("Frame number out of range");
}
fseek(fp, sizeof(Header)+(sizeof(Header) + bytes_per_frame()) *frame_number, SEEK_SET);
return read_impl(buffer);
}
/**
* NormalSubFile methods
*/
@ -26,7 +39,7 @@ NormalSubFile<Header, DataType>::NormalSubFile(std::filesystem::path fname, ssiz
: SubFile<Header, DataType>(fname, rows, cols){};
template <class Header, class DataType> size_t NormalSubFile<Header, DataType>::read_impl(std::byte *buffer) {
return fread(reinterpret_cast<char *>(buffer), sizeof(DataType) * this->rows * this->cols, 1, this->fp);
return fread(buffer, this->bytes_per_frame(), 1, this->fp);
};
@ -38,7 +51,8 @@ template <class Header, class DataType>
ReorderM03SubFile<Header, DataType>::ReorderM03SubFile(std::filesystem::path fname, ssize_t rows, ssize_t cols)
: SubFile<Header, DataType>(fname, rows, cols){};
template <class Header, class DataType> size_t ReorderM03SubFile<Header, DataType>::read_impl(std::byte *buffer) {
template <class Header, class DataType>
size_t ReorderM03SubFile<Header, DataType>::read_impl(std::byte *buffer) {
std::vector<DataType> tmp(this->pixels_per_frame());
size_t rc = fread(reinterpret_cast<char *>(&tmp[0]), this->bytes_per_frame(), 1, this->fp);
@ -69,3 +83,9 @@ template <class Header, class DataType> size_t ReorderM03SubFile<Header, DataTyp
template class NormalSubFile<sls_detector_header, uint16_t>;
template class NormalSubFile<sls_detector_header, uint32_t>;
template class ReorderM03SubFile<sls_detector_header, uint16_t>;
// template size_t ReorderM03SubFile<sls_detector_header, uint16_t>::read_impl(std::byte *buffer);
// template size_t ReorderM03SubFile<sls_detector_header, uint32_t>::read_impl(std::byte *buffer);
// template size_t NormalSubFile<sls_detector_header, uint32_t>::read_impl(std::byte *buffer);
// template size_t NormalSubFile<sls_detector_header, uint16_t>::read_impl(std::byte *buffer);

View File

@ -4,22 +4,32 @@
#include <filesystem>
#include <variant>
template <class Header, class DataType> class SubFile {
class SubFileBase {
protected:
FILE *fp = nullptr;
public:
SubFile(std::filesystem::path fname, ssize_t rows, ssize_t cols);
inline size_t bytes_per_frame() const { return sizeof(DataType) * rows * cols; }
inline size_t pixels_per_frame() const { return rows * cols; }
virtual inline size_t bytes_per_frame() =0;
virtual inline size_t pixels_per_frame() =0;
std::filesystem::path fname;
ssize_t rows{};
ssize_t cols{};
ssize_t n_frames{};
int sub_file_index_{};
size_t read_impl(std::byte *buffer){};
virtual size_t read_impl(std::byte *buffer)=0;
virtual size_t get_frame(std::byte *buffer, int frame_number)=0;
};
template <class Header, class DataType> class SubFile : public SubFileBase{
public:
SubFile(std::filesystem::path fname, ssize_t rows, ssize_t cols);
inline size_t bytes_per_frame() override { return sizeof(DataType) * rows * cols; }
inline size_t pixels_per_frame() override { return rows * cols; }
virtual size_t read_impl(std::byte *buffer)=0;
size_t get_frame(std::byte *buffer, int frame_number);
};
@ -29,11 +39,6 @@ template <class Header, class DataType> class NormalSubFile : public SubFile<Hea
size_t read_impl(std::byte *buffer);
};
// template <class Header, class DataType>
// class FlipSubFile : public SubFile<Header, DataType>{
// size_t read_impl() override;
// };
template <class Header, class DataType> class ReorderM03SubFile : public SubFile<Header, DataType> {
public:
ReorderM03SubFile(std::filesystem::path fname, ssize_t rows, ssize_t cols);
@ -44,4 +49,5 @@ using JungfrauSubFile = NormalSubFile<sls_detector_header, uint16_t>;
using Moench03SubFile = ReorderM03SubFile<sls_detector_header, uint16_t>;
using Mythen3SubFile = NormalSubFile<sls_detector_header, uint32_t>;
using SubFileVariants = std::variant<JungfrauSubFile, Mythen3SubFile, Moench03SubFile>;
// using SubFileVariants = std::variant<JungfrauSubFile, Mythen3SubFile, Moench03SubFile>;

View File

@ -6,7 +6,7 @@
FileFactory FileFactory::getFactory(std::filesystem::path fpath){
FileFactory* FileFactory::getFactory(std::filesystem::path fpath){
// check if file exists
if(!std::filesystem::exists(fpath)){
throw std::runtime_error("File does not exist");
@ -18,7 +18,7 @@ FileFactory FileFactory::getFactory(std::filesystem::path fpath){
}
else if(fpath.extension() == ".json"){
std::cout<<"Loading json file"<<std::endl;
return JsonFileFactory(fpath);
return new JsonFileFactory(fpath);
}
//check if extension is numpy
else if(fpath.extension() == ".npy"){
@ -29,15 +29,15 @@ FileFactory FileFactory::getFactory(std::filesystem::path fpath){
throw std::runtime_error("Unsupported file type");
}
void FileFactory::parse_fname(File& file) {
file.base_path = fpath.parent_path();
file.base_name = fpath.stem();
file.ext = fpath.extension();
void FileFactory::parse_fname(File* file) {
file->base_path = fpath.parent_path();
file->base_name = fpath.stem();
file->ext = fpath.extension();
auto pos = file.base_name.rfind("_");
file.findex = std::stoi(file.base_name.substr(pos + 1));
pos = file.base_name.find("_master_");
file.base_name.erase(pos);
auto pos = file->base_name.rfind("_");
file->findex = std::stoi(file->base_name.substr(pos + 1));
pos = file->base_name.find("_master_");
file->base_name.erase(pos);
}
template <typename Header=sls_detector_header>
@ -56,21 +56,21 @@ template <typename Header=sls_detector_header>
}
void FileFactory::find_geometry(File& file) {
void FileFactory::find_geometry(File* file) {
uint16_t r{};
uint16_t c{};
for (int i = 0; i != file.n_subfiles; ++i) {
auto h = this->read_header(file.data_fname(i, 0));
for (int i = 0; i != file->n_subfiles; ++i) {
auto h = this->read_header(file->data_fname(i, 0));
r = std::max(r, h.row);
c = std::max(c, h.column);
file.positions.push_back({h.row, h.column});
file->positions.push_back({h.row, h.column});
}
r++;
c++;
file.rows = r * file.subfile_rows;
file.cols = c * file.subfile_cols;
file->rows = r * file->subfile_rows;
file->cols = c * file->subfile_cols;
file.rows += (r - 1) * file.cfg.module_gap_row;
file->rows += (r - 1) * file->cfg.module_gap_row;
}

View File

@ -8,14 +8,14 @@ class FileFactory{
protected:
std::filesystem::path fpath;
public:
static FileFactory getFactory(std::filesystem::path);
static FileFactory* getFactory(std::filesystem::path);
// virtual int deleteFile() = 0;
virtual File loadFile(){};//TODO: add option to load all file to memory or keep it on disk
virtual void parse_metadata(File&){};
virtual File* loadFile()=0;//TODO: add option to load all file to memory or keep it on disk
virtual void parse_metadata(File*)=0;
void find_geometry(File&);
void parse_fname(File&);
void find_geometry(File*);
void parse_fname(File*);
template <typename Header> Header read_header(const std::filesystem::path &fname);

View File

@ -1,11 +1,11 @@
#include "JsonFileFactory.hpp"
#include "JsonFile.hpp"
#include "SubFile.hpp"
#include "defs.hpp"
#include "helpers.hpp"
#include <fstream>
#include <iostream>
#include <nlohmann/json.hpp>
#include "SubFile.hpp"
using json = nlohmann::json;
@ -15,52 +15,53 @@ JsonFileFactory::JsonFileFactory(std::filesystem::path fpath) {
this->fpath = fpath;
}
void JsonFileFactory::parse_metadata(File &file) {
void JsonFileFactory::parse_metadata(File* file) {
std::cout << "Parsing metadata" << std::endl;
std::ifstream ifs(file.master_fname());
std::ifstream ifs(file->master_fname());
json j;
ifs >> j;
double v = j["Version"];
file.version = fmt::format("{:.1f}", v);
// std::string tmp;
// j["Detector Type"].get_to(tmp);
file.type = StringTo<DetectorType>(j["Detector Type"].get<std::string>());
file.timing_mode =
StringTo<TimingMode>(j["Timing Mode"].get<std::string>());
file.total_frames = j["Frames in File"];
file.subfile_cols = j["Pixels"]["x"];
file.subfile_rows = j["Pixels"]["y"];
if (file.type == DetectorType::Moench)
file.bitdepth = 16;
else
file.bitdepth = j["Dynamic Range"];
std::cout << "Version: " << v << std::endl;
file->version = fmt::format("{:.1f}", v);
file->type = StringTo<DetectorType>(j["Detector Type"].get<std::string>());
file->timing_mode = StringTo<TimingMode>(j["Timing Mode"].get<std::string>());
file->total_frames = j["Frames in File"];
file->subfile_cols = j["Pixels"]["x"];
file->subfile_rows = j["Pixels"]["y"];
file->max_frames_per_file = j["Max Frames Per File"];
try {
file->bitdepth = j.at("Dynamic Range");
} catch (const json::out_of_range &e) {
std::cerr << "master file does not have Dynamic Range. Defaulting to 16 bit" << '\n';
file->bitdepth = 16;
}
// only Eiger had quad
if (file.type == DetectorType::Eiger) {
file.quad = (j["Quad"] == 1);
if (file->type == DetectorType::Eiger) {
file->quad = (j["Quad"] == 1);
}
}
void JsonFileFactory::open_subfiles(File &file) {
for (int i = 0; i != file.n_subfiles; ++i) {
if (file.type == DetectorType::Jungfrau)
file.subfiles.push_back(JungfrauSubFile(file.data_fname(i, 0), file.subfile_rows, file.subfile_cols));
else if (file.type == DetectorType::Mythen3)
file.subfiles.push_back(Mythen3SubFile(file.data_fname(i, 0), file.subfile_rows, file.subfile_cols));
else if (file.type == DetectorType::Moench)
file.subfiles.push_back(Moench03SubFile(file.data_fname(i, 0), file.subfile_rows, file.subfile_cols));
void JsonFileFactory::open_subfiles(File* file) {
for (int i = 0; i != file->n_subfiles; ++i) {
if (file->type == DetectorType::Jungfrau) {
file->subfiles.push_back(new JungfrauSubFile(file->data_fname(i, 0), file->subfile_rows, file->subfile_cols));
} else if (file->type == DetectorType::Mythen3)
file->subfiles.push_back(new Mythen3SubFile(file->data_fname(i, 0), file->subfile_rows, file->subfile_cols));
else if (file->type == DetectorType::Moench)
file->subfiles.push_back(new Moench03SubFile(file->data_fname(i, 0), file->subfile_rows, file->subfile_cols));
else
throw std::runtime_error("File not supported");
}
}
File JsonFileFactory::loadFile() {
File* JsonFileFactory::loadFile() {
std::cout << "Loading json file" << std::endl;
JsonFile file = JsonFile();
file.fname = fpath;
JsonFile* file = new JsonFile();
file->fname = fpath;
this->parse_fname(file);
this->parse_metadata(file);
file.find_number_of_subfiles();
file->find_number_of_subfiles();
this->find_geometry(file);
this->open_subfiles(file);

View File

@ -5,10 +5,11 @@ class JsonFileFactory: public FileFactory
private:
/* data */
public:
File loadFile() override;
void parse_metadata(File&) override;
File* loadFile() override;
void parse_metadata(File*) override;
JsonFileFactory(std::filesystem::path fpath);
void open_subfiles(File&);
void open_subfiles(File*);
};

View File

@ -2,9 +2,35 @@
#include <iostream>
#include "file_io/file_factory/FileFactory.hpp"
void test(File* f,int frame_number){
std::cout << "frame number: " << frame_number << std::endl;
auto frame = f->get_frame(frame_number);
std::cout << frame.get(0,0) << ' ';
std::cout << frame.get(0,1) << ' ';
std::cout << frame.get(1,0) << ' ';
std::cout << frame.get(511,1023) << std::endl;
}
int main() {
FileFactory fileFactory=FileFactory::getFactory(std::filesystem::path("/home/l_bechir/github/aare")/"data"/"m3_master_0.json");
std::cout<<"filefactory is of instance: "<<typeid(fileFactory).name()<<std::endl;
File f = fileFactory.loadFile();
FileFactory* fileFactory=FileFactory::getFactory(std::filesystem::path("/home/l_bechir/github/aare")/"data"/"jungfrau_single_master_0.json");
File* f = fileFactory->loadFile();
test(f,0);
test(f,1);
test(f,2);
test(f,3);
test(f,4);
test(f,5);
test(f,6);
test(f,7);
test(f,8);
test(f,9);
delete fileFactory;
}