reading raw moench files (PR#3) (#32)

* reading moench and raw files

* read mythen3 files

* read multiport mythen3

* delete .vscode and .env.dev

---------

Co-authored-by: Bechir <bechir.brahem420@gmail.com>
This commit is contained in:
Bechir Braham 2024-03-27 08:58:43 +01:00 committed by GitHub
parent 937acd1138
commit bf216f55c6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
54 changed files with 483 additions and 412 deletions

View File

@ -1 +0,0 @@
export PROJECT_ROOT_DIR=/home/l_bechir/github/aare

2
.gitignore vendored
View File

@ -1,3 +1,5 @@
.vscode
.env.dev
.cproject
.project
bin/

View File

@ -1,23 +0,0 @@
{
"configurations": [
{
"name": "Linux",
"includePath": [
"${workspaceFolder}/file_io/**",
"${workspaceFolder}/core/**",
"${workspaceFolder}/tests/**",
"${workspaceFolder}/tests/**",
"/usr/include"
],
"defines": [],
"compilerPath": "/usr/bin/g++",
"cStandard": "c17",
"cppStandard": "c++17",
"intelliSenseMode": "linux-gcc-x64",
"configurationProvider": "ms-vscode.cmake-tools",
"compilerArgs": [],
"compileCommands": "/home/l_bechir/github/aare/build/compile_commands.json"
}
],
"version": 4
}

55
.vscode/launch.json vendored
View File

@ -1,55 +0,0 @@
{
"configurations": [
{
"name": "(gdb) Launch",
"type": "cppdbg",
"request": "launch",
"program": "${workspaceFolder}/build/examples/json_example",
"args": [],
"stopAtEntry": true,
"cwd": "${fileDirname}",
"environment": [{"name": "PROJECT_ROOT_DIR", "value": "/home/l_bechir/github/aare"}],
"externalConsole": false,
"MIMode": "gdb",
"setupCommands": [
{
"description": "Enable pretty-printing for gdb",
"text": "-enable-pretty-printing",
"ignoreFailures": true
},
{
"description": "Set Disassembly Flavor to Intel",
"text": "-gdb-set disassembly-flavor intel",
"ignoreFailures": true
}
]
},
{
"name": "C/C++: g++ build and debug active file",
"type": "cppdbg",
"request": "launch",
"program": "${fileDirname}/${fileBasenameNoExtension}",
"args": [],
"stopAtEntry": false,
"cwd": "${fileDirname}",
"environment": [],
"externalConsole": false,
"MIMode": "gdb",
"setupCommands": [
{
"description": "Enable pretty-printing for gdb",
"text": "-enable-pretty-printing",
"ignoreFailures": true
},
{
"description": "Set Disassembly Flavor to Intel",
"text": "-gdb-set disassembly-flavor intel",
"ignoreFailures": true
}
],
"preLaunchTask": "C/C++: g++ build active file",
"miDebuggerPath": "/usr/bin/gdb"
}
],
"version": "2.0.0"
}

103
.vscode/settings.json vendored
View File

@ -1,103 +0,0 @@
{
"cmake.configureOnOpen": true,
"files.associations": {
"stdexcept": "cpp",
"cstddef": "cpp",
"cctype": "cpp",
"clocale": "cpp",
"cmath": "cpp",
"cstdarg": "cpp",
"cstdio": "cpp",
"cstdlib": "cpp",
"ctime": "cpp",
"cwchar": "cpp",
"cwctype": "cpp",
"array": "cpp",
"atomic": "cpp",
"bit": "cpp",
"*.tcc": "cpp",
"codecvt": "cpp",
"compare": "cpp",
"concepts": "cpp",
"cstdint": "cpp",
"deque": "cpp",
"string": "cpp",
"unordered_map": "cpp",
"vector": "cpp",
"exception": "cpp",
"algorithm": "cpp",
"functional": "cpp",
"iterator": "cpp",
"memory": "cpp",
"memory_resource": "cpp",
"numeric": "cpp",
"optional": "cpp",
"random": "cpp",
"ratio": "cpp",
"string_view": "cpp",
"system_error": "cpp",
"tuple": "cpp",
"type_traits": "cpp",
"utility": "cpp",
"initializer_list": "cpp",
"iomanip": "cpp",
"iosfwd": "cpp",
"iostream": "cpp",
"istream": "cpp",
"limits": "cpp",
"new": "cpp",
"numbers": "cpp",
"ostream": "cpp",
"sstream": "cpp",
"streambuf": "cpp",
"cinttypes": "cpp",
"typeinfo": "cpp",
"filesystem": "cpp",
"csignal": "cpp",
"cstring": "cpp",
"any": "cpp",
"condition_variable": "cpp",
"forward_list": "cpp",
"list": "cpp",
"map": "cpp",
"set": "cpp",
"unordered_set": "cpp",
"fstream": "cpp",
"mutex": "cpp",
"ranges": "cpp",
"semaphore": "cpp",
"span": "cpp",
"stop_token": "cpp",
"thread": "cpp",
"valarray": "cpp",
"variant": "cpp",
"bitset": "cpp",
"charconv": "cpp",
"chrono": "cpp",
"format": "cpp",
"__bit_reference": "cpp",
"__config": "cpp",
"__hash_table": "cpp",
"__locale": "cpp",
"__node_handle": "cpp",
"__split_buffer": "cpp",
"__threading_support": "cpp",
"__verbose_abort": "cpp",
"execution": "cpp",
"ios": "cpp",
"locale": "cpp",
"__tree": "cpp",
"queue": "cpp",
"stack": "cpp",
"shared_mutex": "cpp",
"cfenv": "cpp",
"complex": "cpp",
"regex": "cpp",
"source_location": "cpp",
"future": "cpp",
"typeindex": "cpp",
"*.in": "c",
"stdfloat": "cpp"
},
"C_Cpp.errorSquiggles": "enabled"
}

25
.vscode/tasks.json vendored
View File

@ -1,25 +0,0 @@
{
"tasks": [
{
"type": "cppbuild",
"label": "C/C++: g++ build active file",
"command": "/usr/bin/g++",
"args": [
"-fdiagnostics-color=always",
"-g",
"${file}",
"-o",
"${fileDirname}/${fileBasenameNoExtension}"
],
"options": {
"cwd": "${fileDirname}"
},
"problemMatcher": [
"$gcc"
],
"group": "build",
"detail": "Task generated by Debugger."
}
],
"version": "2.0.0"
}

View File

@ -11,7 +11,7 @@ add_library(core STATIC ${SourceFiles})
target_include_directories(core PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
target_link_libraries(core PUBLIC fmt::fmt PRIVATE aare_compiler_flags)
target_link_libraries(core PUBLIC fmt::fmt PRIVATE aare_compiler_flags utils)
if (AARE_PYTHON_BINDINGS)
set_property(TARGET core PROPERTY POSITION_INDEPENDENT_CODE ON)
@ -29,5 +29,5 @@ if(AARE_TESTS)
)
target_sources(tests PRIVATE ${TestSources} )
target_link_libraries(tests PRIVATE core)
target_link_libraries(tests PRIVATE core )
endif()

View File

@ -34,7 +34,7 @@ struct xy {
// using image_shape = std::array<ssize_t, 2>;
using dynamic_shape = std::vector<ssize_t>;
enum class DetectorType { Jungfrau, Eiger, Mythen3, Moench };
enum class DetectorType { Jungfrau, Eiger, Mythen3, Moench,ChipTestBoard };
enum class TimingMode {Auto, Trigger};

View File

@ -1,4 +1,5 @@
#include "aare/Frame.hpp"
#include "aare/utils/logger.hpp"
#include <iostream>
#include <cassert>
Frame::Frame(std::byte* bytes, ssize_t rows, ssize_t cols, ssize_t bitdepth):

View File

@ -10,13 +10,14 @@ template <> std::string toString(DetectorType type) {
return "Mythen3";
case DetectorType::Moench:
return "Moench";
case DetectorType::ChipTestBoard:
return "ChipTestBoard";
default:
return "Unknown";
}
}
template <> DetectorType StringTo(std::string name) {
template <> DetectorType StringTo(std::string name) {
if (name == "Jungfrau")
return DetectorType::Jungfrau;
else if (name == "Eiger")
@ -25,18 +26,20 @@ template <> std::string toString(DetectorType type) {
return DetectorType::Mythen3;
else if (name == "Moench")
return DetectorType::Moench;
else if (name == "ChipTestBoard")
return DetectorType::ChipTestBoard;
else {
auto msg = fmt::format("Could not decode dector from: \"{}\"", name);
throw std::runtime_error(msg);
}
}
template <> TimingMode StringTo(std::string mode){
template <> TimingMode StringTo(std::string mode) {
if (mode == "auto")
return TimingMode::Auto;
else if(mode == "trigger")
else if (mode == "trigger")
return TimingMode::Trigger;
else{
else {
auto msg = fmt::format("Could not decode timing mode from: \"{}\"", mode);
throw std::runtime_error(msg);
}

View File

@ -0,0 +1,80 @@
import numpy as np
import matplotlib.pyplot as plt
plt.ion()
header_dt = np.dtype(
[
("Frame Number", "u8"),
("SubFrame Number/ExpLength", "u4"),
("Packet Number", "u4"),
("Bunch ID", "u8"),
("Timestamp", "u8"),
("Module Id", "u2"),
("Row", "u2"),
("Column", "u2"),
("Reserved", "u2"),
("Debug", "u4"),
("Round Robin Number", "u2"),
("Detector Type", "u1"),
("Header Version", "u1"),
("Packets caught mask", "8u8")
]
)
# Read three frames from a jungfrau file with a single interface
frames = 1
parts = 2
frame_cols = 1024
frame_rows = 512
part_cols = 1024
part_rows = 256
parts_data = np.zeros((frames,parts,part_rows,part_cols), dtype = np.uint16)
data = np.zeros((frames,frame_rows,frame_cols), dtype = np.uint16)
header = np.zeros((frames,parts), dtype = header_dt)
for frame in range(frames):
for part in range(parts):
file_name = f'jungfrau_double_d{part}_f{frame}_{0}.raw'
print("Reading file:", file_name)
with open(file_name) as f:
header[frame,part] = np.fromfile(f, dtype=header_dt, count = 1)
parts_data[frame,part] = np.fromfile(f, dtype=np.uint16,count = part_rows*part_cols).reshape(part_rows,part_cols)
data[frame] = np.concatenate((parts_data[frame,0],parts_data[frame,1]),axis=0)
# for frame in range(frames):
# print("Frame:", frame)
# print("Data:\n", data[frame])
# print(data[0,0,0])
# print(data[0,0,1])
# print(data[0,0,50])
print(data[0,0,0])
print(data[0,0,1])
print(data[0,255,1023])
print(data[0,511,1023])
# print()
# print(parts_data[0,0,0,0])
# print(parts_data[0,0,0,1])
# print(parts_data[0,0,1,0])
# print(data.shape)
#fig, ax = plt.subplots()
#im = ax.imshow(data[0])
#im.set_clim(2000,4000)

View File

@ -0,0 +1,40 @@
Version : 6.4
TimeStamp : Wed Aug 31 09:08:49 2022
Detector Type : ChipTestBoard
Timing Mode : auto
Geometry : [1, 1]
Image Size : 360000 bytes
Pixels : [96, 1]
Max Frames Per File : 20000
Frame Discard Policy : nodiscard
Frame Padding : 1
Scan Parameters : [disabled]
Total Frames : 100
Exptime : 100us
Period : 4ms
Ten Giga : 1
ADC Mask : 0xffffffff
Analog Flag : 1
Analog Samples : 5000
Digital Flag : 1
Digital Samples : 5000
Dbit Offset : 0
Dbit Bitset : 0
Frames in File : 100
#Frame Header
Frame Number : 8 bytes
SubFrame Number/ExpLength : 4 bytes
Packet Number : 4 bytes
Bunch ID : 8 bytes
Timestamp : 8 bytes
Module Id : 2 bytes
Row : 2 bytes
Column : 2 bytes
Reserved : 2 bytes
Debug : 4 bytes
Round Robin Number : 2 bytes
Detector Type : 1 byte
Header Version : 1 byte
Packets Caught Mask : 64 bytes

BIN
data/mythen/scan242_d0_f0_3.raw Executable file

Binary file not shown.

BIN
data/mythen/scan242_d1_f0_3.raw Executable file

Binary file not shown.

BIN
data/mythen/scan242_d2_f0_3.raw Executable file

Binary file not shown.

BIN
data/mythen/scan242_d3_f0_3.raw Executable file

Binary file not shown.

View File

@ -0,0 +1,40 @@
Version : 6.2
TimeStamp : Mon Sep 14 16:21:09 2020
Detector Type : Mythen3
Timing Mode : auto
Geometry : [4, 1]
Image Size : 5120 bytes
Pixels : [1280, 1]
Max Frames Per File : 10000
Frame Discard Policy : nodiscard
Frame Padding : 1
Scan Parameters : [disabled]
Total Frames : 1
Dynamic Range : 32
Ten Giga : 1
Period : 0ns
Counter Mask : 0x1
Exptime1 : 0.2s
Exptime2 : 0.2s
Exptime3 : 0.2s
GateDelay1 : 0.1s
GateDelay2 : 0.1s
GateDelay3 : 0.1s
Gates : 1
#Frame Header
Frame Number : 8 bytes
SubFrame Number/ExpLength : 4 bytes
Packet Number : 4 bytes
Bunch ID : 8 bytes
Timestamp : 8 bytes
Module Id : 2 bytes
Row : 2 bytes
Column : 2 bytes
Reserved : 2 bytes
Debug : 4 bytes
Round Robin Number : 2 bytes
Detector Type : 1 byte
Header Version : 1 byte
Packets Caught Mask : 64 bytes

View File

@ -1,4 +1,5 @@
set(EXAMPLE_LIST "json_example;logger_example;numpy_example;multiport_example")
set(EXAMPLE_LIST "json_example;logger_example;numpy_example;multiport_example;raw_example;mythen_example")
foreach(example ${EXAMPLE_LIST})
add_executable(${example} ${example}.cpp)
target_link_libraries(${example} PUBLIC aare PRIVATE aare_compiler_flags)

View File

@ -17,8 +17,7 @@ void test(File *f, int frame_number) {
int main() {
ContextManager ctx_manager;
auto PROJECT_ROOT_DIR = std::filesystem::path(getenv(AARE_ROOT_DIR_VAR));
std::filesystem::path fpath(PROJECT_ROOT_DIR / "data" / "jungfrau_single_master_0.json");
// std::filesystem::path fpath(PROJECT_ROOT_DIR / "data" / "test_numpy_file.npy");
std::filesystem::path fpath(PROJECT_ROOT_DIR / "data" /"jungfrau"/ "jungfrau_single_master_0.json");
std::cout << fpath << std::endl;
@ -26,6 +25,4 @@ int main() {
test(file, 0);
test(file, 2);
test(file, 9);
aare::logger::debug(LOCATION,"Hello", "World");
}

View File

@ -1,7 +1,7 @@
// Your First C++ Program
#include "aare/ContextManager.hpp"
#include <iostream>
#include "aare/utils/logger.hpp"
#include <iostream>
#define AARE_ROOT_DIR_VAR "PROJECT_ROOT_DIR"
@ -9,26 +9,18 @@ void test(File *f, int frame_number) {
std::cout << "frame number: " << frame_number << std::endl;
Frame frame = f->get_frame(frame_number);
std::cout << *((uint16_t *)frame.get(0, 0)) << std::endl;
std::cout << *((uint16_t *)frame.get(0,1)) << std::endl;
std::cout << *((uint16_t *)frame.get(0, 1)) << std::endl;
std::cout << *((uint16_t *)frame.get(255, 1023)) << std::endl;
std::cout << *((uint16_t *)frame.get(511, 1023)) << std::endl;
}
int main() {
ContextManager ctx_manager;
auto PROJECT_ROOT_DIR = std::filesystem::path(getenv(AARE_ROOT_DIR_VAR));
std::filesystem::path fpath(PROJECT_ROOT_DIR / "data" / "jungfrau_double_master_0.json");
std::filesystem::path fpath(PROJECT_ROOT_DIR / "data" / "jungfrau" / "jungfrau_double_master_0.json");
std::cout << fpath << std::endl;
File *file = ctx_manager.get_file(fpath);
test(file, 0);
test(file, 9);
}

View File

@ -0,0 +1,48 @@
// Your First C++ Program
#include "aare/ContextManager.hpp"
#include "aare/utils/logger.hpp"
#include <iostream>
#define AARE_ROOT_DIR_VAR "PROJECT_ROOT_DIR"
void test1(File *f, int frame_number) {
std::cout << "frame number: " << frame_number << std::endl;
Frame frame = f->get_frame(frame_number);
std::cout << *((uint32_t *)frame.get(0, 0)) << std::endl;
std::cout << *((uint32_t *)frame.get(0, 1)) << std::endl;
std::cout << *((uint32_t *)frame.get(0, 3839)) << std::endl;
for (int i = 0; i < 3840; i++) {
uint16_t x = *((uint32_t *)frame.get(0, i));
if (x != i) {
aare::logger::error("error at i", i, "x", x);
}
}
}
void test2(File *f, int frame_number) {
std::cout << "frame number: " << frame_number << std::endl;
Frame frame = f->get_frame(frame_number);
std::cout << *((uint32_t *)frame.get(0, 0)) << std::endl;
std::cout << *((uint32_t *)frame.get(0, 1)) << std::endl;
std::cout << *((uint32_t *)frame.get(0, 1280*4 -1)) << std::endl;
}
int main() {
ContextManager ctx_manager;
auto PROJECT_ROOT_DIR = std::filesystem::path(getenv(AARE_ROOT_DIR_VAR));
if (PROJECT_ROOT_DIR.empty()) {
throw std::runtime_error("environment variable PROJECT_ROOT_DIR is not set");
}
std::filesystem::path fpath(PROJECT_ROOT_DIR / "data" / "mythen" / "m3_master_0.json");
File *file = ctx_manager.get_file(fpath);
test1(file, 0);
fpath = (PROJECT_ROOT_DIR / "data" / "mythen" / "scan242_master_3.raw");
file = ctx_manager.get_file(fpath);
test2(file, 0);
}

View File

@ -4,29 +4,24 @@
#define AARE_ROOT_DIR_VAR "PROJECT_ROOT_DIR"
void test(File* f, int frame_number) {
void test(File *f, int frame_number) {
std::cout << "frame number: " << frame_number << std::endl;
Frame frame = f->get_frame(frame_number);
std::cout << *((uint16_t *)frame.get(0, 0)) << std::endl;
std::cout << *((uint16_t *)frame.get(0, 1)) << std::endl;
std::cout << *((uint16_t *)frame.get(1, 0)) << std::endl;
std::cout << *((uint16_t *)frame.get(49, 49)) << std::endl;
}
int main() {
ContextManager ctx_manager;
auto PROJECT_ROOT_DIR = std::filesystem::path(getenv(AARE_ROOT_DIR_VAR));
// std::filesystem::path fpath(PROJECT_ROOT_DIR / "data" / "jungfrau_single_master_0.json");
std::filesystem::path fpath(PROJECT_ROOT_DIR / "data" / "test_numpy_file.npy");
std::filesystem::path fpath(PROJECT_ROOT_DIR / "data" / "numpy" / "test_numpy_file.npy");
std::cout << fpath << std::endl;
File* file = ctx_manager.get_file(fpath);
File *file = ctx_manager.get_file(fpath);
test(file, 0);
test(file, 2);
test(file, 24);
}

27
examples/raw_example.cpp Normal file
View File

@ -0,0 +1,27 @@
// Your First C++ Program
#include "aare/ContextManager.hpp"
#include <iostream>
#include "aare/utils/logger.hpp"
#define AARE_ROOT_DIR_VAR "PROJECT_ROOT_DIR"
void test(File *f, int frame_number) {
std::cout << "frame number: " << frame_number << std::endl;
Frame frame = f->get_frame(frame_number);
std::cout << *((uint16_t *)frame.get(0, 0)) << std::endl;
std::cout << *((uint16_t *)frame.get(0, 1)) << std::endl;
std::cout << *((uint16_t *)frame.get(0, 95)) << std::endl;
}
int main() {
ContextManager ctx_manager;
auto PROJECT_ROOT_DIR = std::filesystem::path(getenv(AARE_ROOT_DIR_VAR));
if (PROJECT_ROOT_DIR.empty()) {
throw std::runtime_error("environment variable PROJECT_ROOT_DIR is not set");
}
std::filesystem::path fpath(PROJECT_ROOT_DIR / "data" /"moench"/ "moench04_noise_200V_sto_both_100us_no_light_thresh_900_master_0.raw");
File *file = ctx_manager.get_file(fpath);
test(file, 0);
test(file, 2);
test(file, 99);
}

View File

@ -3,8 +3,8 @@ abstract class FileFactory{
{static} +getFactory(path):FileFactory
{abstract} +loadFile(path):File
}
class JsonFileFactory{
+loadFile(path):JsonFile
class RawFileFactory{
+loadFile(path):RawFile
}
class RawFileFactory{
+loadFile(path):RawFile
@ -16,7 +16,7 @@ class NumpyFileFactory{
abstract File{
}
class JsonFile{
class RawFile{
}
class RawFile{
}
@ -25,9 +25,9 @@ class NumpyFile{
FileFactory <|-- RawFileFactory
FileFactory <|-- NumpyFileFactory
FileFactory <|-- JsonFileFactory
FileFactory <|-- RawFileFactory
File <|-- JsonFile
File <|-- RawFile
File <|-- RawFile
File <|-- NumpyFile

View File

@ -8,8 +8,8 @@ set(SourceFiles
${CMAKE_CURRENT_SOURCE_DIR}/src/File.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/FileFactory.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/helpers.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/JsonFile.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/JsonFileFactory.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/RawFile.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/RawFileFactory.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/SubFile.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyFile.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyFileFactory.cpp

View File

@ -1,18 +0,0 @@
#include "aare/FileFactory.hpp"
#include "aare/JsonFile.hpp"
class JsonFileFactory: public FileFactory
{
private:
/* data */
public:
JsonFileFactory(std::filesystem::path fpath);
virtual JsonFile* load_file() override;
void parse_metadata(File*) override;
void parse_fname(File*) override;
void open_subfiles(File*);
sls_detector_header read_header(const std::filesystem::path &fname);
void find_geometry(File*);
};

View File

@ -4,7 +4,7 @@
#include "aare/File.hpp"
class JsonFile : public File {
class RawFile : public File {
using config = RawFileConfig;
public:
@ -38,5 +38,5 @@ class JsonFile : public File {
return this->base_path / fmt::format("{}_d{}_f{}_{}.raw", this->base_name, file_id, mod_id, this->findex);
}
~JsonFile();
~RawFile();
};

View File

@ -1,8 +1,16 @@
#include "aare/File.hpp"
#include <filesystem>
class RawFileFactory{
public:
// RawFileFactory();
// ~RawFileFactory();
File loadFile(std::filesystem::path fpath);
};
#include "aare/FileFactory.hpp"
#include "aare/RawFile.hpp"
class RawFileFactory : public FileFactory {
private:
void parse_json_metadata(RawFile *file);
void parse_raw_metadata(RawFile *file);
public:
RawFileFactory(std::filesystem::path fpath);
virtual RawFile *load_file() override;
void parse_metadata(File *) override;
void parse_fname(File *) override;
void open_subfiles(File *);
sls_detector_header read_header(const std::filesystem::path &fname);
void find_geometry(File *);
};

View File

@ -19,6 +19,8 @@ class SubFile {
std::map<std::pair<DetectorType, int>, pfunc> read_impl_map = {
{{DetectorType::Moench, 16}, &SubFile::read_impl_reorder<uint16_t>},
{{DetectorType::Jungfrau, 16}, &SubFile::read_impl_normal},
{{DetectorType::ChipTestBoard,16}, &SubFile::read_impl_normal},
{{DetectorType::Mythen3, 32}, &SubFile::read_impl_normal}
};

View File

@ -1,7 +1,8 @@
#include "aare/FileFactory.hpp"
#include "aare/File.hpp"
#include "aare/JsonFileFactory.hpp"
#include "aare/RawFileFactory.hpp"
#include "aare/NumpyFileFactory.hpp"
#include "aare/utils/logger.hpp"
#include <iostream>
FileFactory *FileFactory::get_factory(std::filesystem::path fpath) {
@ -10,16 +11,13 @@ FileFactory *FileFactory::get_factory(std::filesystem::path fpath) {
throw std::runtime_error("File does not exist");
}
if (fpath.extension() == ".raw") {
std::cout << "Loading raw file" << std::endl;
throw std::runtime_error("Raw file not implemented");
} else if (fpath.extension() == ".json") {
std::cout << "Loading json file" << std::endl;
return new JsonFileFactory(fpath);
}
if (fpath.extension() == ".raw" || fpath.extension() == ".json"){
aare::logger::info("Loading",fpath.extension(),"file");
return new RawFileFactory(fpath);
}
// check if extension is numpy
else if (fpath.extension() == ".npy") {
std::cout << "Loading numpy file" << std::endl;
aare::logger::info("Loading numpy file");
return new NumpyFileFactory(fpath);
}

View File

@ -1,119 +0,0 @@
#include "aare/JsonFileFactory.hpp"
#include "aare/JsonFile.hpp"
#include "aare/SubFile.hpp"
#include "aare/defs.hpp"
#include "aare/helpers.hpp"
#include "aare/utils/logger.hpp"
#include <fstream>
#include <iostream>
#include <nlohmann/json.hpp>
using json = nlohmann::json;
JsonFileFactory::JsonFileFactory(std::filesystem::path fpath) {
if (not is_master_file(fpath))
throw std::runtime_error("Json file is not a master file");
this->m_fpath = fpath;
}
void JsonFileFactory::parse_metadata(File *_file) {
auto file = dynamic_cast<JsonFile *>(_file);
std::ifstream ifs(file->master_fname());
json j;
ifs >> j;
double v = j["Version"];
std::cout << "Version: " << v << std::endl;
file->version = fmt::format("{:.1f}", v);
file->type = StringTo<DetectorType>(j["Detector Type"].get<std::string>());
file->timing_mode = StringTo<TimingMode>(j["Timing Mode"].get<std::string>());
file->total_frames = j["Frames in File"];
file->subfile_rows = j["Pixels"]["y"];
file->subfile_cols = j["Pixels"]["x"];
file->max_frames_per_file = j["Max Frames Per File"];
try {
file->bitdepth = j.at("Dynamic Range");
} catch (const json::out_of_range &e) {
file->bitdepth = 16;
}
// only Eiger had quad
if (file->type == DetectorType::Eiger) {
file->quad = (j["Quad"] == 1);
}
file->geometry = {j["Geometry"]["y"], j["Geometry"]["x"]};
file->n_subfile_parts = file->geometry.row * file->geometry.col;
}
void JsonFileFactory::open_subfiles(File *_file) {
auto file = dynamic_cast<JsonFile *>(_file);
for (size_t i = 0; i != file->n_subfiles; ++i) {
auto v = std::vector<SubFile *>(file->n_subfile_parts);
for (size_t j = 0; j != file->n_subfile_parts; ++j) {
v[j]=new SubFile(file->data_fname(i, j), file->type, file->subfile_rows,
file->subfile_cols, file->bitdepth);
}
file->subfiles.push_back(v);
}
}
JsonFile *JsonFileFactory::load_file() {
JsonFile *file = new JsonFile();
file->fname = this->m_fpath;
this->parse_fname(file);
this->parse_metadata(file);
file->find_number_of_subfiles();
this->find_geometry(file);
this->open_subfiles(file);
return file;
}
sls_detector_header JsonFileFactory::read_header(const std::filesystem::path &fname) {
sls_detector_header h{};
FILE *fp = fopen(fname.c_str(), "r");
if (!fp)
throw std::runtime_error(fmt::format("Could not open: {} for reading", fname.c_str()));
size_t rc = fread(reinterpret_cast<char *>(&h), sizeof(h), 1, fp);
fclose(fp);
if (rc != 1)
throw std::runtime_error("Could not read header from file");
return h;
}
void JsonFileFactory::find_geometry(File *_file) {
auto file = dynamic_cast<JsonFile *>(_file);
uint16_t r{};
uint16_t c{};
for (size_t i = 0; i < file->n_subfile_parts; i++) {
for (size_t j = 0; j != file->n_subfiles; ++j) {
auto h = this->read_header(file->data_fname(j, i));
r = std::max(r, h.row);
c = std::max(c, h.column);
file->positions.push_back({h.row, h.column});
}
}
r++;
c++;
file->rows = r * file->subfile_rows;
file->cols = c * file->subfile_cols;
file->rows += (r - 1) * file->cfg.module_gap_row;
}
void JsonFileFactory::parse_fname(File *file) {
file->base_path = this->m_fpath.parent_path();
file->base_name = this->m_fpath.stem();
file->ext = this->m_fpath.extension();
auto pos = file->base_name.rfind("_");
file->findex = std::stoi(file->base_name.substr(pos + 1));
pos = file->base_name.find("_master_");
file->base_name.erase(pos);
}

View File

@ -1,7 +1,7 @@
#include "aare/JsonFile.hpp"
#include "aare/RawFile.hpp"
#include "aare/utils/logger.hpp"
Frame JsonFile::get_frame(size_t frame_number) {
Frame RawFile::get_frame(size_t frame_number) {
if (frame_number > this->total_frames) {
throw std::runtime_error("Frame number out of range");
}
@ -39,7 +39,7 @@ Frame JsonFile::get_frame(size_t frame_number) {
return f;
}
JsonFile::~JsonFile() {
RawFile::~RawFile() {
for (auto &vec : subfiles) {
for (auto subfile : vec) {
delete subfile;

View File

@ -0,0 +1,182 @@
#include "aare/RawFileFactory.hpp"
#include "aare/RawFile.hpp"
#include "aare/SubFile.hpp"
#include "aare/defs.hpp"
#include "aare/helpers.hpp"
#include "aare/utils/logger.hpp"
#include <fstream>
#include <iostream>
#include <nlohmann/json.hpp>
using json = nlohmann::json;
RawFileFactory::RawFileFactory(std::filesystem::path fpath) {
if (not is_master_file(fpath))
throw std::runtime_error("Json file is not a master file");
this->m_fpath = fpath;
}
void RawFileFactory::parse_metadata(File *_file) {
auto file = dynamic_cast<RawFile *>(_file);
if (file->ext == ".raw") {
this->parse_raw_metadata(file);
if (file->bitdepth == 0) {
switch (file->type) {
case DetectorType::Eiger:
file->bitdepth = 32;
break;
default:
file->bitdepth = 16;
}
}
} else if (file->ext == ".json") {
this->parse_json_metadata(file);
} else {
throw std::runtime_error("Unsupported file type");
}
file->n_subfile_parts = file->geometry.row * file->geometry.col;
}
void RawFileFactory::parse_raw_metadata(RawFile *file) {
std::ifstream ifs(file->master_fname());
for (std::string line; std::getline(ifs, line);) {
if (line == "#Frame Header")
break;
auto pos = line.find(":");
auto key_pos = pos;
while (key_pos != std::string::npos && std::isspace(line[--key_pos]))
;
if (key_pos != std::string::npos) {
auto key = line.substr(0, key_pos + 1);
auto value = line.substr(pos + 2);
// do the actual parsing
if (key == "Version") {
file->version = value;
} else if (key == "TimeStamp") {
} else if (key == "Detector Type") {
file->type = StringTo<DetectorType>(value);
} else if (key == "Timing Mode") {
file->timing_mode = StringTo<TimingMode>(value);
} else if (key == "Pixels") {
// Total number of pixels cannot be found yet looking at
// submodule
pos = value.find(',');
file->subfile_cols = std::stoi(value.substr(1, pos));
file->subfile_rows = std::stoi(value.substr(pos + 1));
} else if (key == "Total Frames") {
file->total_frames = std::stoi(value);
} else if (key == "Dynamic Range") {
file->bitdepth = std::stoi(value);
} else if (key == "Quad") {
file->quad = (value == "1");
} else if (key == "Max Frames Per File") {
file->max_frames_per_file = std::stoi(value);
} else if (key == "Geometry") {
pos = value.find(',');
file->geometry = {std::stoi(value.substr(1, pos)), std::stoi(value.substr(pos + 1))};
}
}
}
}
void RawFileFactory::parse_json_metadata(RawFile *file) {
std::ifstream ifs(file->master_fname());
json j;
ifs >> j;
double v = j["Version"];
std::cout << "Version: " << v << std::endl;
file->version = fmt::format("{:.1f}", v);
file->type = StringTo<DetectorType>(j["Detector Type"].get<std::string>());
file->timing_mode = StringTo<TimingMode>(j["Timing Mode"].get<std::string>());
file->total_frames = j["Frames in File"];
file->subfile_rows = j["Pixels"]["y"];
file->subfile_cols = j["Pixels"]["x"];
file->max_frames_per_file = j["Max Frames Per File"];
try {
file->bitdepth = j.at("Dynamic Range");
} catch (const json::out_of_range &e) {
file->bitdepth = 16;
}
// only Eiger had quad
if (file->type == DetectorType::Eiger) {
file->quad = (j["Quad"] == 1);
}
file->geometry = {j["Geometry"]["y"], j["Geometry"]["x"]};
}
void RawFileFactory::open_subfiles(File *_file) {
auto file = dynamic_cast<RawFile *>(_file);
for (size_t i = 0; i != file->n_subfiles; ++i) {
auto v = std::vector<SubFile *>(file->n_subfile_parts);
for (size_t j = 0; j != file->n_subfile_parts; ++j) {
v[j] =
new SubFile(file->data_fname(i, j), file->type, file->subfile_rows, file->subfile_cols, file->bitdepth);
}
file->subfiles.push_back(v);
}
}
RawFile *RawFileFactory::load_file() {
RawFile *file = new RawFile();
file->fname = this->m_fpath;
this->parse_fname(file);
this->parse_metadata(file);
file->find_number_of_subfiles();
this->find_geometry(file);
this->open_subfiles(file);
return file;
}
sls_detector_header RawFileFactory::read_header(const std::filesystem::path &fname) {
sls_detector_header h{};
FILE *fp = fopen(fname.c_str(), "r");
if (!fp)
throw std::runtime_error(fmt::format("Could not open: {} for reading", fname.c_str()));
size_t rc = fread(reinterpret_cast<char *>(&h), sizeof(h), 1, fp);
fclose(fp);
if (rc != 1)
throw std::runtime_error("Could not read header from file");
return h;
}
void RawFileFactory::find_geometry(File *_file) {
auto file = dynamic_cast<RawFile *>(_file);
uint16_t r{};
uint16_t c{};
for (size_t i = 0; i < file->n_subfile_parts; i++) {
for (size_t j = 0; j != file->n_subfiles; ++j) {
auto h = this->read_header(file->data_fname(j, i));
r = std::max(r, h.row);
c = std::max(c, h.column);
file->positions.push_back({h.row, h.column});
}
}
r++;
c++;
file->rows = r * file->subfile_rows;
file->cols = c * file->subfile_cols;
file->rows += (r - 1) * file->cfg.module_gap_row;
}
void RawFileFactory::parse_fname(File *file) {
file->base_path = this->m_fpath.parent_path();
file->base_name = this->m_fpath.stem();
file->ext = this->m_fpath.extension();
auto pos = file->base_name.rfind("_");
file->findex = std::stoi(file->base_name.substr(pos + 1));
pos = file->base_name.find("_master_");
file->base_name.erase(pos);
}

View File

@ -12,7 +12,7 @@ SubFile::SubFile(std::filesystem::path fname, DetectorType detector, ssize_t row
this->m_bitdepth = bitdepth;
this->n_frames = std::filesystem::file_size(fname) / (sizeof(sls_detector_header) + rows * cols * bitdepth / 8);
if (read_impl_map.find({detector, bitdepth}) == read_impl_map.end()) {
throw std::runtime_error("Unsupported detector/bitdepth combination");
throw std::runtime_error(LOCATION+"Unsupported detector/bitdepth combination");
}
this->read_impl = read_impl_map.at({detector, bitdepth});

View File

@ -1,17 +1,16 @@
#include <catch2/catch_test_macros.hpp>
#include <fstream>
#include <filesystem>
#include <fstream>
#include "test_config.hpp"
TEST_CASE("Test suite can find data assets"){
auto fpath = test_data_path()/ "test_numpy_file.npy";
TEST_CASE("Test suite can find data assets") {
auto fpath = test_data_path() / "numpy" / "test_numpy_file.npy";
REQUIRE(std::filesystem::exists(fpath));
}
TEST_CASE("Test suite can open data assets"){
auto fpath = test_data_path()/ "test_numpy_file.npy";
TEST_CASE("Test suite can open data assets") {
auto fpath = test_data_path() / "numpy" / "test_numpy_file.npy";
auto f = std::ifstream(fpath, std::ios::binary);
REQUIRE(f.is_open());
}