diff --git a/.env.dev b/.env similarity index 100% rename from .env.dev rename to .env diff --git a/core/include/aare/core/defs.hpp b/core/include/aare/core/defs.hpp index 73e9688..2249075 100644 --- a/core/include/aare/core/defs.hpp +++ b/core/include/aare/core/defs.hpp @@ -46,15 +46,16 @@ struct sls_detector_header { }; struct xy { - int row; - int col; + size_t row; + size_t col; bool operator==(const xy &other) const { return row == other.row && col == other.col; } bool operator!=(const xy &other) const { return !(*this == other); } + std::string to_string() const { return "{ x: " + std::to_string(row) + " y: " + std::to_string(col) + " }"; } }; using dynamic_shape = std::vector; -enum class DetectorType { Jungfrau, Eiger, Mythen3, Moench, ChipTestBoard }; +enum class DetectorType { Jungfrau, Eiger, Mythen3, Moench, ChipTestBoard, Unknown }; enum class TimingMode { Auto, Trigger }; @@ -69,17 +70,4 @@ template <> TimingMode StringTo(const std::string & /*mode*/); using DataTypeVariants = std::variant; -struct RawFileConfig { - int module_gap_row{}; - int module_gap_col{}; - - bool operator==(const RawFileConfig &other) const { - if (module_gap_col != other.module_gap_col) - return false; - if (module_gap_row != other.module_gap_row) - return false; - return true; - } -}; - } // namespace aare \ No newline at end of file diff --git a/data/jungfrau/read_frame.py b/data/jungfrau/read_frame.py index ee5ceaf..9413ccc 100644 --- a/data/jungfrau/read_frame.py +++ b/data/jungfrau/read_frame.py @@ -22,14 +22,15 @@ header_dt = np.dtype( ) # Read three frames from a jungfrau file with a single interface -rows = 512 -cols = 1024 -frames = 10 +rows = 1024 +cols = 512 +frames = 1 data = np.zeros((frames,rows,cols), dtype = np.uint16) header = np.zeros(frames, dtype = header_dt) -for file_id in range(4): - file_name = 'jungfrau_single_d0_f{}_0.raw'.format(file_id) +for frame in range(frames): + + file_name = '/tmp/raw_example_writing_master_' print("Reading file:", file_name) with open(file_name) as f: for i in range(3 if file_id != 3 else 1): diff --git a/data/jungfrau/read_multiport.py b/data/jungfrau/read_multiport.py index de8a4ae..9ff2c9f 100644 --- a/data/jungfrau/read_multiport.py +++ b/data/jungfrau/read_multiport.py @@ -1,6 +1,4 @@ import numpy as np -import matplotlib.pyplot as plt -plt.ion() header_dt = np.dtype( [ @@ -23,9 +21,10 @@ header_dt = np.dtype( # Read three frames from a jungfrau file with a single interface -frames = 1 +frames = 10 parts = 2 - +frame_per_file = 3 +bytes_per_pixel = 2 frame_cols = 1024 frame_rows = 512 @@ -39,14 +38,29 @@ header = np.zeros((frames,parts), dtype = header_dt) +# verify that all parts have the same frame number +for frame in range(frames): + for part in range(parts): + file_name = f'jungfrau_double_d{part}_f{frame//frame_per_file}_{0}.raw' + with open(file_name) as f: + offset = (frame%frame_per_file)*(header_dt.itemsize+part_rows*part_cols*bytes_per_pixel) + # print(f"Reading file: {file_name} at offset {offset}") + header[frame,part] = np.fromfile(f, dtype=header_dt, count = 1,offset=offset) + # print(f"Frame {frame} part {part} frame number: {header[frame,part]['Frame Number']}") + if part > 0: + assert header[frame,part]['Frame Number'] == header[frame,0]['Frame Number'] + +print("[X] All parts have the same frame number\n") + for frame in range(frames): for part in range(parts): - file_name = f'jungfrau_double_d{part}_f{frame}_{0}.raw' - print("Reading file:", file_name) + file_name = f'jungfrau_double_d{part}_f{frame//frame_per_file}_{0}.raw' + # print("Reading file:", file_name) with open(file_name) as f: - header[frame,part] = np.fromfile(f, dtype=header_dt, count = 1) + offset = (frame%frame_per_file)*(header_dt.itemsize+part_rows*part_cols*bytes_per_pixel) + header[frame,part] = np.fromfile(f, dtype=header_dt, count = 1, offset=offset) parts_data[frame,part] = np.fromfile(f, dtype=np.uint16,count = part_rows*part_cols).reshape(part_rows,part_cols) @@ -54,27 +68,20 @@ for frame in range(frames): -# for frame in range(frames): -# print("Frame:", frame) -# print("Data:\n", data[frame]) - -# print(data[0,0,0]) -# print(data[0,0,1]) -# print(data[0,0,50]) -print(data[0,0,0]) -print(data[0,0,1]) -print(data[0,255,1023]) - -print(data[0,511,1023]) -# print() -# print(parts_data[0,0,0,0]) -# print(parts_data[0,0,0,1]) -# print(parts_data[0,0,1,0]) - -# print(data.shape) - - - -#fig, ax = plt.subplots() -#im = ax.imshow(data[0]) -#im.set_clim(2000,4000) +pixel_0_0,pixel_0_1,pixel_1_0,pixel_255_1023,pixel_511_1023,= [],[],[],[],[] +for frame in range(frames): + pixel_0_0.append(data[frame,0,0]) + pixel_0_1.append(data[frame,0,1]) + pixel_1_0.append(data[frame,1,0]) + pixel_255_1023.append(data[frame,255,1023]) + pixel_511_1023.append(data[frame,511,1023]) +print("upper left corner of each frame (pixel_0_0)") +print(pixel_0_0) +print("first pixel on new line of each frame (pixel_1_0)") +print(pixel_1_0) +print("second pixel of the first line of each frame (pixel_0_1)") +print(pixel_0_1) +print("first pixel of the second part on the last line of each frame (pixel_255_1023)") +print(pixel_255_1023) +print("lower right corner of each frame (pixel_511_1023)") +print(pixel_511_1023) diff --git a/data/mythen/CORRECTED_scan242_d0_f0_3.raw b/data/mythen/CORRECTED_scan242_d0_f0_3.raw new file mode 100644 index 0000000..66c0c56 Binary files /dev/null and b/data/mythen/CORRECTED_scan242_d0_f0_3.raw differ diff --git a/data/mythen/CORRECTED_scan242_d1_f0_3.raw b/data/mythen/CORRECTED_scan242_d1_f0_3.raw new file mode 100644 index 0000000..b8db27d Binary files /dev/null and b/data/mythen/CORRECTED_scan242_d1_f0_3.raw differ diff --git a/data/mythen/CORRECTED_scan242_d2_f0_3.raw b/data/mythen/CORRECTED_scan242_d2_f0_3.raw new file mode 100644 index 0000000..df21c94 Binary files /dev/null and b/data/mythen/CORRECTED_scan242_d2_f0_3.raw differ diff --git a/data/mythen/CORRECTED_scan242_d3_f0_3.raw b/data/mythen/CORRECTED_scan242_d3_f0_3.raw new file mode 100644 index 0000000..f91b37c Binary files /dev/null and b/data/mythen/CORRECTED_scan242_d3_f0_3.raw differ diff --git a/data/mythen/CORRECTED_scan242_master_3.raw b/data/mythen/CORRECTED_scan242_master_3.raw new file mode 100755 index 0000000..92305d7 --- /dev/null +++ b/data/mythen/CORRECTED_scan242_master_3.raw @@ -0,0 +1,40 @@ +Version : 6.2 +TimeStamp : Mon Sep 14 16:21:09 2020 + +Detector Type : Mythen3 +Timing Mode : auto +Geometry : [4, 1] +Image Size : 5120 bytes +Pixels : [1280, 1] +Max Frames Per File : 10000 +Frame Discard Policy : nodiscard +Frame Padding : 1 +Scan Parameters : [disabled] +Total Frames : 1 +Dynamic Range : 32 +Ten Giga : 1 +Period : 0ns +Counter Mask : 0x1 +Exptime1 : 0.2s +Exptime2 : 0.2s +Exptime3 : 0.2s +GateDelay1 : 0.1s +GateDelay2 : 0.1s +GateDelay3 : 0.1s +Gates : 1 + +#Frame Header +Frame Number : 8 bytes +SubFrame Number/ExpLength : 4 bytes +Packet Number : 4 bytes +Bunch ID : 8 bytes +Timestamp : 8 bytes +Module Id : 2 bytes +Row : 2 bytes +Column : 2 bytes +Reserved : 2 bytes +Debug : 4 bytes +Round Robin Number : 2 bytes +Detector Type : 1 byte +Header Version : 1 byte +Packets Caught Mask : 64 bytes diff --git a/data/mythen/correct_frame_numbers.py b/data/mythen/correct_frame_numbers.py new file mode 100644 index 0000000..56a84a8 --- /dev/null +++ b/data/mythen/correct_frame_numbers.py @@ -0,0 +1,58 @@ +import numpy as np +import shutil + +header_dt = np.dtype( + [ + ("Frame Number", "u8"), + ("SubFrame Number/ExpLength", "u4"), + ("Packet Number", "u4"), + ("Bunch ID", "u8"), + ("Timestamp", "u8"), + ("Module Id", "u2"), + ("Row", "u2"), + ("Column", "u2"), + ("Reserved", "u2"), + ("Debug", "u4"), + ("Round Robin Number", "u2"), + ("Detector Type", "u1"), + ("Header Version", "u1"), + ("Packets caught mask", "8u8") + ] +) + +# Read three frames from a jungfrau file with a single interface + +frames = 1 +parts = 4 +frame_per_file = 3 +bytes_per_pixel = 4 +frame_cols = 1 +frame_rows = 5120 + +part_cols = 1280 +part_rows = 1 + +header = np.zeros((frames,parts), dtype = header_dt) + + + +# verify that all parts have the same frame number +frame = 0 +i = 55 +for part in range(parts): + file_name_r = f'scan242_d{part}_f{frame//frame_per_file}_{3}.raw' + file_name_w = f'CORRECTED_scan242_d{part}_f{frame//frame_per_file}_{3}.raw' + shutil.copyfile(file_name_r, file_name_w) + + with open(file_name_r) as fr, open(file_name_w, 'r+b') as fw: + # get frame + offset = (frame%frame_per_file)*(header_dt.itemsize+part_rows*part_cols*bytes_per_pixel) + header[frame,part] = np.fromfile(fr, dtype=header_dt, count = 1,offset=offset) + # update frame number + header[frame,part]['Frame Number'] = i + fw.seek(offset) + header[frame,part].tofile(fw) + + +print("[X] Done\n") + diff --git a/data/mythen/read_multiport.py b/data/mythen/read_multiport.py new file mode 100644 index 0000000..51b538c --- /dev/null +++ b/data/mythen/read_multiport.py @@ -0,0 +1,57 @@ +import numpy as np + +header_dt = np.dtype( + [ + ("Frame Number", "u8"), + ("SubFrame Number/ExpLength", "u4"), + ("Packet Number", "u4"), + ("Bunch ID", "u8"), + ("Timestamp", "u8"), + ("Module Id", "u2"), + ("Row", "u2"), + ("Column", "u2"), + ("Reserved", "u2"), + ("Debug", "u4"), + ("Round Robin Number", "u2"), + ("Detector Type", "u1"), + ("Header Version", "u1"), + ("Packets caught mask", "8u8") + ] +) + +# Read three frames from a jungfrau file with a single interface + +frames = 1 +parts = 4 +frame_per_file = 3 +bytes_per_pixel = 4 +frame_cols = 1 +frame_rows = 5120 + +part_cols = 1280 +part_rows = 1 + +header = np.zeros((frames,parts), dtype = header_dt) + + + +# verify that all parts have the same frame number +frame = 0 +sync = True +for part in range(parts): + file_name = f'scan242_d{part}_f{frame//frame_per_file}_{3}.raw' + with open(file_name) as f: + offset = (frame%frame_per_file)*(header_dt.itemsize+part_rows*part_cols*bytes_per_pixel) + # print(f"Reading file: {file_name} at offset {offset}") + header[frame,part] = np.fromfile(f, dtype=header_dt, count = 1,offset=offset) + # print(f"Frame {frame} part {part} frame number: {header[frame,part]['Frame Number']}") + print(f"part {part} frame number: {header[frame,part]['Frame Number']}") + sync = sync and (header[frame,part]['Frame Number'] == header[frame,0]['Frame Number']) + + +if sync: + print("[X] subfiles have the same frame") +else: + print("[X] subfiles do not have the same frame\n") + + diff --git a/data/scripts/read_first_frame_number.py b/data/scripts/read_first_frame_number.py deleted file mode 100644 index 02b19d2..0000000 --- a/data/scripts/read_first_frame_number.py +++ /dev/null @@ -1,29 +0,0 @@ -import numpy as np -from pathlib import Path - -header_dt = np.dtype( - [ - ("Frame Number", "u8"), - ("SubFrame Number/ExpLength", "u4"), - ("Packet Number", "u4"), - ("Bunch ID", "u8"), - ("Timestamp", "u8"), - ("Module Id", "u2"), - ("Row", "u2"), - ("Column", "u2"), - ("Reserved", "u2"), - ("Debug", "u4"), - ("Round Robin Number", "u2"), - ("Detector Type", "u1"), - ("Header Version", "u1"), - ("Packets caught mask", "8u8") - ] -) - - - -with open("data/eiger/eiger_500k_16bit_d0_f0_0.raw", "rb") as f: - for i in range(3): - frame_number = np.fromfile(f, dtype=header_dt, count=1)["Frame Number"][0] - print(frame_number) - f.seek(262144,1) diff --git a/data/scripts/read_multiport.py b/data/scripts/read_multiport.py deleted file mode 100644 index de8a4ae..0000000 --- a/data/scripts/read_multiport.py +++ /dev/null @@ -1,80 +0,0 @@ -import numpy as np -import matplotlib.pyplot as plt -plt.ion() - -header_dt = np.dtype( - [ - ("Frame Number", "u8"), - ("SubFrame Number/ExpLength", "u4"), - ("Packet Number", "u4"), - ("Bunch ID", "u8"), - ("Timestamp", "u8"), - ("Module Id", "u2"), - ("Row", "u2"), - ("Column", "u2"), - ("Reserved", "u2"), - ("Debug", "u4"), - ("Round Robin Number", "u2"), - ("Detector Type", "u1"), - ("Header Version", "u1"), - ("Packets caught mask", "8u8") - ] -) - -# Read three frames from a jungfrau file with a single interface - -frames = 1 -parts = 2 - -frame_cols = 1024 -frame_rows = 512 - -part_cols = 1024 -part_rows = 256 - - -parts_data = np.zeros((frames,parts,part_rows,part_cols), dtype = np.uint16) -data = np.zeros((frames,frame_rows,frame_cols), dtype = np.uint16) -header = np.zeros((frames,parts), dtype = header_dt) - - - - -for frame in range(frames): - - for part in range(parts): - file_name = f'jungfrau_double_d{part}_f{frame}_{0}.raw' - print("Reading file:", file_name) - with open(file_name) as f: - header[frame,part] = np.fromfile(f, dtype=header_dt, count = 1) - parts_data[frame,part] = np.fromfile(f, dtype=np.uint16,count = part_rows*part_cols).reshape(part_rows,part_cols) - - - data[frame] = np.concatenate((parts_data[frame,0],parts_data[frame,1]),axis=0) - - - -# for frame in range(frames): -# print("Frame:", frame) -# print("Data:\n", data[frame]) - -# print(data[0,0,0]) -# print(data[0,0,1]) -# print(data[0,0,50]) -print(data[0,0,0]) -print(data[0,0,1]) -print(data[0,255,1023]) - -print(data[0,511,1023]) -# print() -# print(parts_data[0,0,0,0]) -# print(parts_data[0,0,0,1]) -# print(parts_data[0,0,1,0]) - -# print(data.shape) - - - -#fig, ax = plt.subplots() -#im = ax.imshow(data[0]) -#im.set_clim(2000,4000) diff --git a/data/scripts/verify_rawfile_writing.py b/data/scripts/verify_rawfile_writing.py new file mode 100644 index 0000000..087b5ba --- /dev/null +++ b/data/scripts/verify_rawfile_writing.py @@ -0,0 +1,62 @@ +### +### Verify that the raw file written by the raw_example.cpp are correct +### + + +import numpy as np + +header_dt = np.dtype( + [ + ("Frame Number", "u8"), + ("SubFrame Number/ExpLength", "u4"), + ("Packet Number", "u4"), + ("Bunch ID", "u8"), + ("Timestamp", "u8"), + ("Module Id", "u2"), + ("Row", "u2"), + ("Column", "u2"), + ("Reserved", "u2"), + ("Debug", "u4"), + ("Round Robin Number", "u2"), + ("Detector Type", "u1"), + ("Header Version", "u1"), + ("Packets caught mask", "8u8") + ] +) + +frames = 1 +parts = 1 +frame_per_file = 3 +bytes_per_pixel = 2 +frame_cols = 512 +frame_rows = 1024 + +part_cols = 512 +part_rows = 1024 + + +# parts_data = np.zeros((frames,parts,part_rows,part_cols), dtype = np.uint16) +data = np.zeros((frames,frame_rows,frame_cols), dtype = np.uint16) +header = np.zeros((frames,parts), dtype = header_dt) + + + +# verify that all parts have the same frame number +for frame in range(frames): + for part in range(parts): + file_name = f'/tmp/raw_example_writing_d{part}_f{frame//frame_per_file}_{0}.raw' + with open(file_name) as f: + offset = (frame%frame_per_file)*(header_dt.itemsize+part_rows*part_cols*bytes_per_pixel) + # print(f"Reading file: {file_name} at offset {offset}") + header[frame,part] = np.fromfile(f, dtype=header_dt, count = 1,offset=offset) + # print(f"Frame {frame} part {part} frame number: {header[frame,part]['Frame Number']}") + data[frame] = np.fromfile(f, dtype=np.uint16,count = frame_rows*frame_cols).reshape(frame_rows,frame_cols) + + +for frame in range(frames): + for i,j in np.ndindex(data[frame].shape): + assert(data[frame][i,j] == i+j) + +print("[X] frame data is correct") + + diff --git a/examples/mythen_example.cpp b/examples/mythen_example.cpp index 01bc021..8265f34 100644 --- a/examples/mythen_example.cpp +++ b/examples/mythen_example.cpp @@ -40,7 +40,7 @@ int main() { File file(fpath, "r"); test1(file, 0); - fpath = (PROJECT_ROOT_DIR / "data" / "mythen" / "scan242_master_3.raw"); + fpath = (PROJECT_ROOT_DIR / "data" / "mythen" / "CORRECTED_scan242_master_3.raw"); File file2(fpath, "r"); test2(file2, 0); } \ No newline at end of file diff --git a/examples/raw_example.cpp b/examples/raw_example.cpp index c59ce32..2b48305 100644 --- a/examples/raw_example.cpp +++ b/examples/raw_example.cpp @@ -1,11 +1,9 @@ // Your First C++ Program +#include "aare/aare.hpp" #include "aare/examples/defs.hpp" -#include "aare/file_io/File.hpp" -#include "aare/utils/logger.hpp" #include -using aare::File; -using aare::Frame; +using namespace aare; void test(File &f, int frame_number) { std::cout << "frame number: " << frame_number << '\n'; @@ -26,4 +24,27 @@ int main() { test(file, 0); test(file, 2); test(file, 99); + + std::filesystem::path const path2("/tmp/raw_example_writing.json"); + aare::FileConfig config; + config.version = "1.0"; + config.geometry = {1, 1}; + config.detector_type = aare::DetectorType::Moench; + config.max_frames_per_file = 100; + config.rows = 1024; + config.cols = 512; + config.dtype = aare::DType::UINT16; + File file2(path2, "w", config); + Frame frame(1024, 512, 16); + + for (int i = 0; i < 1024; i++) { + for (int j = 0; j < 512; j++) { + frame.set(i, j, (uint16_t)(i + j)); + } + } + + sls_detector_header header; + header.frameNumber = 0; + file2.write(frame, header); + file2.set_total_frames(1); } \ No newline at end of file diff --git a/file_io/include/aare/file_io/File.hpp b/file_io/include/aare/file_io/File.hpp index ee00714..8ab7f80 100644 --- a/file_io/include/aare/file_io/File.hpp +++ b/file_io/include/aare/file_io/File.hpp @@ -11,6 +11,7 @@ namespace aare { class File { private: FileInterface *file_impl; + bool is_npy = true; public: /** @@ -22,8 +23,8 @@ class File { * @throws std::invalid_argument if the file mode is not supported * */ - File(const std::filesystem::path &fname, const std::string &mode, FileConfig cfg = {}); - void write(Frame &frame); + File(const std::filesystem::path &fname, const std::string &mode, const FileConfig &cfg = {}); + void write(Frame &frame, sls_detector_header header = {}); Frame read(); Frame iread(size_t frame_number); std::vector read(size_t n_frames); @@ -38,6 +39,7 @@ class File { size_t rows() const; size_t cols() const; size_t bitdepth() const; + void set_total_frames(size_t total_frames); /** * @brief Move constructor diff --git a/file_io/include/aare/file_io/FileInterface.hpp b/file_io/include/aare/file_io/FileInterface.hpp index ed89f74..0af109f 100644 --- a/file_io/include/aare/file_io/FileInterface.hpp +++ b/file_io/include/aare/file_io/FileInterface.hpp @@ -15,14 +15,21 @@ namespace aare { * geometry: geometry of the file */ struct FileConfig { - aare::DType dtype = aare::DType(typeid(uint16_t)); + aare::DType dtype{typeid(uint16_t)}; uint64_t rows{}; uint64_t cols{}; - xy geometry{1, 1}; bool operator==(const FileConfig &other) const { - return dtype == other.dtype && rows == other.rows && cols == other.cols && geometry == other.geometry; + return dtype == other.dtype && rows == other.rows && cols == other.cols && geometry == other.geometry && + detector_type == other.detector_type && max_frames_per_file == other.max_frames_per_file; } bool operator!=(const FileConfig &other) const { return !(*this == other); } + + // rawfile specific + std::string version{}; + xy geometry{1, 1}; + DetectorType detector_type{DetectorType::Unknown}; + int max_frames_per_file{}; + size_t total_frames{}; }; /** @@ -38,7 +45,7 @@ class FileInterface { * @return void * @throws std::runtime_error if the function is not implemented */ - virtual void write(Frame &frame) = 0; + // virtual void write(Frame &frame) = 0; /** * @brief write a vector of frames to the file @@ -160,6 +167,8 @@ class FileInterface { virtual ~FileInterface() = default; + void set_total_frames(size_t total_frames) { m_total_frames = total_frames; } + protected: std::string m_mode{}; std::filesystem::path m_fname{}; diff --git a/file_io/include/aare/file_io/NumpyFile.hpp b/file_io/include/aare/file_io/NumpyFile.hpp index 5a72179..3288626 100644 --- a/file_io/include/aare/file_io/NumpyFile.hpp +++ b/file_io/include/aare/file_io/NumpyFile.hpp @@ -27,7 +27,7 @@ class NumpyFile : public FileInterface { */ explicit NumpyFile(const std::filesystem::path &fname, const std::string &mode = "r", FileConfig cfg = {}); - void write(Frame &frame) override; + void write(Frame &frame); Frame read() override { return get_frame(this->current_frame++); } std::vector read(size_t n_frames) override; diff --git a/file_io/include/aare/file_io/RawFile.hpp b/file_io/include/aare/file_io/RawFile.hpp index cc7b399..1047d1a 100644 --- a/file_io/include/aare/file_io/RawFile.hpp +++ b/file_io/include/aare/file_io/RawFile.hpp @@ -5,6 +5,19 @@ namespace aare { +struct ModuleConfig { + int module_gap_row{}; + int module_gap_col{}; + + bool operator==(const ModuleConfig &other) const { + if (module_gap_col != other.module_gap_col) + return false; + if (module_gap_row != other.module_gap_row) + return false; + return true; + } +}; + /** * @brief RawFile class to read .raw and .json files * @note derived from FileInterface @@ -18,13 +31,14 @@ class RawFile : public FileInterface { * @param mode file mode (r, w) * @param cfg file configuration */ - explicit RawFile(const std::filesystem::path &fname, const std::string &mode = "r", const FileConfig &config = {}); + explicit RawFile(const std::filesystem::path &fname, const std::string &mode = "r", + const FileConfig &config = FileConfig{}); /** * @brief write function is not implemented for RawFile * @param frame frame to write */ - void write([[maybe_unused]] Frame &frame) override { throw std::runtime_error("Not implemented"); }; + void write(Frame &frame, sls_detector_header header); Frame read() override { return get_frame(this->current_frame++); }; std::vector read(size_t n_frames) override; void read_into(std::byte *image_buf) override { return get_frame_into(this->current_frame++, image_buf); }; @@ -44,7 +58,15 @@ class RawFile : public FileInterface { size_t pixels_per_frame() override { return m_rows * m_cols; } // goto frame number - void seek(size_t frame_number) override { this->current_frame = frame_number; }; + void seek(size_t frame_number) override { + // check if the frame number is greater than the total frames + // if frame_number == total_frames, then the next read will throw an error + if (frame_number > this->total_frames()) { + throw std::runtime_error( + fmt::format("frame number {} is greater than total frames {}", frame_number, m_total_frames)); + } + this->current_frame = frame_number; + }; // return the position of the file pointer (in number of frames) size_t tell() override { return this->current_frame; }; @@ -88,7 +110,7 @@ class RawFile : public FileInterface { /** * @brief destructor: will delete the subfiles */ - ~RawFile() override; + ~RawFile() noexcept override; size_t total_frames() const override { return m_total_frames; } size_t rows() const override { return m_rows; } @@ -96,19 +118,20 @@ class RawFile : public FileInterface { size_t bitdepth() const override { return m_bitdepth; } private: + void write_master_file(); /** - * @brief read the frame at the given frame number into the image buffer + * @brief read the frame at the given frame index into the image buffer * @param frame_number frame number to read * @param image_buf buffer to store the frame */ - void get_frame_into(size_t frame_number, std::byte *frame_buffer); + void get_frame_into(size_t frame_index, std::byte *frame_buffer); /** - * @brief get the frame at the given frame number + * @brief get the frame at the given frame index * @param frame_number frame number to read * @return Frame */ - Frame get_frame(size_t frame_number); + Frame get_frame(size_t frame_index); /** * @brief parse the file name to get the extension, base name and index @@ -146,6 +169,7 @@ class RawFile : public FileInterface { * @brief open the subfiles */ void open_subfiles(); + void parse_config(const FileConfig &config); size_t n_subfiles{}; size_t n_subfile_parts{}; @@ -153,7 +177,7 @@ class RawFile : public FileInterface { size_t subfile_rows{}, subfile_cols{}; xy geometry{}; std::vector positions; - RawFileConfig cfg{0, 0}; + ModuleConfig cfg{0, 0}; TimingMode timing_mode{}; bool quad{false}; }; diff --git a/file_io/include/aare/file_io/SubFile.hpp b/file_io/include/aare/file_io/SubFile.hpp index f5b2a3d..908cb70 100644 --- a/file_io/include/aare/file_io/SubFile.hpp +++ b/file_io/include/aare/file_io/SubFile.hpp @@ -1,4 +1,5 @@ #pragma once +#include "aare/core/Frame.hpp" #include "aare/core/defs.hpp" #include #include @@ -36,6 +37,7 @@ class SubFile { }; public: + size_t write_part(std::byte *buffer, sls_detector_header header, size_t frame_index); /** * @brief SubFile constructor * @param fname path to the subfile @@ -45,7 +47,8 @@ class SubFile { * @param bitdepth bitdepth of the subfile * @throws std::invalid_argument if the detector,type pair is not supported */ - SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth); + SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth, + const std::string &mode = "r"); /** * @brief read the subfile into a buffer @@ -74,19 +77,22 @@ class SubFile { * @param frame_number frame number to read * @return number of bytes read */ - size_t get_part(std::byte *buffer, size_t frame_number); + size_t get_part(std::byte *buffer, size_t frame_index); size_t frame_number(size_t frame_index); // TODO: define the inlines as variables and assign them in constructor inline size_t bytes_per_part() const { return (m_bitdepth / 8) * m_rows * m_cols; } inline size_t pixels_per_part() const { return m_rows * m_cols; } + ~SubFile(); + protected: FILE *fp = nullptr; size_t m_bitdepth; std::filesystem::path m_fname; size_t m_rows{}; size_t m_cols{}; + std::string m_mode; size_t n_frames{}; int m_sub_file_index_{}; }; diff --git a/file_io/src/File.cpp b/file_io/src/File.cpp index 8848be8..8507186 100644 --- a/file_io/src/File.cpp +++ b/file_io/src/File.cpp @@ -6,7 +6,7 @@ namespace aare { -File::File(const std::filesystem::path &fname, const std::string &mode, FileConfig cfg) { +File::File(const std::filesystem::path &fname, const std::string &mode, const FileConfig &cfg) { if (mode != "r" && mode != "w" && mode != "a") { throw std::invalid_argument("Unsupported file mode"); } @@ -18,6 +18,7 @@ File::File(const std::filesystem::path &fname, const std::string &mode, FileConf if (fname.extension() == ".raw" || fname.extension() == ".json") { aare::logger::debug("Loading raw file"); file_impl = new RawFile(fname, mode, cfg); + is_npy = false; } // check if extension is numpy else if (fname.extension() == ".npy") { @@ -28,7 +29,14 @@ File::File(const std::filesystem::path &fname, const std::string &mode, FileConf } } -void File::write(Frame &frame) { file_impl->write(frame); } +void File::write(Frame &frame, sls_detector_header header) { + if (is_npy) { + aare::logger::info("ignoring header for npy file"); + dynamic_cast(file_impl)->write(frame); + } else { + dynamic_cast(file_impl)->write(frame, header); + } +} Frame File::read() { return file_impl->read(); } size_t File::total_frames() const { return file_impl->total_frames(); } std::vector File::read(size_t n_frames) { return file_impl->read(n_frames); } @@ -42,6 +50,7 @@ size_t File::tell() const { return file_impl->tell(); } size_t File::rows() const { return file_impl->rows(); } size_t File::cols() const { return file_impl->cols(); } size_t File::bitdepth() const { return file_impl->bitdepth(); } +void File::set_total_frames(size_t total_frames) { return file_impl->set_total_frames(total_frames); } File::~File() { delete file_impl; } Frame File::iread(size_t frame_number) { return file_impl->iread(frame_number); } diff --git a/file_io/src/RawFile.cpp b/file_io/src/RawFile.cpp index 10720f2..8c09da8 100644 --- a/file_io/src/RawFile.cpp +++ b/file_io/src/RawFile.cpp @@ -1,5 +1,6 @@ #include "aare/file_io/RawFile.hpp" #include "aare/core/defs.hpp" +#include "aare/utils/json.hpp" #include "aare/utils/logger.hpp" #include #include @@ -9,8 +10,9 @@ using json = nlohmann::json; namespace aare { RawFile::RawFile(const std::filesystem::path &fname, const std::string &mode, const FileConfig &config) { + m_mode = mode; m_fname = fname; - if (mode == "r") { + if (mode == "r" or mode == "r+") { if (config != FileConfig()) { aare::logger::warn( "In read mode it is not necessary to provide a config, the provided config will be ignored"); @@ -21,17 +23,108 @@ RawFile::RawFile(const std::filesystem::path &fname, const std::string &mode, co find_geometry(); open_subfiles(); + } else if (mode == "w" or mode == "w+") { + + if (std::filesystem::exists(fname)) { + // handle mode w as w+ (no overrwriting) + throw std::runtime_error(LOCATION + "File already exists"); + } + + parse_config(config); + parse_fname(); + write_master_file(); + n_subfiles = 1; + n_subfile_parts = 1; + subfile_cols = m_cols; + subfile_rows = m_rows; + open_subfiles(); + } else { throw std::runtime_error(LOCATION + "Unsupported mode"); } } +void RawFile::parse_config(const FileConfig &config) { + m_bitdepth = config.dtype.bitdepth(); + m_total_frames = config.total_frames; + m_rows = config.rows; + m_cols = config.cols; + m_type = config.detector_type; + max_frames_per_file = config.max_frames_per_file; + geometry = config.geometry; + version = config.version; + subfile_rows = config.geometry.row; + subfile_cols = config.geometry.col; + + if (geometry != aare::xy{1, 1}) { + throw std::runtime_error(LOCATION + "Only geometry {1,1} files are supported for writing"); + } +} +void RawFile::write_master_file() { + if (m_ext != ".json") { + throw std::runtime_error(LOCATION + "only json master files are supported for writing"); + } + std::ofstream ofs(master_fname(), std::ios::binary); + std::string ss; + ss.reserve(1024); + ss += "{\n\t"; + aare::write_str(ss, "Version", version); + ss += "\n\t"; + aare::write_digit(ss, "Total Frames", m_total_frames); + ss += "\n\t"; + aare::write_str(ss, "Detector Type", toString(m_type)); + ss += "\n\t"; + aare::write_str(ss, "Geometry", geometry.to_string()); + ss += "\n\t"; + + uint64_t img_size = (m_cols * m_rows) / (geometry.col * geometry.row); + img_size *= m_bitdepth; + aare::write_digit(ss, "Image Size in bytes", img_size); + ss += "\n\t"; + aare::write_digit(ss, "Max Frames Per File", max_frames_per_file); + ss += "\n\t"; + aare::write_digit(ss, "Dynamic Range", m_bitdepth); + ss += "\n\t"; + const aare::xy pixels = {m_rows / geometry.row, m_cols / geometry.col}; + aare::write_str(ss, "Pixels", pixels.to_string()); + ss += "\n\t"; + aare::write_digit(ss, "Number of rows", m_rows); + ss += "\n\t"; + const std::string tmp = "{\n" + " \"Frame Number\": \"8 bytes\",\n" + " \"Exposure Length\": \"4 bytes\",\n" + " \"Packet Number\": \"4 bytes\",\n" + " \"Bunch Id\": \"8 bytes\",\n" + " \"Timestamp\": \"8 bytes\",\n" + " \"Module Id\": \"2 bytes\",\n" + " \"Row\": \"2 bytes\",\n" + " \"Column\": \"2 bytes\",\n" + " \"Reserved\": \"2 bytes\",\n" + " \"Debug\": \"4 bytes\",\n" + " \"RoundRNumber\": \"2 bytes\",\n" + " \"DetType\": \"1 byte\",\n" + " \"Version\": \"1 byte\",\n" + " \"Packet Mask\": \"64 bytes\"\n" + " }"; + + ss += "\"Frame Header Format\":" + tmp + "\n"; + ss += "}"; + ofs << ss; + ofs.close(); +} + void RawFile::open_subfiles() { - for (size_t i = 0; i != n_subfiles; ++i) { - auto v = std::vector(n_subfile_parts); - for (size_t j = 0; j != n_subfile_parts; ++j) { - v[j] = new SubFile(data_fname(i, j), m_type, subfile_rows, subfile_cols, m_bitdepth); + if (m_mode == "r") + for (size_t i = 0; i != n_subfiles; ++i) { + auto v = std::vector(n_subfile_parts); + for (size_t j = 0; j != n_subfile_parts; ++j) { + v[j] = new SubFile(data_fname(i, j), m_type, subfile_rows, subfile_cols, m_bitdepth); + } + subfiles.push_back(v); } + else { + auto v = std::vector(n_subfile_parts); // only one subfile is implemented + v[0] = new SubFile(data_fname(0, 0), m_type, m_rows, m_cols, m_bitdepth, "w"); subfiles.push_back(v); } } @@ -173,51 +266,97 @@ void RawFile::parse_raw_metadata() { max_frames_per_file = std::stoi(value); } else if (key == "Geometry") { pos = value.find(','); - geometry = {std::stoi(value.substr(1, pos)), std::stoi(value.substr(pos + 1))}; + const size_t x = static_cast(std::stoi(value.substr(1, pos))); + const size_t y = static_cast(std::stoi(value.substr(pos + 1))); + + geometry = {x, y}; } } } } void RawFile::parse_fname() { + bool wrong_format = false; m_base_path = m_fname.parent_path(); m_base_name = m_fname.stem(); m_ext = m_fname.extension(); - auto pos = m_base_name.rfind('_'); - m_findex = std::stoi(m_base_name.substr(pos + 1)); - pos = m_base_name.find("_master_"); - m_base_name.erase(pos); + try { + auto pos = m_base_name.rfind('_'); + m_findex = std::stoi(m_base_name.substr(pos + 1)); + } catch (const std::invalid_argument &e) { + m_findex = 0; + wrong_format = true; + } + auto pos = m_base_name.find("_master_"); + if (pos != std::string::npos) { + m_base_name.erase(pos); + wrong_format = true; + } + if (wrong_format and (m_mode == "w+" or m_mode == "w")) { + aare::logger::warn("Master Filename", m_fname, "is not in the correct format"); + aare::logger::warn("using", master_fname(), "as the master file"); + } } -Frame RawFile::get_frame(size_t frame_number) { +Frame RawFile::get_frame(size_t frame_index) { auto f = Frame(this->m_rows, this->m_cols, this->m_bitdepth); std::byte *frame_buffer = f.data(); - get_frame_into(frame_number, frame_buffer); + get_frame_into(frame_index, frame_buffer); return f; } -void RawFile::get_frame_into(size_t frame_number, std::byte *frame_buffer) { - if (frame_number > this->m_total_frames) { +void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer) { + if (frame_index > this->m_total_frames) { throw std::runtime_error(LOCATION + "Frame number out of range"); } - size_t const subfile_id = frame_number / this->max_frames_per_file; - // create frame and get its buffer + std::vector frame_numbers(this->n_subfile_parts); + std::vector frame_indices(this->n_subfile_parts, frame_index); + + if (n_subfile_parts != 1) { + for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) { + auto subfile_id = frame_index / this->max_frames_per_file; + frame_numbers[part_idx] = + this->subfiles[subfile_id][part_idx]->frame_number(frame_index % this->max_frames_per_file); + } + // 1. if frame number vector is the same break + while (std::adjacent_find(frame_numbers.begin(), frame_numbers.end(), std::not_equal_to<>()) != + frame_numbers.end()) { + // 2. find the index of the minimum frame number, + auto min_frame_idx = + std::distance(frame_numbers.begin(), std::min_element(frame_numbers.begin(), frame_numbers.end())); + // 3. increase its index and update its respective frame number + frame_indices[min_frame_idx]++; + // 4. if we can't increase its index => throw error + if (frame_indices[min_frame_idx] >= this->m_total_frames) { + throw std::runtime_error(LOCATION + "Frame number out of range"); + } + auto subfile_id = frame_indices[min_frame_idx] / this->max_frames_per_file; + frame_numbers[min_frame_idx] = this->subfiles[subfile_id][min_frame_idx]->frame_number( + frame_indices[min_frame_idx] % this->max_frames_per_file); + } + } if (this->geometry.col == 1) { // get the part from each subfile and copy it to the frame for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) { + auto corrected_idx = frame_indices[part_idx]; + auto subfile_id = corrected_idx / this->max_frames_per_file; auto part_offset = this->subfiles[subfile_id][part_idx]->bytes_per_part(); this->subfiles[subfile_id][part_idx]->get_part(frame_buffer + part_idx * part_offset, - frame_number % this->max_frames_per_file); + corrected_idx % this->max_frames_per_file); } } else { + // create a buffer that will hold a the frame part auto bytes_per_part = this->subfile_rows * this->subfile_cols * this->m_bitdepth / 8; auto *part_buffer = new std::byte[bytes_per_part]; for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) { - this->subfiles[subfile_id][part_idx]->get_part(part_buffer, frame_number % this->max_frames_per_file); + auto corrected_idx = frame_indices[part_idx]; + auto subfile_id = corrected_idx / this->max_frames_per_file; + + this->subfiles[subfile_id][part_idx]->get_part(part_buffer, corrected_idx % this->max_frames_per_file); for (size_t cur_row = 0; cur_row < (this->subfile_rows); cur_row++) { auto irow = cur_row + (part_idx / this->geometry.col) * this->subfile_rows; auto icol = (part_idx % this->geometry.col) * this->subfile_cols; @@ -231,6 +370,19 @@ void RawFile::get_frame_into(size_t frame_number, std::byte *frame_buffer) { } } +void RawFile::write(Frame &frame, sls_detector_header header) { + if (m_mode == "r") { + throw std::runtime_error(LOCATION + "File is open in read mode"); + } + size_t const subfile_id = this->current_frame / this->max_frames_per_file; + for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) { + + this->subfiles[subfile_id][part_idx]->write_part(frame.data(), header, + this->current_frame % this->max_frames_per_file); + } + this->current_frame++; +} + std::vector RawFile::read(size_t n_frames) { // TODO: implement this in a more efficient way std::vector frames; @@ -256,7 +408,17 @@ size_t RawFile::frame_number(size_t frame_index) { return this->subfiles[subfile_id][0]->frame_number(frame_index % this->max_frames_per_file); } -RawFile::~RawFile() { +RawFile::~RawFile() noexcept { + + // update master file + if (m_mode == "w" or m_mode == "w+" or m_mode == "r+") { + try { + write_master_file(); + } catch (...) { + aare::logger::warn(LOCATION + "Could not update master file"); + } + } + for (auto &vec : subfiles) { for (auto *subfile : vec) { delete subfile; diff --git a/file_io/src/SubFile.cpp b/file_io/src/SubFile.cpp index dc1e085..af30854 100644 --- a/file_io/src/SubFile.cpp +++ b/file_io/src/SubFile.cpp @@ -7,9 +7,9 @@ namespace aare { -SubFile::SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth) - : m_bitdepth(bitdepth), m_fname(fname), m_rows(rows), m_cols(cols), - n_frames(std::filesystem::file_size(fname) / (sizeof(sls_detector_header) + rows * cols * bitdepth / 8)) { +SubFile::SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth, + const std::string &mode) + : m_bitdepth(bitdepth), m_fname(fname), m_rows(rows), m_cols(cols), m_mode(mode) { if (read_impl_map.find({detector, bitdepth}) == read_impl_map.end()) { auto error_msg = LOCATION + "No read_impl function found for detector: " + toString(detector) + @@ -17,25 +17,49 @@ SubFile::SubFile(const std::filesystem::path &fname, DetectorType detector, size throw std::invalid_argument(error_msg); } this->read_impl = read_impl_map.at({detector, bitdepth}); + if (std::filesystem::exists(fname)) { + n_frames = std::filesystem::file_size(fname) / (sizeof(sls_detector_header) + rows * cols * bitdepth / 8); + } else { + n_frames = 0; + } + + if (mode == "r") { + fp = fopen(m_fname.c_str(), "rb"); + } else { + // if file exists, open in read/write mode (without truncating the file) + // if file does not exist, open in write mode + if (std::filesystem::exists(fname)) { + fp = fopen(m_fname.c_str(), "r+b"); + } else { + fp = fopen(m_fname.c_str(), "wb"); + } + } + if (fp == nullptr) { + throw std::runtime_error(LOCATION + "Could not open file for writing"); + } } -size_t SubFile::get_part(std::byte *buffer, size_t frame_number) { - if (frame_number >= n_frames) { +size_t SubFile::get_part(std::byte *buffer, size_t frame_index) { + if (frame_index >= n_frames) { throw std::runtime_error("Frame number out of range"); } // TODO: find a way to avoid opening and closing the file for each frame - aare::logger::debug(LOCATION, "frame:", frame_number, "file:", m_fname.c_str()); - fp = fopen(m_fname.c_str(), "rb"); - if (!fp) { - throw std::runtime_error(fmt::format("Could not open: {} for reading", m_fname.c_str())); - } - fseek(fp, sizeof(sls_detector_header) + (sizeof(sls_detector_header) + bytes_per_part()) * frame_number, // NOLINT + aare::logger::debug(LOCATION, "frame:", frame_index, "file:", m_fname.c_str()); + fseek(fp, sizeof(sls_detector_header) + (sizeof(sls_detector_header) + bytes_per_part()) * frame_index, // NOLINT SEEK_SET); auto ret = (this->*read_impl)(buffer); - if (fclose(fp)) - throw std::runtime_error(LOCATION + "Could not close file"); return ret; } +size_t SubFile::write_part(std::byte *buffer, sls_detector_header header, size_t frame_index) { + if (frame_index > n_frames) { + throw std::runtime_error("Frame number out of range"); + } + fseek(fp, static_cast((sizeof(sls_detector_header) + bytes_per_part()) * frame_index), SEEK_SET); + auto wc = fwrite(reinterpret_cast(&header), sizeof(header), 1, fp); + wc += fwrite(buffer, bytes_per_part(), 1, fp); + + return wc; +} size_t SubFile::read_impl_normal(std::byte *buffer) { return fread(buffer, this->bytes_per_part(), 1, this->fp); } @@ -91,18 +115,18 @@ template size_t SubFile::read_impl_flip(std::byte *buffer) { size_t SubFile::frame_number(size_t frame_index) { sls_detector_header h{}; - fp = fopen(this->m_fname.c_str(), "r"); - if (!fp) - throw std::runtime_error(LOCATION + fmt::format("Could not open: {} for reading", m_fname.c_str())); fseek(fp, (sizeof(sls_detector_header) + bytes_per_part()) * frame_index, SEEK_SET); // NOLINT size_t const rc = fread(reinterpret_cast(&h), sizeof(h), 1, fp); if (rc != 1) throw std::runtime_error(LOCATION + "Could not read header from file"); - if (fclose(fp)) { - throw std::runtime_error(LOCATION + "Could not close file"); - } return h.frameNumber; } +SubFile::~SubFile() { + if (fp) { + fclose(fp); + } +} + } // namespace aare \ No newline at end of file diff --git a/file_io/test/RawFile.test.cpp b/file_io/test/RawFile.test.cpp index 34f2f03..155057f 100644 --- a/file_io/test/RawFile.test.cpp +++ b/file_io/test/RawFile.test.cpp @@ -79,4 +79,36 @@ TEST_CASE("Compare reading from a numpy file with a raw file") { auto npy_frame = npy.read(); CHECK(raw_frame.view() == npy_frame.view()); } -} \ No newline at end of file +} + +TEST_CASE("Read multipart files") { + auto fpath = test_data_path() / "jungfrau" / "jungfrau_double_master_0.json"; + REQUIRE(std::filesystem::exists(fpath)); + + File f(fpath, "r"); + + // we know this file has 10 frames check read_multiport.py for the values + std::vector pixel_0_0 = {2099, 2121, 2108, 2084, 2084, 2118, 2066, 2108, 2112, 2116}; + std::vector pixel_0_1 = {2842, 2796, 2865, 2798, 2805, 2817, 2852, 2789, 2792, 2833}; + std::vector pixel_255_1023 = {2149, 2037, 2115, 2102, 2118, 2090, 2036, 2071, 2073, 2142}; + std::vector pixel_511_1023 = {3231, 3169, 3167, 3162, 3168, 3160, 3171, 3171, 3169, 3171}; + std::vector pixel_1_0 = {2748, 2614, 2665, 2629, 2618, 2630, 2631, 2634, 2577, 2598}; + + for (size_t i = 0; i < 10; i++) { + auto frame = f.read(); + CHECK(frame.rows() == 512); + CHECK(frame.cols() == 1024); + CHECK(frame.view()(0, 0) == pixel_0_0[i]); + CHECK(frame.view()(0, 1) == pixel_0_1[i]); + CHECK(frame.view()(1, 0) == pixel_1_0[i]); + CHECK(frame.view()(255, 1023) == pixel_255_1023[i]); + CHECK(frame.view()(511, 1023) == pixel_511_1023[i]); + } +} + +TEST_CASE("Read file with unordered frames") { + auto fpath = test_data_path() / "mythen" / "scan242_master_3.raw"; + REQUIRE(std::filesystem::exists(fpath)); + File f(fpath, "r"); + REQUIRE_THROWS(f.read()); +} diff --git a/include/aare/aare.hpp b/include/aare/aare.hpp index 78a74d0..92eadf2 100644 --- a/include/aare/aare.hpp +++ b/include/aare/aare.hpp @@ -1 +1,7 @@ -// This is the top level header to include and what most users will use \ No newline at end of file +// This is the top level header to include and what most users will use + +// include all header files +#include "aare/core.hpp" +#include "aare/file_io.hpp" +#include "aare/network_io.hpp" +#include "aare/utils.hpp" \ No newline at end of file diff --git a/include/aare/core.hpp b/include/aare/core.hpp new file mode 100644 index 0000000..1cf2110 --- /dev/null +++ b/include/aare/core.hpp @@ -0,0 +1,8 @@ +#include "aare/core/CircularFifo.hpp" +#include "aare/core/DType.hpp" +#include "aare/core/Frame.hpp" +#include "aare/core/NDArray.hpp" +#include "aare/core/NDView.hpp" +#include "aare/core/defs.hpp" +// #include "aare/core/VariableSizeClusterFinder.hpp" +#include "aare/core/ProducerConsumerQueue.hpp" \ No newline at end of file diff --git a/include/aare/file_io.hpp b/include/aare/file_io.hpp new file mode 100644 index 0000000..d5ee373 --- /dev/null +++ b/include/aare/file_io.hpp @@ -0,0 +1,8 @@ +// ClusterFile.hpp File.hpp FileInterface.hpp NumpyFile.hpp NumpyHelpers.hpp RawFile.hpp SubFile.hpp +#include "aare/file_io/ClusterFile.hpp" +#include "aare/file_io/File.hpp" +#include "aare/file_io/FileInterface.hpp" +#include "aare/file_io/NumpyFile.hpp" +#include "aare/file_io/NumpyHelpers.hpp" +#include "aare/file_io/RawFile.hpp" +#include "aare/file_io/SubFile.hpp" diff --git a/include/aare/network_io.hpp b/include/aare/network_io.hpp new file mode 100644 index 0000000..fd559f0 --- /dev/null +++ b/include/aare/network_io.hpp @@ -0,0 +1,5 @@ +#include "aare/network_io/ZmqHeader.hpp" +#include "aare/network_io/ZmqSocket.hpp" +#include "aare/network_io/ZmqSocketReceiver.hpp" +#include "aare/network_io/ZmqSocketSender.hpp" +#include "aare/network_io/defs.hpp" \ No newline at end of file diff --git a/include/aare/utils.hpp b/include/aare/utils.hpp new file mode 100644 index 0000000..35596af --- /dev/null +++ b/include/aare/utils.hpp @@ -0,0 +1,3 @@ +#include "aare/utils/compare_files.hpp" +#include "aare/utils/json.hpp" +#include "aare/utils/logger.hpp" \ No newline at end of file diff --git a/network_io/src/ZmqHeader.cpp b/network_io/src/ZmqHeader.cpp index 588605e..8d8c889 100644 --- a/network_io/src/ZmqHeader.cpp +++ b/network_io/src/ZmqHeader.cpp @@ -1,66 +1,9 @@ #include "aare/network_io/ZmqHeader.hpp" +#include "aare/utils/json.hpp" #include "simdjson.h" -// helper functions to write json -// append to string for better performance (not tested) - -/** - * @brief write a digit to a string - * takes key and value and outputs->"key": value, - * @tparam T type of value (int, uint32_t, ...) - * @param s string to append to - * @param key key to write - * @param value value to write - * @return void - * @note - * - can't use concepts here because we are using c++17 - */ -template void write_digit(std::string &s, const std::string &key, const T &value) { - s += "\""; - s += key; - s += "\": "; - s += std::to_string(value); - s += ", "; -} -void write_str(std::string &s, const std::string &key, const std::string &value) { - s += "\""; - s += key; - s += "\": \""; - s += value; - s += "\", "; -} -void write_map(std::string &s, const std::string &key, const std::map &value) { - s += "\""; - s += key; - s += "\": {"; - for (const auto &kv : value) { - write_str(s, kv.first, kv.second); - } - // remove last comma or trailing spaces - for (size_t i = s.size() - 1; i > 0; i--) { - if (s[i] == ',' or s[i] == ' ') { - s.pop_back(); - } else - break; - } - s += "}, "; -} -void write_array(std::string &s, const std::string &key, const std::array &value) { - s += "\""; - s += key; - s += "\": ["; - s += std::to_string(value[0]); - s += ", "; - s += std::to_string(value[1]); - s += ", "; - s += std::to_string(value[2]); - s += ", "; - s += std::to_string(value[3]); - s += "], "; -} - namespace aare { std::string ZmqHeader::to_string() const { diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index 8a554b8..401f0a8 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -1,3 +1,4 @@ + add_library(utils STATIC src/logger.cpp) target_include_directories(utils PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include) diff --git a/utils/include/aare/utils/json.hpp b/utils/include/aare/utils/json.hpp new file mode 100644 index 0000000..ab679e5 --- /dev/null +++ b/utils/include/aare/utils/json.hpp @@ -0,0 +1,67 @@ +#pragma once +#include +#include +#include +#include + +// helper functions to write json +// append to string for better performance (not tested) + +namespace aare { + +/** + * @brief write a digit to a string + * takes key and value and outputs->"key": value, + * @tparam T type of value (int, uint32_t, ...) + * @param s string to append to + * @param key key to write + * @param value value to write + * @return void + * @note + * - can't use concepts here because we are using c++17 + */ +template inline void write_digit(std::string &s, const std::string &key, const T &value) { + s += "\""; + s += key; + s += "\": "; + s += std::to_string(value); + s += ", "; +} +inline void write_str(std::string &s, const std::string &key, const std::string &value) { + s += "\""; + s += key; + s += "\": \""; + s += value; + s += "\", "; +} +inline void write_map(std::string &s, const std::string &key, const std::map &value) { + s += "\""; + s += key; + s += "\": {"; + for (const auto &kv : value) { + write_str(s, kv.first, kv.second); + } + // remove last comma or trailing spaces + for (size_t i = s.size() - 1; i > 0; i--) { + if (s[i] == ',' or s[i] == ' ') { + s.pop_back(); + } else + break; + } + s += "}, "; +} +inline void write_array(std::string &s, const std::string &key, const std::array &value) { + s += "\""; + s += key; + s += "\": ["; + s += std::to_string(value[0]); + s += ", "; + s += std::to_string(value[1]); + s += ", "; + s += std::to_string(value[2]); + s += ", "; + s += std::to_string(value[3]); + s += "], "; +} + +} // namespace aare