From 70acfbf4accc7a459c4ed3d4099ab417144b1fbd Mon Sep 17 00:00:00 2001 From: Bechir Braham Date: Tue, 7 May 2024 10:46:24 +0200 Subject: [PATCH] write rawfiles (single file) and read rawfiles in order (#66) * read subfiles with unordered and missing frames * save work debugging * Revert "save work debugging" This reverts commit e791992a05efd754f93a80c980d17397eb4b6045. * Revert "read subfiles with unordered and missing frames" This reverts commit 1177fd129d3690db92e9597ccda62598e5a44d41. * throw when two frames have different frame numbers * write single part RawFile (working beta) * correct total number of frames in master file * add new mythen file with syncd frames * read frames with same frame number * clang-tidy fixes, formatting, add tests * improve readability in loop * fix failing tests --------- Co-authored-by: Bechir --- .env.dev => .env | 0 core/include/aare/core/defs.hpp | 20 +- data/jungfrau/read_frame.py | 11 +- data/jungfrau/read_multiport.py | 69 +++--- data/mythen/CORRECTED_scan242_d0_f0_3.raw | Bin 0 -> 5232 bytes data/mythen/CORRECTED_scan242_d1_f0_3.raw | Bin 0 -> 5232 bytes data/mythen/CORRECTED_scan242_d2_f0_3.raw | Bin 0 -> 5232 bytes data/mythen/CORRECTED_scan242_d3_f0_3.raw | Bin 0 -> 5232 bytes data/mythen/CORRECTED_scan242_master_3.raw | 40 ++++ data/mythen/correct_frame_numbers.py | 58 +++++ data/mythen/read_multiport.py | 57 +++++ data/scripts/read_first_frame_number.py | 29 --- data/scripts/read_multiport.py | 80 ------- data/scripts/verify_rawfile_writing.py | 62 ++++++ examples/mythen_example.cpp | 2 +- examples/raw_example.cpp | 29 ++- file_io/include/aare/file_io/File.hpp | 6 +- .../include/aare/file_io/FileInterface.hpp | 17 +- file_io/include/aare/file_io/NumpyFile.hpp | 2 +- file_io/include/aare/file_io/RawFile.hpp | 42 +++- file_io/include/aare/file_io/SubFile.hpp | 10 +- file_io/src/File.cpp | 13 +- file_io/src/RawFile.cpp | 200 ++++++++++++++++-- file_io/src/SubFile.cpp | 62 ++++-- file_io/test/RawFile.test.cpp | 34 ++- include/aare/aare.hpp | 8 +- include/aare/core.hpp | 8 + include/aare/file_io.hpp | 8 + include/aare/network_io.hpp | 5 + include/aare/utils.hpp | 3 + network_io/src/ZmqHeader.cpp | 59 +----- utils/CMakeLists.txt | 1 + utils/include/aare/utils/json.hpp | 67 ++++++ 33 files changed, 718 insertions(+), 284 deletions(-) rename .env.dev => .env (100%) create mode 100644 data/mythen/CORRECTED_scan242_d0_f0_3.raw create mode 100644 data/mythen/CORRECTED_scan242_d1_f0_3.raw create mode 100644 data/mythen/CORRECTED_scan242_d2_f0_3.raw create mode 100644 data/mythen/CORRECTED_scan242_d3_f0_3.raw create mode 100755 data/mythen/CORRECTED_scan242_master_3.raw create mode 100644 data/mythen/correct_frame_numbers.py create mode 100644 data/mythen/read_multiport.py delete mode 100644 data/scripts/read_first_frame_number.py delete mode 100644 data/scripts/read_multiport.py create mode 100644 data/scripts/verify_rawfile_writing.py create mode 100644 include/aare/core.hpp create mode 100644 include/aare/file_io.hpp create mode 100644 include/aare/network_io.hpp create mode 100644 include/aare/utils.hpp create mode 100644 utils/include/aare/utils/json.hpp diff --git a/.env.dev b/.env similarity index 100% rename from .env.dev rename to .env diff --git a/core/include/aare/core/defs.hpp b/core/include/aare/core/defs.hpp index 73e9688..2249075 100644 --- a/core/include/aare/core/defs.hpp +++ b/core/include/aare/core/defs.hpp @@ -46,15 +46,16 @@ struct sls_detector_header { }; struct xy { - int row; - int col; + size_t row; + size_t col; bool operator==(const xy &other) const { return row == other.row && col == other.col; } bool operator!=(const xy &other) const { return !(*this == other); } + std::string to_string() const { return "{ x: " + std::to_string(row) + " y: " + std::to_string(col) + " }"; } }; using dynamic_shape = std::vector; -enum class DetectorType { Jungfrau, Eiger, Mythen3, Moench, ChipTestBoard }; +enum class DetectorType { Jungfrau, Eiger, Mythen3, Moench, ChipTestBoard, Unknown }; enum class TimingMode { Auto, Trigger }; @@ -69,17 +70,4 @@ template <> TimingMode StringTo(const std::string & /*mode*/); using DataTypeVariants = std::variant; -struct RawFileConfig { - int module_gap_row{}; - int module_gap_col{}; - - bool operator==(const RawFileConfig &other) const { - if (module_gap_col != other.module_gap_col) - return false; - if (module_gap_row != other.module_gap_row) - return false; - return true; - } -}; - } // namespace aare \ No newline at end of file diff --git a/data/jungfrau/read_frame.py b/data/jungfrau/read_frame.py index ee5ceaf..9413ccc 100644 --- a/data/jungfrau/read_frame.py +++ b/data/jungfrau/read_frame.py @@ -22,14 +22,15 @@ header_dt = np.dtype( ) # Read three frames from a jungfrau file with a single interface -rows = 512 -cols = 1024 -frames = 10 +rows = 1024 +cols = 512 +frames = 1 data = np.zeros((frames,rows,cols), dtype = np.uint16) header = np.zeros(frames, dtype = header_dt) -for file_id in range(4): - file_name = 'jungfrau_single_d0_f{}_0.raw'.format(file_id) +for frame in range(frames): + + file_name = '/tmp/raw_example_writing_master_' print("Reading file:", file_name) with open(file_name) as f: for i in range(3 if file_id != 3 else 1): diff --git a/data/jungfrau/read_multiport.py b/data/jungfrau/read_multiport.py index de8a4ae..9ff2c9f 100644 --- a/data/jungfrau/read_multiport.py +++ b/data/jungfrau/read_multiport.py @@ -1,6 +1,4 @@ import numpy as np -import matplotlib.pyplot as plt -plt.ion() header_dt = np.dtype( [ @@ -23,9 +21,10 @@ header_dt = np.dtype( # Read three frames from a jungfrau file with a single interface -frames = 1 +frames = 10 parts = 2 - +frame_per_file = 3 +bytes_per_pixel = 2 frame_cols = 1024 frame_rows = 512 @@ -39,14 +38,29 @@ header = np.zeros((frames,parts), dtype = header_dt) +# verify that all parts have the same frame number +for frame in range(frames): + for part in range(parts): + file_name = f'jungfrau_double_d{part}_f{frame//frame_per_file}_{0}.raw' + with open(file_name) as f: + offset = (frame%frame_per_file)*(header_dt.itemsize+part_rows*part_cols*bytes_per_pixel) + # print(f"Reading file: {file_name} at offset {offset}") + header[frame,part] = np.fromfile(f, dtype=header_dt, count = 1,offset=offset) + # print(f"Frame {frame} part {part} frame number: {header[frame,part]['Frame Number']}") + if part > 0: + assert header[frame,part]['Frame Number'] == header[frame,0]['Frame Number'] + +print("[X] All parts have the same frame number\n") + for frame in range(frames): for part in range(parts): - file_name = f'jungfrau_double_d{part}_f{frame}_{0}.raw' - print("Reading file:", file_name) + file_name = f'jungfrau_double_d{part}_f{frame//frame_per_file}_{0}.raw' + # print("Reading file:", file_name) with open(file_name) as f: - header[frame,part] = np.fromfile(f, dtype=header_dt, count = 1) + offset = (frame%frame_per_file)*(header_dt.itemsize+part_rows*part_cols*bytes_per_pixel) + header[frame,part] = np.fromfile(f, dtype=header_dt, count = 1, offset=offset) parts_data[frame,part] = np.fromfile(f, dtype=np.uint16,count = part_rows*part_cols).reshape(part_rows,part_cols) @@ -54,27 +68,20 @@ for frame in range(frames): -# for frame in range(frames): -# print("Frame:", frame) -# print("Data:\n", data[frame]) - -# print(data[0,0,0]) -# print(data[0,0,1]) -# print(data[0,0,50]) -print(data[0,0,0]) -print(data[0,0,1]) -print(data[0,255,1023]) - -print(data[0,511,1023]) -# print() -# print(parts_data[0,0,0,0]) -# print(parts_data[0,0,0,1]) -# print(parts_data[0,0,1,0]) - -# print(data.shape) - - - -#fig, ax = plt.subplots() -#im = ax.imshow(data[0]) -#im.set_clim(2000,4000) +pixel_0_0,pixel_0_1,pixel_1_0,pixel_255_1023,pixel_511_1023,= [],[],[],[],[] +for frame in range(frames): + pixel_0_0.append(data[frame,0,0]) + pixel_0_1.append(data[frame,0,1]) + pixel_1_0.append(data[frame,1,0]) + pixel_255_1023.append(data[frame,255,1023]) + pixel_511_1023.append(data[frame,511,1023]) +print("upper left corner of each frame (pixel_0_0)") +print(pixel_0_0) +print("first pixel on new line of each frame (pixel_1_0)") +print(pixel_1_0) +print("second pixel of the first line of each frame (pixel_0_1)") +print(pixel_0_1) +print("first pixel of the second part on the last line of each frame (pixel_255_1023)") +print(pixel_255_1023) +print("lower right corner of each frame (pixel_511_1023)") +print(pixel_511_1023) diff --git a/data/mythen/CORRECTED_scan242_d0_f0_3.raw b/data/mythen/CORRECTED_scan242_d0_f0_3.raw new file mode 100644 index 0000000000000000000000000000000000000000..66c0c566d12384a9e904ecff5e6b55ba699b6834 GIT binary patch literal 5232 zcma*n=aQCH5CmWpb41B`i7sKuNf7u%5OdB6445cOFT>07m)B)K>uKu9FTbj;GiRo! zr{}y2=hyuE`|IzOH4lu~@#mepYnP{c!qxK3`EU9E-PR5vIuDH&r~88gBYn@0TIZIB zgFokKLw~K6*HgW-aAr?4r?bjwpHok{>hQ?Ow=dUwy!z7PH*Xzb0Pa z5@-HZt98B2RmZ(ETBp%?_90iT>U6#ns-^Dk*Yn}T`fx$24M znz}iDvtD`CYd?-$>%8fo{kb<^-dq>cbJAjFRb)^V!MBh-^vf4x1H81z6j)Z5$I zQ_0bpwPs#@IonH*e*NiHuSUJnc|6dEU!J?(`7Eb%`mJfv0A9OKb)QHMx+5J>r}gAG z&Bl?VasM0Dv%G$Hy5*ZKFW27Az%|ERU5r!B=M(Go9`x#c?icIv?e_$Jn*$nPzS=Xv zvw=DCdgSPLS6X5mG2YXGoRe1fY5ZJrys7JPwujEVm-;+o?(@dDJnC=kuBMM)dGBJb zoY*criun>d<$KqMPfxBha5n~e`2p{>KEM^@(o&Omr?F++Y49U2)(5_WJZ|Uem#>bx z*tsE~A>JBPcT-}!tMdxt>f?*I1%CSintJNd-Hz>T9Y-F_v9EJ`Kv&&eOsia{_fV&4 zHrUg>T00VVU-7PhcUorAF{>I*Q1|zYufKcHb>AJ(+ZlA8`SrMz7`&tR=zF=_#Z264 z)1^~QJh=nG{=j?DamUe5Q_Kr3yYr)$uG-$f4m1u1^7bmvNt1^B%82cvU+;22=V)LK zp4!>qaBwJ4KOWF>=T#rzm`_`uJFR1Zd^t35_{4D@324$*qpMf#WN;?HaaWTw-#+Jp z69ImE@o1Udx}H`ycqGPIpMXy+q^fy+q%=U@7uwf!3Tl) zp9F6V@W}_BIk>z6?l*ybTI())J_?+CdV}hkW480_0Pl-{r*DI=f?oo&e++Q!VQ)P@ z1#)`$_&MN<{v!A+;Ooo4Y`gJ?%irDL^T7M?reZItY>0Y-J(*(1A2yncK_>X|TIt_lm3vkrD!S?|_^nVY$ z3Er)M=C6SoFTVwRdwU+_oix<>rp05=da=L12j7r358la}yNk`D?QQMPpBgRu*g?OY zXz_b9(2HwNed<>Oef0D-Z>~L@dT_kGyVxG)_q)`K@8<3J@do`yv_YTm&x`%N;q8EL z|E|pEPp`jaTyNuD^w97YeLH^>cufC1{Bm7@rmqp-P zMsUrym}|+jR5ELh%b)&`J7iqPF)`#mjhahahC7<`u~pZ znLrP3^!1$xE(LsF2x#zqZQM_e*nVm>@C$z*{CA%G*?70ySr>kygA^9FP`)*1@gFRytr=+Xwj$p58Q#gxBvhE literal 0 HcmV?d00001 diff --git a/data/mythen/CORRECTED_scan242_d1_f0_3.raw b/data/mythen/CORRECTED_scan242_d1_f0_3.raw new file mode 100644 index 0000000000000000000000000000000000000000..b8db27da9b51961e3f149abc6b90268da3109cc3 GIT binary patch literal 5232 zcmeIyJ5FR#41{6cZ4q>}2_~&BUk6F1K8KwCs2@Gcam;>T1QjoId_y z8GJJ{hN+K7k5{9Ik4vsE57(}I+%S_*3`4V>OwEmEnpf+tcJOJ0gNe_cIonJan(b2O zaf{8^4JTKRs~T^f*Qdc;*tpVG-aDl=eeJc!ZgaTzP49Zyhe!cpzy<^(j!_OG59r(JV`(W{g;oB*O zpF84?%WO4X-CVr9w+fHj?}oQ~ikp-7PTy&(=LUV;tly-3-r@%Nz81B~(e72=-7x$1 zGW==KZ>F2|ab7JGtHZ^`tEb6)c|Okh_G0z$d>nSAuXpHu?5jr$GfN{c_hOeCF|p;fnFhnw~t~;^y`B z(ixXsIP)}^;nS(-uYJ53eZ6uq{+m(5WpBIFZ}vM5I6a*0>)U8n9nLS|# z^ohNnf4_1(>1j4^m!3TS_Nx{1@$h!4*&F5X@C&e=4_Wlx5){q9V& zIGthCdN;e~_o&`2-3(*5xqdIbbmYy!>ZPGvO$-Ysw~MxFc8Kjz$I%{jaql4xlV0A9 zEPNdBap_MDy-QlnnhU4yyQEjl(joS(wUbxFL7$!B=bKDj`n^SXvv~2~OnY@RYI-84m=DRgLa~8h6aP3mV9~Lei{(hMn{WQ0mm&0@~E;&AZoXxIq z*K;qf-dVq!;>H29uPyF4h-4GBZrfl>1|-u&8owR)$ETG{@*eh+S%^})6V$K!)5Af=J6KWXBG!OF)Rzq zH_yZ1H6t%B_qN@kW?tSs@b>Ge7JW&EwXi&CUDu zZp6p@e+&!9@O2x$IC-;all$%ZzW_(O?T3?BkGJ>i=DyYHJkGSsVdG0vc{}+y`82@G Nd0O!O{r&D2_yl7UAa?)& literal 0 HcmV?d00001 diff --git a/data/mythen/CORRECTED_scan242_d2_f0_3.raw b/data/mythen/CORRECTED_scan242_d2_f0_3.raw new file mode 100644 index 0000000000000000000000000000000000000000..df21c945549234f0dfe5ea74a958e656bc8e11d1 GIT binary patch literal 5232 zcmc)MNp4g@5QJechc|Ox*MvU8>dP~t=N><;*`E7s zCb0adZ{@w&*f%5Ew2mI((su7`CUo?rCXPKmJvpA!gK0MATW_;H$L0-Q4If_fl^+_o zA13nQ%pXrYxaZ;X1fDz{+BB=r^O}IOZ^mC|PlG2buC-?fOUIqP=cqrkwx`qF)rRAP z4j!NW#(bRM$?6`LUidY8S~QxAGavfThu*_@N9p8E&iBz-Kh2q+_GYyvX!3^NykXos z>q%Z+QO+uFj%K$e)Z>2Q8PcwY)Z@pRd3`0YmbnVTJe_Ccl2bNCS{5L;- zVDxhIujjr?G}VlcOJ99{ob~F?3a_3zc=q{f&$B&_=RwDQP4F*oRU@xF!BoFF;nur4 z`fz7&b(V+sJlqL7%`l(z^FB^+(TKVEDG;3fj$EJ$%^eSox|ay7t#5+9Geks zwRz^Pzw>;{;iy4F@CMI&y~qo-zU);W_-fHK(>U&NtwLY4J;&xsOKs=0c~YC6J2l|q z(9)NCb4uGC9FBb$??m33Yrbjfg-_wVQJW6FGmO=&Mg6SV3!3`Ile0SG>lt4?dw0Dj zu>G%VF3xhe%_>da`hL`-b2$2GG&8=;QhnO~p3tFh9{An|JUZ^-;AzW|M^>O?AfYV++|M)#K?^!-*#xWPQdnbG^`+cfDJ-w;XibKQmvCp5o z?)9cFemH0QyvX6g;?0Uql_Z#K@>I9vh nzdekc-`d{G?%@3<_&-FB*2zRY)-RuN<>9|iz{~OWKRAvXTr(0`}}?Sd^=cs%Rhg;JAAUvhr9JGyw6$T|Fs-l-nL1`i)_tH$9=x`YJC~#k)D{`+kw1#tS_+%ehlc_Du${BUc@B=T&VycX;mg<=pztUj<_H?gl)`v(}ec;o+$FQ(!Kyf?4}&^C8b!9bUY# zy7k@po0mKFL4WF~?XC3S4xdKn%Y&Y10$;xd@_r6ziu27Q9q;6xr_NYSy7GXId_9Ow zP)$5~^4+ORM~;{rHN^OHZw_is;9kC*&ZfQTOALorwdJ}8bDRLjJZMjV)4r;!chI|7 zzZ-Pu`)=~rcaZ1a+5FY-cSJoto#p6_U#pnCI&$UFQJY`&@Qwp9`R3|v@#Kp29r*N{ zq6>QT9&-Q9#X(k_w*f!D1l6V`_f62*TkA=lK6#U)CoyMQ{P0Ytd->2eki)B)o&@U2 zcQ2=2)Z*J&4`v_-7Z2>!qKCtap7Gq#^m}Y={8^BXXT>vrKJ}-6TH<2zozzvXoJbH|tdX_&j;e}6xHW4)OL4?N-1 zr)`fXkN4jK`n-G*s3Au#E}kjJJ6T~_w@AQ_o#+-f9}rWV$k<= zH-Q>F@@@w0O%MFwn`6H>dNwmqACK+?wD7!*`H1&TX!)DPFPgriqAad6$K+c)4#-kpHI3GnIQP0%^2t0wJPdv~ zkC}t4evkU(rO+2XU0i*r=iX{=-dkT*b>#M4cY_Ju9fx#_~sU)CIYC zde>{eA9U$B>%}~rC&06Z_DfqWKCSq4C%_S}mfp-z4ZYCEh1SG1b8)%6;x(5he(Qi3B@EqQ$>{itL9?(}0O z?mCmc<>apKY^4vlpjS2Q)!|DoR%hPq(`g;Oy2F+4UTt^n$84Nw^8wd_>w!Mx@ZgMx-+FRzX?0brec|Kc(UIG?w}-21 zE!SBO>d~TGj{SapTJrGfsa`JRj>h@5e!RC{%ATKkvht|z1o}P<+<6Z*)b^q8L)Se{ z`@>Q5qd+|$wRq#F$aecAz?G{fvDU*s7g#R`2LVkvptgH)F`j5NUtB(}*#UiC^wYDI zj`tO_$CHmo*DBU~F|*-a%-%=6 -using aare::File; -using aare::Frame; +using namespace aare; void test(File &f, int frame_number) { std::cout << "frame number: " << frame_number << '\n'; @@ -26,4 +24,27 @@ int main() { test(file, 0); test(file, 2); test(file, 99); + + std::filesystem::path const path2("/tmp/raw_example_writing.json"); + aare::FileConfig config; + config.version = "1.0"; + config.geometry = {1, 1}; + config.detector_type = aare::DetectorType::Moench; + config.max_frames_per_file = 100; + config.rows = 1024; + config.cols = 512; + config.dtype = aare::DType::UINT16; + File file2(path2, "w", config); + Frame frame(1024, 512, 16); + + for (int i = 0; i < 1024; i++) { + for (int j = 0; j < 512; j++) { + frame.set(i, j, (uint16_t)(i + j)); + } + } + + sls_detector_header header; + header.frameNumber = 0; + file2.write(frame, header); + file2.set_total_frames(1); } \ No newline at end of file diff --git a/file_io/include/aare/file_io/File.hpp b/file_io/include/aare/file_io/File.hpp index ee00714..8ab7f80 100644 --- a/file_io/include/aare/file_io/File.hpp +++ b/file_io/include/aare/file_io/File.hpp @@ -11,6 +11,7 @@ namespace aare { class File { private: FileInterface *file_impl; + bool is_npy = true; public: /** @@ -22,8 +23,8 @@ class File { * @throws std::invalid_argument if the file mode is not supported * */ - File(const std::filesystem::path &fname, const std::string &mode, FileConfig cfg = {}); - void write(Frame &frame); + File(const std::filesystem::path &fname, const std::string &mode, const FileConfig &cfg = {}); + void write(Frame &frame, sls_detector_header header = {}); Frame read(); Frame iread(size_t frame_number); std::vector read(size_t n_frames); @@ -38,6 +39,7 @@ class File { size_t rows() const; size_t cols() const; size_t bitdepth() const; + void set_total_frames(size_t total_frames); /** * @brief Move constructor diff --git a/file_io/include/aare/file_io/FileInterface.hpp b/file_io/include/aare/file_io/FileInterface.hpp index ed89f74..0af109f 100644 --- a/file_io/include/aare/file_io/FileInterface.hpp +++ b/file_io/include/aare/file_io/FileInterface.hpp @@ -15,14 +15,21 @@ namespace aare { * geometry: geometry of the file */ struct FileConfig { - aare::DType dtype = aare::DType(typeid(uint16_t)); + aare::DType dtype{typeid(uint16_t)}; uint64_t rows{}; uint64_t cols{}; - xy geometry{1, 1}; bool operator==(const FileConfig &other) const { - return dtype == other.dtype && rows == other.rows && cols == other.cols && geometry == other.geometry; + return dtype == other.dtype && rows == other.rows && cols == other.cols && geometry == other.geometry && + detector_type == other.detector_type && max_frames_per_file == other.max_frames_per_file; } bool operator!=(const FileConfig &other) const { return !(*this == other); } + + // rawfile specific + std::string version{}; + xy geometry{1, 1}; + DetectorType detector_type{DetectorType::Unknown}; + int max_frames_per_file{}; + size_t total_frames{}; }; /** @@ -38,7 +45,7 @@ class FileInterface { * @return void * @throws std::runtime_error if the function is not implemented */ - virtual void write(Frame &frame) = 0; + // virtual void write(Frame &frame) = 0; /** * @brief write a vector of frames to the file @@ -160,6 +167,8 @@ class FileInterface { virtual ~FileInterface() = default; + void set_total_frames(size_t total_frames) { m_total_frames = total_frames; } + protected: std::string m_mode{}; std::filesystem::path m_fname{}; diff --git a/file_io/include/aare/file_io/NumpyFile.hpp b/file_io/include/aare/file_io/NumpyFile.hpp index 5a72179..3288626 100644 --- a/file_io/include/aare/file_io/NumpyFile.hpp +++ b/file_io/include/aare/file_io/NumpyFile.hpp @@ -27,7 +27,7 @@ class NumpyFile : public FileInterface { */ explicit NumpyFile(const std::filesystem::path &fname, const std::string &mode = "r", FileConfig cfg = {}); - void write(Frame &frame) override; + void write(Frame &frame); Frame read() override { return get_frame(this->current_frame++); } std::vector read(size_t n_frames) override; diff --git a/file_io/include/aare/file_io/RawFile.hpp b/file_io/include/aare/file_io/RawFile.hpp index cc7b399..1047d1a 100644 --- a/file_io/include/aare/file_io/RawFile.hpp +++ b/file_io/include/aare/file_io/RawFile.hpp @@ -5,6 +5,19 @@ namespace aare { +struct ModuleConfig { + int module_gap_row{}; + int module_gap_col{}; + + bool operator==(const ModuleConfig &other) const { + if (module_gap_col != other.module_gap_col) + return false; + if (module_gap_row != other.module_gap_row) + return false; + return true; + } +}; + /** * @brief RawFile class to read .raw and .json files * @note derived from FileInterface @@ -18,13 +31,14 @@ class RawFile : public FileInterface { * @param mode file mode (r, w) * @param cfg file configuration */ - explicit RawFile(const std::filesystem::path &fname, const std::string &mode = "r", const FileConfig &config = {}); + explicit RawFile(const std::filesystem::path &fname, const std::string &mode = "r", + const FileConfig &config = FileConfig{}); /** * @brief write function is not implemented for RawFile * @param frame frame to write */ - void write([[maybe_unused]] Frame &frame) override { throw std::runtime_error("Not implemented"); }; + void write(Frame &frame, sls_detector_header header); Frame read() override { return get_frame(this->current_frame++); }; std::vector read(size_t n_frames) override; void read_into(std::byte *image_buf) override { return get_frame_into(this->current_frame++, image_buf); }; @@ -44,7 +58,15 @@ class RawFile : public FileInterface { size_t pixels_per_frame() override { return m_rows * m_cols; } // goto frame number - void seek(size_t frame_number) override { this->current_frame = frame_number; }; + void seek(size_t frame_number) override { + // check if the frame number is greater than the total frames + // if frame_number == total_frames, then the next read will throw an error + if (frame_number > this->total_frames()) { + throw std::runtime_error( + fmt::format("frame number {} is greater than total frames {}", frame_number, m_total_frames)); + } + this->current_frame = frame_number; + }; // return the position of the file pointer (in number of frames) size_t tell() override { return this->current_frame; }; @@ -88,7 +110,7 @@ class RawFile : public FileInterface { /** * @brief destructor: will delete the subfiles */ - ~RawFile() override; + ~RawFile() noexcept override; size_t total_frames() const override { return m_total_frames; } size_t rows() const override { return m_rows; } @@ -96,19 +118,20 @@ class RawFile : public FileInterface { size_t bitdepth() const override { return m_bitdepth; } private: + void write_master_file(); /** - * @brief read the frame at the given frame number into the image buffer + * @brief read the frame at the given frame index into the image buffer * @param frame_number frame number to read * @param image_buf buffer to store the frame */ - void get_frame_into(size_t frame_number, std::byte *frame_buffer); + void get_frame_into(size_t frame_index, std::byte *frame_buffer); /** - * @brief get the frame at the given frame number + * @brief get the frame at the given frame index * @param frame_number frame number to read * @return Frame */ - Frame get_frame(size_t frame_number); + Frame get_frame(size_t frame_index); /** * @brief parse the file name to get the extension, base name and index @@ -146,6 +169,7 @@ class RawFile : public FileInterface { * @brief open the subfiles */ void open_subfiles(); + void parse_config(const FileConfig &config); size_t n_subfiles{}; size_t n_subfile_parts{}; @@ -153,7 +177,7 @@ class RawFile : public FileInterface { size_t subfile_rows{}, subfile_cols{}; xy geometry{}; std::vector positions; - RawFileConfig cfg{0, 0}; + ModuleConfig cfg{0, 0}; TimingMode timing_mode{}; bool quad{false}; }; diff --git a/file_io/include/aare/file_io/SubFile.hpp b/file_io/include/aare/file_io/SubFile.hpp index f5b2a3d..908cb70 100644 --- a/file_io/include/aare/file_io/SubFile.hpp +++ b/file_io/include/aare/file_io/SubFile.hpp @@ -1,4 +1,5 @@ #pragma once +#include "aare/core/Frame.hpp" #include "aare/core/defs.hpp" #include #include @@ -36,6 +37,7 @@ class SubFile { }; public: + size_t write_part(std::byte *buffer, sls_detector_header header, size_t frame_index); /** * @brief SubFile constructor * @param fname path to the subfile @@ -45,7 +47,8 @@ class SubFile { * @param bitdepth bitdepth of the subfile * @throws std::invalid_argument if the detector,type pair is not supported */ - SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth); + SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth, + const std::string &mode = "r"); /** * @brief read the subfile into a buffer @@ -74,19 +77,22 @@ class SubFile { * @param frame_number frame number to read * @return number of bytes read */ - size_t get_part(std::byte *buffer, size_t frame_number); + size_t get_part(std::byte *buffer, size_t frame_index); size_t frame_number(size_t frame_index); // TODO: define the inlines as variables and assign them in constructor inline size_t bytes_per_part() const { return (m_bitdepth / 8) * m_rows * m_cols; } inline size_t pixels_per_part() const { return m_rows * m_cols; } + ~SubFile(); + protected: FILE *fp = nullptr; size_t m_bitdepth; std::filesystem::path m_fname; size_t m_rows{}; size_t m_cols{}; + std::string m_mode; size_t n_frames{}; int m_sub_file_index_{}; }; diff --git a/file_io/src/File.cpp b/file_io/src/File.cpp index 8848be8..8507186 100644 --- a/file_io/src/File.cpp +++ b/file_io/src/File.cpp @@ -6,7 +6,7 @@ namespace aare { -File::File(const std::filesystem::path &fname, const std::string &mode, FileConfig cfg) { +File::File(const std::filesystem::path &fname, const std::string &mode, const FileConfig &cfg) { if (mode != "r" && mode != "w" && mode != "a") { throw std::invalid_argument("Unsupported file mode"); } @@ -18,6 +18,7 @@ File::File(const std::filesystem::path &fname, const std::string &mode, FileConf if (fname.extension() == ".raw" || fname.extension() == ".json") { aare::logger::debug("Loading raw file"); file_impl = new RawFile(fname, mode, cfg); + is_npy = false; } // check if extension is numpy else if (fname.extension() == ".npy") { @@ -28,7 +29,14 @@ File::File(const std::filesystem::path &fname, const std::string &mode, FileConf } } -void File::write(Frame &frame) { file_impl->write(frame); } +void File::write(Frame &frame, sls_detector_header header) { + if (is_npy) { + aare::logger::info("ignoring header for npy file"); + dynamic_cast(file_impl)->write(frame); + } else { + dynamic_cast(file_impl)->write(frame, header); + } +} Frame File::read() { return file_impl->read(); } size_t File::total_frames() const { return file_impl->total_frames(); } std::vector File::read(size_t n_frames) { return file_impl->read(n_frames); } @@ -42,6 +50,7 @@ size_t File::tell() const { return file_impl->tell(); } size_t File::rows() const { return file_impl->rows(); } size_t File::cols() const { return file_impl->cols(); } size_t File::bitdepth() const { return file_impl->bitdepth(); } +void File::set_total_frames(size_t total_frames) { return file_impl->set_total_frames(total_frames); } File::~File() { delete file_impl; } Frame File::iread(size_t frame_number) { return file_impl->iread(frame_number); } diff --git a/file_io/src/RawFile.cpp b/file_io/src/RawFile.cpp index 10720f2..8c09da8 100644 --- a/file_io/src/RawFile.cpp +++ b/file_io/src/RawFile.cpp @@ -1,5 +1,6 @@ #include "aare/file_io/RawFile.hpp" #include "aare/core/defs.hpp" +#include "aare/utils/json.hpp" #include "aare/utils/logger.hpp" #include #include @@ -9,8 +10,9 @@ using json = nlohmann::json; namespace aare { RawFile::RawFile(const std::filesystem::path &fname, const std::string &mode, const FileConfig &config) { + m_mode = mode; m_fname = fname; - if (mode == "r") { + if (mode == "r" or mode == "r+") { if (config != FileConfig()) { aare::logger::warn( "In read mode it is not necessary to provide a config, the provided config will be ignored"); @@ -21,17 +23,108 @@ RawFile::RawFile(const std::filesystem::path &fname, const std::string &mode, co find_geometry(); open_subfiles(); + } else if (mode == "w" or mode == "w+") { + + if (std::filesystem::exists(fname)) { + // handle mode w as w+ (no overrwriting) + throw std::runtime_error(LOCATION + "File already exists"); + } + + parse_config(config); + parse_fname(); + write_master_file(); + n_subfiles = 1; + n_subfile_parts = 1; + subfile_cols = m_cols; + subfile_rows = m_rows; + open_subfiles(); + } else { throw std::runtime_error(LOCATION + "Unsupported mode"); } } +void RawFile::parse_config(const FileConfig &config) { + m_bitdepth = config.dtype.bitdepth(); + m_total_frames = config.total_frames; + m_rows = config.rows; + m_cols = config.cols; + m_type = config.detector_type; + max_frames_per_file = config.max_frames_per_file; + geometry = config.geometry; + version = config.version; + subfile_rows = config.geometry.row; + subfile_cols = config.geometry.col; + + if (geometry != aare::xy{1, 1}) { + throw std::runtime_error(LOCATION + "Only geometry {1,1} files are supported for writing"); + } +} +void RawFile::write_master_file() { + if (m_ext != ".json") { + throw std::runtime_error(LOCATION + "only json master files are supported for writing"); + } + std::ofstream ofs(master_fname(), std::ios::binary); + std::string ss; + ss.reserve(1024); + ss += "{\n\t"; + aare::write_str(ss, "Version", version); + ss += "\n\t"; + aare::write_digit(ss, "Total Frames", m_total_frames); + ss += "\n\t"; + aare::write_str(ss, "Detector Type", toString(m_type)); + ss += "\n\t"; + aare::write_str(ss, "Geometry", geometry.to_string()); + ss += "\n\t"; + + uint64_t img_size = (m_cols * m_rows) / (geometry.col * geometry.row); + img_size *= m_bitdepth; + aare::write_digit(ss, "Image Size in bytes", img_size); + ss += "\n\t"; + aare::write_digit(ss, "Max Frames Per File", max_frames_per_file); + ss += "\n\t"; + aare::write_digit(ss, "Dynamic Range", m_bitdepth); + ss += "\n\t"; + const aare::xy pixels = {m_rows / geometry.row, m_cols / geometry.col}; + aare::write_str(ss, "Pixels", pixels.to_string()); + ss += "\n\t"; + aare::write_digit(ss, "Number of rows", m_rows); + ss += "\n\t"; + const std::string tmp = "{\n" + " \"Frame Number\": \"8 bytes\",\n" + " \"Exposure Length\": \"4 bytes\",\n" + " \"Packet Number\": \"4 bytes\",\n" + " \"Bunch Id\": \"8 bytes\",\n" + " \"Timestamp\": \"8 bytes\",\n" + " \"Module Id\": \"2 bytes\",\n" + " \"Row\": \"2 bytes\",\n" + " \"Column\": \"2 bytes\",\n" + " \"Reserved\": \"2 bytes\",\n" + " \"Debug\": \"4 bytes\",\n" + " \"RoundRNumber\": \"2 bytes\",\n" + " \"DetType\": \"1 byte\",\n" + " \"Version\": \"1 byte\",\n" + " \"Packet Mask\": \"64 bytes\"\n" + " }"; + + ss += "\"Frame Header Format\":" + tmp + "\n"; + ss += "}"; + ofs << ss; + ofs.close(); +} + void RawFile::open_subfiles() { - for (size_t i = 0; i != n_subfiles; ++i) { - auto v = std::vector(n_subfile_parts); - for (size_t j = 0; j != n_subfile_parts; ++j) { - v[j] = new SubFile(data_fname(i, j), m_type, subfile_rows, subfile_cols, m_bitdepth); + if (m_mode == "r") + for (size_t i = 0; i != n_subfiles; ++i) { + auto v = std::vector(n_subfile_parts); + for (size_t j = 0; j != n_subfile_parts; ++j) { + v[j] = new SubFile(data_fname(i, j), m_type, subfile_rows, subfile_cols, m_bitdepth); + } + subfiles.push_back(v); } + else { + auto v = std::vector(n_subfile_parts); // only one subfile is implemented + v[0] = new SubFile(data_fname(0, 0), m_type, m_rows, m_cols, m_bitdepth, "w"); subfiles.push_back(v); } } @@ -173,51 +266,97 @@ void RawFile::parse_raw_metadata() { max_frames_per_file = std::stoi(value); } else if (key == "Geometry") { pos = value.find(','); - geometry = {std::stoi(value.substr(1, pos)), std::stoi(value.substr(pos + 1))}; + const size_t x = static_cast(std::stoi(value.substr(1, pos))); + const size_t y = static_cast(std::stoi(value.substr(pos + 1))); + + geometry = {x, y}; } } } } void RawFile::parse_fname() { + bool wrong_format = false; m_base_path = m_fname.parent_path(); m_base_name = m_fname.stem(); m_ext = m_fname.extension(); - auto pos = m_base_name.rfind('_'); - m_findex = std::stoi(m_base_name.substr(pos + 1)); - pos = m_base_name.find("_master_"); - m_base_name.erase(pos); + try { + auto pos = m_base_name.rfind('_'); + m_findex = std::stoi(m_base_name.substr(pos + 1)); + } catch (const std::invalid_argument &e) { + m_findex = 0; + wrong_format = true; + } + auto pos = m_base_name.find("_master_"); + if (pos != std::string::npos) { + m_base_name.erase(pos); + wrong_format = true; + } + if (wrong_format and (m_mode == "w+" or m_mode == "w")) { + aare::logger::warn("Master Filename", m_fname, "is not in the correct format"); + aare::logger::warn("using", master_fname(), "as the master file"); + } } -Frame RawFile::get_frame(size_t frame_number) { +Frame RawFile::get_frame(size_t frame_index) { auto f = Frame(this->m_rows, this->m_cols, this->m_bitdepth); std::byte *frame_buffer = f.data(); - get_frame_into(frame_number, frame_buffer); + get_frame_into(frame_index, frame_buffer); return f; } -void RawFile::get_frame_into(size_t frame_number, std::byte *frame_buffer) { - if (frame_number > this->m_total_frames) { +void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer) { + if (frame_index > this->m_total_frames) { throw std::runtime_error(LOCATION + "Frame number out of range"); } - size_t const subfile_id = frame_number / this->max_frames_per_file; - // create frame and get its buffer + std::vector frame_numbers(this->n_subfile_parts); + std::vector frame_indices(this->n_subfile_parts, frame_index); + + if (n_subfile_parts != 1) { + for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) { + auto subfile_id = frame_index / this->max_frames_per_file; + frame_numbers[part_idx] = + this->subfiles[subfile_id][part_idx]->frame_number(frame_index % this->max_frames_per_file); + } + // 1. if frame number vector is the same break + while (std::adjacent_find(frame_numbers.begin(), frame_numbers.end(), std::not_equal_to<>()) != + frame_numbers.end()) { + // 2. find the index of the minimum frame number, + auto min_frame_idx = + std::distance(frame_numbers.begin(), std::min_element(frame_numbers.begin(), frame_numbers.end())); + // 3. increase its index and update its respective frame number + frame_indices[min_frame_idx]++; + // 4. if we can't increase its index => throw error + if (frame_indices[min_frame_idx] >= this->m_total_frames) { + throw std::runtime_error(LOCATION + "Frame number out of range"); + } + auto subfile_id = frame_indices[min_frame_idx] / this->max_frames_per_file; + frame_numbers[min_frame_idx] = this->subfiles[subfile_id][min_frame_idx]->frame_number( + frame_indices[min_frame_idx] % this->max_frames_per_file); + } + } if (this->geometry.col == 1) { // get the part from each subfile and copy it to the frame for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) { + auto corrected_idx = frame_indices[part_idx]; + auto subfile_id = corrected_idx / this->max_frames_per_file; auto part_offset = this->subfiles[subfile_id][part_idx]->bytes_per_part(); this->subfiles[subfile_id][part_idx]->get_part(frame_buffer + part_idx * part_offset, - frame_number % this->max_frames_per_file); + corrected_idx % this->max_frames_per_file); } } else { + // create a buffer that will hold a the frame part auto bytes_per_part = this->subfile_rows * this->subfile_cols * this->m_bitdepth / 8; auto *part_buffer = new std::byte[bytes_per_part]; for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) { - this->subfiles[subfile_id][part_idx]->get_part(part_buffer, frame_number % this->max_frames_per_file); + auto corrected_idx = frame_indices[part_idx]; + auto subfile_id = corrected_idx / this->max_frames_per_file; + + this->subfiles[subfile_id][part_idx]->get_part(part_buffer, corrected_idx % this->max_frames_per_file); for (size_t cur_row = 0; cur_row < (this->subfile_rows); cur_row++) { auto irow = cur_row + (part_idx / this->geometry.col) * this->subfile_rows; auto icol = (part_idx % this->geometry.col) * this->subfile_cols; @@ -231,6 +370,19 @@ void RawFile::get_frame_into(size_t frame_number, std::byte *frame_buffer) { } } +void RawFile::write(Frame &frame, sls_detector_header header) { + if (m_mode == "r") { + throw std::runtime_error(LOCATION + "File is open in read mode"); + } + size_t const subfile_id = this->current_frame / this->max_frames_per_file; + for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) { + + this->subfiles[subfile_id][part_idx]->write_part(frame.data(), header, + this->current_frame % this->max_frames_per_file); + } + this->current_frame++; +} + std::vector RawFile::read(size_t n_frames) { // TODO: implement this in a more efficient way std::vector frames; @@ -256,7 +408,17 @@ size_t RawFile::frame_number(size_t frame_index) { return this->subfiles[subfile_id][0]->frame_number(frame_index % this->max_frames_per_file); } -RawFile::~RawFile() { +RawFile::~RawFile() noexcept { + + // update master file + if (m_mode == "w" or m_mode == "w+" or m_mode == "r+") { + try { + write_master_file(); + } catch (...) { + aare::logger::warn(LOCATION + "Could not update master file"); + } + } + for (auto &vec : subfiles) { for (auto *subfile : vec) { delete subfile; diff --git a/file_io/src/SubFile.cpp b/file_io/src/SubFile.cpp index dc1e085..af30854 100644 --- a/file_io/src/SubFile.cpp +++ b/file_io/src/SubFile.cpp @@ -7,9 +7,9 @@ namespace aare { -SubFile::SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth) - : m_bitdepth(bitdepth), m_fname(fname), m_rows(rows), m_cols(cols), - n_frames(std::filesystem::file_size(fname) / (sizeof(sls_detector_header) + rows * cols * bitdepth / 8)) { +SubFile::SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth, + const std::string &mode) + : m_bitdepth(bitdepth), m_fname(fname), m_rows(rows), m_cols(cols), m_mode(mode) { if (read_impl_map.find({detector, bitdepth}) == read_impl_map.end()) { auto error_msg = LOCATION + "No read_impl function found for detector: " + toString(detector) + @@ -17,25 +17,49 @@ SubFile::SubFile(const std::filesystem::path &fname, DetectorType detector, size throw std::invalid_argument(error_msg); } this->read_impl = read_impl_map.at({detector, bitdepth}); + if (std::filesystem::exists(fname)) { + n_frames = std::filesystem::file_size(fname) / (sizeof(sls_detector_header) + rows * cols * bitdepth / 8); + } else { + n_frames = 0; + } + + if (mode == "r") { + fp = fopen(m_fname.c_str(), "rb"); + } else { + // if file exists, open in read/write mode (without truncating the file) + // if file does not exist, open in write mode + if (std::filesystem::exists(fname)) { + fp = fopen(m_fname.c_str(), "r+b"); + } else { + fp = fopen(m_fname.c_str(), "wb"); + } + } + if (fp == nullptr) { + throw std::runtime_error(LOCATION + "Could not open file for writing"); + } } -size_t SubFile::get_part(std::byte *buffer, size_t frame_number) { - if (frame_number >= n_frames) { +size_t SubFile::get_part(std::byte *buffer, size_t frame_index) { + if (frame_index >= n_frames) { throw std::runtime_error("Frame number out of range"); } // TODO: find a way to avoid opening and closing the file for each frame - aare::logger::debug(LOCATION, "frame:", frame_number, "file:", m_fname.c_str()); - fp = fopen(m_fname.c_str(), "rb"); - if (!fp) { - throw std::runtime_error(fmt::format("Could not open: {} for reading", m_fname.c_str())); - } - fseek(fp, sizeof(sls_detector_header) + (sizeof(sls_detector_header) + bytes_per_part()) * frame_number, // NOLINT + aare::logger::debug(LOCATION, "frame:", frame_index, "file:", m_fname.c_str()); + fseek(fp, sizeof(sls_detector_header) + (sizeof(sls_detector_header) + bytes_per_part()) * frame_index, // NOLINT SEEK_SET); auto ret = (this->*read_impl)(buffer); - if (fclose(fp)) - throw std::runtime_error(LOCATION + "Could not close file"); return ret; } +size_t SubFile::write_part(std::byte *buffer, sls_detector_header header, size_t frame_index) { + if (frame_index > n_frames) { + throw std::runtime_error("Frame number out of range"); + } + fseek(fp, static_cast((sizeof(sls_detector_header) + bytes_per_part()) * frame_index), SEEK_SET); + auto wc = fwrite(reinterpret_cast(&header), sizeof(header), 1, fp); + wc += fwrite(buffer, bytes_per_part(), 1, fp); + + return wc; +} size_t SubFile::read_impl_normal(std::byte *buffer) { return fread(buffer, this->bytes_per_part(), 1, this->fp); } @@ -91,18 +115,18 @@ template size_t SubFile::read_impl_flip(std::byte *buffer) { size_t SubFile::frame_number(size_t frame_index) { sls_detector_header h{}; - fp = fopen(this->m_fname.c_str(), "r"); - if (!fp) - throw std::runtime_error(LOCATION + fmt::format("Could not open: {} for reading", m_fname.c_str())); fseek(fp, (sizeof(sls_detector_header) + bytes_per_part()) * frame_index, SEEK_SET); // NOLINT size_t const rc = fread(reinterpret_cast(&h), sizeof(h), 1, fp); if (rc != 1) throw std::runtime_error(LOCATION + "Could not read header from file"); - if (fclose(fp)) { - throw std::runtime_error(LOCATION + "Could not close file"); - } return h.frameNumber; } +SubFile::~SubFile() { + if (fp) { + fclose(fp); + } +} + } // namespace aare \ No newline at end of file diff --git a/file_io/test/RawFile.test.cpp b/file_io/test/RawFile.test.cpp index 34f2f03..155057f 100644 --- a/file_io/test/RawFile.test.cpp +++ b/file_io/test/RawFile.test.cpp @@ -79,4 +79,36 @@ TEST_CASE("Compare reading from a numpy file with a raw file") { auto npy_frame = npy.read(); CHECK(raw_frame.view() == npy_frame.view()); } -} \ No newline at end of file +} + +TEST_CASE("Read multipart files") { + auto fpath = test_data_path() / "jungfrau" / "jungfrau_double_master_0.json"; + REQUIRE(std::filesystem::exists(fpath)); + + File f(fpath, "r"); + + // we know this file has 10 frames check read_multiport.py for the values + std::vector pixel_0_0 = {2099, 2121, 2108, 2084, 2084, 2118, 2066, 2108, 2112, 2116}; + std::vector pixel_0_1 = {2842, 2796, 2865, 2798, 2805, 2817, 2852, 2789, 2792, 2833}; + std::vector pixel_255_1023 = {2149, 2037, 2115, 2102, 2118, 2090, 2036, 2071, 2073, 2142}; + std::vector pixel_511_1023 = {3231, 3169, 3167, 3162, 3168, 3160, 3171, 3171, 3169, 3171}; + std::vector pixel_1_0 = {2748, 2614, 2665, 2629, 2618, 2630, 2631, 2634, 2577, 2598}; + + for (size_t i = 0; i < 10; i++) { + auto frame = f.read(); + CHECK(frame.rows() == 512); + CHECK(frame.cols() == 1024); + CHECK(frame.view()(0, 0) == pixel_0_0[i]); + CHECK(frame.view()(0, 1) == pixel_0_1[i]); + CHECK(frame.view()(1, 0) == pixel_1_0[i]); + CHECK(frame.view()(255, 1023) == pixel_255_1023[i]); + CHECK(frame.view()(511, 1023) == pixel_511_1023[i]); + } +} + +TEST_CASE("Read file with unordered frames") { + auto fpath = test_data_path() / "mythen" / "scan242_master_3.raw"; + REQUIRE(std::filesystem::exists(fpath)); + File f(fpath, "r"); + REQUIRE_THROWS(f.read()); +} diff --git a/include/aare/aare.hpp b/include/aare/aare.hpp index 78a74d0..92eadf2 100644 --- a/include/aare/aare.hpp +++ b/include/aare/aare.hpp @@ -1 +1,7 @@ -// This is the top level header to include and what most users will use \ No newline at end of file +// This is the top level header to include and what most users will use + +// include all header files +#include "aare/core.hpp" +#include "aare/file_io.hpp" +#include "aare/network_io.hpp" +#include "aare/utils.hpp" \ No newline at end of file diff --git a/include/aare/core.hpp b/include/aare/core.hpp new file mode 100644 index 0000000..1cf2110 --- /dev/null +++ b/include/aare/core.hpp @@ -0,0 +1,8 @@ +#include "aare/core/CircularFifo.hpp" +#include "aare/core/DType.hpp" +#include "aare/core/Frame.hpp" +#include "aare/core/NDArray.hpp" +#include "aare/core/NDView.hpp" +#include "aare/core/defs.hpp" +// #include "aare/core/VariableSizeClusterFinder.hpp" +#include "aare/core/ProducerConsumerQueue.hpp" \ No newline at end of file diff --git a/include/aare/file_io.hpp b/include/aare/file_io.hpp new file mode 100644 index 0000000..d5ee373 --- /dev/null +++ b/include/aare/file_io.hpp @@ -0,0 +1,8 @@ +// ClusterFile.hpp File.hpp FileInterface.hpp NumpyFile.hpp NumpyHelpers.hpp RawFile.hpp SubFile.hpp +#include "aare/file_io/ClusterFile.hpp" +#include "aare/file_io/File.hpp" +#include "aare/file_io/FileInterface.hpp" +#include "aare/file_io/NumpyFile.hpp" +#include "aare/file_io/NumpyHelpers.hpp" +#include "aare/file_io/RawFile.hpp" +#include "aare/file_io/SubFile.hpp" diff --git a/include/aare/network_io.hpp b/include/aare/network_io.hpp new file mode 100644 index 0000000..fd559f0 --- /dev/null +++ b/include/aare/network_io.hpp @@ -0,0 +1,5 @@ +#include "aare/network_io/ZmqHeader.hpp" +#include "aare/network_io/ZmqSocket.hpp" +#include "aare/network_io/ZmqSocketReceiver.hpp" +#include "aare/network_io/ZmqSocketSender.hpp" +#include "aare/network_io/defs.hpp" \ No newline at end of file diff --git a/include/aare/utils.hpp b/include/aare/utils.hpp new file mode 100644 index 0000000..35596af --- /dev/null +++ b/include/aare/utils.hpp @@ -0,0 +1,3 @@ +#include "aare/utils/compare_files.hpp" +#include "aare/utils/json.hpp" +#include "aare/utils/logger.hpp" \ No newline at end of file diff --git a/network_io/src/ZmqHeader.cpp b/network_io/src/ZmqHeader.cpp index 588605e..8d8c889 100644 --- a/network_io/src/ZmqHeader.cpp +++ b/network_io/src/ZmqHeader.cpp @@ -1,66 +1,9 @@ #include "aare/network_io/ZmqHeader.hpp" +#include "aare/utils/json.hpp" #include "simdjson.h" -// helper functions to write json -// append to string for better performance (not tested) - -/** - * @brief write a digit to a string - * takes key and value and outputs->"key": value, - * @tparam T type of value (int, uint32_t, ...) - * @param s string to append to - * @param key key to write - * @param value value to write - * @return void - * @note - * - can't use concepts here because we are using c++17 - */ -template void write_digit(std::string &s, const std::string &key, const T &value) { - s += "\""; - s += key; - s += "\": "; - s += std::to_string(value); - s += ", "; -} -void write_str(std::string &s, const std::string &key, const std::string &value) { - s += "\""; - s += key; - s += "\": \""; - s += value; - s += "\", "; -} -void write_map(std::string &s, const std::string &key, const std::map &value) { - s += "\""; - s += key; - s += "\": {"; - for (const auto &kv : value) { - write_str(s, kv.first, kv.second); - } - // remove last comma or trailing spaces - for (size_t i = s.size() - 1; i > 0; i--) { - if (s[i] == ',' or s[i] == ' ') { - s.pop_back(); - } else - break; - } - s += "}, "; -} -void write_array(std::string &s, const std::string &key, const std::array &value) { - s += "\""; - s += key; - s += "\": ["; - s += std::to_string(value[0]); - s += ", "; - s += std::to_string(value[1]); - s += ", "; - s += std::to_string(value[2]); - s += ", "; - s += std::to_string(value[3]); - s += "], "; -} - namespace aare { std::string ZmqHeader::to_string() const { diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index 8a554b8..401f0a8 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -1,3 +1,4 @@ + add_library(utils STATIC src/logger.cpp) target_include_directories(utils PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include) diff --git a/utils/include/aare/utils/json.hpp b/utils/include/aare/utils/json.hpp new file mode 100644 index 0000000..ab679e5 --- /dev/null +++ b/utils/include/aare/utils/json.hpp @@ -0,0 +1,67 @@ +#pragma once +#include +#include +#include +#include + +// helper functions to write json +// append to string for better performance (not tested) + +namespace aare { + +/** + * @brief write a digit to a string + * takes key and value and outputs->"key": value, + * @tparam T type of value (int, uint32_t, ...) + * @param s string to append to + * @param key key to write + * @param value value to write + * @return void + * @note + * - can't use concepts here because we are using c++17 + */ +template inline void write_digit(std::string &s, const std::string &key, const T &value) { + s += "\""; + s += key; + s += "\": "; + s += std::to_string(value); + s += ", "; +} +inline void write_str(std::string &s, const std::string &key, const std::string &value) { + s += "\""; + s += key; + s += "\": \""; + s += value; + s += "\", "; +} +inline void write_map(std::string &s, const std::string &key, const std::map &value) { + s += "\""; + s += key; + s += "\": {"; + for (const auto &kv : value) { + write_str(s, kv.first, kv.second); + } + // remove last comma or trailing spaces + for (size_t i = s.size() - 1; i > 0; i--) { + if (s[i] == ',' or s[i] == ' ') { + s.pop_back(); + } else + break; + } + s += "}, "; +} +inline void write_array(std::string &s, const std::string &key, const std::array &value) { + s += "\""; + s += key; + s += "\": ["; + s += std::to_string(value[0]); + s += ", "; + s += std::to_string(value[1]); + s += ", "; + s += std::to_string(value[2]); + s += ", "; + s += std::to_string(value[3]); + s += "], "; +} + +} // namespace aare