write rawfiles (single file) and read rawfiles in order (#66)

* read subfiles with unordered and missing frames

* save work debugging

* Revert "save work debugging"

This reverts commit e791992a05.

* Revert "read subfiles with unordered and missing frames"

This reverts commit 1177fd129d.

* throw when two frames have different frame numbers

* write single part RawFile (working beta)

* correct total number of frames in master file

* add new mythen file with syncd frames

* read frames with same frame number

* clang-tidy fixes, formatting, add tests

* improve readability in loop

* fix failing tests

---------

Co-authored-by: Bechir <bechir.brahem420@gmail.com>

data/mythen/CORRECTED_scan242_master_3.raw

@@ -0,0 +1,40 @@
+Version                    : 6.2
+TimeStamp                  : Mon Sep 14 16:21:09 2020
+
+Detector Type              : Mythen3
+Timing Mode                : auto
+Geometry                   : [4, 1]
+Image Size                 : 5120 bytes
+Pixels                     : [1280, 1]
+Max Frames Per File        : 10000
+Frame Discard Policy       : nodiscard
+Frame Padding              : 1
+Scan Parameters            : [disabled]
+Total Frames               : 1
+Dynamic Range              : 32
+Ten Giga                   : 1
+Period                     : 0ns
+Counter Mask               : 0x1
+Exptime1                   : 0.2s
+Exptime2                   : 0.2s
+Exptime3                   : 0.2s
+GateDelay1                 : 0.1s
+GateDelay2                 : 0.1s
+GateDelay3                 : 0.1s
+Gates                      : 1
+
+#Frame Header
+Frame Number               : 8 bytes
+SubFrame Number/ExpLength  : 4 bytes
+Packet Number              : 4 bytes
+Bunch ID                   : 8 bytes
+Timestamp                  : 8 bytes
+Module Id                  : 2 bytes
+Row                        : 2 bytes
+Column                     : 2 bytes
+Reserved                   : 2 bytes
+Debug                      : 4 bytes
+Round Robin Number         : 2 bytes
+Detector Type              : 1 byte
+Header Version             : 1 byte
+Packets Caught Mask        : 64 bytes

data/mythen/correct_frame_numbers.py

@@ -0,0 +1,58 @@
+import numpy as np
+import shutil
+
+header_dt = np.dtype(
+    [
+        ("Frame Number", "u8"),
+        ("SubFrame Number/ExpLength", "u4"),
+        ("Packet Number", "u4"),
+        ("Bunch ID", "u8"),
+        ("Timestamp", "u8"),
+        ("Module Id", "u2"),
+        ("Row", "u2"),
+        ("Column", "u2"),
+        ("Reserved", "u2"),
+        ("Debug", "u4"),
+        ("Round Robin Number", "u2"),
+        ("Detector Type", "u1"),
+        ("Header Version", "u1"),
+        ("Packets caught mask", "8u8")
+    ]
+)
+
+# Read three frames from a jungfrau file with a single interface
+
+frames = 1
+parts = 4
+frame_per_file = 3
+bytes_per_pixel = 4
+frame_cols = 1
+frame_rows = 5120
+
+part_cols = 1280
+part_rows = 1
+
+header = np.zeros((frames,parts), dtype = header_dt)
+
+
+
+# verify that all parts have the same frame number
+frame = 0
+i = 55
+for part in range(parts):
+    file_name_r = f'scan242_d{part}_f{frame//frame_per_file}_{3}.raw'
+    file_name_w = f'CORRECTED_scan242_d{part}_f{frame//frame_per_file}_{3}.raw'
+    shutil.copyfile(file_name_r, file_name_w)
+
+    with open(file_name_r) as fr, open(file_name_w, 'r+b') as fw:
+        # get frame
+        offset = (frame%frame_per_file)*(header_dt.itemsize+part_rows*part_cols*bytes_per_pixel)
+        header[frame,part] = np.fromfile(fr, dtype=header_dt, count = 1,offset=offset)
+        # update frame number
+        header[frame,part]['Frame Number'] = i
+        fw.seek(offset)
+        header[frame,part].tofile(fw)
+
+
+print("[X] Done\n")
+

data/mythen/read_multiport.py

@@ -0,0 +1,57 @@
+import numpy as np
+
+header_dt = np.dtype(
+    [
+        ("Frame Number", "u8"),
+        ("SubFrame Number/ExpLength", "u4"),
+        ("Packet Number", "u4"),
+        ("Bunch ID", "u8"),
+        ("Timestamp", "u8"),
+        ("Module Id", "u2"),
+        ("Row", "u2"),
+        ("Column", "u2"),
+        ("Reserved", "u2"),
+        ("Debug", "u4"),
+        ("Round Robin Number", "u2"),
+        ("Detector Type", "u1"),
+        ("Header Version", "u1"),
+        ("Packets caught mask", "8u8")
+    ]
+)
+
+# Read three frames from a jungfrau file with a single interface
+
+frames = 1
+parts = 4
+frame_per_file = 3
+bytes_per_pixel = 4
+frame_cols = 1
+frame_rows = 5120
+
+part_cols = 1280
+part_rows = 1
+
+header = np.zeros((frames,parts), dtype = header_dt)
+
+
+
+# verify that all parts have the same frame number
+frame = 0
+sync = True
+for part in range(parts):
+    file_name = f'scan242_d{part}_f{frame//frame_per_file}_{3}.raw'
+    with open(file_name) as f:
+        offset = (frame%frame_per_file)*(header_dt.itemsize+part_rows*part_cols*bytes_per_pixel)
+        # print(f"Reading file: {file_name} at offset {offset}")
+        header[frame,part] = np.fromfile(f, dtype=header_dt, count = 1,offset=offset)
+        # print(f"Frame {frame} part {part} frame number: {header[frame,part]['Frame Number']}")
+        print(f"part {part} frame number: {header[frame,part]['Frame Number']}")
+        sync  = sync and (header[frame,part]['Frame Number'] == header[frame,0]['Frame Number'])
+
+
+if sync:
+    print("[X] subfiles have the same frame")
+else:
+    print("[X] subfiles do not have the same frame\n")
+
+