write rawfiles (single file) and read rawfiles in order (#66)

* read subfiles with unordered and missing frames

* save work debugging

* Revert "save work debugging"

This reverts commit e791992a05.

* Revert "read subfiles with unordered and missing frames"

This reverts commit 1177fd129d.

* throw when two frames have different frame numbers

* write single part RawFile (working beta)

* correct total number of frames in master file

* add new mythen file with syncd frames

* read frames with same frame number

* clang-tidy fixes, formatting, add tests

* improve readability in loop

* fix failing tests

---------

Co-authored-by: Bechir <bechir.brahem420@gmail.com>

data/jungfrau/read_frame.py

@@ -22,14 +22,15 @@ header_dt = np.dtype(
 )
 
 # Read three frames from a jungfrau file with a single interface
-rows = 512
-cols = 1024
-frames = 10
+rows = 1024
+cols = 512
+frames = 1
 
 data = np.zeros((frames,rows,cols), dtype = np.uint16)
 header = np.zeros(frames, dtype = header_dt)
-for file_id in range(4):
-    file_name = 'jungfrau_single_d0_f{}_0.raw'.format(file_id)
+for frame in range(frames):
+    
+    file_name = '/tmp/raw_example_writing_master_'
     print("Reading file:", file_name)
     with open(file_name) as f:
         for i in range(3 if file_id != 3 else 1):

data/jungfrau/read_multiport.py

@@ -1,6 +1,4 @@
 import numpy as np
-import matplotlib.pyplot as plt
-plt.ion()
 
 header_dt = np.dtype(
     [
@@ -23,9 +21,10 @@ header_dt = np.dtype(
 
 # Read three frames from a jungfrau file with a single interface
 
-frames = 1
+frames = 10
 parts = 2
-
+frame_per_file = 3
+bytes_per_pixel = 2
 frame_cols = 1024
 frame_rows = 512
 
@@ -39,14 +38,29 @@ header = np.zeros((frames,parts), dtype = header_dt)
 
 
 
+# verify that all parts have the same frame number
+for frame in range(frames):
+    for part in range(parts):
+        file_name = f'jungfrau_double_d{part}_f{frame//frame_per_file}_{0}.raw'
+        with open(file_name) as f:
+            offset = (frame%frame_per_file)*(header_dt.itemsize+part_rows*part_cols*bytes_per_pixel)
+            # print(f"Reading file: {file_name} at offset {offset}")
+            header[frame,part] = np.fromfile(f, dtype=header_dt, count = 1,offset=offset)
+            # print(f"Frame {frame} part {part} frame number: {header[frame,part]['Frame Number']}")
+            if part > 0:
+                assert header[frame,part]['Frame Number'] == header[frame,0]['Frame Number']
+
+print("[X] All parts have the same frame number\n")
+
 
 for frame in range(frames):
 
     for part in range(parts):
-        file_name = f'jungfrau_double_d{part}_f{frame}_{0}.raw'
-        print("Reading file:", file_name)
+        file_name = f'jungfrau_double_d{part}_f{frame//frame_per_file}_{0}.raw'
+        # print("Reading file:", file_name)
         with open(file_name) as f:
-            header[frame,part] = np.fromfile(f, dtype=header_dt, count = 1)
+            offset = (frame%frame_per_file)*(header_dt.itemsize+part_rows*part_cols*bytes_per_pixel)
+            header[frame,part] = np.fromfile(f, dtype=header_dt, count = 1, offset=offset)
             parts_data[frame,part] = np.fromfile(f, dtype=np.uint16,count = part_rows*part_cols).reshape(part_rows,part_cols)
 
     
@@ -54,27 +68,20 @@ for frame in range(frames):
 
 
 
-# for frame in range(frames):
-#     print("Frame:", frame)
-#     print("Data:\n", data[frame])
-
-# print(data[0,0,0])
-# print(data[0,0,1])
-# print(data[0,0,50])
-print(data[0,0,0])
-print(data[0,0,1])
-print(data[0,255,1023])
-
-print(data[0,511,1023])
-# print()
-# print(parts_data[0,0,0,0])
-# print(parts_data[0,0,0,1])
-# print(parts_data[0,0,1,0])
-
-# print(data.shape)
-
-
-
-#fig, ax = plt.subplots()
-#im = ax.imshow(data[0])
-#im.set_clim(2000,4000)
+pixel_0_0,pixel_0_1,pixel_1_0,pixel_255_1023,pixel_511_1023,= [],[],[],[],[]
+for frame in range(frames):
+    pixel_0_0.append(data[frame,0,0])
+    pixel_0_1.append(data[frame,0,1])
+    pixel_1_0.append(data[frame,1,0])
+    pixel_255_1023.append(data[frame,255,1023])
+    pixel_511_1023.append(data[frame,511,1023])
+print("upper left corner of each frame (pixel_0_0)")
+print(pixel_0_0)
+print("first pixel on new line of each frame (pixel_1_0)")
+print(pixel_1_0)
+print("second pixel of the first line of each frame (pixel_0_1)")
+print(pixel_0_1)
+print("first pixel of the second part on the last line of each frame (pixel_255_1023)")
+print(pixel_255_1023)
+print("lower right corner of each frame (pixel_511_1023)")
+print(pixel_511_1023)

data/mythen/CORRECTED_scan242_master_3.raw

@@ -0,0 +1,40 @@
+Version                    : 6.2
+TimeStamp                  : Mon Sep 14 16:21:09 2020
+
+Detector Type              : Mythen3
+Timing Mode                : auto
+Geometry                   : [4, 1]
+Image Size                 : 5120 bytes
+Pixels                     : [1280, 1]
+Max Frames Per File        : 10000
+Frame Discard Policy       : nodiscard
+Frame Padding              : 1
+Scan Parameters            : [disabled]
+Total Frames               : 1
+Dynamic Range              : 32
+Ten Giga                   : 1
+Period                     : 0ns
+Counter Mask               : 0x1
+Exptime1                   : 0.2s
+Exptime2                   : 0.2s
+Exptime3                   : 0.2s
+GateDelay1                 : 0.1s
+GateDelay2                 : 0.1s
+GateDelay3                 : 0.1s
+Gates                      : 1
+
+#Frame Header
+Frame Number               : 8 bytes
+SubFrame Number/ExpLength  : 4 bytes
+Packet Number              : 4 bytes
+Bunch ID                   : 8 bytes
+Timestamp                  : 8 bytes
+Module Id                  : 2 bytes
+Row                        : 2 bytes
+Column                     : 2 bytes
+Reserved                   : 2 bytes
+Debug                      : 4 bytes
+Round Robin Number         : 2 bytes
+Detector Type              : 1 byte
+Header Version             : 1 byte
+Packets Caught Mask        : 64 bytes

data/mythen/correct_frame_numbers.py

@@ -0,0 +1,58 @@
+import numpy as np
+import shutil
+
+header_dt = np.dtype(
+    [
+        ("Frame Number", "u8"),
+        ("SubFrame Number/ExpLength", "u4"),
+        ("Packet Number", "u4"),
+        ("Bunch ID", "u8"),
+        ("Timestamp", "u8"),
+        ("Module Id", "u2"),
+        ("Row", "u2"),
+        ("Column", "u2"),
+        ("Reserved", "u2"),
+        ("Debug", "u4"),
+        ("Round Robin Number", "u2"),
+        ("Detector Type", "u1"),
+        ("Header Version", "u1"),
+        ("Packets caught mask", "8u8")
+    ]
+)
+
+# Read three frames from a jungfrau file with a single interface
+
+frames = 1
+parts = 4
+frame_per_file = 3
+bytes_per_pixel = 4
+frame_cols = 1
+frame_rows = 5120
+
+part_cols = 1280
+part_rows = 1
+
+header = np.zeros((frames,parts), dtype = header_dt)
+
+
+
+# verify that all parts have the same frame number
+frame = 0
+i = 55
+for part in range(parts):
+    file_name_r = f'scan242_d{part}_f{frame//frame_per_file}_{3}.raw'
+    file_name_w = f'CORRECTED_scan242_d{part}_f{frame//frame_per_file}_{3}.raw'
+    shutil.copyfile(file_name_r, file_name_w)
+
+    with open(file_name_r) as fr, open(file_name_w, 'r+b') as fw:
+        # get frame
+        offset = (frame%frame_per_file)*(header_dt.itemsize+part_rows*part_cols*bytes_per_pixel)
+        header[frame,part] = np.fromfile(fr, dtype=header_dt, count = 1,offset=offset)
+        # update frame number
+        header[frame,part]['Frame Number'] = i
+        fw.seek(offset)
+        header[frame,part].tofile(fw)
+
+
+print("[X] Done\n")
+

data/mythen/read_multiport.py

@@ -0,0 +1,57 @@
+import numpy as np
+
+header_dt = np.dtype(
+    [
+        ("Frame Number", "u8"),
+        ("SubFrame Number/ExpLength", "u4"),
+        ("Packet Number", "u4"),
+        ("Bunch ID", "u8"),
+        ("Timestamp", "u8"),
+        ("Module Id", "u2"),
+        ("Row", "u2"),
+        ("Column", "u2"),
+        ("Reserved", "u2"),
+        ("Debug", "u4"),
+        ("Round Robin Number", "u2"),
+        ("Detector Type", "u1"),
+        ("Header Version", "u1"),
+        ("Packets caught mask", "8u8")
+    ]
+)
+
+# Read three frames from a jungfrau file with a single interface
+
+frames = 1
+parts = 4
+frame_per_file = 3
+bytes_per_pixel = 4
+frame_cols = 1
+frame_rows = 5120
+
+part_cols = 1280
+part_rows = 1
+
+header = np.zeros((frames,parts), dtype = header_dt)
+
+
+
+# verify that all parts have the same frame number
+frame = 0
+sync = True
+for part in range(parts):
+    file_name = f'scan242_d{part}_f{frame//frame_per_file}_{3}.raw'
+    with open(file_name) as f:
+        offset = (frame%frame_per_file)*(header_dt.itemsize+part_rows*part_cols*bytes_per_pixel)
+        # print(f"Reading file: {file_name} at offset {offset}")
+        header[frame,part] = np.fromfile(f, dtype=header_dt, count = 1,offset=offset)
+        # print(f"Frame {frame} part {part} frame number: {header[frame,part]['Frame Number']}")
+        print(f"part {part} frame number: {header[frame,part]['Frame Number']}")
+        sync  = sync and (header[frame,part]['Frame Number'] == header[frame,0]['Frame Number'])
+
+
+if sync:
+    print("[X] subfiles have the same frame")
+else:
+    print("[X] subfiles do not have the same frame\n")
+
+

data/scripts/read_first_frame_number.py

@@ -1,29 +0,0 @@
-import numpy as np
-from pathlib import Path
-
-header_dt = np.dtype(
-    [
-        ("Frame Number", "u8"),
-        ("SubFrame Number/ExpLength", "u4"),
-        ("Packet Number", "u4"),
-        ("Bunch ID", "u8"),
-        ("Timestamp", "u8"),
-        ("Module Id", "u2"),
-        ("Row", "u2"),
-        ("Column", "u2"),
-        ("Reserved", "u2"),
-        ("Debug", "u4"),
-        ("Round Robin Number", "u2"),
-        ("Detector Type", "u1"),
-        ("Header Version", "u1"),
-        ("Packets caught mask", "8u8")
-    ]
-)
-
-
-
-with open("data/eiger/eiger_500k_16bit_d0_f0_0.raw", "rb") as f:
-    for i in range(3):
-        frame_number = np.fromfile(f, dtype=header_dt, count=1)["Frame Number"][0]
-        print(frame_number)
-        f.seek(262144,1)

data/scripts/read_multiport.py

@@ -1,80 +0,0 @@
-import numpy as np
-import matplotlib.pyplot as plt
-plt.ion()
-
-header_dt = np.dtype(
-    [
-        ("Frame Number", "u8"),
-        ("SubFrame Number/ExpLength", "u4"),
-        ("Packet Number", "u4"),
-        ("Bunch ID", "u8"),
-        ("Timestamp", "u8"),
-        ("Module Id", "u2"),
-        ("Row", "u2"),
-        ("Column", "u2"),
-        ("Reserved", "u2"),
-        ("Debug", "u4"),
-        ("Round Robin Number", "u2"),
-        ("Detector Type", "u1"),
-        ("Header Version", "u1"),
-        ("Packets caught mask", "8u8")
-    ]
-)
-
-# Read three frames from a jungfrau file with a single interface
-
-frames = 1
-parts = 2
-
-frame_cols = 1024
-frame_rows = 512
-
-part_cols = 1024
-part_rows = 256
-
-
-parts_data = np.zeros((frames,parts,part_rows,part_cols), dtype = np.uint16)
-data = np.zeros((frames,frame_rows,frame_cols), dtype = np.uint16)
-header = np.zeros((frames,parts), dtype = header_dt)
-
-
-
-
-for frame in range(frames):
-
-    for part in range(parts):
-        file_name = f'jungfrau_double_d{part}_f{frame}_{0}.raw'
-        print("Reading file:", file_name)
-        with open(file_name) as f:
-            header[frame,part] = np.fromfile(f, dtype=header_dt, count = 1)
-            parts_data[frame,part] = np.fromfile(f, dtype=np.uint16,count = part_rows*part_cols).reshape(part_rows,part_cols)
-
-    
-    data[frame] = np.concatenate((parts_data[frame,0],parts_data[frame,1]),axis=0)
-
-
-
-# for frame in range(frames):
-#     print("Frame:", frame)
-#     print("Data:\n", data[frame])
-
-# print(data[0,0,0])
-# print(data[0,0,1])
-# print(data[0,0,50])
-print(data[0,0,0])
-print(data[0,0,1])
-print(data[0,255,1023])
-
-print(data[0,511,1023])
-# print()
-# print(parts_data[0,0,0,0])
-# print(parts_data[0,0,0,1])
-# print(parts_data[0,0,1,0])
-
-# print(data.shape)
-
-
-
-#fig, ax = plt.subplots()
-#im = ax.imshow(data[0])
-#im.set_clim(2000,4000)

data/scripts/verify_rawfile_writing.py

@@ -0,0 +1,62 @@
+###
+### Verify that the raw file written by the raw_example.cpp are correct
+###
+
+
+import numpy as np
+
+header_dt = np.dtype(
+    [
+        ("Frame Number", "u8"),
+        ("SubFrame Number/ExpLength", "u4"),
+        ("Packet Number", "u4"),
+        ("Bunch ID", "u8"),
+        ("Timestamp", "u8"),
+        ("Module Id", "u2"),
+        ("Row", "u2"),
+        ("Column", "u2"),
+        ("Reserved", "u2"),
+        ("Debug", "u4"),
+        ("Round Robin Number", "u2"),
+        ("Detector Type", "u1"),
+        ("Header Version", "u1"),
+        ("Packets caught mask", "8u8")
+    ]
+)
+
+frames = 1
+parts = 1
+frame_per_file = 3
+bytes_per_pixel = 2
+frame_cols = 512
+frame_rows = 1024
+
+part_cols = 512
+part_rows = 1024
+
+
+# parts_data = np.zeros((frames,parts,part_rows,part_cols), dtype = np.uint16)
+data = np.zeros((frames,frame_rows,frame_cols), dtype = np.uint16)
+header = np.zeros((frames,parts), dtype = header_dt)
+
+
+
+# verify that all parts have the same frame number
+for frame in range(frames):
+    for part in range(parts):
+        file_name = f'/tmp/raw_example_writing_d{part}_f{frame//frame_per_file}_{0}.raw'
+        with open(file_name) as f:
+            offset = (frame%frame_per_file)*(header_dt.itemsize+part_rows*part_cols*bytes_per_pixel)
+            # print(f"Reading file: {file_name} at offset {offset}")
+            header[frame,part] = np.fromfile(f, dtype=header_dt, count = 1,offset=offset)
+            # print(f"Frame {frame} part {part} frame number: {header[frame,part]['Frame Number']}")
+            data[frame] = np.fromfile(f, dtype=np.uint16,count = frame_rows*frame_cols).reshape(frame_rows,frame_cols)
+            
+
+for frame in range(frames):
+    for i,j in np.ndindex(data[frame].shape):
+        assert(data[frame][i,j] == i+j)
+
+print("[X] frame data is correct")
+        
+