Keep scans in a list instead of a dict

pyzebra/app/panel_ccl_integrate.py

@@ -87,8 +87,8 @@ def create():
     proposal_textinput.on_change("value", proposal_textinput_callback)
 
     def _init_datatable():
-        scan_list = list(det_data["scan"].keys())
+        scan_list = list(range(len(det_data["scan"])))
-        hkl = [f'{m["h_index"]} {m["k_index"]} {m["l_index"]}' for m in det_data["scan"].values()]
+        hkl = [f'{m["h_index"]} {m["k_index"]} {m["l_index"]}' for m in det_data["scan"]]
         scan_table_source.data.update(
             scan=scan_list,
             hkl=hkl,
@@ -159,8 +159,8 @@ def create():
     append_upload_button.on_change("value", append_upload_button_callback)
 
     def _update_table():
-        num_of_peaks = [len(scan.get("peak_indexes", [])) for scan in det_data["scan"].values()]
+        num_of_peaks = [len(scan.get("peak_indexes", [])) for scan in det_data["scan"]]
-        fit_ok = [(1 if "fit" in scan else 0) for scan in det_data["scan"].values()]
+        fit_ok = [(1 if "fit" in scan else 0) for scan in det_data["scan"]]
         scan_table_source.data.update(peaks=num_of_peaks, fit=fit_ok)
 
     def _update_plot(scan):
@@ -447,7 +447,7 @@ def create():
 
     def peakfind_all_button_callback():
         peakfind_params = _get_peakfind_params()
-        for scan in det_data["scan"].values():
+        for scan in det_data["scan"]:
             pyzebra.ccl_findpeaks(scan, **peakfind_params)
 
         _update_table()
@@ -479,7 +479,7 @@ def create():
 
     def fit_all_button_callback():
         fit_params = _get_fit_params()
-        for scan in det_data["scan"].values():
+        for scan in det_data["scan"]:
             # fit_params are updated inplace within `fitccl`
             pyzebra.fitccl(scan, **deepcopy(fit_params))
 
@@ -515,7 +515,8 @@ def create():
             export_data = deepcopy(det_data)
             for s, export in zip(scan_table_source.data["scan"], scan_table_source.data["export"]):
                 if not export:
-                    del export_data["scan"][s]
+                    if "fit" in export_data["scan"][s]:
+                        del export_data["scan"][s]["fit"]
 
             pyzebra.export_1D(
                 export_data,

pyzebra/app/panel_param_study.py

@@ -96,7 +96,7 @@ def create():
     proposal_textinput.on_change("value", proposal_textinput_callback)
 
     def _init_datatable():
-        scan_list = list(det_data["scan"].keys())
+        scan_list = list(range(len(det_data["scan"])))
         file_list = []
         extra_meta = det_data.get("extra_meta", {})
         for scan_id in scan_list:
@@ -184,8 +184,8 @@ def create():
     append_upload_button.on_change("value", append_upload_button_callback)
 
     def _update_table():
-        num_of_peaks = [len(scan.get("peak_indexes", [])) for scan in det_data["scan"].values()]
+        num_of_peaks = [len(scan.get("peak_indexes", [])) for scan in det_data["scan"]]
-        fit_ok = [(1 if "fit" in scan else 0) for scan in det_data["scan"].values()]
+        fit_ok = [(1 if "fit" in scan else 0) for scan in det_data["scan"]]
         scan_table_source.data.update(peaks=num_of_peaks, fit=fit_ok)
 
     def _update_plot():
@@ -554,7 +554,7 @@ def create():
 
     def peakfind_all_button_callback():
         peakfind_params = _get_peakfind_params()
-        for scan in det_data["scan"].values():
+        for scan in det_data["scan"]:
             pyzebra.ccl_findpeaks(scan, **peakfind_params)
 
         _update_table()
@@ -586,7 +586,7 @@ def create():
 
     def fit_all_button_callback():
         fit_params = _get_fit_params()
-        for scan in det_data["scan"].values():
+        for scan in det_data["scan"]:
             # fit_params are updated inplace within `fitccl`
             pyzebra.fitccl(scan, **deepcopy(fit_params))
 
@@ -649,7 +649,8 @@ def create():
             export_data = deepcopy(det_data)
             for s, export in zip(scan_table_source.data["scan"], scan_table_source.data["export"]):
                 if not export:
-                    del export_data["scan"][s]
+                    if "fit" in export_data["scan"][s]:
+                        del export_data["scan"][s]["fit"]
 
             pyzebra.export_1D(
                 export_data,

pyzebra/ccl_io.py

@@ -133,7 +133,7 @@ def parse_1D(fileobj, data_type):
             break
 
     # read data
-    scan = {}
+    scan = []
     if data_type == ".ccl":
         ccl_first_line = (*CCL_FIRST_LINE, *CCL_ANGLES[metadata["zebra_mode"]])
         ccl_second_line = CCL_SECOND_LINE
@@ -162,7 +162,7 @@ def parse_1D(fileobj, data_type):
                 counts.extend(map(int, next(fileobj).split()))
             s["Counts"] = counts
 
-            scan[s["scan_number"]] = s
+            scan.append(s)
 
     elif data_type == ".dat":
         # skip the first 2 rows, the third row contans the column names
@@ -203,12 +203,12 @@ def parse_1D(fileobj, data_type):
         s["nu_angle"] = metadata["nu"]
 
         s["scan_number"] = 1
-        scan[s["scan_number"]] = dict(s)
+        scan.append(dict(s))
 
     else:
         print("Unknown file extention")
 
-    for s in scan.values():
+    for s in scan:
         if s["h_index"].is_integer() and s["k_index"].is_integer() and s["l_index"].is_integer():
             s["h_index"] = int(s["h_index"])
             s["k_index"] = int(s["k_index"])
@@ -231,12 +231,11 @@ def export_1D(data, path, area_method=AREA_METHODS[0], lorentz=False, hkl_precis
     zebra_mode = data["meta"]["zebra_mode"]
     file_content = {".comm": [], ".incomm": []}
 
-    for key, scan in data["scan"].items():
+    for ind, scan in enumerate(data["scan"]):
         if "fit" not in scan:
-            print("Scan skipped - no fit value for:", key)
             continue
 
-        scan_str = f"{key:6}"
+        ind_str = f"{ind:6}"
 
         h, k, l = scan["h_index"], scan["k_index"], scan["l_index"]
         if scan["indices"] == "hkl":
@@ -267,7 +266,7 @@ def export_1D(data, path, area_method=AREA_METHODS[0], lorentz=False, hkl_precis
             ang_str = ang_str + f"{scan[angle]:8}"
 
         ref = file_content[".comm"] if scan["indices"] == "hkl" else file_content[".incomm"]
-        ref.append(scan_str + hkl_str + area_str + ang_str + "\n")
+        ref.append(ind_str + hkl_str + area_str + ang_str + "\n")
 
     for ext, content in file_content.items():
         if content:

pyzebra/merge_function.py

@@ -13,8 +13,7 @@ def create_tuples(x, y, y_err):
 
 
 def normalize_all(dictionary, monitor=100000):
-    for keys in dictionary["scan"]:
+    for scan in dictionary["scan"]:
-        scan = dictionary["scan"][keys]
         counts = np.array(scan["Counts"])
         sigma = np.sqrt(counts) if "sigma" not in scan else scan["sigma"]
         monitor_ratio = monitor / scan["monitor"]
@@ -161,8 +160,8 @@ def merge_dups(dictionary):
         "gamma_angle": 0.05,
     }
 
-    for i in list(dictionary["scan"]):
+    for i in range(len(dictionary["scan"])):
-        for j in list(dictionary["scan"]):
+        for j in range(len(dictionary["scan"])):
             if i == j:
                 continue
             else:
@@ -181,8 +180,8 @@ def merge_dups(dictionary):
 
 
 def add_scan(dict1, dict2, scan_to_add):
-    max_scan = np.max(list(dict1["scan"]))
+    max_scan = len(dict1["scan"])
-    dict1["scan"][max_scan + 1] = dict2["scan"][scan_to_add]
+    dict1["scan"].append(dict2["scan"][scan_to_add])
     if dict1.get("extra_meta") is None:
         dict1["extra_meta"] = {}
     dict1["extra_meta"][max_scan + 1] = dict2["meta"]
@@ -196,8 +195,8 @@ def process(dict1, dict2, angles, precision):
         # check UB matrixes
         if check_UB(dict1, dict2):
             # iterate over second dict and check for matches
-            for i in list(dict2["scan"]):
+            for i in range(len(dict2["scan"])):
-                for j in list(dict1["scan"]):
+                for j in range(len(dict1["scan"])):
                     if check_angles(dict1["scan"][j], dict2["scan"][i], angles, precision):
                         # angles good, see the mag and temp
                         if check_temp_mag(dict1["scan"][j], dict2["scan"][i]):
@@ -276,7 +275,7 @@ def add_dict(dict1, dict2):
     # this is for the qscan case
     except KeyError:
         print("Zebra mode not specified")
-    max_measurement_dict1 = max([keys for keys in dict1["scan"]])
+    max_measurement_dict1 = len(dict1["scan"])
     new_filenames = np.arange(
         max_measurement_dict1 + 1, max_measurement_dict1 + 1 + len(dict2["scan"])
     )
@@ -286,9 +285,9 @@ def add_dict(dict1, dict2):
 
     new_meta_name = "meta" + str(dict2["meta"]["original_filename"])
     if new_meta_name not in dict1:
-        for keys, name in zip(dict2["scan"], new_filenames):
+        for keys, name in zip(range(len(dict2["scan"])), new_filenames):
             dict2["scan"][keys]["file_of_origin"] = str(dict2["meta"]["original_filename"])
-            dict1["scan"][name] = dict2["scan"][keys]
+            dict1["scan"].append(dict2["scan"][keys])
             dict1["extra_meta"][name] = dict2["meta"]
 
         dict1[new_meta_name] = dict2["meta"]

pyzebra/param_study_moduls.py

@@ -51,7 +51,7 @@ def load_dats(filepath):
             else:
 
                 dict1 = add_dict(dict1, load_1D(file_list[i]))
-        dict1["scan"][i + 1]["params"] = {}
+        dict1["scan"].append({})
         if data_type == "txt":
             for x in range(len(col_names) - 1):
                 dict1["scan"][i + 1]["params"][col_names[x + 1]] = float(file_list[i][x + 1])
@@ -76,7 +76,7 @@ def create_dataframe(dict1, variables):
         print(keys)
 
     # populate the dict
-    for keys in dict1["scan"]:
+    for keys in range(len(dict1["scan"])):
         if "file_of_origin" in dict1["scan"][keys]:
             pull_dict["filenames"].append(dict1["scan"][keys]["file_of_origin"].split("/")[-1])
         else:
@@ -298,7 +298,7 @@ def add_dict(dict1, dict2):
     # this is for the qscan case
     except KeyError:
         print("Zebra mode not specified")
-    max_measurement_dict1 = max([keys for keys in dict1["scan"]])
+    max_measurement_dict1 = len(dict1["scan"])
     new_filenames = np.arange(
         max_measurement_dict1 + 1, max_measurement_dict1 + 1 + len(dict2["scan"])
     )
@@ -352,8 +352,8 @@ def scan_dict(dict, precision=0.5):
         return
 
     d = {}
-    for i in dict["scan"]:
+    for i in range(len(dict["scan"])):
-        for j in dict["scan"]:
+        for j in range(len(dict["scan"])):
             if dict["scan"][i] != dict["scan"][j]:
                 itup = list()
                 for k in angles:
@@ -399,7 +399,7 @@ def variables(dictionary):
     # find all variables that are in all scans
     stdev_precision = 0.05
     all_vars = list()
-    for keys in dictionary["scan"]:
+    for keys in range(len(dictionary["scan"])):
         all_vars.append([key for key in dictionary["scan"][keys] if key != "params"])
         if dictionary["scan"][keys]["params"]:
             all_vars.append(key for key in dictionary["scan"][keys]["params"])
@@ -432,7 +432,7 @@ def variables(dictionary):
     # check for primary variable, needs to be list, we dont suspect the
     # primary variable be as a parameter (be in scan[params])
     primary_candidates = list()
-    for key in dictionary["scan"]:
+    for key in range(len(dictionary["scan"])):
         for i in inall_red:
             if isinstance(_finditem(dictionary["scan"][key], i), list):
                 if np.std(_finditem(dictionary["scan"][key], i)) > stdev_precision:
@@ -454,7 +454,7 @@ def variables(dictionary):
     # print("secondary candidates", secondary_candidates)
     # select arrays and floats and ints
     second_round_secondary_candidates = list()
-    for key in dictionary["scan"]:
+    for key in range(len(dictionary["scan"])):
         for i in secondary_candidates:
             if isinstance(_finditem(dictionary["scan"][key], i), float):
                 second_round_secondary_candidates.append(i)
@@ -475,7 +475,7 @@ def variables(dictionary):
     third_round_sec_candidates = list()
     for i in second_round_secondary_candidates:
         check_array = list()
-        for keys in dictionary["scan"]:
+        for keys in range(len(dictionary["scan"])):
             check_array.append(np.average(_finditem(dictionary["scan"][keys], i)))
         # print(i, check_array, np.std(check_array))
         if np.std(check_array) > stdev_precision: