Don't filter pulses by missing events but use pulse filter mask instead

eos/event_analysis.py

@@ -19,7 +19,7 @@ class ExtractWalltime(EventDataAction):
         wallTime = extract_walltime(dataset.data.events.tof,
                                     dataset.data.packets.start_index,
                                     dataset.data.packets.time)
-        logging.debug(f'        expending event stream by wallTime')
+        logging.debug(f'          expending event stream by wallTime')
         new_events = append_fields(dataset.data.events, [('wallTime', wallTime.dtype)])
         new_events.wallTime = wallTime
         dataset.data.events = new_events
@@ -186,6 +186,7 @@ class FilterByLog(EventDataAction):
         if self.remove_switchpulse:
             switched = fltr_pulses[:-1] & ~fltr_pulses[1:]
             fltr_pulses[:-1] &= ~switched
+        dataset.data.pulses.mask += EVENT_BITMASKS[filter_variable]*np.logical_not(fltr_pulses)
         goodTimeS = dataset.data.pulses.time[fltr_pulses]
         filter_e = np.logical_not(np.isin(dataset.data.events.wallTime, goodTimeS))
         dataset.data.events.mask += EVENT_BITMASKS[filter_variable]*filter_e

eos/event_data_types.py

@@ -37,7 +37,7 @@ class DA00EventBuffer:
 # Structured datatypes used for event streams
 EVENT_TYPE = np.dtype([('tof', np.float64), ('pixelID', np.uint32), ('mask', np.int32)])
 PACKET_TYPE = np.dtype([('start_index', np.uint32), ('time', np.int64)])
-PULSE_TYPE = np.dtype([('time', np.int64), ('monitor', np.float32)])
+PULSE_TYPE = np.dtype([('time', np.int64), ('monitor', np.float32), ('mask', np.int32)])
 PC_TYPE = np.dtype([('current', np.float32), ('time', np.int64)])
 BM_TYPE = np.dtype([('value', np.int32), ('time', np.int64)])
 LOG_TYPE = np.dtype([('value', np.float32), ('time', np.int64)])

eos/event_handling.py

@@ -134,6 +134,7 @@ class FilterMonitorThreshold(EventDataAction):
             raise ValueError(
                     "FilterMonitorThreshold requires walltTime to be extracted, please run ExtractWalltime first")
         low_current_filter = dataset.data.pulses.monitor>2*dataset.timing.tau*self.lowCurrentThreshold*1e-3
+        dataset.data.pulses.mask[np.logical_not(low_current_filter)] = EVENT_BITMASKS['MonitorThreshold']
         dataset.data.pulses.monitor[np.logical_not(low_current_filter)] = 0.
         goodTimeS = dataset.data.pulses.time[low_current_filter]
         filter_e = np.logical_not(np.isin(dataset.data.events.wallTime, goodTimeS))
@@ -197,19 +198,25 @@ class ApplyMask(EventDataAction):
         logging.info(f'      number of events: total = {pre_filter:7d}, after filtering = {post_filter:7d}')
         if d.device_logs == {} or not hasattr(dataset, 'update_info_from_logs'):
             return
-        if 'wallTime' in d.events.dtype.names:
-            # filter pulses and logs to allow update of header information
-            from .helpers import add_log_to_pulses
-            times = np.unique(d.events.wallTime)
-            # make sure all log variables are associated with pulses
-            for key, log in d.device_logs.items():
-                if not key in d.pulses.dtype.names:
-                    # interpolate the parameter values for all existing pulses
-                    add_log_to_pulses(key, dataset)
-            # remove all pulses that have no more events
-            d.pulses = d.pulses[np.isin(d.pulses.time, times)]
-            for key, log in d.device_logs.items():
-                d.device_logs[key] = np.recarray(d.pulses.shape, dtype = log.dtype)
-                d.device_logs[key].time = d.pulses.time
-                d.device_logs[key].value = d.pulses[key]
-            dataset.update_info_from_logs()
+        # filter pulses, not all masks apply to pulses
+        pre_filter_pulses = d.pulses.shape[0]
+        # make sure all log variables are associated with pulses
+        for key, log in d.device_logs.items():
+            if not key in d.pulses.dtype.names:
+                from .helpers import add_log_to_pulses
+                # interpolate the parameter values for all existing pulses
+                add_log_to_pulses(key, dataset)
+        if self.bitmask_filter is None:
+            d.pulses = d.pulses[d.pulses.mask==0]
+        else:
+            # remove the provided bitmask_filter bits from the events
+            # this means that all bits that are set in bitmask_filter will NOT be used to filter events
+            fltr = (d.pulses.mask & (~self.bitmask_filter)) == 0
+            d.pulses = d.pulses[fltr]
+        post_filter_pulses = d.pulses.shape[0]
+        logging.info(f'      number of pulses: total = {pre_filter_pulses:7d}, after filtering = {post_filter_pulses:7d}')
+        for key, log in d.device_logs.items():
+            d.device_logs[key] = np.recarray(d.pulses.shape, dtype = log.dtype)
+            d.device_logs[key].time = d.pulses.time
+            d.device_logs[key].value = d.pulses[key]
+        dataset.update_info_from_logs()

eos/file_reader.py

@@ -549,6 +549,7 @@ class AmorEventData(AmorHeader):
         pulses = np.recarray(pulseTimeS.shape, dtype=PULSE_TYPE)
         pulses.time = pulseTimeS
         pulses.monitor = 1. # default is monitor pulses as it requires no calculation
+        pulses.mask = 0
         # apply filter in case the events were filtered
         if (self.first_index>0 or not self.EOF):
             pulses = pulses[(pulses.time>=packets.time[0])&(pulses.time<=packets.time[-1])]