From 94f57fb4f0192e6e369e60dbc13f01a9a77fdcb3 Mon Sep 17 00:00:00 2001 From: Artur Glavic Date: Wed, 9 Sep 2026 11:07:25 +0200 Subject: [PATCH] Don't filter pulses by missing events but use pulse filter mask instead --- eos/event_analysis.py | 3 ++- eos/event_data_types.py | 2 +- eos/event_handling.py | 39 +++++++++++++++++++++++---------------- eos/file_reader.py | 1 + 4 files changed, 27 insertions(+), 18 deletions(-) diff --git a/eos/event_analysis.py b/eos/event_analysis.py index d7c2d3a..3750941 100644 --- a/eos/event_analysis.py +++ b/eos/event_analysis.py @@ -19,7 +19,7 @@ class ExtractWalltime(EventDataAction): wallTime = extract_walltime(dataset.data.events.tof, dataset.data.packets.start_index, dataset.data.packets.time) - logging.debug(f' expending event stream by wallTime') + logging.debug(f' expending event stream by wallTime') new_events = append_fields(dataset.data.events, [('wallTime', wallTime.dtype)]) new_events.wallTime = wallTime dataset.data.events = new_events @@ -186,6 +186,7 @@ class FilterByLog(EventDataAction): if self.remove_switchpulse: switched = fltr_pulses[:-1] & ~fltr_pulses[1:] fltr_pulses[:-1] &= ~switched + dataset.data.pulses.mask += EVENT_BITMASKS[filter_variable]*np.logical_not(fltr_pulses) goodTimeS = dataset.data.pulses.time[fltr_pulses] filter_e = np.logical_not(np.isin(dataset.data.events.wallTime, goodTimeS)) dataset.data.events.mask += EVENT_BITMASKS[filter_variable]*filter_e diff --git a/eos/event_data_types.py b/eos/event_data_types.py index 27c682c..39dfa41 100644 --- a/eos/event_data_types.py +++ b/eos/event_data_types.py @@ -37,7 +37,7 @@ class DA00EventBuffer: # Structured datatypes used for event streams EVENT_TYPE = np.dtype([('tof', np.float64), ('pixelID', np.uint32), ('mask', np.int32)]) PACKET_TYPE = np.dtype([('start_index', np.uint32), ('time', np.int64)]) -PULSE_TYPE = np.dtype([('time', np.int64), ('monitor', np.float32)]) +PULSE_TYPE = np.dtype([('time', np.int64), ('monitor', np.float32), ('mask', np.int32)]) PC_TYPE = np.dtype([('current', np.float32), ('time', np.int64)]) BM_TYPE = np.dtype([('value', np.int32), ('time', np.int64)]) LOG_TYPE = np.dtype([('value', np.float32), ('time', np.int64)]) diff --git a/eos/event_handling.py b/eos/event_handling.py index 76169f4..09ce599 100644 --- a/eos/event_handling.py +++ b/eos/event_handling.py @@ -134,6 +134,7 @@ class FilterMonitorThreshold(EventDataAction): raise ValueError( "FilterMonitorThreshold requires walltTime to be extracted, please run ExtractWalltime first") low_current_filter = dataset.data.pulses.monitor>2*dataset.timing.tau*self.lowCurrentThreshold*1e-3 + dataset.data.pulses.mask[np.logical_not(low_current_filter)] = EVENT_BITMASKS['MonitorThreshold'] dataset.data.pulses.monitor[np.logical_not(low_current_filter)] = 0. goodTimeS = dataset.data.pulses.time[low_current_filter] filter_e = np.logical_not(np.isin(dataset.data.events.wallTime, goodTimeS)) @@ -197,19 +198,25 @@ class ApplyMask(EventDataAction): logging.info(f' number of events: total = {pre_filter:7d}, after filtering = {post_filter:7d}') if d.device_logs == {} or not hasattr(dataset, 'update_info_from_logs'): return - if 'wallTime' in d.events.dtype.names: - # filter pulses and logs to allow update of header information - from .helpers import add_log_to_pulses - times = np.unique(d.events.wallTime) - # make sure all log variables are associated with pulses - for key, log in d.device_logs.items(): - if not key in d.pulses.dtype.names: - # interpolate the parameter values for all existing pulses - add_log_to_pulses(key, dataset) - # remove all pulses that have no more events - d.pulses = d.pulses[np.isin(d.pulses.time, times)] - for key, log in d.device_logs.items(): - d.device_logs[key] = np.recarray(d.pulses.shape, dtype = log.dtype) - d.device_logs[key].time = d.pulses.time - d.device_logs[key].value = d.pulses[key] - dataset.update_info_from_logs() + # filter pulses, not all masks apply to pulses + pre_filter_pulses = d.pulses.shape[0] + # make sure all log variables are associated with pulses + for key, log in d.device_logs.items(): + if not key in d.pulses.dtype.names: + from .helpers import add_log_to_pulses + # interpolate the parameter values for all existing pulses + add_log_to_pulses(key, dataset) + if self.bitmask_filter is None: + d.pulses = d.pulses[d.pulses.mask==0] + else: + # remove the provided bitmask_filter bits from the events + # this means that all bits that are set in bitmask_filter will NOT be used to filter events + fltr = (d.pulses.mask & (~self.bitmask_filter)) == 0 + d.pulses = d.pulses[fltr] + post_filter_pulses = d.pulses.shape[0] + logging.info(f' number of pulses: total = {pre_filter_pulses:7d}, after filtering = {post_filter_pulses:7d}') + for key, log in d.device_logs.items(): + d.device_logs[key] = np.recarray(d.pulses.shape, dtype = log.dtype) + d.device_logs[key].time = d.pulses.time + d.device_logs[key].value = d.pulses[key] + dataset.update_info_from_logs() diff --git a/eos/file_reader.py b/eos/file_reader.py index 3bc1806..02bdcc6 100644 --- a/eos/file_reader.py +++ b/eos/file_reader.py @@ -549,6 +549,7 @@ class AmorEventData(AmorHeader): pulses = np.recarray(pulseTimeS.shape, dtype=PULSE_TYPE) pulses.time = pulseTimeS pulses.monitor = 1. # default is monitor pulses as it requires no calculation + pulses.mask = 0 # apply filter in case the events were filtered if (self.first_index>0 or not self.EOF): pulses = pulses[(pulses.time>=packets.time[0])&(pulses.time<=packets.time[-1])]