diff --git a/libeos/event_analysis.py b/libeos/event_analysis.py index 8df19dc..45c1dc5 100644 --- a/libeos/event_analysis.py +++ b/libeos/event_analysis.py @@ -21,7 +21,7 @@ class AnalyzePixelIDs(EventDataAction): def perform_action(self, dataset: EventDatasetProtocol) ->None: d = dataset.data - pixelLookUp = self.resolve_pixels() + delta_z, pixelLookUp = self.resolve_pixels() # TODO: change numba implementation to use native pixelID type (detZ, detXdist, delta, mask) = filter_project_x( pixelLookUp, d.events.pixelID.astype(np.int64), self.yRange[0], self.yRange[1] @@ -34,7 +34,7 @@ class AnalyzePixelIDs(EventDataAction): ana_events.delta = delta ana_events.mask += np.logical_not(mask)*EVENT_BITMASKS['yRange'] d.events = ana_events - dataset.geometry.delta_z = self.delta_z + dataset.geometry.delta_z = delta_z def resolve_pixels(self): """determine spatial coordinats and angles from pixel number""" @@ -48,8 +48,9 @@ class AnalyzePixelIDs(EventDataAction): bladeAngle = np.rad2deg( 2. * np.arcsin(0.5*Detector.bladeZ / Detector.distance) ) delta = (Detector.nBlades/2. - bladeNr) * bladeAngle \ - np.rad2deg( np.arctan(bZi*Detector.dZ / ( Detector.distance + bZi * Detector.dX) ) ) - self.delta_z = delta[detYi==1] - return np.vstack((detYi.T, detZi.T, detX.T, delta.T)).T + delta_z = delta[detYi==1] + pixel_lookup=np.vstack((detYi.T, detZi.T, detX.T, delta.T)).T + return delta_z, pixel_lookup class TofTimeCorrection(EventDataAction): def __init__(self, correct_chopper_opening: bool = True): @@ -71,7 +72,7 @@ class CalculateWavelength(EventDataAction): if not 'detXdist' in dataset.data.events.dtype.names: raise ValueError("CalculateWavelength requires dataset with analyzed pixels, perform AnalyzePixelIDs first") - self.lamdaMax = const.lamdaCut+1.e13*dataset.timing.tau*const.hdm/(dataset.geometry.chopperDetectorDistance+124.) + #lamdaMax = const.lamdaCut+1.e13*dataset.timing.tau*const.hdm/(dataset.geometry.chopperDetectorDistance+124.) # lambda lamda = (1.e13*const.hdm)*d.events.tof/(dataset.geometry.chopperDetectorDistance+d.events.detXdist) diff --git a/libeos/event_data_types.py b/libeos/event_data_types.py index ef7cebd..1073395 100644 --- a/libeos/event_data_types.py +++ b/libeos/event_data_types.py @@ -5,6 +5,7 @@ from typing import List, Optional, Protocol, Tuple from dataclasses import dataclass from .header import Header from abc import ABC, abstractmethod +from hashlib import sha256 import numpy as np import logging @@ -105,6 +106,14 @@ class EventDataAction(ABC): output += f'{key}={value}, ' return output.rstrip(', ')+')' + def action_hash(self)->bytes: + # generate a unique hash that encodes this action with its configuration parameters + mh = sha256() + mh.update(self.__class__.__name__.encode()) + for key,value in sorted(self.__dict__.items()): + mh.update(repr(value).encode()) + return mh.hexdigest() + class CombinedAction(EventDataAction): """ Used to perform multiple actions in one call. Stores a sequence of actions @@ -129,3 +138,9 @@ class CombinedAction(EventDataAction): for ai in self._actions[1:]: output += ' | '+repr(ai) return output + + def action_hash(self)->bytes: + mh = sha256() + for action in self._actions: + mh.update(action.action_hash().encode()) + return mh.hexdigest() diff --git a/libeos/normalisation.py b/libeos/normalisation.py index 8c8a6d6..7f2578d 100644 --- a/libeos/normalisation.py +++ b/libeos/normalisation.py @@ -4,8 +4,7 @@ Defines how to normalize a focusing reflectometry dataset by a reference measure import logging import os import numpy as np -from typing import List - +from typing import List, Optional from .event_data_types import EventDatasetProtocol from .header import Header @@ -43,14 +42,17 @@ class LZNormalisation: self.file_list = [os.path.basename(entry) for entry in reference.file_list] @classmethod - def from_file(cls, filename) -> 'LZNormalisation': - logging.warning(f'normalisation matrix: found and using {filename}') + def from_file(cls, filename, check_hash=None) -> Optional['LZNormalisation']: self = super().__new__(cls) with open(filename, 'rb') as fh: + hash = str(np.load(fh, allow_pickle=True)) self.file_list = np.load(fh, allow_pickle=True) self.angle = np.load(fh, allow_pickle=True) self.norm = np.load(fh, allow_pickle=True) self.monitor = np.load(fh, allow_pickle=True) + if check_hash is not None and hash != check_hash: + logging.info(' file hash does not match this reduction configuration') + raise ValueError('file hash does not match this reduction configuration') return self @classmethod @@ -63,8 +65,9 @@ class LZNormalisation: self.monitor = 1. return self - def safe(self, filename): + def safe(self, filename, hash): with open(filename, 'wb') as fh: + np.save(fh, hash, allow_pickle=False) np.save(fh, np.array(self.file_list), allow_pickle=False) np.save(fh, np.array(self.angle), allow_pickle=False) np.save(fh, self.norm, allow_pickle=False) diff --git a/libeos/reduction.py b/libeos/reduction.py index 6f02305..f8c5644 100644 --- a/libeos/reduction.py +++ b/libeos/reduction.py @@ -378,23 +378,32 @@ class AmorReduction: outputPath = self.output_config.outputPath normalisation_list = self.path_resolver.expand_file_list(short_notation) name = '_'.join(map(str, normalisation_list)) - n_path = os.path.join(outputPath, f'{name}_{str(self.experiment_config.monitorType)}.norm') + n_path = os.path.join(outputPath, f'{name}.norm') + self.norm = None if os.path.exists(n_path): - logging.warning(f'normalisation matrix: found and using {n_path}') - self.norm = LZNormalisation.from_file(n_path) - self.header.measurement_additional_files = self.norm.file_list - else: + logging.debug(f'trying to load matrix from file {n_path}') + try: + self.norm = LZNormalisation.from_file(n_path, self.normevent_actions.action_hash()) + except (ValueError, EOFError): + self.norm =None + else: + logging.warning(f'normalisation matrix: found and using {n_path}') + if self.norm is None: + # in case file does not exist or the action hash doesn't match, create new normalization logging.warning(f'normalisation matrix: using the files {normalisation_list}') normalization_files = list(map(self.path_resolver.get_path, normalisation_list)) reference = AmorEventData(normalization_files[0]) - for nfi in normalization_files[1:]: - reference.append(AmorEventData(nfi)) self.normevent_actions(reference) + for nfi in normalization_files[1:]: + toadd = AmorEventData(nfi) + self.normevent_actions(toadd) + reference.append(toadd) self.norm = LZNormalisation(reference, self.reduction_config.normalisationMethod, self.grid) if reference.data.events.shape[0] > 1e6: - self.norm.safe(n_path) + self.norm.safe(n_path, self.normevent_actions.action_hash()) + self.header.measurement_additional_files = self.norm.file_list self.header.reduction.corrections.append('normalisation with \'additional files\'') def project_on_lz(self, fromHDF, norm_lz, normAngle, lamda_e, detZ_e): diff --git a/tests/test_full_analysis.py b/tests/test_full_analysis.py index 445b9ff..1235a02 100644 --- a/tests/test_full_analysis.py +++ b/tests/test_full_analysis.py @@ -37,7 +37,7 @@ class FullAmorTest(TestCase): def tearDown(self): self.pr.disable() - for fi in ['../test_results/test.Rqz.ort', '../test_results/5952_a.norm']: + for fi in ['../test_results/test.Rqz.ort', '../test_results/5952.norm']: try: os.unlink(os.path.join(self.reader_config.rawPath[0], fi)) except FileNotFoundError: