From f9d30236b4eaaa344822bf1da931649e7b910dc4 Mon Sep 17 00:00:00 2001 From: Sven Augustin Date: Mon, 23 Mar 2020 14:27:32 +0000 Subject: [PATCH] re-wrote take_pedestal --- slic/daq/diacounter.py | 11 ++++- slic/daq/pedestals.py | 99 +++++++++++++++++++++--------------------- 2 files changed, 59 insertions(+), 51 deletions(-) diff --git a/slic/daq/diacounter.py b/slic/daq/diacounter.py index a20be8699..393999d29 100644 --- a/slic/daq/diacounter.py +++ b/slic/daq/diacounter.py @@ -8,7 +8,7 @@ from ..utils.printing import printable_dict_of_dicts from .acquisition import Acquisition from .basecounter import BaseCounter from .utils import can_create_file, SwissFELPaths -from .pedestals import find_last_pedestal +from .pedestals import find_last_pedestal, take_pedestal @@ -122,6 +122,15 @@ class DIACounter(BaseCounter): def get_last_pedestal(self): return find_last_pedestal(self.active_clients, self.paths.pede) + def take_pedestal(self, analyze=True, n_pulses=1000, n_bad_modules=0, freq=25, user=None): + instrument = self.instrument + pgroup = self.pgroup + api_address = self.api_address + raw_dir = self.paths.raw + res_dir = self.paths.res + exptime = EXPTIME[instrument] + return take_pedestal(instrument, pgroup, api_address, raw_dir, res_dir, analyze, n_pulses, n_bad_modules, freq, exptime, user) + def __repr__(self): clients = self.active_clients clients = ", ".join(clients) diff --git a/slic/daq/pedestals.py b/slic/daq/pedestals.py index be52cc8a8..8f67dc3d5 100644 --- a/slic/daq/pedestals.py +++ b/slic/daq/pedestals.py @@ -2,11 +2,14 @@ import os from glob import glob from datetime import datetime from collections import defaultdict +from jungfrau_utils.scripts.jungfrau_run_pedestals import run as ju_record_raw_pedestal + TIMESTAMP_FORMAT = "%Y%m%d_%H%M" + def find_last_pedestal(clients, directory="."): clients = set(clients) clients = clients - {"bsread"} @@ -36,7 +39,6 @@ def find_last_pedestal(clients, directory="."): return res - def extract_timestamp(fn): fn = strip_folder(fn) base = fn.split(".")[0] @@ -56,61 +58,58 @@ def strip_folder(fn): +def take_pedestal(instrument, pgroup, api_address, raw_dir, res_dir, analyze_locally, n_pulses, n_bad_modules, freq, exptime, user): + make_missing_dir(raw_dir) + make_missing_dir(res_dir) + now = datetime.now().strftime(TIMESTAMP_FORMAT) + file_base = "pedestal_" + now + raw_file_base = raw_dir + file_base + res_file_base = res_dir + file_base - - - - -def take_pedestal(self, n_frames, analyze=True, n_bad_modules=0, update_config=True, freq=25): - from jungfrau_utils.scripts.jungfrau_run_pedestals import run as jungfrau_utils_run - directory = "/sf/%s/data/p%d/raw/JF_pedestals/" % (self.instrument, self.pgroup) - if not os.path.exists(directory): - print("Directory %s not existing, AND I CAN NOT CREATE IT, CALL DIMA" % directory) - #os.makedirs(directory) - - res_dir = directory.replace("/raw/", "/res/") - if not os.path.exists(res_dir): - print("Directory %s not existing, creating it" % res_dir) - os.makedirs(res_dir) - os.chmod(res_dir, 0o775) - filename = "pedestal_%s" % datetime.now().strftime("%Y%m%d_%H%M") period = 1 / freq - jungfrau_utils_run(self.api_address, filename, directory, self.pgroup, period, self.detector_config["exptime"], - n_frames, 1, analyze, n_bad_modules, self.instrument) - - if update_config: - self.pede_file = (directory + filename).replace("raw/", "res/").replace(".h5", ".res.h5") - print("Pedestal file updated to %s" % self.pede_file) - return self.pede_file - -#TODO: ?!? - if analyze: - pedestals_taken = Path(directory).glob(filename + "*") - print( - "Analysis of pedestal data is outsourced to batch farm, user credentials required." - ) - user = input("enter user name for analysis on sf batch farm: ") - commandstr = [ - f"ssh {user}@sf-cn-1 source /sf/{self.instrument}/bin/anaconda_env" - ] - for ped in pedestals_taken: - commandstr.append( - f"sbatch jungfrau_create_pedestals --filename {ped.as_posix()} --directory {res_dir} --verbosity 4" - ) - os.system("\;".join(commandstr)) - - - - - - - - - + trigger = 1 + ju_record_raw_pedestal(api_address, file_base, raw_dir, pgroup, period, exptime, n_pulses, trigger, analyze_locally, n_bad_modules, instrument) + if not analyze_locally: + analyze_pedestal_on_cluster(instrument, raw_file_base, res_dir, user) + + return res_file_base + + +def make_missing_dir(d): + if os.path.exists(d): + return + + msg = "Directory \"{}\" does not exist, creating it...".format(d) + print(msg) + + os.makedirs(d) + os.chmod(d, 0o775) + + +def analyze_pedestal_on_cluster(instrument, raw_file_base, res_dir, user=None): + pedestals_taken = raw_file_base + "*" + pedestals_taken = glob(pedestals_taken) + + if not pedestals_taken: + return + + if user is None: + print("Analysis of raw pedestal data on SF computing nodes requires user credentials.") + user = input("Enter user name: ") + + cmd = f"ssh {user}@sf-cn-1 source /sf/{instrument}/bin/anaconda_env" + commands = [cmd] + + for ped in pedestals_taken: + cmd = f"sbatch jungfrau_create_pedestals --filename {ped} --directory {res_dir} --verbosity 4" + commands.append(cmd) + + cmd = "\;".join(commands) + os.system(cmd)