diff --git a/tests/scripts/test_frame_synchronizer.py b/tests/scripts/test_frame_synchronizer.py index cbb37e01e..b8468070c 100644 --- a/tests/scripts/test_frame_synchronizer.py +++ b/tests/scripts/test_frame_synchronizer.py @@ -3,242 +3,137 @@ ''' This file is used to start up simulators, frame synchronizer, pull sockets, acquire, test and kill them finally. ''' -import argparse -import os, sys, subprocess, time -import shlex, traceback, json + +import sys, time +import traceback, json + +from slsdet import Detector +from slsdet.defines import DEFAULT_TCP_RX_PORTNO + +from utils_for_test import ( + Log, + LogLevel, + RuntimeException, + checkIfProcessRunning, + killProcess, + cleanup, + cleanSharedmemory, + startProcessInBackground, + startProcessInBackgroundWithLogFile, + startDetectorVirtualServer, + loadConfig, + ParseArguments +) + +LOG_PREFIX_FNAME = '/tmp/slsFrameSynchronizer_test' +MAIN_LOG_FNAME = LOG_PREFIX_FNAME + '_log.txt' +PULL_SOCKET_PREFIX_FNAME = LOG_PREFIX_FNAME + '_pull_socket_' -from slsdet import Detector, detectorType, detectorSettings -from slsdet.defines import DEFAULT_TCP_RX_PORTNO, DEFAULT_UDP_DST_PORTNO -SERVER_START_PORTNO=1900 - -from utils_for_test import Log, LogLevel - - -class RuntimeException (Exception): - def __init__ (self, message): - super().__init__(Log(LogLevel.ERROR, message)) - -def checkIfProcessRunning(processName): - cmd = f"pgrep -f {processName}" - res = subprocess.getoutput(cmd) - return res.strip().splitlines() - - -def killProcess(name, fp): - pids = checkIfProcessRunning(name) - if pids: - Log(LogLevel.INFO, f"Killing '{name}' processes with PIDs: {', '.join(pids)}", fp) - for pid in pids: - try: - p = subprocess.run(['kill', pid]) - if p.returncode != 0 and bool(checkIfProcessRunning(name)): - raise RuntimeException(f"Could not kill {name} with pid {pid}") - except Exception as e: - raise RuntimeException(f"Failed to kill process {name} pid:{pid}. Exception occured: [code:{e}, msg:{e.stderr}]") - #else: - # Log(LogLevel.INFO, 'process not running : ' + name) - - -def cleanup(fp): - ''' - kill both servers, receivers and clean shared memory - ''' - Log(LogLevel.INFO, 'Cleaning up') - Log(LogLevel.INFO, 'Cleaning up', fp) - killProcess('DetectorServer_virtual', fp) - killProcess('slsReceiver', fp) - killProcess('slsMultiReceiver', fp) - killProcess('slsFrameSynchronizer', fp) - killProcess('frameSynchronizerPullSocket', fp) - cleanSharedmemory(fp) - -def cleanSharedmemory(fp): - Log(LogLevel.INFO, 'Cleaning up shared memory...', fp) - try: - p = subprocess.run(['sls_detector_get', 'free'], stdout=fp, stderr=fp) - except: - raise RuntimeException('Could not free shared memory') - -def startProcessInBackground(name, fp): - try: - # in background and dont print output - p = subprocess.Popen(shlex.split(name), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, restore_signals=False) - Log(LogLevel.INFO, 'Starting up ' + name + ' ...', fp) - except Exception as e: - raise RuntimeException(f'Could not start {name}:{e}') - -def startServers(name, num_mods): - Log(LogLevel.INFO, 'Starting server') - for i in range(num_mods): - port_no = SERVER_START_PORTNO + (i * 2) - startProcessInBackground(name + 'DetectorServer_virtual -p' + str(port_no), fp) - time.sleep(6) - -def startFrameSynchronizerPullSocket(fname, fp): - Log(LogLevel.INFO, 'Starting sync pull socket') - Log(LogLevel.INFO, f"Starting up Synchronizer pull socket. Log: {fname}", fp) - Log(LogLevel.INFO, f"Synchronizer pull socket log: {fname}") +def startFrameSynchronizerPullSocket(name, fp): + fname = PULL_SOCKET_PREFIX_FNAME + name + '.txt' cmd = ['python', '-u', 'frameSynchronizerPullSocket.py'] - try: - with open(fname, 'w') as fp: - subprocess.Popen(cmd, stdout=fp, stderr=fp, text=True) - except Exception as e: - raise RuntimeException(f"failed to start synchronizer pull socket: {e}") + startProcessInBackgroundWithLogFile(cmd, fp, fname) + def startFrameSynchronizer(num_mods, fp): - Log(LogLevel.INFO, 'Starting frame synchronizer') + cmd = ['slsFrameSynchronizer', str(DEFAULT_TCP_RX_PORTNO), str(num_mods)] # in 10.0.0 - #startProcessInBackground('slsFrameSynchronizer -n ' + str(num_mods) + ' -p ' + str(DEFAULT_TCP_RX_PORTNO)) - startProcessInBackground('slsFrameSynchronizer ' + str(DEFAULT_TCP_RX_PORTNO) + ' ' + str(num_mods), fp) - tStartup = 1 * num_mods - time.sleep(tStartup) + #cmd = ['slsFrameSynchronizer', '-p', str(DEFAULT_TCP_RX_PORTNO), '-n', str(num_mods)] + startProcessInBackground(cmd, fp) + time.sleep(1) -def loadConfig(name, num_mods, rx_hostname, settingsdir, num_frames, fp): - Log(LogLevel.INFO, 'Loading config') - Log(LogLevel.INFO, 'Loading config', fp) - try: - d = Detector() - d.virtual = [num_mods, SERVER_START_PORTNO] - d.udp_dstport = DEFAULT_UDP_DST_PORTNO - if name == 'eiger': - d.udp_dstport2 = DEFAULT_UDP_DST_PORTNO + 1 +def acquire(): + Log(LogLevel.INFO, 'Acquiring') + Log(LogLevel.INFO, 'Acquiring', fp) + d = Detector() + d.acquire() - d.rx_hostname = rx_hostname - d.udp_dstip = 'auto' - if name != "eiger": - d.udp_srcip = 'auto' - if name == "jungfrau" or name == "moench" or name == "xilinx_ctb": - d.powerchip = 1 +def testFramesCaught(name, num_frames): + d = Detector() + fnum = d.rx_framescaught[0] + if fnum != num_frames: + raise RuntimeException(f"{name} caught only {fnum}. Expected {num_frames}") + + Log(LogLevel.INFOGREEN, f'Frames caught test passed for {name}') + Log(LogLevel.INFOGREEN, f'Frames caught test passed for {name}', fp) - if d.type == detectorType.XILINX_CHIPTESTBOARD: - d.configureTransceiver() - d.frames = num_frames - except Exception as e: - raise RuntimeException(f'Could not load config for {name}. Error: {str(e)}') +def testZmqHeadetTypeCount(name, num_mods, num_frames, fp): -def validate_htype_counts(log_path, num_mods, num_ports_per_module, num_frames): + Log(LogLevel.INFO, f"Testing Zmq Header type count for {name}") + Log(LogLevel.INFO, f"Testing Zmq Header type count for {name}", fp) htype_counts = { "header": 0, "series_end": 0, "module": 0 } - # get a count of each htype from file - with open(log_path, 'r') as f: - for line in f: - line = line.strip() - if not line or not line.startswith('{'): - continue - try: - data = json.loads(line) - htype = data.get("htype") - if htype in htype_counts: - htype_counts[htype] += 1 - except json.JSONDecodeError: - continue # or log malformed line - - for htype, expected_count in [("header", num_mods), ("series_end", num_mods), ("module", num_ports_per_module * num_mods * num_frames)]: - if htype_counts[htype] != expected_count: - msg = f"Expected {expected_count} '{htype}' entries, found {htype_counts[htype]}" - raise RuntimeException(msg) - -def startTests(name, num_mods, num_frames, fp, file_pull_socket): - Log(LogLevel.INFO, 'Tests for ' + name) - Log(LogLevel.INFO, 'Tests for ' + name, fp) - cmd = 'tests --abort [.cmdcall] -s -o ' + fname - - d = Detector() - num_ports_per_module = d.numinterfaces - if name == "gotthard2": - num_ports_per_module = 1 - d.acquire() - fnum = d.rx_framescaught[0] - if fnum != num_frames: - raise RuntimeException(f"{name} caught only {fnum}. Expected {num_frames}") - - validate_htype_counts(file_pull_socket, num_mods, num_ports_per_module, num_frames) - Log(LogLevel.INFOGREEN, f"Log file htype checks passed for {name}", fp) - - -# parse cmd line for rx_hostname and settingspath using the argparse library -parser = argparse.ArgumentParser(description = 'automated tests with the virtual detector servers') -parser.add_argument('rx_hostname', nargs='?', default='localhost', help = 'hostname/ip of the current machine') -parser.add_argument('settingspath', nargs='?', default='../../settingsdir', help = 'Relative or absolut path to the settingspath') -parser.add_argument('-n', '--num-mods', nargs='?', default=2, type=int, help = 'Number of modules to test with') -parser.add_argument('-f', '--num-frames', nargs='?', default=1, type=int, help = 'Number of frames to test with') -parser.add_argument('-s', '--servers', nargs='*', help='Detector servers to run') -args = parser.parse_args() - -if args.servers is None: - servers = [ - 'eiger', - 'jungfrau', - 'mythen3', - 'gotthard2', - 'ctb', - 'moench', - 'xilinx_ctb' - ] -else: - servers = args.servers - - -Log(LogLevel.INFO, 'Arguments:\nrx_hostname: ' + args.rx_hostname + '\nsettingspath: \'' + args.settingspath + '\nservers: \'' + ' '.join(servers) + '\nnum_mods: \'' + str(args.num_mods) + '\nnum_frames: \'' + str(args.num_frames) + '\'') - - -# redirect to file -prefix_fname = '/tmp/slsFrameSynchronizer_test' -original_stdout = sys.stdout -original_stderr = sys.stderr -fname = prefix_fname + '_log.txt' -Log(LogLevel.INFOBLUE, '\nLog File: ' + fname) - -with open(fname, 'w') as fp: - try: - testError = False - for server in servers: - try: - Log(LogLevel.INFOBLUE, '\nSynchonizer tests for ' + server, fp) - Log(LogLevel.INFOBLUE, '\nSynchonizer tests for ' + server) - - # cmd tests for det - cleanup(fp) - startServers(server, args.num_mods) - file_pull_socket = prefix_fname + '_pull_socket_' + server + '.txt' - startFrameSynchronizerPullSocket(file_pull_socket, fp) - startFrameSynchronizer(args.num_mods, fp) - loadConfig(server, args.num_mods, args.rx_hostname, args.settingspath, args.num_frames, fp) - startTests(server, args.num_mods, args.num_frames, fp, file_pull_socket) - cleanup(fp) - - except Exception as e: - # redirect to terminal - sys.stdout = original_stdout - sys.stderr = original_stderr - Log(LogLevel.INFORED, f'Exception caught while testing {server}. Cleaning up...') - with open(fname, 'a') as fp_error: - traceback.print_exc(file=fp_error) # This will log the full traceback - - testError = True - cleanup(fp) - break - - # redirect to terminal - sys.stdout = original_stdout - sys.stderr = original_stderr - if not testError: - Log(LogLevel.INFOGREEN, 'Passed all sync tests\n' + str(servers)) - + # get a count of each htype from file + pull_socket_fname = PULL_SOCKET_PREFIX_FNAME + name + '.txt' + with open(pull_socket_fname, 'r') as log_fp: + for line in log_fp: + line = line.strip() + if not line or not line.startswith('{'): + continue + try: + data = json.loads(line) + htype = data.get("htype") + if htype in htype_counts: + htype_counts[htype] += 1 + except json.JSONDecodeError: + continue + # test if file contents matches expected counts + d = Detector() + num_ports_per_module = 1 if name == "gotthard2" else d.numinterfaces + total_num_frame_parts = num_ports_per_module * num_mods * num_frames + for htype, expected_count in [("header", num_mods), ("series_end", num_mods), ("module", total_num_frame_parts)]: + if htype_counts[htype] != expected_count: + msg = f"Expected {expected_count} '{htype}' entries, found {htype_counts[htype]}" + raise RuntimeException(msg) except Exception as e: - # redirect to terminal - sys.stdout = original_stdout - sys.stderr = original_stderr - Log(LogLevel.INFORED, f'Exception caught with general testing. Cleaning up...') - cleanup(fp) - \ No newline at end of file + raise RuntimeException(f'Failed to get zmq header count type. Error:{e}') + + Log(LogLevel.INFOGREEN, f"Zmq Header type count test passed for {name}") + Log(LogLevel.INFOGREEN, f"Zmq Header type count test passed for {name}", fp) + + +def startTestsForAll(args, fp): + for server in args.servers: + try: + Log(LogLevel.INFOBLUE, f'Synchronizer Tests for {server}') + Log(LogLevel.INFOBLUE, f'Synchronizer Tests for {server}', fp) + cleanup(fp) + startDetectorVirtualServer(server, args.num_mods, fp) + startFrameSynchronizerPullSocket(server, fp) + startFrameSynchronizer(args.num_mods, fp) + loadConfig(name=server, rx_hostname=args.rx_hostname, settingsdir=args.settingspath, fp=fp, num_mods=args.num_mods, num_frames=args.num_frames) + acquire() + testFramesCaught(server, args.num_frames) + testZmqHeadetTypeCount(server, args.num_mods, args.num_frames, fp) + except Exception as e: + raise RuntimeException(f'Synchronizer Tests failed') + + Log(LogLevel.INFOGREEN, 'Passed all synchronizer tests for all detectors \n' + str(args.servers)) + + +if __name__ == '__main__': + args = ParseArguments(description='Automated tests to test frame synchronizer', default_num_mods=2) + + Log(LogLevel.INFOBLUE, '\nLog File: ' + MAIN_LOG_FNAME + '\n') + + with open(MAIN_LOG_FNAME, 'w') as fp: + try: + startTestsForAll(args, fp) + cleanup(fp) + except Exception as e: + with open(MAIN_LOG_FNAME, 'a') as fp_error: + traceback.print_exc(file=fp_error) + cleanup(fp) + Log(LogLevel.ERROR, f'Tests Failed.') + + diff --git a/tests/scripts/test_simulators.py b/tests/scripts/test_simulators.py index 480418f3c..fc0a54a34 100644 --- a/tests/scripts/test_simulators.py +++ b/tests/scripts/test_simulators.py @@ -4,240 +4,86 @@ This file is used to start up simulators, receivers and run all the tests on them and finally kill the simulators and receivers. ''' import argparse -import os, sys, subprocess, time +import sys, subprocess, time, traceback -from slsdet import Detector, detectorType, detectorSettings -from slsdet.defines import DEFAULT_TCP_CNTRL_PORTNO, DEFAULT_TCP_RX_PORTNO, DEFAULT_UDP_DST_PORTNO -HALFMOD2_TCP_CNTRL_PORTNO=1955 -HALFMOD2_TCP_RX_PORTNO=1957 +from slsdet import Detector +from slsdet.defines import DEFAULT_TCP_RX_PORTNO -from utils_for_test import Log, LogLevel - -class RuntimeException (Exception): - def __init__ (self, message): - super().__init__(Log(LogLevel.INFORED, message)) - -def checkIfProcessRunning(processName): - cmd = f"pgrep -f {processName}" - res = subprocess.getoutput(cmd) - return res.strip().splitlines() +from utils_for_test import ( + Log, + LogLevel, + RuntimeException, + checkIfProcessRunning, + killProcess, + cleanup, + cleanSharedmemory, + startProcessInBackground, + runProcessWithLogFile, + startDetectorVirtualServer, + loadConfig, + ParseArguments +) -def killProcess(name): - pids = checkIfProcessRunning(name) - if pids: - Log(LogLevel.INFOGREEN, f"Killing '{name}' processes with PIDs: {', '.join(pids)}") - for pid in pids: - try: - p = subprocess.run(['kill', pid]) - if p.returncode != 0 and bool(checkIfProcessRunning(name)): - raise RuntimeException(f"Could not kill {name} with pid {pid}") - except Exception as e: - raise RuntimeException(f"Failed to kill process {name} pid:{pid}. Exception occured: [code:{e}, msg:{e.stderr}]") - #else: - # Log(LogLevel.INFO, 'process not running : ' + name) +LOG_PREFIX_FNAME = '/tmp/slsDetectorPackage_virtual_test' +MAIN_LOG_FNAME = LOG_PREFIX_FNAME + '_log.txt' +GENERAL_TESTS_LOG_FNAME = LOG_PREFIX_FNAME + '_results_general.txt' +CMD_TEST_LOG_PREFIX_FNAME = LOG_PREFIX_FNAME + '_results_cmd_' -def cleanup(fp): - ''' - kill both servers, receivers and clean shared memory - ''' - Log(LogLevel.INFOGREEN, 'Cleaning up...') - killProcess('DetectorServer_virtual') - killProcess('slsReceiver') - killProcess('slsMultiReceiver') - cleanSharedmemory(fp) - -def cleanSharedmemory(fp): - Log(LogLevel.INFOGREEN, 'Cleaning up shared memory...') - try: - p = subprocess.run(['sls_detector_get', 'free'], stdout=fp, stderr=fp) - except: - raise RuntimeException('Could not free shared memory') - -def startProcessInBackground(name): - try: - # in background and dont print output - p = subprocess.Popen(name.split(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, restore_signals=False) - Log(LogLevel.INFOGREEN, 'Starting up ' + name + ' ...') - except Exception as e: - raise RuntimeException(f'Could not start {name}:{e}') - -def startServer(name): - - startProcessInBackground(name + 'DetectorServer_virtual') - # second half - if name == 'eiger': - startProcessInBackground(name + 'DetectorServer_virtual -p' + str(HALFMOD2_TCP_CNTRL_PORTNO)) - tStartup = 6 - Log(LogLevel.INFO, 'Takes ' + str(tStartup) + ' seconds... Please be patient') - time.sleep(tStartup) - -def startReceiver(name): - startProcessInBackground('slsReceiver') - # second half - if name == 'eiger': - startProcessInBackground('slsReceiver -t' + str(HALFMOD2_TCP_RX_PORTNO)) - time.sleep(2) - -def loadConfig(name, rx_hostname, settingsdir): - Log(LogLevel.INFOGREEN, 'Loading config') - try: - d = Detector() - if name == 'eiger': - d.hostname = 'localhost:' + str(DEFAULT_TCP_CNTRL_PORTNO) + '+localhost:' + str(HALFMOD2_TCP_CNTRL_PORTNO) - #d.udp_dstport = {2: 50003} - # will set up for every module - d.udp_dstport = DEFAULT_UDP_DST_PORTNO - d.udp_dstport2 = DEFAULT_UDP_DST_PORTNO + 1 - d.rx_hostname = rx_hostname + ':' + str(DEFAULT_TCP_RX_PORTNO) + '+' + rx_hostname + ':' + str(HALFMOD2_TCP_RX_PORTNO) - d.udp_dstip = 'auto' - d.trimen = [4500, 5400, 6400] - d.settingspath = settingsdir + '/eiger/' - d.setThresholdEnergy(4500, detectorSettings.STANDARD) - else: - d.hostname = 'localhost' - d.rx_hostname = rx_hostname - d.udp_dstip = 'auto' - d.udp_dstip = 'auto' - if d.type == detectorType.GOTTHARD: - d.udp_srcip = d.udp_dstip - else: - d.udp_srcip = 'auto' - if d.type == detectorType.JUNGFRAU or d.type == detectorType.MOENCH or d.type == detectorType.XILINX_CHIPTESTBOARD: - d.powerchip = 1 - if d.type == detectorType.XILINX_CHIPTESTBOARD: - d.configureTransceiver() - except: - raise RuntimeException('Could not load config for ' + name) - -def startCmdTests(name, fp, fname): - Log(LogLevel.INFOGREEN, 'Cmd Tests for ' + name) - cmd = 'tests --abort [.cmdcall] -s -o ' + fname - try: - subprocess.run(cmd.split(), stdout=fp, stderr=fp, check=True, text=True) - except subprocess.CalledProcessError as e: - pass - - with open (fname, 'r') as f: - for line in f: - if "FAILED" in line: - msg = 'Cmd tests failed for ' + name + '!!!' - sys.stdout = original_stdout - Log(LogLevel.ERROR, f"{msg}\n{line}") - sys.stdout = fp - raise Exception(msg) - - Log(LogLevel.INFOGREEN, 'Cmd Tests successful for ' + name) - -def startGeneralTests(fp, fname): - Log(LogLevel.INFOGREEN, 'General Tests') - cmd = 'tests --abort -s -o ' + fname - try: - subprocess.run(cmd.split(), stdout=fp, stderr=fp, check=True, text=True) - except subprocess.CalledProcessError as e: - pass - - with open (fname, 'r') as f: - for line in f: - if "FAILED" in line: - msg = 'General tests failed !!!' - sys.stdout = original_stdout - Log(LogLevel.ERROR, msg + '\n' + line) - sys.stdout = fp - raise Exception(msg) - - Log(LogLevel.INFOGREEN, 'General Tests successful') - - - -# parse cmd line for rx_hostname and settingspath using the argparse library -parser = argparse.ArgumentParser(description = 'automated tests with the virtual detector servers') -parser.add_argument('rx_hostname', nargs='?', default='localhost', help = 'hostname/ip of the current machine') -parser.add_argument('settingspath', nargs='?', default='../../settingsdir', help = 'Relative or absolut path to the settingspath') -parser.add_argument('-s', '--servers', help='Detector servers to run', nargs='*') -args = parser.parse_args() - -if args.servers is None: - servers = [ - 'eiger', - 'jungfrau', - 'mythen3', - 'gotthard2', - 'gotthard', - 'ctb', - 'moench', - 'xilinx_ctb' - ] -else: - servers = args.servers - - -Log(LogLevel.INFO, 'Arguments:\nrx_hostname: ' + args.rx_hostname + '\nsettingspath: \'' + args.settingspath + '\nservers: \'' + ' '.join(servers) + '\'') - - -# redirect to file -prefix_fname = '/tmp/slsDetectorPackage_virtual_test' -original_stdout = sys.stdout -original_stderr = sys.stderr -fname = prefix_fname + '_log.txt' -Log(LogLevel.INFOBLUE, '\nLog File: ' + fname) - -with open(fname, 'w') as fp: - - - - # general tests - file_results = prefix_fname + '_results_general.txt' - Log(LogLevel.INFOBLUE, 'General tests (results: ' + file_results + ')') - sys.stdout = fp - sys.stderr = fp - Log(LogLevel.INFOBLUE, 'General tests (results: ' + file_results + ')') +def startReceiver(num_mods, fp): + if num_mods == 1: + cmd = ['slsReceiver'] + else: + cmd = ['slsMultiReceiver', str(DEFAULT_TCP_RX_PORTNO), str(num_mods)] + # in 10.0.0 + #cmd = ['slsMultiReceiver', '-p', str(DEFAULT_TCP_RX_PORTNO), '-n', str(num_mods)] + startProcessInBackground(cmd, fp) + time.sleep(1) +def startGeneralTests(fp): + fname = GENERAL_TESTS_LOG_FNAME + cmd = ['tests', '--abort', '-s'] try: cleanup(fp) - startGeneralTests(fp, file_results) - cleanup(fp) - - testError = False - for server in servers: - try: - # print to terminal for progress - sys.stdout = original_stdout - sys.stderr = original_stderr - file_results = prefix_fname + '_results_cmd_' + server + '.txt' - Log(LogLevel.INFOBLUE, 'Cmd tests for ' + server + ' (results: ' + file_results + ')') - sys.stdout = fp - sys.stderr = fp - Log(LogLevel.INFOBLUE, 'Cmd tests for ' + server + ' (results: ' + file_results + ')') - - # cmd tests for det - cleanup(fp) - startServer(server) - startReceiver(server) - loadConfig(server, args.rx_hostname, args.settingspath) - startCmdTests(server, fp, file_results) - cleanup(fp) - - except Exception as e: - # redirect to terminal - sys.stdout = original_stdout - sys.stderr = original_stderr - Log(LogLevel.INFORED, f'Exception caught while testing {server}. Cleaning up...') - testError = True - break - - # redirect to terminal - sys.stdout = original_stdout - sys.stderr = original_stderr - if not testError: - Log(LogLevel.INFOGREEN, 'Passed all tests for virtual detectors \n' + str(servers)) - - + runProcessWithLogFile('General Tests', cmd, fp, fname) except Exception as e: - # redirect to terminal - sys.stdout = original_stdout - sys.stderr = original_stderr - Log(LogLevel.INFORED, f'Exception caught with general testing. Cleaning up...') - cleanSharedmemory(sys.stdout) - \ No newline at end of file + raise RuntimeException(f'General tests failed.') from e + + +def startCmdTestsForAll(args, fp): + for server in args.servers: + try: + num_mods = 2 if server == 'eiger' else 1 + fname = CMD_TEST_LOG_PREFIX_FNAME + server + '.txt' + cmd = ['tests', '--abort', '[.cmdcall]', '-s'] + + Log(LogLevel.INFOBLUE, f'Starting Cmd Tests for {server}') + cleanup(fp) + startDetectorVirtualServer(name=server, num_mods=num_mods, fp=fp) + startReceiver(num_mods, fp) + loadConfig(name=server, rx_hostname=args.rx_hostname, settingsdir=args.settingspath, fp=fp, num_mods=num_mods) + runProcessWithLogFile('Cmd Tests for ' + server, cmd, fp, fname) + except Exception as e: + raise RuntimeException(f'Cmd Tests failed for {server}.') + + Log(LogLevel.INFOGREEN, 'Passed all tests for all detectors \n' + str(args.servers)) + + +if __name__ == '__main__': + args = ParseArguments('Automated tests with the virtual detector servers') + if args.num_mods > 1: + raise RuntimeException(f'Cannot support multiple modules at the moment (except Eiger).') + + Log(LogLevel.INFOBLUE, '\nLog File: ' + MAIN_LOG_FNAME + '\n') + + with open(MAIN_LOG_FNAME, 'w') as fp: + try: + startGeneralTests(fp) + startCmdTestsForAll(args, fp) + cleanup(fp) + except Exception as e: + with open(MAIN_LOG_FNAME, 'a') as fp_error: + traceback.print_exc(file=fp_error) + cleanup(fp) + Log(LogLevel.ERROR, f'Tests Failed.') diff --git a/tests/scripts/utils_for_test.py b/tests/scripts/utils_for_test.py index 5b59994c7..2969d5599 100644 --- a/tests/scripts/utils_for_test.py +++ b/tests/scripts/utils_for_test.py @@ -4,12 +4,17 @@ This file is used for common utils used for integration tests between simulators and receivers. ''' -import sys +import sys, subprocess, time, argparse from enum import Enum from colorama import Fore, Style, init +from slsdet import Detector, detectorSettings +from slsdet.defines import DEFAULT_TCP_RX_PORTNO, DEFAULT_UDP_DST_PORTNO +SERVER_START_PORTNO=1900 + init(autoreset=True) + class LogLevel(Enum): INFO = 0 INFORED = 1 @@ -19,12 +24,14 @@ class LogLevel(Enum): ERROR = 5 DEBUG = 6 + LOG_LABELS = { LogLevel.WARNING: "WARNING: ", LogLevel.ERROR: "ERROR: ", LogLevel.DEBUG: "DEBUG: " } + LOG_COLORS = { LogLevel.INFO: Fore.WHITE, LogLevel.INFORED: Fore.RED, @@ -35,7 +42,204 @@ LOG_COLORS = { LogLevel.DEBUG: Fore.CYAN } + def Log(level: LogLevel, message: str, stream=sys.stdout): color = LOG_COLORS.get(level, Fore.WHITE) label = LOG_LABELS.get(level, "") - print(f"{color}{label}{message}{Style.RESET_ALL}", file=stream, flush=True) \ No newline at end of file + print(f"{color}{label}{message}{Style.RESET_ALL}", file=stream, flush=True) + + +class RuntimeException (Exception): + def __init__ (self, message): + Log(LogLevel.ERROR, message) + super().__init__(message) + + +def checkIfProcessRunning(processName): + cmd = f"pgrep -f {processName}" + res = subprocess.getoutput(cmd) + return res.strip().splitlines() + + +def killProcess(name, fp): + pids = checkIfProcessRunning(name) + if pids: + Log(LogLevel.INFO, f"Killing '{name}' processes with PIDs: {', '.join(pids)}", fp) + for pid in pids: + try: + p = subprocess.run(['kill', pid]) + if p.returncode != 0 and bool(checkIfProcessRunning(name)): + raise RuntimeException(f"Could not kill {name} with pid {pid}") + except Exception as e: + raise RuntimeException(f"Failed to kill process {name} pid:{pid}. Exception occured: [code:{e}, msg:{e.stderr}]") + #else: + # Log(LogLevel.INFO, 'process not running : ' + name) + + +def cleanSharedmemory(fp): + Log(LogLevel.INFO, 'Cleaning up shared memory', fp) + try: + p = subprocess.run(['sls_detector_get', 'free'], stdout=fp, stderr=fp) + except: + raise RuntimeException('Could not free shared memory') + + +def cleanup(fp): + Log(LogLevel.INFO, 'Cleaning up') + Log(LogLevel.INFO, 'Cleaning up', fp) + killProcess('DetectorServer_virtual', fp) + killProcess('slsReceiver', fp) + killProcess('slsMultiReceiver', fp) + killProcess('slsFrameSynchronizer', fp) + killProcess('frameSynchronizerPullSocket', fp) + cleanSharedmemory(fp) + + +def startProcessInBackground(cmd, fp): + Log(LogLevel.INFO, 'Starting up ' + ' '.join(cmd)) + Log(LogLevel.INFO, 'Starting up ' + ' '.join(cmd), fp) + try: + p = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, restore_signals=False) + except Exception as e: + raise RuntimeException(f'Failed to start {cmd}:{e}') + + +def startProcessInBackgroundWithLogFile(cmd, fp, log_file_name): + Log(LogLevel.INFOBLUE, 'Starting up ' + ' '.join(cmd) + '. Log: ' + log_file_name) + Log(LogLevel.INFOBLUE, 'Starting up ' + ' '.join(cmd) + '. Log: ' + log_file_name, fp) + try: + with open(log_file_name, 'w') as log_fp: + subprocess.Popen(cmd, stdout=log_fp, stderr=log_fp, text=True) + except Exception as e: + raise RuntimeException(f'Failed to start {cmd}:{e}') + + +def runProcessWithLogFile(name, cmd, fp, log_file_name): + Log(LogLevel.INFOBLUE, 'Running ' + name + '. Log: ' + log_file_name) + Log(LogLevel.INFOBLUE, 'Running ' + name + '. Log: ' + log_file_name, fp) + Log(LogLevel.INFOBLUE, 'Cmd: ' + ' '.join(cmd), fp) + try: + with open(log_file_name, 'w') as log_fp: + subprocess.run(cmd, stdout=log_fp, stderr=log_fp, check=True, text=True) + except subprocess.CalledProcessError as e: + pass + except Exception as e: + Log(LogLevel.ERROR, f'Failed to run {name}:{e}', fp) + raise RuntimeException(f'Failed to run {name}:{e}') + + with open (log_file_name, 'r') as f: + for line in f: + if "FAILED" in line: + raise RuntimeException(f'{line}') + + Log(LogLevel.INFOGREEN, name + ' successful!\n') + Log(LogLevel.INFOGREEN, name + ' successful!\n', fp) + + +def startDetectorVirtualServer(name :str, num_mods, fp): + for i in range(num_mods): + port_no = SERVER_START_PORTNO + (i * 2) + cmd = [name + 'DetectorServer_virtual', '-p', str(port_no)] + startProcessInBackground(cmd, fp) + match name: + case 'jungfrau': + time.sleep(7) + case 'gotthard2': + time.sleep(5) + case _: + time.sleep(3) + + +def connectToVirtualServers(name, num_mods): + try: + d = Detector() + except Exception as e: + raise RuntimeException(f'Could not create Detector object for {name}. Error: {str(e)}') + + counts_sec = 5 + while (counts_sec != 0): + try: + d.virtual = [num_mods, SERVER_START_PORTNO] + break + except Exception as e: + # stop server still not up, wait a bit longer + if "Cannot connect to" in str(e): + Log(LogLevel.WARNING, f'Still waiting for {name} virtual server to be up...{counts_sec}s left') + time.sleep(1) + counts_sec -= 1 + else: + raise + + return d + + +def loadConfig(name, rx_hostname, settingsdir, fp, num_mods = 1, num_frames = 1): + Log(LogLevel.INFO, 'Loading config') + Log(LogLevel.INFO, 'Loading config', fp) + try: + d = connectToVirtualServers(name, num_mods) + d.udp_dstport = DEFAULT_UDP_DST_PORTNO + if name == 'eiger': + d.udp_dstport2 = DEFAULT_UDP_DST_PORTNO + 1 + + d.rx_hostname = rx_hostname + d.udp_dstip = 'auto' + if name != "eiger": + if name == "gotthard": + d.udp_srcip = d.udp_dstip + else: + d.udp_srcip = 'auto' + + if name == "jungfrau" or name == "moench" or name == "xilinx_ctb": + d.powerchip = 1 + + if name == "xilinx_ctb": + d.configureTransceiver() + + if name == "eiger": + d.trimen = [4500, 5400, 6400] + d.settingspath = settingsdir + '/eiger/' + d.setThresholdEnergy(4500, detectorSettings.STANDARD) + + d.frames = num_frames + except Exception as e: + raise RuntimeException(f'Could not load config for {name}. Error: {str(e)}') + + +def ParseArguments(description, default_num_mods=1): + parser = argparse.ArgumentParser(description) + + parser.add_argument('rx_hostname', nargs='?', default='localhost', + help='Hostname/IP of the current machine') + parser.add_argument('settingspath', nargs='?', default='../../settingsdir', + help='Relative or absolute path to the settings directory') + parser.add_argument('-n', '--num-mods', nargs='?', default=default_num_mods, type=int, + help='Number of modules to test with') + parser.add_argument('-f', '--num-frames', nargs='?', default=1, type=int, + help='Number of frames to test with') + parser.add_argument('-s', '--servers', nargs='*', + help='Detector servers to run') + + args = parser.parse_args() + + # Set default server list if not provided + if args.servers is None: + args.servers = [ + 'eiger', + 'jungfrau', + 'mythen3', + 'gotthard2', + 'gotthard' + 'ctb', + 'moench', + 'xilinx_ctb' + ] + + Log(LogLevel.INFO, 'Arguments:\n' + + 'rx_hostname: ' + args.rx_hostname + + '\nsettingspath: \'' + args.settingspath + + '\nservers: \'' + ' '.join(args.servers) + + '\nnum_mods: \'' + str(args.num_mods) + + '\nnum_frames: \'' + str(args.num_frames) + '\'') + + return args