From af386a736e3d8f894ff1b552ac74d96153ac32ed Mon Sep 17 00:00:00 2001 From: Dhanya Thattil Date: Fri, 2 May 2025 16:19:34 +0200 Subject: [PATCH 01/16] first draft of frame synchonizer test. also updated tests updated test_simiulator.py --- python/CMakeLists.txt | 7 +- .../tests/Caller/test-Caller-rx.cpp | 36 +-- tests/CMakeLists.txt | 1 + tests/scripts/test_frame_synchronizer.py | 223 ++++++++++++++++++ tests/scripts/test_simulators.py | 184 +++++++-------- 5 files changed, 340 insertions(+), 111 deletions(-) create mode 100644 tests/scripts/test_frame_synchronizer.py diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 1bd44d267..36c0c35a7 100755 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -64,6 +64,10 @@ configure_file( scripts/test_virtual.py ${CMAKE_BINARY_DIR}/test_virtual.py ) +configure_file(scripts/frameSynchronizerPullSocket.py + ${CMAKE_BINARY_DIR}/frameSynchronizerPullSocket.py COPYONLY) + + configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/../VERSION ${CMAKE_BINARY_DIR}/bin/slsdet/VERSION ) @@ -76,4 +80,5 @@ if(SLS_INSTALL_PYTHONEXT) install(FILES ${PYTHON_FILES} DESTINATION ${CMAKE_INSTALL_PREFIX}/python/slsdet) install(FILES ../VERSION DESTINATION ${CMAKE_INSTALL_PREFIX}/python/slsdet) -endif() \ No newline at end of file +endif() + diff --git a/slsDetectorSoftware/tests/Caller/test-Caller-rx.cpp b/slsDetectorSoftware/tests/Caller/test-Caller-rx.cpp index 38fe14e86..66cc474d6 100644 --- a/slsDetectorSoftware/tests/Caller/test-Caller-rx.cpp +++ b/slsDetectorSoftware/tests/Caller/test-Caller-rx.cpp @@ -445,23 +445,25 @@ TEST_CASE("rx_arping", "[.cmdcall][.rx]") { Detector det; Caller caller(&det); auto prev_val = det.getRxArping(); - { - std::ostringstream oss; - caller.call("rx_arping", {"1"}, -1, PUT, oss); - REQUIRE(oss.str() == "rx_arping 1\n"); - } - { - std::ostringstream oss; - caller.call("rx_arping", {}, -1, GET, oss); - REQUIRE(oss.str() == "rx_arping 1\n"); - } - { - std::ostringstream oss; - caller.call("rx_arping", {"0"}, -1, PUT, oss); - REQUIRE(oss.str() == "rx_arping 0\n"); - } - for (int i = 0; i != det.size(); ++i) { - det.setRxArping(prev_val[i], {i}); + if (det.getDestinationUDPIP()[0].str() != "127.0.0.1") { + { + std::ostringstream oss; + caller.call("rx_arping", {"1"}, -1, PUT, oss); + REQUIRE(oss.str() == "rx_arping 1\n"); + } + { + std::ostringstream oss; + caller.call("rx_arping", {}, -1, GET, oss); + REQUIRE(oss.str() == "rx_arping 1\n"); + } + { + std::ostringstream oss; + caller.call("rx_arping", {"0"}, -1, PUT, oss); + REQUIRE(oss.str() == "rx_arping 0\n"); + } + for (int i = 0; i != det.size(); ++i) { + det.setRxArping(prev_val[i], {i}); + } } } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index f222cc599..7f717667d 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -60,3 +60,4 @@ include(Catch) catch_discover_tests(tests) configure_file(scripts/test_simulators.py ${CMAKE_BINARY_DIR}/bin/test_simulators.py COPYONLY) +configure_file(scripts/test_frame_synchronizer.py ${CMAKE_BINARY_DIR}/bin/test_frame_synchronizer.py COPYONLY) diff --git a/tests/scripts/test_frame_synchronizer.py b/tests/scripts/test_frame_synchronizer.py new file mode 100644 index 000000000..ab7febe4a --- /dev/null +++ b/tests/scripts/test_frame_synchronizer.py @@ -0,0 +1,223 @@ +# SPDX-License-Identifier: LGPL-3.0-or-other +# Copyright (C) 2021 Contributors to the SLS Detector Package +''' +This file is used to start up simulators, receivers and run all the tests on them and finally kill the simulators and receivers. +''' +import argparse +import os, sys, subprocess, time, colorama +import shlex + +from colorama import Fore, Style +from slsdet import Detector, detectorType, detectorSettings +from slsdet.defines import DEFAULT_TCP_RX_PORTNO, DEFAULT_UDP_DST_PORTNO +SERVER_START_PORTNO=1900 + + +colorama.init(autoreset=True) + +def Log(color, message): + print(f"{color}{message}{Style.RESET_ALL}", flush=True) + +class RuntimeException (Exception): + def __init__ (self, message): + super().__init__(Log(Fore.RED, message)) + +def checkIfProcessRunning(processName): + cmd = f"pgrep -f {processName}" + res = subprocess.getoutput(cmd) + return res.strip().splitlines() + + +def killProcess(name): + pids = checkIfProcessRunning(name) + if pids: + Log(Fore.GREEN, f"Killing '{name}' processes with PIDs: {', '.join(pids)}") + for pid in pids: + try: + p = subprocess.run(['kill', pid]) + if p.returncode != 0 and bool(checkIfProcessRunning(name)): + raise RuntimeException(f"Could not kill {name} with pid {pid}") + except Exception as e: + Log(Fore.RED, f"Failed to kill process {name} pid:{pid}. Exception occured: [code:{e}, msg:{e.stderr}]") + raise + #else: + # Log(Fore.WHITE, 'process not running : ' + name) + + +def cleanup(fp): + ''' + kill both servers, receivers and clean shared memory + ''' + Log(Fore.GREEN, 'Cleaning up...') + killProcess('DetectorServer_virtual') + killProcess('slsReceiver') + killProcess('slsMultiReceiver') + killProcess('slsFrameSynchronizer') + killProcess('frameSynchronizerPullSocket') + cleanSharedmemory(fp) + +def cleanSharedmemory(fp): + Log(Fore.GREEN, 'Cleaning up shared memory...') + try: + p = subprocess.run(['sls_detector_get', 'free'], stdout=fp, stderr=fp) + except: + Log(Fore.RED, 'Could not free shared memory') + raise + +def startProcessInBackground(name): + try: + # in background and dont print output + p = subprocess.Popen(shlex.split(name), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, restore_signals=False) + Log(Fore.GREEN, 'Starting up ' + name + ' ...') + except Exception as e: + Log(Fore.RED, f'Could not start {name}:{e}') + raise + +def startServers(name, num_mods): + for i in range(num_mods): + port_no = SERVER_START_PORTNO + (i * 2) + startProcessInBackground(name + 'DetectorServer_virtual -p' + str(port_no)) + time.sleep(6) + +def startFrameSynchronizerPullSocket(): + startProcessInBackground('python frameSynchronizerPullSocket.py') + tStartup = 4 + Log(Fore.WHITE, 'Takes ' + str(tStartup) + ' seconds... Please be patient') + time.sleep(tStartup) + if not checkIfProcessRunning('frameSynchonizerPull'): + Log(Fore.RED, "Could not start pull socket. Its not running.") + raise + +def startFrameSynchronizer(num_mods): + Log(Fore.GREEN, "Going to start frame synchonizer") + # in 10.0.0 + #startProcessInBackground('slsFrameSynchronizer -n ' + str(num_mods) + ' -p ' + str(DEFAULT_TCP_RX_PORTNO)) + startProcessInBackground('slsFrameSynchronizer ' + str(DEFAULT_TCP_RX_PORTNO) + ' ' + str(num_mods)) + tStartup = 1 * num_mods + Log(Fore.WHITE, 'Takes ' + str(tStartup) + ' seconds... Please be patient') + time.sleep(tStartup) + +def loadConfig(name, num_mods, rx_hostname, settingsdir, num_frames): + Log(Fore.GREEN, 'Loading config') + try: + d = Detector() + d.virtual = [num_mods, SERVER_START_PORTNO] + d.udp_dstport = DEFAULT_UDP_DST_PORTNO + + if name == 'eiger': + d.udp_dstport2 = DEFAULT_UDP_DST_PORTNO + 1 + + d.rx_hostname = rx_hostname + d.udp_dstip = 'auto' + d.udp_srcip = 'auto' + + if name == 'eiger': + d.trimen = [4500, 5400, 6400] + d.settingspath = settingsdir + '/eiger/' + d.setThresholdEnergy(4500, detectorSettings.STANDARD) + elif d.type == detectorType.JUNGFRAU or d.type == detectorType.MOENCH or d.type == detectorType.XILINX_CHIPTESTBOARD: + d.powerchip = 1 + + if d.type == detectorType.XILINX_CHIPTESTBOARD: + d.configureTransceiver() + + d.frames = num_frames + except Exception as e: + Log(Fore.RED, f'Could not load config for {name}. Error: {str(e)}') + raise + +def startTests(name, fp, fname, num_frames): + Log(Fore.GREEN, 'Tests for ' + name) + cmd = 'tests --abort [.cmdcall] -s -o ' + fname + + d = Detector() + d.acquire() + fnum = d.rx_framescaught[0] + if fnum == num_frames: + Log(Fore.RED, "{name} caught only {fnum}. Expected {num_frames}") + raise + + Log(Fore.GREEN, 'Tests successful for ' + name) + + +# parse cmd line for rx_hostname and settingspath using the argparse library +parser = argparse.ArgumentParser(description = 'automated tests with the virtual detector servers') +parser.add_argument('rx_hostname', nargs='?', default='localhost', help = 'hostname/ip of the current machine') +parser.add_argument('settingspath', nargs='?', default='../../settingsdir', help = 'Relative or absolut path to the settingspath') +parser.add_argument('-n', '--num-mods', nargs='?', default=2, type=int, help = 'Number of modules to test with') +parser.add_argument('-f', '--num-frames', nargs='?', default=1, type=int, help = 'Number of frames to test with') +parser.add_argument('-s', '--servers', nargs='*', help='Detector servers to run') +args = parser.parse_args() + +if args.servers is None: + servers = [ + #'eiger', + 'jungfrau', + #'mythen3', + #'gotthard2', + #'ctb', + #'moench', + #'xilinx_ctb' + ] +else: + servers = args.servers + + +Log(Fore.WHITE, 'Arguments:\nrx_hostname: ' + args.rx_hostname + '\nsettingspath: \'' + args.settingspath + '\nservers: \'' + ' '.join(servers) + '\nnum_mods: \'' + str(args.num_mods) + '\nnum_frames: \'' + str(args.num_frames) + '\'') + + +# redirect to file +prefix_fname = '/tmp/slsFrameSynchronizer_test' +original_stdout = sys.stdout +original_stderr = sys.stderr +fname = prefix_fname + '_log.txt' +Log(Fore.BLUE, '\nLog File: ' + fname) + +with open(fname, 'w') as fp: + + try: + cleanup(fp) + + testError = False + for server in servers: + try: + # print to terminal for progress + sys.stdout = original_stdout + sys.stderr = original_stderr + file_results = prefix_fname + '_results_cmd_' + server + '.txt' + Log(Fore.BLUE, 'Synchonizer tests for ' + server + ' (results: ' + file_results + ')') + sys.stdout = fp + sys.stderr = fp + Log(Fore.BLUE, 'Synchonizer tests for ' + server + ' (results: ' + file_results + ')') + + # cmd tests for det + cleanup(fp) + startServers(server, args.num_mods) + startFrameSynchronizerPullSocket() + startFrameSynchronizer(args.num_mods) + loadConfig(server, args.num_mods, args.rx_hostname, args.settingspath, args.num_frames) + startTests(server, fp, file_results, args.num_frames) + cleanup(fp) + + except Exception as e: + # redirect to terminal + sys.stdout = original_stdout + sys.stderr = original_stderr + Log(Fore.RED, f'Exception caught while testing {server}. Cleaning up...') + testError = True + break + + # redirect to terminal + sys.stdout = original_stdout + sys.stderr = original_stderr + if not testError: + Log(Fore.GREEN, 'Passed all sync tests\n' + str(servers)) + + + except Exception as e: + # redirect to terminal + sys.stdout = original_stdout + sys.stderr = original_stderr + Log(Fore.RED, f'Exception caught with general testing. Cleaning up...') + cleanSharedmemory(sys.stdout) + \ No newline at end of file diff --git a/tests/scripts/test_simulators.py b/tests/scripts/test_simulators.py index 34f7ca9ba..7715ede32 100644 --- a/tests/scripts/test_simulators.py +++ b/tests/scripts/test_simulators.py @@ -4,9 +4,9 @@ This file is used to start up simulators, receivers and run all the tests on them and finally kill the simulators and receivers. ''' import argparse -import os, sys, subprocess, time, colorama, signal +import os, sys, subprocess, time, colorama -from colorama import Fore +from colorama import Fore, Style from slsdet import Detector, detectorType, detectorSettings from slsdet.defines import DEFAULT_TCP_CNTRL_PORTNO, DEFAULT_TCP_RX_PORTNO, DEFAULT_UDP_DST_PORTNO HALFMOD2_TCP_CNTRL_PORTNO=1955 @@ -14,68 +14,46 @@ HALFMOD2_TCP_RX_PORTNO=1957 colorama.init(autoreset=True) +def Log(color, message): + print(f"{color}{message}{Style.RESET_ALL}", flush=True) + class RuntimeException (Exception): def __init__ (self, message): - super().__init__(Fore.RED + message) + super().__init__(Log(Fore.RED, message)) -def Log(color, message): - print('\n' + color + message, flush=True) - - def checkIfProcessRunning(processName): - cmd = "ps -ef | grep " + processName - print(cmd) - res=subprocess.getoutput(cmd) - print(res) - # eg. of output - #l_user 250506 243295 0 14:38 pts/5 00:00:00 /bin/sh -c ps -ef | grep slsReceiver - #l_user 250508 250506 0 14:38 pts/5 00:00:00 grep slsReceiver - - print('how many') - cmd = "ps -ef | grep " + processName + " | wc -l" - print(cmd) - res=subprocess.getoutput(cmd) - print(res) - - if res == '2': - return False - return True + cmd = f"pgrep -f {processName}" + res = subprocess.getoutput(cmd) + return res.strip().splitlines() def killProcess(name): - if checkIfProcessRunning(name): - Log(Fore.GREEN, 'killing ' + name) - p = subprocess.run(['killall', name]) - if p.returncode != 0: - raise RuntimeException('killall failed for ' + name) - else: - print('process not running : ' + name) + pids = checkIfProcessRunning(name) + if pids: + Log(Fore.GREEN, f"Killing '{name}' processes with PIDs: {', '.join(pids)}") + for pid in pids: + try: + p = subprocess.run(['kill', pid]) + if p.returncode != 0 and bool(checkIfProcessRunning(name)): + raise RuntimeException(f"Could not kill {name} with pid {pid}") + except Exception as e: + Log(Fore.RED, f"Failed to kill process {name} pid:{pid}. Exception occured: [code:{e}, msg:{e.stderr}]") + raise + #else: + # Log(Fore.WHITE, 'process not running : ' + name) -def killAllStaleProcesses(): - killProcess('eigerDetectorServer_virtual') - killProcess('jungfrauDetectorServer_virtual') - killProcess('mythen3DetectorServer_virtual') - killProcess('gotthard2DetectorServer_virtual') - killProcess('gotthardDetectorServer_virtual') - killProcess('ctbDetectorServer_virtual') - killProcess('moenchDetectorServer_virtual') - killProcess('xilinx_ctbDetectorServer_virtual') - killProcess('slsReceiver') - killProcess('slsMultiReceiver') - cleanSharedmemory() - -def cleanup(name): +def cleanup(fp): ''' kill both servers, receivers and clean shared memory ''' Log(Fore.GREEN, 'Cleaning up...') - killProcess(name + 'DetectorServer_virtual') + killProcess('DetectorServer_virtual') killProcess('slsReceiver') killProcess('slsMultiReceiver') - cleanSharedmemory() + cleanSharedmemory(fp) -def cleanSharedmemory(): +def cleanSharedmemory(fp): Log(Fore.GREEN, 'Cleaning up shared memory...') try: p = subprocess.run(['sls_detector_get', 'free'], stdout=fp, stderr=fp) @@ -88,8 +66,8 @@ def startProcessInBackground(name): # in background and dont print output p = subprocess.Popen(name.split(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, restore_signals=False) Log(Fore.GREEN, 'Starting up ' + name + ' ...') - except: - Log(Fore.RED, 'Could not start ' + name) + except Exception as e: + Log(Fore.RED, f'Could not start {name}:{e}') raise def startServer(name): @@ -128,6 +106,7 @@ def loadConfig(name, rx_hostname, settingsdir): d.hostname = 'localhost' d.rx_hostname = rx_hostname d.udp_dstip = 'auto' + d.udp_dstip = 'auto' if d.type == detectorType.GOTTHARD: d.udp_srcip = d.udp_dstip else: @@ -143,14 +122,19 @@ def loadConfig(name, rx_hostname, settingsdir): def startCmdTests(name, fp, fname): Log(Fore.GREEN, 'Cmd Tests for ' + name) cmd = 'tests --abort [.cmdcall] -s -o ' + fname - p = subprocess.run(cmd.split(), stdout=fp, stderr=fp, check=True, text=True) - p.check_returncode() + try: + subprocess.run(cmd.split(), stdout=fp, stderr=fp, check=True, text=True) + except subprocess.CalledProcessError as e: + pass with open (fname, 'r') as f: for line in f: if "FAILED" in line: msg = 'Cmd tests failed for ' + name + '!!!' + sys.stdout = original_stdout Log(Fore.RED, msg) + Log(Fore.RED, line) + sys.stdout = fp raise Exception(msg) Log(Fore.GREEN, 'Cmd Tests successful for ' + name) @@ -158,14 +142,18 @@ def startCmdTests(name, fp, fname): def startGeneralTests(fp, fname): Log(Fore.GREEN, 'General Tests') cmd = 'tests --abort -s -o ' + fname - p = subprocess.run(cmd.split(), stdout=fp, stderr=fp, check=True, text=True) - p.check_returncode() + try: + subprocess.run(cmd.split(), stdout=fp, stderr=fp, check=True, text=True) + except subprocess.CalledProcessError as e: + pass with open (fname, 'r') as f: for line in f: if "FAILED" in line: msg = 'General tests failed !!!' - Log(Fore.RED, msg) + sys.stdout = original_stdout + Log(Fore.RED, msg + '\n' + line) + sys.stdout = fp raise Exception(msg) Log(Fore.GREEN, 'General Tests successful') @@ -174,12 +162,10 @@ def startGeneralTests(fp, fname): # parse cmd line for rx_hostname and settingspath using the argparse library parser = argparse.ArgumentParser(description = 'automated tests with the virtual detector servers') -parser.add_argument('rx_hostname', help = 'hostname/ip of the current machine') -parser.add_argument('settingspath', help = 'Relative or absolut path to the settingspath') +parser.add_argument('rx_hostname', nargs='?', default='localhost', help = 'hostname/ip of the current machine') +parser.add_argument('settingspath', nargs='?', default='../../settingsdir', help = 'Relative or absolut path to the settingspath') parser.add_argument('-s', '--servers', help='Detector servers to run', nargs='*') args = parser.parse_args() -if args.rx_hostname == 'localhost': - raise RuntimeException('Cannot use localhost for rx_hostname for the tests (fails for rx_arping for eg.)') if args.servers is None: servers = [ @@ -196,7 +182,7 @@ else: servers = args.servers -Log(Fore.WHITE, 'Arguments:\nrx_hostname: ' + args.rx_hostname + '\nsettingspath: \'' + args.settingspath + '\'') +Log(Fore.WHITE, 'Arguments:\nrx_hostname: ' + args.rx_hostname + '\nsettingspath: \'' + args.settingspath + '\nservers: \'' + ' '.join(servers) + '\'') # redirect to file @@ -208,46 +194,58 @@ Log(Fore.BLUE, '\nLog File: ' + fname) with open(fname, 'w') as fp: + + # general tests file_results = prefix_fname + '_results_general.txt' Log(Fore.BLUE, 'General tests (results: ' + file_results + ')') sys.stdout = fp sys.stderr = fp Log(Fore.BLUE, 'General tests (results: ' + file_results + ')') - startGeneralTests(fp, file_results) - killAllStaleProcesses() + try: + startGeneralTests(fp, file_results) + cleanup(fp) - for server in servers: - try: - # print to terminal for progress - sys.stdout = original_stdout - sys.stderr = original_stderr - file_results = prefix_fname + '_results_cmd_' + server + '.txt' - Log(Fore.BLUE, 'Cmd tests for ' + server + ' (results: ' + file_results + ')') - sys.stdout = fp - sys.stderr = fp - Log(Fore.BLUE, 'Cmd tests for ' + server + ' (results: ' + file_results + ')') - - # cmd tests for det - cleanup(server) - startServer(server) - startReceiver(server) - loadConfig(server, args.rx_hostname, args.settingspath) - startCmdTests(server, fp, file_results) - cleanup(server) - except: - Log(Fore.RED, 'Exception caught. Cleaning up.') - cleanup(server) - sys.stdout = original_stdout - sys.stderr = original_stderr - Log(Fore.RED, 'Cmd tests failed for ' + server + '!!!') - raise + testError = False + for server in servers: + try: + # print to terminal for progress + sys.stdout = original_stdout + sys.stderr = original_stderr + file_results = prefix_fname + '_results_cmd_' + server + '.txt' + Log(Fore.BLUE, 'Cmd tests for ' + server + ' (results: ' + file_results + ')') + sys.stdout = fp + sys.stderr = fp + Log(Fore.BLUE, 'Cmd tests for ' + server + ' (results: ' + file_results + ')') + + # cmd tests for det + cleanup(fp) + startServer(server) + startReceiver(server) + loadConfig(server, args.rx_hostname, args.settingspath) + startCmdTests(server, fp, file_results) + cleanup(fp) + + except Exception as e: + # redirect to terminal + sys.stdout = original_stdout + sys.stderr = original_stderr + Log(Fore.RED, f'Exception caught while testing {server}. Cleaning up...') + testError = True + break + + # redirect to terminal + sys.stdout = original_stdout + sys.stderr = original_stderr + if not testError: + Log(Fore.GREEN, 'Passed all tests for virtual detectors \n' + str(servers)) - Log(Fore.GREEN, 'Passed all tests for virtual detectors \n' + str(servers)) - -# redirect to terminal -sys.stdout = original_stdout -sys.stderr = original_stderr -Log(Fore.GREEN, 'Passed all tests for virtual detectors \n' + str(servers) + '\nYayyyy! :) ') \ No newline at end of file + except Exception as e: + # redirect to terminal + sys.stdout = original_stdout + sys.stderr = original_stderr + Log(Fore.RED, f'Exception caught with general testing. Cleaning up...') + cleanSharedmemory(sys.stdout) + \ No newline at end of file From 0f4d10912bed4e94c4a06fb482e655d554f01244 Mon Sep 17 00:00:00 2001 From: Dhanya Thattil Date: Fri, 2 May 2025 22:48:48 +0200 Subject: [PATCH 02/16] test frame synchonizer now runs for all modules (only jungfrua enableD) with num modules and num frames as arguments (also servers) and checks the output log of pull socket to ensure there are expected number of headers, moduels and series end htypes --- python/CMakeLists.txt | 2 +- tests/scripts/test_frame_synchronizer.py | 113 ++++++++++++++--------- 2 files changed, 72 insertions(+), 43 deletions(-) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 36c0c35a7..9765e55d1 100755 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -65,7 +65,7 @@ configure_file( scripts/test_virtual.py ) configure_file(scripts/frameSynchronizerPullSocket.py - ${CMAKE_BINARY_DIR}/frameSynchronizerPullSocket.py COPYONLY) + ${CMAKE_BINARY_DIR}/bin/frameSynchronizerPullSocket.py COPYONLY) configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/../VERSION diff --git a/tests/scripts/test_frame_synchronizer.py b/tests/scripts/test_frame_synchronizer.py index ab7febe4a..2d88b6fcb 100644 --- a/tests/scripts/test_frame_synchronizer.py +++ b/tests/scripts/test_frame_synchronizer.py @@ -5,7 +5,7 @@ This file is used to start up simulators, receivers and run all the tests on the ''' import argparse import os, sys, subprocess, time, colorama -import shlex +import shlex, traceback, json from colorama import Fore, Style from slsdet import Detector, detectorType, detectorSettings @@ -15,8 +15,8 @@ SERVER_START_PORTNO=1900 colorama.init(autoreset=True) -def Log(color, message): - print(f"{color}{message}{Style.RESET_ALL}", flush=True) +def Log(color, message, stream=sys.stdout): + print(f"{color}{message}{Style.RESET_ALL}", file=stream, flush=True) class RuntimeException (Exception): def __init__ (self, message): @@ -28,10 +28,10 @@ def checkIfProcessRunning(processName): return res.strip().splitlines() -def killProcess(name): +def killProcess(name, fp): pids = checkIfProcessRunning(name) if pids: - Log(Fore.GREEN, f"Killing '{name}' processes with PIDs: {', '.join(pids)}") + Log(Fore.WHITE, f"Killing '{name}' processes with PIDs: {', '.join(pids)}", fp) for pid in pids: try: p = subprocess.run(['kill', pid]) @@ -48,16 +48,17 @@ def cleanup(fp): ''' kill both servers, receivers and clean shared memory ''' - Log(Fore.GREEN, 'Cleaning up...') - killProcess('DetectorServer_virtual') - killProcess('slsReceiver') - killProcess('slsMultiReceiver') - killProcess('slsFrameSynchronizer') - killProcess('frameSynchronizerPullSocket') + Log(Fore.WHITE, 'Cleaning up') + Log(Fore.WHITE, 'Cleaning up', fp) + killProcess('DetectorServer_virtual', fp) + killProcess('slsReceiver', fp) + killProcess('slsMultiReceiver', fp) + killProcess('slsFrameSynchronizer', fp) + killProcess('frameSynchronizerPullSocket', fp) cleanSharedmemory(fp) def cleanSharedmemory(fp): - Log(Fore.GREEN, 'Cleaning up shared memory...') + Log(Fore.WHITE, 'Cleaning up shared memory...', fp) try: p = subprocess.run(['sls_detector_get', 'free'], stdout=fp, stderr=fp) except: @@ -68,37 +69,41 @@ def startProcessInBackground(name): try: # in background and dont print output p = subprocess.Popen(shlex.split(name), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, restore_signals=False) - Log(Fore.GREEN, 'Starting up ' + name + ' ...') + Log(Fore.WHITE, 'Starting up ' + name + ' ...', fp) except Exception as e: Log(Fore.RED, f'Could not start {name}:{e}') raise def startServers(name, num_mods): + Log(Fore.WHITE, 'Starting server') for i in range(num_mods): port_no = SERVER_START_PORTNO + (i * 2) startProcessInBackground(name + 'DetectorServer_virtual -p' + str(port_no)) time.sleep(6) -def startFrameSynchronizerPullSocket(): - startProcessInBackground('python frameSynchronizerPullSocket.py') - tStartup = 4 - Log(Fore.WHITE, 'Takes ' + str(tStartup) + ' seconds... Please be patient') - time.sleep(tStartup) - if not checkIfProcessRunning('frameSynchonizerPull'): - Log(Fore.RED, "Could not start pull socket. Its not running.") +def startFrameSynchronizerPullSocket(fname, fp): + Log(Fore.WHITE, 'Starting sync pull socket') + Log(Fore.WHITE, f"Starting up Synchronizer pull socket. Log: {fname}", fp) + Log(Fore.WHITE, f"Synchronizer pull socket log: {fname}") + cmd = ['python', '-u', 'frameSynchronizerPullSocket.py'] + try: + with open(fname, 'w') as fp: + subprocess.Popen(cmd, stdout=fp, stderr=fp, text=True) + except Exception as e: + Log(Fore.RED, f"failed to start synchronizer pull socket: {e}") raise def startFrameSynchronizer(num_mods): - Log(Fore.GREEN, "Going to start frame synchonizer") + Log(Fore.WHITE, 'Starting frame synchronizer') # in 10.0.0 #startProcessInBackground('slsFrameSynchronizer -n ' + str(num_mods) + ' -p ' + str(DEFAULT_TCP_RX_PORTNO)) startProcessInBackground('slsFrameSynchronizer ' + str(DEFAULT_TCP_RX_PORTNO) + ' ' + str(num_mods)) tStartup = 1 * num_mods - Log(Fore.WHITE, 'Takes ' + str(tStartup) + ' seconds... Please be patient') time.sleep(tStartup) -def loadConfig(name, num_mods, rx_hostname, settingsdir, num_frames): - Log(Fore.GREEN, 'Loading config') +def loadConfig(name, num_mods, rx_hostname, settingsdir, num_frames, fp): + Log(Fore.WHITE, 'Loading config') + Log(Fore.WHITE, 'Loading config', fp) try: d = Detector() d.virtual = [num_mods, SERVER_START_PORTNO] @@ -126,18 +131,46 @@ def loadConfig(name, num_mods, rx_hostname, settingsdir, num_frames): Log(Fore.RED, f'Could not load config for {name}. Error: {str(e)}') raise -def startTests(name, fp, fname, num_frames): - Log(Fore.GREEN, 'Tests for ' + name) +def validate_htype_counts(log_path, num_mods, num_frames): + htype_counts = { + "header": 0, + "series_end": 0, + "module": 0 + } + + with open(log_path, 'r') as f: + for line in f: + line = line.strip() + if not line or not line.startswith('{'): + continue + try: + data = json.loads(line) + htype = data.get("htype") + if htype in htype_counts: + htype_counts[htype] += 1 + except json.JSONDecodeError: + continue # or log malformed line + + for htype, expected_count in [("header", num_mods), ("series_end", num_mods * num_frames), ("module", num_mods)]: + if htype_counts[htype] != expected_count: + msg = f"Expected 2 '{htype}' entries, found {htype_counts[htype]}" + Log(Fore.RED, msg) + raise RuntimeError(msg) + +def startTests(name, num_mods, num_frames, fp, file_pull_socket): + Log(Fore.WHITE, 'Tests for ' + name) + Log(Fore.WHITE, 'Tests for ' + name, fp) cmd = 'tests --abort [.cmdcall] -s -o ' + fname d = Detector() d.acquire() fnum = d.rx_framescaught[0] - if fnum == num_frames: - Log(Fore.RED, "{name} caught only {fnum}. Expected {num_frames}") + if fnum != num_frames: + Log(Fore.RED, f"{name} caught only {fnum}. Expected {num_frames}") raise - Log(Fore.GREEN, 'Tests successful for ' + name) + validate_htype_counts(file_pull_socket, num_mods, num_frames) + Log(Fore.GREEN, f"Log file htype checks passed for {name}", fp) # parse cmd line for rx_hostname and settingspath using the argparse library @@ -176,27 +209,20 @@ Log(Fore.BLUE, '\nLog File: ' + fname) with open(fname, 'w') as fp: try: - cleanup(fp) - testError = False for server in servers: try: - # print to terminal for progress - sys.stdout = original_stdout - sys.stderr = original_stderr - file_results = prefix_fname + '_results_cmd_' + server + '.txt' - Log(Fore.BLUE, 'Synchonizer tests for ' + server + ' (results: ' + file_results + ')') - sys.stdout = fp - sys.stderr = fp - Log(Fore.BLUE, 'Synchonizer tests for ' + server + ' (results: ' + file_results + ')') + Log(Fore.BLUE, '\nSynchonizer tests for ' + server, fp) + Log(Fore.BLUE, '\nSynchonizer tests for ' + server) # cmd tests for det cleanup(fp) startServers(server, args.num_mods) - startFrameSynchronizerPullSocket() + file_pull_socket = prefix_fname + '_pull_socket_' + server + '.txt' + startFrameSynchronizerPullSocket(file_pull_socket, fp) startFrameSynchronizer(args.num_mods) - loadConfig(server, args.num_mods, args.rx_hostname, args.settingspath, args.num_frames) - startTests(server, fp, file_results, args.num_frames) + loadConfig(server, args.num_mods, args.rx_hostname, args.settingspath, args.num_frames, fp) + startTests(server, args.num_mods, args.num_frames, fp, file_pull_socket) cleanup(fp) except Exception as e: @@ -204,6 +230,9 @@ with open(fname, 'w') as fp: sys.stdout = original_stdout sys.stderr = original_stderr Log(Fore.RED, f'Exception caught while testing {server}. Cleaning up...') + with open(fname, 'a') as fp_error: + traceback.print_exc(file=fp_error) # This will log the full traceback + testError = True break From 6c604e234076abe93513aa4eac3a38f9f9905f38 Mon Sep 17 00:00:00 2001 From: Dhanya Thattil Date: Sat, 3 May 2025 00:11:55 +0200 Subject: [PATCH 03/16] small fixes, but bug introduced here 'https://github.com/slsdetectorgroup/slsDetectorPackage/pull/968/commits/40895b01488662184710f8362c30b20a7828da7b' --- slsReceiverSoftware/src/FrameSynchronizerApp.cpp | 9 +++++---- tests/scripts/test_frame_synchronizer.py | 3 ++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/slsReceiverSoftware/src/FrameSynchronizerApp.cpp b/slsReceiverSoftware/src/FrameSynchronizerApp.cpp index 3d793e46f..96275e1a7 100644 --- a/slsReceiverSoftware/src/FrameSynchronizerApp.cpp +++ b/slsReceiverSoftware/src/FrameSynchronizerApp.cpp @@ -130,7 +130,7 @@ std::set get_valid_fnums(const PortFrameMap &port_frame_map) { // collect all unique frame numbers from all ports std::set unique_fnums; - for (auto it = port_frame_map.begin(); it != port_frame_map.begin(); ++it) { + for (auto it = port_frame_map.begin(); it != port_frame_map.end(); ++it) { const FrameMap &frame_map = it->second; for (auto frame = frame_map.begin(); frame != frame_map.end(); ++frame) { @@ -151,9 +151,10 @@ std::set get_valid_fnums(const PortFrameMap &port_frame_map) { LOG(sls::logDEBUG) << "Fnum " << fnum << " is missing in port " << port; // invalid: fnum greater than all in that port - auto last_frame = std::prev(frame_map.end()); - auto last_fnum = last_frame->first; - if (fnum > last_fnum) { + auto last_frame = + frame_map.upper_bound(fnum); // std::prev(frame_map.end()); + // auto last_fnum = last_frame->first; + if (last_frame == frame_map.end()) { //(fnum > last_fnum) { LOG(sls::logDEBUG) << "And no larger fnum found. Fnum " << fnum << " is invalid.\n"; is_valid = false; diff --git a/tests/scripts/test_frame_synchronizer.py b/tests/scripts/test_frame_synchronizer.py index 2d88b6fcb..f901672cf 100644 --- a/tests/scripts/test_frame_synchronizer.py +++ b/tests/scripts/test_frame_synchronizer.py @@ -234,6 +234,7 @@ with open(fname, 'w') as fp: traceback.print_exc(file=fp_error) # This will log the full traceback testError = True + cleanup(fp) break # redirect to terminal @@ -248,5 +249,5 @@ with open(fname, 'w') as fp: sys.stdout = original_stdout sys.stderr = original_stderr Log(Fore.RED, f'Exception caught with general testing. Cleaning up...') - cleanSharedmemory(sys.stdout) + cleanup(fp) \ No newline at end of file From 0ecc6030c6aa0eb3a47bf59f448e26ef24b76b76 Mon Sep 17 00:00:00 2001 From: Dhanya Thattil Date: Mon, 5 May 2025 14:03:00 +0200 Subject: [PATCH 04/16] fixed fram sync issue. mainly due to zmq msg being cleaned up before sending it --- .../src/FrameSynchronizerApp.cpp | 54 +++++++++++-------- 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/slsReceiverSoftware/src/FrameSynchronizerApp.cpp b/slsReceiverSoftware/src/FrameSynchronizerApp.cpp index 96275e1a7..22983dc88 100644 --- a/slsReceiverSoftware/src/FrameSynchronizerApp.cpp +++ b/slsReceiverSoftware/src/FrameSynchronizerApp.cpp @@ -107,11 +107,11 @@ void zmq_free(void *data, void *hint) { delete[] static_cast(data); } void print_frames(const PortFrameMap &frame_port_map) { LOG(sls::logDEBUG) << "Printing frames"; for (const auto &it : frame_port_map) { - uint16_t udpPort = it.first; + const uint16_t udpPort = it.first; const auto &frame_map = it.second; LOG(sls::logDEBUG) << "UDP port: " << udpPort; for (const auto &frame : frame_map) { - uint64_t fnum = frame.first; + const uint64_t fnum = frame.first; const auto &msg_list = frame.second; LOG(sls::logDEBUG) << " acq index: " << fnum << '[' << msg_list.size() << ']'; @@ -130,31 +130,32 @@ std::set get_valid_fnums(const PortFrameMap &port_frame_map) { // collect all unique frame numbers from all ports std::set unique_fnums; - for (auto it = port_frame_map.begin(); it != port_frame_map.end(); ++it) { - const FrameMap &frame_map = it->second; - for (auto frame = frame_map.begin(); frame != frame_map.end(); - ++frame) { - unique_fnums.insert(frame->first); + for (const auto &it : port_frame_map) { + const FrameMap &frame_map = it.second; + for (const auto &frame : frame_map) { + unique_fnums.insert(frame.first); } } // collect valid frame numbers for (auto &fnum : unique_fnums) { bool is_valid = true; - for (auto it = port_frame_map.begin(); it != port_frame_map.end(); - ++it) { - uint16_t port = it->first; - const FrameMap &frame_map = it->second; + for (const auto &it : port_frame_map) { + const uint16_t port = it.first; + const FrameMap &frame_map = it.second; auto frame = frame_map.find(fnum); // invalid: fnum missing in one port if (frame == frame_map.end()) { LOG(sls::logDEBUG) << "Fnum " << fnum << " is missing in port " << port; + /* // invalid: fnum greater than all in that port - auto last_frame = - frame_map.upper_bound(fnum); // std::prev(frame_map.end()); - // auto last_fnum = last_frame->first; - if (last_frame == frame_map.end()) { //(fnum > last_fnum) { + auto last_frame = std::prev(frame_map.end()); + auto last_fnum = last_frame->first; + if (fnum > last_fnum) { + */ + auto upper_frame = frame_map.upper_bound(fnum); + if (upper_frame == frame_map.end()) { LOG(sls::logDEBUG) << "And no larger fnum found. Fnum " << fnum << " is invalid.\n"; is_valid = false; @@ -224,29 +225,38 @@ void Correlate(FrameStatus *stat) { // sending all valid fnum data packets for (const auto &fnum : valid_fnums) { ZmqMsgList msg_list; - PortFrameMap &port_frame_map = stat->frames; - for (auto it = port_frame_map.begin(); - it != port_frame_map.end(); ++it) { - uint16_t port = it->first; - const FrameMap &frame_map = it->second; + // PortFrameMap &port_frame_map = stat->frames; + for (const auto &it : stat->frames) { + const uint16_t port = it.first; + const FrameMap &frame_map = it.second; auto frame = frame_map.find(fnum); if (frame != frame_map.end()) { msg_list.insert(msg_list.end(), stat->frames[port][fnum].begin(), stat->frames[port][fnum].end()); // clean up - for (zmq_msg_t *msg : stat->frames[port][fnum]) { + /*for (zmq_msg_t *msg : stat->frames[port][fnum]) { if (msg) { zmq_msg_close(msg); delete msg; } - } + }*/ stat->frames[port].erase(fnum); } } LOG(printHeadersLevel) << "Sending data packets for fnum " << fnum; zmq_send_multipart(socket, msg_list); + for (const auto &it : stat->frames) { + const uint16_t port = it.first; + // clean up + for (zmq_msg_t *msg : stat->frames[port][fnum]) { + if (msg) { + zmq_msg_close(msg); + delete msg; + } + } + } } } // sending all end packets From 15030ab73256f3266e444239afa71c137992b611 Mon Sep 17 00:00:00 2001 From: Dhanya Thattil Date: Mon, 5 May 2025 14:04:06 +0200 Subject: [PATCH 05/16] minor --- slsReceiverSoftware/src/FrameSynchronizerApp.cpp | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/slsReceiverSoftware/src/FrameSynchronizerApp.cpp b/slsReceiverSoftware/src/FrameSynchronizerApp.cpp index 22983dc88..c37b9b9db 100644 --- a/slsReceiverSoftware/src/FrameSynchronizerApp.cpp +++ b/slsReceiverSoftware/src/FrameSynchronizerApp.cpp @@ -148,12 +148,6 @@ std::set get_valid_fnums(const PortFrameMap &port_frame_map) { if (frame == frame_map.end()) { LOG(sls::logDEBUG) << "Fnum " << fnum << " is missing in port " << port; - /* - // invalid: fnum greater than all in that port - auto last_frame = std::prev(frame_map.end()); - auto last_fnum = last_frame->first; - if (fnum > last_fnum) { - */ auto upper_frame = frame_map.upper_bound(fnum); if (upper_frame == frame_map.end()) { LOG(sls::logDEBUG) << "And no larger fnum found. Fnum " @@ -234,13 +228,6 @@ void Correlate(FrameStatus *stat) { msg_list.insert(msg_list.end(), stat->frames[port][fnum].begin(), stat->frames[port][fnum].end()); - // clean up - /*for (zmq_msg_t *msg : stat->frames[port][fnum]) { - if (msg) { - zmq_msg_close(msg); - delete msg; - } - }*/ stat->frames[port].erase(fnum); } } From b00a37bc16f04a63ac0591411051fb4742616212 Mon Sep 17 00:00:00 2001 From: Dhanya Thattil Date: Mon, 5 May 2025 14:05:08 +0200 Subject: [PATCH 06/16] minor --- slsReceiverSoftware/src/FrameSynchronizerApp.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/slsReceiverSoftware/src/FrameSynchronizerApp.cpp b/slsReceiverSoftware/src/FrameSynchronizerApp.cpp index c37b9b9db..f9658e5ee 100644 --- a/slsReceiverSoftware/src/FrameSynchronizerApp.cpp +++ b/slsReceiverSoftware/src/FrameSynchronizerApp.cpp @@ -219,7 +219,6 @@ void Correlate(FrameStatus *stat) { // sending all valid fnum data packets for (const auto &fnum : valid_fnums) { ZmqMsgList msg_list; - // PortFrameMap &port_frame_map = stat->frames; for (const auto &it : stat->frames) { const uint16_t port = it.first; const FrameMap &frame_map = it.second; From 9889a31f816a4a9814e7949a4cc4a526fb674e80 Mon Sep 17 00:00:00 2001 From: Dhanya Thattil Date: Mon, 5 May 2025 17:22:47 +0200 Subject: [PATCH 07/16] works for all detectors --- tests/scripts/test_frame_synchronizer.py | 43 ++++++++++++------------ tests/scripts/test_simulators.py | 1 + 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/tests/scripts/test_frame_synchronizer.py b/tests/scripts/test_frame_synchronizer.py index f901672cf..621654520 100644 --- a/tests/scripts/test_frame_synchronizer.py +++ b/tests/scripts/test_frame_synchronizer.py @@ -65,7 +65,7 @@ def cleanSharedmemory(fp): Log(Fore.RED, 'Could not free shared memory') raise -def startProcessInBackground(name): +def startProcessInBackground(name, fp): try: # in background and dont print output p = subprocess.Popen(shlex.split(name), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, restore_signals=False) @@ -78,7 +78,7 @@ def startServers(name, num_mods): Log(Fore.WHITE, 'Starting server') for i in range(num_mods): port_no = SERVER_START_PORTNO + (i * 2) - startProcessInBackground(name + 'DetectorServer_virtual -p' + str(port_no)) + startProcessInBackground(name + 'DetectorServer_virtual -p' + str(port_no), fp) time.sleep(6) def startFrameSynchronizerPullSocket(fname, fp): @@ -93,11 +93,11 @@ def startFrameSynchronizerPullSocket(fname, fp): Log(Fore.RED, f"failed to start synchronizer pull socket: {e}") raise -def startFrameSynchronizer(num_mods): +def startFrameSynchronizer(num_mods, fp): Log(Fore.WHITE, 'Starting frame synchronizer') # in 10.0.0 #startProcessInBackground('slsFrameSynchronizer -n ' + str(num_mods) + ' -p ' + str(DEFAULT_TCP_RX_PORTNO)) - startProcessInBackground('slsFrameSynchronizer ' + str(DEFAULT_TCP_RX_PORTNO) + ' ' + str(num_mods)) + startProcessInBackground('slsFrameSynchronizer ' + str(DEFAULT_TCP_RX_PORTNO) + ' ' + str(num_mods), fp) tStartup = 1 * num_mods time.sleep(tStartup) @@ -114,13 +114,10 @@ def loadConfig(name, num_mods, rx_hostname, settingsdir, num_frames, fp): d.rx_hostname = rx_hostname d.udp_dstip = 'auto' - d.udp_srcip = 'auto' + if name != "eiger": + d.udp_srcip = 'auto' - if name == 'eiger': - d.trimen = [4500, 5400, 6400] - d.settingspath = settingsdir + '/eiger/' - d.setThresholdEnergy(4500, detectorSettings.STANDARD) - elif d.type == detectorType.JUNGFRAU or d.type == detectorType.MOENCH or d.type == detectorType.XILINX_CHIPTESTBOARD: + if name == "jungfrau" or name == "moench" or name == "xilinx_ctb": d.powerchip = 1 if d.type == detectorType.XILINX_CHIPTESTBOARD: @@ -131,13 +128,14 @@ def loadConfig(name, num_mods, rx_hostname, settingsdir, num_frames, fp): Log(Fore.RED, f'Could not load config for {name}. Error: {str(e)}') raise -def validate_htype_counts(log_path, num_mods, num_frames): +def validate_htype_counts(log_path, num_mods, num_ports_per_module, num_frames): htype_counts = { "header": 0, "series_end": 0, "module": 0 } + # get a count of each htype from file with open(log_path, 'r') as f: for line in f: line = line.strip() @@ -151,9 +149,9 @@ def validate_htype_counts(log_path, num_mods, num_frames): except json.JSONDecodeError: continue # or log malformed line - for htype, expected_count in [("header", num_mods), ("series_end", num_mods * num_frames), ("module", num_mods)]: + for htype, expected_count in [("header", num_mods), ("series_end", num_mods), ("module", num_ports_per_module * num_mods * num_frames)]: if htype_counts[htype] != expected_count: - msg = f"Expected 2 '{htype}' entries, found {htype_counts[htype]}" + msg = f"Expected {expected_count} '{htype}' entries, found {htype_counts[htype]}" Log(Fore.RED, msg) raise RuntimeError(msg) @@ -163,13 +161,16 @@ def startTests(name, num_mods, num_frames, fp, file_pull_socket): cmd = 'tests --abort [.cmdcall] -s -o ' + fname d = Detector() + num_ports_per_module = d.numinterfaces + if name == "gotthard2": + num_ports_per_module = 1 d.acquire() fnum = d.rx_framescaught[0] if fnum != num_frames: Log(Fore.RED, f"{name} caught only {fnum}. Expected {num_frames}") raise - validate_htype_counts(file_pull_socket, num_mods, num_frames) + validate_htype_counts(file_pull_socket, num_mods, num_ports_per_module, num_frames) Log(Fore.GREEN, f"Log file htype checks passed for {name}", fp) @@ -184,13 +185,13 @@ args = parser.parse_args() if args.servers is None: servers = [ - #'eiger', + 'eiger', 'jungfrau', - #'mythen3', - #'gotthard2', - #'ctb', - #'moench', - #'xilinx_ctb' + 'mythen3', + 'gotthard2', + 'ctb', + 'moench', + 'xilinx_ctb' ] else: servers = args.servers @@ -220,7 +221,7 @@ with open(fname, 'w') as fp: startServers(server, args.num_mods) file_pull_socket = prefix_fname + '_pull_socket_' + server + '.txt' startFrameSynchronizerPullSocket(file_pull_socket, fp) - startFrameSynchronizer(args.num_mods) + startFrameSynchronizer(args.num_mods, fp) loadConfig(server, args.num_mods, args.rx_hostname, args.settingspath, args.num_frames, fp) startTests(server, args.num_mods, args.num_frames, fp, file_pull_socket) cleanup(fp) diff --git a/tests/scripts/test_simulators.py b/tests/scripts/test_simulators.py index 7715ede32..6c59c50c0 100644 --- a/tests/scripts/test_simulators.py +++ b/tests/scripts/test_simulators.py @@ -204,6 +204,7 @@ with open(fname, 'w') as fp: Log(Fore.BLUE, 'General tests (results: ' + file_results + ')') try: + cleanup(fp) startGeneralTests(fp, file_results) cleanup(fp) From 2c93d40e81dba77a236c0e8f0c99b63642ee6e12 Mon Sep 17 00:00:00 2001 From: Dhanya Thattil Date: Tue, 6 May 2025 17:37:40 +0200 Subject: [PATCH 08/16] moved log out to utils --- tests/scripts/test_frame_synchronizer.py | 80 +++++++++++------------- tests/scripts/test_simulators.py | 67 +++++++++----------- tests/scripts/utils_for_test.py | 41 ++++++++++++ 3 files changed, 105 insertions(+), 83 deletions(-) create mode 100644 tests/scripts/utils_for_test.py diff --git a/tests/scripts/test_frame_synchronizer.py b/tests/scripts/test_frame_synchronizer.py index 621654520..cbb37e01e 100644 --- a/tests/scripts/test_frame_synchronizer.py +++ b/tests/scripts/test_frame_synchronizer.py @@ -1,26 +1,23 @@ # SPDX-License-Identifier: LGPL-3.0-or-other # Copyright (C) 2021 Contributors to the SLS Detector Package ''' -This file is used to start up simulators, receivers and run all the tests on them and finally kill the simulators and receivers. +This file is used to start up simulators, frame synchronizer, pull sockets, acquire, test and kill them finally. ''' import argparse -import os, sys, subprocess, time, colorama +import os, sys, subprocess, time import shlex, traceback, json -from colorama import Fore, Style + from slsdet import Detector, detectorType, detectorSettings from slsdet.defines import DEFAULT_TCP_RX_PORTNO, DEFAULT_UDP_DST_PORTNO SERVER_START_PORTNO=1900 +from utils_for_test import Log, LogLevel -colorama.init(autoreset=True) - -def Log(color, message, stream=sys.stdout): - print(f"{color}{message}{Style.RESET_ALL}", file=stream, flush=True) class RuntimeException (Exception): def __init__ (self, message): - super().__init__(Log(Fore.RED, message)) + super().__init__(Log(LogLevel.ERROR, message)) def checkIfProcessRunning(processName): cmd = f"pgrep -f {processName}" @@ -31,25 +28,24 @@ def checkIfProcessRunning(processName): def killProcess(name, fp): pids = checkIfProcessRunning(name) if pids: - Log(Fore.WHITE, f"Killing '{name}' processes with PIDs: {', '.join(pids)}", fp) + Log(LogLevel.INFO, f"Killing '{name}' processes with PIDs: {', '.join(pids)}", fp) for pid in pids: try: p = subprocess.run(['kill', pid]) if p.returncode != 0 and bool(checkIfProcessRunning(name)): raise RuntimeException(f"Could not kill {name} with pid {pid}") except Exception as e: - Log(Fore.RED, f"Failed to kill process {name} pid:{pid}. Exception occured: [code:{e}, msg:{e.stderr}]") - raise + raise RuntimeException(f"Failed to kill process {name} pid:{pid}. Exception occured: [code:{e}, msg:{e.stderr}]") #else: - # Log(Fore.WHITE, 'process not running : ' + name) + # Log(LogLevel.INFO, 'process not running : ' + name) def cleanup(fp): ''' kill both servers, receivers and clean shared memory ''' - Log(Fore.WHITE, 'Cleaning up') - Log(Fore.WHITE, 'Cleaning up', fp) + Log(LogLevel.INFO, 'Cleaning up') + Log(LogLevel.INFO, 'Cleaning up', fp) killProcess('DetectorServer_virtual', fp) killProcess('slsReceiver', fp) killProcess('slsMultiReceiver', fp) @@ -58,43 +54,40 @@ def cleanup(fp): cleanSharedmemory(fp) def cleanSharedmemory(fp): - Log(Fore.WHITE, 'Cleaning up shared memory...', fp) + Log(LogLevel.INFO, 'Cleaning up shared memory...', fp) try: p = subprocess.run(['sls_detector_get', 'free'], stdout=fp, stderr=fp) except: - Log(Fore.RED, 'Could not free shared memory') - raise + raise RuntimeException('Could not free shared memory') def startProcessInBackground(name, fp): try: # in background and dont print output p = subprocess.Popen(shlex.split(name), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, restore_signals=False) - Log(Fore.WHITE, 'Starting up ' + name + ' ...', fp) + Log(LogLevel.INFO, 'Starting up ' + name + ' ...', fp) except Exception as e: - Log(Fore.RED, f'Could not start {name}:{e}') - raise + raise RuntimeException(f'Could not start {name}:{e}') def startServers(name, num_mods): - Log(Fore.WHITE, 'Starting server') + Log(LogLevel.INFO, 'Starting server') for i in range(num_mods): port_no = SERVER_START_PORTNO + (i * 2) startProcessInBackground(name + 'DetectorServer_virtual -p' + str(port_no), fp) time.sleep(6) def startFrameSynchronizerPullSocket(fname, fp): - Log(Fore.WHITE, 'Starting sync pull socket') - Log(Fore.WHITE, f"Starting up Synchronizer pull socket. Log: {fname}", fp) - Log(Fore.WHITE, f"Synchronizer pull socket log: {fname}") + Log(LogLevel.INFO, 'Starting sync pull socket') + Log(LogLevel.INFO, f"Starting up Synchronizer pull socket. Log: {fname}", fp) + Log(LogLevel.INFO, f"Synchronizer pull socket log: {fname}") cmd = ['python', '-u', 'frameSynchronizerPullSocket.py'] try: with open(fname, 'w') as fp: subprocess.Popen(cmd, stdout=fp, stderr=fp, text=True) except Exception as e: - Log(Fore.RED, f"failed to start synchronizer pull socket: {e}") - raise + raise RuntimeException(f"failed to start synchronizer pull socket: {e}") def startFrameSynchronizer(num_mods, fp): - Log(Fore.WHITE, 'Starting frame synchronizer') + Log(LogLevel.INFO, 'Starting frame synchronizer') # in 10.0.0 #startProcessInBackground('slsFrameSynchronizer -n ' + str(num_mods) + ' -p ' + str(DEFAULT_TCP_RX_PORTNO)) startProcessInBackground('slsFrameSynchronizer ' + str(DEFAULT_TCP_RX_PORTNO) + ' ' + str(num_mods), fp) @@ -102,8 +95,8 @@ def startFrameSynchronizer(num_mods, fp): time.sleep(tStartup) def loadConfig(name, num_mods, rx_hostname, settingsdir, num_frames, fp): - Log(Fore.WHITE, 'Loading config') - Log(Fore.WHITE, 'Loading config', fp) + Log(LogLevel.INFO, 'Loading config') + Log(LogLevel.INFO, 'Loading config', fp) try: d = Detector() d.virtual = [num_mods, SERVER_START_PORTNO] @@ -125,8 +118,7 @@ def loadConfig(name, num_mods, rx_hostname, settingsdir, num_frames, fp): d.frames = num_frames except Exception as e: - Log(Fore.RED, f'Could not load config for {name}. Error: {str(e)}') - raise + raise RuntimeException(f'Could not load config for {name}. Error: {str(e)}') def validate_htype_counts(log_path, num_mods, num_ports_per_module, num_frames): htype_counts = { @@ -152,12 +144,11 @@ def validate_htype_counts(log_path, num_mods, num_ports_per_module, num_frames): for htype, expected_count in [("header", num_mods), ("series_end", num_mods), ("module", num_ports_per_module * num_mods * num_frames)]: if htype_counts[htype] != expected_count: msg = f"Expected {expected_count} '{htype}' entries, found {htype_counts[htype]}" - Log(Fore.RED, msg) - raise RuntimeError(msg) + raise RuntimeException(msg) def startTests(name, num_mods, num_frames, fp, file_pull_socket): - Log(Fore.WHITE, 'Tests for ' + name) - Log(Fore.WHITE, 'Tests for ' + name, fp) + Log(LogLevel.INFO, 'Tests for ' + name) + Log(LogLevel.INFO, 'Tests for ' + name, fp) cmd = 'tests --abort [.cmdcall] -s -o ' + fname d = Detector() @@ -167,11 +158,10 @@ def startTests(name, num_mods, num_frames, fp, file_pull_socket): d.acquire() fnum = d.rx_framescaught[0] if fnum != num_frames: - Log(Fore.RED, f"{name} caught only {fnum}. Expected {num_frames}") - raise + raise RuntimeException(f"{name} caught only {fnum}. Expected {num_frames}") validate_htype_counts(file_pull_socket, num_mods, num_ports_per_module, num_frames) - Log(Fore.GREEN, f"Log file htype checks passed for {name}", fp) + Log(LogLevel.INFOGREEN, f"Log file htype checks passed for {name}", fp) # parse cmd line for rx_hostname and settingspath using the argparse library @@ -197,7 +187,7 @@ else: servers = args.servers -Log(Fore.WHITE, 'Arguments:\nrx_hostname: ' + args.rx_hostname + '\nsettingspath: \'' + args.settingspath + '\nservers: \'' + ' '.join(servers) + '\nnum_mods: \'' + str(args.num_mods) + '\nnum_frames: \'' + str(args.num_frames) + '\'') +Log(LogLevel.INFO, 'Arguments:\nrx_hostname: ' + args.rx_hostname + '\nsettingspath: \'' + args.settingspath + '\nservers: \'' + ' '.join(servers) + '\nnum_mods: \'' + str(args.num_mods) + '\nnum_frames: \'' + str(args.num_frames) + '\'') # redirect to file @@ -205,7 +195,7 @@ prefix_fname = '/tmp/slsFrameSynchronizer_test' original_stdout = sys.stdout original_stderr = sys.stderr fname = prefix_fname + '_log.txt' -Log(Fore.BLUE, '\nLog File: ' + fname) +Log(LogLevel.INFOBLUE, '\nLog File: ' + fname) with open(fname, 'w') as fp: @@ -213,8 +203,8 @@ with open(fname, 'w') as fp: testError = False for server in servers: try: - Log(Fore.BLUE, '\nSynchonizer tests for ' + server, fp) - Log(Fore.BLUE, '\nSynchonizer tests for ' + server) + Log(LogLevel.INFOBLUE, '\nSynchonizer tests for ' + server, fp) + Log(LogLevel.INFOBLUE, '\nSynchonizer tests for ' + server) # cmd tests for det cleanup(fp) @@ -230,7 +220,7 @@ with open(fname, 'w') as fp: # redirect to terminal sys.stdout = original_stdout sys.stderr = original_stderr - Log(Fore.RED, f'Exception caught while testing {server}. Cleaning up...') + Log(LogLevel.INFORED, f'Exception caught while testing {server}. Cleaning up...') with open(fname, 'a') as fp_error: traceback.print_exc(file=fp_error) # This will log the full traceback @@ -242,13 +232,13 @@ with open(fname, 'w') as fp: sys.stdout = original_stdout sys.stderr = original_stderr if not testError: - Log(Fore.GREEN, 'Passed all sync tests\n' + str(servers)) + Log(LogLevel.INFOGREEN, 'Passed all sync tests\n' + str(servers)) except Exception as e: # redirect to terminal sys.stdout = original_stdout sys.stderr = original_stderr - Log(Fore.RED, f'Exception caught with general testing. Cleaning up...') + Log(LogLevel.INFORED, f'Exception caught with general testing. Cleaning up...') cleanup(fp) \ No newline at end of file diff --git a/tests/scripts/test_simulators.py b/tests/scripts/test_simulators.py index 6c59c50c0..480418f3c 100644 --- a/tests/scripts/test_simulators.py +++ b/tests/scripts/test_simulators.py @@ -4,22 +4,18 @@ This file is used to start up simulators, receivers and run all the tests on them and finally kill the simulators and receivers. ''' import argparse -import os, sys, subprocess, time, colorama +import os, sys, subprocess, time -from colorama import Fore, Style from slsdet import Detector, detectorType, detectorSettings from slsdet.defines import DEFAULT_TCP_CNTRL_PORTNO, DEFAULT_TCP_RX_PORTNO, DEFAULT_UDP_DST_PORTNO HALFMOD2_TCP_CNTRL_PORTNO=1955 HALFMOD2_TCP_RX_PORTNO=1957 -colorama.init(autoreset=True) - -def Log(color, message): - print(f"{color}{message}{Style.RESET_ALL}", flush=True) +from utils_for_test import Log, LogLevel class RuntimeException (Exception): def __init__ (self, message): - super().__init__(Log(Fore.RED, message)) + super().__init__(Log(LogLevel.INFORED, message)) def checkIfProcessRunning(processName): cmd = f"pgrep -f {processName}" @@ -30,45 +26,42 @@ def checkIfProcessRunning(processName): def killProcess(name): pids = checkIfProcessRunning(name) if pids: - Log(Fore.GREEN, f"Killing '{name}' processes with PIDs: {', '.join(pids)}") + Log(LogLevel.INFOGREEN, f"Killing '{name}' processes with PIDs: {', '.join(pids)}") for pid in pids: try: p = subprocess.run(['kill', pid]) if p.returncode != 0 and bool(checkIfProcessRunning(name)): raise RuntimeException(f"Could not kill {name} with pid {pid}") except Exception as e: - Log(Fore.RED, f"Failed to kill process {name} pid:{pid}. Exception occured: [code:{e}, msg:{e.stderr}]") - raise + raise RuntimeException(f"Failed to kill process {name} pid:{pid}. Exception occured: [code:{e}, msg:{e.stderr}]") #else: - # Log(Fore.WHITE, 'process not running : ' + name) + # Log(LogLevel.INFO, 'process not running : ' + name) def cleanup(fp): ''' kill both servers, receivers and clean shared memory ''' - Log(Fore.GREEN, 'Cleaning up...') + Log(LogLevel.INFOGREEN, 'Cleaning up...') killProcess('DetectorServer_virtual') killProcess('slsReceiver') killProcess('slsMultiReceiver') cleanSharedmemory(fp) def cleanSharedmemory(fp): - Log(Fore.GREEN, 'Cleaning up shared memory...') + Log(LogLevel.INFOGREEN, 'Cleaning up shared memory...') try: p = subprocess.run(['sls_detector_get', 'free'], stdout=fp, stderr=fp) except: - Log(Fore.RED, 'Could not free shared memory') - raise + raise RuntimeException('Could not free shared memory') def startProcessInBackground(name): try: # in background and dont print output p = subprocess.Popen(name.split(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, restore_signals=False) - Log(Fore.GREEN, 'Starting up ' + name + ' ...') + Log(LogLevel.INFOGREEN, 'Starting up ' + name + ' ...') except Exception as e: - Log(Fore.RED, f'Could not start {name}:{e}') - raise + raise RuntimeException(f'Could not start {name}:{e}') def startServer(name): @@ -77,7 +70,7 @@ def startServer(name): if name == 'eiger': startProcessInBackground(name + 'DetectorServer_virtual -p' + str(HALFMOD2_TCP_CNTRL_PORTNO)) tStartup = 6 - Log(Fore.WHITE, 'Takes ' + str(tStartup) + ' seconds... Please be patient') + Log(LogLevel.INFO, 'Takes ' + str(tStartup) + ' seconds... Please be patient') time.sleep(tStartup) def startReceiver(name): @@ -88,7 +81,7 @@ def startReceiver(name): time.sleep(2) def loadConfig(name, rx_hostname, settingsdir): - Log(Fore.GREEN, 'Loading config') + Log(LogLevel.INFOGREEN, 'Loading config') try: d = Detector() if name == 'eiger': @@ -116,11 +109,10 @@ def loadConfig(name, rx_hostname, settingsdir): if d.type == detectorType.XILINX_CHIPTESTBOARD: d.configureTransceiver() except: - Log(Fore.RED, 'Could not load config for ' + name) - raise + raise RuntimeException('Could not load config for ' + name) def startCmdTests(name, fp, fname): - Log(Fore.GREEN, 'Cmd Tests for ' + name) + Log(LogLevel.INFOGREEN, 'Cmd Tests for ' + name) cmd = 'tests --abort [.cmdcall] -s -o ' + fname try: subprocess.run(cmd.split(), stdout=fp, stderr=fp, check=True, text=True) @@ -132,15 +124,14 @@ def startCmdTests(name, fp, fname): if "FAILED" in line: msg = 'Cmd tests failed for ' + name + '!!!' sys.stdout = original_stdout - Log(Fore.RED, msg) - Log(Fore.RED, line) + Log(LogLevel.ERROR, f"{msg}\n{line}") sys.stdout = fp raise Exception(msg) - Log(Fore.GREEN, 'Cmd Tests successful for ' + name) + Log(LogLevel.INFOGREEN, 'Cmd Tests successful for ' + name) def startGeneralTests(fp, fname): - Log(Fore.GREEN, 'General Tests') + Log(LogLevel.INFOGREEN, 'General Tests') cmd = 'tests --abort -s -o ' + fname try: subprocess.run(cmd.split(), stdout=fp, stderr=fp, check=True, text=True) @@ -152,11 +143,11 @@ def startGeneralTests(fp, fname): if "FAILED" in line: msg = 'General tests failed !!!' sys.stdout = original_stdout - Log(Fore.RED, msg + '\n' + line) + Log(LogLevel.ERROR, msg + '\n' + line) sys.stdout = fp raise Exception(msg) - Log(Fore.GREEN, 'General Tests successful') + Log(LogLevel.INFOGREEN, 'General Tests successful') @@ -182,7 +173,7 @@ else: servers = args.servers -Log(Fore.WHITE, 'Arguments:\nrx_hostname: ' + args.rx_hostname + '\nsettingspath: \'' + args.settingspath + '\nservers: \'' + ' '.join(servers) + '\'') +Log(LogLevel.INFO, 'Arguments:\nrx_hostname: ' + args.rx_hostname + '\nsettingspath: \'' + args.settingspath + '\nservers: \'' + ' '.join(servers) + '\'') # redirect to file @@ -190,7 +181,7 @@ prefix_fname = '/tmp/slsDetectorPackage_virtual_test' original_stdout = sys.stdout original_stderr = sys.stderr fname = prefix_fname + '_log.txt' -Log(Fore.BLUE, '\nLog File: ' + fname) +Log(LogLevel.INFOBLUE, '\nLog File: ' + fname) with open(fname, 'w') as fp: @@ -198,10 +189,10 @@ with open(fname, 'w') as fp: # general tests file_results = prefix_fname + '_results_general.txt' - Log(Fore.BLUE, 'General tests (results: ' + file_results + ')') + Log(LogLevel.INFOBLUE, 'General tests (results: ' + file_results + ')') sys.stdout = fp sys.stderr = fp - Log(Fore.BLUE, 'General tests (results: ' + file_results + ')') + Log(LogLevel.INFOBLUE, 'General tests (results: ' + file_results + ')') try: cleanup(fp) @@ -215,10 +206,10 @@ with open(fname, 'w') as fp: sys.stdout = original_stdout sys.stderr = original_stderr file_results = prefix_fname + '_results_cmd_' + server + '.txt' - Log(Fore.BLUE, 'Cmd tests for ' + server + ' (results: ' + file_results + ')') + Log(LogLevel.INFOBLUE, 'Cmd tests for ' + server + ' (results: ' + file_results + ')') sys.stdout = fp sys.stderr = fp - Log(Fore.BLUE, 'Cmd tests for ' + server + ' (results: ' + file_results + ')') + Log(LogLevel.INFOBLUE, 'Cmd tests for ' + server + ' (results: ' + file_results + ')') # cmd tests for det cleanup(fp) @@ -232,7 +223,7 @@ with open(fname, 'w') as fp: # redirect to terminal sys.stdout = original_stdout sys.stderr = original_stderr - Log(Fore.RED, f'Exception caught while testing {server}. Cleaning up...') + Log(LogLevel.INFORED, f'Exception caught while testing {server}. Cleaning up...') testError = True break @@ -240,13 +231,13 @@ with open(fname, 'w') as fp: sys.stdout = original_stdout sys.stderr = original_stderr if not testError: - Log(Fore.GREEN, 'Passed all tests for virtual detectors \n' + str(servers)) + Log(LogLevel.INFOGREEN, 'Passed all tests for virtual detectors \n' + str(servers)) except Exception as e: # redirect to terminal sys.stdout = original_stdout sys.stderr = original_stderr - Log(Fore.RED, f'Exception caught with general testing. Cleaning up...') + Log(LogLevel.INFORED, f'Exception caught with general testing. Cleaning up...') cleanSharedmemory(sys.stdout) \ No newline at end of file diff --git a/tests/scripts/utils_for_test.py b/tests/scripts/utils_for_test.py new file mode 100644 index 000000000..5b59994c7 --- /dev/null +++ b/tests/scripts/utils_for_test.py @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: LGPL-3.0-or-other +# Copyright (C) 2021 Contributors to the SLS Detector Package +''' +This file is used for common utils used for integration tests between simulators and receivers. +''' + +import sys +from enum import Enum +from colorama import Fore, Style, init + +init(autoreset=True) + +class LogLevel(Enum): + INFO = 0 + INFORED = 1 + INFOGREEN = 2 + INFOBLUE = 3 + WARNING = 4 + ERROR = 5 + DEBUG = 6 + +LOG_LABELS = { + LogLevel.WARNING: "WARNING: ", + LogLevel.ERROR: "ERROR: ", + LogLevel.DEBUG: "DEBUG: " +} + +LOG_COLORS = { + LogLevel.INFO: Fore.WHITE, + LogLevel.INFORED: Fore.RED, + LogLevel.INFOGREEN: Fore.GREEN, + LogLevel.INFOBLUE: Fore.BLUE, + LogLevel.WARNING: Fore.YELLOW, + LogLevel.ERROR: Fore.RED, + LogLevel.DEBUG: Fore.CYAN +} + +def Log(level: LogLevel, message: str, stream=sys.stdout): + color = LOG_COLORS.get(level, Fore.WHITE) + label = LOG_LABELS.get(level, "") + print(f"{color}{label}{message}{Style.RESET_ALL}", file=stream, flush=True) \ No newline at end of file From 684f511bf811fec06f53ab7930155bfadf630a7a Mon Sep 17 00:00:00 2001 From: Dhanya Thattil Date: Tue, 6 May 2025 17:38:07 +0200 Subject: [PATCH 09/16] copy in make --- tests/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7f717667d..4811f1798 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -61,3 +61,4 @@ catch_discover_tests(tests) configure_file(scripts/test_simulators.py ${CMAKE_BINARY_DIR}/bin/test_simulators.py COPYONLY) configure_file(scripts/test_frame_synchronizer.py ${CMAKE_BINARY_DIR}/bin/test_frame_synchronizer.py COPYONLY) +configure_file(scripts/utils_for_test.py ${CMAKE_BINARY_DIR}/bin/utils_for_test.py COPYONLY) From 03eded30f9fe0d58a2e933f1c9c8931728320759 Mon Sep 17 00:00:00 2001 From: Dhanya Thattil Date: Wed, 7 May 2025 13:57:12 +0200 Subject: [PATCH 10/16] refactored test_simulator and test_frame_synchronizer tests --- tests/scripts/test_frame_synchronizer.py | 329 ++++++++--------------- tests/scripts/test_simulators.py | 300 +++++---------------- tests/scripts/utils_for_test.py | 208 +++++++++++++- 3 files changed, 391 insertions(+), 446 deletions(-) diff --git a/tests/scripts/test_frame_synchronizer.py b/tests/scripts/test_frame_synchronizer.py index cbb37e01e..b8468070c 100644 --- a/tests/scripts/test_frame_synchronizer.py +++ b/tests/scripts/test_frame_synchronizer.py @@ -3,242 +3,137 @@ ''' This file is used to start up simulators, frame synchronizer, pull sockets, acquire, test and kill them finally. ''' -import argparse -import os, sys, subprocess, time -import shlex, traceback, json + +import sys, time +import traceback, json + +from slsdet import Detector +from slsdet.defines import DEFAULT_TCP_RX_PORTNO + +from utils_for_test import ( + Log, + LogLevel, + RuntimeException, + checkIfProcessRunning, + killProcess, + cleanup, + cleanSharedmemory, + startProcessInBackground, + startProcessInBackgroundWithLogFile, + startDetectorVirtualServer, + loadConfig, + ParseArguments +) + +LOG_PREFIX_FNAME = '/tmp/slsFrameSynchronizer_test' +MAIN_LOG_FNAME = LOG_PREFIX_FNAME + '_log.txt' +PULL_SOCKET_PREFIX_FNAME = LOG_PREFIX_FNAME + '_pull_socket_' -from slsdet import Detector, detectorType, detectorSettings -from slsdet.defines import DEFAULT_TCP_RX_PORTNO, DEFAULT_UDP_DST_PORTNO -SERVER_START_PORTNO=1900 - -from utils_for_test import Log, LogLevel - - -class RuntimeException (Exception): - def __init__ (self, message): - super().__init__(Log(LogLevel.ERROR, message)) - -def checkIfProcessRunning(processName): - cmd = f"pgrep -f {processName}" - res = subprocess.getoutput(cmd) - return res.strip().splitlines() - - -def killProcess(name, fp): - pids = checkIfProcessRunning(name) - if pids: - Log(LogLevel.INFO, f"Killing '{name}' processes with PIDs: {', '.join(pids)}", fp) - for pid in pids: - try: - p = subprocess.run(['kill', pid]) - if p.returncode != 0 and bool(checkIfProcessRunning(name)): - raise RuntimeException(f"Could not kill {name} with pid {pid}") - except Exception as e: - raise RuntimeException(f"Failed to kill process {name} pid:{pid}. Exception occured: [code:{e}, msg:{e.stderr}]") - #else: - # Log(LogLevel.INFO, 'process not running : ' + name) - - -def cleanup(fp): - ''' - kill both servers, receivers and clean shared memory - ''' - Log(LogLevel.INFO, 'Cleaning up') - Log(LogLevel.INFO, 'Cleaning up', fp) - killProcess('DetectorServer_virtual', fp) - killProcess('slsReceiver', fp) - killProcess('slsMultiReceiver', fp) - killProcess('slsFrameSynchronizer', fp) - killProcess('frameSynchronizerPullSocket', fp) - cleanSharedmemory(fp) - -def cleanSharedmemory(fp): - Log(LogLevel.INFO, 'Cleaning up shared memory...', fp) - try: - p = subprocess.run(['sls_detector_get', 'free'], stdout=fp, stderr=fp) - except: - raise RuntimeException('Could not free shared memory') - -def startProcessInBackground(name, fp): - try: - # in background and dont print output - p = subprocess.Popen(shlex.split(name), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, restore_signals=False) - Log(LogLevel.INFO, 'Starting up ' + name + ' ...', fp) - except Exception as e: - raise RuntimeException(f'Could not start {name}:{e}') - -def startServers(name, num_mods): - Log(LogLevel.INFO, 'Starting server') - for i in range(num_mods): - port_no = SERVER_START_PORTNO + (i * 2) - startProcessInBackground(name + 'DetectorServer_virtual -p' + str(port_no), fp) - time.sleep(6) - -def startFrameSynchronizerPullSocket(fname, fp): - Log(LogLevel.INFO, 'Starting sync pull socket') - Log(LogLevel.INFO, f"Starting up Synchronizer pull socket. Log: {fname}", fp) - Log(LogLevel.INFO, f"Synchronizer pull socket log: {fname}") +def startFrameSynchronizerPullSocket(name, fp): + fname = PULL_SOCKET_PREFIX_FNAME + name + '.txt' cmd = ['python', '-u', 'frameSynchronizerPullSocket.py'] - try: - with open(fname, 'w') as fp: - subprocess.Popen(cmd, stdout=fp, stderr=fp, text=True) - except Exception as e: - raise RuntimeException(f"failed to start synchronizer pull socket: {e}") + startProcessInBackgroundWithLogFile(cmd, fp, fname) + def startFrameSynchronizer(num_mods, fp): - Log(LogLevel.INFO, 'Starting frame synchronizer') + cmd = ['slsFrameSynchronizer', str(DEFAULT_TCP_RX_PORTNO), str(num_mods)] # in 10.0.0 - #startProcessInBackground('slsFrameSynchronizer -n ' + str(num_mods) + ' -p ' + str(DEFAULT_TCP_RX_PORTNO)) - startProcessInBackground('slsFrameSynchronizer ' + str(DEFAULT_TCP_RX_PORTNO) + ' ' + str(num_mods), fp) - tStartup = 1 * num_mods - time.sleep(tStartup) + #cmd = ['slsFrameSynchronizer', '-p', str(DEFAULT_TCP_RX_PORTNO), '-n', str(num_mods)] + startProcessInBackground(cmd, fp) + time.sleep(1) -def loadConfig(name, num_mods, rx_hostname, settingsdir, num_frames, fp): - Log(LogLevel.INFO, 'Loading config') - Log(LogLevel.INFO, 'Loading config', fp) - try: - d = Detector() - d.virtual = [num_mods, SERVER_START_PORTNO] - d.udp_dstport = DEFAULT_UDP_DST_PORTNO - if name == 'eiger': - d.udp_dstport2 = DEFAULT_UDP_DST_PORTNO + 1 +def acquire(): + Log(LogLevel.INFO, 'Acquiring') + Log(LogLevel.INFO, 'Acquiring', fp) + d = Detector() + d.acquire() - d.rx_hostname = rx_hostname - d.udp_dstip = 'auto' - if name != "eiger": - d.udp_srcip = 'auto' - if name == "jungfrau" or name == "moench" or name == "xilinx_ctb": - d.powerchip = 1 +def testFramesCaught(name, num_frames): + d = Detector() + fnum = d.rx_framescaught[0] + if fnum != num_frames: + raise RuntimeException(f"{name} caught only {fnum}. Expected {num_frames}") + + Log(LogLevel.INFOGREEN, f'Frames caught test passed for {name}') + Log(LogLevel.INFOGREEN, f'Frames caught test passed for {name}', fp) - if d.type == detectorType.XILINX_CHIPTESTBOARD: - d.configureTransceiver() - d.frames = num_frames - except Exception as e: - raise RuntimeException(f'Could not load config for {name}. Error: {str(e)}') +def testZmqHeadetTypeCount(name, num_mods, num_frames, fp): -def validate_htype_counts(log_path, num_mods, num_ports_per_module, num_frames): + Log(LogLevel.INFO, f"Testing Zmq Header type count for {name}") + Log(LogLevel.INFO, f"Testing Zmq Header type count for {name}", fp) htype_counts = { "header": 0, "series_end": 0, "module": 0 } - # get a count of each htype from file - with open(log_path, 'r') as f: - for line in f: - line = line.strip() - if not line or not line.startswith('{'): - continue - try: - data = json.loads(line) - htype = data.get("htype") - if htype in htype_counts: - htype_counts[htype] += 1 - except json.JSONDecodeError: - continue # or log malformed line - - for htype, expected_count in [("header", num_mods), ("series_end", num_mods), ("module", num_ports_per_module * num_mods * num_frames)]: - if htype_counts[htype] != expected_count: - msg = f"Expected {expected_count} '{htype}' entries, found {htype_counts[htype]}" - raise RuntimeException(msg) - -def startTests(name, num_mods, num_frames, fp, file_pull_socket): - Log(LogLevel.INFO, 'Tests for ' + name) - Log(LogLevel.INFO, 'Tests for ' + name, fp) - cmd = 'tests --abort [.cmdcall] -s -o ' + fname - - d = Detector() - num_ports_per_module = d.numinterfaces - if name == "gotthard2": - num_ports_per_module = 1 - d.acquire() - fnum = d.rx_framescaught[0] - if fnum != num_frames: - raise RuntimeException(f"{name} caught only {fnum}. Expected {num_frames}") - - validate_htype_counts(file_pull_socket, num_mods, num_ports_per_module, num_frames) - Log(LogLevel.INFOGREEN, f"Log file htype checks passed for {name}", fp) - - -# parse cmd line for rx_hostname and settingspath using the argparse library -parser = argparse.ArgumentParser(description = 'automated tests with the virtual detector servers') -parser.add_argument('rx_hostname', nargs='?', default='localhost', help = 'hostname/ip of the current machine') -parser.add_argument('settingspath', nargs='?', default='../../settingsdir', help = 'Relative or absolut path to the settingspath') -parser.add_argument('-n', '--num-mods', nargs='?', default=2, type=int, help = 'Number of modules to test with') -parser.add_argument('-f', '--num-frames', nargs='?', default=1, type=int, help = 'Number of frames to test with') -parser.add_argument('-s', '--servers', nargs='*', help='Detector servers to run') -args = parser.parse_args() - -if args.servers is None: - servers = [ - 'eiger', - 'jungfrau', - 'mythen3', - 'gotthard2', - 'ctb', - 'moench', - 'xilinx_ctb' - ] -else: - servers = args.servers - - -Log(LogLevel.INFO, 'Arguments:\nrx_hostname: ' + args.rx_hostname + '\nsettingspath: \'' + args.settingspath + '\nservers: \'' + ' '.join(servers) + '\nnum_mods: \'' + str(args.num_mods) + '\nnum_frames: \'' + str(args.num_frames) + '\'') - - -# redirect to file -prefix_fname = '/tmp/slsFrameSynchronizer_test' -original_stdout = sys.stdout -original_stderr = sys.stderr -fname = prefix_fname + '_log.txt' -Log(LogLevel.INFOBLUE, '\nLog File: ' + fname) - -with open(fname, 'w') as fp: - try: - testError = False - for server in servers: - try: - Log(LogLevel.INFOBLUE, '\nSynchonizer tests for ' + server, fp) - Log(LogLevel.INFOBLUE, '\nSynchonizer tests for ' + server) - - # cmd tests for det - cleanup(fp) - startServers(server, args.num_mods) - file_pull_socket = prefix_fname + '_pull_socket_' + server + '.txt' - startFrameSynchronizerPullSocket(file_pull_socket, fp) - startFrameSynchronizer(args.num_mods, fp) - loadConfig(server, args.num_mods, args.rx_hostname, args.settingspath, args.num_frames, fp) - startTests(server, args.num_mods, args.num_frames, fp, file_pull_socket) - cleanup(fp) - - except Exception as e: - # redirect to terminal - sys.stdout = original_stdout - sys.stderr = original_stderr - Log(LogLevel.INFORED, f'Exception caught while testing {server}. Cleaning up...') - with open(fname, 'a') as fp_error: - traceback.print_exc(file=fp_error) # This will log the full traceback - - testError = True - cleanup(fp) - break - - # redirect to terminal - sys.stdout = original_stdout - sys.stderr = original_stderr - if not testError: - Log(LogLevel.INFOGREEN, 'Passed all sync tests\n' + str(servers)) - + # get a count of each htype from file + pull_socket_fname = PULL_SOCKET_PREFIX_FNAME + name + '.txt' + with open(pull_socket_fname, 'r') as log_fp: + for line in log_fp: + line = line.strip() + if not line or not line.startswith('{'): + continue + try: + data = json.loads(line) + htype = data.get("htype") + if htype in htype_counts: + htype_counts[htype] += 1 + except json.JSONDecodeError: + continue + # test if file contents matches expected counts + d = Detector() + num_ports_per_module = 1 if name == "gotthard2" else d.numinterfaces + total_num_frame_parts = num_ports_per_module * num_mods * num_frames + for htype, expected_count in [("header", num_mods), ("series_end", num_mods), ("module", total_num_frame_parts)]: + if htype_counts[htype] != expected_count: + msg = f"Expected {expected_count} '{htype}' entries, found {htype_counts[htype]}" + raise RuntimeException(msg) except Exception as e: - # redirect to terminal - sys.stdout = original_stdout - sys.stderr = original_stderr - Log(LogLevel.INFORED, f'Exception caught with general testing. Cleaning up...') - cleanup(fp) - \ No newline at end of file + raise RuntimeException(f'Failed to get zmq header count type. Error:{e}') + + Log(LogLevel.INFOGREEN, f"Zmq Header type count test passed for {name}") + Log(LogLevel.INFOGREEN, f"Zmq Header type count test passed for {name}", fp) + + +def startTestsForAll(args, fp): + for server in args.servers: + try: + Log(LogLevel.INFOBLUE, f'Synchronizer Tests for {server}') + Log(LogLevel.INFOBLUE, f'Synchronizer Tests for {server}', fp) + cleanup(fp) + startDetectorVirtualServer(server, args.num_mods, fp) + startFrameSynchronizerPullSocket(server, fp) + startFrameSynchronizer(args.num_mods, fp) + loadConfig(name=server, rx_hostname=args.rx_hostname, settingsdir=args.settingspath, fp=fp, num_mods=args.num_mods, num_frames=args.num_frames) + acquire() + testFramesCaught(server, args.num_frames) + testZmqHeadetTypeCount(server, args.num_mods, args.num_frames, fp) + except Exception as e: + raise RuntimeException(f'Synchronizer Tests failed') + + Log(LogLevel.INFOGREEN, 'Passed all synchronizer tests for all detectors \n' + str(args.servers)) + + +if __name__ == '__main__': + args = ParseArguments(description='Automated tests to test frame synchronizer', default_num_mods=2) + + Log(LogLevel.INFOBLUE, '\nLog File: ' + MAIN_LOG_FNAME + '\n') + + with open(MAIN_LOG_FNAME, 'w') as fp: + try: + startTestsForAll(args, fp) + cleanup(fp) + except Exception as e: + with open(MAIN_LOG_FNAME, 'a') as fp_error: + traceback.print_exc(file=fp_error) + cleanup(fp) + Log(LogLevel.ERROR, f'Tests Failed.') + + diff --git a/tests/scripts/test_simulators.py b/tests/scripts/test_simulators.py index 480418f3c..fc0a54a34 100644 --- a/tests/scripts/test_simulators.py +++ b/tests/scripts/test_simulators.py @@ -4,240 +4,86 @@ This file is used to start up simulators, receivers and run all the tests on them and finally kill the simulators and receivers. ''' import argparse -import os, sys, subprocess, time +import sys, subprocess, time, traceback -from slsdet import Detector, detectorType, detectorSettings -from slsdet.defines import DEFAULT_TCP_CNTRL_PORTNO, DEFAULT_TCP_RX_PORTNO, DEFAULT_UDP_DST_PORTNO -HALFMOD2_TCP_CNTRL_PORTNO=1955 -HALFMOD2_TCP_RX_PORTNO=1957 +from slsdet import Detector +from slsdet.defines import DEFAULT_TCP_RX_PORTNO -from utils_for_test import Log, LogLevel - -class RuntimeException (Exception): - def __init__ (self, message): - super().__init__(Log(LogLevel.INFORED, message)) - -def checkIfProcessRunning(processName): - cmd = f"pgrep -f {processName}" - res = subprocess.getoutput(cmd) - return res.strip().splitlines() +from utils_for_test import ( + Log, + LogLevel, + RuntimeException, + checkIfProcessRunning, + killProcess, + cleanup, + cleanSharedmemory, + startProcessInBackground, + runProcessWithLogFile, + startDetectorVirtualServer, + loadConfig, + ParseArguments +) -def killProcess(name): - pids = checkIfProcessRunning(name) - if pids: - Log(LogLevel.INFOGREEN, f"Killing '{name}' processes with PIDs: {', '.join(pids)}") - for pid in pids: - try: - p = subprocess.run(['kill', pid]) - if p.returncode != 0 and bool(checkIfProcessRunning(name)): - raise RuntimeException(f"Could not kill {name} with pid {pid}") - except Exception as e: - raise RuntimeException(f"Failed to kill process {name} pid:{pid}. Exception occured: [code:{e}, msg:{e.stderr}]") - #else: - # Log(LogLevel.INFO, 'process not running : ' + name) +LOG_PREFIX_FNAME = '/tmp/slsDetectorPackage_virtual_test' +MAIN_LOG_FNAME = LOG_PREFIX_FNAME + '_log.txt' +GENERAL_TESTS_LOG_FNAME = LOG_PREFIX_FNAME + '_results_general.txt' +CMD_TEST_LOG_PREFIX_FNAME = LOG_PREFIX_FNAME + '_results_cmd_' -def cleanup(fp): - ''' - kill both servers, receivers and clean shared memory - ''' - Log(LogLevel.INFOGREEN, 'Cleaning up...') - killProcess('DetectorServer_virtual') - killProcess('slsReceiver') - killProcess('slsMultiReceiver') - cleanSharedmemory(fp) - -def cleanSharedmemory(fp): - Log(LogLevel.INFOGREEN, 'Cleaning up shared memory...') - try: - p = subprocess.run(['sls_detector_get', 'free'], stdout=fp, stderr=fp) - except: - raise RuntimeException('Could not free shared memory') - -def startProcessInBackground(name): - try: - # in background and dont print output - p = subprocess.Popen(name.split(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, restore_signals=False) - Log(LogLevel.INFOGREEN, 'Starting up ' + name + ' ...') - except Exception as e: - raise RuntimeException(f'Could not start {name}:{e}') - -def startServer(name): - - startProcessInBackground(name + 'DetectorServer_virtual') - # second half - if name == 'eiger': - startProcessInBackground(name + 'DetectorServer_virtual -p' + str(HALFMOD2_TCP_CNTRL_PORTNO)) - tStartup = 6 - Log(LogLevel.INFO, 'Takes ' + str(tStartup) + ' seconds... Please be patient') - time.sleep(tStartup) - -def startReceiver(name): - startProcessInBackground('slsReceiver') - # second half - if name == 'eiger': - startProcessInBackground('slsReceiver -t' + str(HALFMOD2_TCP_RX_PORTNO)) - time.sleep(2) - -def loadConfig(name, rx_hostname, settingsdir): - Log(LogLevel.INFOGREEN, 'Loading config') - try: - d = Detector() - if name == 'eiger': - d.hostname = 'localhost:' + str(DEFAULT_TCP_CNTRL_PORTNO) + '+localhost:' + str(HALFMOD2_TCP_CNTRL_PORTNO) - #d.udp_dstport = {2: 50003} - # will set up for every module - d.udp_dstport = DEFAULT_UDP_DST_PORTNO - d.udp_dstport2 = DEFAULT_UDP_DST_PORTNO + 1 - d.rx_hostname = rx_hostname + ':' + str(DEFAULT_TCP_RX_PORTNO) + '+' + rx_hostname + ':' + str(HALFMOD2_TCP_RX_PORTNO) - d.udp_dstip = 'auto' - d.trimen = [4500, 5400, 6400] - d.settingspath = settingsdir + '/eiger/' - d.setThresholdEnergy(4500, detectorSettings.STANDARD) - else: - d.hostname = 'localhost' - d.rx_hostname = rx_hostname - d.udp_dstip = 'auto' - d.udp_dstip = 'auto' - if d.type == detectorType.GOTTHARD: - d.udp_srcip = d.udp_dstip - else: - d.udp_srcip = 'auto' - if d.type == detectorType.JUNGFRAU or d.type == detectorType.MOENCH or d.type == detectorType.XILINX_CHIPTESTBOARD: - d.powerchip = 1 - if d.type == detectorType.XILINX_CHIPTESTBOARD: - d.configureTransceiver() - except: - raise RuntimeException('Could not load config for ' + name) - -def startCmdTests(name, fp, fname): - Log(LogLevel.INFOGREEN, 'Cmd Tests for ' + name) - cmd = 'tests --abort [.cmdcall] -s -o ' + fname - try: - subprocess.run(cmd.split(), stdout=fp, stderr=fp, check=True, text=True) - except subprocess.CalledProcessError as e: - pass - - with open (fname, 'r') as f: - for line in f: - if "FAILED" in line: - msg = 'Cmd tests failed for ' + name + '!!!' - sys.stdout = original_stdout - Log(LogLevel.ERROR, f"{msg}\n{line}") - sys.stdout = fp - raise Exception(msg) - - Log(LogLevel.INFOGREEN, 'Cmd Tests successful for ' + name) - -def startGeneralTests(fp, fname): - Log(LogLevel.INFOGREEN, 'General Tests') - cmd = 'tests --abort -s -o ' + fname - try: - subprocess.run(cmd.split(), stdout=fp, stderr=fp, check=True, text=True) - except subprocess.CalledProcessError as e: - pass - - with open (fname, 'r') as f: - for line in f: - if "FAILED" in line: - msg = 'General tests failed !!!' - sys.stdout = original_stdout - Log(LogLevel.ERROR, msg + '\n' + line) - sys.stdout = fp - raise Exception(msg) - - Log(LogLevel.INFOGREEN, 'General Tests successful') - - - -# parse cmd line for rx_hostname and settingspath using the argparse library -parser = argparse.ArgumentParser(description = 'automated tests with the virtual detector servers') -parser.add_argument('rx_hostname', nargs='?', default='localhost', help = 'hostname/ip of the current machine') -parser.add_argument('settingspath', nargs='?', default='../../settingsdir', help = 'Relative or absolut path to the settingspath') -parser.add_argument('-s', '--servers', help='Detector servers to run', nargs='*') -args = parser.parse_args() - -if args.servers is None: - servers = [ - 'eiger', - 'jungfrau', - 'mythen3', - 'gotthard2', - 'gotthard', - 'ctb', - 'moench', - 'xilinx_ctb' - ] -else: - servers = args.servers - - -Log(LogLevel.INFO, 'Arguments:\nrx_hostname: ' + args.rx_hostname + '\nsettingspath: \'' + args.settingspath + '\nservers: \'' + ' '.join(servers) + '\'') - - -# redirect to file -prefix_fname = '/tmp/slsDetectorPackage_virtual_test' -original_stdout = sys.stdout -original_stderr = sys.stderr -fname = prefix_fname + '_log.txt' -Log(LogLevel.INFOBLUE, '\nLog File: ' + fname) - -with open(fname, 'w') as fp: - - - - # general tests - file_results = prefix_fname + '_results_general.txt' - Log(LogLevel.INFOBLUE, 'General tests (results: ' + file_results + ')') - sys.stdout = fp - sys.stderr = fp - Log(LogLevel.INFOBLUE, 'General tests (results: ' + file_results + ')') +def startReceiver(num_mods, fp): + if num_mods == 1: + cmd = ['slsReceiver'] + else: + cmd = ['slsMultiReceiver', str(DEFAULT_TCP_RX_PORTNO), str(num_mods)] + # in 10.0.0 + #cmd = ['slsMultiReceiver', '-p', str(DEFAULT_TCP_RX_PORTNO), '-n', str(num_mods)] + startProcessInBackground(cmd, fp) + time.sleep(1) +def startGeneralTests(fp): + fname = GENERAL_TESTS_LOG_FNAME + cmd = ['tests', '--abort', '-s'] try: cleanup(fp) - startGeneralTests(fp, file_results) - cleanup(fp) - - testError = False - for server in servers: - try: - # print to terminal for progress - sys.stdout = original_stdout - sys.stderr = original_stderr - file_results = prefix_fname + '_results_cmd_' + server + '.txt' - Log(LogLevel.INFOBLUE, 'Cmd tests for ' + server + ' (results: ' + file_results + ')') - sys.stdout = fp - sys.stderr = fp - Log(LogLevel.INFOBLUE, 'Cmd tests for ' + server + ' (results: ' + file_results + ')') - - # cmd tests for det - cleanup(fp) - startServer(server) - startReceiver(server) - loadConfig(server, args.rx_hostname, args.settingspath) - startCmdTests(server, fp, file_results) - cleanup(fp) - - except Exception as e: - # redirect to terminal - sys.stdout = original_stdout - sys.stderr = original_stderr - Log(LogLevel.INFORED, f'Exception caught while testing {server}. Cleaning up...') - testError = True - break - - # redirect to terminal - sys.stdout = original_stdout - sys.stderr = original_stderr - if not testError: - Log(LogLevel.INFOGREEN, 'Passed all tests for virtual detectors \n' + str(servers)) - - + runProcessWithLogFile('General Tests', cmd, fp, fname) except Exception as e: - # redirect to terminal - sys.stdout = original_stdout - sys.stderr = original_stderr - Log(LogLevel.INFORED, f'Exception caught with general testing. Cleaning up...') - cleanSharedmemory(sys.stdout) - \ No newline at end of file + raise RuntimeException(f'General tests failed.') from e + + +def startCmdTestsForAll(args, fp): + for server in args.servers: + try: + num_mods = 2 if server == 'eiger' else 1 + fname = CMD_TEST_LOG_PREFIX_FNAME + server + '.txt' + cmd = ['tests', '--abort', '[.cmdcall]', '-s'] + + Log(LogLevel.INFOBLUE, f'Starting Cmd Tests for {server}') + cleanup(fp) + startDetectorVirtualServer(name=server, num_mods=num_mods, fp=fp) + startReceiver(num_mods, fp) + loadConfig(name=server, rx_hostname=args.rx_hostname, settingsdir=args.settingspath, fp=fp, num_mods=num_mods) + runProcessWithLogFile('Cmd Tests for ' + server, cmd, fp, fname) + except Exception as e: + raise RuntimeException(f'Cmd Tests failed for {server}.') + + Log(LogLevel.INFOGREEN, 'Passed all tests for all detectors \n' + str(args.servers)) + + +if __name__ == '__main__': + args = ParseArguments('Automated tests with the virtual detector servers') + if args.num_mods > 1: + raise RuntimeException(f'Cannot support multiple modules at the moment (except Eiger).') + + Log(LogLevel.INFOBLUE, '\nLog File: ' + MAIN_LOG_FNAME + '\n') + + with open(MAIN_LOG_FNAME, 'w') as fp: + try: + startGeneralTests(fp) + startCmdTestsForAll(args, fp) + cleanup(fp) + except Exception as e: + with open(MAIN_LOG_FNAME, 'a') as fp_error: + traceback.print_exc(file=fp_error) + cleanup(fp) + Log(LogLevel.ERROR, f'Tests Failed.') diff --git a/tests/scripts/utils_for_test.py b/tests/scripts/utils_for_test.py index 5b59994c7..2969d5599 100644 --- a/tests/scripts/utils_for_test.py +++ b/tests/scripts/utils_for_test.py @@ -4,12 +4,17 @@ This file is used for common utils used for integration tests between simulators and receivers. ''' -import sys +import sys, subprocess, time, argparse from enum import Enum from colorama import Fore, Style, init +from slsdet import Detector, detectorSettings +from slsdet.defines import DEFAULT_TCP_RX_PORTNO, DEFAULT_UDP_DST_PORTNO +SERVER_START_PORTNO=1900 + init(autoreset=True) + class LogLevel(Enum): INFO = 0 INFORED = 1 @@ -19,12 +24,14 @@ class LogLevel(Enum): ERROR = 5 DEBUG = 6 + LOG_LABELS = { LogLevel.WARNING: "WARNING: ", LogLevel.ERROR: "ERROR: ", LogLevel.DEBUG: "DEBUG: " } + LOG_COLORS = { LogLevel.INFO: Fore.WHITE, LogLevel.INFORED: Fore.RED, @@ -35,7 +42,204 @@ LOG_COLORS = { LogLevel.DEBUG: Fore.CYAN } + def Log(level: LogLevel, message: str, stream=sys.stdout): color = LOG_COLORS.get(level, Fore.WHITE) label = LOG_LABELS.get(level, "") - print(f"{color}{label}{message}{Style.RESET_ALL}", file=stream, flush=True) \ No newline at end of file + print(f"{color}{label}{message}{Style.RESET_ALL}", file=stream, flush=True) + + +class RuntimeException (Exception): + def __init__ (self, message): + Log(LogLevel.ERROR, message) + super().__init__(message) + + +def checkIfProcessRunning(processName): + cmd = f"pgrep -f {processName}" + res = subprocess.getoutput(cmd) + return res.strip().splitlines() + + +def killProcess(name, fp): + pids = checkIfProcessRunning(name) + if pids: + Log(LogLevel.INFO, f"Killing '{name}' processes with PIDs: {', '.join(pids)}", fp) + for pid in pids: + try: + p = subprocess.run(['kill', pid]) + if p.returncode != 0 and bool(checkIfProcessRunning(name)): + raise RuntimeException(f"Could not kill {name} with pid {pid}") + except Exception as e: + raise RuntimeException(f"Failed to kill process {name} pid:{pid}. Exception occured: [code:{e}, msg:{e.stderr}]") + #else: + # Log(LogLevel.INFO, 'process not running : ' + name) + + +def cleanSharedmemory(fp): + Log(LogLevel.INFO, 'Cleaning up shared memory', fp) + try: + p = subprocess.run(['sls_detector_get', 'free'], stdout=fp, stderr=fp) + except: + raise RuntimeException('Could not free shared memory') + + +def cleanup(fp): + Log(LogLevel.INFO, 'Cleaning up') + Log(LogLevel.INFO, 'Cleaning up', fp) + killProcess('DetectorServer_virtual', fp) + killProcess('slsReceiver', fp) + killProcess('slsMultiReceiver', fp) + killProcess('slsFrameSynchronizer', fp) + killProcess('frameSynchronizerPullSocket', fp) + cleanSharedmemory(fp) + + +def startProcessInBackground(cmd, fp): + Log(LogLevel.INFO, 'Starting up ' + ' '.join(cmd)) + Log(LogLevel.INFO, 'Starting up ' + ' '.join(cmd), fp) + try: + p = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, restore_signals=False) + except Exception as e: + raise RuntimeException(f'Failed to start {cmd}:{e}') + + +def startProcessInBackgroundWithLogFile(cmd, fp, log_file_name): + Log(LogLevel.INFOBLUE, 'Starting up ' + ' '.join(cmd) + '. Log: ' + log_file_name) + Log(LogLevel.INFOBLUE, 'Starting up ' + ' '.join(cmd) + '. Log: ' + log_file_name, fp) + try: + with open(log_file_name, 'w') as log_fp: + subprocess.Popen(cmd, stdout=log_fp, stderr=log_fp, text=True) + except Exception as e: + raise RuntimeException(f'Failed to start {cmd}:{e}') + + +def runProcessWithLogFile(name, cmd, fp, log_file_name): + Log(LogLevel.INFOBLUE, 'Running ' + name + '. Log: ' + log_file_name) + Log(LogLevel.INFOBLUE, 'Running ' + name + '. Log: ' + log_file_name, fp) + Log(LogLevel.INFOBLUE, 'Cmd: ' + ' '.join(cmd), fp) + try: + with open(log_file_name, 'w') as log_fp: + subprocess.run(cmd, stdout=log_fp, stderr=log_fp, check=True, text=True) + except subprocess.CalledProcessError as e: + pass + except Exception as e: + Log(LogLevel.ERROR, f'Failed to run {name}:{e}', fp) + raise RuntimeException(f'Failed to run {name}:{e}') + + with open (log_file_name, 'r') as f: + for line in f: + if "FAILED" in line: + raise RuntimeException(f'{line}') + + Log(LogLevel.INFOGREEN, name + ' successful!\n') + Log(LogLevel.INFOGREEN, name + ' successful!\n', fp) + + +def startDetectorVirtualServer(name :str, num_mods, fp): + for i in range(num_mods): + port_no = SERVER_START_PORTNO + (i * 2) + cmd = [name + 'DetectorServer_virtual', '-p', str(port_no)] + startProcessInBackground(cmd, fp) + match name: + case 'jungfrau': + time.sleep(7) + case 'gotthard2': + time.sleep(5) + case _: + time.sleep(3) + + +def connectToVirtualServers(name, num_mods): + try: + d = Detector() + except Exception as e: + raise RuntimeException(f'Could not create Detector object for {name}. Error: {str(e)}') + + counts_sec = 5 + while (counts_sec != 0): + try: + d.virtual = [num_mods, SERVER_START_PORTNO] + break + except Exception as e: + # stop server still not up, wait a bit longer + if "Cannot connect to" in str(e): + Log(LogLevel.WARNING, f'Still waiting for {name} virtual server to be up...{counts_sec}s left') + time.sleep(1) + counts_sec -= 1 + else: + raise + + return d + + +def loadConfig(name, rx_hostname, settingsdir, fp, num_mods = 1, num_frames = 1): + Log(LogLevel.INFO, 'Loading config') + Log(LogLevel.INFO, 'Loading config', fp) + try: + d = connectToVirtualServers(name, num_mods) + d.udp_dstport = DEFAULT_UDP_DST_PORTNO + if name == 'eiger': + d.udp_dstport2 = DEFAULT_UDP_DST_PORTNO + 1 + + d.rx_hostname = rx_hostname + d.udp_dstip = 'auto' + if name != "eiger": + if name == "gotthard": + d.udp_srcip = d.udp_dstip + else: + d.udp_srcip = 'auto' + + if name == "jungfrau" or name == "moench" or name == "xilinx_ctb": + d.powerchip = 1 + + if name == "xilinx_ctb": + d.configureTransceiver() + + if name == "eiger": + d.trimen = [4500, 5400, 6400] + d.settingspath = settingsdir + '/eiger/' + d.setThresholdEnergy(4500, detectorSettings.STANDARD) + + d.frames = num_frames + except Exception as e: + raise RuntimeException(f'Could not load config for {name}. Error: {str(e)}') + + +def ParseArguments(description, default_num_mods=1): + parser = argparse.ArgumentParser(description) + + parser.add_argument('rx_hostname', nargs='?', default='localhost', + help='Hostname/IP of the current machine') + parser.add_argument('settingspath', nargs='?', default='../../settingsdir', + help='Relative or absolute path to the settings directory') + parser.add_argument('-n', '--num-mods', nargs='?', default=default_num_mods, type=int, + help='Number of modules to test with') + parser.add_argument('-f', '--num-frames', nargs='?', default=1, type=int, + help='Number of frames to test with') + parser.add_argument('-s', '--servers', nargs='*', + help='Detector servers to run') + + args = parser.parse_args() + + # Set default server list if not provided + if args.servers is None: + args.servers = [ + 'eiger', + 'jungfrau', + 'mythen3', + 'gotthard2', + 'gotthard' + 'ctb', + 'moench', + 'xilinx_ctb' + ] + + Log(LogLevel.INFO, 'Arguments:\n' + + 'rx_hostname: ' + args.rx_hostname + + '\nsettingspath: \'' + args.settingspath + + '\nservers: \'' + ' '.join(args.servers) + + '\nnum_mods: \'' + str(args.num_mods) + + '\nnum_frames: \'' + str(args.num_frames) + '\'') + + return args From f5bf5095250e51251df8618a039a1f332426a615 Mon Sep 17 00:00:00 2001 From: Dhanya Thattil Date: Wed, 7 May 2025 13:58:21 +0200 Subject: [PATCH 11/16] typo --- tests/scripts/utils_for_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/utils_for_test.py b/tests/scripts/utils_for_test.py index 2969d5599..540fcdcd9 100644 --- a/tests/scripts/utils_for_test.py +++ b/tests/scripts/utils_for_test.py @@ -229,7 +229,7 @@ def ParseArguments(description, default_num_mods=1): 'jungfrau', 'mythen3', 'gotthard2', - 'gotthard' + 'gotthard', 'ctb', 'moench', 'xilinx_ctb' From 5f6dea2d6d3017050226da637e9bd64b3dbc994e Mon Sep 17 00:00:00 2001 From: Dhanya Thattil Date: Wed, 7 May 2025 14:06:30 +0200 Subject: [PATCH 12/16] fix for gotthard to work --- tests/scripts/test_frame_synchronizer.py | 1 + tests/scripts/utils_for_test.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/tests/scripts/test_frame_synchronizer.py b/tests/scripts/test_frame_synchronizer.py index b8468070c..dc3c68281 100644 --- a/tests/scripts/test_frame_synchronizer.py +++ b/tests/scripts/test_frame_synchronizer.py @@ -115,6 +115,7 @@ def startTestsForAll(args, fp): acquire() testFramesCaught(server, args.num_frames) testZmqHeadetTypeCount(server, args.num_mods, args.num_frames, fp) + Log(LogLevel.INFO, '\n') except Exception as e: raise RuntimeException(f'Synchronizer Tests failed') diff --git a/tests/scripts/utils_for_test.py b/tests/scripts/utils_for_test.py index 540fcdcd9..138ed4076 100644 --- a/tests/scripts/utils_for_test.py +++ b/tests/scripts/utils_for_test.py @@ -140,6 +140,8 @@ def startDetectorVirtualServer(name :str, num_mods, fp): for i in range(num_mods): port_no = SERVER_START_PORTNO + (i * 2) cmd = [name + 'DetectorServer_virtual', '-p', str(port_no)] + if name == 'gotthard': + cmd += ['-m', '1'] startProcessInBackground(cmd, fp) match name: case 'jungfrau': From a85ad6cd848a8d20d1c071391dd31cfafe724a1f Mon Sep 17 00:00:00 2001 From: Dhanya Thattil Date: Wed, 7 May 2025 15:58:28 +0200 Subject: [PATCH 13/16] minor --- tests/scripts/test_frame_synchronizer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/scripts/test_frame_synchronizer.py b/tests/scripts/test_frame_synchronizer.py index dc3c68281..384a33b1e 100644 --- a/tests/scripts/test_frame_synchronizer.py +++ b/tests/scripts/test_frame_synchronizer.py @@ -44,7 +44,7 @@ def startFrameSynchronizer(num_mods, fp): time.sleep(1) -def acquire(): +def acquire(fp): Log(LogLevel.INFO, 'Acquiring') Log(LogLevel.INFO, 'Acquiring', fp) d = Detector() @@ -112,7 +112,7 @@ def startTestsForAll(args, fp): startFrameSynchronizerPullSocket(server, fp) startFrameSynchronizer(args.num_mods, fp) loadConfig(name=server, rx_hostname=args.rx_hostname, settingsdir=args.settingspath, fp=fp, num_mods=args.num_mods, num_frames=args.num_frames) - acquire() + acquire(fp) testFramesCaught(server, args.num_frames) testZmqHeadetTypeCount(server, args.num_mods, args.num_frames, fp) Log(LogLevel.INFO, '\n') From c0ae090be0b50c983f3213b2b0b8ff7c6d9c6b4a Mon Sep 17 00:00:00 2001 From: Dhanya Thattil Date: Wed, 7 May 2025 15:56:41 +0200 Subject: [PATCH 14/16] better error messageS --- tests/scripts/test_frame_synchronizer.py | 4 ++-- tests/scripts/test_simulators.py | 2 +- tests/scripts/utils_for_test.py | 14 +++++++------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/scripts/test_frame_synchronizer.py b/tests/scripts/test_frame_synchronizer.py index 384a33b1e..87c784cf7 100644 --- a/tests/scripts/test_frame_synchronizer.py +++ b/tests/scripts/test_frame_synchronizer.py @@ -96,7 +96,7 @@ def testZmqHeadetTypeCount(name, num_mods, num_frames, fp): msg = f"Expected {expected_count} '{htype}' entries, found {htype_counts[htype]}" raise RuntimeException(msg) except Exception as e: - raise RuntimeException(f'Failed to get zmq header count type. Error:{e}') + raise RuntimeException(f'Failed to get zmq header count type. Error:{str(e)}') from e Log(LogLevel.INFOGREEN, f"Zmq Header type count test passed for {name}") Log(LogLevel.INFOGREEN, f"Zmq Header type count test passed for {name}", fp) @@ -117,7 +117,7 @@ def startTestsForAll(args, fp): testZmqHeadetTypeCount(server, args.num_mods, args.num_frames, fp) Log(LogLevel.INFO, '\n') except Exception as e: - raise RuntimeException(f'Synchronizer Tests failed') + raise RuntimeException(f'Synchronizer Tests failed') from e Log(LogLevel.INFOGREEN, 'Passed all synchronizer tests for all detectors \n' + str(args.servers)) diff --git a/tests/scripts/test_simulators.py b/tests/scripts/test_simulators.py index fc0a54a34..d6f4371d9 100644 --- a/tests/scripts/test_simulators.py +++ b/tests/scripts/test_simulators.py @@ -65,7 +65,7 @@ def startCmdTestsForAll(args, fp): loadConfig(name=server, rx_hostname=args.rx_hostname, settingsdir=args.settingspath, fp=fp, num_mods=num_mods) runProcessWithLogFile('Cmd Tests for ' + server, cmd, fp, fname) except Exception as e: - raise RuntimeException(f'Cmd Tests failed for {server}.') + raise RuntimeException(f'Cmd Tests failed for {server}.') from e Log(LogLevel.INFOGREEN, 'Passed all tests for all detectors \n' + str(args.servers)) diff --git a/tests/scripts/utils_for_test.py b/tests/scripts/utils_for_test.py index 138ed4076..5f209899a 100644 --- a/tests/scripts/utils_for_test.py +++ b/tests/scripts/utils_for_test.py @@ -71,7 +71,7 @@ def killProcess(name, fp): if p.returncode != 0 and bool(checkIfProcessRunning(name)): raise RuntimeException(f"Could not kill {name} with pid {pid}") except Exception as e: - raise RuntimeException(f"Failed to kill process {name} pid:{pid}. Exception occured: [code:{e}, msg:{e.stderr}]") + raise RuntimeException(f"Failed to kill process {name} pid:{pid}. Error: {str(e)}]") from e #else: # Log(LogLevel.INFO, 'process not running : ' + name) @@ -101,7 +101,7 @@ def startProcessInBackground(cmd, fp): try: p = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, restore_signals=False) except Exception as e: - raise RuntimeException(f'Failed to start {cmd}:{e}') + raise RuntimeException(f'Failed to start {cmd}:{str(e)}') from e def startProcessInBackgroundWithLogFile(cmd, fp, log_file_name): @@ -111,7 +111,7 @@ def startProcessInBackgroundWithLogFile(cmd, fp, log_file_name): with open(log_file_name, 'w') as log_fp: subprocess.Popen(cmd, stdout=log_fp, stderr=log_fp, text=True) except Exception as e: - raise RuntimeException(f'Failed to start {cmd}:{e}') + raise RuntimeException(f'Failed to start {cmd}:{str(e)}') from e def runProcessWithLogFile(name, cmd, fp, log_file_name): @@ -124,8 +124,8 @@ def runProcessWithLogFile(name, cmd, fp, log_file_name): except subprocess.CalledProcessError as e: pass except Exception as e: - Log(LogLevel.ERROR, f'Failed to run {name}:{e}', fp) - raise RuntimeException(f'Failed to run {name}:{e}') + Log(LogLevel.ERROR, f'Failed to run {name}:{str(e)}', fp) + raise RuntimeException(f'Failed to run {name}:{str(e)}') with open (log_file_name, 'r') as f: for line in f: @@ -156,7 +156,7 @@ def connectToVirtualServers(name, num_mods): try: d = Detector() except Exception as e: - raise RuntimeException(f'Could not create Detector object for {name}. Error: {str(e)}') + raise RuntimeException(f'Could not create Detector object for {name}. Error: {str(e)}') from e counts_sec = 5 while (counts_sec != 0): @@ -205,7 +205,7 @@ def loadConfig(name, rx_hostname, settingsdir, fp, num_mods = 1, num_frames = 1) d.frames = num_frames except Exception as e: - raise RuntimeException(f'Could not load config for {name}. Error: {str(e)}') + raise RuntimeException(f'Could not load config for {name}. Error: {str(e)}') from e def ParseArguments(description, default_num_mods=1): From 674865120b3d167532ce23e54cd41a26cf8763d1 Mon Sep 17 00:00:00 2001 From: Dhanya Thattil Date: Wed, 7 May 2025 16:03:43 +0200 Subject: [PATCH 15/16] typo --- tests/scripts/utils_for_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/utils_for_test.py b/tests/scripts/utils_for_test.py index 5f209899a..389bfad4a 100644 --- a/tests/scripts/utils_for_test.py +++ b/tests/scripts/utils_for_test.py @@ -71,7 +71,7 @@ def killProcess(name, fp): if p.returncode != 0 and bool(checkIfProcessRunning(name)): raise RuntimeException(f"Could not kill {name} with pid {pid}") except Exception as e: - raise RuntimeException(f"Failed to kill process {name} pid:{pid}. Error: {str(e)}]") from e + raise RuntimeException(f"Failed to kill process {name} pid:{pid}. Error: {str(e)}") from e #else: # Log(LogLevel.INFO, 'process not running : ' + name) From 2c4cd807480f7facf001e07b2a6027f381fefea3 Mon Sep 17 00:00:00 2001 From: Dhanya Thattil Date: Wed, 7 May 2025 16:33:50 +0200 Subject: [PATCH 16/16] moving the erasure of the fnum to after sending the zmg packets and also deleteing all old frames when end of acquisition --- .../src/FrameSynchronizerApp.cpp | 31 +++++++++++++++---- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/slsReceiverSoftware/src/FrameSynchronizerApp.cpp b/slsReceiverSoftware/src/FrameSynchronizerApp.cpp index f9658e5ee..2dc84c30b 100644 --- a/slsReceiverSoftware/src/FrameSynchronizerApp.cpp +++ b/slsReceiverSoftware/src/FrameSynchronizerApp.cpp @@ -227,20 +227,24 @@ void Correlate(FrameStatus *stat) { msg_list.insert(msg_list.end(), stat->frames[port][fnum].begin(), stat->frames[port][fnum].end()); - stat->frames[port].erase(fnum); } } LOG(printHeadersLevel) << "Sending data packets for fnum " << fnum; zmq_send_multipart(socket, msg_list); + // clean up for (const auto &it : stat->frames) { const uint16_t port = it.first; - // clean up - for (zmq_msg_t *msg : stat->frames[port][fnum]) { - if (msg) { - zmq_msg_close(msg); - delete msg; + const FrameMap &frame_map = it.second; + auto frame = frame_map.find(fnum); + if (frame != frame_map.end()) { + for (zmq_msg_t *msg : frame->second) { + if (msg) { + zmq_msg_close(msg); + delete msg; + } } + stat->frames[port].erase(fnum); } } } @@ -256,6 +260,21 @@ void Correlate(FrameStatus *stat) { } } stat->ends.clear(); + // clean up old frames + for (auto &it : stat->frames) { + FrameMap &frame_map = it.second; + for (auto &frame : frame_map) { + for (zmq_msg_t *msg : frame.second) { + if (msg) { + zmq_msg_close(msg); + delete msg; + } + } + frame.second.clear(); + } + frame_map.clear(); + } + stat->frames.clear(); } } }