From 4d7d3c9138b9a2a4520e06b44e96a39dc244733d Mon Sep 17 00:00:00 2001 From: Dhanya Thattil Date: Fri, 25 Apr 2025 09:09:49 +0200 Subject: [PATCH 1/3] upgrading to c++17 from c++11 and patch command has to be found before applying patch on libzmq (#1195) --- CMakeLists.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b1d035d76..6b3127da1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -44,6 +44,10 @@ endif() # Patch is applied in the FetchContent_Declare set(SLS_LIBZMQ_VERSION "4.3.4") +find_program(PATCH_EXECUTABLE patch) +if(NOT PATCH_EXECUTABLE) + message(FATAL_ERROR "The 'patch' tool is required for patching lib zeromq. Please install it.") +endif() if(SLS_FETCH_ZMQ_FROM_GITHUB) # Opt in to pull down a zmq version from github instead of @@ -216,7 +220,7 @@ endif() # to control options for the libraries if(NOT TARGET slsProjectOptions) add_library(slsProjectOptions INTERFACE) - target_compile_features(slsProjectOptions INTERFACE cxx_std_11) + target_compile_features(slsProjectOptions INTERFACE cxx_std_17) endif() if (NOT TARGET slsProjectWarnings) From 5fa2402ec53ad6937517e5576132ea1466f8a40d Mon Sep 17 00:00:00 2001 From: Dhanya Thattil Date: Fri, 25 Apr 2025 10:14:15 +0200 Subject: [PATCH 2/3] Dev/allow localhost for virtual tests (#1190) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * remove the check for localhost being used in rx_hostname for python test for simulators, run rx_arping test only if hostname is not 'localhost' * fix tests for fpath: cannot set back to empty anymore (empty is default) * default rx_hostname arg = localhost, and default settings path =../../settingsdir * changed virtual tests script for better printout on exceptions * fix for catching generaltests exceptions and exiting instead of continuing * fix minor * fixed shared memeory tests to include current env and fixed prints for errors --------- Co-authored-by: Erik Fröjdh --- .../tests/Caller/test-Caller-rx.cpp | 39 ++--- .../tests/test-SharedMemory.cpp | 25 +++- tests/scripts/test_simulators.py | 138 +++++++++--------- 3 files changed, 114 insertions(+), 88 deletions(-) diff --git a/slsDetectorSoftware/tests/Caller/test-Caller-rx.cpp b/slsDetectorSoftware/tests/Caller/test-Caller-rx.cpp index 508abc59d..fcf0de97d 100644 --- a/slsDetectorSoftware/tests/Caller/test-Caller-rx.cpp +++ b/slsDetectorSoftware/tests/Caller/test-Caller-rx.cpp @@ -445,23 +445,25 @@ TEST_CASE("rx_arping", "[.cmdcall][.rx]") { Detector det; Caller caller(&det); auto prev_val = det.getRxArping(); - { - std::ostringstream oss; - caller.call("rx_arping", {"1"}, -1, PUT, oss); - REQUIRE(oss.str() == "rx_arping 1\n"); - } - { - std::ostringstream oss; - caller.call("rx_arping", {}, -1, GET, oss); - REQUIRE(oss.str() == "rx_arping 1\n"); - } - { - std::ostringstream oss; - caller.call("rx_arping", {"0"}, -1, PUT, oss); - REQUIRE(oss.str() == "rx_arping 0\n"); - } - for (int i = 0; i != det.size(); ++i) { - det.setRxArping(prev_val[i], {i}); + if (det.getDestinationUDPIP()[0].str() != "127.0.0.1") { + { + std::ostringstream oss; + caller.call("rx_arping", {"1"}, -1, PUT, oss); + REQUIRE(oss.str() == "rx_arping 1\n"); + } + { + std::ostringstream oss; + caller.call("rx_arping", {}, -1, GET, oss); + REQUIRE(oss.str() == "rx_arping 1\n"); + } + { + std::ostringstream oss; + caller.call("rx_arping", {"0"}, -1, PUT, oss); + REQUIRE(oss.str() == "rx_arping 0\n"); + } + for (int i = 0; i != det.size(); ++i) { + det.setRxArping(prev_val[i], {i}); + } } } @@ -583,6 +585,9 @@ TEST_CASE("fpath", "[.cmdcall]") { REQUIRE(oss.str() == "fpath /tmp\n"); } for (int i = 0; i != det.size(); ++i) { + if (prev_val[i].empty()) { + continue; + } det.setFilePath(prev_val[i], {i}); } } diff --git a/slsDetectorSoftware/tests/test-SharedMemory.cpp b/slsDetectorSoftware/tests/test-SharedMemory.cpp index 5ba3c20a2..4dff6d5d7 100644 --- a/slsDetectorSoftware/tests/test-SharedMemory.cpp +++ b/slsDetectorSoftware/tests/test-SharedMemory.cpp @@ -18,11 +18,16 @@ struct Data { constexpr int shm_id = 10; TEST_CASE("Create SharedMemory read and write", "[detector]") { + const char *env_p = std::getenv("SLSDETNAME"); + std::string env_name = env_p ? ("_" + std::string(env_p)) : ""; SharedMemory shm(shm_id, -1); + if (shm.exists()) { + shm.removeSharedMemory(); + } shm.createSharedMemory(); CHECK(shm.getName() == std::string("/slsDetectorPackage_detector_") + - std::to_string(shm_id)); + std::to_string(shm_id) + env_name); shm()->x = 3; shm()->y = 5.7; @@ -90,10 +95,12 @@ TEST_CASE("Open two shared memories to the same place", "[detector]") { } TEST_CASE("Move SharedMemory", "[detector]") { + const char *env_p = std::getenv("SLSDETNAME"); + std::string env_name = env_p ? ("_" + std::string(env_p)) : ""; SharedMemory shm(shm_id, -1); CHECK(shm.getName() == std::string("/slsDetectorPackage_detector_") + - std::to_string(shm_id)); + std::to_string(shm_id) + env_name); shm.createSharedMemory(); shm()->x = 9; @@ -104,15 +111,19 @@ TEST_CASE("Move SharedMemory", "[detector]") { REQUIRE_THROWS( shm()); // trying to access should throw instead of returning a nullptr CHECK(shm2.getName() == std::string("/slsDetectorPackage_detector_") + - std::to_string(shm_id)); + std::to_string(shm_id) + env_name); shm2.removeSharedMemory(); } TEST_CASE("Create several shared memories", "[detector]") { + const char *env_p = std::getenv("SLSDETNAME"); + std::string env_name = env_p ? ("_" + std::string(env_p)) : ""; + constexpr int N = 5; std::vector> v; v.reserve(N); for (int i = 0; i != N; ++i) { + std::cout << "i:" << i << std::endl; v.emplace_back(shm_id + i, -1); CHECK(v[i].exists() == false); v[i].createSharedMemory(); @@ -123,7 +134,7 @@ TEST_CASE("Create several shared memories", "[detector]") { for (int i = 0; i != N; ++i) { CHECK(*v[i]() == i); CHECK(v[i].getName() == std::string("/slsDetectorPackage_detector_") + - std::to_string(i + shm_id)); + std::to_string(i + shm_id) + env_name); } for (int i = 0; i != N; ++i) { @@ -133,8 +144,12 @@ TEST_CASE("Create several shared memories", "[detector]") { } TEST_CASE("Create create a shared memory with a tag") { + const char *env_p = std::getenv("SLSDETNAME"); + std::string env_name = env_p ? ("_" + std::string(env_p)) : ""; + SharedMemory shm(0, -1, "ctbdacs"); - REQUIRE(shm.getName() == "/slsDetectorPackage_detector_0_ctbdacs"); + REQUIRE(shm.getName() == + "/slsDetectorPackage_detector_0" + env_name + "_ctbdacs"); } TEST_CASE("Create create a shared memory with a tag when SLSDETNAME is set") { diff --git a/tests/scripts/test_simulators.py b/tests/scripts/test_simulators.py index ce7fbd420..ea580c5e6 100644 --- a/tests/scripts/test_simulators.py +++ b/tests/scripts/test_simulators.py @@ -4,7 +4,7 @@ This file is used to start up simulators, receivers and run all the tests on them and finally kill the simulators and receivers. ''' import argparse -import os, sys, subprocess, time, colorama, signal +import os, sys, subprocess, time, colorama from colorama import Fore from slsdet import Detector, detectorType, detectorSettings @@ -23,23 +23,9 @@ def Log(color, message): def checkIfProcessRunning(processName): - cmd = "ps -ef | grep " + processName - print(cmd) - res=subprocess.getoutput(cmd) - print(res) - # eg. of output - #l_user 250506 243295 0 14:38 pts/5 00:00:00 /bin/sh -c ps -ef | grep slsReceiver - #l_user 250508 250506 0 14:38 pts/5 00:00:00 grep slsReceiver - - print('how many') - cmd = "ps -ef | grep " + processName + " | wc -l" - print(cmd) - res=subprocess.getoutput(cmd) - print(res) - - if res == '2': - return False - return True + cmd = f"pgrep -f {processName}" + res = subprocess.getoutput(cmd) + return bool(res.strip()) def killProcess(name): @@ -52,7 +38,7 @@ def killProcess(name): print('process not running : ' + name) -def killAllStaleProcesses(): +def killAllStaleProcesses(fp): killProcess('eigerDetectorServer_virtual') killProcess('jungfrauDetectorServer_virtual') killProcess('mythen3DetectorServer_virtual') @@ -62,9 +48,9 @@ def killAllStaleProcesses(): killProcess('xilinx_ctbDetectorServer_virtual') killProcess('slsReceiver') killProcess('slsMultiReceiver') - cleanSharedmemory() + cleanSharedmemory(fp) -def cleanup(name): +def cleanup(name, fp): ''' kill both servers, receivers and clean shared memory ''' @@ -72,9 +58,9 @@ def cleanup(name): killProcess(name + 'DetectorServer_virtual') killProcess('slsReceiver') killProcess('slsMultiReceiver') - cleanSharedmemory() + cleanSharedmemory(fp) -def cleanSharedmemory(): +def cleanSharedmemory(fp): Log(Fore.GREEN, 'Cleaning up shared memory...') try: p = subprocess.run(['sls_detector_get', 'free'], stdout=fp, stderr=fp) @@ -87,8 +73,8 @@ def startProcessInBackground(name): # in background and dont print output p = subprocess.Popen(name.split(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, restore_signals=False) Log(Fore.GREEN, 'Starting up ' + name + ' ...') - except: - Log(Fore.RED, 'Could not start ' + name) + except Exception as e: + Log(Fore.RED, f'Could not start {name}:{e}') raise def startServer(name): @@ -139,14 +125,19 @@ def loadConfig(name, rx_hostname, settingsdir): def startCmdTests(name, fp, fname): Log(Fore.GREEN, 'Cmd Tests for ' + name) cmd = 'tests --abort [.cmdcall] -s -o ' + fname - p = subprocess.run(cmd.split(), stdout=fp, stderr=fp, check=True, text=True) - p.check_returncode() + try: + subprocess.run(cmd.split(), stdout=fp, stderr=fp, check=True, text=True) + except subprocess.CalledProcessError as e: + pass with open (fname, 'r') as f: for line in f: if "FAILED" in line: msg = 'Cmd tests failed for ' + name + '!!!' + sys.stdout = original_stdout Log(Fore.RED, msg) + Log(Fore.RED, line) + sys.stdout = fp raise Exception(msg) Log(Fore.GREEN, 'Cmd Tests successful for ' + name) @@ -154,14 +145,18 @@ def startCmdTests(name, fp, fname): def startGeneralTests(fp, fname): Log(Fore.GREEN, 'General Tests') cmd = 'tests --abort -s -o ' + fname - p = subprocess.run(cmd.split(), stdout=fp, stderr=fp, check=True, text=True) - p.check_returncode() + try: + subprocess.run(cmd.split(), stdout=fp, stderr=fp, check=True, text=True) + except subprocess.CalledProcessError as e: + pass with open (fname, 'r') as f: for line in f: if "FAILED" in line: msg = 'General tests failed !!!' - Log(Fore.RED, msg) + sys.stdout = original_stdout + Log(Fore.RED, msg + '\n' + line) + sys.stdout = fp raise Exception(msg) Log(Fore.GREEN, 'General Tests successful') @@ -170,12 +165,10 @@ def startGeneralTests(fp, fname): # parse cmd line for rx_hostname and settingspath using the argparse library parser = argparse.ArgumentParser(description = 'automated tests with the virtual detector servers') -parser.add_argument('rx_hostname', help = 'hostname/ip of the current machine') -parser.add_argument('settingspath', help = 'Relative or absolut path to the settingspath') +parser.add_argument('rx_hostname', nargs='?', default='localhost', help = 'hostname/ip of the current machine') +parser.add_argument('settingspath', nargs='?', default='../../settingsdir', help = 'Relative or absolut path to the settingspath') parser.add_argument('-s', '--servers', help='Detector servers to run', nargs='*') args = parser.parse_args() -if args.rx_hostname == 'localhost': - raise RuntimeException('Cannot use localhost for rx_hostname for the tests (fails for rx_arping for eg.)') if args.servers is None: servers = [ @@ -203,46 +196,59 @@ Log(Fore.BLUE, '\nLog File: ' + fname) with open(fname, 'w') as fp: + + # general tests file_results = prefix_fname + '_results_general.txt' Log(Fore.BLUE, 'General tests (results: ' + file_results + ')') sys.stdout = fp sys.stderr = fp Log(Fore.BLUE, 'General tests (results: ' + file_results + ')') - startGeneralTests(fp, file_results) - killAllStaleProcesses() + try: + startGeneralTests(fp, file_results) + killAllStaleProcesses(fp) - for server in servers: - try: - # print to terminal for progress - sys.stdout = original_stdout - sys.stderr = original_stderr - file_results = prefix_fname + '_results_cmd_' + server + '.txt' - Log(Fore.BLUE, 'Cmd tests for ' + server + ' (results: ' + file_results + ')') - sys.stdout = fp - sys.stderr = fp - Log(Fore.BLUE, 'Cmd tests for ' + server + ' (results: ' + file_results + ')') - - # cmd tests for det - cleanup(server) - startServer(server) - startReceiver(server) - loadConfig(server, args.rx_hostname, args.settingspath) - startCmdTests(server, fp, file_results) - cleanup(server) - except: - Log(Fore.RED, 'Exception caught. Cleaning up.') - cleanup(server) - sys.stdout = original_stdout - sys.stderr = original_stderr - Log(Fore.RED, 'Cmd tests failed for ' + server + '!!!') - raise + testError = False + for server in servers: + try: + # print to terminal for progress + sys.stdout = original_stdout + sys.stderr = original_stderr + file_results = prefix_fname + '_results_cmd_' + server + '.txt' + Log(Fore.BLUE, 'Cmd tests for ' + server + ' (results: ' + file_results + ')') + sys.stdout = fp + sys.stderr = fp + Log(Fore.BLUE, 'Cmd tests for ' + server + ' (results: ' + file_results + ')') + + # cmd tests for det + cleanup(server, fp) + startServer(server) + startReceiver(server) + loadConfig(server, args.rx_hostname, args.settingspath) + startCmdTests(server, fp, file_results) + cleanup(server, fp) + + except Exception as e: + # redirect to terminal + sys.stdout = original_stdout + sys.stderr = original_stderr + Log(Fore.RED, f'Exception caught while testing {server}. Cleaning up...') + testError = True + break + + # redirect to terminal + sys.stdout = original_stdout + sys.stderr = original_stderr + if not testError: + Log(Fore.GREEN, 'Passed all tests for virtual detectors \n' + str(servers)) - Log(Fore.GREEN, 'Passed all tests for virtual detectors \n' + str(servers)) + except Exception as e: + # redirect to terminal + sys.stdout = original_stdout + sys.stderr = original_stderr + Log(Fore.RED, f'Exception caught with general testing. Cleaning up...') + cleanSharedmemory(sys.stdout) + -# redirect to terminal -sys.stdout = original_stdout -sys.stderr = original_stderr -Log(Fore.GREEN, 'Passed all tests for virtual detectors \n' + str(servers) + '\nYayyyy! :) ') \ No newline at end of file From 625f4353fbc3bcd4102022d3050e93767d3cca3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Fr=C3=B6jdh?= Date: Fri, 25 Apr 2025 12:04:45 +0200 Subject: [PATCH 3/3] Dev/gitea docker (#1194) * gitea workflows for RH8 and RH9 * using our docker images --- .gitea/workflows/rh8-native.yml | 29 +++++++++++++++++++++++++++++ .gitea/workflows/rh9-native.yml | 27 +++++++++++++++++++++++++++ CMakeLists.txt | 3 ++- 3 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 .gitea/workflows/rh8-native.yml create mode 100644 .gitea/workflows/rh9-native.yml diff --git a/.gitea/workflows/rh8-native.yml b/.gitea/workflows/rh8-native.yml new file mode 100644 index 000000000..51d754125 --- /dev/null +++ b/.gitea/workflows/rh8-native.yml @@ -0,0 +1,29 @@ +name: Build on RHEL8 + +on: + push: + workflow_dispatch: + +permissions: + contents: read + +jobs: + build: + runs-on: "ubuntu-latest" + container: + image: gitea.psi.ch/detectors/rhel8-detectors-dev + steps: + - name: Clone repository + run: | + echo Cloning ${{ github.ref_name }} + git clone https://${{secrets.GITHUB_TOKEN}}@gitea.psi.ch/${{ github.repository }}.git --branch=${{ github.ref_name }} . + + - name: Build library + run: | + mkdir build && cd build + cmake .. -DSLS_USE_PYTHON=ON -DSLS_USE_TESTS=ON + make -j 2 + + - name: C++ unit tests + working-directory: ${{gitea.workspace}}/build + run: ctest \ No newline at end of file diff --git a/.gitea/workflows/rh9-native.yml b/.gitea/workflows/rh9-native.yml new file mode 100644 index 000000000..890b09edf --- /dev/null +++ b/.gitea/workflows/rh9-native.yml @@ -0,0 +1,27 @@ +name: Build on RHEL9 + +on: + push: + workflow_dispatch: + +permissions: + contents: read + +jobs: + build: + runs-on: "ubuntu-latest" + container: + image: gitea.psi.ch/detectors/rhel9-detectors-dev + steps: + - uses: actions/checkout@v4 + + + - name: Build library + run: | + mkdir build && cd build + cmake .. -DSLS_USE_PYTHON=ON -DSLS_USE_TESTS=ON + make -j 2 + + - name: C++ unit tests + working-directory: ${{gitea.workspace}}/build + run: ctest \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 6b3127da1..212281f68 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -333,7 +333,8 @@ if (SLS_USE_INTEGRATION_TESTS) endif (SLS_USE_INTEGRATION_TESTS) if (SLS_USE_PYTHON) - find_package (Python 3.8 COMPONENTS Interpreter Development) + find_package (Python 3.8 COMPONENTS Interpreter Development.Module REQUIRED) + set(PYBIND11_FINDPYTHON ON) # Needed for RH8 if(SLS_FETCH_PYBIND11_FROM_GITHUB) FetchContent_Declare( pybind11