Build Packages / build:rpm (ubuntu2404_nocuda) (push) Successful in 8m30s
Build Packages / build:rpm (rocky8_nocuda) (push) Successful in 10m13s
Build Packages / build:rpm (ubuntu2204_nocuda) (push) Successful in 9m45s
Build Packages / build:rpm (rocky9_nocuda) (push) Successful in 11m13s
Build Packages / build:rpm (rocky8_sls9) (push) Successful in 9m51s
Build Packages / build:rpm (rocky8) (push) Successful in 8m29s
Build Packages / build:rpm (rocky9_sls9) (push) Successful in 9m31s
Build Packages / build:rpm (rocky9) (push) Successful in 9m42s
Build Packages / build:rpm (ubuntu2204) (push) Successful in 8m47s
Build Packages / build:rpm (ubuntu2404) (push) Successful in 8m23s
Build Packages / Generate python client (push) Successful in 19s
Build Packages / Build documentation (push) Successful in 38s
Build Packages / Create release (push) Skipped
Build Packages / XDS test (durin plugin) (push) Successful in 6m18s
Build Packages / XDS test (neggia plugin) (push) Successful in 6m4s
Build Packages / XDS test (JFJoch plugin) (push) Successful in 6m35s
Build Packages / DIALS test (push) Successful in 10m39s
Build Packages / Unit tests (push) Successful in 1h24m58s
NUMA CPU/memory pinning is no longer worthwhile: the FPGA DMA buffers are placed device-local by the kernel (dma_alloc_coherent), the big RAM ring buffer is random-access (first-touch handles placement), and GPU work is already spread across all visible devices. So drop the pinning entirely and with it libnuma. - Delete NUMAHWPolicy; the only concern worth keeping - GPU selection - is done directly via pin_gpu() (round-robin over visible GPUs) in the indexer pool and the Lite analysis threads. CPU-only threads (FPGA acquire/pedestal/summation/frame-transform) no longer bind anything. - Drop get_gpu_numa_node() (sysfs lookup) - only SelectGPUAndItsNUMA used it. - numa_policy broker setting is deprecated and ignored (kept in the API for backward compatibility; warns once on startup). - Remove NUMA_LIBRARY / numa.h / numaif.h detection from CMake. - Docs: drop the NUMA dependency, remove the numa_policy config example, and document running multiple brokers on disjoint GPUs via CUDA_VISIBLE_DEVICES. - Remove NUMA_GPU_REVIEW.md (the planning note; this work is now done). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
142 lines
4.2 KiB
C++
142 lines
4.2 KiB
C++
// SPDX-FileCopyrightText: 2024 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
|
// SPDX-License-Identifier: GPL-3.0-only
|
|
// Using OpenAPI licensed with Apache License 2.0
|
|
|
|
#include <csignal>
|
|
#include <fstream>
|
|
#include <iostream>
|
|
#include <vector>
|
|
|
|
#include <nlohmann/json.hpp>
|
|
|
|
#include "../common/Logger.h"
|
|
#include "../common/print_license.h"
|
|
#include "../detector_control/DectrisSimplonClient.h"
|
|
#include "../writer/HDF5Objects.h"
|
|
|
|
#include "JFJochBrokerHttp.h"
|
|
#include "JFJochBrokerParser.h"
|
|
|
|
static httplib::Server *httpServer = nullptr;
|
|
|
|
static void sigHandler [[noreturn]] (int sig) {
|
|
switch (sig) {
|
|
case SIGINT:
|
|
case SIGQUIT:
|
|
case SIGTERM:
|
|
case SIGHUP:
|
|
default:
|
|
if (httpServer)
|
|
httpServer->stop();
|
|
break;
|
|
}
|
|
std::exit(0);
|
|
}
|
|
|
|
static void setUpUnixSignals(std::vector<int> quitSignals) {
|
|
sigset_t blocking_mask;
|
|
sigemptyset(&blocking_mask);
|
|
for (auto sig : quitSignals)
|
|
sigaddset(&blocking_mask, sig);
|
|
|
|
struct sigaction sa;
|
|
sa.sa_handler = sigHandler;
|
|
sa.sa_mask = blocking_mask;
|
|
sa.sa_flags = 0;
|
|
|
|
for (auto sig : quitSignals)
|
|
sigaction(sig, &sa, nullptr);
|
|
}
|
|
|
|
int main(int argc, char **argv) {
|
|
RegisterHDF5Filter();
|
|
print_license("jfjoch_broker");
|
|
|
|
if ((argc == 1) || (argc > 3)) {
|
|
std::cout << "Usage ./jfjoch_broker <JSON config> {<TCP http port>}" << std::endl;
|
|
return EXIT_FAILURE;
|
|
}
|
|
|
|
uint16_t http_port = 5232;
|
|
if (argc >= 3) http_port = static_cast<uint16_t>(std::atoi(argv[2]));
|
|
|
|
Logger logger("jfjoch_broker");
|
|
|
|
org::openapitools::server::model::Jfjoch_settings settings;
|
|
std::ifstream file(argv[1]);
|
|
try {
|
|
nlohmann::json input = nlohmann::json::parse(file);
|
|
settings = input;
|
|
settings.validate();
|
|
logger.Info("JSON configuration file read properly");
|
|
} catch (const std::exception &e) {
|
|
logger.Error("Error reading JSON configuration file: " + std::string(e.what()));
|
|
return EXIT_FAILURE;
|
|
}
|
|
|
|
if (settings.numaPolicyIsSet())
|
|
logger.Warning("NUMA policy is deprecated and ignored - processed are split over all the CPUs/GPUs in the system");
|
|
if (settings.verboseIsSet() && settings.isVerbose())
|
|
logger.Verbose(true);
|
|
else
|
|
logger.Verbose(false);
|
|
|
|
std::unique_ptr<ImagePusher> image_pusher = ParseImagePusher(settings);
|
|
if (image_pusher)
|
|
logger.Info(image_pusher->PrintSetup());
|
|
|
|
std::vector<DetectorSetup> det_setup;
|
|
for (const auto &d: settings.getDetector())
|
|
det_setup.push_back(ParseDetectorSetup(d));
|
|
|
|
if (det_setup.empty()) {
|
|
logger.Error("At least one detector need to be defined in the config file");
|
|
return EXIT_FAILURE;
|
|
}
|
|
|
|
DiffractionExperiment experiment(det_setup[0]);
|
|
ParseFacilityConfiguration(settings, experiment);
|
|
|
|
SpotFindingSettings spot_finding_settings = ParseSpotFindingSettings(settings);
|
|
|
|
AcquisitionDeviceGroup aq_devices;
|
|
ParseAcquisitionDeviceGroup(settings, aq_devices);
|
|
experiment.DataStreams(aq_devices.size());
|
|
|
|
int32_t send_buffer_size_MiB = settings.getImageBufferMiB();
|
|
std::unique_ptr<JFJochReceiverService> receiver =
|
|
std::make_unique<JFJochReceiverService>(aq_devices, logger, *image_pusher, send_buffer_size_MiB);
|
|
ParseReceiverSettings(settings, *receiver);
|
|
|
|
JFJochBrokerHttp broker(experiment, spot_finding_settings);
|
|
broker.FrontendDirectory(settings.getFrontendDirectory());
|
|
|
|
for (const auto &d: det_setup)
|
|
broker.AddDetectorSetup(d);
|
|
|
|
if (receiver)
|
|
broker.Services().Receiver(receiver.get());
|
|
|
|
httplib::Server server;
|
|
httpServer = &server;
|
|
|
|
server.Get("/", [](const httplib::Request &req, httplib::Response &res) {
|
|
res.status = 302; // Found (temporary redirect)
|
|
res.set_header("Location", "/frontend");
|
|
});
|
|
server.set_mount_point("/frontend", settings.getFrontendDirectory());
|
|
|
|
signal(SIGPIPE, SIG_IGN);
|
|
std::vector<int> sigs{SIGQUIT, SIGINT, SIGTERM, SIGHUP};
|
|
setUpUnixSignals(sigs);
|
|
|
|
broker.attach(server);
|
|
|
|
if (!server.listen("0.0.0.0", http_port)) {
|
|
logger.Error("Failed to start HTTP server");
|
|
return EXIT_FAILURE;
|
|
}
|
|
|
|
return EXIT_SUCCESS;
|
|
}
|