Follow-ups to the Idle-vs-Error acquisition failure split: - JFJochServices::Stop: raise the critical detector fault BEFORE the ordinary receiver/writer exception. Both can be set at once (detector not idle, then receiver->Stop() also throws); throwing the ordinary one first masked the detector fault and returned the broker to Idle instead of Error, skipping the required re-initialisation. - SLSDetectorWrapper::Stop/Deactivate: rethrow a JFJochException as-is (as Start already does) so InternalStop's JFJochCriticalException is not downgraded to an ordinary JFJochException by the generic catch. - JFJochServices::Start: wrap the whole best-effort receiver cleanup (Cancel + Stop) in the catch so a throwing Cancel cannot replace the original detector exception before it is re-raised. Also fix the "cannot create indexing pool" failure seen when retrying initialize after a failed acquisition start: a failed start leaves the receiver (and its GPU resources) alive until the next Start, but the retried initialize rebuilds the indexer pool first, so the fresh GPU indexer had to coexist with the stale receiver and its init failed. JFJochReceiverService::Indexing now releases the previous receiver before (re)building the pool (safe: only runs when the receiver is idle), which also removes a dangling pool pointer held by that receiver. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
485 lines
20 KiB
C++
485 lines
20 KiB
C++
// SPDX-FileCopyrightText: 2024 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
|
// SPDX-License-Identifier: GPL-3.0-only
|
|
|
|
#include <thread>
|
|
|
|
#include "../common/JFJochException.h"
|
|
#include "../common/Definitions.h"
|
|
#include "SLSDetectorWrapper.h"
|
|
|
|
namespace {
|
|
// Value reported for temperature / high voltage of an inactive (excluded)
|
|
// half-module, so that the position in the logical vector is preserved.
|
|
constexpr int64_t kInactiveModulePlaceholder = -1;
|
|
}
|
|
|
|
SLSDetectorWrapper::SLSDetectorWrapper() {
|
|
logger.Info("SLS detector package {} client {}", det.getPackageVersion(), det.getClientVersion());
|
|
}
|
|
|
|
std::vector<int64_t> SLSDetectorWrapper::MapToLogical(const std::vector<int64_t>& sls_values,
|
|
int64_t placeholder) const {
|
|
std::vector<int64_t> out(n_logical_units, placeholder);
|
|
for (size_t s = 0; (s < sls_values.size()) && (s < sls_to_logical.size()); s++)
|
|
out[sls_to_logical[s]] = sls_values[s];
|
|
return out;
|
|
}
|
|
|
|
void SLSDetectorWrapper::Initialize(DiffractionExperiment& experiment,
|
|
const std::vector<AcquisitionDeviceNetConfig>& net_config) {
|
|
if (experiment.GetDetectorType() == DetectorType::DECTRIS)
|
|
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
|
|
"DectrisDetectorWrapper needs PSI detector");
|
|
logger.Info("Initialize detector {}", experiment.GetDetectorSetup().GetDescription());
|
|
|
|
det_type = experiment.GetDetectorSetup().GetDetectorType();
|
|
try {
|
|
// Per-(half-)module hostnames in logical order.
|
|
// EIGER: one entry per half-module, logical index = 2 * module + half
|
|
// JUNGFRAU: one entry per module
|
|
// An empty entry marks an inactive half-module: it is excluded from the
|
|
// slsDetectorPackage (never connected, never configured) but keeps its
|
|
// slot in the logical numbering.
|
|
auto module_hostname = experiment.GetDetectorModuleHostname();
|
|
|
|
n_logical_units = static_cast<int>(module_hostname.size());
|
|
|
|
std::vector<std::string> active_hostname;
|
|
sls_to_logical.clear();
|
|
for (int k = 0; k < n_logical_units; k++) {
|
|
if (!module_hostname[k].empty()) {
|
|
active_hostname.push_back(module_hostname[k]);
|
|
sls_to_logical.push_back(k);
|
|
} else {
|
|
logger.Info("Logical half-module {} is inactive (empty hostname) - excluded from detector", k);
|
|
}
|
|
}
|
|
|
|
if (active_hostname.empty())
|
|
throw JFJochException(JFJochExceptionCategory::Detector,
|
|
"No active module hostname provided");
|
|
|
|
det.setHostname(active_hostname);
|
|
|
|
// Reverse map: logical index -> SLS index (-1 if inactive).
|
|
std::vector<int> logical_to_sls(n_logical_units, -1);
|
|
for (size_t s = 0; s < sls_to_logical.size(); s++)
|
|
logical_to_sls[sls_to_logical[s]] = static_cast<int>(s);
|
|
|
|
auto mod_cfg = experiment.GetDetectorModuleConfig(net_config);
|
|
|
|
if (det_type == DetectorType::JUNGFRAU) {
|
|
if (n_logical_units != experiment.GetModulesNum())
|
|
throw JFJochException(JFJochExceptionCategory::Detector,
|
|
"Hostname vector size must equal module count for JUNGFRAU");
|
|
if (sls_to_logical.size() != static_cast<size_t>(n_logical_units))
|
|
throw JFJochException(JFJochExceptionCategory::Detector,
|
|
"Inactive modules are not supported for JUNGFRAU");
|
|
if (det.size() != experiment.GetModulesNum()) {
|
|
logger.Error("Discrepancy in module number between DAQ and detector");
|
|
throw JFJochException(JFJochExceptionCategory::Detector,
|
|
"Discrepancy in module number between DAQ and detector");
|
|
}
|
|
|
|
// always try to stop new detector first
|
|
det.stopDetector();
|
|
|
|
det.setNumberofUDPInterfaces(experiment.GetUDPInterfaceCount());
|
|
for (int i = 0; i < mod_cfg.size(); i++) {
|
|
logger.Info("Configure network for module {}", i);
|
|
|
|
auto &cfg = mod_cfg[i];
|
|
|
|
det.setSourceUDPIP(sls::IpAddr(cfg.ipv4_src_addr_1), {i});
|
|
det.setSourceUDPMAC(sls::MacAddr(BASE_DETECTOR_MAC + i * 2), {i});
|
|
|
|
det.setDestinationUDPPort(16384 + (cfg.data_stream<<8) + cfg.module_id_in_data_stream * 2, i);
|
|
det.setDestinationUDPIP(sls::IpAddr(cfg.ipv4_dest_addr_1), {i});
|
|
det.setDestinationUDPMAC(sls::MacAddr(cfg.mac_addr_dest_1), {i});
|
|
|
|
if (experiment.GetUDPInterfaceCount() == 2) {
|
|
det.setSourceUDPIP2(sls::IpAddr(cfg.ipv4_src_addr_2), {i});
|
|
det.setSourceUDPMAC2(sls::MacAddr(BASE_DETECTOR_MAC + i * 2 + 1), {i});
|
|
det.setDestinationUDPPort2(16384 + (cfg.data_stream<<8) + cfg.module_id_in_data_stream * 2 + 1, i);
|
|
det.setDestinationUDPIP2(sls::IpAddr(cfg.ipv4_dest_addr_2), {i});
|
|
det.setDestinationUDPMAC2(sls::MacAddr(cfg.mac_addr_dest_2), {i});
|
|
}
|
|
uint32_t tmp = (cfg.module_id_in_data_stream * 2) % UINT16_MAX;
|
|
uint32_t column_id_register = ((tmp + 1) << 16) | tmp;
|
|
|
|
det.writeRegister(0x7C, column_id_register, {i});
|
|
}
|
|
det.setTemperatureControl(true);
|
|
det.setThresholdTemperature(experiment.GetDetectorSetup().GetTempThreshold_degC(), {0});;
|
|
|
|
auto tx_delay = experiment.GetDetectorSetup().GetTxDelay();
|
|
if (tx_delay.size() == experiment.GetModulesNum()) {
|
|
for (int i = 0 ; i < tx_delay.size(); i++)
|
|
det.setTransmissionDelayFrame(tx_delay[i], {i});
|
|
}
|
|
|
|
if (experiment.GetUDPInterfaceCount() == 2)
|
|
det.setReadoutSpeed(slsDetectorDefs::speedLevel::FULL_SPEED);
|
|
else
|
|
det.setReadoutSpeed(slsDetectorDefs::speedLevel::HALF_SPEED);
|
|
|
|
det.setAutoComparatorDisable(true);
|
|
if (!det.getPowerChip().squash(false)) {
|
|
det.setPowerChip(true);
|
|
std::this_thread::sleep_for(std::chrono::seconds(5));
|
|
}
|
|
|
|
} else if (det_type == DetectorType::EIGER) {
|
|
if (n_logical_units != 2 * experiment.GetModulesNum())
|
|
throw JFJochException(JFJochExceptionCategory::Detector,
|
|
"Hostname vector size must equal 2 x module count for EIGER");
|
|
if (det.size() != sls_to_logical.size()) {
|
|
logger.Error("Discrepancy in active module number between DAQ ({}) and detector ({})",
|
|
sls_to_logical.size(), det.size());
|
|
throw JFJochException(JFJochExceptionCategory::Detector,
|
|
"Discrepancy in module number between DAQ and detector");
|
|
}
|
|
det.setInterruptSubframe(true);
|
|
det.setTenGiga(true);
|
|
|
|
auto trim_directory = experiment.GetDetectorSetup().GetTrimFileDirectory();
|
|
if (!trim_directory.empty()) {
|
|
// Hardcoded for now - need to make it nicer
|
|
std::vector<int> trim_en = experiment.GetDetectorSetup().GetTrimEnergies_eV();
|
|
if (trim_en.empty())
|
|
throw JFJochException(JFJochExceptionCategory::Detector, "Trimming energies not provided");
|
|
det.setTrimEnergies(trim_en);
|
|
det.setSettingsPath(trim_directory);
|
|
}
|
|
auto trim_files = experiment.GetDetectorSetup().GetTrimFileNames();
|
|
|
|
for (int i = 0; i < mod_cfg.size(); i++) {
|
|
auto &cfg = mod_cfg[i];
|
|
|
|
for (int h = 0; h < 2; h++) {
|
|
int logical = 2 * i + h;
|
|
int s = logical_to_sls[logical];
|
|
if (s < 0) {
|
|
logger.Info("Skip inactive half-module {} (no slsDetectorPackage configuration)", logical);
|
|
continue;
|
|
}
|
|
|
|
logger.Info("Configure network for half-module {} (SLS index {})", logical, s);
|
|
|
|
// The UDP destination port carries the LOGICAL module id and
|
|
// half, so the surviving modules land at the correct place in
|
|
// the assembled image regardless of the SLS index shift.
|
|
uint16_t port = 16384 + (cfg.data_stream << 8) + cfg.module_id_in_data_stream * 2 + h;
|
|
det.setDestinationUDPPort(port, s);
|
|
det.setDestinationUDPPort2(port, s);
|
|
|
|
det.setSourceUDPIP(sls::IpAddr(cfg.ipv4_src_addr_1), {s});
|
|
det.setDestinationUDPIP(sls::IpAddr(cfg.ipv4_dest_addr_1), {s});
|
|
det.setDestinationUDPMAC(sls::MacAddr(cfg.mac_addr_dest_1), {s});
|
|
//det.setRow(static_cast<uint32_t>(cfg.module_id_in_data_stream * 2 + h), {s});
|
|
|
|
if (!trim_files.empty() && (logical < static_cast<int>(trim_files.size())))
|
|
det.loadTrimbits(trim_files[logical], {s});
|
|
}
|
|
}
|
|
}
|
|
det.setHighVoltage(experiment.GetDetectorSetup().GetHighVoltage());
|
|
} catch (const std::exception &e) {
|
|
logger.ErrorException(e);
|
|
throw JFJochException(JFJochExceptionCategory::Detector, e.what());
|
|
}
|
|
logger.Info(" ... done");
|
|
}
|
|
|
|
void SLSDetectorWrapper::Start(const DiffractionExperiment& experiment) {
|
|
logger.Info("Start");
|
|
|
|
if (det.size() != sls_to_logical.size())
|
|
throw JFJochException(JFJochExceptionCategory::Detector,
|
|
"Discrepancy in module number between DAQ and detector");
|
|
|
|
if (det_type == DetectorType::EIGER) {
|
|
auto energy_threshold_ev = experiment.GetEigerThreshold_keV() * 1000.0f;
|
|
if (det.getThresholdEnergy().squash(0) != energy_threshold_ev)
|
|
det.setThresholdEnergy(energy_threshold_ev);
|
|
|
|
// For EIGER exposure time needs to be updated for every measurements
|
|
det.setDynamicRange(experiment.GetEigerBitDepth());
|
|
det.setPeriod(experiment.GetDetectorPeriod());
|
|
det.setExptime(experiment.GetFrameCountTime());
|
|
}
|
|
|
|
try {
|
|
InternalStop();
|
|
|
|
det.setNextFrameNumber(1);
|
|
det.setNumberOfFrames(experiment.GetFrameNumPerTrigger() / experiment.GetStorageCellNumber());
|
|
det.setNumberOfTriggers(experiment.GetNumTriggers());
|
|
|
|
det.startDetector();
|
|
} catch (const JFJochException &) {
|
|
throw; // already a categorised JFJoch exception (incl. critical detector error) - keep its type
|
|
} catch (std::exception &e) {
|
|
logger.ErrorException(e);
|
|
throw JFJochException(JFJochExceptionCategory::Detector, e.what());
|
|
}
|
|
|
|
logger.Info(" ... done");
|
|
}
|
|
|
|
void SLSDetectorWrapper::InternalStop() {
|
|
// Assume it is executed in try-catch!
|
|
auto state = GetState();
|
|
if (state == DetectorState::ERROR)
|
|
throw JFJochCriticalException("Detector in error state");
|
|
else if ((state == DetectorState::BUSY) || (state == DetectorState::WAITING)) {
|
|
try {
|
|
det.stopDetector();
|
|
} catch (...) {
|
|
// Sometimes stop gives problem - ignore these
|
|
logger.Warning("Problem with stopping the detector - ignored.");
|
|
}
|
|
std::this_thread::sleep_for(std::chrono::milliseconds(10));
|
|
state = GetState();
|
|
if (state != DetectorState::IDLE)
|
|
throw JFJochException(JFJochExceptionCategory::Detector,
|
|
"Detector busy and cannot be stopped");
|
|
}
|
|
}
|
|
|
|
void SLSDetectorWrapper::Deactivate() {
|
|
logger.Info("Deactivate");
|
|
try {
|
|
InternalStop();
|
|
if (det_type == DetectorType::JUNGFRAU) {
|
|
det.setHighVoltage(0);
|
|
std::this_thread::sleep_for(std::chrono::seconds(5));
|
|
|
|
det.setPowerChip(false);
|
|
det.setMaster(false, 0);
|
|
det.setSynchronization(false);
|
|
}
|
|
} catch (const JFJochException &) {
|
|
throw; // already categorised (incl. critical detector error) - keep its type
|
|
} catch (std::exception &e) {
|
|
logger.ErrorException(e);
|
|
throw JFJochException(JFJochExceptionCategory::Detector, e.what());
|
|
}
|
|
logger.Info(" ... done");
|
|
}
|
|
|
|
void SLSDetectorWrapper::Stop() {
|
|
logger.Info("Stop");
|
|
try {
|
|
InternalStop();
|
|
} catch (const JFJochException &) {
|
|
throw; // already categorised (incl. critical detector error) - keep its type
|
|
} catch (std::exception &e) {
|
|
logger.ErrorException(e);
|
|
throw JFJochException(JFJochExceptionCategory::Detector, e.what());
|
|
}
|
|
logger.Info(" ... done");
|
|
}
|
|
|
|
void SLSDetectorWrapper::Trigger() {
|
|
logger.Info("Trigger");
|
|
try {
|
|
det.sendSoftwareTrigger();
|
|
} catch (std::exception &e) {
|
|
logger.ErrorException(e);
|
|
throw JFJochException(JFJochExceptionCategory::Detector, e.what());
|
|
}
|
|
logger.Info(" ... done");
|
|
}
|
|
|
|
DetectorState SLSDetectorWrapper::GetState() const {
|
|
if (det.empty())
|
|
return DetectorState::NOT_CONNECTED;
|
|
try {
|
|
bool is_idle = true;
|
|
bool is_waiting = true;
|
|
for (const auto & i : det.getDetectorStatus()) {
|
|
if (i == slsDetectorDefs::runStatus::ERROR)
|
|
return DetectorState::ERROR;
|
|
if ((i != slsDetectorDefs::runStatus::IDLE) &&
|
|
(i != slsDetectorDefs::runStatus::STOPPED) &&
|
|
(i != slsDetectorDefs::runStatus::RUN_FINISHED))
|
|
is_idle = false;
|
|
if (i != slsDetectorDefs::WAITING)
|
|
is_waiting = false;
|
|
}
|
|
if (is_idle)
|
|
return DetectorState::IDLE;
|
|
else if (is_waiting)
|
|
return DetectorState::WAITING;
|
|
else
|
|
return DetectorState::BUSY;
|
|
} catch (std::exception &e) {
|
|
throw JFJochException(JFJochExceptionCategory::Detector, e.what());
|
|
}
|
|
}
|
|
|
|
int64_t SLSDetectorWrapper::GetNumberOfTriggersLeft() const {
|
|
int64_t ret = 0;
|
|
for (const auto & i : det.getNumberOfTriggersLeft()) {
|
|
if (i > ret)
|
|
ret = i;
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
DetectorPowerState SLSDetectorWrapper::GetPowerState() const {
|
|
uint64_t count_on = 0;
|
|
for (const auto &i : det.getPowerChip()) {
|
|
if (i) count_on++;
|
|
}
|
|
|
|
if (count_on == 0)
|
|
return DetectorPowerState::OFF;
|
|
else if (count_on == det.size())
|
|
return DetectorPowerState::ON;
|
|
else
|
|
return DetectorPowerState::PARTIAL;
|
|
}
|
|
|
|
int64_t SLSDetectorWrapper::GetFirmwareVersion() const {
|
|
try {
|
|
auto result = det.getFirmwareVersion();
|
|
return result.squash(0x0);
|
|
} catch (std::exception &e) {
|
|
throw JFJochException(JFJochExceptionCategory::Detector, e.what());
|
|
}
|
|
}
|
|
|
|
std::string SLSDetectorWrapper::GetDetectorServerVersion() const {
|
|
try {
|
|
auto result = det.getDetectorServerVersion();
|
|
return result.squash("mixed");
|
|
} catch (std::exception &e) {
|
|
throw JFJochException(JFJochExceptionCategory::Detector, e.what());
|
|
}
|
|
}
|
|
|
|
std::vector<int64_t> SLSDetectorWrapper::GetFPGATemperatures() const {
|
|
try {
|
|
auto result = det.getTemperature(slsDetectorDefs::TEMPERATURE_FPGA);
|
|
std::vector<int64_t> sls_values;
|
|
sls_values.reserve(result.size());
|
|
for (int i = 0; i < result.size(); i++)
|
|
sls_values.push_back(result[i]);
|
|
// Report at the logical (Jungfraujoch) module position; inactive
|
|
// half-modules are filled with a placeholder.
|
|
return MapToLogical(sls_values, kInactiveModulePlaceholder);
|
|
} catch (std::exception &e) {
|
|
throw JFJochException(JFJochExceptionCategory::Detector, e.what());
|
|
}
|
|
}
|
|
|
|
std::vector<int64_t> SLSDetectorWrapper::GetHighVoltage() const {
|
|
try {
|
|
auto result = det.getHighVoltage();
|
|
std::vector<int64_t> sls_values;
|
|
for (int i : result)
|
|
sls_values.push_back(i);
|
|
return MapToLogical(sls_values, kInactiveModulePlaceholder);
|
|
} catch (std::exception &e) {
|
|
throw JFJochException(JFJochExceptionCategory::Detector, e.what());
|
|
}
|
|
}
|
|
|
|
DetectorStatus SLSDetectorWrapper::GetStatus() const {
|
|
DetectorStatus status{};
|
|
|
|
if (det.empty())
|
|
return DetectorStatus{
|
|
.detector_state = DetectorState::NOT_CONNECTED,
|
|
.power_state = DetectorPowerState::OFF,
|
|
.detector_server_version = "N/A"
|
|
};
|
|
|
|
status.detector_server_version = GetDetectorServerVersion();
|
|
status.detector_state = GetState();
|
|
if (det_type == DetectorType::JUNGFRAU) {
|
|
status.power_state = GetPowerState();
|
|
status.remaining_triggers = GetNumberOfTriggersLeft();
|
|
} else {
|
|
status.power_state = DetectorPowerState::ON;
|
|
status.remaining_triggers = -1;
|
|
}
|
|
|
|
status.temperature_fpga_degC = GetFPGATemperatures();
|
|
status.high_voltage_V = GetHighVoltage();
|
|
return status;
|
|
}
|
|
|
|
void SLSDetectorWrapper::Configure(const DiffractionExperiment &experiment) {
|
|
if (det_type == DetectorType::JUNGFRAU) {
|
|
if (experiment.IsFixedGainG1()) {
|
|
if ((experiment.GetDetectorMode() == DetectorMode::PedestalG0) ||
|
|
(experiment.GetDetectorMode() == DetectorMode::PedestalG2))
|
|
throw JFJochException(JFJochExceptionCategory::Detector,
|
|
"Pedestal G0/G2 doesn't make sense for fixed G1 mode");
|
|
det.setGainMode(slsDetectorDefs::FIX_G1);
|
|
} else {
|
|
switch (experiment.GetDetectorMode()) {
|
|
case DetectorMode::PedestalG1:
|
|
det.setGainMode(slsDetectorDefs::gainMode::FORCE_SWITCH_G1);
|
|
break;
|
|
case DetectorMode::PedestalG2:
|
|
det.setGainMode(slsDetectorDefs::gainMode::FORCE_SWITCH_G2);
|
|
break;
|
|
default:
|
|
det.setGainMode(slsDetectorDefs::gainMode::DYNAMIC);
|
|
break;
|
|
}
|
|
}
|
|
|
|
det.setStorageCellStart(experiment.GetStorageCellStart());
|
|
det.setNumberOfAdditionalStorageCells(experiment.GetStorageCellNumber() - 1);
|
|
det.setStorageCellDelay(experiment.GetStorageCellDelay() - MIN_STORAGE_CELL_DELAY);
|
|
|
|
if (experiment.IsUsingGainHG0())
|
|
det.setSettings(slsDetectorDefs::HIGHGAIN0);
|
|
else
|
|
det.setSettings(slsDetectorDefs::GAIN0);
|
|
|
|
if (experiment.IsDetectorModuleSync()) {
|
|
det.setMaster(true, 0);
|
|
det.setSynchronization(true);
|
|
}
|
|
det.setDelayAfterTrigger(experiment.GetDetectorDelay());
|
|
} else if (det_type == DetectorType::EIGER) {
|
|
auto energy_threshold = experiment.GetDetectorSettings().GetEigerThreshold_keV();
|
|
if (energy_threshold.has_value())
|
|
det.setThresholdEnergy(std::lround(energy_threshold.value() * 1000.0));
|
|
}
|
|
|
|
switch (experiment.GetDetectorTiming()) {
|
|
case DetectorTiming::Auto:
|
|
det.setTimingMode(slsDetectorDefs::timingMode::AUTO_TIMING);
|
|
break;
|
|
case DetectorTiming::Trigger:
|
|
det.setTimingMode(slsDetectorDefs::timingMode::TRIGGER_EXPOSURE);
|
|
break;
|
|
case DetectorTiming::Burst:
|
|
if (det_type == DetectorType::JUNGFRAU)
|
|
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
|
|
"Burst timing mode not supported with JUNGFRAU");
|
|
det.setTimingMode(slsDetectorDefs::timingMode::BURST_TRIGGER);
|
|
break;
|
|
case DetectorTiming::Gated:
|
|
if (det_type == DetectorType::JUNGFRAU)
|
|
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
|
|
"Gated timing mode not supported with JUNGFRAU");
|
|
det.setTimingMode(slsDetectorDefs::timingMode::GATED);
|
|
break;
|
|
}
|
|
|
|
det.setPeriod(experiment.GetDetectorPeriod());
|
|
det.setExptime(experiment.GetFrameCountTime());
|
|
}
|
|
|
|
void SLSDetectorWrapper::LoadPixelMask(PixelMask &mask) {
|
|
// Do nothing
|
|
}
|