IndexerThreadPool collapsed two different outcomes into the same empty reply. A worker that threw set result = nullptr, and so did a dispatch that never found a free worker; both then became a default-constructed IndexerResult, indistinguishable downstream from an indexer that ran and found no lattice. The first says nothing about the frame at all - the indexer never looked, and it will fail again on the next one - while the second is a real negative result about the crystal. The visible cost was the advice a failed run gave. With the card full, rugnux printed "Indexer thread 0 failed: CUDA (GPU) error" and then ended with "Two-pass rotation indexing found no lattice. Check the beam centre (--beam-x / --beam-y), raise --max-spots ..." - sending the operator to look at geometry that was never wrong, for a machine that was simply out of memory. None of those remedies can help when the frames were not examined. IndexerResult gains an optional error, set by the worker and by the pool's own catch; the remaining nullptr path keeps its meaning of "not attempted" and deliberately carries no error. RotationIndexer records it and exposes GetIndexerError(), and rugnux's first pass branches on it at the throw site, naming the resource failure instead. The error travels as DATA through image_analysis/ rather than as an exception, because RotationIndexer::RunIndexing() is on the online path - IndexAndRefine calls it on a schedule from the broker and the receiver, where dropping a frame is the right failure and killing a live acquisition is not. Only rugnux, which owns the "this run is over" decision, turns it into one. The failure result is byte-identical to the default-constructed one it replaces, so any_executed and the online frame-drop behaviour are unchanged. The new IndexerError category exists because the category is only the display prefix on what() - Category() is read nowhere - and the old line read "Processing failed: Input parameter invalid" for a GPU fault, which is the same defect one layer up. SpotFinderError is the precedent. msg.indexing_result is deliberately left alone: of its three states, "not attempted" is the honest one for a frame the indexer never examined, and asserting false would be the same collapse again. Verified on a squeezed card (246 MB free, -N 4): exit 1, zero dropped frames, the new message, and no mention of --beam-x. On a free card, against the previous binary at -N 6 on the same input, the logs differ only in paths and timing - same space group, cell and merge statistics. Targeted Catch2 cases pass, including three that drive the pool through the online path. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Lc5JG6kJqZoCWaoZ43JGTW
193 lines
7.2 KiB
C++
193 lines
7.2 KiB
C++
// SPDX-FileCopyrightText: 2024 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
|
// SPDX-License-Identifier: GPL-3.0-only
|
|
|
|
#pragma once
|
|
|
|
#include <cstring>
|
|
#include <string>
|
|
#include <exception>
|
|
|
|
enum class JFJochExceptionCategory {
|
|
JSON,
|
|
InputParameterAboveMax,
|
|
InputParameterBelowMin,
|
|
InputParameterInvalid,
|
|
WrongNumber,
|
|
WrongUnit,
|
|
ArrayOutOfBounds,
|
|
HDF5,
|
|
IBVerbs,
|
|
RESTCommandUnknown,
|
|
WrongDAQState,
|
|
WrongReceiverState,
|
|
Detector,
|
|
MemAllocFailed,
|
|
AcquisitionDeviceError,
|
|
GPUCUDAError,
|
|
GainFileOpenError,
|
|
MockFileOpenError,
|
|
Compression,
|
|
SharedMemory,
|
|
ZeroMQ,
|
|
ServiceUndefined,
|
|
TriggerError,
|
|
FileWriteError,
|
|
ZSTDCompressionError,
|
|
LogstashConnectionFailure,
|
|
TIFFGeneratorError,
|
|
gRPCError,
|
|
PreviewError,
|
|
HardwareParityError,
|
|
SpotFinderError,
|
|
PCIeError,
|
|
CBORError,
|
|
UDPError,
|
|
CommunicationError,
|
|
SimplonError,
|
|
CalibrationError,
|
|
IndexerError
|
|
};
|
|
|
|
class JFJochException : public std::exception {
|
|
protected:
|
|
std::string msg;
|
|
JFJochExceptionCategory category;
|
|
static std::string DecodeCategory(JFJochExceptionCategory category) {
|
|
switch (category) {
|
|
case JFJochExceptionCategory::JSON:
|
|
return "JSON parsing or analysis error";
|
|
case JFJochExceptionCategory::InputParameterAboveMax:
|
|
return "Input parameter above max";
|
|
case JFJochExceptionCategory::InputParameterBelowMin:
|
|
return "Input parameter below min";
|
|
case JFJochExceptionCategory::InputParameterInvalid:
|
|
return "Input parameter invalid";
|
|
case JFJochExceptionCategory::WrongNumber:
|
|
return "Wrong number";
|
|
case JFJochExceptionCategory::WrongUnit:
|
|
return "Wrong unit";
|
|
case JFJochExceptionCategory::ArrayOutOfBounds:
|
|
return "Array out of bounds";
|
|
case JFJochExceptionCategory::HDF5:
|
|
return "HDF5 error";
|
|
case JFJochExceptionCategory::IBVerbs:
|
|
return "Infiniband error";
|
|
case JFJochExceptionCategory::RESTCommandUnknown:
|
|
return "REST command unknown";
|
|
case JFJochExceptionCategory::WrongDAQState:
|
|
return "DAQ state error";
|
|
case JFJochExceptionCategory::Detector:
|
|
return "Detector error";
|
|
case JFJochExceptionCategory::MemAllocFailed:
|
|
return "Memory error";
|
|
case JFJochExceptionCategory::AcquisitionDeviceError:
|
|
return "Acquisition device error";
|
|
case JFJochExceptionCategory::GPUCUDAError:
|
|
return "CUDA (GPU) error";
|
|
case JFJochExceptionCategory::GainFileOpenError:
|
|
return "Gain file open error";
|
|
case JFJochExceptionCategory::MockFileOpenError:
|
|
return "Mock file open error";
|
|
case JFJochExceptionCategory::Compression:
|
|
return "Compression error";
|
|
case JFJochExceptionCategory::SharedMemory:
|
|
return "Shared memory error";
|
|
case JFJochExceptionCategory::ZeroMQ:
|
|
return "ZeroMQ error";
|
|
case JFJochExceptionCategory::ServiceUndefined:
|
|
return "Service undefined error";
|
|
case JFJochExceptionCategory::TriggerError:
|
|
return "Trigger error";
|
|
case JFJochExceptionCategory::FileWriteError:
|
|
return "File writer error";
|
|
case JFJochExceptionCategory::ZSTDCompressionError:
|
|
return "Zstd compressor error";
|
|
case JFJochExceptionCategory::LogstashConnectionFailure:
|
|
return "Logstash connection error";
|
|
case JFJochExceptionCategory::TIFFGeneratorError:
|
|
return "TIFF writer error";
|
|
case JFJochExceptionCategory::gRPCError:
|
|
return "Remote node (gRPC) error";
|
|
case JFJochExceptionCategory::PreviewError:
|
|
return "Preview error";
|
|
case JFJochExceptionCategory::HardwareParityError:
|
|
return "Parity error (HW)";
|
|
case JFJochExceptionCategory::SpotFinderError:
|
|
return "Spot finder";
|
|
case JFJochExceptionCategory::PCIeError:
|
|
return "PCIe driver error";
|
|
case JFJochExceptionCategory::CBORError:
|
|
return "CBOR serialization error";
|
|
case JFJochExceptionCategory::UDPError:
|
|
return "UDP simulator error";
|
|
case JFJochExceptionCategory::CommunicationError:
|
|
return "Communication error";
|
|
case JFJochExceptionCategory::SimplonError:
|
|
return "Simplon API error";
|
|
case JFJochExceptionCategory::CalibrationError:
|
|
return "Detector calibration error";
|
|
case JFJochExceptionCategory::IndexerError:
|
|
return "Indexer error";
|
|
default:
|
|
return "";
|
|
}
|
|
}
|
|
public:
|
|
JFJochException(JFJochExceptionCategory in_val, const std::string &description) noexcept :
|
|
category(in_val) {
|
|
try {
|
|
msg = DecodeCategory(category) + " (" + description + ")";
|
|
} catch (...) {}
|
|
}
|
|
|
|
JFJochException(const JFJochException &e) noexcept {
|
|
try {
|
|
category = e.category;
|
|
msg = e.msg;
|
|
} catch (...) {}
|
|
}
|
|
|
|
[[nodiscard]] const char *what() const noexcept override {
|
|
return msg.c_str();
|
|
}
|
|
|
|
[[nodiscard]] JFJochExceptionCategory Category() const noexcept {
|
|
return category;
|
|
}
|
|
};
|
|
|
|
class WrongDAQStateException : public JFJochException {
|
|
public:
|
|
explicit WrongDAQStateException(const std::string &description)
|
|
: JFJochException(JFJochExceptionCategory::WrongDAQState, description) {}
|
|
};
|
|
|
|
// Thrown when an operation leaves a subsystem (typically the detector) in an undefined state that a
|
|
// simple retry cannot recover from - the broker must re-initialise. The state machine catches this
|
|
// separately and goes to the Error state, whereas ordinary acquisition failures return to Idle.
|
|
class JFJochCriticalException : public JFJochException {
|
|
public:
|
|
explicit JFJochCriticalException(const std::string &description,
|
|
JFJochExceptionCategory in_category = JFJochExceptionCategory::Detector)
|
|
: JFJochException(in_category, description) {}
|
|
};
|
|
|
|
// A PCIe/FPGA hardware fault leaves the acquisition device in an undefined state that a retry cannot
|
|
// recover from, so it is critical: the state machine forces re-initialisation (Error state) rather
|
|
// than returning to Idle as it does for ordinary acquisition failures.
|
|
class PCIeDeviceException : public JFJochCriticalException {
|
|
public:
|
|
explicit PCIeDeviceException(const std::string &description)
|
|
: JFJochCriticalException(description, JFJochExceptionCategory::PCIeError) {
|
|
msg += " (" + std::string(strerror(errno)) + ")";
|
|
}
|
|
};
|
|
|
|
// Thrown by JFJochBitShuffleCompressor::Compress when the compressed output does not fit in the
|
|
// caller-provided destination buffer. Lets callers (e.g. the receiver) drop just that frame.
|
|
class CompressionBufferTooSmallException : public JFJochException {
|
|
public:
|
|
explicit CompressionBufferTooSmallException(const std::string &description)
|
|
: JFJochException(JFJochExceptionCategory::Compression, description) {}
|
|
};
|