leonarski_fandClaude Opus 5.5
509a54ef92
So that rugnux can process a collection while it is written, the broker reports each data file once
a writer has closed it and renamed it into place:
- Writer: FileWriter takes a callback, called for every closed data file. StreamWriter uses it on TCP
to send an acknowledgement with ack_for = FILE_CLOSED (new TCPFrameType 10): file number and the
images the file holds. TCP protocol version 5. Files closed at END are reported before the END
acknowledgement.
- LiveCollection (image_pusher): the start message the writer of the master file received, the
closed files, the end. Fed by the TCP pusher (FILE_CLOSED acknowledgements; end once every writer
acknowledged END) and by the in-process HDF5 pusher (the FileWriter callback). The ZeroMQ pusher
has no back channel and offers none.
- Two routes, protected by the dataset's bearer tokens like the other dataset routes:
- GET /live/start.cbor: that start message;
- GET /live/events: server-sent events (collection, file, end, superseded) with the history first
and a keepalive every 10 s. One follower at a time; a second gets 409.
Documented in jfjoch_api.yaml; they add no schema, so the generated C++ model is unchanged.
- The HTTP server's thread pool is set explicitly (16, up to 64): an event stream holds a thread for
the whole collection.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SVmAWnzCmRKAXVUCdc4iNi
72 lines
2.7 KiB
C++
72 lines
2.7 KiB
C++
// SPDX-FileCopyrightText: 2025 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
|
// SPDX-License-Identifier: GPL-3.0-only
|
|
|
|
#pragma once
|
|
|
|
#include <cstdint>
|
|
|
|
constexpr uint32_t JFJOCH_TCP_MAGIC = 0x4A464A54; // JFJT
|
|
constexpr uint32_t JFJOCH_TCP_VERSION = 5;
|
|
|
|
// Upper bound on a single frame's payload. A receiver rejects any header claiming more
|
|
// before allocating for it: an uncapped payload_size would otherwise drive a huge
|
|
// resize() and take down the receiving thread. 1 GiB is far above any real image frame.
|
|
constexpr uint64_t JFJOCH_TCP_MAX_PAYLOAD_SIZE = static_cast<uint64_t>(1) << 30;
|
|
|
|
enum class TCPFrameType : uint16_t {
|
|
START = 1,
|
|
DATA = 2,
|
|
CALIBRATION = 3,
|
|
END = 4,
|
|
ACK = 5,
|
|
CANCEL = 6,
|
|
KEEPALIVE = 7,
|
|
// Writer -> pusher liveness/backpressure heartbeat. Tells the pusher the writer
|
|
// is alive but busy (e.g. stalled on a slow filesystem) and that it should keep
|
|
// waiting for a proper ACK rather than declaring the connection dead. Carries the
|
|
// writer's current FIFO occupancy in the ack_fifo_* fields.
|
|
BUSY = 8,
|
|
// Pusher -> writer dry run, sent before the detector is armed. Carries the same CBOR
|
|
// StartMessage a START would, but the writer only checks that it could write the output
|
|
// (path, directory, no file in the way) and answers with an ACK; it opens nothing and
|
|
// leaves its state untouched. A rejected PREFLIGHT is not fatal - the run was never
|
|
// started, so the connection stays usable for the next attempt.
|
|
PREFLIGHT = 9,
|
|
// Writer -> pusher, sent as an ACK with ack_for = FILE_CLOSED: a data file has been closed and
|
|
// renamed into place. image_number is the file number (from 0), ack_processed_images the images
|
|
// the file holds. Lets the broker tell a reader following the collection which files it can read.
|
|
FILE_CLOSED = 10,
|
|
};
|
|
|
|
enum class TCPAckCode : uint16_t {
|
|
None = 0,
|
|
StartFailed = 1,
|
|
DataWriteFailed = 2,
|
|
EndFailed = 3,
|
|
DiskQuotaExceeded = 4,
|
|
NoSpaceLeft = 5,
|
|
PermissionDenied = 6,
|
|
IoError = 7,
|
|
ProtocolError = 8
|
|
};
|
|
|
|
constexpr uint32_t TCP_ACK_FLAG_OK = 1u << 0;
|
|
constexpr uint32_t TCP_ACK_FLAG_FATAL = 1u << 1;
|
|
constexpr uint32_t TCP_ACK_FLAG_HAS_ERROR_TEXT = 1u << 2;
|
|
|
|
struct alignas(64) TcpFrameHeader {
|
|
uint32_t magic = JFJOCH_TCP_MAGIC;
|
|
uint16_t version = JFJOCH_TCP_VERSION ;
|
|
uint16_t type = 0;
|
|
uint64_t image_number = 0;
|
|
uint64_t payload_size = 0;
|
|
uint32_t socket_number = 0;
|
|
uint32_t flags = 0;
|
|
uint64_t run_number = 0;
|
|
|
|
uint32_t ack_processed_images = 0;
|
|
uint16_t ack_code = 0;
|
|
uint16_t ack_for = 0;
|
|
uint64_t ack_fifo_occupancy = 0;
|
|
uint64_t ack_fifo_max_occupancy = 0;
|
|
}; |