diff --git a/CLAUDE.md b/CLAUDE.md index cc308d96..48e59478 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -94,6 +94,13 @@ cd tests/test_data && python jfjoch_broker_test.py # feeds a test image, sta images over ZeroMQ → `jfjoch_writer` (`writer/`) consumes the stream and writes NXmx HDF5. The broker also emits a low-rate preview stream and a metadata stream (`preview/`). +**Writer file split:** one acquisition produces one `_master.h5` plus many `_data_NNNNNN.h5` +files. Dataset-wide metadata (geometry, detector config, ROI/azimuthal definitions — anything +fixed for the whole run) is written to the **master** file in `writer/HDF5NXmx.cpp` (the `NXmx` +class). Per-image arrays (one entry per frame) are written to the **data** files by the +`HDF5DataFilePlugin` subclasses in `writer/`. Put shared metadata in `NXmx`, not in a data-file +plugin. + **`jfjoch_broker`** (`broker/`) is the central online service: HTTP/REST + OpenAPI control plane, FPGA configuration, image building, ZeroMQ output. `JFJochStateMachine` drives acquisition state; `JFJochServices` wires the pieces; `OpenAPIConvert`/`JFJochBrokerParser` translate between the diff --git a/common/JFJochMessages.h b/common/JFJochMessages.h index eee16f6e..11a29806 100644 --- a/common/JFJochMessages.h +++ b/common/JFJochMessages.h @@ -270,6 +270,9 @@ struct StartMessage { std::map> pixel_mask; + // Per-pixel ROI bitmask (converted geometry), bit i set for ROI i in rois. + std::vector roi_map; + std::map threshold_energy; std::optional total_flux; diff --git a/docs/HDF5.md b/docs/HDF5.md index 49e72cbd..0b6d38cf 100644 --- a/docs/HDF5.md +++ b/docs/HDF5.md @@ -319,9 +319,10 @@ In the master file these per-image groups are exposed through `/entry/reflection | `image_count` | `[n_images, φ_bins, q_bins]` | | pixels contributing per bin | | `map` | `[y, x]` | | pixel→bin mapping (master file only) | -### 4.4 `/entry/roi/` — regions of interest +### 4.4 `/entry/roi` — regions of interest -One sub-group per configured ROI, each with `[n_images]` vectors: +`/entry/roi/` has one sub-group per configured ROI. The **per-image result +vectors** `[n_images]` live in the data files: | Dataset | Meaning | |---------|---------| @@ -331,6 +332,21 @@ One sub-group per configured ROI, each with `[n_images]` vectors: | `npixel` | number of valid pixels | | `x`, `y` | intensity-weighted centroid | +The **dataset-wide ROI definition** (geometry, fixed for the whole acquisition) lives in the +master file in the same sub-group: + +| Dataset | Meaning | +|---------|---------| +| `bit_index` | which bit of `roi_map` (below) marks this ROI | +| `type` | `box`, `circle` or `azim` | +| `min_x_pxl`, `max_x_pxl`, `min_y_pxl`, `max_y_pxl` | box bounds (type `box`) | +| `center_x_pxl`, `center_y_pxl`, `radius_pxl` | circle (type `circle`) | +| `q_min_recipA`, `q_max_recipA` | Q range (type `azim`) | +| `phi_min_deg`, `phi_max_deg` | azimuthal-angle sector (type `azim`, omitted for a full ring) | + +`/entry/roi/roi_map` `[y, x]` (master file only) is a `uint16` per-pixel bitmask: bit `bit_index` +is set for every pixel belonging to that ROI, so an ROI's footprint can be recovered exactly. + ### 4.5 `/entry/image` — per-image pixel statistics `[n_images]` vectors: `max_value`, `min_value` (viable min/max, excluding error/saturated pixels), diff --git a/frame_serialize/CBORStream2Deserializer.cpp b/frame_serialize/CBORStream2Deserializer.cpp index 4c01990c..b61e9c67 100644 --- a/frame_serialize/CBORStream2Deserializer.cpp +++ b/frame_serialize/CBORStream2Deserializer.cpp @@ -1056,6 +1056,14 @@ namespace { image.GetWidth() * image.GetHeight()); } + void ProcessROIMapElement(StartMessage &message, CborValue &value) { + CompressedImage image = GetCBORMultidimTypedArray(value); + if (image.GetMode() == CompressedImageMode::Uint16) + JFJochDecompress(message.roi_map, image.GetCompressionAlgorithm(), + image.GetCompressed(), image.GetCompressedSize(), + image.GetWidth() * image.GetHeight()); + } + std::optional ProcessHDF5Format(int input) { auto tmp = static_cast(input); switch (tmp) { @@ -1221,6 +1229,8 @@ namespace { ProcessPixelMaskElement(message, value); else if (key == "az_int_map") ProcessAzintMapElement(message, value); + else if (key == "roi_map") + ProcessROIMapElement(message, value); else if (key == "channels") GetCBORStringArray(value, message.channels); else if (key == "detector_translation") diff --git a/frame_serialize/CBORStream2Serializer.cpp b/frame_serialize/CBORStream2Serializer.cpp index 3e361ef3..0cf19abd 100644 --- a/frame_serialize/CBORStream2Serializer.cpp +++ b/frame_serialize/CBORStream2Serializer.cpp @@ -446,6 +446,23 @@ inline void CBOR_ENC_AZINT_MAP(CborEncoder &encoder, const StartMessage &msg) { CBOR_ENC_2D_TYPED_ARRAY(encoder, image); } +inline void CBOR_ENC_ROI_MAP(CborEncoder &encoder, const StartMessage &msg) { + if (msg.roi_map.empty()) + return; + + if (msg.roi_map.size() != msg.image_size_x * msg.image_size_y) + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, + "Mismatch in size of ROI map"); + + JFJochBitShuffleCompressor compressor(CompressionAlgorithm::BSHUF_LZ4); + auto mask_compressed = compressor.Compress(msg.roi_map); + CompressedImage image(mask_compressed.data(), mask_compressed.size(), + msg.image_size_x, msg.image_size_y, + CompressedImageMode::Uint16, + CompressionAlgorithm::BSHUF_LZ4, "roi_map"); + CBOR_ENC_2D_TYPED_ARRAY(encoder, image); +} + inline void CBOR_ENC(CborEncoder &encoder, const char* key, const UnitCell &val) { CborEncoder mapEncoder; @@ -667,6 +684,7 @@ void CBORStream2Serializer::SerializeSequenceStart(const StartMessage& message) CBOR_ENC_PIXEL_MASK(mapEncoder, message); CBOR_ENC_AZINT_MAP(mapEncoder, message); + CBOR_ENC_ROI_MAP(mapEncoder, message); CBOR_ENC(mapEncoder, "channels", message.channels); CBOR_ENC(mapEncoder, "max_spot_count", message.max_spot_count); diff --git a/receiver/JFJochReceiver.cpp b/receiver/JFJochReceiver.cpp index 752013ff..666cc16a 100644 --- a/receiver/JFJochReceiver.cpp +++ b/receiver/JFJochReceiver.cpp @@ -119,6 +119,8 @@ void JFJochReceiver::SendStartMessage() { } message.writer_notification_zmq_addr = image_pusher.GetWriterNotificationSocketAddress(); message.rois = experiment.ROI().ExportMetadata(); + if (!experiment.ROI().empty()) + message.roi_map = experiment.ExportROIMap(); message.max_spot_count = experiment.GetMaxSpotCount(); diff --git a/tests/CBORTest.cpp b/tests/CBORTest.cpp index 35bbed6f..347aef50 100644 --- a/tests/CBORTest.cpp +++ b/tests/CBORTest.cpp @@ -870,6 +870,29 @@ TEST_CASE("CBORSerialize_Start_AzintMap", "[CBOR]") { CHECK(deserialized->start_message->az_int_map == azint_map); } +TEST_CASE("CBORSerialize_Start_ROIMap", "[CBOR]") { + std::vector buffer(8 * 1024 * 1024); + CBORStream2Serializer serializer(buffer.data(), buffer.size()); + + std::vector roi_map(32 * 15, 0); + roi_map[0] = (1 << 0) | (1 << 3); + roi_map[32 * 14] = (1 << 2); + + StartMessage msg{ + .image_size_x = 32, + .image_size_y = 15, + .roi_map = roi_map + }; + + REQUIRE_NOTHROW(serializer.SerializeSequenceStart(msg)); + + auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize()); + REQUIRE(deserialized); + REQUIRE(deserialized->msg_type == CBORImageType::START); + REQUIRE(deserialized->start_message); + CHECK(deserialized->start_message->roi_map == roi_map); +} + TEST_CASE("CBORSerialize_Image_Spots", "[CBOR]") { std::vector buffer(8 * 1024 * 1024); diff --git a/tools/jfjoch_process.cpp b/tools/jfjoch_process.cpp index ffb707bc..79702b50 100644 --- a/tools/jfjoch_process.cpp +++ b/tools/jfjoch_process.cpp @@ -760,6 +760,9 @@ int main(int argc, char **argv) { start_message.az_int_bin_to_phi = mapping.GetBinToPhi(); start_message.pixel_mask["default"] = pixel_mask.GetMask(experiment); + start_message.rois = experiment.ROI().ExportMetadata(); + if (!experiment.ROI().empty()) + start_message.roi_map = experiment.ExportROIMap(); start_message.max_spot_count = experiment.GetMaxSpotCount(); start_message.master_suffix = "process"; diff --git a/writer/HDF5NXmx.cpp b/writer/HDF5NXmx.cpp index 252e83f6..d666c390 100644 --- a/writer/HDF5NXmx.cpp +++ b/writer/HDF5NXmx.cpp @@ -51,6 +51,7 @@ NXmx::NXmx(const StartMessage &start) Attenuator(start); UserData(start); MX(start); + ROI(start); Fluorescence(start); } @@ -444,6 +445,57 @@ void NXmx::MX(const StartMessage &start) { } } +static void WriteROIDefinition(const HDF5Object &group, const ROIConfig &def) { + switch (def.type) { + case ROIConfig::ROIType::Box: + SaveScalar(group, "type", "box"); + SaveScalar(group, "min_x_pxl", def.box.xmin); + SaveScalar(group, "max_x_pxl", def.box.xmax); + SaveScalar(group, "min_y_pxl", def.box.ymin); + SaveScalar(group, "max_y_pxl", def.box.ymax); + break; + case ROIConfig::ROIType::Circle: + SaveScalar(group, "type", "circle"); + SaveScalar(group, "center_x_pxl", def.circle.x); + SaveScalar(group, "center_y_pxl", def.circle.y); + SaveScalar(group, "radius_pxl", def.circle.r); + break; + case ROIConfig::ROIType::Azim: + SaveScalar(group, "type", "azim"); + SaveScalar(group, "q_min_recipA", def.azim.qmin); + SaveScalar(group, "q_max_recipA", def.azim.qmax); + // phi_min == phi_max means a full ring; only record a sector. + if (def.azim.phi_min != def.azim.phi_max) { + SaveScalar(group, "phi_min_deg", def.azim.phi_min); + SaveScalar(group, "phi_max_deg", def.azim.phi_max); + } + break; + } +} + +void NXmx::ROI(const StartMessage &start) { + if (start.rois.empty()) + return; + + HDF5Group roi_group(*hdf5_file, "/entry/roi"); + roi_group.NXClass("NXcollection"); + + if (!start.roi_map.empty()) { + // Per-pixel ROI bitmask: bit i (the bit_index below) marks pixels in ROI i. + CompressionAlgorithm roi_alg = (start.file_format == FileWriterFormat::NXmxLegacy) + ? CompressionAlgorithm::NO_COMPRESSION + : CompressionAlgorithm::BSHUF_LZ4; + std::vector dims = {start.image_size_y, start.image_size_x}; + roi_group.SaveVector("roi_map", start.roi_map, dims, roi_alg); + } + + for (size_t i = 0; i < start.rois.size(); i++) { + HDF5Group g(roi_group, start.rois[i].name); + SaveScalar(g, "bit_index", static_cast(i)); + WriteROIDefinition(g, start.rois[i]); + } +} + void NXmx::DetectorModule(const std::string &name, const std::vector &origin, const std::vector &size, const std::vector &fast_axis, const std::vector &slow_axis, const std::string &nx_axis, double pixel_size_mm) { diff --git a/writer/HDF5NXmx.h b/writer/HDF5NXmx.h index 30612dc0..427f1895 100644 --- a/writer/HDF5NXmx.h +++ b/writer/HDF5NXmx.h @@ -54,6 +54,7 @@ class NXmx { void EndResultVectors(const EndMessage &end); void UserData(const StartMessage &start); void MX(const StartMessage &start); + void ROI(const StartMessage &start); void Fluorescence(const StartMessage &start); public: