mx/Jungfraujoch
2
1
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases 67 Wiki Activity

writer: remove CBF and TIFF writers; only HDF5 is written

Browse Source 
Delete CBFWriter and the TIFF write path from jfjoch_writer so the writer
only produces the NXmx HDF5 formats. Drop CBF/TIFF from the internal
FileWriterFormat enum and reject them in FileWriterSettings.

The CBF/TIFF values are kept (marked deprecated) in the file_writer_format
OpenAPI enum for back compatibility: incoming requests using them are now
rejected in OpenAPIConvert with a clear "no longer supported" error, and a
stale CBF/TIFF value on the CBOR wire decodes to unset rather than a removed
enumerator.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
leonarski_f
2026-07-15 10:11:39 +02:00
co-authored by Claude Opus 4.8
parent 135765e5b5
commit 4bdf9309dd
12 changed files with 8 additions and 339 deletions
Download Patch File Download Diff File Expand all files Collapse all files
broker/OpenAPIConvert.cpp
+3 -8
View File
@@ -838,12 +838,6 @@ org::openapitools::server::model::File_writer_format Convert(FileWriterFormat in
case FileWriterFormat::NXmxIntegrated:
ret.setValue(org::openapitools::server::model::File_writer_format::eFile_writer_format::NXMXINTEGRATED);
break;
case FileWriterFormat::CBF:
ret.setValue(org::openapitools::server::model::File_writer_format::eFile_writer_format::CBF);
break;
case FileWriterFormat::TIFF:
ret.setValue(org::openapitools::server::model::File_writer_format::eFile_writer_format::TIFF);
break;
case FileWriterFormat::NoFile:
ret.setValue(org::openapitools::server::model::File_writer_format::eFile_writer_format::NOFILEWRITTEN);
break;
@@ -864,9 +858,10 @@ FileWriterFormat Convert(const org::openapitools::server::model::File_writer_for
case org::openapitools::server::model::File_writer_format::eFile_writer_format::NXMXINTEGRATED:
return FileWriterFormat::NXmxIntegrated;
case org::openapitools::server::model::File_writer_format::eFile_writer_format::CBF:
return FileWriterFormat::CBF;
case org::openapitools::server::model::File_writer_format::eFile_writer_format::TIFF:
return FileWriterFormat::TIFF;
// Deprecated, kept in the OpenAPI enum for back compatibility only - only HDF5 is written now.
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"CBF and TIFF file formats are no longer supported");
case org::openapitools::server::model::File_writer_format::eFile_writer_format::NOFILEWRITTEN:
return FileWriterFormat::NoFile;
default:
broker/jfjoch_api.yaml
+3 -3
View File
@@ -726,11 +726,11 @@ components:
description: |
NoFileWritten - no files are written at all
NXmxOnlyData - only data files are written, no master file
NXmxLegacy - legacy format with soft links to data files in the master file; necessary for DECTRIS Albula 4.0 and DECTRIS Neggia
NXmxLegacy - legacy format with soft links to data files in the master file; necessary for DECTRIS Albula 4.0 and DECTRIS Neggia
NXmxVDS - newer format with virtual dataset linking data files in the master file, also includes better metadata handling
NXmxIntegrated - single HDF5 per dataset
CBF - CBF format (limited metadata)
TIFF - TIFF format (no metadata)
CBF - DEPRECATED, no longer supported; kept for back compatibility only. Requests using this value are rejected. Only HDF5 formats are written.
TIFF - DEPRECATED, no longer supported; kept for back compatibility only. Requests using this value are rejected. Only HDF5 formats are written.
file_writer_settings:
type: object
properties:
common/FileWriterSettings.cpp
-2
View File
@@ -15,8 +15,6 @@ FileWriterSettings &FileWriterSettings::FileFormat(FileWriterFormat input) {
case FileWriterFormat::NXmxLegacy:
case FileWriterFormat::NXmxVDS:
case FileWriterFormat::NXmxIntegrated:
case FileWriterFormat::TIFF:
case FileWriterFormat::CBF:
case FileWriterFormat::NoFile:
hdf5_master_format_version = input;
return *this;
common/JFJochMessages.h
+2 -2
View File
@@ -33,8 +33,8 @@ enum class FileWriterFormat : int {
NXmxLegacy = 1,
NXmxVDS = 2,
NXmxIntegrated = 3,
CBF = 4,
TIFF = 5,
// 4 (CBF) and 5 (TIFF) removed - only HDF5 is written now. The values are kept
// (deprecated) in the OpenAPI enum in broker/jfjoch_api.yaml for back compatibility.
NoFile = 6
};
frame_serialize/CBORStream2Deserializer.cpp
-2
View File
@@ -1073,8 +1073,6 @@ namespace {
case FileWriterFormat::NXmxLegacy:
case FileWriterFormat::NXmxIntegrated:
case FileWriterFormat::NXmxVDS:
case FileWriterFormat::CBF:
case FileWriterFormat::TIFF:
case FileWriterFormat::NoFile:
return tmp;
default:
tests/HDF5WritingTest.cpp
-95
View File
@@ -1255,101 +1255,6 @@ TEST_CASE("HDF5Writer_Link_zero_images", "[HDF5][Full]") {
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("FileWriter_CBF_16bit", "[HDF5][Full]") {
{
DetectorSetup det = DetJF4M("DET1");
det.SerialNumber("1");
DiffractionExperiment x(det);
std::vector<SpotToSave> spots;
x.FilePrefix("lyso_cbf_16").ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION);
x.PixelSigned(true).BitDepthImage(16);
x.SetFileWriterFormat(FileWriterFormat::CBF);
StartMessage start_message;
x.FillMessage(start_message);
FileWriter file_set(start_message);
std::vector<int16_t> image(x.GetPixelsNum(), 45);
for (int i = 0; i < x.GetImageNum(); i++) {
DataMessage message{};
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
message.spots = spots;
message.number = i;
REQUIRE_NOTHROW(file_set.Write(message));
}
auto v = file_set.Finalize();
REQUIRE(!file_set.GetZMQAddr());
}
}
TEST_CASE("FileWriter_CBF", "[HDF5][Full]") {
{
DetectorSetup det = DetJF4M("DET1");
det.SerialNumber("1");
DiffractionExperiment x(det);
std::vector<SpotToSave> spots;
x.FilePrefix("lyso_cbf").ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION);
x.PixelSigned(false).BitDepthImage(32);
x.SetFileWriterFormat(FileWriterFormat::CBF);
StartMessage start_message;
x.FillMessage(start_message);
FileWriter file_set(start_message);
std::vector<uint32_t> image(x.GetPixelsNum());
for (int i = 0; i < image.size(); i++)
image[i] = i;
for (int i = 0; i < x.GetImageNum(); i++) {
DataMessage message{};
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
message.spots = spots;
message.number = i;
REQUIRE_NOTHROW(file_set.Write(message));
}
auto v = file_set.Finalize();
REQUIRE(!file_set.GetZMQAddr());
}
}
TEST_CASE("FileWriter_TIFF", "[HDF5][Full]") {
{
DetectorSetup det = DetJF4M("DET1");
det.SerialNumber("1");
DiffractionExperiment x(det);
std::vector<SpotToSave> spots;
x.FilePrefix("lyso_tiff").ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION);
x.SetFileWriterFormat(FileWriterFormat::TIFF);
StartMessage start_message;
x.FillMessage(start_message);
FileWriter file_set(start_message);
std::vector<int16_t> image(x.GetPixelsNum(), 45);
for (int i = 0; i < x.GetImageNum(); i++) {
DataMessage message{};
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
message.spots = spots;
message.number = i;
REQUIRE_NOTHROW(file_set.Write(message));
}
auto v = file_set.Finalize();
REQUIRE(!file_set.GetZMQAddr());
}
}
TEST_CASE("HDF5Objects_VDS_reverse_contiguous", "[HDF5][Unit]") {
{
RegisterHDF5Filter();
tests/JFJochReceiverProcessingTest.cpp
-35
View File
@@ -1543,41 +1543,6 @@ TEST_CASE("JFJochIntegrationTest_HDF5FilePusher_republish", "[JFJochReceiver]")
REQUIRE(!pull_socket.Receive(msg, true));
}
TEST_CASE("JFJochIntegrationTest_HDF5FilePusher_cbf", "[JFJochReceiver]") {
Logger logger(Catch::getResultCapture().getCurrentTestName());
RegisterHDF5Filter();
const uint16_t nthreads = 4;
DiffractionExperiment experiment(DetJF4M());
experiment.ImagesPerTrigger(5).NumTriggers(1).UseInternalPacketGenerator(true)
.FilePrefix("cbf_test").JungfrauConvPhotonCnt(true)
.DetectorDistance_mm(75).BeamY_pxl(1136).BeamX_pxl(1090).IncidentEnergy_keV(12.4)
.SetFileWriterFormat(FileWriterFormat::CBF);
JFCalibration calibration(experiment);
PixelMask pixel_mask(experiment);
// Setup acquisition device
AcquisitionDeviceGroup aq_devices;
aq_devices.Add(std::make_unique<HLSSimulatedDevice>(0, 64));
HDF5FilePusher pusher;
JFJochReceiverService service(aq_devices, logger, pusher);
service.NumThreads(nthreads);
service.Indexing(experiment.GetIndexingSettings());
service.Start(experiment, pixel_mask, &calibration);
auto receiver_out = service.Stop();
CHECK(receiver_out.efficiency == 1.0);
CHECK(receiver_out.status.images_collected == 5);
CHECK(receiver_out.status.images_sent == 5);
CHECK(!receiver_out.status.cancelled);
}
TEST_CASE("JFJochIntegrationTest_HDF5FilePusher_Raw", "[JFJochReceiver]") {
Logger logger(Catch::getResultCapture().getCurrentTestName());
writer/CBFWriter.cpp
-149
View File
@@ -1,149 +0,0 @@
// SPDX-FileCopyrightText: 2024 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include "CBFWriter.h"
#include <utility>
#include "../include/spdlog/fmt/fmt.h"
#include "../compression/JFJochDecompress.h"
template <class T>
void write(uint8_t* arr, T val, size_t &offset) {
auto ptr = reinterpret_cast<T *>(arr + offset);
*ptr = val;
offset += sizeof(T);
}
template <class T>
void CBFWriter::EncodeCBF(const T *image, size_t num_elem, std::vector<uint8_t> &output) {
size_t output_loc = 0;
int64_t cval = 0;
for (int i = 0; i < num_elem; i++) {
if (output_loc > output.size() - 6)
output.resize(output.size() * 2);
int64_t nval;
if (image[i] > saturation_value)
nval = saturation_value;
else if ((image[i] < 0) || (pixel_mask[i] != 0))
// For Bitshuffle compression INTx_MIN is the best value for special things, at it is one bit different from 0
// For CBF compression -1 is much better, as it reduces differences in counts
nval = -1;
else
nval = image[i];
int64_t diff = nval - cval;
if (abs(diff) <= 127)
write(output.data(), static_cast<int8_t>(diff), output_loc);
else {
write(output.data(), static_cast<uint8_t>(0x80), output_loc);
if (abs(diff) <= 32767)
write(output.data(), static_cast<int16_t>(diff), output_loc);
else {
write(output.data(), static_cast<uint16_t>(0x8000), output_loc);
write(output.data(), static_cast<int32_t>(diff), output_loc);
}
}
cval = nval; // current value
}
output.resize(output_loc);
}
CBFWriter::CBFWriter(StartMessage in_start_msg)
: start_msg(std::move(in_start_msg)),
saturation_value(std::min<int64_t>(start_msg.saturation_value, (1L<<24))),
pixel_mask(start_msg.image_size_x * start_msg.image_size_y, 0) {
header += fmt::format("\r\n# Detector: {:s}, S/N {:s}\r\n",
start_msg.detector_description,
start_msg.detector_serial_number);
header += fmt::format("# Silicon sensor, thickness {:.6f} m\r\n", start_msg.sensor_thickness);
header += fmt::format("# Exposure_time {:.6f} s\r\n", start_msg.count_time);
header += fmt::format("# Exposure_period {:.6f} s\r\n", start_msg.frame_time);
header += fmt::format("# Count_cutoff {:d} counts\r\n", saturation_value);
header += fmt::format("# Wavelength {:.3f} A\r\n", start_msg.incident_wavelength);
header += fmt::format("# Detector_distance {:.4f} m\r\n", start_msg.detector_distance);
header += fmt::format("# Beam_xy ({:.1f}, {:.1f}) pixels\r\n", start_msg.beam_center_x, start_msg.beam_center_y);
if (start_msg.goniometer.has_value())
header += fmt::format("# Start_angle {:f} deg.\r\n# Angle_increment {:f} deg.\r\n",
start_msg.goniometer->GetStart_deg(),
start_msg.goniometer->GetIncrement_deg());
if ((start_msg.pixel_mask.size() == 1) && (start_msg.pixel_mask.begin()->second.size() == pixel_mask.size())) {
for (int i = 0; i < pixel_mask.size(); i++)
pixel_mask[i] = start_msg.pixel_mask.begin()->second[i];
}
}
void CBFWriter::WriteImage(const DataMessage &data_msg) {
std::string filename = fmt::format("{:s}_{:06d}.cbf", start_msg.file_prefix, data_msg.number);
FILE *fh = fopen(filename.c_str(), "wb");
if (!fh)
throw (JFJochException(JFJochExceptionCategory::FileWriteError, "Cannot create file " + filename));
auto npixel = data_msg.image.GetWidth() * data_msg.image.GetHeight();
std::vector<uint8_t> image;
data_msg.image.GetUncompressed(image);
std::vector<uint8_t> output(npixel * sizeof(uint32_t) * 2);
switch (data_msg.image.GetMode()) {
case CompressedImageMode::Int8:
EncodeCBF((int8_t *) image.data(), npixel, output);
break;
case CompressedImageMode::Int16:
EncodeCBF((int16_t *) image.data(), npixel, output);
break;
case CompressedImageMode::Int32:
EncodeCBF((int32_t *) image.data(), npixel, output);
break;
case CompressedImageMode::Uint8:
EncodeCBF((uint8_t *) image.data(), npixel, output);
break;
case CompressedImageMode::Uint16:
EncodeCBF((uint16_t *) image.data(), npixel, output);
break;
case CompressedImageMode::Uint32:
EncodeCBF((uint32_t *) image.data(), npixel, output);
break;
default:
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"RGB/float modes not supported in CBF writer");
}
fputs("###CBF: VERSION 1.7.11\r\n\r\n", fh);
fputs(fmt::format("data_image_{:06d}\r\n\r\n", data_msg.number).c_str(), fh);
fputs("_array_data.header_convention SLS_1.0\r\n", fh);
fputs("_array_data.header_contents\r\n;\r\n\r\n", fh);
fputs(header.c_str(), fh);
fputs(";\r\n", fh);
fputs("_array_data.data", fh);
fputs("\r\n;\r\n", fh);
fputs("--CIF-BINARY-FORMAT-SECTION--\r\n", fh);
fputs("Content-Type: application/octet-stream;\r\n", fh);
fputs(" conversions=\"x-CBF_BYTE_OFFSET\"\r\n", fh);
fputs("Content-Transfer-Encoding: BINARY\r\n", fh);
fputs(fmt::format("X-Binary-Size: {:d}\r\n", output.size()).c_str(), fh);
fputs("X-Binary-ID: 1\r\n", fh);
fputs("X-Binary-Element-Type: \"signed 32-bit integer\"\r\n", fh);
fputs("X-Binary-Element-Byte-Order: LITTLE_ENDIAN\r\n", fh);
// fputs("Content-MD5: {md5_hash}\r\n", fh);
fputs(fmt::format("X-Binary-Number-of-Elements: {:d}\r\n", npixel).c_str(), fh);
fputs(fmt::format("X-Binary-Size-Fastest-Dimension: {:d}\r\n", data_msg.image.GetWidth()).c_str(), fh);
fputs(fmt::format("X-Binary-Size-Second-Dimension: {:d}\r\n", data_msg.image.GetHeight()).c_str(), fh);
fputs("X-Binary-Size-Padding: 0\r\n", fh);
fputs("\x0C\x1A\x04\xD5", fh);
fwrite(output.data(), sizeof(uint8_t), output.size(), fh);
fputs("\r\n--CIF-BINARY-FORMAT-SECTION----\r\n;\r\n\r\n",fh);
fclose(fh);
}
writer/CBFWriter.h
-22
View File
@@ -1,22 +0,0 @@
// SPDX-FileCopyrightText: 2024 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#pragma once
#include "../common/JFJochMessages.h"
class CBFWriter {
StartMessage start_msg;
const int64_t saturation_value;
std::string header;
std::vector<uint32_t> pixel_mask;
template <class T>
void EncodeCBF(const T *image, size_t num_elem, std::vector<uint8_t> &output);
public:
CBFWriter(StartMessage start_msg);
void WriteImage(const DataMessage& data_msg);
};
writer/CMakeLists.txt
-2
View File
@@ -29,8 +29,6 @@ ADD_LIBRARY(JFJochWriter STATIC
HDF5DataFilePluginReflection.h
HDF5DataFilePluginPerformance.cpp
HDF5DataFilePluginPerformance.h
CBFWriter.cpp
CBFWriter.h
)
TARGET_LINK_LIBRARIES(JFJochWriter JFJochPreview JFJochLogger JFJochHDF5Wrappers CBORStream2FrameSerialize)
writer/FileWriter.cpp
-16
View File
@@ -8,7 +8,6 @@
#include "../common/CheckPath.h"
#include "../common/Logger.h"
#include "../common/JFJochException.h"
#include "../preview/JFJochTIFF.h"
#include "JFJochDecompress.h"
FileWriter::FileWriter(const StartMessage &request, bool check_overwrite_at_start, bool trusted_path)
@@ -33,9 +32,6 @@ FileWriter::FileWriter(const StartMessage &request, bool check_overwrite_at_star
case FileWriterFormat::NXmxIntegrated:
CreateHDF5MasterFile(request);
break;
case FileWriterFormat::CBF:
cbf_writer = std::make_unique<CBFWriter>(start_message);
break;
default:
// Do nothing
break;
@@ -51,24 +47,12 @@ void FileWriter::Write(const DataMessage &msg) {
case FileWriterFormat::NXmxIntegrated:
WriteHDF5(msg);
break;
case FileWriterFormat::CBF:
cbf_writer->WriteImage(msg);
break;
case FileWriterFormat::TIFF:
WriteTIFF(msg);
break;
case FileWriterFormat::NoFile:
// Do nothing
break;
}
}
void FileWriter::WriteTIFF(const DataMessage &msg) {
const std::string file_name = fmt::format("{:s}{:06d}.tiff", start_message.file_prefix, msg.number);
WriteTIFFToFile(file_name,msg.image);
}
void FileWriter::WriteHDF5(const DataMessage& msg) {
std::lock_guard<std::mutex> lock(hdf5_mutex);
if (msg.image.GetCompressedSize() == 0)
writer/FileWriter.h
-3
View File
@@ -9,7 +9,6 @@
#include "../common/JFJochMessages.h"
#include "../common/ZMQWrappers.h"
#include "HDF5NXmx.h"
#include "CBFWriter.h"
#include <unordered_set>
class FileWriter {
@@ -20,7 +19,6 @@ class FileWriter {
std::vector<std::unique_ptr<HDF5DataFile> > files;
std::vector<HDF5DataFileStatistics> stats;
std::unique_ptr<ZMQSocket> finalized_file_socket;
std::unique_ptr<CBFWriter> cbf_writer;
std::unordered_set<uint64_t> closed_files;
constexpr static uint64_t close_file_lag_images = 1000;
@@ -45,7 +43,6 @@ public:
explicit FileWriter(const StartMessage &request, bool check_overwrite_at_start = true,
bool trusted_path = false);
void Write(const DataMessage& msg);
void WriteTIFF(const DataMessage& msg);
void WriteHDF5(const DataMessage& msg);
void WriteHDF5(const CompressedImage& msg);
void WriteHDF5(const EndMessage& msg);