diff --git a/broker/OpenAPIConvert.cpp b/broker/OpenAPIConvert.cpp index 61b02aba..3c3ebc8d 100644 --- a/broker/OpenAPIConvert.cpp +++ b/broker/OpenAPIConvert.cpp @@ -838,12 +838,6 @@ org::openapitools::server::model::File_writer_format Convert(FileWriterFormat in case FileWriterFormat::NXmxIntegrated: ret.setValue(org::openapitools::server::model::File_writer_format::eFile_writer_format::NXMXINTEGRATED); break; - case FileWriterFormat::CBF: - ret.setValue(org::openapitools::server::model::File_writer_format::eFile_writer_format::CBF); - break; - case FileWriterFormat::TIFF: - ret.setValue(org::openapitools::server::model::File_writer_format::eFile_writer_format::TIFF); - break; case FileWriterFormat::NoFile: ret.setValue(org::openapitools::server::model::File_writer_format::eFile_writer_format::NOFILEWRITTEN); break; @@ -864,9 +858,10 @@ FileWriterFormat Convert(const org::openapitools::server::model::File_writer_for case org::openapitools::server::model::File_writer_format::eFile_writer_format::NXMXINTEGRATED: return FileWriterFormat::NXmxIntegrated; case org::openapitools::server::model::File_writer_format::eFile_writer_format::CBF: - return FileWriterFormat::CBF; case org::openapitools::server::model::File_writer_format::eFile_writer_format::TIFF: - return FileWriterFormat::TIFF; + // Deprecated, kept in the OpenAPI enum for back compatibility only - only HDF5 is written now. + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, + "CBF and TIFF file formats are no longer supported"); case org::openapitools::server::model::File_writer_format::eFile_writer_format::NOFILEWRITTEN: return FileWriterFormat::NoFile; default: diff --git a/broker/jfjoch_api.yaml b/broker/jfjoch_api.yaml index 39ded72d..e3d2a7f4 100644 --- a/broker/jfjoch_api.yaml +++ b/broker/jfjoch_api.yaml @@ -726,11 +726,11 @@ components: description: | NoFileWritten - no files are written at all NXmxOnlyData - only data files are written, no master file - NXmxLegacy - legacy format with soft links to data files in the master file; necessary for DECTRIS Albula 4.0 and DECTRIS Neggia + NXmxLegacy - legacy format with soft links to data files in the master file; necessary for DECTRIS Albula 4.0 and DECTRIS Neggia NXmxVDS - newer format with virtual dataset linking data files in the master file, also includes better metadata handling NXmxIntegrated - single HDF5 per dataset - CBF - CBF format (limited metadata) - TIFF - TIFF format (no metadata) + CBF - DEPRECATED, no longer supported; kept for back compatibility only. Requests using this value are rejected. Only HDF5 formats are written. + TIFF - DEPRECATED, no longer supported; kept for back compatibility only. Requests using this value are rejected. Only HDF5 formats are written. file_writer_settings: type: object properties: diff --git a/common/FileWriterSettings.cpp b/common/FileWriterSettings.cpp index d62cfd32..9facb6c0 100644 --- a/common/FileWriterSettings.cpp +++ b/common/FileWriterSettings.cpp @@ -15,8 +15,6 @@ FileWriterSettings &FileWriterSettings::FileFormat(FileWriterFormat input) { case FileWriterFormat::NXmxLegacy: case FileWriterFormat::NXmxVDS: case FileWriterFormat::NXmxIntegrated: - case FileWriterFormat::TIFF: - case FileWriterFormat::CBF: case FileWriterFormat::NoFile: hdf5_master_format_version = input; return *this; diff --git a/common/JFJochMessages.h b/common/JFJochMessages.h index ef7ba2cb..a884d4c9 100644 --- a/common/JFJochMessages.h +++ b/common/JFJochMessages.h @@ -33,8 +33,8 @@ enum class FileWriterFormat : int { NXmxLegacy = 1, NXmxVDS = 2, NXmxIntegrated = 3, - CBF = 4, - TIFF = 5, + // 4 (CBF) and 5 (TIFF) removed - only HDF5 is written now. The values are kept + // (deprecated) in the OpenAPI enum in broker/jfjoch_api.yaml for back compatibility. NoFile = 6 }; diff --git a/frame_serialize/CBORStream2Deserializer.cpp b/frame_serialize/CBORStream2Deserializer.cpp index 877c71a2..9c36c8a8 100644 --- a/frame_serialize/CBORStream2Deserializer.cpp +++ b/frame_serialize/CBORStream2Deserializer.cpp @@ -1073,8 +1073,6 @@ namespace { case FileWriterFormat::NXmxLegacy: case FileWriterFormat::NXmxIntegrated: case FileWriterFormat::NXmxVDS: - case FileWriterFormat::CBF: - case FileWriterFormat::TIFF: case FileWriterFormat::NoFile: return tmp; default: diff --git a/tests/HDF5WritingTest.cpp b/tests/HDF5WritingTest.cpp index 39a73a47..07e469b3 100644 --- a/tests/HDF5WritingTest.cpp +++ b/tests/HDF5WritingTest.cpp @@ -1255,101 +1255,6 @@ TEST_CASE("HDF5Writer_Link_zero_images", "[HDF5][Full]") { REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); } -TEST_CASE("FileWriter_CBF_16bit", "[HDF5][Full]") { - { - DetectorSetup det = DetJF4M("DET1"); - det.SerialNumber("1"); - DiffractionExperiment x(det); - std::vector spots; - - x.FilePrefix("lyso_cbf_16").ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION); - x.PixelSigned(true).BitDepthImage(16); - x.SetFileWriterFormat(FileWriterFormat::CBF); - StartMessage start_message; - x.FillMessage(start_message); - - FileWriter file_set(start_message); - std::vector image(x.GetPixelsNum(), 45); - - for (int i = 0; i < x.GetImageNum(); i++) { - DataMessage message{}; - message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum()); - message.spots = spots; - message.number = i; - - REQUIRE_NOTHROW(file_set.Write(message)); - } - - auto v = file_set.Finalize(); - - REQUIRE(!file_set.GetZMQAddr()); - } -} - -TEST_CASE("FileWriter_CBF", "[HDF5][Full]") { - { - DetectorSetup det = DetJF4M("DET1"); - det.SerialNumber("1"); - DiffractionExperiment x(det); - std::vector spots; - - x.FilePrefix("lyso_cbf").ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION); - x.PixelSigned(false).BitDepthImage(32); - x.SetFileWriterFormat(FileWriterFormat::CBF); - StartMessage start_message; - x.FillMessage(start_message); - - FileWriter file_set(start_message); - std::vector image(x.GetPixelsNum()); - - for (int i = 0; i < image.size(); i++) - image[i] = i; - - for (int i = 0; i < x.GetImageNum(); i++) { - DataMessage message{}; - message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum()); - message.spots = spots; - message.number = i; - - REQUIRE_NOTHROW(file_set.Write(message)); - } - - auto v = file_set.Finalize(); - - REQUIRE(!file_set.GetZMQAddr()); - } -} - -TEST_CASE("FileWriter_TIFF", "[HDF5][Full]") { - { - DetectorSetup det = DetJF4M("DET1"); - det.SerialNumber("1"); - DiffractionExperiment x(det); - std::vector spots; - - x.FilePrefix("lyso_tiff").ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION); - x.SetFileWriterFormat(FileWriterFormat::TIFF); - StartMessage start_message; - x.FillMessage(start_message); - - FileWriter file_set(start_message); - std::vector image(x.GetPixelsNum(), 45); - - for (int i = 0; i < x.GetImageNum(); i++) { - DataMessage message{}; - message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum()); - message.spots = spots; - message.number = i; - - REQUIRE_NOTHROW(file_set.Write(message)); - } - - auto v = file_set.Finalize(); - - REQUIRE(!file_set.GetZMQAddr()); - } -} - TEST_CASE("HDF5Objects_VDS_reverse_contiguous", "[HDF5][Unit]") { { RegisterHDF5Filter(); diff --git a/tests/JFJochReceiverProcessingTest.cpp b/tests/JFJochReceiverProcessingTest.cpp index 801baebd..4eab4af1 100644 --- a/tests/JFJochReceiverProcessingTest.cpp +++ b/tests/JFJochReceiverProcessingTest.cpp @@ -1543,41 +1543,6 @@ TEST_CASE("JFJochIntegrationTest_HDF5FilePusher_republish", "[JFJochReceiver]") REQUIRE(!pull_socket.Receive(msg, true)); } -TEST_CASE("JFJochIntegrationTest_HDF5FilePusher_cbf", "[JFJochReceiver]") { - Logger logger(Catch::getResultCapture().getCurrentTestName()); - - RegisterHDF5Filter(); - - const uint16_t nthreads = 4; - - DiffractionExperiment experiment(DetJF4M()); - experiment.ImagesPerTrigger(5).NumTriggers(1).UseInternalPacketGenerator(true) - .FilePrefix("cbf_test").JungfrauConvPhotonCnt(true) - .DetectorDistance_mm(75).BeamY_pxl(1136).BeamX_pxl(1090).IncidentEnergy_keV(12.4) - .SetFileWriterFormat(FileWriterFormat::CBF); - - JFCalibration calibration(experiment); - PixelMask pixel_mask(experiment); - - // Setup acquisition device - AcquisitionDeviceGroup aq_devices; - aq_devices.Add(std::make_unique(0, 64)); - - HDF5FilePusher pusher; - - JFJochReceiverService service(aq_devices, logger, pusher); - service.NumThreads(nthreads); - service.Indexing(experiment.GetIndexingSettings()); - - service.Start(experiment, pixel_mask, &calibration); - auto receiver_out = service.Stop(); - - CHECK(receiver_out.efficiency == 1.0); - CHECK(receiver_out.status.images_collected == 5); - CHECK(receiver_out.status.images_sent == 5); - CHECK(!receiver_out.status.cancelled); -} - TEST_CASE("JFJochIntegrationTest_HDF5FilePusher_Raw", "[JFJochReceiver]") { Logger logger(Catch::getResultCapture().getCurrentTestName()); diff --git a/writer/CBFWriter.cpp b/writer/CBFWriter.cpp deleted file mode 100644 index c8a86037..00000000 --- a/writer/CBFWriter.cpp +++ /dev/null @@ -1,149 +0,0 @@ -// SPDX-FileCopyrightText: 2024 Filip Leonarski, Paul Scherrer Institute -// SPDX-License-Identifier: GPL-3.0-only - -#include "CBFWriter.h" - -#include - -#include "../include/spdlog/fmt/fmt.h" -#include "../compression/JFJochDecompress.h" - -template -void write(uint8_t* arr, T val, size_t &offset) { - auto ptr = reinterpret_cast(arr + offset); - *ptr = val; - offset += sizeof(T); -} - -template -void CBFWriter::EncodeCBF(const T *image, size_t num_elem, std::vector &output) { - size_t output_loc = 0; - int64_t cval = 0; - - for (int i = 0; i < num_elem; i++) { - if (output_loc > output.size() - 6) - output.resize(output.size() * 2); - - int64_t nval; - if (image[i] > saturation_value) - nval = saturation_value; - else if ((image[i] < 0) || (pixel_mask[i] != 0)) - // For Bitshuffle compression INTx_MIN is the best value for special things, at it is one bit different from 0 - // For CBF compression -1 is much better, as it reduces differences in counts - nval = -1; - else - nval = image[i]; - - int64_t diff = nval - cval; - if (abs(diff) <= 127) - write(output.data(), static_cast(diff), output_loc); - else { - write(output.data(), static_cast(0x80), output_loc); - if (abs(diff) <= 32767) - write(output.data(), static_cast(diff), output_loc); - else { - write(output.data(), static_cast(0x8000), output_loc); - write(output.data(), static_cast(diff), output_loc); - } - } - cval = nval; // current value - } - output.resize(output_loc); -} - -CBFWriter::CBFWriter(StartMessage in_start_msg) - : start_msg(std::move(in_start_msg)), - saturation_value(std::min(start_msg.saturation_value, (1L<<24))), - pixel_mask(start_msg.image_size_x * start_msg.image_size_y, 0) { - - header += fmt::format("\r\n# Detector: {:s}, S/N {:s}\r\n", - start_msg.detector_description, - start_msg.detector_serial_number); - - header += fmt::format("# Silicon sensor, thickness {:.6f} m\r\n", start_msg.sensor_thickness); - header += fmt::format("# Exposure_time {:.6f} s\r\n", start_msg.count_time); - header += fmt::format("# Exposure_period {:.6f} s\r\n", start_msg.frame_time); - header += fmt::format("# Count_cutoff {:d} counts\r\n", saturation_value); - header += fmt::format("# Wavelength {:.3f} A\r\n", start_msg.incident_wavelength); - header += fmt::format("# Detector_distance {:.4f} m\r\n", start_msg.detector_distance); - header += fmt::format("# Beam_xy ({:.1f}, {:.1f}) pixels\r\n", start_msg.beam_center_x, start_msg.beam_center_y); - - if (start_msg.goniometer.has_value()) - header += fmt::format("# Start_angle {:f} deg.\r\n# Angle_increment {:f} deg.\r\n", - start_msg.goniometer->GetStart_deg(), - start_msg.goniometer->GetIncrement_deg()); - - if ((start_msg.pixel_mask.size() == 1) && (start_msg.pixel_mask.begin()->second.size() == pixel_mask.size())) { - for (int i = 0; i < pixel_mask.size(); i++) - pixel_mask[i] = start_msg.pixel_mask.begin()->second[i]; - } -} - -void CBFWriter::WriteImage(const DataMessage &data_msg) { - std::string filename = fmt::format("{:s}_{:06d}.cbf", start_msg.file_prefix, data_msg.number); - FILE *fh = fopen(filename.c_str(), "wb"); - if (!fh) - throw (JFJochException(JFJochExceptionCategory::FileWriteError, "Cannot create file " + filename)); - - auto npixel = data_msg.image.GetWidth() * data_msg.image.GetHeight(); - - std::vector image; - data_msg.image.GetUncompressed(image); - - std::vector output(npixel * sizeof(uint32_t) * 2); - - switch (data_msg.image.GetMode()) { - case CompressedImageMode::Int8: - EncodeCBF((int8_t *) image.data(), npixel, output); - break; - case CompressedImageMode::Int16: - EncodeCBF((int16_t *) image.data(), npixel, output); - break; - case CompressedImageMode::Int32: - EncodeCBF((int32_t *) image.data(), npixel, output); - break; - case CompressedImageMode::Uint8: - EncodeCBF((uint8_t *) image.data(), npixel, output); - break; - case CompressedImageMode::Uint16: - EncodeCBF((uint16_t *) image.data(), npixel, output); - break; - case CompressedImageMode::Uint32: - EncodeCBF((uint32_t *) image.data(), npixel, output); - break; - default: - throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, - "RGB/float modes not supported in CBF writer"); - } - - fputs("###CBF: VERSION 1.7.11\r\n\r\n", fh); - fputs(fmt::format("data_image_{:06d}\r\n\r\n", data_msg.number).c_str(), fh); - fputs("_array_data.header_convention SLS_1.0\r\n", fh); - fputs("_array_data.header_contents\r\n;\r\n\r\n", fh); - fputs(header.c_str(), fh); - - fputs(";\r\n", fh); - - fputs("_array_data.data", fh); - fputs("\r\n;\r\n", fh); - fputs("--CIF-BINARY-FORMAT-SECTION--\r\n", fh); - fputs("Content-Type: application/octet-stream;\r\n", fh); - fputs(" conversions=\"x-CBF_BYTE_OFFSET\"\r\n", fh); - fputs("Content-Transfer-Encoding: BINARY\r\n", fh); - fputs(fmt::format("X-Binary-Size: {:d}\r\n", output.size()).c_str(), fh); - fputs("X-Binary-ID: 1\r\n", fh); - fputs("X-Binary-Element-Type: \"signed 32-bit integer\"\r\n", fh); - fputs("X-Binary-Element-Byte-Order: LITTLE_ENDIAN\r\n", fh); - // fputs("Content-MD5: {md5_hash}\r\n", fh); - fputs(fmt::format("X-Binary-Number-of-Elements: {:d}\r\n", npixel).c_str(), fh); - fputs(fmt::format("X-Binary-Size-Fastest-Dimension: {:d}\r\n", data_msg.image.GetWidth()).c_str(), fh); - fputs(fmt::format("X-Binary-Size-Second-Dimension: {:d}\r\n", data_msg.image.GetHeight()).c_str(), fh); - fputs("X-Binary-Size-Padding: 0\r\n", fh); - - fputs("\x0C\x1A\x04\xD5", fh); - - fwrite(output.data(), sizeof(uint8_t), output.size(), fh); - fputs("\r\n--CIF-BINARY-FORMAT-SECTION----\r\n;\r\n\r\n",fh); - - fclose(fh); -} diff --git a/writer/CBFWriter.h b/writer/CBFWriter.h deleted file mode 100644 index 4158f49c..00000000 --- a/writer/CBFWriter.h +++ /dev/null @@ -1,22 +0,0 @@ -// SPDX-FileCopyrightText: 2024 Filip Leonarski, Paul Scherrer Institute -// SPDX-License-Identifier: GPL-3.0-only - -#pragma once - -#include "../common/JFJochMessages.h" - -class CBFWriter { - StartMessage start_msg; - const int64_t saturation_value; - std::string header; - std::vector pixel_mask; - - template - void EncodeCBF(const T *image, size_t num_elem, std::vector &output); -public: - CBFWriter(StartMessage start_msg); - void WriteImage(const DataMessage& data_msg); -}; - - - diff --git a/writer/CMakeLists.txt b/writer/CMakeLists.txt index d9222b83..766d3b8d 100644 --- a/writer/CMakeLists.txt +++ b/writer/CMakeLists.txt @@ -29,8 +29,6 @@ ADD_LIBRARY(JFJochWriter STATIC HDF5DataFilePluginReflection.h HDF5DataFilePluginPerformance.cpp HDF5DataFilePluginPerformance.h - CBFWriter.cpp - CBFWriter.h ) TARGET_LINK_LIBRARIES(JFJochWriter JFJochPreview JFJochLogger JFJochHDF5Wrappers CBORStream2FrameSerialize) diff --git a/writer/FileWriter.cpp b/writer/FileWriter.cpp index 1c5d6508..316a69b5 100644 --- a/writer/FileWriter.cpp +++ b/writer/FileWriter.cpp @@ -8,7 +8,6 @@ #include "../common/CheckPath.h" #include "../common/Logger.h" #include "../common/JFJochException.h" -#include "../preview/JFJochTIFF.h" #include "JFJochDecompress.h" FileWriter::FileWriter(const StartMessage &request, bool check_overwrite_at_start, bool trusted_path) @@ -33,9 +32,6 @@ FileWriter::FileWriter(const StartMessage &request, bool check_overwrite_at_star case FileWriterFormat::NXmxIntegrated: CreateHDF5MasterFile(request); break; - case FileWriterFormat::CBF: - cbf_writer = std::make_unique(start_message); - break; default: // Do nothing break; @@ -51,24 +47,12 @@ void FileWriter::Write(const DataMessage &msg) { case FileWriterFormat::NXmxIntegrated: WriteHDF5(msg); break; - case FileWriterFormat::CBF: - cbf_writer->WriteImage(msg); - break; - case FileWriterFormat::TIFF: - WriteTIFF(msg); - break; case FileWriterFormat::NoFile: // Do nothing break; } } -void FileWriter::WriteTIFF(const DataMessage &msg) { - const std::string file_name = fmt::format("{:s}{:06d}.tiff", start_message.file_prefix, msg.number); - WriteTIFFToFile(file_name,msg.image); -} - - void FileWriter::WriteHDF5(const DataMessage& msg) { std::lock_guard lock(hdf5_mutex); if (msg.image.GetCompressedSize() == 0) diff --git a/writer/FileWriter.h b/writer/FileWriter.h index ae3e5bd4..338b0bae 100644 --- a/writer/FileWriter.h +++ b/writer/FileWriter.h @@ -9,7 +9,6 @@ #include "../common/JFJochMessages.h" #include "../common/ZMQWrappers.h" #include "HDF5NXmx.h" -#include "CBFWriter.h" #include class FileWriter { @@ -20,7 +19,6 @@ class FileWriter { std::vector > files; std::vector stats; std::unique_ptr finalized_file_socket; - std::unique_ptr cbf_writer; std::unordered_set closed_files; constexpr static uint64_t close_file_lag_images = 1000; @@ -45,7 +43,6 @@ public: explicit FileWriter(const StartMessage &request, bool check_overwrite_at_start = true, bool trusted_path = false); void Write(const DataMessage& msg); - void WriteTIFF(const DataMessage& msg); void WriteHDF5(const DataMessage& msg); void WriteHDF5(const CompressedImage& msg); void WriteHDF5(const EndMessage& msg);