added command fformatoptions for hdf5 filters

This commit is contained in:
ext-muller_m committed 2026-11-12 12:29:56 +01:00
1 parent 06b5713155
commit 4466ea3dbd
23 files changed
+597 -56

No files matched your search

-6
View File
@@ -221,12 +221,6 @@ else()
endif()
option(SLS_USE_HDF5 "HDF5 File format" OFF)
set(SLS_HDF5_FILTER_ID "0" CACHE STRING "HDF5 filter ID (0 disables)")
set(SLS_HDF5_FILTER_PARAMS "" CACHE STRING "HDF5 filter params, comma-separated")
string(REPLACE "," ";" SLS_HDF5_FILTER_PARAMS "${SLS_HDF5_FILTER_PARAMS}") # mmh ugly but oke - in case people use it with commas in the command line
list(LENGTH SLS_HDF5_FILTER_PARAMS NUM_FILTER_PARAMS)
string(REPLACE ";" "," SLS_HDF5_FILTER_PARAMS_CPP "${SLS_HDF5_FILTER_PARAMS}")
message(STATUS "HDF5 FILTER PARAMETERS: ${SLS_HDF5_FILTER_PARAMS_CPP}")
option(SLS_BUILD_SHARED_LIBRARIES "Build shared libaries" OFF)
option(SLS_USE_TEXTCLIENT "Text Client" ON)
option(SLS_USE_DETECTOR "Detector libs" ON)
+32
View File
@@ -27,6 +27,38 @@ Setup
sls_detector_put fformat hdf5
Filters (Compression)
----------------------
HDF5 filters (eg. compression) for the image dataset in the data files are set using command `fformatoptions <commandline.html#term-fformatoptions-filter-id-param-1-param-2-...-or-none>`_ after setting the file format to hdf5. The default is no filters.
* Each filter is the HDF5 filter id followed by its parameters (cd_values), all unsigned integers.
* Multiple filters are applied in the given order.
* Filters that are not part of the HDF5 library (eg. 32008 for bitshuffle) must be available to the receiver as plugins found in **HDF5_PLUGIN_PATH**. Setting a filter that is not available to the receiver will throw an error.
* The same filter plugins must be available to the application reading the data files.
* Changing the file format resets the filters.
* List of existing filter plugins and information on writing custom filters: `HDF5 Link <https://github.com/HDFGroup/hdf5_plugins/blob/master/docs/RegisteredFilterPlugins.md>`_.
.. code-block:: bash
# gzip (filter id 1) with compression level 6
sls_detector_put fformatoptions 1 6
# shuffle (filter id 2) followed by gzip
sls_detector_put fformatoptions [2] [1,6]
# no filters
sls_detector_put fformatoptions none
# get filters
sls_detector_get fformatoptions
Master File
-------------
+27
View File
@@ -1080,6 +1080,33 @@ class Detector(CppDetectorApi):
def fformat(self, format):
ut.set_using_dict(self.setFileFormat, format)
@property
@element
def fformatoptions(self):
"""
Options specific to the file format (fformat) of the data files written by the receiver.
Note
-----
Default is no options. They are reset when the file format is changed. \n
[HDF5] List of HDF5 filters (eg. compression) applied in the given order to the image dataset. Each filter is a list with the HDF5 filter id followed by its parameters (cd_values). Filters that are not part of the HDF5 library (eg. 32008 for bitshuffle) must be available to the receiver as plugins found in HDF5_PLUGIN_PATH. \n
[Binary] No options available.
Example
--------
>>> d.fformat = fileFormat.HDF5
>>> d.fformatoptions = [[1, 6]] # gzip with compression level 6
>>> d.fformatoptions = [[2], [1, 6]] # shuffle followed by gzip
>>> d.fformatoptions
[[2], [1, 6]]
>>> d.fformatoptions = [] # no filters
"""
return self.getFileFormatOptions()
@fformatoptions.setter
def fformatoptions(self, value):
ut.set_using_dict(self.setFileFormatOptions, value)
@property
@element
def findex(self):
@@ -2585,6 +2585,14 @@ rx_roi:
PUT:
argc: -1
fformatoptions:
is_description: true
actions:
GET:
argc: 0
PUT:
argc: -1
ratecorr:
is_description: true
actions:
+17 -1
View File
@@ -1052,9 +1052,25 @@ class Detector {
Result<defs::fileFormat> getFileFormat(Positions pos = {}) const;
/** default binary, Options: BINARY, HDF5 (library must be compiled with
* this option) */
* this option). Changing the file format resets the file format options.
*/
void setFileFormat(defs::fileFormat f, Positions pos = {});
Result<std::vector<std::vector<uint32_t>>>
getFileFormatOptions(Positions pos = {}) const;
/** Options specific to the current file format. Default is no options.
* They are reset when the file format is changed. \n
* [HDF5] List of HDF5 filters (eg. compression) applied in the given
* order to the image dataset. Each filter is a list with the HDF5 filter
* id followed by its parameters (cd_values). eg. {{2}, {1, 6}} for
* shuffle followed by gzip with compression level 6. Filters that are
* not part of the HDF5 library must be available as plugins to the
* receiver (HDF5_PLUGIN_PATH). \n
* [Binary] No options available. */
void setFileFormatOptions(const std::vector<std::vector<uint32_t>> &options,
Positions pos = {});
Result<std::string> getFilePath(Positions pos = {}) const;
/** Default is "/". If path does not exist and fwrite enabled, it will try
+170
View File
@@ -5,7 +5,9 @@
#include "sls/logger.h"
#include "sls/string_utils.h"
#include <cctype>
#include <iostream>
#include <limits>
#include <thread>
namespace sls {
@@ -883,6 +885,174 @@ defs::ROI Caller::parseRoi(const std::vector<std::string> &parts) {
return roi;
}
namespace {
bool isListSeparator(char c) {
return std::isspace(static_cast<unsigned char>(c)) || c == ',';
}
/** Strict conversion, as std::stoul accepts signs and trailing characters */
uint32_t parseUnsignedInt(const std::string &s) {
const std::string error =
"Could not convert '" + s + "' to a 32 bit unsigned integer";
if (s.empty() || !std::isdigit(static_cast<unsigned char>(s[0]))) {
throw RuntimeError(error);
}
const int base = s.find("0x") != std::string::npos ? 16 : 10;
size_t pos = 0;
unsigned long long value = 0;
try {
value = std::stoull(s, &pos, base);
} catch (const std::exception &) {
throw RuntimeError(error);
}
if (pos != s.size() || value > std::numeric_limits<uint32_t>::max()) {
throw RuntimeError(error);
}
return static_cast<uint32_t>(value);
}
/** Space or comma separated list of unsigned integers */
std::vector<uint32_t> parseUnsignedIntList(const std::string &list) {
std::vector<uint32_t> values;
std::string item;
for (char c : list) {
if (!isListSeparator(c)) {
item += c;
} else if (!item.empty()) {
values.push_back(parseUnsignedInt(item));
item.clear();
}
}
if (!item.empty()) {
values.push_back(parseUnsignedInt(item));
}
return values;
}
/**
* Parses the arguments of fformatoptions to a list of options, where each
* option is a list of unsigned integers.
* Without brackets all the arguments are a single option: 1 6
* With brackets every bracket pair is an option: [2] [1,6]
* The format printed by the get is accepted as well: [[2], [1, 6]]
*/
std::vector<std::vector<uint32_t>>
parseFileFormatOptions(const std::vector<std::string> &args) {
const std::string input = join(args, ' ');
std::vector<std::vector<uint32_t>> options;
if (input.find_first_of("[]") == std::string::npos) {
options.push_back(parseUnsignedIntList(input));
return options;
}
std::string list;
bool inList = false;
int depth = 0;
for (char c : input) {
// only separators allowed between the bracket pairs of the options
const bool separator = isListSeparator(c) || c == ';';
if (c == '[') {
if (++depth > 2) {
throw RuntimeError("Too many nested brackets");
}
if (!list.empty()) {
throw RuntimeError("Found '" + list +
"' in front of a nested bracket");
}
inList = true;
} else if (c == ']') {
if (--depth < 0) {
throw RuntimeError("Mismatched brackets");
}
if (inList) {
options.push_back(parseUnsignedIntList(list));
list.clear();
inList = false;
}
} else if (inList && (!separator || !list.empty())) {
list += c;
} else if (!separator) {
throw RuntimeError(std::string("Found '") + c +
"' outside of brackets");
}
}
if (depth != 0) {
throw RuntimeError("Mismatched brackets");
}
// [] or [[]] is an empty list of options
if (options.size() == 1 && options[0].empty()) {
options.clear();
}
return options;
}
} // namespace
std::string Caller::fformatoptions(int action) {
std::ostringstream os;
std::string helpMessage =
std::string("[filter id] [param 1] [param 2]... or [none]\n") +
"\tOptions specific to the file format (fformat) of the data files "
"written by the receiver. Default is none.\n"
"\tThey are reset to none when the file format is changed.\n\n"
"\t[hdf5] List of HDF5 filters (eg. compression) applied in the "
"given order to the image dataset.\n"
"\tA filter is the HDF5 filter id followed by its parameters "
"(cd_values), all unsigned integers.\n"
"\tFor example, to set a single filter (gzip with compression level "
"6): 1 6\n\n"
"\tTo specify multiple filters, use square brackets between filters "
"and commas inside for each filter. \n"
"\tInside each bracket, no spaces allowed.\n"
"\tIf you use semicolon (along with '['and ']' to separate filters), \n"
"\tenclose the entire list in quotes.\n"
"\tExamples (shuffle followed by gzip):\n"
"\t [2] [1,6]\n"
"\t \"[2];[1,6]\"\n\n"
"\tNotes:\n"
"\t- Filters that are not part of the HDF5 library (eg. 32008 for "
"bitshuffle) must be available to the receiver as plugins found in "
"HDF5_PLUGIN_PATH.\n"
"\t- Use 'none' to remove all filters.\n\n"
"\t[binary] No options available.\n";
if (action == defs::HELP_ACTION) {
os << helpMessage;
} else if (action == defs::GET_ACTION) {
if (!args.empty()) {
WrongNumberOfParameters(0);
}
auto t = det->getFileFormatOptions(std::vector<int>{det_id});
os << OutString(t) << '\n';
} else if (action == defs::PUT_ACTION) {
if (args.empty()) {
WrongNumberOfParameters(1);
}
std::vector<std::vector<uint32_t>> t;
// 'none' option already covered as t is empty by default
if (args.size() != 1 || args[0] != "none") {
try {
t = parseFileFormatOptions(args);
} catch (const std::exception &e) {
throw RuntimeError(
"Could not parse file format options: " +
std::string(e.what()) + ". Use sls_detector_help " + cmd +
" to get the right syntax expected.");
}
}
det->setFileFormatOptions(t, std::vector<int>{det_id});
os << ToString(t) << '\n';
} else {
throw RuntimeError("Unknown action");
}
return os.str();
}
std::string Caller::ratecorr(int action) {
std::ostringstream os;
if (action == defs::HELP_ACTION) {
+10
View File
@@ -1498,6 +1498,16 @@ void Detector::setFileFormat(defs::fileFormat f, Positions pos) {
pimpl->Parallel(&Module::setFileFormat, pos, f);
}
Result<std::vector<std::vector<uint32_t>>>
Detector::getFileFormatOptions(Positions pos) const {
return pimpl->Parallel(&Module::getFileFormatOptions, pos);
}
void Detector::setFileFormatOptions(
const std::vector<std::vector<uint32_t>> &options, Positions pos) {
pimpl->Parallel(&Module::setFileFormatOptions, pos, options);
}
Result<std::string> Detector::getFilePath(Positions pos) const {
return pimpl->Parallel(&Module::getFilePath, pos);
}
+18
View File
@@ -1570,6 +1570,24 @@ void Module::setFileFormat(fileFormat f) {
sendToReceiver(F_SET_RECEIVER_FILE_FORMAT, f, nullptr);
}
std::vector<std::vector<uint32_t>> Module::getFileFormatOptions() const {
// sent as a single list with every option preceded by its size
return unflattenSizePrefixed(
sendToReceiverVarVector<uint32_t>(F_GET_RECEIVER_FILE_FORMAT_OPTIONS));
}
void Module::setFileFormatOptions(
const std::vector<std::vector<uint32_t>> &options) {
for (const auto &option : options) {
if (option.empty()) {
throw RuntimeError("File format option cannot be empty");
}
}
// sent as a single list with every option preceded by its size
sendToReceiverVarVector(F_SET_RECEIVER_FILE_FORMAT_OPTIONS,
flattenSizePrefixed(options));
}
std::string Module::getFilePath() const {
char ret[MAX_STR_LENGTH]{};
sendToReceiver(F_GET_RECEIVER_FILE_PATH, nullptr, ret);
+3
View File
@@ -325,6 +325,9 @@ class Module : public virtual slsDetectorDefs {
* ************************************************/
fileFormat getFileFormat() const;
void setFileFormat(fileFormat f);
std::vector<std::vector<uint32_t>> getFileFormatOptions() const;
void
setFileFormatOptions(const std::vector<std::vector<uint32_t>> &options);
std::string getFilePath() const;
void setFilePath(const std::string &path);
std::string getFileName() const;
@@ -988,6 +988,107 @@ TEST_CASE("fformat", "[.detectorintegration]") {
}
}
TEST_CASE("fformatoptions", "[.detectorintegration]") {
Detector det;
Caller caller(&det);
auto prev_format = det.getFileFormat();
auto prev_val = det.getFileFormatOptions();
// binary has no options
det.setFileFormat(defs::BINARY);
{
std::ostringstream oss;
caller.call("fformatoptions", {}, -1, GET, oss);
REQUIRE(oss.str() == "fformatoptions []\n");
}
{
std::ostringstream oss;
caller.call("fformatoptions", {"none"}, -1, PUT, oss);
REQUIRE(oss.str() == "fformatoptions []\n");
}
REQUIRE_THROWS(caller.call("fformatoptions", {"1", "6"}, -1, PUT));
REQUIRE_THROWS(caller.call("fformatoptions", {}, -1, PUT));
REQUIRE_THROWS(caller.call("fformatoptions", {"1"}, -1, GET));
#ifdef HDF5C
// hdf5 filters (1: gzip, 2: shuffle, 3: fletcher32 are part of hdf5 lib)
det.setFileFormat(defs::HDF5);
{
std::ostringstream oss;
caller.call("fformatoptions", {}, -1, GET, oss);
REQUIRE(oss.str() == "fformatoptions []\n");
}
{
std::ostringstream oss;
caller.call("fformatoptions", {"1", "6"}, -1, PUT, oss);
REQUIRE(oss.str() == "fformatoptions [[1, 6]]\n");
}
{
std::ostringstream oss;
caller.call("fformatoptions", {}, -1, GET, oss);
REQUIRE(oss.str() == "fformatoptions [[1, 6]]\n");
}
{
std::ostringstream oss;
caller.call("fformatoptions", {"[2]", "[1,6]"}, -1, PUT, oss);
REQUIRE(oss.str() == "fformatoptions [[2], [1, 6]]\n");
}
{
std::ostringstream oss;
caller.call("fformatoptions", {}, -1, GET, oss);
REQUIRE(oss.str() == "fformatoptions [[2], [1, 6]]\n");
}
{
std::ostringstream oss;
caller.call("fformatoptions", {"[2];[1,6];[3]"}, -1, PUT, oss);
REQUIRE(oss.str() == "fformatoptions [[2], [1, 6], [3]]\n");
}
// format printed by the get
{
std::ostringstream oss;
caller.call("fformatoptions", {"[[2],", "[1,", "6]]"}, -1, PUT, oss);
REQUIRE(oss.str() == "fformatoptions [[2], [1, 6]]\n");
}
// syntax
REQUIRE_THROWS(caller.call("fformatoptions", {"[2]", "1", "6"}, -1, PUT));
REQUIRE_THROWS(caller.call("fformatoptions", {"[2", "[1,6]"}, -1, PUT));
REQUIRE_THROWS(caller.call("fformatoptions", {"[2]", "[]"}, -1, PUT));
REQUIRE_THROWS(caller.call("fformatoptions", {"1", "-6"}, -1, PUT));
REQUIRE_THROWS(caller.call("fformatoptions", {"1", "6.5"}, -1, PUT));
REQUIRE_THROWS(caller.call("fformatoptions", {"gzip"}, -1, PUT));
// invalid filter id, repeated filter, filter not available
REQUIRE_THROWS(caller.call("fformatoptions", {"0"}, -1, PUT));
REQUIRE_THROWS(caller.call("fformatoptions", {"65536"}, -1, PUT));
REQUIRE_THROWS(caller.call("fformatoptions", {"[1,6]", "[1,4]"}, -1, PUT));
REQUIRE_THROWS(caller.call("fformatoptions", {"65535"}, -1, PUT));
// unchanged after failed attempts
{
std::ostringstream oss;
caller.call("fformatoptions", {}, -1, GET, oss);
REQUIRE(oss.str() == "fformatoptions [[2], [1, 6]]\n");
}
// changing file format resets the options
det.setFileFormat(defs::BINARY);
det.setFileFormat(defs::HDF5);
{
std::ostringstream oss;
caller.call("fformatoptions", {}, -1, GET, oss);
REQUIRE(oss.str() == "fformatoptions []\n");
}
REQUIRE_NOTHROW(caller.call("fformatoptions", {"1", "6"}, -1, PUT));
{
std::ostringstream oss;
caller.call("fformatoptions", {"none"}, -1, PUT, oss);
REQUIRE(oss.str() == "fformatoptions []\n");
}
#endif
for (int i = 0; i != det.size(); ++i) {
det.setFileFormat(prev_format[i], {i});
det.setFileFormatOptions(prev_val[i], {i});
}
}
TEST_CASE("fpath", "[.detectorintegration]") {
Detector det;
Caller caller(&det);
-4
View File
@@ -59,10 +59,6 @@ target_compile_definitions(slsReceiverObject
# HDF5
if (SLS_USE_HDF5)
if (HDF5_FOUND)
target_compile_definitions(slsReceiverObject PRIVATE
SLS_HDF5_FILTER_ID=${SLS_HDF5_FILTER_ID}
SLS_HDF5_FILTER_PARAMS=${SLS_HDF5_FILTER_PARAMS_CPP}
SLS_NUM_FILTER_PARAMS=${NUM_FILTER_PARAMS})
target_link_libraries(slsReceiverObject PUBLIC
${HDF5_LIBRARIES})
target_include_directories(slsReceiverObject PUBLIC
@@ -219,6 +219,8 @@ int ClientInterface::functionTable(){
flist[F_RECEIVER_GET_UDP_DATASTREAM] = &ClientInterface::get_port_udp_datastream;
flist[F_RECEIVER_SET_UDP_PORT_DISABLE_META] = &ClientInterface::set_udp_port_disable_meta;
flist[F_RECEIVER_GET_UDP_PORT_DISABLE_META] = &ClientInterface::get_udp_port_disable_meta;
flist[F_GET_RECEIVER_FILE_FORMAT_OPTIONS] = &ClientInterface::get_file_format_options;
flist[F_SET_RECEIVER_FILE_FORMAT_OPTIONS] = &ClientInterface::set_file_format_options;
for (int i = NUM_DET_FUNCTIONS + 1; i < NUM_REC_FUNCTIONS ; i++) {
@@ -1044,6 +1046,29 @@ int ClientInterface::get_file_format(Interface &socket) {
return socket.sendResult(retval);
}
int ClientInterface::set_file_format_options(Interface &socket) {
// received as a single list with every option preceded by its size
auto args = socket.receiveVariableArgs<std::vector<uint32_t>>();
auto options = unflattenSizePrefixed(args);
verifyIdle(socket);
LOG(logDEBUG1) << "Setting file format options:" << ToString(options);
try {
impl()->setFileFormatOptions(options);
} catch (const std::exception &e) {
throw RuntimeError("Could not set file format options to " +
ToString(options) + " [" + std::string(e.what()) +
']');
}
return socket.Send(OK);
}
int ClientInterface::get_file_format_options(Interface &socket) {
auto options = impl()->getFileFormatOptions();
LOG(logDEBUG1) << "File Format Options: " << ToString(options);
// sent as a single list with every option preceded by its size
return socket.sendVariableResult(flattenSizePrefixed(options));
}
int ClientInterface::set_streaming_port(Interface &socket) {
auto port = socket.Receive<uint16_t>();
try {
@@ -107,6 +107,8 @@ class ClientInterface : private virtual slsDetectorDefs {
int set_flip_rows(ServerInterface &socket);
int set_file_format(ServerInterface &socket);
int get_file_format(ServerInterface &socket);
int set_file_format_options(ServerInterface &socket);
int get_file_format_options(ServerInterface &socket);
int set_streaming_port(ServerInterface &socket);
int get_streaming_port(ServerInterface &socket);
int set_silent_mode(ServerInterface &socket);
+6 -7
View File
@@ -148,12 +148,11 @@ void DataProcessor::SetupFileWriter(const bool filewriteEnable,
}
}
void DataProcessor::CreateFirstFiles(const std::filesystem::path &filePath,
const std::string &fileNamePrefix,
const uint64_t fileIndex,
const bool overWriteEnable,
const bool silentMode,
const bool udpDataStream) {
void DataProcessor::CreateFirstFiles(
const std::filesystem::path &filePath, const std::string &fileNamePrefix,
const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode,
const bool udpDataStream,
const std::vector<std::vector<uint32_t>> &fileFormatOptions) {
if (dataFile == nullptr) {
throw RuntimeError("file object not contstructed");
}
@@ -181,7 +180,7 @@ void DataProcessor::CreateFirstFiles(const std::filesystem::path &filePath,
dataFile->CreateFirstHDF5DataFile(
filePath, fileNamePrefix, fileIndex, overWriteEnable, silentMode,
udpPortNumber, generalData->framesPerFile, nTotalFrames, nx, ny,
generalData->dynamicRange);
generalData->dynamicRange, fileFormatOptions);
break;
#endif
case BINARY:
+3 -1
View File
@@ -63,7 +63,9 @@ class DataProcessor : private virtual slsDetectorDefs, public ThreadObject {
void CreateFirstFiles(const std::filesystem::path &filePath,
const std::string &fileNamePrefix,
const uint64_t fileIndex, const bool overWriteEnable,
const bool silentMode, const bool udpDataStream);
const bool silentMode, const bool udpDataStream,
const std::vector<std::vector<uint32_t>>
&fileFormatOptions);
#ifdef HDF5C
uint32_t GetFilesInAcquisition() const;
std::string CreateVirtualFile(const std::filesystem::path &filePath,
+2 -1
View File
@@ -62,7 +62,8 @@ class File : private virtual slsDetectorDefs {
const bool overWriteEnable, const bool silentMode,
const uint16_t udpPortNumber, const uint32_t maxFramesPerFile,
const uint64_t numImages, const uint32_t nPixelsX,
const uint32_t nPixelsY, const uint32_t dynamicRange) {
const uint32_t nPixelsY, const uint32_t dynamicRange,
const std::vector<std::vector<uint32_t>> &filters) {
LOG(logERROR)
<< "This is a generic function CreateFirstHDF5DataFile that "
"should be overloaded by a derived class";
+55 -30
View File
@@ -2,6 +2,7 @@
// Copyright (C) 2021 Contributors to the SLS Detector Package
#include "HDF5DataFile.h"
#include "receiver_defs.h"
#include "sls/ToString.h"
#include <iomanip>
#include <sstream>
@@ -57,6 +58,40 @@ std::vector<H5::DataType> HDF5DataFile::GetParameterDataTypes() const {
slsDetectorDefs::fileFormat HDF5DataFile::GetFileFormat() const { return HDF5; }
void HDF5DataFile::ValidateFilters(
const std::vector<std::vector<uint32_t>> &filters) {
if (filters.size() > static_cast<size_t>(H5Z_MAX_NFILTERS)) {
throw RuntimeError("Too many HDF5 filters (" +
std::to_string(filters.size()) + "). Maximum is " +
std::to_string(H5Z_MAX_NFILTERS));
}
std::vector<uint32_t> ids;
for (const auto &filter : filters) {
if (filter.empty()) {
throw RuntimeError("HDF5 filter must start with the filter id");
}
const auto id = filter[0];
if (id == static_cast<uint32_t>(H5Z_FILTER_NONE) ||
id > static_cast<uint32_t>(H5Z_FILTER_MAX)) {
throw RuntimeError("Invalid HDF5 filter id " + std::to_string(id) +
". Must be between 1 and " +
std::to_string(H5Z_FILTER_MAX));
}
if (std::find(ids.begin(), ids.end(), id) != ids.end()) {
throw RuntimeError("HDF5 filter id " + std::to_string(id) +
" is used more than once");
}
ids.push_back(id);
// filters that are not part of the hdf5 library are loaded as plugins
if (H5Zfilter_avail(static_cast<H5Z_filter_t>(id)) <= 0) {
throw RuntimeError(
"HDF5 filter id " + std::to_string(id) +
" is not available in the receiver. Filters that are not "
"part of the HDF5 library must be found in HDF5_PLUGIN_PATH");
}
}
}
void HDF5DataFile::CloseFile() {
std::lock_guard<std::mutex> lock(*hdf5Lib);
try {
@@ -92,7 +127,7 @@ void HDF5DataFile::CreateFirstHDF5DataFile(
const uint64_t fIndex, const bool owEnable, const bool sMode,
const uint16_t uPortNumber, const uint32_t mFramesPerFile,
const uint64_t nImages, const uint32_t nX, const uint32_t nY,
const uint32_t dr) {
const uint32_t dr, const std::vector<std::vector<uint32_t>> &hdf5Filters) {
subFileIndex = 0;
numFramesInFile = 0;
@@ -104,6 +139,7 @@ void HDF5DataFile::CreateFirstHDF5DataFile(
nPixelsX = nX;
nPixelsY = nY;
dynamicRange = dr;
filters = hdf5Filters;
m_filePath = filePath;
fileNamePrefix = fNamePrefix;
@@ -137,7 +173,7 @@ void HDF5DataFile::CreateFile() {
'_' + std::to_string(fileIndex) + ".h5");
fileName = p.string();
std::lock_guard<std::mutex> lock(*hdf5Lib);
std::unique_lock<std::mutex> lock(*hdf5Lib);
uint64_t framestosave =
((maxFramesPerFile == 0) ? numImages : // infinite images
@@ -195,33 +231,14 @@ void HDF5DataFile::CreateFile() {
// plistPara.setFillValue(dataType, &fill_value);
plist.setChunk(DATA_RANK, dimsChunk);
// option to add generic filter to property list
/* TODO:
- as command instead of compile time ?
- allow nested filters ?
*/
if (SLS_HDF5_FILTER_ID > 0) {
// HDF5 filters are dynamically registered --> check if the filter
// is available
const auto filterId = static_cast<H5Z_filter_t>(SLS_HDF5_FILTER_ID);
if (H5Zfilter_avail(filterId) <= 0) {
throw RuntimeError("HDF5 filter ID " +
std::to_string(SLS_HDF5_FILTER_ID) +
" is not available, check HDF5_PLUGIN_PATH");
}
std::for_each(hdf5FilterParameters.begin(),
hdf5FilterParameters.end(), [](unsigned int param) {
LOG(logINFO)
<< "HDF5 filter parameter: " << param;
});
// apply filter to property list
if (hdf5FilterParameters.empty()) {
plist.setFilter(filterId, 0, 0, nullptr);
} else {
plist.setFilter(filterId, 0, hdf5FilterParameters.size(),
hdf5FilterParameters.data());
}
// filters of the image dataset, applied in the given order
// (set with the file format options, validated when set)
for (const auto &filter : filters) {
// filter id followed by its parameters
const auto filterId = static_cast<H5Z_filter_t>(filter[0]);
const size_t numParams = filter.size() - 1;
plist.setFilter(filterId, H5Z_FLAG_MANDATORY, numParams,
numParams == 0 ? nullptr : filter.data() + 1);
}
plistPara.setChunk(PARA_RANK, dimsChunkPara);
@@ -237,8 +254,16 @@ void HDF5DataFile::CreateFile() {
}
} catch (const H5::Exception &error) {
error.printErrorStack();
// CloseFile() wants the same mutex --> have to unlock here
lock.unlock();
CloseFile();
throw RuntimeError("Could not create HDF5 handles in object " + index);
std::string msg =
"Could not create HDF5 handles in object " + std::to_string(index);
if (!filters.empty()) {
msg += ". Check the file format options (HDF5 filters " +
ToString(filters) + ")";
}
throw RuntimeError(msg);
}
if (!silentMode) {
LOG(logINFO) << "[" << udpPortNumber
+12 -5
View File
@@ -3,20 +3,24 @@
#pragma once
#include "File.h"
#include <array>
#include <mutex>
namespace sls {
constexpr std::array<unsigned int, SLS_NUM_FILTER_PARAMS> hdf5FilterParameters{
SLS_HDF5_FILTER_PARAMS};
class HDF5DataFile : private virtual slsDetectorDefs, public File {
public:
HDF5DataFile(const int index, std::mutex *hdf5Lib);
~HDF5DataFile();
/**
* Throws if a filter id is invalid, used more than once or not available.
* Each filter is a list with the HDF5 filter id followed by its
* parameters (cd_values). The caller must hold the hdf5 library mutex.
*/
static void
ValidateFilters(const std::vector<std::vector<uint32_t>> &filters);
fileFormat GetFileFormat() const override;
std::string GetFileName() const override;
uint32_t GetFilesInAcquisition() const override;
@@ -32,7 +36,9 @@ class HDF5DataFile : private virtual slsDetectorDefs, public File {
const bool sMode, const uint16_t uPortNumber,
const uint32_t mFramesPerFile,
const uint64_t nImages, const uint32_t nX,
const uint32_t nY, const uint32_t dr) override;
const uint32_t nY, const uint32_t dr,
const std::vector<std::vector<uint32_t>>
&hdf5Filters) override;
void WriteToFile(char *imageData, sls_receiver_header &header,
const int imageSize, const uint64_t currentFrameNumber,
@@ -68,6 +74,7 @@ class HDF5DataFile : private virtual slsDetectorDefs, public File {
uint32_t nPixelsX{0};
uint32_t nPixelsY{0};
uint32_t dynamicRange{0};
std::vector<std::vector<uint32_t>> filters;
std::filesystem::path m_filePath{};
std::string fileNamePrefix{};
+35 -1
View File
@@ -5,6 +5,9 @@
#include "DataStreamer.h"
#include "Fifo.h"
#include "GeneralData.h"
#ifdef HDF5C
#include "HDF5DataFile.h"
#endif
#include "Listener.h"
#include "MasterAttributes.h"
#include "sls/ToString.h"
@@ -486,11 +489,41 @@ void Implementation::setFileFormat(const fileFormat f) {
}
for (const auto &it : dataProcessor)
it->SetupFileWriter(fileWriteEnable, fileFormatType, &hdf5LibMutex);
// options belong to the previous file format
if (!fileFormatOptions.empty()) {
fileFormatOptions.clear();
LOG(logINFO) << "File Format Options reset";
}
}
LOG(logINFO) << "File Format: " << ToString(fileFormatType);
}
std::vector<std::vector<uint32_t>>
Implementation::getFileFormatOptions() const {
return fileFormatOptions;
}
void Implementation::setFileFormatOptions(
const std::vector<std::vector<uint32_t>> &options) {
switch (fileFormatType) {
#ifdef HDF5C
case HDF5: {
std::lock_guard<std::mutex> lock(hdf5LibMutex);
HDF5DataFile::ValidateFilters(options);
} break;
#endif
default:
if (!options.empty()) {
throw RuntimeError("No file format options available for " +
ToString(fileFormatType) + " file format");
}
break;
}
fileFormatOptions = options;
LOG(logINFO) << "File Format Options: " << ToString(fileFormatOptions);
}
std::string Implementation::getFilePath() const { return filePath.string(); }
void Implementation::setFilePath(const std::string &c) {
@@ -910,7 +943,8 @@ void Implementation::SetupWriter() {
std::to_string(modulePos * generalData->numUDPInterfaces + i);
dataProcessor[i]->CreateFirstFiles(filePath, fileNamePrefix,
fileIndex, overwriteEnable,
silentMode, udpDataStream[i]);
silentMode, udpDataStream[i],
fileFormatOptions);
}
} catch (const RuntimeError &e) {
shutDownUDPSockets();
+3
View File
@@ -71,6 +71,8 @@ class Implementation : private virtual slsDetectorDefs {
* ************************************************/
fileFormat getFileFormat() const;
void setFileFormat(slsDetectorDefs::fileFormat f);
std::vector<std::vector<uint32_t>> getFileFormatOptions() const;
void setFileFormatOptions(const std::vector<std::vector<uint32_t>> &options);
std::string getFilePath() const;
/* check for existence */
void setFilePath(const std::string &c);
@@ -318,6 +320,7 @@ class Implementation : private virtual slsDetectorDefs {
// file parameters
fileFormat fileFormatType{BINARY};
std::vector<std::vector<uint32_t>> fileFormatOptions;
std::filesystem::path filePath{};
std::string fileName{"run"};
uint64_t fileIndex{0};
@@ -14,6 +14,7 @@
// #include <utility> //support pair in vectors
#include "sls/TypeTraits.h"
#include "sls/sls_detector_exceptions.h"
namespace sls {
@@ -193,6 +194,42 @@ stableRemoveDuplicates(T &c) {
return false;
}
/**
* @brief Flattens a list of lists into a single list, where every inner list
* is preceded by its size. Used to send a list of variable sized lists over
* the socket. Reversed by unflattenSizePrefixed.
* eg. {{2}, {1, 6}} becomes {1, 2, 2, 1, 6}
*/
template <typename T>
std::vector<T> flattenSizePrefixed(const std::vector<std::vector<T>> &lists) {
std::vector<T> flat;
for (const auto &list : lists) {
flat.push_back(static_cast<T>(list.size()));
flat.insert(flat.end(), list.begin(), list.end());
}
return flat;
}
/**
* @brief Reverses flattenSizePrefixed.
* Throws if the sizes in the flat list do not match its length.
*/
template <typename T>
std::vector<std::vector<T>> unflattenSizePrefixed(const std::vector<T> &flat) {
std::vector<std::vector<T>> lists;
size_t pos = 0;
while (pos < flat.size()) {
const auto size = static_cast<size_t>(flat[pos++]);
if (size > flat.size() - pos) {
throw RuntimeError("Could not unflatten list: size of inner list "
"exceeds the remaining number of elements");
}
lists.emplace_back(flat.begin() + pos, flat.begin() + pos + size);
pos += size;
}
return lists;
}
} // namespace sls
#endif // CONTAINER_UTILS_H
@@ -428,6 +428,8 @@ enum detFuncs {
F_RECEIVER_GET_UDP_DATASTREAM,
F_RECEIVER_SET_UDP_PORT_DISABLE_META,
F_RECEIVER_GET_UDP_PORT_DISABLE_META,
F_GET_RECEIVER_FILE_FORMAT_OPTIONS,
F_SET_RECEIVER_FILE_FORMAT_OPTIONS,
NUM_REC_FUNCTIONS
};
@@ -851,6 +853,8 @@ const char* getFunctionNameFromEnum(enum detFuncs func) {
case F_RECEIVER_GET_UDP_DATASTREAM: return "F_RECEIVER_GET_UDP_DATASTREAM";
case F_RECEIVER_SET_UDP_PORT_DISABLE_META: return "F_RECEIVER_SET_UDP_PORT_DISABLE_META";
case F_RECEIVER_GET_UDP_PORT_DISABLE_META: return "F_RECEIVER_GET_UDP_PORT_DISABLE_META";
case F_GET_RECEIVER_FILE_FORMAT_OPTIONS: return "F_GET_RECEIVER_FILE_FORMAT_OPTIONS";
case F_SET_RECEIVER_FILE_FORMAT_OPTIONS: return "F_SET_RECEIVER_FILE_FORMAT_OPTIONS";
case NUM_REC_FUNCTIONS: return "NUM_REC_FUNCTIONS";
@@ -181,6 +181,33 @@ TEST_CASE("remove duplicates but keep order, pattern ") {
CHECK(v == std::vector<int>{8, 1, 2, 3});
}
TEST_CASE("flatten list of lists with size prefix and back") {
std::vector<std::vector<uint32_t>> lists{{2}, {1, 6}, {32008, 0, 2}};
auto flat = flattenSizePrefixed(lists);
CHECK(flat == std::vector<uint32_t>{1, 2, 2, 1, 6, 3, 32008, 0, 2});
CHECK(unflattenSizePrefixed(flat) == lists);
}
TEST_CASE("flatten keeps empty inner lists") {
std::vector<std::vector<uint32_t>> lists{{}, {5}, {}};
auto flat = flattenSizePrefixed(lists);
CHECK(flat == std::vector<uint32_t>{0, 1, 5, 0});
CHECK(unflattenSizePrefixed(flat) == lists);
}
TEST_CASE("flatten empty list of lists") {
std::vector<std::vector<uint32_t>> lists;
auto flat = flattenSizePrefixed(lists);
CHECK(flat.empty());
CHECK(unflattenSizePrefixed(flat).empty());
}
TEST_CASE("unflatten throws if size exceeds remaining elements") {
CHECK_THROWS(unflattenSizePrefixed(std::vector<uint32_t>{2, 1}));
CHECK_THROWS(unflattenSizePrefixed(std::vector<uint32_t>{1, 5, 3, 1, 2}));
CHECK_THROWS(unflattenSizePrefixed(std::vector<uint32_t>{0xFFFFFFFF, 1}));
}
TEST_CASE("remove duplicated empty vector") {
std::vector<int> v;
auto r = removeDuplicates(v);