mirror of
https://github.com/slsdetectorgroup/slsDetectorPackage.git
synced 2026-11-18 17:24:04 +01:00
added command fformatoptions for hdf5 filters
This commit is contained in:
1 parent
06b5713155
commit
4466ea3dbd
23 files changed
+597
-56
No files matched your search
@@ -221,12 +221,6 @@ else()
|
||||
endif()
|
||||
|
||||
option(SLS_USE_HDF5 "HDF5 File format" OFF)
|
||||
set(SLS_HDF5_FILTER_ID "0" CACHE STRING "HDF5 filter ID (0 disables)")
|
||||
set(SLS_HDF5_FILTER_PARAMS "" CACHE STRING "HDF5 filter params, comma-separated")
|
||||
string(REPLACE "," ";" SLS_HDF5_FILTER_PARAMS "${SLS_HDF5_FILTER_PARAMS}") # mmh ugly but oke - in case people use it with commas in the command line
|
||||
list(LENGTH SLS_HDF5_FILTER_PARAMS NUM_FILTER_PARAMS)
|
||||
string(REPLACE ";" "," SLS_HDF5_FILTER_PARAMS_CPP "${SLS_HDF5_FILTER_PARAMS}")
|
||||
message(STATUS "HDF5 FILTER PARAMETERS: ${SLS_HDF5_FILTER_PARAMS_CPP}")
|
||||
option(SLS_BUILD_SHARED_LIBRARIES "Build shared libaries" OFF)
|
||||
option(SLS_USE_TEXTCLIENT "Text Client" ON)
|
||||
option(SLS_USE_DETECTOR "Detector libs" ON)
|
||||
|
||||
@@ -27,6 +27,38 @@ Setup
|
||||
sls_detector_put fformat hdf5
|
||||
|
||||
|
||||
Filters (Compression)
|
||||
----------------------
|
||||
|
||||
HDF5 filters (eg. compression) for the image dataset in the data files are set using command `fformatoptions <commandline.html#term-fformatoptions-filter-id-param-1-param-2-...-or-none>`_ after setting the file format to hdf5. The default is no filters.
|
||||
|
||||
* Each filter is the HDF5 filter id followed by its parameters (cd_values), all unsigned integers.
|
||||
|
||||
* Multiple filters are applied in the given order.
|
||||
|
||||
* Filters that are not part of the HDF5 library (eg. 32008 for bitshuffle) must be available to the receiver as plugins found in **HDF5_PLUGIN_PATH**. Setting a filter that is not available to the receiver will throw an error.
|
||||
|
||||
* The same filter plugins must be available to the application reading the data files.
|
||||
|
||||
* Changing the file format resets the filters.
|
||||
|
||||
* List of existing filter plugins and information on writing custom filters: `HDF5 Link <https://github.com/HDFGroup/hdf5_plugins/blob/master/docs/RegisteredFilterPlugins.md>`_.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
# gzip (filter id 1) with compression level 6
|
||||
sls_detector_put fformatoptions 1 6
|
||||
|
||||
# shuffle (filter id 2) followed by gzip
|
||||
sls_detector_put fformatoptions [2] [1,6]
|
||||
|
||||
# no filters
|
||||
sls_detector_put fformatoptions none
|
||||
|
||||
# get filters
|
||||
sls_detector_get fformatoptions
|
||||
|
||||
|
||||
Master File
|
||||
-------------
|
||||
|
||||
|
||||
@@ -1080,6 +1080,33 @@ class Detector(CppDetectorApi):
|
||||
def fformat(self, format):
|
||||
ut.set_using_dict(self.setFileFormat, format)
|
||||
|
||||
@property
|
||||
@element
|
||||
def fformatoptions(self):
|
||||
"""
|
||||
Options specific to the file format (fformat) of the data files written by the receiver.
|
||||
|
||||
Note
|
||||
-----
|
||||
Default is no options. They are reset when the file format is changed. \n
|
||||
[HDF5] List of HDF5 filters (eg. compression) applied in the given order to the image dataset. Each filter is a list with the HDF5 filter id followed by its parameters (cd_values). Filters that are not part of the HDF5 library (eg. 32008 for bitshuffle) must be available to the receiver as plugins found in HDF5_PLUGIN_PATH. \n
|
||||
[Binary] No options available.
|
||||
|
||||
Example
|
||||
--------
|
||||
>>> d.fformat = fileFormat.HDF5
|
||||
>>> d.fformatoptions = [[1, 6]] # gzip with compression level 6
|
||||
>>> d.fformatoptions = [[2], [1, 6]] # shuffle followed by gzip
|
||||
>>> d.fformatoptions
|
||||
[[2], [1, 6]]
|
||||
>>> d.fformatoptions = [] # no filters
|
||||
"""
|
||||
return self.getFileFormatOptions()
|
||||
|
||||
@fformatoptions.setter
|
||||
def fformatoptions(self, value):
|
||||
ut.set_using_dict(self.setFileFormatOptions, value)
|
||||
|
||||
@property
|
||||
@element
|
||||
def findex(self):
|
||||
|
||||
@@ -2585,6 +2585,14 @@ rx_roi:
|
||||
PUT:
|
||||
argc: -1
|
||||
|
||||
fformatoptions:
|
||||
is_description: true
|
||||
actions:
|
||||
GET:
|
||||
argc: 0
|
||||
PUT:
|
||||
argc: -1
|
||||
|
||||
ratecorr:
|
||||
is_description: true
|
||||
actions:
|
||||
|
||||
@@ -1052,9 +1052,25 @@ class Detector {
|
||||
Result<defs::fileFormat> getFileFormat(Positions pos = {}) const;
|
||||
|
||||
/** default binary, Options: BINARY, HDF5 (library must be compiled with
|
||||
* this option) */
|
||||
* this option). Changing the file format resets the file format options.
|
||||
*/
|
||||
void setFileFormat(defs::fileFormat f, Positions pos = {});
|
||||
|
||||
Result<std::vector<std::vector<uint32_t>>>
|
||||
getFileFormatOptions(Positions pos = {}) const;
|
||||
|
||||
/** Options specific to the current file format. Default is no options.
|
||||
* They are reset when the file format is changed. \n
|
||||
* [HDF5] List of HDF5 filters (eg. compression) applied in the given
|
||||
* order to the image dataset. Each filter is a list with the HDF5 filter
|
||||
* id followed by its parameters (cd_values). eg. {{2}, {1, 6}} for
|
||||
* shuffle followed by gzip with compression level 6. Filters that are
|
||||
* not part of the HDF5 library must be available as plugins to the
|
||||
* receiver (HDF5_PLUGIN_PATH). \n
|
||||
* [Binary] No options available. */
|
||||
void setFileFormatOptions(const std::vector<std::vector<uint32_t>> &options,
|
||||
Positions pos = {});
|
||||
|
||||
Result<std::string> getFilePath(Positions pos = {}) const;
|
||||
|
||||
/** Default is "/". If path does not exist and fwrite enabled, it will try
|
||||
|
||||
@@ -5,7 +5,9 @@
|
||||
#include "sls/logger.h"
|
||||
#include "sls/string_utils.h"
|
||||
|
||||
#include <cctype>
|
||||
#include <iostream>
|
||||
#include <limits>
|
||||
#include <thread>
|
||||
|
||||
namespace sls {
|
||||
@@ -883,6 +885,174 @@ defs::ROI Caller::parseRoi(const std::vector<std::string> &parts) {
|
||||
return roi;
|
||||
}
|
||||
|
||||
namespace {
|
||||
|
||||
bool isListSeparator(char c) {
|
||||
return std::isspace(static_cast<unsigned char>(c)) || c == ',';
|
||||
}
|
||||
|
||||
/** Strict conversion, as std::stoul accepts signs and trailing characters */
|
||||
uint32_t parseUnsignedInt(const std::string &s) {
|
||||
const std::string error =
|
||||
"Could not convert '" + s + "' to a 32 bit unsigned integer";
|
||||
if (s.empty() || !std::isdigit(static_cast<unsigned char>(s[0]))) {
|
||||
throw RuntimeError(error);
|
||||
}
|
||||
const int base = s.find("0x") != std::string::npos ? 16 : 10;
|
||||
size_t pos = 0;
|
||||
unsigned long long value = 0;
|
||||
try {
|
||||
value = std::stoull(s, &pos, base);
|
||||
} catch (const std::exception &) {
|
||||
throw RuntimeError(error);
|
||||
}
|
||||
if (pos != s.size() || value > std::numeric_limits<uint32_t>::max()) {
|
||||
throw RuntimeError(error);
|
||||
}
|
||||
return static_cast<uint32_t>(value);
|
||||
}
|
||||
|
||||
/** Space or comma separated list of unsigned integers */
|
||||
std::vector<uint32_t> parseUnsignedIntList(const std::string &list) {
|
||||
std::vector<uint32_t> values;
|
||||
std::string item;
|
||||
for (char c : list) {
|
||||
if (!isListSeparator(c)) {
|
||||
item += c;
|
||||
} else if (!item.empty()) {
|
||||
values.push_back(parseUnsignedInt(item));
|
||||
item.clear();
|
||||
}
|
||||
}
|
||||
if (!item.empty()) {
|
||||
values.push_back(parseUnsignedInt(item));
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the arguments of fformatoptions to a list of options, where each
|
||||
* option is a list of unsigned integers.
|
||||
* Without brackets all the arguments are a single option: 1 6
|
||||
* With brackets every bracket pair is an option: [2] [1,6]
|
||||
* The format printed by the get is accepted as well: [[2], [1, 6]]
|
||||
*/
|
||||
std::vector<std::vector<uint32_t>>
|
||||
parseFileFormatOptions(const std::vector<std::string> &args) {
|
||||
const std::string input = join(args, ' ');
|
||||
std::vector<std::vector<uint32_t>> options;
|
||||
if (input.find_first_of("[]") == std::string::npos) {
|
||||
options.push_back(parseUnsignedIntList(input));
|
||||
return options;
|
||||
}
|
||||
|
||||
std::string list;
|
||||
bool inList = false;
|
||||
int depth = 0;
|
||||
for (char c : input) {
|
||||
// only separators allowed between the bracket pairs of the options
|
||||
const bool separator = isListSeparator(c) || c == ';';
|
||||
if (c == '[') {
|
||||
if (++depth > 2) {
|
||||
throw RuntimeError("Too many nested brackets");
|
||||
}
|
||||
if (!list.empty()) {
|
||||
throw RuntimeError("Found '" + list +
|
||||
"' in front of a nested bracket");
|
||||
}
|
||||
inList = true;
|
||||
} else if (c == ']') {
|
||||
if (--depth < 0) {
|
||||
throw RuntimeError("Mismatched brackets");
|
||||
}
|
||||
if (inList) {
|
||||
options.push_back(parseUnsignedIntList(list));
|
||||
list.clear();
|
||||
inList = false;
|
||||
}
|
||||
} else if (inList && (!separator || !list.empty())) {
|
||||
list += c;
|
||||
} else if (!separator) {
|
||||
throw RuntimeError(std::string("Found '") + c +
|
||||
"' outside of brackets");
|
||||
}
|
||||
}
|
||||
if (depth != 0) {
|
||||
throw RuntimeError("Mismatched brackets");
|
||||
}
|
||||
// [] or [[]] is an empty list of options
|
||||
if (options.size() == 1 && options[0].empty()) {
|
||||
options.clear();
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
std::string Caller::fformatoptions(int action) {
|
||||
std::ostringstream os;
|
||||
std::string helpMessage =
|
||||
std::string("[filter id] [param 1] [param 2]... or [none]\n") +
|
||||
"\tOptions specific to the file format (fformat) of the data files "
|
||||
"written by the receiver. Default is none.\n"
|
||||
"\tThey are reset to none when the file format is changed.\n\n"
|
||||
|
||||
"\t[hdf5] List of HDF5 filters (eg. compression) applied in the "
|
||||
"given order to the image dataset.\n"
|
||||
"\tA filter is the HDF5 filter id followed by its parameters "
|
||||
"(cd_values), all unsigned integers.\n"
|
||||
"\tFor example, to set a single filter (gzip with compression level "
|
||||
"6): 1 6\n\n"
|
||||
|
||||
"\tTo specify multiple filters, use square brackets between filters "
|
||||
"and commas inside for each filter. \n"
|
||||
"\tInside each bracket, no spaces allowed.\n"
|
||||
"\tIf you use semicolon (along with '['and ']' to separate filters), \n"
|
||||
"\tenclose the entire list in quotes.\n"
|
||||
"\tExamples (shuffle followed by gzip):\n"
|
||||
"\t [2] [1,6]\n"
|
||||
"\t \"[2];[1,6]\"\n\n"
|
||||
|
||||
"\tNotes:\n"
|
||||
"\t- Filters that are not part of the HDF5 library (eg. 32008 for "
|
||||
"bitshuffle) must be available to the receiver as plugins found in "
|
||||
"HDF5_PLUGIN_PATH.\n"
|
||||
"\t- Use 'none' to remove all filters.\n\n"
|
||||
|
||||
"\t[binary] No options available.\n";
|
||||
|
||||
if (action == defs::HELP_ACTION) {
|
||||
os << helpMessage;
|
||||
} else if (action == defs::GET_ACTION) {
|
||||
if (!args.empty()) {
|
||||
WrongNumberOfParameters(0);
|
||||
}
|
||||
auto t = det->getFileFormatOptions(std::vector<int>{det_id});
|
||||
os << OutString(t) << '\n';
|
||||
} else if (action == defs::PUT_ACTION) {
|
||||
if (args.empty()) {
|
||||
WrongNumberOfParameters(1);
|
||||
}
|
||||
std::vector<std::vector<uint32_t>> t;
|
||||
// 'none' option already covered as t is empty by default
|
||||
if (args.size() != 1 || args[0] != "none") {
|
||||
try {
|
||||
t = parseFileFormatOptions(args);
|
||||
} catch (const std::exception &e) {
|
||||
throw RuntimeError(
|
||||
"Could not parse file format options: " +
|
||||
std::string(e.what()) + ". Use sls_detector_help " + cmd +
|
||||
" to get the right syntax expected.");
|
||||
}
|
||||
}
|
||||
det->setFileFormatOptions(t, std::vector<int>{det_id});
|
||||
os << ToString(t) << '\n';
|
||||
} else {
|
||||
throw RuntimeError("Unknown action");
|
||||
}
|
||||
return os.str();
|
||||
}
|
||||
|
||||
std::string Caller::ratecorr(int action) {
|
||||
std::ostringstream os;
|
||||
if (action == defs::HELP_ACTION) {
|
||||
|
||||
@@ -1498,6 +1498,16 @@ void Detector::setFileFormat(defs::fileFormat f, Positions pos) {
|
||||
pimpl->Parallel(&Module::setFileFormat, pos, f);
|
||||
}
|
||||
|
||||
Result<std::vector<std::vector<uint32_t>>>
|
||||
Detector::getFileFormatOptions(Positions pos) const {
|
||||
return pimpl->Parallel(&Module::getFileFormatOptions, pos);
|
||||
}
|
||||
|
||||
void Detector::setFileFormatOptions(
|
||||
const std::vector<std::vector<uint32_t>> &options, Positions pos) {
|
||||
pimpl->Parallel(&Module::setFileFormatOptions, pos, options);
|
||||
}
|
||||
|
||||
Result<std::string> Detector::getFilePath(Positions pos) const {
|
||||
return pimpl->Parallel(&Module::getFilePath, pos);
|
||||
}
|
||||
|
||||
@@ -1570,6 +1570,24 @@ void Module::setFileFormat(fileFormat f) {
|
||||
sendToReceiver(F_SET_RECEIVER_FILE_FORMAT, f, nullptr);
|
||||
}
|
||||
|
||||
std::vector<std::vector<uint32_t>> Module::getFileFormatOptions() const {
|
||||
// sent as a single list with every option preceded by its size
|
||||
return unflattenSizePrefixed(
|
||||
sendToReceiverVarVector<uint32_t>(F_GET_RECEIVER_FILE_FORMAT_OPTIONS));
|
||||
}
|
||||
|
||||
void Module::setFileFormatOptions(
|
||||
const std::vector<std::vector<uint32_t>> &options) {
|
||||
for (const auto &option : options) {
|
||||
if (option.empty()) {
|
||||
throw RuntimeError("File format option cannot be empty");
|
||||
}
|
||||
}
|
||||
// sent as a single list with every option preceded by its size
|
||||
sendToReceiverVarVector(F_SET_RECEIVER_FILE_FORMAT_OPTIONS,
|
||||
flattenSizePrefixed(options));
|
||||
}
|
||||
|
||||
std::string Module::getFilePath() const {
|
||||
char ret[MAX_STR_LENGTH]{};
|
||||
sendToReceiver(F_GET_RECEIVER_FILE_PATH, nullptr, ret);
|
||||
|
||||
@@ -325,6 +325,9 @@ class Module : public virtual slsDetectorDefs {
|
||||
* ************************************************/
|
||||
fileFormat getFileFormat() const;
|
||||
void setFileFormat(fileFormat f);
|
||||
std::vector<std::vector<uint32_t>> getFileFormatOptions() const;
|
||||
void
|
||||
setFileFormatOptions(const std::vector<std::vector<uint32_t>> &options);
|
||||
std::string getFilePath() const;
|
||||
void setFilePath(const std::string &path);
|
||||
std::string getFileName() const;
|
||||
|
||||
@@ -988,6 +988,107 @@ TEST_CASE("fformat", "[.detectorintegration]") {
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("fformatoptions", "[.detectorintegration]") {
|
||||
Detector det;
|
||||
Caller caller(&det);
|
||||
auto prev_format = det.getFileFormat();
|
||||
auto prev_val = det.getFileFormatOptions();
|
||||
|
||||
// binary has no options
|
||||
det.setFileFormat(defs::BINARY);
|
||||
{
|
||||
std::ostringstream oss;
|
||||
caller.call("fformatoptions", {}, -1, GET, oss);
|
||||
REQUIRE(oss.str() == "fformatoptions []\n");
|
||||
}
|
||||
{
|
||||
std::ostringstream oss;
|
||||
caller.call("fformatoptions", {"none"}, -1, PUT, oss);
|
||||
REQUIRE(oss.str() == "fformatoptions []\n");
|
||||
}
|
||||
REQUIRE_THROWS(caller.call("fformatoptions", {"1", "6"}, -1, PUT));
|
||||
REQUIRE_THROWS(caller.call("fformatoptions", {}, -1, PUT));
|
||||
REQUIRE_THROWS(caller.call("fformatoptions", {"1"}, -1, GET));
|
||||
|
||||
#ifdef HDF5C
|
||||
// hdf5 filters (1: gzip, 2: shuffle, 3: fletcher32 are part of hdf5 lib)
|
||||
det.setFileFormat(defs::HDF5);
|
||||
{
|
||||
std::ostringstream oss;
|
||||
caller.call("fformatoptions", {}, -1, GET, oss);
|
||||
REQUIRE(oss.str() == "fformatoptions []\n");
|
||||
}
|
||||
{
|
||||
std::ostringstream oss;
|
||||
caller.call("fformatoptions", {"1", "6"}, -1, PUT, oss);
|
||||
REQUIRE(oss.str() == "fformatoptions [[1, 6]]\n");
|
||||
}
|
||||
{
|
||||
std::ostringstream oss;
|
||||
caller.call("fformatoptions", {}, -1, GET, oss);
|
||||
REQUIRE(oss.str() == "fformatoptions [[1, 6]]\n");
|
||||
}
|
||||
{
|
||||
std::ostringstream oss;
|
||||
caller.call("fformatoptions", {"[2]", "[1,6]"}, -1, PUT, oss);
|
||||
REQUIRE(oss.str() == "fformatoptions [[2], [1, 6]]\n");
|
||||
}
|
||||
{
|
||||
std::ostringstream oss;
|
||||
caller.call("fformatoptions", {}, -1, GET, oss);
|
||||
REQUIRE(oss.str() == "fformatoptions [[2], [1, 6]]\n");
|
||||
}
|
||||
{
|
||||
std::ostringstream oss;
|
||||
caller.call("fformatoptions", {"[2];[1,6];[3]"}, -1, PUT, oss);
|
||||
REQUIRE(oss.str() == "fformatoptions [[2], [1, 6], [3]]\n");
|
||||
}
|
||||
// format printed by the get
|
||||
{
|
||||
std::ostringstream oss;
|
||||
caller.call("fformatoptions", {"[[2],", "[1,", "6]]"}, -1, PUT, oss);
|
||||
REQUIRE(oss.str() == "fformatoptions [[2], [1, 6]]\n");
|
||||
}
|
||||
// syntax
|
||||
REQUIRE_THROWS(caller.call("fformatoptions", {"[2]", "1", "6"}, -1, PUT));
|
||||
REQUIRE_THROWS(caller.call("fformatoptions", {"[2", "[1,6]"}, -1, PUT));
|
||||
REQUIRE_THROWS(caller.call("fformatoptions", {"[2]", "[]"}, -1, PUT));
|
||||
REQUIRE_THROWS(caller.call("fformatoptions", {"1", "-6"}, -1, PUT));
|
||||
REQUIRE_THROWS(caller.call("fformatoptions", {"1", "6.5"}, -1, PUT));
|
||||
REQUIRE_THROWS(caller.call("fformatoptions", {"gzip"}, -1, PUT));
|
||||
// invalid filter id, repeated filter, filter not available
|
||||
REQUIRE_THROWS(caller.call("fformatoptions", {"0"}, -1, PUT));
|
||||
REQUIRE_THROWS(caller.call("fformatoptions", {"65536"}, -1, PUT));
|
||||
REQUIRE_THROWS(caller.call("fformatoptions", {"[1,6]", "[1,4]"}, -1, PUT));
|
||||
REQUIRE_THROWS(caller.call("fformatoptions", {"65535"}, -1, PUT));
|
||||
// unchanged after failed attempts
|
||||
{
|
||||
std::ostringstream oss;
|
||||
caller.call("fformatoptions", {}, -1, GET, oss);
|
||||
REQUIRE(oss.str() == "fformatoptions [[2], [1, 6]]\n");
|
||||
}
|
||||
// changing file format resets the options
|
||||
det.setFileFormat(defs::BINARY);
|
||||
det.setFileFormat(defs::HDF5);
|
||||
{
|
||||
std::ostringstream oss;
|
||||
caller.call("fformatoptions", {}, -1, GET, oss);
|
||||
REQUIRE(oss.str() == "fformatoptions []\n");
|
||||
}
|
||||
REQUIRE_NOTHROW(caller.call("fformatoptions", {"1", "6"}, -1, PUT));
|
||||
{
|
||||
std::ostringstream oss;
|
||||
caller.call("fformatoptions", {"none"}, -1, PUT, oss);
|
||||
REQUIRE(oss.str() == "fformatoptions []\n");
|
||||
}
|
||||
#endif
|
||||
|
||||
for (int i = 0; i != det.size(); ++i) {
|
||||
det.setFileFormat(prev_format[i], {i});
|
||||
det.setFileFormatOptions(prev_val[i], {i});
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("fpath", "[.detectorintegration]") {
|
||||
Detector det;
|
||||
Caller caller(&det);
|
||||
|
||||
@@ -59,10 +59,6 @@ target_compile_definitions(slsReceiverObject
|
||||
# HDF5
|
||||
if (SLS_USE_HDF5)
|
||||
if (HDF5_FOUND)
|
||||
target_compile_definitions(slsReceiverObject PRIVATE
|
||||
SLS_HDF5_FILTER_ID=${SLS_HDF5_FILTER_ID}
|
||||
SLS_HDF5_FILTER_PARAMS=${SLS_HDF5_FILTER_PARAMS_CPP}
|
||||
SLS_NUM_FILTER_PARAMS=${NUM_FILTER_PARAMS})
|
||||
target_link_libraries(slsReceiverObject PUBLIC
|
||||
${HDF5_LIBRARIES})
|
||||
target_include_directories(slsReceiverObject PUBLIC
|
||||
|
||||
@@ -219,6 +219,8 @@ int ClientInterface::functionTable(){
|
||||
flist[F_RECEIVER_GET_UDP_DATASTREAM] = &ClientInterface::get_port_udp_datastream;
|
||||
flist[F_RECEIVER_SET_UDP_PORT_DISABLE_META] = &ClientInterface::set_udp_port_disable_meta;
|
||||
flist[F_RECEIVER_GET_UDP_PORT_DISABLE_META] = &ClientInterface::get_udp_port_disable_meta;
|
||||
flist[F_GET_RECEIVER_FILE_FORMAT_OPTIONS] = &ClientInterface::get_file_format_options;
|
||||
flist[F_SET_RECEIVER_FILE_FORMAT_OPTIONS] = &ClientInterface::set_file_format_options;
|
||||
|
||||
|
||||
for (int i = NUM_DET_FUNCTIONS + 1; i < NUM_REC_FUNCTIONS ; i++) {
|
||||
@@ -1044,6 +1046,29 @@ int ClientInterface::get_file_format(Interface &socket) {
|
||||
return socket.sendResult(retval);
|
||||
}
|
||||
|
||||
int ClientInterface::set_file_format_options(Interface &socket) {
|
||||
// received as a single list with every option preceded by its size
|
||||
auto args = socket.receiveVariableArgs<std::vector<uint32_t>>();
|
||||
auto options = unflattenSizePrefixed(args);
|
||||
verifyIdle(socket);
|
||||
LOG(logDEBUG1) << "Setting file format options:" << ToString(options);
|
||||
try {
|
||||
impl()->setFileFormatOptions(options);
|
||||
} catch (const std::exception &e) {
|
||||
throw RuntimeError("Could not set file format options to " +
|
||||
ToString(options) + " [" + std::string(e.what()) +
|
||||
']');
|
||||
}
|
||||
return socket.Send(OK);
|
||||
}
|
||||
|
||||
int ClientInterface::get_file_format_options(Interface &socket) {
|
||||
auto options = impl()->getFileFormatOptions();
|
||||
LOG(logDEBUG1) << "File Format Options: " << ToString(options);
|
||||
// sent as a single list with every option preceded by its size
|
||||
return socket.sendVariableResult(flattenSizePrefixed(options));
|
||||
}
|
||||
|
||||
int ClientInterface::set_streaming_port(Interface &socket) {
|
||||
auto port = socket.Receive<uint16_t>();
|
||||
try {
|
||||
|
||||
@@ -107,6 +107,8 @@ class ClientInterface : private virtual slsDetectorDefs {
|
||||
int set_flip_rows(ServerInterface &socket);
|
||||
int set_file_format(ServerInterface &socket);
|
||||
int get_file_format(ServerInterface &socket);
|
||||
int set_file_format_options(ServerInterface &socket);
|
||||
int get_file_format_options(ServerInterface &socket);
|
||||
int set_streaming_port(ServerInterface &socket);
|
||||
int get_streaming_port(ServerInterface &socket);
|
||||
int set_silent_mode(ServerInterface &socket);
|
||||
|
||||
@@ -148,12 +148,11 @@ void DataProcessor::SetupFileWriter(const bool filewriteEnable,
|
||||
}
|
||||
}
|
||||
|
||||
void DataProcessor::CreateFirstFiles(const std::filesystem::path &filePath,
|
||||
const std::string &fileNamePrefix,
|
||||
const uint64_t fileIndex,
|
||||
const bool overWriteEnable,
|
||||
const bool silentMode,
|
||||
const bool udpDataStream) {
|
||||
void DataProcessor::CreateFirstFiles(
|
||||
const std::filesystem::path &filePath, const std::string &fileNamePrefix,
|
||||
const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode,
|
||||
const bool udpDataStream,
|
||||
const std::vector<std::vector<uint32_t>> &fileFormatOptions) {
|
||||
if (dataFile == nullptr) {
|
||||
throw RuntimeError("file object not contstructed");
|
||||
}
|
||||
@@ -181,7 +180,7 @@ void DataProcessor::CreateFirstFiles(const std::filesystem::path &filePath,
|
||||
dataFile->CreateFirstHDF5DataFile(
|
||||
filePath, fileNamePrefix, fileIndex, overWriteEnable, silentMode,
|
||||
udpPortNumber, generalData->framesPerFile, nTotalFrames, nx, ny,
|
||||
generalData->dynamicRange);
|
||||
generalData->dynamicRange, fileFormatOptions);
|
||||
break;
|
||||
#endif
|
||||
case BINARY:
|
||||
|
||||
@@ -63,7 +63,9 @@ class DataProcessor : private virtual slsDetectorDefs, public ThreadObject {
|
||||
void CreateFirstFiles(const std::filesystem::path &filePath,
|
||||
const std::string &fileNamePrefix,
|
||||
const uint64_t fileIndex, const bool overWriteEnable,
|
||||
const bool silentMode, const bool udpDataStream);
|
||||
const bool silentMode, const bool udpDataStream,
|
||||
const std::vector<std::vector<uint32_t>>
|
||||
&fileFormatOptions);
|
||||
#ifdef HDF5C
|
||||
uint32_t GetFilesInAcquisition() const;
|
||||
std::string CreateVirtualFile(const std::filesystem::path &filePath,
|
||||
|
||||
@@ -62,7 +62,8 @@ class File : private virtual slsDetectorDefs {
|
||||
const bool overWriteEnable, const bool silentMode,
|
||||
const uint16_t udpPortNumber, const uint32_t maxFramesPerFile,
|
||||
const uint64_t numImages, const uint32_t nPixelsX,
|
||||
const uint32_t nPixelsY, const uint32_t dynamicRange) {
|
||||
const uint32_t nPixelsY, const uint32_t dynamicRange,
|
||||
const std::vector<std::vector<uint32_t>> &filters) {
|
||||
LOG(logERROR)
|
||||
<< "This is a generic function CreateFirstHDF5DataFile that "
|
||||
"should be overloaded by a derived class";
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
// Copyright (C) 2021 Contributors to the SLS Detector Package
|
||||
#include "HDF5DataFile.h"
|
||||
#include "receiver_defs.h"
|
||||
#include "sls/ToString.h"
|
||||
|
||||
#include <iomanip>
|
||||
#include <sstream>
|
||||
@@ -57,6 +58,40 @@ std::vector<H5::DataType> HDF5DataFile::GetParameterDataTypes() const {
|
||||
|
||||
slsDetectorDefs::fileFormat HDF5DataFile::GetFileFormat() const { return HDF5; }
|
||||
|
||||
void HDF5DataFile::ValidateFilters(
|
||||
const std::vector<std::vector<uint32_t>> &filters) {
|
||||
if (filters.size() > static_cast<size_t>(H5Z_MAX_NFILTERS)) {
|
||||
throw RuntimeError("Too many HDF5 filters (" +
|
||||
std::to_string(filters.size()) + "). Maximum is " +
|
||||
std::to_string(H5Z_MAX_NFILTERS));
|
||||
}
|
||||
std::vector<uint32_t> ids;
|
||||
for (const auto &filter : filters) {
|
||||
if (filter.empty()) {
|
||||
throw RuntimeError("HDF5 filter must start with the filter id");
|
||||
}
|
||||
const auto id = filter[0];
|
||||
if (id == static_cast<uint32_t>(H5Z_FILTER_NONE) ||
|
||||
id > static_cast<uint32_t>(H5Z_FILTER_MAX)) {
|
||||
throw RuntimeError("Invalid HDF5 filter id " + std::to_string(id) +
|
||||
". Must be between 1 and " +
|
||||
std::to_string(H5Z_FILTER_MAX));
|
||||
}
|
||||
if (std::find(ids.begin(), ids.end(), id) != ids.end()) {
|
||||
throw RuntimeError("HDF5 filter id " + std::to_string(id) +
|
||||
" is used more than once");
|
||||
}
|
||||
ids.push_back(id);
|
||||
// filters that are not part of the hdf5 library are loaded as plugins
|
||||
if (H5Zfilter_avail(static_cast<H5Z_filter_t>(id)) <= 0) {
|
||||
throw RuntimeError(
|
||||
"HDF5 filter id " + std::to_string(id) +
|
||||
" is not available in the receiver. Filters that are not "
|
||||
"part of the HDF5 library must be found in HDF5_PLUGIN_PATH");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void HDF5DataFile::CloseFile() {
|
||||
std::lock_guard<std::mutex> lock(*hdf5Lib);
|
||||
try {
|
||||
@@ -92,7 +127,7 @@ void HDF5DataFile::CreateFirstHDF5DataFile(
|
||||
const uint64_t fIndex, const bool owEnable, const bool sMode,
|
||||
const uint16_t uPortNumber, const uint32_t mFramesPerFile,
|
||||
const uint64_t nImages, const uint32_t nX, const uint32_t nY,
|
||||
const uint32_t dr) {
|
||||
const uint32_t dr, const std::vector<std::vector<uint32_t>> &hdf5Filters) {
|
||||
|
||||
subFileIndex = 0;
|
||||
numFramesInFile = 0;
|
||||
@@ -104,6 +139,7 @@ void HDF5DataFile::CreateFirstHDF5DataFile(
|
||||
nPixelsX = nX;
|
||||
nPixelsY = nY;
|
||||
dynamicRange = dr;
|
||||
filters = hdf5Filters;
|
||||
|
||||
m_filePath = filePath;
|
||||
fileNamePrefix = fNamePrefix;
|
||||
@@ -137,7 +173,7 @@ void HDF5DataFile::CreateFile() {
|
||||
'_' + std::to_string(fileIndex) + ".h5");
|
||||
fileName = p.string();
|
||||
|
||||
std::lock_guard<std::mutex> lock(*hdf5Lib);
|
||||
std::unique_lock<std::mutex> lock(*hdf5Lib);
|
||||
|
||||
uint64_t framestosave =
|
||||
((maxFramesPerFile == 0) ? numImages : // infinite images
|
||||
@@ -195,33 +231,14 @@ void HDF5DataFile::CreateFile() {
|
||||
// plistPara.setFillValue(dataType, &fill_value);
|
||||
plist.setChunk(DATA_RANK, dimsChunk);
|
||||
|
||||
// option to add generic filter to property list
|
||||
/* TODO:
|
||||
- as command instead of compile time ?
|
||||
- allow nested filters ?
|
||||
*/
|
||||
if (SLS_HDF5_FILTER_ID > 0) {
|
||||
// HDF5 filters are dynamically registered --> check if the filter
|
||||
// is available
|
||||
const auto filterId = static_cast<H5Z_filter_t>(SLS_HDF5_FILTER_ID);
|
||||
if (H5Zfilter_avail(filterId) <= 0) {
|
||||
throw RuntimeError("HDF5 filter ID " +
|
||||
std::to_string(SLS_HDF5_FILTER_ID) +
|
||||
" is not available, check HDF5_PLUGIN_PATH");
|
||||
}
|
||||
|
||||
std::for_each(hdf5FilterParameters.begin(),
|
||||
hdf5FilterParameters.end(), [](unsigned int param) {
|
||||
LOG(logINFO)
|
||||
<< "HDF5 filter parameter: " << param;
|
||||
});
|
||||
// apply filter to property list
|
||||
if (hdf5FilterParameters.empty()) {
|
||||
plist.setFilter(filterId, 0, 0, nullptr);
|
||||
} else {
|
||||
plist.setFilter(filterId, 0, hdf5FilterParameters.size(),
|
||||
hdf5FilterParameters.data());
|
||||
}
|
||||
// filters of the image dataset, applied in the given order
|
||||
// (set with the file format options, validated when set)
|
||||
for (const auto &filter : filters) {
|
||||
// filter id followed by its parameters
|
||||
const auto filterId = static_cast<H5Z_filter_t>(filter[0]);
|
||||
const size_t numParams = filter.size() - 1;
|
||||
plist.setFilter(filterId, H5Z_FLAG_MANDATORY, numParams,
|
||||
numParams == 0 ? nullptr : filter.data() + 1);
|
||||
}
|
||||
plistPara.setChunk(PARA_RANK, dimsChunkPara);
|
||||
|
||||
@@ -237,8 +254,16 @@ void HDF5DataFile::CreateFile() {
|
||||
}
|
||||
} catch (const H5::Exception &error) {
|
||||
error.printErrorStack();
|
||||
// CloseFile() wants the same mutex --> have to unlock here
|
||||
lock.unlock();
|
||||
CloseFile();
|
||||
throw RuntimeError("Could not create HDF5 handles in object " + index);
|
||||
std::string msg =
|
||||
"Could not create HDF5 handles in object " + std::to_string(index);
|
||||
if (!filters.empty()) {
|
||||
msg += ". Check the file format options (HDF5 filters " +
|
||||
ToString(filters) + ")";
|
||||
}
|
||||
throw RuntimeError(msg);
|
||||
}
|
||||
if (!silentMode) {
|
||||
LOG(logINFO) << "[" << udpPortNumber
|
||||
|
||||
@@ -3,20 +3,24 @@
|
||||
#pragma once
|
||||
|
||||
#include "File.h"
|
||||
#include <array>
|
||||
#include <mutex>
|
||||
|
||||
namespace sls {
|
||||
|
||||
constexpr std::array<unsigned int, SLS_NUM_FILTER_PARAMS> hdf5FilterParameters{
|
||||
SLS_HDF5_FILTER_PARAMS};
|
||||
|
||||
class HDF5DataFile : private virtual slsDetectorDefs, public File {
|
||||
|
||||
public:
|
||||
HDF5DataFile(const int index, std::mutex *hdf5Lib);
|
||||
~HDF5DataFile();
|
||||
|
||||
/**
|
||||
* Throws if a filter id is invalid, used more than once or not available.
|
||||
* Each filter is a list with the HDF5 filter id followed by its
|
||||
* parameters (cd_values). The caller must hold the hdf5 library mutex.
|
||||
*/
|
||||
static void
|
||||
ValidateFilters(const std::vector<std::vector<uint32_t>> &filters);
|
||||
|
||||
fileFormat GetFileFormat() const override;
|
||||
std::string GetFileName() const override;
|
||||
uint32_t GetFilesInAcquisition() const override;
|
||||
@@ -32,7 +36,9 @@ class HDF5DataFile : private virtual slsDetectorDefs, public File {
|
||||
const bool sMode, const uint16_t uPortNumber,
|
||||
const uint32_t mFramesPerFile,
|
||||
const uint64_t nImages, const uint32_t nX,
|
||||
const uint32_t nY, const uint32_t dr) override;
|
||||
const uint32_t nY, const uint32_t dr,
|
||||
const std::vector<std::vector<uint32_t>>
|
||||
&hdf5Filters) override;
|
||||
|
||||
void WriteToFile(char *imageData, sls_receiver_header &header,
|
||||
const int imageSize, const uint64_t currentFrameNumber,
|
||||
@@ -68,6 +74,7 @@ class HDF5DataFile : private virtual slsDetectorDefs, public File {
|
||||
uint32_t nPixelsX{0};
|
||||
uint32_t nPixelsY{0};
|
||||
uint32_t dynamicRange{0};
|
||||
std::vector<std::vector<uint32_t>> filters;
|
||||
|
||||
std::filesystem::path m_filePath{};
|
||||
std::string fileNamePrefix{};
|
||||
|
||||
@@ -5,6 +5,9 @@
|
||||
#include "DataStreamer.h"
|
||||
#include "Fifo.h"
|
||||
#include "GeneralData.h"
|
||||
#ifdef HDF5C
|
||||
#include "HDF5DataFile.h"
|
||||
#endif
|
||||
#include "Listener.h"
|
||||
#include "MasterAttributes.h"
|
||||
#include "sls/ToString.h"
|
||||
@@ -486,11 +489,41 @@ void Implementation::setFileFormat(const fileFormat f) {
|
||||
}
|
||||
for (const auto &it : dataProcessor)
|
||||
it->SetupFileWriter(fileWriteEnable, fileFormatType, &hdf5LibMutex);
|
||||
// options belong to the previous file format
|
||||
if (!fileFormatOptions.empty()) {
|
||||
fileFormatOptions.clear();
|
||||
LOG(logINFO) << "File Format Options reset";
|
||||
}
|
||||
}
|
||||
|
||||
LOG(logINFO) << "File Format: " << ToString(fileFormatType);
|
||||
}
|
||||
|
||||
std::vector<std::vector<uint32_t>>
|
||||
Implementation::getFileFormatOptions() const {
|
||||
return fileFormatOptions;
|
||||
}
|
||||
|
||||
void Implementation::setFileFormatOptions(
|
||||
const std::vector<std::vector<uint32_t>> &options) {
|
||||
switch (fileFormatType) {
|
||||
#ifdef HDF5C
|
||||
case HDF5: {
|
||||
std::lock_guard<std::mutex> lock(hdf5LibMutex);
|
||||
HDF5DataFile::ValidateFilters(options);
|
||||
} break;
|
||||
#endif
|
||||
default:
|
||||
if (!options.empty()) {
|
||||
throw RuntimeError("No file format options available for " +
|
||||
ToString(fileFormatType) + " file format");
|
||||
}
|
||||
break;
|
||||
}
|
||||
fileFormatOptions = options;
|
||||
LOG(logINFO) << "File Format Options: " << ToString(fileFormatOptions);
|
||||
}
|
||||
|
||||
std::string Implementation::getFilePath() const { return filePath.string(); }
|
||||
|
||||
void Implementation::setFilePath(const std::string &c) {
|
||||
@@ -910,7 +943,8 @@ void Implementation::SetupWriter() {
|
||||
std::to_string(modulePos * generalData->numUDPInterfaces + i);
|
||||
dataProcessor[i]->CreateFirstFiles(filePath, fileNamePrefix,
|
||||
fileIndex, overwriteEnable,
|
||||
silentMode, udpDataStream[i]);
|
||||
silentMode, udpDataStream[i],
|
||||
fileFormatOptions);
|
||||
}
|
||||
} catch (const RuntimeError &e) {
|
||||
shutDownUDPSockets();
|
||||
|
||||
@@ -71,6 +71,8 @@ class Implementation : private virtual slsDetectorDefs {
|
||||
* ************************************************/
|
||||
fileFormat getFileFormat() const;
|
||||
void setFileFormat(slsDetectorDefs::fileFormat f);
|
||||
std::vector<std::vector<uint32_t>> getFileFormatOptions() const;
|
||||
void setFileFormatOptions(const std::vector<std::vector<uint32_t>> &options);
|
||||
std::string getFilePath() const;
|
||||
/* check for existence */
|
||||
void setFilePath(const std::string &c);
|
||||
@@ -318,6 +320,7 @@ class Implementation : private virtual slsDetectorDefs {
|
||||
|
||||
// file parameters
|
||||
fileFormat fileFormatType{BINARY};
|
||||
std::vector<std::vector<uint32_t>> fileFormatOptions;
|
||||
std::filesystem::path filePath{};
|
||||
std::string fileName{"run"};
|
||||
uint64_t fileIndex{0};
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
// #include <utility> //support pair in vectors
|
||||
|
||||
#include "sls/TypeTraits.h"
|
||||
#include "sls/sls_detector_exceptions.h"
|
||||
|
||||
namespace sls {
|
||||
|
||||
@@ -193,6 +194,42 @@ stableRemoveDuplicates(T &c) {
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Flattens a list of lists into a single list, where every inner list
|
||||
* is preceded by its size. Used to send a list of variable sized lists over
|
||||
* the socket. Reversed by unflattenSizePrefixed.
|
||||
* eg. {{2}, {1, 6}} becomes {1, 2, 2, 1, 6}
|
||||
*/
|
||||
template <typename T>
|
||||
std::vector<T> flattenSizePrefixed(const std::vector<std::vector<T>> &lists) {
|
||||
std::vector<T> flat;
|
||||
for (const auto &list : lists) {
|
||||
flat.push_back(static_cast<T>(list.size()));
|
||||
flat.insert(flat.end(), list.begin(), list.end());
|
||||
}
|
||||
return flat;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Reverses flattenSizePrefixed.
|
||||
* Throws if the sizes in the flat list do not match its length.
|
||||
*/
|
||||
template <typename T>
|
||||
std::vector<std::vector<T>> unflattenSizePrefixed(const std::vector<T> &flat) {
|
||||
std::vector<std::vector<T>> lists;
|
||||
size_t pos = 0;
|
||||
while (pos < flat.size()) {
|
||||
const auto size = static_cast<size_t>(flat[pos++]);
|
||||
if (size > flat.size() - pos) {
|
||||
throw RuntimeError("Could not unflatten list: size of inner list "
|
||||
"exceeds the remaining number of elements");
|
||||
}
|
||||
lists.emplace_back(flat.begin() + pos, flat.begin() + pos + size);
|
||||
pos += size;
|
||||
}
|
||||
return lists;
|
||||
}
|
||||
|
||||
} // namespace sls
|
||||
|
||||
#endif // CONTAINER_UTILS_H
|
||||
@@ -428,6 +428,8 @@ enum detFuncs {
|
||||
F_RECEIVER_GET_UDP_DATASTREAM,
|
||||
F_RECEIVER_SET_UDP_PORT_DISABLE_META,
|
||||
F_RECEIVER_GET_UDP_PORT_DISABLE_META,
|
||||
F_GET_RECEIVER_FILE_FORMAT_OPTIONS,
|
||||
F_SET_RECEIVER_FILE_FORMAT_OPTIONS,
|
||||
|
||||
NUM_REC_FUNCTIONS
|
||||
};
|
||||
@@ -851,6 +853,8 @@ const char* getFunctionNameFromEnum(enum detFuncs func) {
|
||||
case F_RECEIVER_GET_UDP_DATASTREAM: return "F_RECEIVER_GET_UDP_DATASTREAM";
|
||||
case F_RECEIVER_SET_UDP_PORT_DISABLE_META: return "F_RECEIVER_SET_UDP_PORT_DISABLE_META";
|
||||
case F_RECEIVER_GET_UDP_PORT_DISABLE_META: return "F_RECEIVER_GET_UDP_PORT_DISABLE_META";
|
||||
case F_GET_RECEIVER_FILE_FORMAT_OPTIONS: return "F_GET_RECEIVER_FILE_FORMAT_OPTIONS";
|
||||
case F_SET_RECEIVER_FILE_FORMAT_OPTIONS: return "F_SET_RECEIVER_FILE_FORMAT_OPTIONS";
|
||||
|
||||
|
||||
case NUM_REC_FUNCTIONS: return "NUM_REC_FUNCTIONS";
|
||||
|
||||
@@ -181,6 +181,33 @@ TEST_CASE("remove duplicates but keep order, pattern ") {
|
||||
CHECK(v == std::vector<int>{8, 1, 2, 3});
|
||||
}
|
||||
|
||||
TEST_CASE("flatten list of lists with size prefix and back") {
|
||||
std::vector<std::vector<uint32_t>> lists{{2}, {1, 6}, {32008, 0, 2}};
|
||||
auto flat = flattenSizePrefixed(lists);
|
||||
CHECK(flat == std::vector<uint32_t>{1, 2, 2, 1, 6, 3, 32008, 0, 2});
|
||||
CHECK(unflattenSizePrefixed(flat) == lists);
|
||||
}
|
||||
|
||||
TEST_CASE("flatten keeps empty inner lists") {
|
||||
std::vector<std::vector<uint32_t>> lists{{}, {5}, {}};
|
||||
auto flat = flattenSizePrefixed(lists);
|
||||
CHECK(flat == std::vector<uint32_t>{0, 1, 5, 0});
|
||||
CHECK(unflattenSizePrefixed(flat) == lists);
|
||||
}
|
||||
|
||||
TEST_CASE("flatten empty list of lists") {
|
||||
std::vector<std::vector<uint32_t>> lists;
|
||||
auto flat = flattenSizePrefixed(lists);
|
||||
CHECK(flat.empty());
|
||||
CHECK(unflattenSizePrefixed(flat).empty());
|
||||
}
|
||||
|
||||
TEST_CASE("unflatten throws if size exceeds remaining elements") {
|
||||
CHECK_THROWS(unflattenSizePrefixed(std::vector<uint32_t>{2, 1}));
|
||||
CHECK_THROWS(unflattenSizePrefixed(std::vector<uint32_t>{1, 5, 3, 1, 2}));
|
||||
CHECK_THROWS(unflattenSizePrefixed(std::vector<uint32_t>{0xFFFFFFFF, 1}));
|
||||
}
|
||||
|
||||
TEST_CASE("remove duplicated empty vector") {
|
||||
std::vector<int> v;
|
||||
auto r = removeDuplicates(v);
|
||||
|
||||
Reference in new issue
Block a user