diff --git a/CMakeLists.txt b/CMakeLists.txt index adc5e223a..41ee30021 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -221,12 +221,6 @@ else() endif() option(SLS_USE_HDF5 "HDF5 File format" OFF) -set(SLS_HDF5_FILTER_ID "0" CACHE STRING "HDF5 filter ID (0 disables)") -set(SLS_HDF5_FILTER_PARAMS "" CACHE STRING "HDF5 filter params, comma-separated") -string(REPLACE "," ";" SLS_HDF5_FILTER_PARAMS "${SLS_HDF5_FILTER_PARAMS}") # mmh ugly but oke - in case people use it with commas in the command line -list(LENGTH SLS_HDF5_FILTER_PARAMS NUM_FILTER_PARAMS) -string(REPLACE ";" "," SLS_HDF5_FILTER_PARAMS_CPP "${SLS_HDF5_FILTER_PARAMS}") -message(STATUS "HDF5 FILTER PARAMETERS: ${SLS_HDF5_FILTER_PARAMS_CPP}") option(SLS_BUILD_SHARED_LIBRARIES "Build shared libaries" OFF) option(SLS_USE_TEXTCLIENT "Text Client" ON) option(SLS_USE_DETECTOR "Detector libs" ON) diff --git a/docs/src/hdf5fileformat.rst b/docs/src/hdf5fileformat.rst index baff1970f..64243adf2 100644 --- a/docs/src/hdf5fileformat.rst +++ b/docs/src/hdf5fileformat.rst @@ -27,6 +27,38 @@ Setup sls_detector_put fformat hdf5 +Filters (Compression) +---------------------- + +HDF5 filters (eg. compression) for the image dataset in the data files are set using command `fformatoptions `_ after setting the file format to hdf5. The default is no filters. + +* Each filter is the HDF5 filter id followed by its parameters (cd_values), all unsigned integers. + +* Multiple filters are applied in the given order. + +* Filters that are not part of the HDF5 library (eg. 32008 for bitshuffle) must be available to the receiver as plugins found in **HDF5_PLUGIN_PATH**. Setting a filter that is not available to the receiver will throw an error. + +* The same filter plugins must be available to the application reading the data files. + +* Changing the file format resets the filters. + +* List of existing filter plugins and information on writing custom filters: `HDF5 Link `_. + +.. code-block:: bash + + # gzip (filter id 1) with compression level 6 + sls_detector_put fformatoptions 1 6 + + # shuffle (filter id 2) followed by gzip + sls_detector_put fformatoptions [2] [1,6] + + # no filters + sls_detector_put fformatoptions none + + # get filters + sls_detector_get fformatoptions + + Master File ------------- diff --git a/python/slsdet/detector.py b/python/slsdet/detector.py index 581de9337..b72a8deec 100755 --- a/python/slsdet/detector.py +++ b/python/slsdet/detector.py @@ -1080,6 +1080,33 @@ class Detector(CppDetectorApi): def fformat(self, format): ut.set_using_dict(self.setFileFormat, format) + @property + @element + def fformatoptions(self): + """ + Options specific to the file format (fformat) of the data files written by the receiver. + + Note + ----- + Default is no options. They are reset when the file format is changed. \n + [HDF5] List of HDF5 filters (eg. compression) applied in the given order to the image dataset. Each filter is a list with the HDF5 filter id followed by its parameters (cd_values). Filters that are not part of the HDF5 library (eg. 32008 for bitshuffle) must be available to the receiver as plugins found in HDF5_PLUGIN_PATH. \n + [Binary] No options available. + + Example + -------- + >>> d.fformat = fileFormat.HDF5 + >>> d.fformatoptions = [[1, 6]] # gzip with compression level 6 + >>> d.fformatoptions = [[2], [1, 6]] # shuffle followed by gzip + >>> d.fformatoptions + [[2], [1, 6]] + >>> d.fformatoptions = [] # no filters + """ + return self.getFileFormatOptions() + + @fformatoptions.setter + def fformatoptions(self, value): + ut.set_using_dict(self.setFileFormatOptions, value) + @property @element def findex(self): diff --git a/slsDetectorSoftware/generator/commands.yaml b/slsDetectorSoftware/generator/commands.yaml index d9dd5deba..e9b3e5620 100644 --- a/slsDetectorSoftware/generator/commands.yaml +++ b/slsDetectorSoftware/generator/commands.yaml @@ -2585,6 +2585,14 @@ rx_roi: PUT: argc: -1 +fformatoptions: + is_description: true + actions: + GET: + argc: 0 + PUT: + argc: -1 + ratecorr: is_description: true actions: diff --git a/slsDetectorSoftware/include/sls/Detector.h b/slsDetectorSoftware/include/sls/Detector.h index 7ddd5680c..3ae2e7b5c 100644 --- a/slsDetectorSoftware/include/sls/Detector.h +++ b/slsDetectorSoftware/include/sls/Detector.h @@ -1052,9 +1052,25 @@ class Detector { Result getFileFormat(Positions pos = {}) const; /** default binary, Options: BINARY, HDF5 (library must be compiled with - * this option) */ + * this option). Changing the file format resets the file format options. + */ void setFileFormat(defs::fileFormat f, Positions pos = {}); + Result>> + getFileFormatOptions(Positions pos = {}) const; + + /** Options specific to the current file format. Default is no options. + * They are reset when the file format is changed. \n + * [HDF5] List of HDF5 filters (eg. compression) applied in the given + * order to the image dataset. Each filter is a list with the HDF5 filter + * id followed by its parameters (cd_values). eg. {{2}, {1, 6}} for + * shuffle followed by gzip with compression level 6. Filters that are + * not part of the HDF5 library must be available as plugins to the + * receiver (HDF5_PLUGIN_PATH). \n + * [Binary] No options available. */ + void setFileFormatOptions(const std::vector> &options, + Positions pos = {}); + Result getFilePath(Positions pos = {}) const; /** Default is "/". If path does not exist and fwrite enabled, it will try diff --git a/slsDetectorSoftware/src/CallerSpecial.cpp b/slsDetectorSoftware/src/CallerSpecial.cpp index e7d7977d2..c58d79d46 100644 --- a/slsDetectorSoftware/src/CallerSpecial.cpp +++ b/slsDetectorSoftware/src/CallerSpecial.cpp @@ -5,7 +5,9 @@ #include "sls/logger.h" #include "sls/string_utils.h" +#include #include +#include #include namespace sls { @@ -883,6 +885,174 @@ defs::ROI Caller::parseRoi(const std::vector &parts) { return roi; } +namespace { + +bool isListSeparator(char c) { + return std::isspace(static_cast(c)) || c == ','; +} + +/** Strict conversion, as std::stoul accepts signs and trailing characters */ +uint32_t parseUnsignedInt(const std::string &s) { + const std::string error = + "Could not convert '" + s + "' to a 32 bit unsigned integer"; + if (s.empty() || !std::isdigit(static_cast(s[0]))) { + throw RuntimeError(error); + } + const int base = s.find("0x") != std::string::npos ? 16 : 10; + size_t pos = 0; + unsigned long long value = 0; + try { + value = std::stoull(s, &pos, base); + } catch (const std::exception &) { + throw RuntimeError(error); + } + if (pos != s.size() || value > std::numeric_limits::max()) { + throw RuntimeError(error); + } + return static_cast(value); +} + +/** Space or comma separated list of unsigned integers */ +std::vector parseUnsignedIntList(const std::string &list) { + std::vector values; + std::string item; + for (char c : list) { + if (!isListSeparator(c)) { + item += c; + } else if (!item.empty()) { + values.push_back(parseUnsignedInt(item)); + item.clear(); + } + } + if (!item.empty()) { + values.push_back(parseUnsignedInt(item)); + } + return values; +} + +/** + * Parses the arguments of fformatoptions to a list of options, where each + * option is a list of unsigned integers. + * Without brackets all the arguments are a single option: 1 6 + * With brackets every bracket pair is an option: [2] [1,6] + * The format printed by the get is accepted as well: [[2], [1, 6]] + */ +std::vector> +parseFileFormatOptions(const std::vector &args) { + const std::string input = join(args, ' '); + std::vector> options; + if (input.find_first_of("[]") == std::string::npos) { + options.push_back(parseUnsignedIntList(input)); + return options; + } + + std::string list; + bool inList = false; + int depth = 0; + for (char c : input) { + // only separators allowed between the bracket pairs of the options + const bool separator = isListSeparator(c) || c == ';'; + if (c == '[') { + if (++depth > 2) { + throw RuntimeError("Too many nested brackets"); + } + if (!list.empty()) { + throw RuntimeError("Found '" + list + + "' in front of a nested bracket"); + } + inList = true; + } else if (c == ']') { + if (--depth < 0) { + throw RuntimeError("Mismatched brackets"); + } + if (inList) { + options.push_back(parseUnsignedIntList(list)); + list.clear(); + inList = false; + } + } else if (inList && (!separator || !list.empty())) { + list += c; + } else if (!separator) { + throw RuntimeError(std::string("Found '") + c + + "' outside of brackets"); + } + } + if (depth != 0) { + throw RuntimeError("Mismatched brackets"); + } + // [] or [[]] is an empty list of options + if (options.size() == 1 && options[0].empty()) { + options.clear(); + } + return options; +} + +} // namespace + +std::string Caller::fformatoptions(int action) { + std::ostringstream os; + std::string helpMessage = + std::string("[filter id] [param 1] [param 2]... or [none]\n") + + "\tOptions specific to the file format (fformat) of the data files " + "written by the receiver. Default is none.\n" + "\tThey are reset to none when the file format is changed.\n\n" + + "\t[hdf5] List of HDF5 filters (eg. compression) applied in the " + "given order to the image dataset.\n" + "\tA filter is the HDF5 filter id followed by its parameters " + "(cd_values), all unsigned integers.\n" + "\tFor example, to set a single filter (gzip with compression level " + "6): 1 6\n\n" + + "\tTo specify multiple filters, use square brackets between filters " + "and commas inside for each filter. \n" + "\tInside each bracket, no spaces allowed.\n" + "\tIf you use semicolon (along with '['and ']' to separate filters), \n" + "\tenclose the entire list in quotes.\n" + "\tExamples (shuffle followed by gzip):\n" + "\t [2] [1,6]\n" + "\t \"[2];[1,6]\"\n\n" + + "\tNotes:\n" + "\t- Filters that are not part of the HDF5 library (eg. 32008 for " + "bitshuffle) must be available to the receiver as plugins found in " + "HDF5_PLUGIN_PATH.\n" + "\t- Use 'none' to remove all filters.\n\n" + + "\t[binary] No options available.\n"; + + if (action == defs::HELP_ACTION) { + os << helpMessage; + } else if (action == defs::GET_ACTION) { + if (!args.empty()) { + WrongNumberOfParameters(0); + } + auto t = det->getFileFormatOptions(std::vector{det_id}); + os << OutString(t) << '\n'; + } else if (action == defs::PUT_ACTION) { + if (args.empty()) { + WrongNumberOfParameters(1); + } + std::vector> t; + // 'none' option already covered as t is empty by default + if (args.size() != 1 || args[0] != "none") { + try { + t = parseFileFormatOptions(args); + } catch (const std::exception &e) { + throw RuntimeError( + "Could not parse file format options: " + + std::string(e.what()) + ". Use sls_detector_help " + cmd + + " to get the right syntax expected."); + } + } + det->setFileFormatOptions(t, std::vector{det_id}); + os << ToString(t) << '\n'; + } else { + throw RuntimeError("Unknown action"); + } + return os.str(); +} + std::string Caller::ratecorr(int action) { std::ostringstream os; if (action == defs::HELP_ACTION) { diff --git a/slsDetectorSoftware/src/Detector.cpp b/slsDetectorSoftware/src/Detector.cpp index 58f88d836..0baacc735 100644 --- a/slsDetectorSoftware/src/Detector.cpp +++ b/slsDetectorSoftware/src/Detector.cpp @@ -1498,6 +1498,16 @@ void Detector::setFileFormat(defs::fileFormat f, Positions pos) { pimpl->Parallel(&Module::setFileFormat, pos, f); } +Result>> +Detector::getFileFormatOptions(Positions pos) const { + return pimpl->Parallel(&Module::getFileFormatOptions, pos); +} + +void Detector::setFileFormatOptions( + const std::vector> &options, Positions pos) { + pimpl->Parallel(&Module::setFileFormatOptions, pos, options); +} + Result Detector::getFilePath(Positions pos) const { return pimpl->Parallel(&Module::getFilePath, pos); } diff --git a/slsDetectorSoftware/src/Module.cpp b/slsDetectorSoftware/src/Module.cpp index 49475057e..9030673c7 100644 --- a/slsDetectorSoftware/src/Module.cpp +++ b/slsDetectorSoftware/src/Module.cpp @@ -1570,6 +1570,24 @@ void Module::setFileFormat(fileFormat f) { sendToReceiver(F_SET_RECEIVER_FILE_FORMAT, f, nullptr); } +std::vector> Module::getFileFormatOptions() const { + // sent as a single list with every option preceded by its size + return unflattenSizePrefixed( + sendToReceiverVarVector(F_GET_RECEIVER_FILE_FORMAT_OPTIONS)); +} + +void Module::setFileFormatOptions( + const std::vector> &options) { + for (const auto &option : options) { + if (option.empty()) { + throw RuntimeError("File format option cannot be empty"); + } + } + // sent as a single list with every option preceded by its size + sendToReceiverVarVector(F_SET_RECEIVER_FILE_FORMAT_OPTIONS, + flattenSizePrefixed(options)); +} + std::string Module::getFilePath() const { char ret[MAX_STR_LENGTH]{}; sendToReceiver(F_GET_RECEIVER_FILE_PATH, nullptr, ret); diff --git a/slsDetectorSoftware/src/Module.h b/slsDetectorSoftware/src/Module.h index 3228671a9..63c9e98dd 100644 --- a/slsDetectorSoftware/src/Module.h +++ b/slsDetectorSoftware/src/Module.h @@ -325,6 +325,9 @@ class Module : public virtual slsDetectorDefs { * ************************************************/ fileFormat getFileFormat() const; void setFileFormat(fileFormat f); + std::vector> getFileFormatOptions() const; + void + setFileFormatOptions(const std::vector> &options); std::string getFilePath() const; void setFilePath(const std::string &path); std::string getFileName() const; diff --git a/slsDetectorSoftware/tests/Caller/test-Caller-rx.cpp b/slsDetectorSoftware/tests/Caller/test-Caller-rx.cpp index fa0f0d443..530fcf1f6 100644 --- a/slsDetectorSoftware/tests/Caller/test-Caller-rx.cpp +++ b/slsDetectorSoftware/tests/Caller/test-Caller-rx.cpp @@ -988,6 +988,107 @@ TEST_CASE("fformat", "[.detectorintegration]") { } } +TEST_CASE("fformatoptions", "[.detectorintegration]") { + Detector det; + Caller caller(&det); + auto prev_format = det.getFileFormat(); + auto prev_val = det.getFileFormatOptions(); + + // binary has no options + det.setFileFormat(defs::BINARY); + { + std::ostringstream oss; + caller.call("fformatoptions", {}, -1, GET, oss); + REQUIRE(oss.str() == "fformatoptions []\n"); + } + { + std::ostringstream oss; + caller.call("fformatoptions", {"none"}, -1, PUT, oss); + REQUIRE(oss.str() == "fformatoptions []\n"); + } + REQUIRE_THROWS(caller.call("fformatoptions", {"1", "6"}, -1, PUT)); + REQUIRE_THROWS(caller.call("fformatoptions", {}, -1, PUT)); + REQUIRE_THROWS(caller.call("fformatoptions", {"1"}, -1, GET)); + +#ifdef HDF5C + // hdf5 filters (1: gzip, 2: shuffle, 3: fletcher32 are part of hdf5 lib) + det.setFileFormat(defs::HDF5); + { + std::ostringstream oss; + caller.call("fformatoptions", {}, -1, GET, oss); + REQUIRE(oss.str() == "fformatoptions []\n"); + } + { + std::ostringstream oss; + caller.call("fformatoptions", {"1", "6"}, -1, PUT, oss); + REQUIRE(oss.str() == "fformatoptions [[1, 6]]\n"); + } + { + std::ostringstream oss; + caller.call("fformatoptions", {}, -1, GET, oss); + REQUIRE(oss.str() == "fformatoptions [[1, 6]]\n"); + } + { + std::ostringstream oss; + caller.call("fformatoptions", {"[2]", "[1,6]"}, -1, PUT, oss); + REQUIRE(oss.str() == "fformatoptions [[2], [1, 6]]\n"); + } + { + std::ostringstream oss; + caller.call("fformatoptions", {}, -1, GET, oss); + REQUIRE(oss.str() == "fformatoptions [[2], [1, 6]]\n"); + } + { + std::ostringstream oss; + caller.call("fformatoptions", {"[2];[1,6];[3]"}, -1, PUT, oss); + REQUIRE(oss.str() == "fformatoptions [[2], [1, 6], [3]]\n"); + } + // format printed by the get + { + std::ostringstream oss; + caller.call("fformatoptions", {"[[2],", "[1,", "6]]"}, -1, PUT, oss); + REQUIRE(oss.str() == "fformatoptions [[2], [1, 6]]\n"); + } + // syntax + REQUIRE_THROWS(caller.call("fformatoptions", {"[2]", "1", "6"}, -1, PUT)); + REQUIRE_THROWS(caller.call("fformatoptions", {"[2", "[1,6]"}, -1, PUT)); + REQUIRE_THROWS(caller.call("fformatoptions", {"[2]", "[]"}, -1, PUT)); + REQUIRE_THROWS(caller.call("fformatoptions", {"1", "-6"}, -1, PUT)); + REQUIRE_THROWS(caller.call("fformatoptions", {"1", "6.5"}, -1, PUT)); + REQUIRE_THROWS(caller.call("fformatoptions", {"gzip"}, -1, PUT)); + // invalid filter id, repeated filter, filter not available + REQUIRE_THROWS(caller.call("fformatoptions", {"0"}, -1, PUT)); + REQUIRE_THROWS(caller.call("fformatoptions", {"65536"}, -1, PUT)); + REQUIRE_THROWS(caller.call("fformatoptions", {"[1,6]", "[1,4]"}, -1, PUT)); + REQUIRE_THROWS(caller.call("fformatoptions", {"65535"}, -1, PUT)); + // unchanged after failed attempts + { + std::ostringstream oss; + caller.call("fformatoptions", {}, -1, GET, oss); + REQUIRE(oss.str() == "fformatoptions [[2], [1, 6]]\n"); + } + // changing file format resets the options + det.setFileFormat(defs::BINARY); + det.setFileFormat(defs::HDF5); + { + std::ostringstream oss; + caller.call("fformatoptions", {}, -1, GET, oss); + REQUIRE(oss.str() == "fformatoptions []\n"); + } + REQUIRE_NOTHROW(caller.call("fformatoptions", {"1", "6"}, -1, PUT)); + { + std::ostringstream oss; + caller.call("fformatoptions", {"none"}, -1, PUT, oss); + REQUIRE(oss.str() == "fformatoptions []\n"); + } +#endif + + for (int i = 0; i != det.size(); ++i) { + det.setFileFormat(prev_format[i], {i}); + det.setFileFormatOptions(prev_val[i], {i}); + } +} + TEST_CASE("fpath", "[.detectorintegration]") { Detector det; Caller caller(&det); diff --git a/slsReceiverSoftware/CMakeLists.txt b/slsReceiverSoftware/CMakeLists.txt index d68b36f12..8c00cd8eb 100755 --- a/slsReceiverSoftware/CMakeLists.txt +++ b/slsReceiverSoftware/CMakeLists.txt @@ -59,10 +59,6 @@ target_compile_definitions(slsReceiverObject # HDF5 if (SLS_USE_HDF5) if (HDF5_FOUND) - target_compile_definitions(slsReceiverObject PRIVATE - SLS_HDF5_FILTER_ID=${SLS_HDF5_FILTER_ID} - SLS_HDF5_FILTER_PARAMS=${SLS_HDF5_FILTER_PARAMS_CPP} - SLS_NUM_FILTER_PARAMS=${NUM_FILTER_PARAMS}) target_link_libraries(slsReceiverObject PUBLIC ${HDF5_LIBRARIES}) target_include_directories(slsReceiverObject PUBLIC diff --git a/slsReceiverSoftware/src/ClientInterface.cpp b/slsReceiverSoftware/src/ClientInterface.cpp index 9cc70d535..08167efd0 100644 --- a/slsReceiverSoftware/src/ClientInterface.cpp +++ b/slsReceiverSoftware/src/ClientInterface.cpp @@ -219,6 +219,8 @@ int ClientInterface::functionTable(){ flist[F_RECEIVER_GET_UDP_DATASTREAM] = &ClientInterface::get_port_udp_datastream; flist[F_RECEIVER_SET_UDP_PORT_DISABLE_META] = &ClientInterface::set_udp_port_disable_meta; flist[F_RECEIVER_GET_UDP_PORT_DISABLE_META] = &ClientInterface::get_udp_port_disable_meta; + flist[F_GET_RECEIVER_FILE_FORMAT_OPTIONS] = &ClientInterface::get_file_format_options; + flist[F_SET_RECEIVER_FILE_FORMAT_OPTIONS] = &ClientInterface::set_file_format_options; for (int i = NUM_DET_FUNCTIONS + 1; i < NUM_REC_FUNCTIONS ; i++) { @@ -1044,6 +1046,29 @@ int ClientInterface::get_file_format(Interface &socket) { return socket.sendResult(retval); } +int ClientInterface::set_file_format_options(Interface &socket) { + // received as a single list with every option preceded by its size + auto args = socket.receiveVariableArgs>(); + auto options = unflattenSizePrefixed(args); + verifyIdle(socket); + LOG(logDEBUG1) << "Setting file format options:" << ToString(options); + try { + impl()->setFileFormatOptions(options); + } catch (const std::exception &e) { + throw RuntimeError("Could not set file format options to " + + ToString(options) + " [" + std::string(e.what()) + + ']'); + } + return socket.Send(OK); +} + +int ClientInterface::get_file_format_options(Interface &socket) { + auto options = impl()->getFileFormatOptions(); + LOG(logDEBUG1) << "File Format Options: " << ToString(options); + // sent as a single list with every option preceded by its size + return socket.sendVariableResult(flattenSizePrefixed(options)); +} + int ClientInterface::set_streaming_port(Interface &socket) { auto port = socket.Receive(); try { diff --git a/slsReceiverSoftware/src/ClientInterface.h b/slsReceiverSoftware/src/ClientInterface.h index 07ba77a7d..1b3b87192 100644 --- a/slsReceiverSoftware/src/ClientInterface.h +++ b/slsReceiverSoftware/src/ClientInterface.h @@ -107,6 +107,8 @@ class ClientInterface : private virtual slsDetectorDefs { int set_flip_rows(ServerInterface &socket); int set_file_format(ServerInterface &socket); int get_file_format(ServerInterface &socket); + int set_file_format_options(ServerInterface &socket); + int get_file_format_options(ServerInterface &socket); int set_streaming_port(ServerInterface &socket); int get_streaming_port(ServerInterface &socket); int set_silent_mode(ServerInterface &socket); diff --git a/slsReceiverSoftware/src/DataProcessor.cpp b/slsReceiverSoftware/src/DataProcessor.cpp index f6359274c..4ca3f431d 100644 --- a/slsReceiverSoftware/src/DataProcessor.cpp +++ b/slsReceiverSoftware/src/DataProcessor.cpp @@ -148,12 +148,11 @@ void DataProcessor::SetupFileWriter(const bool filewriteEnable, } } -void DataProcessor::CreateFirstFiles(const std::filesystem::path &filePath, - const std::string &fileNamePrefix, - const uint64_t fileIndex, - const bool overWriteEnable, - const bool silentMode, - const bool udpDataStream) { +void DataProcessor::CreateFirstFiles( + const std::filesystem::path &filePath, const std::string &fileNamePrefix, + const uint64_t fileIndex, const bool overWriteEnable, const bool silentMode, + const bool udpDataStream, + const std::vector> &fileFormatOptions) { if (dataFile == nullptr) { throw RuntimeError("file object not contstructed"); } @@ -181,7 +180,7 @@ void DataProcessor::CreateFirstFiles(const std::filesystem::path &filePath, dataFile->CreateFirstHDF5DataFile( filePath, fileNamePrefix, fileIndex, overWriteEnable, silentMode, udpPortNumber, generalData->framesPerFile, nTotalFrames, nx, ny, - generalData->dynamicRange); + generalData->dynamicRange, fileFormatOptions); break; #endif case BINARY: diff --git a/slsReceiverSoftware/src/DataProcessor.h b/slsReceiverSoftware/src/DataProcessor.h index 0f9dadaef..0ce9943aa 100644 --- a/slsReceiverSoftware/src/DataProcessor.h +++ b/slsReceiverSoftware/src/DataProcessor.h @@ -63,7 +63,9 @@ class DataProcessor : private virtual slsDetectorDefs, public ThreadObject { void CreateFirstFiles(const std::filesystem::path &filePath, const std::string &fileNamePrefix, const uint64_t fileIndex, const bool overWriteEnable, - const bool silentMode, const bool udpDataStream); + const bool silentMode, const bool udpDataStream, + const std::vector> + &fileFormatOptions); #ifdef HDF5C uint32_t GetFilesInAcquisition() const; std::string CreateVirtualFile(const std::filesystem::path &filePath, diff --git a/slsReceiverSoftware/src/File.h b/slsReceiverSoftware/src/File.h index 18bc32496..54b754481 100644 --- a/slsReceiverSoftware/src/File.h +++ b/slsReceiverSoftware/src/File.h @@ -62,7 +62,8 @@ class File : private virtual slsDetectorDefs { const bool overWriteEnable, const bool silentMode, const uint16_t udpPortNumber, const uint32_t maxFramesPerFile, const uint64_t numImages, const uint32_t nPixelsX, - const uint32_t nPixelsY, const uint32_t dynamicRange) { + const uint32_t nPixelsY, const uint32_t dynamicRange, + const std::vector> &filters) { LOG(logERROR) << "This is a generic function CreateFirstHDF5DataFile that " "should be overloaded by a derived class"; diff --git a/slsReceiverSoftware/src/HDF5DataFile.cpp b/slsReceiverSoftware/src/HDF5DataFile.cpp index b058732b4..d17e7bc5b 100644 --- a/slsReceiverSoftware/src/HDF5DataFile.cpp +++ b/slsReceiverSoftware/src/HDF5DataFile.cpp @@ -2,6 +2,7 @@ // Copyright (C) 2021 Contributors to the SLS Detector Package #include "HDF5DataFile.h" #include "receiver_defs.h" +#include "sls/ToString.h" #include #include @@ -57,6 +58,40 @@ std::vector HDF5DataFile::GetParameterDataTypes() const { slsDetectorDefs::fileFormat HDF5DataFile::GetFileFormat() const { return HDF5; } +void HDF5DataFile::ValidateFilters( + const std::vector> &filters) { + if (filters.size() > static_cast(H5Z_MAX_NFILTERS)) { + throw RuntimeError("Too many HDF5 filters (" + + std::to_string(filters.size()) + "). Maximum is " + + std::to_string(H5Z_MAX_NFILTERS)); + } + std::vector ids; + for (const auto &filter : filters) { + if (filter.empty()) { + throw RuntimeError("HDF5 filter must start with the filter id"); + } + const auto id = filter[0]; + if (id == static_cast(H5Z_FILTER_NONE) || + id > static_cast(H5Z_FILTER_MAX)) { + throw RuntimeError("Invalid HDF5 filter id " + std::to_string(id) + + ". Must be between 1 and " + + std::to_string(H5Z_FILTER_MAX)); + } + if (std::find(ids.begin(), ids.end(), id) != ids.end()) { + throw RuntimeError("HDF5 filter id " + std::to_string(id) + + " is used more than once"); + } + ids.push_back(id); + // filters that are not part of the hdf5 library are loaded as plugins + if (H5Zfilter_avail(static_cast(id)) <= 0) { + throw RuntimeError( + "HDF5 filter id " + std::to_string(id) + + " is not available in the receiver. Filters that are not " + "part of the HDF5 library must be found in HDF5_PLUGIN_PATH"); + } + } +} + void HDF5DataFile::CloseFile() { std::lock_guard lock(*hdf5Lib); try { @@ -92,7 +127,7 @@ void HDF5DataFile::CreateFirstHDF5DataFile( const uint64_t fIndex, const bool owEnable, const bool sMode, const uint16_t uPortNumber, const uint32_t mFramesPerFile, const uint64_t nImages, const uint32_t nX, const uint32_t nY, - const uint32_t dr) { + const uint32_t dr, const std::vector> &hdf5Filters) { subFileIndex = 0; numFramesInFile = 0; @@ -104,6 +139,7 @@ void HDF5DataFile::CreateFirstHDF5DataFile( nPixelsX = nX; nPixelsY = nY; dynamicRange = dr; + filters = hdf5Filters; m_filePath = filePath; fileNamePrefix = fNamePrefix; @@ -137,7 +173,7 @@ void HDF5DataFile::CreateFile() { '_' + std::to_string(fileIndex) + ".h5"); fileName = p.string(); - std::lock_guard lock(*hdf5Lib); + std::unique_lock lock(*hdf5Lib); uint64_t framestosave = ((maxFramesPerFile == 0) ? numImages : // infinite images @@ -195,33 +231,14 @@ void HDF5DataFile::CreateFile() { // plistPara.setFillValue(dataType, &fill_value); plist.setChunk(DATA_RANK, dimsChunk); - // option to add generic filter to property list - /* TODO: - - as command instead of compile time ? - - allow nested filters ? - */ - if (SLS_HDF5_FILTER_ID > 0) { - // HDF5 filters are dynamically registered --> check if the filter - // is available - const auto filterId = static_cast(SLS_HDF5_FILTER_ID); - if (H5Zfilter_avail(filterId) <= 0) { - throw RuntimeError("HDF5 filter ID " + - std::to_string(SLS_HDF5_FILTER_ID) + - " is not available, check HDF5_PLUGIN_PATH"); - } - - std::for_each(hdf5FilterParameters.begin(), - hdf5FilterParameters.end(), [](unsigned int param) { - LOG(logINFO) - << "HDF5 filter parameter: " << param; - }); - // apply filter to property list - if (hdf5FilterParameters.empty()) { - plist.setFilter(filterId, 0, 0, nullptr); - } else { - plist.setFilter(filterId, 0, hdf5FilterParameters.size(), - hdf5FilterParameters.data()); - } + // filters of the image dataset, applied in the given order + // (set with the file format options, validated when set) + for (const auto &filter : filters) { + // filter id followed by its parameters + const auto filterId = static_cast(filter[0]); + const size_t numParams = filter.size() - 1; + plist.setFilter(filterId, H5Z_FLAG_MANDATORY, numParams, + numParams == 0 ? nullptr : filter.data() + 1); } plistPara.setChunk(PARA_RANK, dimsChunkPara); @@ -237,8 +254,16 @@ void HDF5DataFile::CreateFile() { } } catch (const H5::Exception &error) { error.printErrorStack(); + // CloseFile() wants the same mutex --> have to unlock here + lock.unlock(); CloseFile(); - throw RuntimeError("Could not create HDF5 handles in object " + index); + std::string msg = + "Could not create HDF5 handles in object " + std::to_string(index); + if (!filters.empty()) { + msg += ". Check the file format options (HDF5 filters " + + ToString(filters) + ")"; + } + throw RuntimeError(msg); } if (!silentMode) { LOG(logINFO) << "[" << udpPortNumber diff --git a/slsReceiverSoftware/src/HDF5DataFile.h b/slsReceiverSoftware/src/HDF5DataFile.h index 0d9014986..b2030b5f6 100644 --- a/slsReceiverSoftware/src/HDF5DataFile.h +++ b/slsReceiverSoftware/src/HDF5DataFile.h @@ -3,20 +3,24 @@ #pragma once #include "File.h" -#include #include namespace sls { -constexpr std::array hdf5FilterParameters{ - SLS_HDF5_FILTER_PARAMS}; - class HDF5DataFile : private virtual slsDetectorDefs, public File { public: HDF5DataFile(const int index, std::mutex *hdf5Lib); ~HDF5DataFile(); + /** + * Throws if a filter id is invalid, used more than once or not available. + * Each filter is a list with the HDF5 filter id followed by its + * parameters (cd_values). The caller must hold the hdf5 library mutex. + */ + static void + ValidateFilters(const std::vector> &filters); + fileFormat GetFileFormat() const override; std::string GetFileName() const override; uint32_t GetFilesInAcquisition() const override; @@ -32,7 +36,9 @@ class HDF5DataFile : private virtual slsDetectorDefs, public File { const bool sMode, const uint16_t uPortNumber, const uint32_t mFramesPerFile, const uint64_t nImages, const uint32_t nX, - const uint32_t nY, const uint32_t dr) override; + const uint32_t nY, const uint32_t dr, + const std::vector> + &hdf5Filters) override; void WriteToFile(char *imageData, sls_receiver_header &header, const int imageSize, const uint64_t currentFrameNumber, @@ -68,6 +74,7 @@ class HDF5DataFile : private virtual slsDetectorDefs, public File { uint32_t nPixelsX{0}; uint32_t nPixelsY{0}; uint32_t dynamicRange{0}; + std::vector> filters; std::filesystem::path m_filePath{}; std::string fileNamePrefix{}; diff --git a/slsReceiverSoftware/src/Implementation.cpp b/slsReceiverSoftware/src/Implementation.cpp index 60c704578..0895594c5 100644 --- a/slsReceiverSoftware/src/Implementation.cpp +++ b/slsReceiverSoftware/src/Implementation.cpp @@ -5,6 +5,9 @@ #include "DataStreamer.h" #include "Fifo.h" #include "GeneralData.h" +#ifdef HDF5C +#include "HDF5DataFile.h" +#endif #include "Listener.h" #include "MasterAttributes.h" #include "sls/ToString.h" @@ -486,11 +489,41 @@ void Implementation::setFileFormat(const fileFormat f) { } for (const auto &it : dataProcessor) it->SetupFileWriter(fileWriteEnable, fileFormatType, &hdf5LibMutex); + // options belong to the previous file format + if (!fileFormatOptions.empty()) { + fileFormatOptions.clear(); + LOG(logINFO) << "File Format Options reset"; + } } LOG(logINFO) << "File Format: " << ToString(fileFormatType); } +std::vector> +Implementation::getFileFormatOptions() const { + return fileFormatOptions; +} + +void Implementation::setFileFormatOptions( + const std::vector> &options) { + switch (fileFormatType) { +#ifdef HDF5C + case HDF5: { + std::lock_guard lock(hdf5LibMutex); + HDF5DataFile::ValidateFilters(options); + } break; +#endif + default: + if (!options.empty()) { + throw RuntimeError("No file format options available for " + + ToString(fileFormatType) + " file format"); + } + break; + } + fileFormatOptions = options; + LOG(logINFO) << "File Format Options: " << ToString(fileFormatOptions); +} + std::string Implementation::getFilePath() const { return filePath.string(); } void Implementation::setFilePath(const std::string &c) { @@ -910,7 +943,8 @@ void Implementation::SetupWriter() { std::to_string(modulePos * generalData->numUDPInterfaces + i); dataProcessor[i]->CreateFirstFiles(filePath, fileNamePrefix, fileIndex, overwriteEnable, - silentMode, udpDataStream[i]); + silentMode, udpDataStream[i], + fileFormatOptions); } } catch (const RuntimeError &e) { shutDownUDPSockets(); diff --git a/slsReceiverSoftware/src/Implementation.h b/slsReceiverSoftware/src/Implementation.h index 88c9de797..7feb5a062 100644 --- a/slsReceiverSoftware/src/Implementation.h +++ b/slsReceiverSoftware/src/Implementation.h @@ -71,6 +71,8 @@ class Implementation : private virtual slsDetectorDefs { * ************************************************/ fileFormat getFileFormat() const; void setFileFormat(slsDetectorDefs::fileFormat f); + std::vector> getFileFormatOptions() const; + void setFileFormatOptions(const std::vector> &options); std::string getFilePath() const; /* check for existence */ void setFilePath(const std::string &c); @@ -318,6 +320,7 @@ class Implementation : private virtual slsDetectorDefs { // file parameters fileFormat fileFormatType{BINARY}; + std::vector> fileFormatOptions; std::filesystem::path filePath{}; std::string fileName{"run"}; uint64_t fileIndex{0}; diff --git a/slsSupportLib/include/sls/container_utils.h b/slsSupportLib/include/sls/container_utils.h index 13ac893f9..4905fef80 100644 --- a/slsSupportLib/include/sls/container_utils.h +++ b/slsSupportLib/include/sls/container_utils.h @@ -14,6 +14,7 @@ // #include //support pair in vectors #include "sls/TypeTraits.h" +#include "sls/sls_detector_exceptions.h" namespace sls { @@ -193,6 +194,42 @@ stableRemoveDuplicates(T &c) { return false; } +/** + * @brief Flattens a list of lists into a single list, where every inner list + * is preceded by its size. Used to send a list of variable sized lists over + * the socket. Reversed by unflattenSizePrefixed. + * eg. {{2}, {1, 6}} becomes {1, 2, 2, 1, 6} + */ +template +std::vector flattenSizePrefixed(const std::vector> &lists) { + std::vector flat; + for (const auto &list : lists) { + flat.push_back(static_cast(list.size())); + flat.insert(flat.end(), list.begin(), list.end()); + } + return flat; +} + +/** + * @brief Reverses flattenSizePrefixed. + * Throws if the sizes in the flat list do not match its length. + */ +template +std::vector> unflattenSizePrefixed(const std::vector &flat) { + std::vector> lists; + size_t pos = 0; + while (pos < flat.size()) { + const auto size = static_cast(flat[pos++]); + if (size > flat.size() - pos) { + throw RuntimeError("Could not unflatten list: size of inner list " + "exceeds the remaining number of elements"); + } + lists.emplace_back(flat.begin() + pos, flat.begin() + pos + size); + pos += size; + } + return lists; +} + } // namespace sls #endif // CONTAINER_UTILS_H diff --git a/slsSupportLib/include/sls/sls_detector_funcs.h b/slsSupportLib/include/sls/sls_detector_funcs.h index e27f76e87..d3278721e 100755 --- a/slsSupportLib/include/sls/sls_detector_funcs.h +++ b/slsSupportLib/include/sls/sls_detector_funcs.h @@ -428,6 +428,8 @@ enum detFuncs { F_RECEIVER_GET_UDP_DATASTREAM, F_RECEIVER_SET_UDP_PORT_DISABLE_META, F_RECEIVER_GET_UDP_PORT_DISABLE_META, + F_GET_RECEIVER_FILE_FORMAT_OPTIONS, + F_SET_RECEIVER_FILE_FORMAT_OPTIONS, NUM_REC_FUNCTIONS }; @@ -851,6 +853,8 @@ const char* getFunctionNameFromEnum(enum detFuncs func) { case F_RECEIVER_GET_UDP_DATASTREAM: return "F_RECEIVER_GET_UDP_DATASTREAM"; case F_RECEIVER_SET_UDP_PORT_DISABLE_META: return "F_RECEIVER_SET_UDP_PORT_DISABLE_META"; case F_RECEIVER_GET_UDP_PORT_DISABLE_META: return "F_RECEIVER_GET_UDP_PORT_DISABLE_META"; + case F_GET_RECEIVER_FILE_FORMAT_OPTIONS: return "F_GET_RECEIVER_FILE_FORMAT_OPTIONS"; + case F_SET_RECEIVER_FILE_FORMAT_OPTIONS: return "F_SET_RECEIVER_FILE_FORMAT_OPTIONS"; case NUM_REC_FUNCTIONS: return "NUM_REC_FUNCTIONS"; diff --git a/slsSupportLib/tests/test-container_utils.cpp b/slsSupportLib/tests/test-container_utils.cpp index e546662bb..c2118e563 100644 --- a/slsSupportLib/tests/test-container_utils.cpp +++ b/slsSupportLib/tests/test-container_utils.cpp @@ -181,6 +181,33 @@ TEST_CASE("remove duplicates but keep order, pattern ") { CHECK(v == std::vector{8, 1, 2, 3}); } +TEST_CASE("flatten list of lists with size prefix and back") { + std::vector> lists{{2}, {1, 6}, {32008, 0, 2}}; + auto flat = flattenSizePrefixed(lists); + CHECK(flat == std::vector{1, 2, 2, 1, 6, 3, 32008, 0, 2}); + CHECK(unflattenSizePrefixed(flat) == lists); +} + +TEST_CASE("flatten keeps empty inner lists") { + std::vector> lists{{}, {5}, {}}; + auto flat = flattenSizePrefixed(lists); + CHECK(flat == std::vector{0, 1, 5, 0}); + CHECK(unflattenSizePrefixed(flat) == lists); +} + +TEST_CASE("flatten empty list of lists") { + std::vector> lists; + auto flat = flattenSizePrefixed(lists); + CHECK(flat.empty()); + CHECK(unflattenSizePrefixed(flat).empty()); +} + +TEST_CASE("unflatten throws if size exceeds remaining elements") { + CHECK_THROWS(unflattenSizePrefixed(std::vector{2, 1})); + CHECK_THROWS(unflattenSizePrefixed(std::vector{1, 5, 3, 1, 2})); + CHECK_THROWS(unflattenSizePrefixed(std::vector{0xFFFFFFFF, 1})); +} + TEST_CASE("remove duplicated empty vector") { std::vector v; auto r = removeDuplicates(v);