mirror of
https://github.com/paulscherrerinstitute/sf_daq_buffer.git
synced 2026-06-08 05:48:51 +02:00
Rename folder of core library
This commit is contained in:
@@ -1,49 +0,0 @@
|
||||
SRC_DIR = ./src
|
||||
OBJ_DIR = ./obj
|
||||
BIN_DIR = ./bin
|
||||
MKDIR = mkdir -p
|
||||
|
||||
CC = g++
|
||||
CFLAGS = -Wall -Wfatal-errors -fPIC -pthread -std=c++11 -I./include -I${CONDA_PREFIX}/include
|
||||
LDFLAGS = -L${CONDA_PREFIX}/lib -L/usr/lib64 -lzmq -lhdf5 -lhdf5_hl -lhdf5_cpp -lhdf5_hl_cpp -lboost_system -lboost_regex -lboost_thread -lpthread
|
||||
|
||||
UNAME := $(shell uname)
|
||||
ifeq ($(UNAME), Linux)
|
||||
SOFLAGS += -shared -Wl,-soname,libcpp_h5_writer.so
|
||||
endif
|
||||
ifeq ($(UNAME), Darwin)
|
||||
SOFLAGS += -shared -Wl,-install_name,libcpp_h5_writer.so
|
||||
endif
|
||||
|
||||
HEADERS = $(wildcard $(SRC_DIR)/*.hpp)
|
||||
SRCS = $(wildcard $(SRC_DIR)/*.cpp)
|
||||
OBJS = $(patsubst $(SRC_DIR)/%.cpp, $(OBJ_DIR)/%.o, $(SRCS))
|
||||
|
||||
libcpp_h5_writer: build_dirs $(OBJS)
|
||||
$(CC) $(SOFLAGS) -o $(BIN_DIR)/libcpp_h5_writer.so $(OBJS) $(LDFLAGS)
|
||||
|
||||
debug: CFLAGS += -DDEBUG_OUTPUT
|
||||
debug: libcpp_h5_writer
|
||||
|
||||
perf: CFLAGS += -DPERF_OUTPUT
|
||||
perf: libcpp_h5_writer
|
||||
|
||||
deploy: libcpp_h5_writer
|
||||
cp bin/* ${CONDA_PREFIX}/lib
|
||||
mkdir -p ${CONDA_PREFIX}/include/cpp_h5_writer
|
||||
cp src/*.hpp ${CONDA_PREFIX}/include/cpp_h5_writer
|
||||
cp include/date.h ${CONDA_PREFIX}/include/cpp_h5_writer
|
||||
mkdir -p ${CONDA_PREFIX}/include/cpp_h5_writer/compression
|
||||
cp src/compression/*.h ${CONDA_PREFIX}/include/cpp_h5_writer/compression
|
||||
|
||||
$(OBJ_DIR)/%.o: $(SRC_DIR)/%.cpp
|
||||
$(CC) $(CFLAGS) $(LDFLAGS) -c -o $@ $<
|
||||
|
||||
build_dirs:
|
||||
$(MKDIR) $(OBJ_DIR) $(BIN_DIR)
|
||||
|
||||
clean:
|
||||
rm -rf $(OBJ_DIR) $(BIN_DIR)
|
||||
|
||||
test: build_dirs $(OBJS)
|
||||
$(CC) $(CFLAGS) test/test_main.cpp $(OBJS) $(LDFLAGS) -lgtest_main -lgtest -o $(BIN_DIR)/execute_tests
|
||||
@@ -1,7 +0,0 @@
|
||||
make
|
||||
|
||||
cp bin/* ${PREFIX}/lib
|
||||
|
||||
mkdir -p ${PREFIX}/include/cpp_h5_writer
|
||||
cp src/*.hpp ${PREFIX}/include/cpp_h5_writer
|
||||
cp include/date.h ${CONDA_PREFIX}/include/cpp_h5_writer
|
||||
@@ -1,27 +0,0 @@
|
||||
package:
|
||||
name: lib_cpp_h5_writer
|
||||
version: 0.1.0
|
||||
|
||||
build:
|
||||
number: 0
|
||||
|
||||
source:
|
||||
path: ..
|
||||
|
||||
requirements:
|
||||
build:
|
||||
- make
|
||||
- gcc
|
||||
- cppzmq ==4.2.1
|
||||
- hdf5 ==1.10.1
|
||||
- boost ==1.61.0
|
||||
|
||||
run:
|
||||
- cppzmq ==4.2.1
|
||||
- hdf5 ==1.10.1
|
||||
- boost ==1.61.0
|
||||
|
||||
|
||||
about:
|
||||
summary: "C++ library for a ZMQ to H5 writer."
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
-8319
File diff suppressed because it is too large
Load Diff
@@ -1,66 +0,0 @@
|
||||
#include <iostream>
|
||||
|
||||
#include "H5Format.hpp"
|
||||
#include "BufferedWriter.hpp"
|
||||
|
||||
|
||||
using namespace std;
|
||||
|
||||
BufferedWriter::BufferedWriter(const std::string& filename, size_t total_frames, unique_ptr<MetadataBuffer>&& metadata_buffer,
|
||||
hsize_t frames_per_file, hsize_t initial_dataset_size, hsize_t dataset_increase_step) :
|
||||
H5Writer(filename, frames_per_file, initial_dataset_size, dataset_increase_step),
|
||||
total_frames(total_frames), metadata_buffer(move(metadata_buffer))
|
||||
{
|
||||
#ifdef DEBUG_OUTPUT
|
||||
using namespace date;
|
||||
cout << "[" << std::chrono::system_clock::now() << "]";
|
||||
cout << "[BufferedWriter::BufferedWriter] Creating buffered writer";
|
||||
cout << " with filename " << filename;
|
||||
cout << " and total_frames " << total_frames;
|
||||
cout << " and frames_per_file " << frames_per_file;
|
||||
cout << " and initial_dataset_size " << initial_dataset_size;
|
||||
cout << endl;
|
||||
#endif
|
||||
}
|
||||
|
||||
void BufferedWriter::cache_metadata(string name, uint64_t frame_index, const char* data)
|
||||
{
|
||||
auto relative_frame_index = get_relative_data_index(frame_index);
|
||||
metadata_buffer->add_metadata_to_buffer(name, relative_frame_index, data);
|
||||
}
|
||||
|
||||
void BufferedWriter::write_metadata_to_file()
|
||||
{
|
||||
auto header_values_type = metadata_buffer->get_header_values_type();
|
||||
|
||||
if (header_values_type) {
|
||||
for (const auto& header_type : *header_values_type) {
|
||||
auto& dataset_name = header_type.first;
|
||||
auto& header_data_type = header_type.second;
|
||||
|
||||
vector<size_t> data_shape = {header_data_type.value_shape};
|
||||
|
||||
create_dataset(dataset_name, data_shape, header_data_type.type, header_data_type.endianness, false,
|
||||
metadata_buffer->get_n_images());
|
||||
|
||||
H5::AtomType dataset_data_type(H5FormatUtils::get_dataset_data_type(header_data_type.type));
|
||||
dataset_data_type.setOrder(H5T_ORDER_LE);
|
||||
|
||||
auto& dataset = datasets.at(dataset_name);
|
||||
dataset.write(metadata_buffer->get_metadata_values(dataset_name).get(), dataset_data_type);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::unique_ptr<BufferedWriter> get_buffered_writer(const string& filename, size_t total_frames,
|
||||
std::unique_ptr<MetadataBuffer> metadata_buffer, hsize_t frames_per_file, hsize_t dataset_increase_step)
|
||||
{
|
||||
size_t initial_dataset_size = frames_per_file != 0 ? frames_per_file : total_frames;
|
||||
|
||||
if (filename == "/dev/null") {
|
||||
return unique_ptr<BufferedWriter>(new DummyBufferedWriter());
|
||||
} else {
|
||||
return unique_ptr<BufferedWriter>(new BufferedWriter(filename, total_frames, move(metadata_buffer),
|
||||
frames_per_file, initial_dataset_size, dataset_increase_step));
|
||||
}
|
||||
}
|
||||
@@ -1,52 +0,0 @@
|
||||
#ifndef BUFFEREDWRITER_H
|
||||
#define BUFFEREDWRITER_H
|
||||
|
||||
#include "H5Writer.hpp"
|
||||
#include "MetadataBuffer.hpp"
|
||||
|
||||
class BufferedWriter : public H5Writer
|
||||
{
|
||||
size_t total_frames;
|
||||
std::unique_ptr<MetadataBuffer> metadata_buffer;
|
||||
|
||||
public:
|
||||
BufferedWriter(const std::string& filename, size_t total_frames, std::unique_ptr<MetadataBuffer>&& metadata_buffer,
|
||||
hsize_t frames_per_file=0, hsize_t initial_dataset_size=1000, hsize_t dataset_increase_step=1000);
|
||||
virtual void cache_metadata(std::string name, uint64_t frame_index, const char* data);
|
||||
virtual void write_metadata_to_file();
|
||||
};
|
||||
|
||||
class DummyBufferedWriter : public BufferedWriter, public DummyH5Writer
|
||||
{
|
||||
public:
|
||||
DummyBufferedWriter() : BufferedWriter("/dev/null", 0, 0, 0, 0) {}
|
||||
void cache_metadata(std::string name, uint64_t frame_index, const char* data) override {}
|
||||
void write_metadata_to_file() override {}
|
||||
|
||||
bool is_file_open() const override
|
||||
{ return DummyH5Writer::is_file_open(); }
|
||||
|
||||
void create_file(const hsize_t frame_chunk=1) override
|
||||
{ return DummyH5Writer::create_file(frame_chunk); }
|
||||
|
||||
void close_file() override
|
||||
{ return DummyH5Writer::close_file(); }
|
||||
|
||||
void write_data(const std::string& dataset_name, const size_t data_index, const char* data, const std::vector<size_t>& data_shape,
|
||||
const size_t data_bytes_size, const std::string& data_type, const std::string& endianness) override
|
||||
{
|
||||
return DummyH5Writer::write_data (
|
||||
dataset_name, data_index, data, data_shape, data_bytes_size, data_type, endianness );
|
||||
}
|
||||
|
||||
H5::H5File& get_h5_file() override
|
||||
{ return DummyH5Writer::get_h5_file(); }
|
||||
|
||||
bool is_data_for_current_file(const size_t data_index) override
|
||||
{ return DummyH5Writer::is_data_for_current_file(data_index); }
|
||||
};
|
||||
|
||||
std::unique_ptr<BufferedWriter> get_buffered_writer(const std::string& filename, size_t total_frames,
|
||||
std::unique_ptr<MetadataBuffer> metadata_buffer, hsize_t frames_per_file=0, hsize_t dataset_increase_step=1000);
|
||||
|
||||
#endif
|
||||
@@ -1,391 +0,0 @@
|
||||
#include <string>
|
||||
#include <sstream>
|
||||
#include <stdexcept>
|
||||
#include <iostream>
|
||||
|
||||
#include "config.hpp"
|
||||
#include "H5Format.hpp"
|
||||
|
||||
using namespace std;
|
||||
|
||||
|
||||
hsize_t H5FormatUtils::expand_dataset(H5::DataSet& dataset, hsize_t frame_index, hsize_t dataset_increase_step)
|
||||
{
|
||||
const auto& data_space = dataset.getSpace();
|
||||
|
||||
int dataset_rank = data_space.getSimpleExtentNdims();
|
||||
hsize_t dataset_dimension[dataset_rank];
|
||||
|
||||
data_space.getSimpleExtentDims(dataset_dimension);
|
||||
dataset_dimension[0] = frame_index + dataset_increase_step;
|
||||
|
||||
#ifdef DEBUG_OUTPUT
|
||||
using namespace date;
|
||||
cout << "[" << std::chrono::system_clock::now() << "]";
|
||||
cout << "[H5FormatUtils::expand_dataset] Expanding dataspace to size (";
|
||||
for (int i=0; i<dataset_rank; ++i) {
|
||||
cout << dataset_dimension[i] << ",";
|
||||
}
|
||||
cout << ")" << endl;
|
||||
#endif
|
||||
|
||||
dataset.extend(dataset_dimension);
|
||||
|
||||
return dataset_dimension[0];
|
||||
}
|
||||
|
||||
void H5FormatUtils::compact_dataset(H5::DataSet& dataset, hsize_t max_frame_index)
|
||||
{
|
||||
// Only chunked datasets can be resized.
|
||||
if (H5D_CHUNKED != dataset.getCreatePlist().getLayout()) {
|
||||
#ifdef DEBUG_OUTPUT
|
||||
using namespace date;
|
||||
cout << "[" << std::chrono::system_clock::now() << "]";
|
||||
cout << "[H5FormatUtils::compact_dataset] Not compact a contiguous dataset." << endl;
|
||||
#endif
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const auto& data_space = dataset.getSpace();
|
||||
|
||||
int dataset_rank = data_space.getSimpleExtentNdims();
|
||||
hsize_t dataset_dimension[dataset_rank];
|
||||
|
||||
data_space.getSimpleExtentDims(dataset_dimension);
|
||||
dataset_dimension[0] = max_frame_index + 1;
|
||||
|
||||
#ifdef DEBUG_OUTPUT
|
||||
using namespace date;
|
||||
cout << "[" << std::chrono::system_clock::now() << "]";
|
||||
cout << "[H5FormatUtils::compact_dataset] Compacting dataspace to size (";
|
||||
for (int i=0; i<dataset_rank; ++i) {
|
||||
cout << dataset_dimension[i] << ",";
|
||||
}
|
||||
cout << ")" << endl;
|
||||
#endif
|
||||
|
||||
dataset.extend(dataset_dimension);
|
||||
}
|
||||
|
||||
H5::Group H5FormatUtils::create_group(H5::Group& target, const string& name)
|
||||
{
|
||||
return target.createGroup(name.c_str());
|
||||
}
|
||||
|
||||
const boost::any& H5FormatUtils::get_value_from_reference(const string& dataset_name,
|
||||
const boost::any& value_reference, const unordered_map<string, boost::any>& values)
|
||||
{
|
||||
try {
|
||||
auto reference_string = boost::any_cast<string>(value_reference);
|
||||
|
||||
#ifdef DEBUG_OUTPUT
|
||||
using namespace date;
|
||||
cout << "[" << std::chrono::system_clock::now() << "]";
|
||||
cout << "[H5FormatUtils::get_value_from_reference] Getting dataset '"<< dataset_name;
|
||||
cout << "' reference value '" << reference_string << "'." << endl;
|
||||
#endif
|
||||
|
||||
return values.at(reference_string);
|
||||
|
||||
} catch (const boost::bad_any_cast& exception) {
|
||||
stringstream error_message;
|
||||
using namespace date;
|
||||
error_message << "[" << chrono::system_clock::now() << "]";
|
||||
error_message << "Cannot convert dataset " << dataset_name << " value reference to string." << endl;
|
||||
|
||||
throw runtime_error(error_message.str());
|
||||
|
||||
} catch (const out_of_range& exception){
|
||||
stringstream error_message;
|
||||
using namespace date;
|
||||
error_message << "[" << std::chrono::system_clock::now() << "]";
|
||||
error_message << "Dataset " << dataset_name << " value reference " << boost::any_cast<string>(value_reference);
|
||||
error_message << " not present in values map." << endl;
|
||||
|
||||
throw runtime_error(error_message.str());
|
||||
}
|
||||
}
|
||||
|
||||
const H5::PredType& H5FormatUtils::get_dataset_data_type(const string& type)
|
||||
{
|
||||
#ifdef DEBUG_OUTPUT
|
||||
using namespace date;
|
||||
cout << "[" << std::chrono::system_clock::now() << "]";
|
||||
cout << "[H5FormatUtils::get_dataset_data_type] Getting dataset type for received frame type " << type << endl;
|
||||
#endif
|
||||
|
||||
if (type == "uint8") {
|
||||
return H5::PredType::NATIVE_UINT8;
|
||||
|
||||
} else if (type == "uint16") {
|
||||
return H5::PredType::NATIVE_UINT16;
|
||||
|
||||
} else if (type == "uint32") {
|
||||
return H5::PredType::NATIVE_UINT32;
|
||||
|
||||
} else if (type == "uint64") {
|
||||
return H5::PredType::NATIVE_UINT64;
|
||||
|
||||
} else if (type == "int8") {
|
||||
return H5::PredType::NATIVE_INT8;
|
||||
|
||||
} else if (type == "int16") {
|
||||
return H5::PredType::NATIVE_INT16;
|
||||
|
||||
} else if (type == "int32") {
|
||||
return H5::PredType::NATIVE_INT32;
|
||||
|
||||
} else if (type == "int64") {
|
||||
return H5::PredType::NATIVE_INT64;
|
||||
|
||||
} else {
|
||||
// We cannot really convert this attribute.
|
||||
stringstream error_message;
|
||||
using namespace date;
|
||||
error_message << "[" << std::chrono::system_clock::now() << "]";
|
||||
error_message << "[H5FormatUtils::get_dataset_data_type] Unsupported dataset data_type " << type << endl;
|
||||
|
||||
throw runtime_error(error_message.str());
|
||||
}
|
||||
}
|
||||
|
||||
H5::DataSet H5FormatUtils::write_dataset(H5::Group& target, const h5_dataset& dataset,
|
||||
const unordered_map<string, boost::any>& values)
|
||||
{
|
||||
const string& name = dataset.name;
|
||||
boost::any value;
|
||||
|
||||
// Value is stored directly in the struct.
|
||||
if (dataset.data_location == IMMEDIATE){
|
||||
value = dataset.value;
|
||||
// Value in struct is just a string reference to into the values map.
|
||||
} else {
|
||||
value = H5FormatUtils::get_value_from_reference(name, dataset.value, values);
|
||||
}
|
||||
|
||||
if (dataset.data_type == NX_CHAR || dataset.data_type == NX_DATE_TIME || dataset.data_type == NXnote) {
|
||||
// Attempt to convert to const char * (string "literals" cause that).
|
||||
try {
|
||||
return H5FormatUtils::write_dataset(target, name, string(boost::any_cast<const char*>(value)));
|
||||
} catch (const boost::bad_any_cast& exception) {}
|
||||
|
||||
// Atempt to convert to string.
|
||||
try {
|
||||
return H5FormatUtils::write_dataset(target, name, boost::any_cast<string>(value));
|
||||
} catch (const boost::bad_any_cast& exception) {}
|
||||
|
||||
// We cannot really convert this attribute.
|
||||
stringstream error_message;
|
||||
using namespace date;
|
||||
error_message << "[" << std::chrono::system_clock::now() << "]";
|
||||
error_message << "Cannot convert dataset " << name << " to string or const char*." << endl;
|
||||
|
||||
throw runtime_error(error_message.str());
|
||||
|
||||
} else if (dataset.data_type == NX_INT) {
|
||||
try {
|
||||
return H5FormatUtils::write_dataset(target, name, boost::any_cast<int>(value));
|
||||
} catch (const boost::bad_any_cast& exception) {}
|
||||
|
||||
// We cannot really convert this attribute.
|
||||
stringstream error_message;
|
||||
using namespace date;
|
||||
error_message << "[" << std::chrono::system_clock::now() << "]";
|
||||
error_message << "Cannot convert dataset " << name << " to NX_INT." << endl;
|
||||
|
||||
throw runtime_error(error_message.str());
|
||||
} else if (dataset.data_type == NX_FLOAT || dataset.data_type == NX_NUMBER) {
|
||||
try {
|
||||
return H5FormatUtils::write_dataset(target, name, boost::any_cast<double>(value));
|
||||
} catch (const boost::bad_any_cast& exception) {}
|
||||
|
||||
// We cannot really convert this attribute.
|
||||
stringstream error_message;
|
||||
using namespace date;
|
||||
error_message << "[" << std::chrono::system_clock::now() << "]";
|
||||
error_message << "Cannot convert dataset " << name << " to NX_FLOAT." << endl;
|
||||
|
||||
throw runtime_error(error_message.str());
|
||||
} else {
|
||||
stringstream error_message;
|
||||
using namespace date;
|
||||
error_message << "[" << std::chrono::system_clock::now() << "]";
|
||||
error_message << "Unsupported dataset type for dataset " << name << "." << endl;
|
||||
|
||||
throw runtime_error(error_message.str());
|
||||
}
|
||||
}
|
||||
|
||||
H5::DataSet H5FormatUtils::write_dataset(H5::Group& target, const string& name, double value)
|
||||
{
|
||||
H5::DataSpace att_space(H5S_SCALAR);
|
||||
auto data_type = H5::PredType::NATIVE_DOUBLE;
|
||||
|
||||
H5::DataSet dataset = target.createDataSet(name.c_str(), data_type , att_space);
|
||||
dataset.write(&value, data_type);
|
||||
|
||||
return dataset;
|
||||
}
|
||||
|
||||
H5::DataSet H5FormatUtils::write_dataset(H5::Group& target, const string& name, int value)
|
||||
{
|
||||
H5::DataSpace att_space(H5S_SCALAR);
|
||||
auto data_type = H5::PredType::NATIVE_INT;
|
||||
|
||||
H5::DataSet dataset = target.createDataSet(name.c_str(), data_type, att_space);
|
||||
dataset.write(&value, data_type);
|
||||
|
||||
return dataset;
|
||||
}
|
||||
|
||||
H5::DataSet H5FormatUtils::write_dataset(H5::Group& target, const string& name, const string& value)
|
||||
{
|
||||
H5::DataSpace att_space(H5S_SCALAR);
|
||||
H5::DataType data_type = H5::StrType(0, H5T_VARIABLE);
|
||||
|
||||
H5::DataSet dataset = target.createDataSet(name.c_str(), data_type ,att_space);
|
||||
|
||||
dataset.write(value, data_type);
|
||||
|
||||
return dataset;
|
||||
}
|
||||
|
||||
void H5FormatUtils::write_attribute(H5::H5Object& target, const string& name, const string& value)
|
||||
{
|
||||
H5::DataSpace att_space(H5S_SCALAR);
|
||||
H5::DataType data_type = H5::StrType(H5::PredType::C_S1, H5T_VARIABLE);
|
||||
|
||||
auto h5_attribute = target.createAttribute(name.c_str(), data_type, att_space);
|
||||
h5_attribute.write(data_type, value.c_str());
|
||||
}
|
||||
|
||||
void H5FormatUtils::write_attribute(H5::H5Object& target, const string& name, int value)
|
||||
{
|
||||
H5::DataSpace att_space(H5S_SCALAR);
|
||||
auto data_type = H5::PredType::NATIVE_INT;
|
||||
|
||||
auto h5_attribute = target.createAttribute(name.c_str(), data_type, att_space);
|
||||
h5_attribute.write(data_type, &value);
|
||||
}
|
||||
|
||||
void H5FormatUtils::write_attribute(H5::H5Object& target, const h5_attr& attribute,
|
||||
const unordered_map<string, boost::any>& values)
|
||||
{
|
||||
string name = attribute.name;
|
||||
boost::any value;
|
||||
|
||||
// Value is stored directly in the struct.
|
||||
if (attribute.data_location == IMMEDIATE){
|
||||
value = attribute.value;
|
||||
// Value in struct is just a string reference to into the values map.
|
||||
} else {
|
||||
value = H5FormatUtils::get_value_from_reference(name, attribute.value, values);
|
||||
}
|
||||
|
||||
if (attribute.data_type == NX_CHAR) {
|
||||
// Attempt to convert to const char * (string "literals" cause that).
|
||||
try {
|
||||
H5FormatUtils::write_attribute(target, name, string(boost::any_cast<const char*>(value)));
|
||||
return;
|
||||
} catch (const boost::bad_any_cast& exception) {}
|
||||
|
||||
// Atempt to convert to string.
|
||||
try {
|
||||
H5FormatUtils::write_attribute(target, name, boost::any_cast<string>(value));
|
||||
return;
|
||||
} catch (const boost::bad_any_cast& exception) {}
|
||||
|
||||
// We cannot really convert this attribute.
|
||||
stringstream error_message;
|
||||
using namespace date;
|
||||
error_message << "[" << std::chrono::system_clock::now() << "]";
|
||||
error_message << "Cannot convert attribute " << name << " to string or const char*." << endl;
|
||||
|
||||
throw runtime_error(error_message.str());
|
||||
|
||||
} else if (attribute.data_type == NX_INT) {
|
||||
try {
|
||||
H5FormatUtils::write_attribute(target, name, boost::any_cast<int>(value));
|
||||
return;
|
||||
} catch (const boost::bad_any_cast& exception) {}
|
||||
|
||||
// We cannot really convert this attribute.
|
||||
stringstream error_message;
|
||||
using namespace date;
|
||||
error_message << "[" << std::chrono::system_clock::now() << "]";
|
||||
error_message << "Cannot convert attribute " << name << " to INT." << endl;
|
||||
|
||||
throw runtime_error(error_message.str());
|
||||
}
|
||||
}
|
||||
|
||||
void H5FormatUtils::write_format_data(H5::Group& file_node, const h5_parent& format_node,
|
||||
const std::unordered_map<std::string, h5_value>& values)
|
||||
{
|
||||
auto process_items = [&format_node, &values](H5::Group& node_group){
|
||||
for (const auto item_ptr : format_node.items) {
|
||||
|
||||
const h5_base& item = *item_ptr;
|
||||
|
||||
if (item.node_type == GROUP) {
|
||||
auto sub_group = dynamic_cast<const h5_group&>(item);
|
||||
|
||||
write_format_data(node_group, sub_group, values);
|
||||
|
||||
} else if (item.node_type == ATTRIBUTE) {
|
||||
auto sub_attribute = dynamic_cast<const h5_attr&>(item);
|
||||
|
||||
H5FormatUtils::write_attribute(node_group, sub_attribute, values);
|
||||
|
||||
} else if (item.node_type == DATASET) {
|
||||
auto sub_dataset = dynamic_cast<const h5_dataset&>(item);
|
||||
auto current_dataset = H5FormatUtils::write_dataset(node_group, sub_dataset, values);
|
||||
|
||||
for (const auto dataset_attr_ptr : sub_dataset.items) {
|
||||
const h5_base& dataset_attr = *dataset_attr_ptr;
|
||||
|
||||
// You can specify only attributes inside a dataset.
|
||||
if (dataset_attr.node_type != ATTRIBUTE) {
|
||||
stringstream error_message;
|
||||
using namespace date;
|
||||
error_message << "[" << std::chrono::system_clock::now() << "]";
|
||||
error_message << "Invalid element " << dataset_attr.name << " on dataset " << sub_dataset.name << ". Only attributes allowd.";
|
||||
|
||||
throw invalid_argument( error_message.str() );
|
||||
}
|
||||
|
||||
auto sub_attribute = dynamic_cast<const h5_attr&>(dataset_attr);
|
||||
|
||||
H5FormatUtils::write_attribute(current_dataset, sub_attribute, values);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if (format_node.node_type == GROUP) {
|
||||
auto x = H5FormatUtils::create_group(file_node, format_node.name);
|
||||
process_items(x);
|
||||
}else {
|
||||
process_items(file_node);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
void H5FormatUtils::write_format(H5::H5File& file, const H5Format& format,
|
||||
const std::unordered_map<std::string, h5_value>& input_values)
|
||||
{
|
||||
auto format_definition = format.get_format_definition();
|
||||
auto default_values = format.get_default_values();
|
||||
|
||||
auto format_values(default_values);
|
||||
|
||||
format.add_input_values(format_values, input_values);
|
||||
format.add_calculated_values(format_values);
|
||||
|
||||
write_format_data(file, format_definition, format_values);
|
||||
|
||||
for (const auto& mapping : format.get_dataset_move_mapping()) {
|
||||
file.move(mapping.first, mapping.second);
|
||||
}
|
||||
}
|
||||
@@ -1,143 +0,0 @@
|
||||
#ifndef H5FORMAT_H
|
||||
#define H5FORMAT_H
|
||||
|
||||
#include <string>
|
||||
#include <list>
|
||||
#include <unordered_map>
|
||||
#include <H5Cpp.h>
|
||||
#include <memory>
|
||||
#include <tuple>
|
||||
#include <boost/any.hpp>
|
||||
#include <chrono>
|
||||
#include "date.h"
|
||||
|
||||
typedef boost::any h5_value;
|
||||
|
||||
enum NODE_TYPE
|
||||
{
|
||||
EMPTY_ROOT,
|
||||
ATTRIBUTE,
|
||||
DATASET,
|
||||
GROUP
|
||||
};
|
||||
|
||||
enum DATA_TYPE
|
||||
{
|
||||
NX_FLOAT,
|
||||
NX_CHAR,
|
||||
NX_INT,
|
||||
NX_DATE_TIME,
|
||||
NX_NUMBER,
|
||||
NXnote
|
||||
};
|
||||
|
||||
enum DATA_LOCATION
|
||||
{
|
||||
IMMEDIATE,
|
||||
REFERENCE
|
||||
};
|
||||
|
||||
|
||||
struct h5_base
|
||||
{
|
||||
h5_base(const std::string& name, NODE_TYPE node_type) : name(name), node_type(node_type){};
|
||||
virtual ~h5_base(){}
|
||||
std::string name;
|
||||
NODE_TYPE node_type;
|
||||
};
|
||||
|
||||
struct h5_data_base
|
||||
{
|
||||
h5_data_base(DATA_TYPE data_type, DATA_LOCATION data_location) : data_type(data_type), data_location(data_location) {};
|
||||
|
||||
virtual ~h5_data_base(){}
|
||||
|
||||
DATA_TYPE data_type;
|
||||
DATA_LOCATION data_location;
|
||||
};
|
||||
|
||||
struct h5_parent: public h5_base
|
||||
{
|
||||
h5_parent(const std::string& name, NODE_TYPE node_type, const std::list<std::shared_ptr<h5_base>>& items) :
|
||||
h5_base(name, node_type), items(items) {};
|
||||
|
||||
std::list<std::shared_ptr<h5_base>> items;
|
||||
};
|
||||
|
||||
struct h5_group : public h5_parent
|
||||
{
|
||||
h5_group(const std::string& name, const std::list<std::shared_ptr<h5_base>>& items={}) :
|
||||
h5_parent(name, GROUP, items) {};
|
||||
};
|
||||
|
||||
struct h5_dataset : public h5_parent, public h5_data_base
|
||||
{
|
||||
h5_dataset(const std::string& name, const std::string& value, DATA_TYPE data_type, const std::list<std::shared_ptr<h5_base>>& items={})
|
||||
: h5_parent(name, DATASET, items), h5_data_base(data_type, REFERENCE), value(value) {};
|
||||
|
||||
std::string value;
|
||||
};
|
||||
|
||||
struct h5_attr : public h5_base, public h5_data_base
|
||||
{
|
||||
h5_attr(const std::string& name, const h5_value& value, DATA_TYPE data_types, DATA_LOCATION data_location=IMMEDIATE)
|
||||
: h5_base(name, ATTRIBUTE), h5_data_base(data_types, data_location), value(value){};
|
||||
|
||||
h5_value value;
|
||||
};
|
||||
|
||||
class H5Format
|
||||
{
|
||||
public:
|
||||
virtual ~H5Format(){};
|
||||
|
||||
virtual const std::unordered_map<std::string, DATA_TYPE>& get_input_value_type() const = 0;
|
||||
|
||||
virtual const std::unordered_map<std::string, boost::any>& get_default_values() const = 0;
|
||||
|
||||
virtual const h5_parent& get_format_definition() const = 0;
|
||||
|
||||
virtual void add_calculated_values(std::unordered_map<std::string, boost::any>& values) const = 0;
|
||||
|
||||
virtual void add_input_values(std::unordered_map<std::string, boost::any>& values,
|
||||
const std::unordered_map<std::string, boost::any>& input_values) const = 0;
|
||||
|
||||
virtual const std::unordered_map<std::string, std::string>& get_dataset_move_mapping() const = 0;
|
||||
};
|
||||
|
||||
namespace H5FormatUtils
|
||||
{
|
||||
hsize_t expand_dataset(H5::DataSet& dataset, hsize_t frame_index, hsize_t dataset_increase_step);
|
||||
|
||||
void compact_dataset(H5::DataSet& dataset, hsize_t max_frame_index);
|
||||
|
||||
H5::Group create_group(H5::Group& target, const std::string& name);
|
||||
const H5::PredType& get_dataset_data_type(const std::string& type);
|
||||
|
||||
H5::DataSet write_dataset(H5::Group& target, const h5_dataset& dataset,
|
||||
const std::unordered_map<std::string, boost::any>& values);
|
||||
|
||||
H5::DataSet write_dataset(H5::Group& target, const std::string& name, double value);
|
||||
|
||||
H5::DataSet write_dataset(H5::Group& target, const std::string& name, int value);
|
||||
|
||||
H5::DataSet write_dataset(H5::Group& target, const std::string& name, const std::string& value);
|
||||
|
||||
void write_attribute(H5::H5Object& target, const h5_attr& attribute,
|
||||
const std::unordered_map<std::string, boost::any>& values);
|
||||
|
||||
void write_attribute(H5::H5Object& target, const std::string& name, const std::string& value);
|
||||
|
||||
void write_attribute(H5::H5Object& target, const std::string& name, int value);
|
||||
|
||||
const boost::any& get_value_from_reference(const std::string& dataset_name,
|
||||
const boost::any& value_reference, const std::unordered_map<std::string, boost::any>& values);
|
||||
|
||||
void write_format_data(H5::Group& file_node, const h5_parent& format_node,
|
||||
const std::unordered_map<std::string, h5_value>& values);
|
||||
|
||||
void write_format(H5::H5File& file, const H5Format& format,
|
||||
const std::unordered_map<std::string, h5_value>& input_values);
|
||||
};
|
||||
|
||||
#endif
|
||||
@@ -1,334 +0,0 @@
|
||||
#include <sstream>
|
||||
#include <stdexcept>
|
||||
#include <iostream>
|
||||
|
||||
#include "H5Writer.hpp"
|
||||
#include "H5Format.hpp"
|
||||
|
||||
extern "C"
|
||||
{
|
||||
#include "H5DOpublic.h"
|
||||
}
|
||||
|
||||
using namespace std;
|
||||
|
||||
std::unique_ptr<H5Writer> get_h5_writer(const string& filename, hsize_t frames_per_file,
|
||||
hsize_t initial_dataset_size, hsize_t dataset_increase_step)
|
||||
{
|
||||
if (filename == "/dev/null") {
|
||||
return unique_ptr<H5Writer>(new DummyH5Writer());
|
||||
} else {
|
||||
return unique_ptr<H5Writer>(new H5Writer(filename, frames_per_file, initial_dataset_size, dataset_increase_step));
|
||||
}
|
||||
}
|
||||
|
||||
H5Writer::H5Writer(const std::string& filename, hsize_t frames_per_file, hsize_t initial_dataset_size,
|
||||
hsize_t dataset_increase_step) :
|
||||
filename(filename), frames_per_file(frames_per_file),
|
||||
initial_dataset_size(initial_dataset_size), dataset_increase_step(dataset_increase_step)
|
||||
{
|
||||
#ifdef DEBUG_OUTPUT
|
||||
using namespace date;
|
||||
cout << "[" << std::chrono::system_clock::now() << "]";
|
||||
cout << "[H5Writer::H5Writer] Creating chunked writer";
|
||||
cout << " with filename " << filename;
|
||||
cout << " and frames_per_file " << frames_per_file;
|
||||
cout << " and initial_dataset_size " << initial_dataset_size;
|
||||
cout << endl;
|
||||
#endif
|
||||
}
|
||||
|
||||
H5Writer::~H5Writer()
|
||||
{
|
||||
close_file();
|
||||
}
|
||||
|
||||
void H5Writer::close_file()
|
||||
{
|
||||
if (is_file_open()) {
|
||||
|
||||
#ifdef DEBUG_OUTPUT
|
||||
using namespace date;
|
||||
cout << "[" << std::chrono::system_clock::now() << "]";
|
||||
cout << "[H5Writer::close_file] Closing file." << endl;
|
||||
#endif
|
||||
|
||||
hsize_t min_frame_in_dataset = 0;
|
||||
if (frames_per_file) {
|
||||
min_frame_in_dataset = (current_frame_chunk - 1) * frames_per_file;
|
||||
}
|
||||
|
||||
// max_data_index is relative to the current file.
|
||||
hsize_t max_frame_in_dataset = max_data_index + min_frame_in_dataset;
|
||||
|
||||
// Frame indexing starts at 1 (for some reason).
|
||||
auto image_nr_low = min_frame_in_dataset + 1;
|
||||
auto image_nr_high = max_frame_in_dataset + 1;
|
||||
|
||||
#ifdef DEBUG_OUTPUT
|
||||
using namespace date;
|
||||
cout << "[" << std::chrono::system_clock::now() << "]";
|
||||
cout << "[H5Writer::close_file] Setting datasets attribute image_nr_low=" << image_nr_low;
|
||||
cout << " and image_nr_high=" << image_nr_high << endl;
|
||||
#endif
|
||||
|
||||
for (const auto& dataset_map : datasets) {
|
||||
auto dataset = dataset_map.second;
|
||||
|
||||
H5FormatUtils::compact_dataset(dataset, max_data_index);
|
||||
|
||||
H5FormatUtils::write_attribute(dataset, "image_nr_low", image_nr_low);
|
||||
H5FormatUtils::write_attribute(dataset, "image_nr_high", image_nr_high);
|
||||
}
|
||||
|
||||
file.close();
|
||||
|
||||
} else {
|
||||
#ifdef DEBUG_OUTPUT
|
||||
using namespace date;
|
||||
cout << "[" << std::chrono::system_clock::now() << "]";
|
||||
cout << "[H5Writer::close_file] Trying to close an already closed file." << endl;
|
||||
#endif
|
||||
}
|
||||
|
||||
// Cleanup.
|
||||
datasets.clear();
|
||||
datasets_current_size.clear();
|
||||
|
||||
current_frame_chunk = 0;
|
||||
max_data_index = 0;
|
||||
}
|
||||
|
||||
void H5Writer::write_data(const string& dataset_name, const size_t data_index, const char* data,
|
||||
const std::vector<size_t>& data_shape, const size_t data_bytes_size, const string& data_type, const string& endianness)
|
||||
{
|
||||
try {
|
||||
|
||||
// Define the ofset of the currently received image in the file.
|
||||
hsize_t relative_data_index = prepare_storage_for_data(dataset_name, data_index, data_shape, data_type, endianness);
|
||||
|
||||
// Define the offset where to write the data.
|
||||
size_t data_rank = data_shape.size();
|
||||
hsize_t offset[data_rank+1];
|
||||
|
||||
offset[0] = relative_data_index;
|
||||
for (uint index=0; index<data_rank; ++index) {
|
||||
offset[index+1] = 0;
|
||||
}
|
||||
|
||||
// No compression for now.
|
||||
uint32_t filters = 0;
|
||||
|
||||
const auto& dataset = datasets.at(dataset_name);
|
||||
|
||||
if( H5DOwrite_chunk(dataset.getId(), H5P_DEFAULT, filters, offset, data_bytes_size, data) )
|
||||
{
|
||||
stringstream error_message;
|
||||
using namespace date;
|
||||
error_message << "[" << std::chrono::system_clock::now() << "]";
|
||||
error_message << "Error while writing dataset " << dataset_name << " chunk to file at offset ";
|
||||
error_message << relative_data_index << "." << endl;
|
||||
|
||||
throw invalid_argument( error_message.str() );
|
||||
}
|
||||
} catch (...) {
|
||||
using namespace date;
|
||||
cout << "[" << std::chrono::system_clock::now() << "]";
|
||||
cout << "[H5Writer::write_data] Error while trying to write data to dataset " << dataset_name << endl;
|
||||
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
void H5Writer::create_dataset(const string& dataset_name, const vector<size_t>& data_shape,
|
||||
const string& data_type, const string& endianness, bool chunked, hsize_t dataset_size)
|
||||
{
|
||||
// Number of dimensions in each data point.
|
||||
const size_t data_rank = data_shape.size();
|
||||
// The +1 dimension is to account for the sequence of data points (time).
|
||||
const hsize_t dataset_rank = data_rank + 1;
|
||||
|
||||
hsize_t dataset_dimension[dataset_rank];
|
||||
hsize_t max_dataset_dimension[dataset_rank];
|
||||
hsize_t dataset_chunking[dataset_rank];
|
||||
|
||||
// This should be equivalent to the total number of frames in this file.
|
||||
dataset_dimension[0] = dataset_size;
|
||||
// The maximum dataset size is the same as the number of images.
|
||||
max_dataset_dimension[0] = dataset_size;
|
||||
// Chunking is always set to a single data point.
|
||||
dataset_chunking[0] = 1;
|
||||
|
||||
for (size_t index=0; index<data_rank; ++index) {
|
||||
dataset_dimension[index+1] = data_shape[index];
|
||||
max_dataset_dimension[index+1] = data_shape[index];
|
||||
dataset_chunking[index+1] = data_shape[index];
|
||||
}
|
||||
|
||||
// Create a chunked dataset if needed.
|
||||
H5::DSetCreatPropList dataset_properties;
|
||||
if (chunked) {
|
||||
dataset_properties.setChunk(dataset_rank, dataset_chunking);
|
||||
|
||||
// Chunked datasets can be resized without limits.
|
||||
max_dataset_dimension[0] = H5S_UNLIMITED;
|
||||
}
|
||||
|
||||
H5::DataSpace dataspace(dataset_rank, dataset_dimension, max_dataset_dimension);
|
||||
|
||||
#ifdef DEBUG_OUTPUT
|
||||
using namespace date;
|
||||
cout << "[" << std::chrono::system_clock::now() << "]";
|
||||
cout << "[H5Writer::create_dataset] Creating dataspace of size (";
|
||||
for (hsize_t i=0; i<dataset_rank; ++i) {
|
||||
cout << dataset_dimension[i] << ",";
|
||||
}
|
||||
cout << ")" << endl;
|
||||
#endif
|
||||
|
||||
H5::AtomType dataset_data_type(H5FormatUtils::get_dataset_data_type(data_type));
|
||||
|
||||
if (endianness == "big") {
|
||||
dataset_data_type.setOrder(H5T_ORDER_BE);
|
||||
} else {
|
||||
dataset_data_type.setOrder(H5T_ORDER_LE);
|
||||
}
|
||||
|
||||
auto dataset = file.createDataSet(dataset_name.c_str(), dataset_data_type, dataspace, dataset_properties);
|
||||
|
||||
datasets.insert({dataset_name, dataset});
|
||||
datasets_current_size.insert({dataset_name, initial_dataset_size});
|
||||
}
|
||||
|
||||
void H5Writer::create_file(hsize_t frame_chunk)
|
||||
{
|
||||
|
||||
if (file.getId() != -1) {
|
||||
close_file();
|
||||
}
|
||||
|
||||
auto target_filename = filename;
|
||||
|
||||
// In case frames_per_file is > 0, the filename variable is a template for the filename.
|
||||
if (frames_per_file) {
|
||||
#ifdef DEBUG_OUTPUT
|
||||
using namespace date;
|
||||
cout << "[" << std::chrono::system_clock::now() << "]";
|
||||
cout << "[H5Writer::create_file] Frames per file is defined. Format " << filename << " with frame_chunk " << frame_chunk << endl;
|
||||
#endif
|
||||
|
||||
// Space for 10 digits should be enough.
|
||||
char buffer[filename.length() + 10];
|
||||
|
||||
sprintf(buffer, filename.c_str(), frame_chunk);
|
||||
target_filename = string(buffer);
|
||||
}
|
||||
|
||||
#ifdef DEBUG_OUTPUT
|
||||
using namespace date;
|
||||
cout << "[" << std::chrono::system_clock::now() << "]";
|
||||
cout << "[H5Writer::create_file] Creating filename " << target_filename << endl;
|
||||
#endif
|
||||
|
||||
file = H5::H5File(target_filename.c_str(), H5F_ACC_TRUNC);
|
||||
|
||||
if (file.getId() == -1) {
|
||||
stringstream error_message;
|
||||
using namespace date;
|
||||
error_message << "[" << std::chrono::system_clock::now() << "]";
|
||||
error_message << "Cannot create new file with filename " << target_filename << endl;
|
||||
|
||||
throw runtime_error(error_message.str());
|
||||
}
|
||||
|
||||
// New file created - set this files chunk number.
|
||||
current_frame_chunk = frame_chunk;
|
||||
}
|
||||
|
||||
bool H5Writer::is_file_open() const
|
||||
{
|
||||
return (file.getId() != -1);
|
||||
}
|
||||
|
||||
inline size_t H5Writer::get_relative_data_index(const size_t data_index)
|
||||
{
|
||||
// No file roll over.
|
||||
if (frames_per_file == 0) {
|
||||
return data_index;
|
||||
}
|
||||
|
||||
size_t destination_file_index = data_index / frames_per_file;
|
||||
size_t relative_data_index = data_index - (destination_file_index * frames_per_file);
|
||||
|
||||
return relative_data_index;
|
||||
}
|
||||
|
||||
inline bool H5Writer::is_data_for_current_file(const size_t data_index)
|
||||
{
|
||||
if (frames_per_file) {
|
||||
hsize_t frame_chunk = (data_index / frames_per_file) + 1;
|
||||
|
||||
// This frames does not go into this file.
|
||||
if (frame_chunk != current_frame_chunk) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
hsize_t H5Writer::prepare_storage_for_data(const string& dataset_name, const size_t data_index, const std::vector<size_t>& data_shape,
|
||||
const string& data_type, const string& endianness)
|
||||
{
|
||||
// Check if we have to create a new file.
|
||||
if (!is_data_for_current_file(data_index)) {
|
||||
// Calculate to which file (1 based) the data_index belongs.
|
||||
hsize_t frame_chunk = (data_index / frames_per_file) + 1;
|
||||
create_file(frame_chunk);
|
||||
}
|
||||
|
||||
// Open the file if needed.
|
||||
if (!is_file_open()) {
|
||||
create_file();
|
||||
}
|
||||
|
||||
// Create the dataset if we don't have it yet.
|
||||
if (datasets.find(dataset_name) == datasets.end()) {
|
||||
create_dataset(dataset_name, data_shape, data_type, endianness, true, initial_dataset_size);
|
||||
}
|
||||
|
||||
hsize_t current_dataset_size = datasets_current_size.at(dataset_name);
|
||||
|
||||
hsize_t relative_data_index = get_relative_data_index(data_index);
|
||||
|
||||
// Expand the dataset if needed.
|
||||
if (relative_data_index > current_dataset_size) {
|
||||
auto dataset = datasets.at(dataset_name);
|
||||
|
||||
hsize_t new_dataset_size = H5FormatUtils::expand_dataset(dataset, relative_data_index, dataset_increase_step);
|
||||
datasets_current_size[dataset_name] = new_dataset_size;
|
||||
}
|
||||
|
||||
// Keep track of the max index in this file - needed for shrinking the dataset at the end.
|
||||
if (relative_data_index > max_data_index) {
|
||||
max_data_index = relative_data_index;
|
||||
}
|
||||
|
||||
return relative_data_index;
|
||||
}
|
||||
|
||||
H5::H5File& H5Writer::get_h5_file()
|
||||
{
|
||||
return file;
|
||||
}
|
||||
|
||||
H5::H5File& DummyH5Writer::get_h5_file()
|
||||
{
|
||||
stringstream error_message;
|
||||
using namespace date;
|
||||
error_message << "[" << std::chrono::system_clock::now() << "]";
|
||||
error_message << "Cannot get the H5 file with the dummy writer." << endl;
|
||||
|
||||
throw runtime_error(error_message.str());
|
||||
};
|
||||
|
||||
@@ -1,73 +0,0 @@
|
||||
#ifndef H5WRITER_H
|
||||
#define H5WRITER_H
|
||||
|
||||
#include <unordered_map>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
#include <H5Cpp.h>
|
||||
#include <chrono>
|
||||
#include "date.h"
|
||||
|
||||
class H5Writer
|
||||
{
|
||||
protected:
|
||||
// Initialized in constructor.
|
||||
const std::string filename;
|
||||
hsize_t frames_per_file;
|
||||
hsize_t initial_dataset_size;
|
||||
hsize_t dataset_increase_step = 0;
|
||||
|
||||
// State variables.
|
||||
hsize_t max_data_index = 0;
|
||||
hsize_t current_frame_chunk = 0;
|
||||
|
||||
H5::H5File file;
|
||||
std::unordered_map<std::string, H5::DataSet> datasets;
|
||||
std::unordered_map<std::string, hsize_t> datasets_current_size;
|
||||
|
||||
hsize_t prepare_storage_for_data(const std::string& dataset_name, const size_t data_index, const std::vector<size_t>& data_shape,
|
||||
const std::string& data_type, const std::string& endianness);
|
||||
|
||||
void create_dataset(const std::string& dataset_name, const std::vector<size_t>& data_shape,
|
||||
const std::string& data_type, const std::string& endianness, bool chunked, hsize_t dataset_size);
|
||||
|
||||
size_t get_relative_data_index(const size_t data_index);
|
||||
|
||||
public:
|
||||
H5Writer(const std::string& filename, hsize_t frames_per_file=0, hsize_t initial_dataset_size=1000, hsize_t dataset_increase_step=1000);
|
||||
virtual ~H5Writer();
|
||||
virtual bool is_file_open() const;
|
||||
virtual void create_file(const hsize_t frame_chunk=1);
|
||||
virtual void close_file();
|
||||
virtual void write_data(const std::string& dataset_name, const size_t data_index, const char* data, const std::vector<size_t>& data_shape,
|
||||
const size_t data_bytes_size, const std::string& data_type, const std::string& endianness);
|
||||
virtual H5::H5File& get_h5_file();
|
||||
virtual bool is_data_for_current_file(const size_t data_index);
|
||||
|
||||
};
|
||||
|
||||
class DummyH5Writer : public H5Writer
|
||||
{
|
||||
public:
|
||||
DummyH5Writer() : H5Writer("/dev/null", 0, 0, 0) {}
|
||||
|
||||
bool is_file_open() const override
|
||||
{ return false; }
|
||||
|
||||
void create_file(const hsize_t frame_chunk=1) override {}
|
||||
|
||||
void close_file() override {}
|
||||
|
||||
void write_data(const std::string& dataset_name, const size_t data_index, const char* data, const std::vector<size_t>& data_shape,
|
||||
const size_t data_bytes_size, const std::string& data_type, const std::string& endianness) override {}
|
||||
|
||||
H5::H5File& get_h5_file() override;
|
||||
|
||||
bool is_data_for_current_file(const size_t data_index) override
|
||||
{ return true; }
|
||||
};
|
||||
|
||||
std::unique_ptr<H5Writer> get_h5_writer(const std::string& filename, hsize_t frames_per_file=0,
|
||||
hsize_t initial_dataset_size=1000, hsize_t dataset_increase_step=1000);
|
||||
|
||||
#endif
|
||||
@@ -1,84 +0,0 @@
|
||||
#include <iostream>
|
||||
#include <stdexcept>
|
||||
|
||||
#include "date.h"
|
||||
#include "MetadataBuffer.hpp"
|
||||
|
||||
using namespace std;
|
||||
|
||||
MetadataBuffer::MetadataBuffer(uint64_t n_images, shared_ptr<unordered_map<string, HeaderDataType>> header_values_type) :
|
||||
n_images(n_images), header_values_type(header_values_type)
|
||||
{
|
||||
if (header_values_type) {
|
||||
for (const auto& header_type : *header_values_type) {
|
||||
auto& name = header_type.first;
|
||||
auto& header_data_type = header_type.second;
|
||||
|
||||
size_t bytes_size_per_frame = header_data_type.value_shape * header_data_type.value_bytes_size;
|
||||
size_t buffer_size_bytes = n_images * bytes_size_per_frame;
|
||||
|
||||
shared_ptr<char> buffer(new char[buffer_size_bytes](), std::default_delete<char[]>());
|
||||
metadata_buffer.insert({name, buffer});
|
||||
metadata_length_bytes.insert({name, bytes_size_per_frame});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void MetadataBuffer::add_metadata_to_buffer(string name, uint64_t frame_index, const char* data)
|
||||
{
|
||||
if (frame_index >= n_images) {
|
||||
stringstream error_message;
|
||||
using namespace date;
|
||||
error_message << "[" << std::chrono::system_clock::now() << "] ";
|
||||
error_message << "Requested frame_index " << frame_index << " ";
|
||||
error_message << "for metadata " << name << " in metadata buffer is out of range. ";
|
||||
error_message << "Number of available slots: " << n_images << endl;
|
||||
|
||||
throw runtime_error(error_message.str());
|
||||
}
|
||||
|
||||
auto metadata = metadata_buffer.find(name);
|
||||
|
||||
if (metadata == metadata_buffer.end()) {
|
||||
stringstream error_message;
|
||||
using namespace date;
|
||||
error_message << "[" << std::chrono::system_clock::now() << "] ";
|
||||
error_message << "Cannot add data to metadata buffer because '" << name << "' is not declared."<< endl;
|
||||
|
||||
throw runtime_error(error_message.str());
|
||||
}
|
||||
|
||||
size_t bytes_size_per_frame = metadata_length_bytes.at(name);
|
||||
size_t buffer_offset = frame_index * bytes_size_per_frame;
|
||||
|
||||
char* buffer = metadata->second.get();
|
||||
buffer += buffer_offset;
|
||||
|
||||
memcpy(buffer, data, bytes_size_per_frame);
|
||||
}
|
||||
|
||||
shared_ptr<char> MetadataBuffer::get_metadata_values(string name)
|
||||
{
|
||||
auto metadata = metadata_buffer.find(name);
|
||||
|
||||
if (metadata == metadata_buffer.end()) {
|
||||
stringstream error_message;
|
||||
using namespace date;
|
||||
error_message << "[" << std::chrono::system_clock::now() << "] ";
|
||||
error_message << "Cannot get data from metadata buffer because '" << name << "' is not declared."<< endl;
|
||||
|
||||
throw runtime_error(error_message.str());
|
||||
}
|
||||
|
||||
return metadata->second;
|
||||
}
|
||||
|
||||
shared_ptr<unordered_map<string, HeaderDataType>> MetadataBuffer::get_header_values_type()
|
||||
{
|
||||
return header_values_type;
|
||||
}
|
||||
|
||||
uint64_t MetadataBuffer::get_n_images()
|
||||
{
|
||||
return n_images;
|
||||
}
|
||||
@@ -1,26 +0,0 @@
|
||||
#ifndef METADATABUFFER_H
|
||||
#define METADATABUFFER_H
|
||||
|
||||
#include <unordered_map>
|
||||
#include <string>
|
||||
|
||||
#include "ZmqReceiver.hpp"
|
||||
|
||||
class MetadataBuffer
|
||||
{
|
||||
const uint64_t n_images;
|
||||
const std::shared_ptr<std::unordered_map<std::string, HeaderDataType>> header_values_type;
|
||||
|
||||
protected:
|
||||
std::unordered_map<std::string, std::shared_ptr<char>> metadata_buffer;
|
||||
std::unordered_map<std::string, size_t> metadata_length_bytes;
|
||||
|
||||
public:
|
||||
MetadataBuffer(uint64_t n_images, std::shared_ptr<std::unordered_map<std::string, HeaderDataType>> header_values_type);
|
||||
void add_metadata_to_buffer(std::string name, uint64_t frame_index, const char* data);
|
||||
std::shared_ptr<char> get_metadata_values(std::string name);
|
||||
std::shared_ptr<std::unordered_map<std::string, HeaderDataType>> get_header_values_type();
|
||||
uint64_t get_n_images();
|
||||
};
|
||||
|
||||
#endif
|
||||
@@ -1,118 +0,0 @@
|
||||
#include <cstdlib>
|
||||
#include <chrono>
|
||||
#include <unistd.h>
|
||||
#include <stdexcept>
|
||||
#include <iostream>
|
||||
#include <memory>
|
||||
#include <boost/thread.hpp>
|
||||
#include <future>
|
||||
#include <vector>
|
||||
|
||||
#include "RestApi.hpp"
|
||||
#include "ProcessManager.hpp"
|
||||
#include "config.hpp"
|
||||
#include "BufferedWriter.hpp"
|
||||
#include "compression/compression.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
ProcessManager::ProcessManager(
|
||||
WriterManager& writer_manager,
|
||||
ZmqReceiver& receiver,
|
||||
RingBuffer& ring_buffer,
|
||||
const H5Format& format,
|
||||
uint16_t rest_port,
|
||||
const string& bsread_rest_address,
|
||||
hsize_t frames_per_file) :
|
||||
writer_manager(writer_manager),
|
||||
receiver(receiver),
|
||||
ring_buffer(ring_buffer),
|
||||
format(format),
|
||||
rest_port(rest_port),
|
||||
bsread_rest_address(bsread_rest_address),
|
||||
frames_per_file(frames_per_file)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
void ProcessManager::run_receivers(uint8_t n_receiving_threads)
|
||||
{
|
||||
|
||||
|
||||
RestApi::start_rest_api(writer_manager, rest_port);
|
||||
|
||||
|
||||
// In case SIGINT stopped the rest_api.
|
||||
writer_manager.stop();
|
||||
|
||||
|
||||
#ifdef DEBUG_OUTPUT
|
||||
using namespace date;
|
||||
cout << "[" << std::chrono::system_clock::now() << "]";
|
||||
cout << "[ProcessManager::run_writer] Writer properly stopped." << endl;
|
||||
#endif
|
||||
}
|
||||
|
||||
void ProcessManager::receive_zmq()
|
||||
{
|
||||
receiver.connect();
|
||||
|
||||
while (writer_manager.is_running()) {
|
||||
|
||||
auto frame = receiver.receive();
|
||||
|
||||
// In case no message is available before the timeout, both pointers are NULL.
|
||||
if (!frame.first || !writer_manager.receive_frame()){
|
||||
continue;
|
||||
}
|
||||
|
||||
auto frame_metadata = frame.first;
|
||||
auto frame_data = frame.second;
|
||||
|
||||
#ifdef DEBUG_OUTPUT
|
||||
using namespace date;
|
||||
cout << "[" << std::chrono::system_clock::now() << "]";
|
||||
cout << "[ProcessManager::receive_zmq] Processing FrameMetadata";
|
||||
cout << " with frame_index " << frame_metadata->frame_index;
|
||||
cout << " and frame_shape [" << frame_metadata->frame_shape[0] << ", " << frame_metadata->frame_shape[1] << "]";
|
||||
cout << " and endianness " << frame_metadata->endianness;
|
||||
cout << " and type " << frame_metadata->type;
|
||||
cout << " and frame_bytes_size " << frame_metadata->frame_bytes_size;
|
||||
cout << "." << endl;
|
||||
#endif
|
||||
|
||||
char* buffer = ring_buffer.reserve(frame_metadata);
|
||||
|
||||
size_t max_buffer_size = compression::get_bitshuffle_max_buffer_size(
|
||||
frame_metadata->frame_bytes_size, 1);
|
||||
|
||||
if (max_buffer_size > ring_buffer.get_slot_size()) {
|
||||
|
||||
}
|
||||
|
||||
auto compressed_size = compression::compress_bitshuffle(
|
||||
static_cast<const char*>(frame_data),
|
||||
frame_metadata->frame_bytes_size,
|
||||
1,
|
||||
buffer);
|
||||
|
||||
#ifdef DEBUG_OUTPUT
|
||||
using namespace date;
|
||||
cout << "[" << std::chrono::system_clock::now() << "]";
|
||||
cout << "[ProcessManager::receive_zmq] Compressed image from ";
|
||||
cout << frame_metadata->frame_bytes_size << " bytes to ";
|
||||
cout << compressed_size << " bytes." << endl;
|
||||
#endif
|
||||
|
||||
frame_metadata->frame_bytes_size = compressed_size;
|
||||
|
||||
ring_buffer.commit(frame_metadata);
|
||||
}
|
||||
|
||||
#ifdef DEBUG_OUTPUT
|
||||
using namespace date;
|
||||
cout << "[" << std::chrono::system_clock::now() << "]";
|
||||
cout << "[ProcessManager::receive_zmq] Receiver thread stopped." << endl;
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
#ifndef PROCESSMANAGER_H
|
||||
#define PROCESSMANAGER_H
|
||||
|
||||
#include "WriterManager.hpp"
|
||||
#include "H5Format.hpp"
|
||||
#include "RingBuffer.hpp"
|
||||
#include "ZmqReceiver.hpp"
|
||||
#include <chrono>
|
||||
#include "date.h"
|
||||
|
||||
class ProcessManager
|
||||
{
|
||||
WriterManager& writer_manager;
|
||||
ZmqReceiver& receiver;
|
||||
RingBuffer& ring_buffer;
|
||||
const H5Format& format;
|
||||
|
||||
uint16_t rest_port;
|
||||
const std::string& bsread_rest_address;
|
||||
hsize_t frames_per_file;
|
||||
|
||||
public:
|
||||
ProcessManager(WriterManager& writer_manager,
|
||||
ZmqReceiver& receiver,
|
||||
RingBuffer& ring_buffer,
|
||||
const H5Format& format,
|
||||
uint16_t rest_port,
|
||||
const std::string& bsread_rest_address,
|
||||
hsize_t frames_per_file=0);
|
||||
|
||||
void receive_zmq();
|
||||
void run_receivers(uint8_t n_receiving_threads);
|
||||
|
||||
};
|
||||
|
||||
#endif
|
||||
@@ -1,68 +0,0 @@
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
|
||||
#include "crow_all.h"
|
||||
#include "RestApi.hpp"
|
||||
|
||||
using namespace std;
|
||||
|
||||
void RestApi::start_rest_api(WriterManager& writer_manager, uint16_t port)
|
||||
{
|
||||
|
||||
#ifdef DEBUG_OUTPUT
|
||||
cout << "[rest_interface::start_rest_api] Starting rest interface on port ";
|
||||
cout << port << endl;
|
||||
#endif
|
||||
|
||||
crow::SimpleApp app;
|
||||
|
||||
CROW_ROUTE(app, "/stop")([&](){
|
||||
writer_manager.stop();
|
||||
|
||||
crow::json::wvalue result;
|
||||
result["state"] = "ok";
|
||||
result["status"] = writer_manager.get_status();
|
||||
|
||||
return result;
|
||||
});
|
||||
|
||||
CROW_ROUTE (app, "/status") ([&](){
|
||||
crow::json::wvalue result;
|
||||
result["state"] = "ok";
|
||||
result["status"] = writer_manager.get_status();
|
||||
|
||||
return result;
|
||||
});
|
||||
|
||||
CROW_ROUTE (app, "/statistics") ([&](){
|
||||
crow::json::wvalue result;
|
||||
|
||||
for (const auto& item : writer_manager.get_statistics()) {
|
||||
result[item.first] = item.second;
|
||||
}
|
||||
|
||||
result["state"] = "ok";
|
||||
result["status"] = writer_manager.get_status();
|
||||
|
||||
return result;
|
||||
});
|
||||
|
||||
CROW_ROUTE (app, "/start").methods("POST"_method) ([&](const crow::request& req)
|
||||
{
|
||||
auto request_parameters = crow::json::load(req.body);
|
||||
|
||||
int n_frames = request_parameters["n_frames"].i();
|
||||
string output_file = request_parameters["output_file"].s();
|
||||
int user_id = request_parameters["user_id"].i();
|
||||
|
||||
writer_manager.start(output_file, n_frames, user_id);
|
||||
|
||||
crow::json::wvalue result;
|
||||
result["message"] = "Writer started.";
|
||||
result["status"] = writer_manager.get_status();
|
||||
return result;
|
||||
});
|
||||
|
||||
app.loglevel(crow::LogLevel::ERROR);
|
||||
app.port(port).run();
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
#ifndef RESTAPI_H
|
||||
#define RESTAPI_H
|
||||
|
||||
#include "WriterManager.hpp"
|
||||
#include <chrono>
|
||||
#include "date.h"
|
||||
|
||||
namespace RestApi
|
||||
{
|
||||
void start_rest_api(WriterManager& writer_manager, uint16_t port);
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -1,270 +0,0 @@
|
||||
#include <stdexcept>
|
||||
#include <sstream>
|
||||
#include <cstring>
|
||||
#include <iostream>
|
||||
#include <cstddef>
|
||||
|
||||
#include "RingBuffer.hpp"
|
||||
|
||||
using namespace std;
|
||||
|
||||
RingBuffer::RingBuffer(size_t n_slots) : n_slots(n_slots), ringbuffer_slots(n_slots, 0)
|
||||
{
|
||||
#ifdef DEBUG_OUTPUT
|
||||
using namespace date;
|
||||
using namespace chrono;
|
||||
cout << "[" << system_clock::now() << "]";
|
||||
cout << "[RingBuffer::RingBuffer] Creating ring buffer";
|
||||
cout << " with n_slots " << n_slots << endl;
|
||||
#endif
|
||||
}
|
||||
|
||||
RingBuffer::~RingBuffer()
|
||||
{
|
||||
// If the frame buffer is allocated, free it.
|
||||
if (frame_data_buffer != NULL) {
|
||||
free(frame_data_buffer);
|
||||
frame_data_buffer = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
void RingBuffer::initialize(size_t slot_size)
|
||||
{
|
||||
if (frame_data_buffer) {
|
||||
stringstream error_message;
|
||||
using namespace date;
|
||||
using namespace chrono;
|
||||
error_message << "[" << system_clock::now() << "]";
|
||||
error_message << "[RingBuffer::initialize] Ring buffer";
|
||||
error_message << " already initialized." << endl;
|
||||
|
||||
throw runtime_error(error_message.str());
|
||||
}
|
||||
|
||||
#ifdef DEBUG_OUTPUT
|
||||
using namespace date;
|
||||
using namespace chrono;
|
||||
cout << "[" << system_clock::now() << "]";
|
||||
cout << "[RingBuffer::initialize] Initializing ring buffer";
|
||||
cout << " with slot_size " << slot_size << endl;
|
||||
#endif
|
||||
|
||||
this->write_index = 0;
|
||||
this->slot_size = slot_size;
|
||||
this->buffer_size = slot_size * n_slots;
|
||||
this->frame_data_buffer = new char[buffer_size];
|
||||
this->buffer_used_slots = 0;
|
||||
this->ring_buffer_initialized = true;
|
||||
|
||||
#ifdef DEBUG_OUTPUT
|
||||
using namespace date;
|
||||
using namespace chrono;
|
||||
cout << "[" << system_clock::now() << "]";
|
||||
cout << "[RingBuffer::initialize] Total buffer_size " << buffer_size << endl;
|
||||
#endif
|
||||
}
|
||||
|
||||
char* RingBuffer::reserve(shared_ptr<FrameMetadata> frame_metadata)
|
||||
{
|
||||
if (!ring_buffer_initialized) {
|
||||
initialize(frame_metadata->frame_bytes_size);
|
||||
}
|
||||
|
||||
// All images must fit in the ring buffer slot.
|
||||
if (frame_metadata->frame_bytes_size > slot_size) {
|
||||
stringstream error_message;
|
||||
|
||||
using namespace date;
|
||||
using namespace chrono;
|
||||
error_message << "[" << system_clock::now() << "]";
|
||||
error_message << "[RingBuffer::reserve] Received frame index ";
|
||||
error_message << frame_metadata->frame_index;
|
||||
error_message << " that is too large for ring buffer slot. ";
|
||||
error_message << "Slot size " << slot_size << ", but frame bytes size ";
|
||||
error_message << frame_metadata->frame_bytes_size << endl;
|
||||
|
||||
throw runtime_error(error_message.str());
|
||||
}
|
||||
|
||||
// Check and reserve slot in the buffer.
|
||||
{
|
||||
lock_guard<mutex> lock(ringbuffer_slots_mutex);
|
||||
|
||||
if (!ringbuffer_slots[write_index]) {
|
||||
ringbuffer_slots[write_index] = 1;
|
||||
|
||||
frame_metadata->buffer_slot_index = write_index;
|
||||
|
||||
#ifdef DEBUG_OUTPUT
|
||||
using namespace date;
|
||||
using namespace chrono;
|
||||
cout << "[" << system_clock::now() << "]";
|
||||
cout << "[RingBuafer::reserve] Ring buffer slot ";
|
||||
cout << frame_metadata->buffer_slot_index;
|
||||
cout << " reserved for frame_index ";
|
||||
cout << frame_metadata->frame_index << endl;
|
||||
#endif
|
||||
|
||||
write_index = (write_index + 1) % n_slots;
|
||||
buffer_used_slots++;
|
||||
|
||||
} else {
|
||||
stringstream error_message;
|
||||
|
||||
using namespace date;
|
||||
using namespace chrono;
|
||||
error_message << "[" << system_clock::now() << "]";
|
||||
error_message << "[RingBuffer::reserve] Ring buffer is full.";
|
||||
error_message << " Collision at write_index = " << write_index << endl;
|
||||
|
||||
throw runtime_error(error_message.str());
|
||||
}
|
||||
}
|
||||
|
||||
// The slot is already reserved, no need for synchronization.
|
||||
return get_buffer_slot_address(frame_metadata->buffer_slot_index);
|
||||
}
|
||||
|
||||
void RingBuffer::commit(shared_ptr<FrameMetadata> frame_metadata)
|
||||
{
|
||||
lock_guard<mutex> lock(frame_metadata_queue_mutex);
|
||||
|
||||
frame_metadata_queue.push_back(frame_metadata);
|
||||
|
||||
#ifdef DEBUG_OUTPUT
|
||||
using namespace date;
|
||||
cout << "[" << std::chrono::system_clock::now() << "]";
|
||||
cout << "[RingBuffer::write] Metadata for frame_index ";
|
||||
cout << frame_metadata->frame_index << " added to metadata queue." << endl;
|
||||
#endif
|
||||
}
|
||||
|
||||
char* RingBuffer::get_buffer_slot_address(size_t buffer_slot_index)
|
||||
{
|
||||
char* slot_memory_address = frame_data_buffer + (buffer_slot_index * slot_size);
|
||||
|
||||
// Check if the memory address is valid.
|
||||
if (slot_memory_address > frame_data_buffer + buffer_size) {
|
||||
stringstream error_message;
|
||||
|
||||
using namespace date;
|
||||
using namespace chrono;
|
||||
error_message << "[" << system_clock::now() << "]";
|
||||
error_message << "[RingBuffer::get_buffer_slot_address] Calculated";
|
||||
error_message << " ring buffer address is out of bound for buffer_slot_index ";
|
||||
error_message << buffer_slot_index << endl;
|
||||
|
||||
throw runtime_error(error_message.str());
|
||||
}
|
||||
|
||||
return slot_memory_address;
|
||||
}
|
||||
|
||||
pair<shared_ptr<FrameMetadata>, char*> RingBuffer::read()
|
||||
{
|
||||
shared_ptr<FrameMetadata> frame_metadata;
|
||||
|
||||
{
|
||||
lock_guard<mutex> lock(frame_metadata_queue_mutex);
|
||||
|
||||
// A NULL char* indicates that there are no available data in the ring buffer.
|
||||
if (frame_metadata_queue.empty()) {
|
||||
return {NULL, NULL};
|
||||
}
|
||||
|
||||
frame_metadata = frame_metadata_queue.front();
|
||||
frame_metadata_queue.pop_front();
|
||||
}
|
||||
|
||||
#ifdef DEBUG_OUTPUT
|
||||
using namespace date;
|
||||
using namespace chrono;
|
||||
cout << "[" << system_clock::now() << "]";
|
||||
cout << "[RingBuffer::read] Received metadata for frame_index ";
|
||||
cout << frame_metadata->frame_index << endl;
|
||||
#endif
|
||||
|
||||
// Check if the references ring buffer slot is valid.
|
||||
{
|
||||
lock_guard<mutex> lock(ringbuffer_slots_mutex);
|
||||
|
||||
if (!ringbuffer_slots[frame_metadata->buffer_slot_index]) {
|
||||
stringstream error_message;
|
||||
|
||||
using namespace date;
|
||||
using namespace chrono;
|
||||
error_message << "[" << system_clock::now() << "]";
|
||||
error_message << "[RingBuffer::read] Ring buffer slot";
|
||||
error_message << " referenced in message header ";
|
||||
error_message << frame_metadata->buffer_slot_index << " is empty." << endl;
|
||||
|
||||
throw runtime_error(error_message.str());
|
||||
}
|
||||
}
|
||||
|
||||
char* slot_memory_address = get_buffer_slot_address(frame_metadata->buffer_slot_index);
|
||||
|
||||
return {frame_metadata, slot_memory_address};
|
||||
}
|
||||
|
||||
void RingBuffer::release(size_t buffer_slot_index)
|
||||
{
|
||||
// Cannot release a slot index that is out of range.
|
||||
if (buffer_slot_index >= n_slots) {
|
||||
stringstream error_message;
|
||||
|
||||
using namespace date;
|
||||
using namespace chrono;
|
||||
error_message << "[" << system_clock::now() << "]";
|
||||
error_message << "[RingBuffer::release] Slot index to release ";
|
||||
error_message << buffer_slot_index << " is out of range.";
|
||||
error_message << " Ring buffer n_slots = " << n_slots << endl;
|
||||
|
||||
throw runtime_error(error_message.str());
|
||||
}
|
||||
|
||||
{
|
||||
lock_guard<mutex> lock(ringbuffer_slots_mutex);
|
||||
|
||||
if (ringbuffer_slots[buffer_slot_index]) {
|
||||
ringbuffer_slots[buffer_slot_index] = 0;
|
||||
|
||||
buffer_used_slots--;
|
||||
|
||||
} else {
|
||||
stringstream error_message;
|
||||
|
||||
using namespace date;
|
||||
using namespace chrono;
|
||||
error_message << "[" << system_clock::now() << "]";
|
||||
error_message << "[RingBuffer::release] Cannot release empty";
|
||||
error_message << " ring buffer slot " << buffer_slot_index << endl;
|
||||
|
||||
throw runtime_error(error_message.str());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool RingBuffer::is_empty()
|
||||
{
|
||||
lock_guard<mutex> lock(ringbuffer_slots_mutex);
|
||||
|
||||
return buffer_used_slots == 0;
|
||||
}
|
||||
|
||||
void RingBuffer::clear()
|
||||
{
|
||||
lock_guard<mutex> lock_slots(ringbuffer_slots_mutex);
|
||||
lock_guard<mutex> lock_metadata(frame_metadata_queue_mutex);
|
||||
|
||||
write_index = 0;
|
||||
buffer_used_slots = 0;
|
||||
ringbuffer_slots = vector<bool>(n_slots, 0);
|
||||
frame_metadata_queue.clear();
|
||||
}
|
||||
|
||||
size_t RingBuffer::get_slot_size()
|
||||
{
|
||||
return slot_size;
|
||||
}
|
||||
|
||||
@@ -1,64 +0,0 @@
|
||||
#ifndef RINGBUFFER_H
|
||||
#define RINGBUFFER_H
|
||||
|
||||
#include <list>
|
||||
#include <vector>
|
||||
#include <map>
|
||||
#include <mutex>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <boost/any.hpp>
|
||||
#include <chrono>
|
||||
#include "date.h"
|
||||
|
||||
struct FrameMetadata
|
||||
{
|
||||
// Ring buffer needed data.
|
||||
size_t buffer_slot_index;
|
||||
size_t frame_bytes_size;
|
||||
|
||||
// Image header data.
|
||||
uint64_t frame_index;
|
||||
std::string endianness;
|
||||
std::string type;
|
||||
std::vector<size_t> frame_shape;
|
||||
|
||||
// Pass additional header values.
|
||||
std::map<std::string, std::shared_ptr<char>> header_values;
|
||||
};
|
||||
|
||||
class RingBuffer
|
||||
{
|
||||
// Initialized in constructor.
|
||||
size_t n_slots = 0;
|
||||
std::vector<bool> ringbuffer_slots;
|
||||
|
||||
// Set in initialize().
|
||||
size_t slot_size = 0;
|
||||
size_t buffer_size = 0;
|
||||
char* frame_data_buffer = NULL;
|
||||
size_t write_index = 0;
|
||||
size_t buffer_used_slots = 0;
|
||||
bool ring_buffer_initialized = false;
|
||||
|
||||
std::list< std::shared_ptr<FrameMetadata> > frame_metadata_queue;
|
||||
std::mutex frame_metadata_queue_mutex;
|
||||
std::mutex ringbuffer_slots_mutex;
|
||||
|
||||
char* get_buffer_slot_address(size_t buffer_slot_index);
|
||||
|
||||
public:
|
||||
RingBuffer(size_t n_slots);
|
||||
virtual ~RingBuffer();
|
||||
void initialize(size_t slot_size);
|
||||
|
||||
char* reserve(std::shared_ptr<FrameMetadata> metadata);
|
||||
void commit(std::shared_ptr<FrameMetadata> metadata);
|
||||
std::pair<std::shared_ptr<FrameMetadata>, char*> read();
|
||||
void release(size_t buffer_slot_index);
|
||||
bool is_empty();
|
||||
void clear();
|
||||
size_t get_slot_size();
|
||||
};
|
||||
|
||||
#endif
|
||||
@@ -1,490 +0,0 @@
|
||||
#include <iostream>
|
||||
#include <sstream>
|
||||
|
||||
#include <boost/thread.hpp>
|
||||
#include "WriterManager.hpp"
|
||||
#include "MetadataBuffer.hpp"
|
||||
#include "BufferedWriter.hpp"
|
||||
#include "config.hpp"
|
||||
|
||||
using namespace std;
|
||||
|
||||
void writer_utils::set_process_id(int user_id)
|
||||
{
|
||||
|
||||
#ifdef DEBUG_OUTPUT
|
||||
using namespace date;
|
||||
using namespace chrono;
|
||||
cout << "[" << system_clock::now() << "]";
|
||||
cout << "[writer_utils::set_process_id] Setting process user to ";
|
||||
cout << user_id << endl;
|
||||
#endif
|
||||
|
||||
if (setegid(user_id)) {
|
||||
stringstream error_message;
|
||||
|
||||
using namespace date;
|
||||
using namespace chrono;
|
||||
error_message << "[" << system_clock::now() << "]";
|
||||
error_message << "[writer_utils::set_process_id] Cannot set group_id to ";
|
||||
error_message << user_id << endl;
|
||||
|
||||
throw runtime_error(error_message.str());
|
||||
}
|
||||
|
||||
if (seteuid(user_id)) {
|
||||
stringstream error_message;
|
||||
|
||||
using namespace date;
|
||||
using namespace chrono;
|
||||
error_message << "[" << system_clock::now() << "]";
|
||||
error_message << "[writer_utils::set_process_id] Cannot set user_id to ";
|
||||
error_message << user_id << endl;
|
||||
|
||||
throw runtime_error(error_message.str());
|
||||
}
|
||||
}
|
||||
|
||||
void writer_utils::create_destination_folder(const string& output_file)
|
||||
{
|
||||
auto file_separator_index = output_file.rfind('/');
|
||||
|
||||
if (file_separator_index != string::npos) {
|
||||
string output_folder(output_file.substr(0, file_separator_index));
|
||||
|
||||
using namespace date;
|
||||
using namespace chrono;
|
||||
cout << "[" << system_clock::now() << "]";
|
||||
cout << "[writer_utils::create_destination_folder] Creating folder ";
|
||||
cout << output_folder << endl;
|
||||
|
||||
string create_folder_command("mkdir -p " + output_folder);
|
||||
system(create_folder_command.c_str());
|
||||
}
|
||||
}
|
||||
|
||||
WriterManager::WriterManager(
|
||||
ZmqReceiver& receiver,
|
||||
RingBuffer& ring_buffer,
|
||||
const H5Format& format,
|
||||
std::shared_ptr<header_map> header_values_type,
|
||||
uint8_t n_receiving_threads,
|
||||
hsize_t frames_per_file):
|
||||
receiver(receiver),
|
||||
ring_buffer(ring_buffer),
|
||||
format(format),
|
||||
header_values_type(header_values_type),
|
||||
n_receiving_threads(n_receiving_threads),
|
||||
frames_per_file(frames_per_file),
|
||||
receiving_threads(frames_per_file),
|
||||
logs(10)
|
||||
{
|
||||
running_flag = true;
|
||||
|
||||
writing_flag = false;
|
||||
receiving_flag = false;
|
||||
|
||||
n_frames_to_receive = 0;
|
||||
n_frames_to_write = 0;
|
||||
|
||||
#ifdef DEBUG_OUTPUT
|
||||
using namespace date;
|
||||
using namespace chrono;
|
||||
|
||||
cout << "[" << system_clock::now() << "]";
|
||||
cout << "[WriterManager::WriterManager]"
|
||||
cout << " Writer manager initialized." << endl;
|
||||
#endif
|
||||
}
|
||||
|
||||
WriterManager::~WriterManager()
|
||||
{
|
||||
stop();
|
||||
}
|
||||
|
||||
void WriterManager::stop()
|
||||
{
|
||||
#ifdef DEBUG_OUTPUT
|
||||
using namespace date;
|
||||
using namespace chrono;
|
||||
cout << "[" << system_clock::now() << "]";
|
||||
cout << "[WriterManager::stop] Stopping the writer." << endl;
|
||||
#endif
|
||||
|
||||
running_flag = false;
|
||||
|
||||
for (auto& thread_ptr:receiving_threads) {
|
||||
thread_ptr->join();
|
||||
}
|
||||
receiving_threads.clear();
|
||||
|
||||
writing_thread.join();
|
||||
}
|
||||
|
||||
string WriterManager::get_status()
|
||||
{
|
||||
if (writing_flag) {
|
||||
return "writing";
|
||||
} else if (running_flag) {
|
||||
return "ready";
|
||||
} else {
|
||||
return "Error.. I guess. This shouldn't be possible?";
|
||||
}
|
||||
}
|
||||
|
||||
unordered_map<string, uint64_t> WriterManager::get_statistics() const
|
||||
{
|
||||
unordered_map<string, uint64_t> result = {
|
||||
{"n_frames_receive", n_frames_to_receive.load()},
|
||||
{"n_frames_to_write", n_frames_to_write.load()}
|
||||
};
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
void WriterManager::start(const string output_file,
|
||||
const int n_frames,
|
||||
const int user_id)
|
||||
{
|
||||
|
||||
#ifdef DEBUG_OUTPUT
|
||||
using namespace date;
|
||||
using namespace chrono;
|
||||
cout << "[" << system_clock::now() << "]";
|
||||
cout << "[WriterManager::start] Starting with parameters: ";
|
||||
cout << "\toutput_file: " << output_file;
|
||||
cout << "\tn_frames: " << n_frames;
|
||||
cout << "\tuser_id: " << user_id;
|
||||
cout << endl;
|
||||
#endif
|
||||
|
||||
n_frames_to_write = n_frames;
|
||||
writing_flag = true;
|
||||
|
||||
n_frames_to_receive = n_frames;
|
||||
receiving_flag = true;
|
||||
|
||||
|
||||
writing_thread = boost::thread(&WriterManager::write_h5,
|
||||
this,
|
||||
output_file,
|
||||
n_frames);
|
||||
|
||||
for (uint8_t i_rec=0; i_rec < n_receiving_threads; i_rec++) {
|
||||
receiving_threads.push_back(unique_ptr<boost::thread>(
|
||||
new boost::thread(&WriterManager::receive_zmq, this)
|
||||
));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
bool WriterManager::is_running() const
|
||||
{
|
||||
return running_flag.load();
|
||||
}
|
||||
|
||||
bool WriterManager::is_writing() const
|
||||
{
|
||||
return writing_flag.load();
|
||||
}
|
||||
|
||||
bool WriterManager::should_receive_frame() {
|
||||
if (n_frames_to_receive > 0) {
|
||||
return (n_frames_to_receive.fetch_sub(1) >= 0);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
bool WriterManager::should_write_frame() {
|
||||
if (n_frames_to_write > 0) {
|
||||
return (n_frames_to_write.fetch_sub(1) >= 0);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
void WriterManager::writing_completed() {
|
||||
writing_flag = false;
|
||||
|
||||
#ifdef DEBUG_OUTPUT
|
||||
using namespace date;
|
||||
using namespace chrono;
|
||||
cout << "[" << system_clock::now() << "]";
|
||||
cout << "[WriterManager::writing_completed]"
|
||||
cout << " Writing has finished." << endl;
|
||||
#endif
|
||||
|
||||
//TODO: Send this event somewhere somehow?
|
||||
}
|
||||
|
||||
void WriterManager::writing_error(string error) {
|
||||
|
||||
}
|
||||
|
||||
void WriterManager::write_h5_format(H5::H5File& file) {
|
||||
|
||||
try {
|
||||
H5FormatUtils::write_format(file, format, {});
|
||||
} catch (const runtime_error& ex) {
|
||||
using namespace date;
|
||||
using namespace chrono;
|
||||
|
||||
cout << "[" << system_clock::now() << "]";
|
||||
cout << "[ProcessManager::write_h5_format] Error while";
|
||||
cout << " trying to write file format: "<< ex.what() << endl;
|
||||
}
|
||||
}
|
||||
|
||||
void WriterManager::write_h5(const string output_file, const uint64_t n_frames)
|
||||
{
|
||||
try {
|
||||
|
||||
size_t metadata_buffer_size =
|
||||
frames_per_file != 0 ? frames_per_file : n_frames;
|
||||
|
||||
auto metadata_buffer = unique_ptr<MetadataBuffer>(
|
||||
new MetadataBuffer(metadata_buffer_size,
|
||||
header_values_type));
|
||||
|
||||
auto writer = get_buffered_writer(
|
||||
output_file,
|
||||
n_frames,
|
||||
move(metadata_buffer),
|
||||
frames_per_file,
|
||||
config::dataset_increase_step);
|
||||
|
||||
writer->create_file();
|
||||
|
||||
auto raw_frames_dataset_name = config::raw_image_dataset_name;
|
||||
|
||||
uint64_t last_pulse_id = 0;
|
||||
|
||||
while(is_writing() || !ring_buffer.is_empty()) {
|
||||
|
||||
if (ring_buffer.is_empty()) {
|
||||
boost::this_thread::sleep_for(
|
||||
boost::chrono::milliseconds(
|
||||
config::ring_buffer_read_retry_interval));
|
||||
continue;
|
||||
}
|
||||
|
||||
const pair< shared_ptr<FrameMetadata>, char* > received_data =
|
||||
ring_buffer.read();
|
||||
|
||||
// NULL pointer means that the ringbuffer->read() timeouted.
|
||||
if(!received_data.first) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// The acquisition stops when there are no more frames to write.
|
||||
if (!should_write_frame()) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Write file format before rolling to next file.
|
||||
if (!writer->is_data_for_current_file(
|
||||
received_data.first->frame_index)) {
|
||||
|
||||
#ifdef DEBUG_OUTPUT
|
||||
using namespace date;
|
||||
using namespace chrono;
|
||||
|
||||
cout << "[" << system_clock::now() << "]";
|
||||
cout << "[WriterManager::write_h5] Frame index ";
|
||||
cout << received_data.first->frame_index;
|
||||
cout << " does not belong to current file. ";
|
||||
cout << " Write format before switching file." << endl;
|
||||
#endif
|
||||
|
||||
writer->write_metadata_to_file();
|
||||
|
||||
write_h5_format(writer->get_h5_file());
|
||||
}
|
||||
|
||||
#ifdef PERF_OUTPUT
|
||||
using namespace date;
|
||||
using namespace chrono;
|
||||
|
||||
auto start_time_frame = system_clock::now();
|
||||
#endif
|
||||
|
||||
// Write image data.
|
||||
writer->write_data(raw_frames_dataset_name,
|
||||
received_data.first->frame_index,
|
||||
received_data.second,
|
||||
received_data.first->frame_shape,
|
||||
received_data.first->frame_bytes_size,
|
||||
received_data.first->type,
|
||||
received_data.first->endianness);
|
||||
|
||||
#ifdef PERF_OUTPUT
|
||||
using namespace date;
|
||||
using namespace chrono;
|
||||
|
||||
auto frame_time_difference = system_clock::now() - start_time_frame;
|
||||
|
||||
auto frame_diff_ms =
|
||||
duration<float, milli>(frame_time_difference).count();
|
||||
|
||||
cout << "[" << system_clock::now() << "]";
|
||||
cout << "[WriterManager::write_h5] Frame index ";
|
||||
cout << received_data.first->frame_index;
|
||||
cout << " written in " << frame_diff_ms << " ms." << endl;
|
||||
#endif
|
||||
|
||||
ring_buffer.release(received_data.first->buffer_slot_index);
|
||||
|
||||
#ifdef PERF_OUTPUT
|
||||
using namespace date;
|
||||
using namespace chrono;
|
||||
|
||||
auto start_time_metadata = system_clock::now();
|
||||
#endif
|
||||
|
||||
// Write image metadata if mapping specified.
|
||||
if (header_values_type) {
|
||||
|
||||
for (const auto& header_type : *header_values_type) {
|
||||
|
||||
auto& name = header_type.first;
|
||||
auto value = received_data.first->header_values.at(name);
|
||||
|
||||
// TODO: Ugly hack until we get the start sequence in bsread.
|
||||
if (name == "pulse_id") {
|
||||
if (!last_pulse_id) {
|
||||
last_pulse_id = *(reinterpret_cast<uint64_t*>(value.get()));
|
||||
//notify_first_pulse_id(last_pulse_id);
|
||||
} else {
|
||||
last_pulse_id = *(reinterpret_cast<uint64_t*>(value.get()));
|
||||
}
|
||||
}
|
||||
|
||||
writer->cache_metadata(name, received_data.first->frame_index, value.get());
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef PERF_OUTPUT
|
||||
using namespace date;
|
||||
using namespace chrono;
|
||||
|
||||
auto metadata_time_difference = system_clock::now() - start_time_metadata;
|
||||
auto metadata_diff_ms = duration<float, milli>(metadata_time_difference).count();
|
||||
|
||||
cout << "[" << system_clock::now() << "]";
|
||||
cout << "[ProcessManager::write_h5] Frame metadata index ";
|
||||
cout << received_data.first->frame_index << " written in " << metadata_diff_ms << " ms." << endl;
|
||||
#endif
|
||||
}
|
||||
|
||||
// Send the last_pulse_id only if it was set.
|
||||
if (last_pulse_id) {
|
||||
//notify_last_pulse_id(last_pulse_id);
|
||||
}
|
||||
|
||||
if (writer->is_file_open()) {
|
||||
#ifdef DEBUG_OUTPUT
|
||||
using namespace date;
|
||||
using namespace chrono;
|
||||
|
||||
cout << "[" << system_clock::now() << "]";
|
||||
cout << "[ProcessManager::write] Writing file format." << endl;
|
||||
#endif
|
||||
|
||||
writer->write_metadata_to_file();
|
||||
|
||||
write_h5_format(writer->get_h5_file());
|
||||
}
|
||||
|
||||
#ifdef DEBUG_OUTPUT
|
||||
using namespace date;
|
||||
using namespace chrono;
|
||||
|
||||
cout << "[" << system_clock::now() << "]";
|
||||
cout << "[ProcessManager::write] Closing file " << output_file << endl;
|
||||
#endif
|
||||
|
||||
writer->close_file();
|
||||
|
||||
#ifdef DEBUG_OUTPUT
|
||||
using namespace date;
|
||||
using namespace chrono;
|
||||
|
||||
cout << "[" << system_clock::now() << "]";
|
||||
cout << "[ProcessManager::write] Writer thread stopped." << endl;
|
||||
#endif
|
||||
|
||||
writing_completed();
|
||||
|
||||
} catch (const exception& ex) {
|
||||
writing_error(ex.what());
|
||||
}
|
||||
}
|
||||
|
||||
void WriterManager::receive_zmq()
|
||||
{
|
||||
receiver.connect();
|
||||
|
||||
while (is_running()) {
|
||||
|
||||
auto frame = receiver.receive();
|
||||
|
||||
// If receive timeout, both pointers are NULL.
|
||||
if (!frame.first || !receive_frame()){
|
||||
continue;
|
||||
}
|
||||
|
||||
auto frame_metadata = frame.first;
|
||||
auto frame_data = frame.second;
|
||||
|
||||
#ifdef DEBUG_OUTPUT
|
||||
using namespace date;
|
||||
using namespace chrono;
|
||||
cout << "[" << system_clock::now() << "]";
|
||||
cout << "[ProcessManager::receive_zmq] Processing FrameMetadata";
|
||||
cout << " with frame_index " << frame_metadata->frame_index;
|
||||
cout << " and frame_shape [" << frame_metadata->frame_shape[0];
|
||||
cout << ", " << frame_metadata->frame_shape[1] << "]";
|
||||
cout << " and endianness " << frame_metadata->endianness;
|
||||
cout << " and type " << frame_metadata->type;
|
||||
cout << " and frame_bytes_size ";
|
||||
cout << frame_metadata->frame_bytes_size << "." << endl;
|
||||
#endif
|
||||
|
||||
char* buffer = ring_buffer.reserve(frame_metadata);
|
||||
|
||||
size_t max_buffer_size = compression::get_bitshuffle_max_buffer_size(
|
||||
frame_metadata->frame_bytes_size, 1);
|
||||
|
||||
if (max_buffer_size > ring_buffer.get_slot_size()) {
|
||||
|
||||
}
|
||||
|
||||
auto compressed_size = compression::compress_bitshuffle(
|
||||
static_cast<const char*>(frame_data),
|
||||
frame_metadata->frame_bytes_size,
|
||||
1,
|
||||
buffer);
|
||||
|
||||
#ifdef DEBUG_OUTPUT
|
||||
using namespace date;
|
||||
using namespace chrono;
|
||||
cout << "[" << system_clock::now() << "]";
|
||||
cout << "[WriterManager::receive_zmq] Compressed image from ";
|
||||
cout << frame_metadata->frame_bytes_size << " bytes to ";
|
||||
cout << compressed_size << " bytes." << endl;
|
||||
#endif
|
||||
|
||||
frame_metadata->frame_bytes_size = compressed_size;
|
||||
|
||||
ring_buffer.commit(frame_metadata);
|
||||
}
|
||||
|
||||
#ifdef DEBUG_OUTPUT
|
||||
using namespace date;
|
||||
cout << "[" << std::chrono::system_clock::now() << "]";
|
||||
cout << "[WriterManager::receive_zmq] Receiver thread stopped." << endl;
|
||||
#endif
|
||||
}
|
||||
@@ -1,99 +0,0 @@
|
||||
#ifndef WRITERMANAGER_H
|
||||
#define WRITERMANAGER_H
|
||||
|
||||
#include <unordered_map>
|
||||
#include <string>
|
||||
#include <atomic>
|
||||
#include <mutex>
|
||||
#include <boost/any.hpp>
|
||||
#include <boost/thread.hpp>
|
||||
#include <chrono>
|
||||
#include "date.h"
|
||||
#include <deque>
|
||||
#include <memory>
|
||||
#include <list>
|
||||
|
||||
#include "ZmqReceiver.hpp"
|
||||
#include "RingBuffer.hpp"
|
||||
#include "H5Format.hpp"
|
||||
#include "compression/compression.h"
|
||||
|
||||
namespace writer_utils {
|
||||
void set_process_id(int user_id);
|
||||
void create_destination_folder(const std::string& output_file);
|
||||
}
|
||||
|
||||
struct WriterManagerLog
|
||||
{
|
||||
std::string filename;
|
||||
std::string status;
|
||||
|
||||
uint64_t n_requested_frames;
|
||||
uint64_t n_received_frames;
|
||||
uint64_t n_written_frames;
|
||||
};
|
||||
|
||||
|
||||
class WriterManager
|
||||
{
|
||||
// Initialize in constructor.
|
||||
|
||||
std::atomic<bool> running_flag;
|
||||
|
||||
std::atomic<bool> writing_flag;
|
||||
std::atomic<bool> receiving_flag;
|
||||
|
||||
std::atomic<int64_t> n_frames_to_receive;
|
||||
std::atomic<int64_t> n_frames_to_write;
|
||||
|
||||
protected:
|
||||
RingBuffer& ring_buffer;
|
||||
|
||||
ZmqReceiver& receiver;
|
||||
uint8_t n_receiving_threads;
|
||||
|
||||
const H5Format& format;
|
||||
hsize_t frames_per_file;
|
||||
|
||||
std::list<std::unique_ptr<boost::thread>> receiving_threads;
|
||||
boost::thread writing_thread;
|
||||
|
||||
typedef std::unordered_map<std::string, HeaderDataType> header_map;
|
||||
std::shared_ptr<header_map> header_values_type = NULL;
|
||||
|
||||
const std::deque<WriterManagerLog> logs;
|
||||
|
||||
void receive_zmq();
|
||||
void write_h5(std::string output_file, uint64_t n_frames);
|
||||
|
||||
void write_h5_format(H5::H5File& file);
|
||||
|
||||
public:
|
||||
|
||||
WriterManager(ZmqReceiver& receiver,
|
||||
RingBuffer& ring_buffer,
|
||||
const H5Format& format,
|
||||
std::shared_ptr<header_map> header_values_type,
|
||||
uint8_t n_receiving_threads,
|
||||
hsize_t frames_per_file=0);
|
||||
|
||||
virtual ~WriterManager();
|
||||
|
||||
void start(std::string output_file, int n_frames, int user_id);
|
||||
void stop();
|
||||
|
||||
std::string get_status();
|
||||
std::unordered_map<std::string, uint64_t> get_statistics() const;
|
||||
|
||||
bool receive_frame();
|
||||
bool is_running() const;
|
||||
bool is_writing() const;
|
||||
|
||||
bool should_write_frame();
|
||||
bool should_receive_frame();
|
||||
|
||||
void writing_completed();
|
||||
void writing_error(std::string error);
|
||||
};
|
||||
|
||||
#endif
|
||||
@@ -1,255 +0,0 @@
|
||||
#include <iostream>
|
||||
#include <stdexcept>
|
||||
|
||||
#include "config.hpp"
|
||||
#include "ZmqReceiver.hpp"
|
||||
#include "H5Format.hpp"
|
||||
|
||||
using namespace std;
|
||||
namespace pt = boost::property_tree;
|
||||
|
||||
HeaderDataType::HeaderDataType(const std::string& type, size_t shape) :
|
||||
type(type), value_shape(shape), endianness("little"), is_array(true) {
|
||||
value_bytes_size = get_type_byte_size(type);
|
||||
}
|
||||
|
||||
HeaderDataType::HeaderDataType(const std::string& type) :
|
||||
type(type), value_shape(1), endianness("little"), is_array(false) {
|
||||
value_bytes_size = get_type_byte_size(type);
|
||||
}
|
||||
|
||||
size_t get_type_byte_size(const string& type)
|
||||
{
|
||||
if (type == "uint8" || type== "int8") {
|
||||
return 1;
|
||||
|
||||
} else if (type == "uint16" || type == "int16") {
|
||||
return 2;
|
||||
|
||||
} else if (type == "uint32" || type == "int32" || type == "float32") {
|
||||
return 4;
|
||||
|
||||
} else if (type == "uint64" || type == "int64" || type == "float64") {
|
||||
return 8;
|
||||
|
||||
} else {
|
||||
stringstream error_message;
|
||||
using namespace date;
|
||||
error_message << "[" << std::chrono::system_clock::now() << "]";
|
||||
error_message << "[ZmqReceiver::get_type_byte_size] Unsupported data type " << type << endl;
|
||||
|
||||
throw runtime_error(error_message.str());
|
||||
}
|
||||
}
|
||||
|
||||
ZmqReceiver::ZmqReceiver(const std::string& connect_address, const int n_io_threads, const int receive_timeout,
|
||||
shared_ptr<unordered_map<string, HeaderDataType>> header_values_type) :
|
||||
connect_address(connect_address), n_io_threads(n_io_threads),
|
||||
receive_timeout(receive_timeout), receiver(NULL), header_values_type(header_values_type)
|
||||
|
||||
{
|
||||
#ifdef DEBUG_OUTPUT
|
||||
using namespace date;
|
||||
cout << "[" << std::chrono::system_clock::now() << "]";
|
||||
cout << "[ZmqReceiver::ZmqReceiver] Creating ZMQ receiver with";
|
||||
cout << " connect_address " << connect_address;
|
||||
cout << " n_io_threads " << n_io_threads;
|
||||
cout << " receive_timeout " << receive_timeout;
|
||||
cout << endl;
|
||||
#endif
|
||||
|
||||
message_header = zmq::message_t(config::zmq_buffer_size_header);
|
||||
message_data = zmq::message_t(config::zmq_buffer_size_data);
|
||||
}
|
||||
|
||||
|
||||
ZmqReceiver::ZmqReceiver(const ZmqReceiver& other) :
|
||||
ZmqReceiver(other.connect_address, other.n_io_threads, other.receive_timeout, other.header_values_type)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
void ZmqReceiver::connect()
|
||||
{
|
||||
#ifdef DEBUG_OUTPUT
|
||||
using namespace date;
|
||||
cout << "[" << std::chrono::system_clock::now() << "]";
|
||||
cout << "[ZmqReceiver::connect] Connecting to address " << connect_address;
|
||||
cout << " with n_io_threads " << n_io_threads << endl;
|
||||
#endif
|
||||
|
||||
context = make_shared<zmq::context_t>(n_io_threads);
|
||||
receiver = make_shared<zmq::socket_t>(*context, ZMQ_PULL);
|
||||
|
||||
receiver->setsockopt(ZMQ_RCVTIMEO, receive_timeout);
|
||||
receiver->connect(connect_address);
|
||||
}
|
||||
|
||||
pair<shared_ptr<FrameMetadata>, char*> ZmqReceiver::receive()
|
||||
{
|
||||
if (!receiver) {
|
||||
stringstream error_message;
|
||||
using namespace date;
|
||||
error_message << "[" << std::chrono::system_clock::now() << "]";
|
||||
error_message << "[ZmqReceiver::receive] Cannot receive before connecting. ";
|
||||
error_message << "Connect first." << endl;
|
||||
|
||||
throw runtime_error(error_message.str());
|
||||
}
|
||||
|
||||
// Get the message header.
|
||||
if (!receiver->recv(&message_header)){
|
||||
return {NULL, NULL};
|
||||
}
|
||||
|
||||
auto header_string = string(static_cast<char*>(message_header.data()), message_header.size());
|
||||
auto frame_metadata = read_json_header(header_string);
|
||||
|
||||
// Get the message data.
|
||||
if (!receiver->recv(&message_data)) {
|
||||
using namespace date;
|
||||
cout << "[" << std::chrono::system_clock::now() << "]";
|
||||
cout << "[ZmqReceiver::receive] Error while reading from ZMQ. Frame index " << frame_metadata->frame_index << " lost.";
|
||||
cout << " Trying to continue with the next frame." << endl;
|
||||
|
||||
return {NULL, NULL};
|
||||
}
|
||||
|
||||
frame_metadata->frame_bytes_size = message_data.size();
|
||||
|
||||
return {frame_metadata, static_cast<char*>(message_data.data())};
|
||||
}
|
||||
|
||||
shared_ptr<FrameMetadata> ZmqReceiver::read_json_header(const string& header)
|
||||
{
|
||||
try {
|
||||
|
||||
stringstream header_stream;
|
||||
header_stream << header << endl;
|
||||
pt::read_json(header_stream, json_header);
|
||||
|
||||
auto header_data = make_shared<FrameMetadata>();
|
||||
|
||||
header_data->frame_index = json_header.get<uint64_t>("frame");
|
||||
|
||||
for (const auto& item : json_header.get_child("shape")) {
|
||||
header_data->frame_shape.push_back(item.second.get_value<size_t>());
|
||||
}
|
||||
|
||||
// Array 1.0 specified little endian as the default encoding.
|
||||
header_data->endianness = json_header.get("endianness", "little");
|
||||
|
||||
header_data->type = json_header.get<string>("type");
|
||||
|
||||
if (header_values_type) {
|
||||
for (const auto& value_mapping : *header_values_type) {
|
||||
|
||||
const auto& name = value_mapping.first;
|
||||
const auto& header_data_type = value_mapping.second;
|
||||
|
||||
auto value = get_value_from_json(json_header, name, header_data_type);
|
||||
|
||||
header_data->header_values.insert(
|
||||
{name, value}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return header_data;
|
||||
|
||||
} catch (...) {
|
||||
using namespace date;
|
||||
cout << "[" << std::chrono::system_clock::now() << "]";
|
||||
cout << "[ZmqReceiver::read_json_header] Error while interpreting the JSON header. Header string: " << header << endl;
|
||||
cout << "Expected JSON header format: " << endl;
|
||||
|
||||
if (header_values_type) {
|
||||
for (const auto& value_mapping : *header_values_type) {
|
||||
cout << "\t" << value_mapping.first << ":" << value_mapping.second.type;
|
||||
cout << "[" << value_mapping.second.value_shape << "]" << endl;
|
||||
}
|
||||
} else {
|
||||
cout << "\tExpected header value types is a null pointer." << endl;
|
||||
}
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
void copy_value_to_buffer(char* buffer, const size_t offset, const pt::ptree& json_value, const HeaderDataType& header_data_type)
|
||||
{
|
||||
if (header_data_type.type == "uint8") {
|
||||
auto value = json_value.get_value<uint8_t>();
|
||||
memcpy(buffer + offset, reinterpret_cast<char*>(&value), header_data_type.value_bytes_size);
|
||||
|
||||
} else if (header_data_type.type == "uint16") {
|
||||
auto value = json_value.get_value<uint16_t>();
|
||||
memcpy(buffer + offset, reinterpret_cast<char*>(&value), header_data_type.value_bytes_size);
|
||||
|
||||
} else if (header_data_type.type == "uint32") {
|
||||
auto value = json_value.get_value<uint32_t>();
|
||||
memcpy(buffer + offset, reinterpret_cast<char*>(&value), header_data_type.value_bytes_size);
|
||||
|
||||
} else if (header_data_type.type == "uint64") {
|
||||
auto value = json_value.get_value<uint64_t>();
|
||||
memcpy(buffer + offset, reinterpret_cast<char*>(&value), header_data_type.value_bytes_size);
|
||||
|
||||
} else if (header_data_type.type == "int8") {
|
||||
auto value = json_value.get_value<int8_t>();
|
||||
memcpy(buffer + offset, reinterpret_cast<char*>(&value), header_data_type.value_bytes_size);
|
||||
|
||||
} else if (header_data_type.type == "int16") {
|
||||
auto value = json_value.get_value<int16_t>();
|
||||
memcpy(buffer + offset, reinterpret_cast<char*>(&value), header_data_type.value_bytes_size);
|
||||
|
||||
} else if (header_data_type.type == "int32") {
|
||||
auto value = json_value.get_value<int32_t>();
|
||||
memcpy(buffer + offset, reinterpret_cast<char*>(&value), header_data_type.value_bytes_size);
|
||||
|
||||
} else if (header_data_type.type == "int64") {
|
||||
auto value = json_value.get_value<int64_t>();
|
||||
memcpy(buffer + offset, reinterpret_cast<char*>(&value), header_data_type.value_bytes_size);
|
||||
|
||||
} else if (header_data_type.type == "float32") {
|
||||
auto value = json_value.get_value<float>();
|
||||
memcpy(buffer + offset, reinterpret_cast<char*>(&value), header_data_type.value_bytes_size);
|
||||
|
||||
} else if (header_data_type.type == "float64") {
|
||||
auto value = json_value.get_value<double>();
|
||||
memcpy(buffer + offset, reinterpret_cast<char*>(&value), header_data_type.value_bytes_size);
|
||||
|
||||
} else {
|
||||
// We cannot really convert this attribute.
|
||||
stringstream error_message;
|
||||
using namespace date;
|
||||
error_message << "[" << std::chrono::system_clock::now() << "]";
|
||||
error_message << "[ZmqReceiver::get_value_from_json] Unsupported header data type " << header_data_type.type << endl;
|
||||
|
||||
throw runtime_error(error_message.str());
|
||||
}
|
||||
}
|
||||
|
||||
shared_ptr<char> get_value_from_json(const pt::ptree& json_header, const string& name, const HeaderDataType& header_data_type)
|
||||
{
|
||||
char* buffer = new char[header_data_type.value_bytes_size * header_data_type.value_shape];
|
||||
|
||||
if (header_data_type.is_array) {
|
||||
size_t index = 0;
|
||||
|
||||
for (const auto& item : json_header.get_child(name)) {
|
||||
auto offset = index * header_data_type.value_bytes_size;
|
||||
copy_value_to_buffer(buffer, offset, item.second, header_data_type);
|
||||
|
||||
++index;
|
||||
}
|
||||
|
||||
} else {
|
||||
copy_value_to_buffer(buffer, 0, json_header.get_child(name), header_data_type);
|
||||
}
|
||||
|
||||
return shared_ptr<char>(buffer, default_delete<char[]>());
|
||||
}
|
||||
|
||||
const shared_ptr<unordered_map<string, HeaderDataType>> ZmqReceiver::get_header_values_type() const
|
||||
{
|
||||
return header_values_type;
|
||||
}
|
||||
@@ -1,68 +0,0 @@
|
||||
#ifndef ZMQRECEIVER_H
|
||||
#define ZMQRECEIVER_H
|
||||
|
||||
#include <string>
|
||||
#include <memory>
|
||||
#include <tuple>
|
||||
#include <zmq.hpp>
|
||||
#include <vector>
|
||||
#include <memory>
|
||||
#include <unordered_map>
|
||||
#include <boost/property_tree/json_parser.hpp>
|
||||
#include <chrono>
|
||||
#include "date.h"
|
||||
|
||||
#include "RingBuffer.hpp"
|
||||
|
||||
struct HeaderDataType
|
||||
{
|
||||
std::string type;
|
||||
size_t value_shape;
|
||||
std::string endianness;
|
||||
size_t value_bytes_size;
|
||||
bool is_array;
|
||||
|
||||
HeaderDataType(const std::string& type);
|
||||
HeaderDataType(const std::string& type, size_t shape);
|
||||
};
|
||||
|
||||
size_t get_type_byte_size(const std::string& type);
|
||||
|
||||
void copy_value_to_buffer(const char* buffer, size_t offset, const boost::property_tree::ptree& json_value,
|
||||
const HeaderDataType& header_data_type);
|
||||
|
||||
std::shared_ptr<char> get_value_from_json(const boost::property_tree::ptree& json_header,
|
||||
const std::string& name, const HeaderDataType& header_data_type);
|
||||
|
||||
class ZmqReceiver
|
||||
{
|
||||
const std::string connect_address;
|
||||
const int n_io_threads;
|
||||
const int receive_timeout;
|
||||
std::shared_ptr<zmq::socket_t> receiver = NULL;
|
||||
std::shared_ptr<zmq::context_t> context = NULL;
|
||||
zmq::message_t message_header;
|
||||
zmq::message_t message_data;
|
||||
boost::property_tree::ptree json_header;
|
||||
|
||||
std::shared_ptr<std::unordered_map<std::string, HeaderDataType>> header_values_type = NULL;
|
||||
|
||||
public:
|
||||
ZmqReceiver(const std::string& connect_address, const int n_io_threads, const int receive_timeout,
|
||||
std::shared_ptr<std::unordered_map<std::string, HeaderDataType>> header_values_type=NULL);
|
||||
|
||||
ZmqReceiver(const ZmqReceiver& other);
|
||||
|
||||
virtual ~ZmqReceiver(){};
|
||||
|
||||
void connect();
|
||||
|
||||
std::shared_ptr<FrameMetadata> read_json_header(const std::string& header);
|
||||
|
||||
std::pair<std::shared_ptr<FrameMetadata>, char*> receive();
|
||||
|
||||
const std::shared_ptr<std::unordered_map<std::string, HeaderDataType>> get_header_values_type() const;
|
||||
|
||||
};
|
||||
|
||||
#endif
|
||||
@@ -1,165 +0,0 @@
|
||||
/*
|
||||
* Bitshuffle - Filter for improving compression of typed binary data.
|
||||
*
|
||||
* Author: Kiyoshi Masui <kiyo@physics.ubc.ca>
|
||||
* Website: http://www.github.com/kiyo-masui/bitshuffle
|
||||
* Created: 2014
|
||||
*
|
||||
* See LICENSE file for details about copyright and rights to use.
|
||||
*
|
||||
*/
|
||||
|
||||
#include "bitshuffle.h"
|
||||
#include "bitshuffle_core.h"
|
||||
#include "bitshuffle_internals.h"
|
||||
#include "lz4.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
|
||||
// Constants.
|
||||
// Use fast decompression instead of safe decompression for LZ4.
|
||||
#define BSHUF_LZ4_DECOMPRESS_FAST
|
||||
|
||||
|
||||
// Macros.
|
||||
#define CHECK_ERR_FREE_LZ(count, buf) if (count < 0) { \
|
||||
free(buf); return count - 1000; }
|
||||
|
||||
|
||||
/* Bitshuffle and compress a single block. */
|
||||
int64_t bshuf_compress_lz4_block(ioc_chain *C_ptr, \
|
||||
const size_t size, const size_t elem_size) {
|
||||
|
||||
int64_t nbytes, count;
|
||||
void *tmp_buf_bshuf;
|
||||
void *tmp_buf_lz4;
|
||||
size_t this_iter;
|
||||
const void *in;
|
||||
void *out;
|
||||
|
||||
tmp_buf_bshuf = malloc(size * elem_size);
|
||||
if (tmp_buf_bshuf == NULL) return -1;
|
||||
|
||||
tmp_buf_lz4 = malloc(LZ4_compressBound(size * elem_size));
|
||||
if (tmp_buf_lz4 == NULL){
|
||||
free(tmp_buf_bshuf);
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
||||
in = ioc_get_in(C_ptr, &this_iter);
|
||||
ioc_set_next_in(C_ptr, &this_iter, (void*) ((char*) in + size * elem_size));
|
||||
|
||||
count = bshuf_trans_bit_elem(in, tmp_buf_bshuf, size, elem_size);
|
||||
if (count < 0) {
|
||||
free(tmp_buf_lz4);
|
||||
free(tmp_buf_bshuf);
|
||||
return count;
|
||||
}
|
||||
nbytes = LZ4_compress((const char*) tmp_buf_bshuf, (char*) tmp_buf_lz4, size * elem_size);
|
||||
free(tmp_buf_bshuf);
|
||||
CHECK_ERR_FREE_LZ(nbytes, tmp_buf_lz4);
|
||||
|
||||
out = ioc_get_out(C_ptr, &this_iter);
|
||||
ioc_set_next_out(C_ptr, &this_iter, (void *) ((char *) out + nbytes + 4));
|
||||
|
||||
bshuf_write_uint32_BE(out, nbytes);
|
||||
memcpy((char *) out + 4, tmp_buf_lz4, nbytes);
|
||||
|
||||
free(tmp_buf_lz4);
|
||||
|
||||
return nbytes + 4;
|
||||
}
|
||||
|
||||
|
||||
/* Decompress and bitunshuffle a single block. */
|
||||
int64_t bshuf_decompress_lz4_block(ioc_chain *C_ptr,
|
||||
const size_t size, const size_t elem_size) {
|
||||
|
||||
int64_t nbytes, count;
|
||||
void *out, *tmp_buf;
|
||||
const void *in;
|
||||
size_t this_iter;
|
||||
int32_t nbytes_from_header;
|
||||
|
||||
in = ioc_get_in(C_ptr, &this_iter);
|
||||
nbytes_from_header = bshuf_read_uint32_BE(in);
|
||||
ioc_set_next_in(C_ptr, &this_iter,
|
||||
(void*) ((char*) in + nbytes_from_header + 4));
|
||||
|
||||
out = ioc_get_out(C_ptr, &this_iter);
|
||||
ioc_set_next_out(C_ptr, &this_iter,
|
||||
(void *) ((char *) out + size * elem_size));
|
||||
|
||||
tmp_buf = malloc(size * elem_size);
|
||||
if (tmp_buf == NULL) return -1;
|
||||
|
||||
#ifdef BSHUF_LZ4_DECOMPRESS_FAST
|
||||
nbytes = LZ4_decompress_fast((const char*) in + 4, (char*) tmp_buf, size * elem_size);
|
||||
CHECK_ERR_FREE_LZ(nbytes, tmp_buf);
|
||||
if (nbytes != nbytes_from_header) {
|
||||
free(tmp_buf);
|
||||
return -91;
|
||||
}
|
||||
#else
|
||||
nbytes = LZ4_decompress_safe((const char*) in + 4, (char *) tmp_buf, nbytes_from_header,
|
||||
size * elem_size);
|
||||
CHECK_ERR_FREE_LZ(nbytes, tmp_buf);
|
||||
if (nbytes != size * elem_size) {
|
||||
free(tmp_buf);
|
||||
return -91;
|
||||
}
|
||||
nbytes = nbytes_from_header;
|
||||
#endif
|
||||
count = bshuf_untrans_bit_elem(tmp_buf, out, size, elem_size);
|
||||
CHECK_ERR_FREE(count, tmp_buf);
|
||||
nbytes += 4;
|
||||
|
||||
free(tmp_buf);
|
||||
return nbytes;
|
||||
}
|
||||
|
||||
|
||||
/* ---- Public functions ----
|
||||
*
|
||||
* See header file for description and usage.
|
||||
*
|
||||
*/
|
||||
|
||||
size_t bshuf_compress_lz4_bound(const size_t size,
|
||||
const size_t elem_size, size_t block_size) {
|
||||
|
||||
size_t bound, leftover;
|
||||
|
||||
if (block_size == 0) {
|
||||
block_size = bshuf_default_block_size(elem_size);
|
||||
}
|
||||
if (block_size % BSHUF_BLOCKED_MULT) return -81;
|
||||
|
||||
// Note that each block gets a 4 byte header.
|
||||
// Size of full blocks.
|
||||
bound = (LZ4_compressBound(block_size * elem_size) + 4) * (size / block_size);
|
||||
// Size of partial blocks, if any.
|
||||
leftover = ((size % block_size) / BSHUF_BLOCKED_MULT) * BSHUF_BLOCKED_MULT;
|
||||
if (leftover) bound += LZ4_compressBound(leftover * elem_size) + 4;
|
||||
// Size of uncompressed data not fitting into any blocks.
|
||||
bound += (size % BSHUF_BLOCKED_MULT) * elem_size;
|
||||
return bound;
|
||||
}
|
||||
|
||||
|
||||
int64_t bshuf_compress_lz4(const void* in, void* out, const size_t size,
|
||||
const size_t elem_size, size_t block_size) {
|
||||
return bshuf_blocked_wrap_fun(&bshuf_compress_lz4_block, in, out, size,
|
||||
elem_size, block_size);
|
||||
}
|
||||
|
||||
|
||||
int64_t bshuf_decompress_lz4(const void* in, void* out, const size_t size,
|
||||
const size_t elem_size, size_t block_size) {
|
||||
return bshuf_blocked_wrap_fun(&bshuf_decompress_lz4_block, in, out, size,
|
||||
elem_size, block_size);
|
||||
}
|
||||
|
||||
@@ -1,123 +0,0 @@
|
||||
/*
|
||||
* Bitshuffle - Filter for improving compression of typed binary data.
|
||||
*
|
||||
* This file is part of Bitshuffle
|
||||
* Author: Kiyoshi Masui <kiyo@physics.ubc.ca>
|
||||
* Website: http://www.github.com/kiyo-masui/bitshuffle
|
||||
* Created: 2014
|
||||
*
|
||||
* See LICENSE file for details about copyright and rights to use.
|
||||
*
|
||||
*
|
||||
* Header File
|
||||
*
|
||||
* Worker routines return an int64_t which is the number of bytes processed
|
||||
* if positive or an error code if negative.
|
||||
*
|
||||
* Error codes:
|
||||
* -1 : Failed to allocate memory.
|
||||
* -11 : Missing SSE.
|
||||
* -12 : Missing AVX.
|
||||
* -80 : Input size not a multiple of 8.
|
||||
* -81 : block_size not multiple of 8.
|
||||
* -91 : Decompression error, wrong number of bytes processed.
|
||||
* -1YYY : Error internal to compression routine with error code -YYY.
|
||||
*/
|
||||
|
||||
|
||||
#ifndef BITSHUFFLE_H
|
||||
#define BITSHUFFLE_H
|
||||
|
||||
#include <stdlib.h>
|
||||
#include "bitshuffle_core.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* ---- bshuf_compress_lz4_bound ----
|
||||
*
|
||||
* Bound on size of data compressed with *bshuf_compress_lz4*.
|
||||
*
|
||||
* Parameters
|
||||
* ----------
|
||||
* size : number of elements in input
|
||||
* elem_size : element size of typed data
|
||||
* block_size : Process in blocks of this many elements. Pass 0 to
|
||||
* select automatically (recommended).
|
||||
*
|
||||
* Returns
|
||||
* -------
|
||||
* Bound on compressed data size.
|
||||
*
|
||||
*/
|
||||
size_t bshuf_compress_lz4_bound(const size_t size,
|
||||
const size_t elem_size, size_t block_size);
|
||||
|
||||
|
||||
/* ---- bshuf_compress_lz4 ----
|
||||
*
|
||||
* Bitshuffled and compress the data using LZ4.
|
||||
*
|
||||
* Transpose within elements, in blocks of data of *block_size* elements then
|
||||
* compress the blocks using LZ4. In the output buffer, each block is prefixed
|
||||
* by a 4 byte integer giving the compressed size of that block.
|
||||
*
|
||||
* Output buffer must be large enough to hold the compressed data. This could
|
||||
* be in principle substantially larger than the input buffer. Use the routine
|
||||
* *bshuf_compress_lz4_bound* to get an upper limit.
|
||||
*
|
||||
* Parameters
|
||||
* ----------
|
||||
* in : input buffer, must be of size * elem_size bytes
|
||||
* out : output buffer, must be large enough to hold data.
|
||||
* size : number of elements in input
|
||||
* elem_size : element size of typed data
|
||||
* block_size : Process in blocks of this many elements. Pass 0 to
|
||||
* select automatically (recommended).
|
||||
*
|
||||
* Returns
|
||||
* -------
|
||||
* number of bytes used in output buffer, negative error-code if failed.
|
||||
*
|
||||
*/
|
||||
int64_t bshuf_compress_lz4(const void* in, void* out, const size_t size, const size_t
|
||||
elem_size, size_t block_size);
|
||||
|
||||
|
||||
/* ---- bshuf_decompress_lz4 ----
|
||||
*
|
||||
* Undo compression and bitshuffling.
|
||||
*
|
||||
* Decompress data then un-bitshuffle it in blocks of *block_size* elements.
|
||||
*
|
||||
* To properly unshuffle bitshuffled data, *size*, *elem_size* and *block_size*
|
||||
* must patch the parameters used to compress the data.
|
||||
*
|
||||
* NOT TO BE USED WITH UNTRUSTED DATA: This routine uses the function
|
||||
* LZ4_decompress_fast from LZ4, which does not protect against maliciously
|
||||
* formed datasets. By modifying the compressed data, this function could be
|
||||
* coerced into leaving the boundaries of the input buffer.
|
||||
*
|
||||
* Parameters
|
||||
* ----------
|
||||
* in : input buffer
|
||||
* out : output buffer, must be of size * elem_size bytes
|
||||
* size : number of elements in input
|
||||
* elem_size : element size of typed data
|
||||
* block_size : Process in blocks of this many elements. Pass 0 to
|
||||
* select automatically (recommended).
|
||||
*
|
||||
* Returns
|
||||
* -------
|
||||
* number of bytes consumed in *input* buffer, negative error-code if failed.
|
||||
*
|
||||
*/
|
||||
int64_t bshuf_decompress_lz4(const void* in, void* out, const size_t size,
|
||||
const size_t elem_size, size_t block_size);
|
||||
|
||||
#ifdef __cplusplus
|
||||
} // extern "C"
|
||||
#endif
|
||||
|
||||
#endif // BITSHUFFLE_H
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,156 +0,0 @@
|
||||
/*
|
||||
* Bitshuffle - Filter for improving compression of typed binary data.
|
||||
*
|
||||
* This file is part of Bitshuffle
|
||||
* Author: Kiyoshi Masui <kiyo@physics.ubc.ca>
|
||||
* Website: http://www.github.com/kiyo-masui/bitshuffle
|
||||
* Created: 2014
|
||||
*
|
||||
* See LICENSE file for details about copyright and rights to use.
|
||||
*
|
||||
*
|
||||
* Header File
|
||||
*
|
||||
* Worker routines return an int64_t which is the number of bytes processed
|
||||
* if positive or an error code if negative.
|
||||
*
|
||||
* Error codes:
|
||||
* -1 : Failed to allocate memory.
|
||||
* -11 : Missing SSE.
|
||||
* -12 : Missing AVX.
|
||||
* -80 : Input size not a multiple of 8.
|
||||
* -81 : block_size not multiple of 8.
|
||||
* -91 : Decompression error, wrong number of bytes processed.
|
||||
* -1YYY : Error internal to compression routine with error code -YYY.
|
||||
*/
|
||||
|
||||
|
||||
#ifndef BITSHUFFLE_CORE_H
|
||||
#define BITSHUFFLE_CORE_H
|
||||
|
||||
// We assume GNU g++ defining `__cplusplus` has stdint.h
|
||||
//#if (defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199900L) || defined(__cplusplus)
|
||||
#include <stdint.h>
|
||||
//#else
|
||||
// typedef unsigned char uint8_t;
|
||||
// typedef unsigned short uint16_t;
|
||||
// typedef unsigned int uint32_t;
|
||||
// typedef signed int int32_t;
|
||||
// typedef unsigned long long uint64_t;
|
||||
// typedef long long int64_t;
|
||||
//#endif
|
||||
|
||||
#include <stdlib.h>
|
||||
|
||||
|
||||
// These are usually set in the setup.py.
|
||||
#ifndef BSHUF_VERSION_MAJOR
|
||||
#define BSHUF_VERSION_MAJOR 0
|
||||
#define BSHUF_VERSION_MINOR 3
|
||||
#define BSHUF_VERSION_POINT 3
|
||||
#endif
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* --- bshuf_using_SSE2 ----
|
||||
*
|
||||
* Whether routines where compiled with the SSE2 instruction set.
|
||||
*
|
||||
* Returns
|
||||
* -------
|
||||
* 1 if using SSE2, 0 otherwise.
|
||||
*
|
||||
*/
|
||||
int bshuf_using_SSE2(void);
|
||||
|
||||
|
||||
/* ---- bshuf_using_AVX2 ----
|
||||
*
|
||||
* Whether routines where compiled with the AVX2 instruction set.
|
||||
*
|
||||
* Returns
|
||||
* -------
|
||||
* 1 if using AVX2, 0 otherwise.
|
||||
*
|
||||
*/
|
||||
int bshuf_using_AVX2(void);
|
||||
|
||||
|
||||
/* ---- bshuf_default_block_size ----
|
||||
*
|
||||
* The default block size as function of element size.
|
||||
*
|
||||
* This is the block size used by the blocked routines (any routine
|
||||
* taking a *block_size* argument) when the block_size is not provided
|
||||
* (zero is passed).
|
||||
*
|
||||
* The results of this routine are guaranteed to be stable such that
|
||||
* shuffled/compressed data can always be decompressed.
|
||||
*
|
||||
* Parameters
|
||||
* ----------
|
||||
* elem_size : element size of data to be shuffled/compressed.
|
||||
*
|
||||
*/
|
||||
size_t bshuf_default_block_size(const size_t elem_size);
|
||||
|
||||
|
||||
/* ---- bshuf_bitshuffle ----
|
||||
*
|
||||
* Bitshuffle the data.
|
||||
*
|
||||
* Transpose the bits within elements, in blocks of *block_size*
|
||||
* elements.
|
||||
*
|
||||
* Parameters
|
||||
* ----------
|
||||
* in : input buffer, must be of size * elem_size bytes
|
||||
* out : output buffer, must be of size * elem_size bytes
|
||||
* size : number of elements in input
|
||||
* elem_size : element size of typed data
|
||||
* block_size : Do transpose in blocks of this many elements. Pass 0 to
|
||||
* select automatically (recommended).
|
||||
*
|
||||
* Returns
|
||||
* -------
|
||||
* number of bytes processed, negative error-code if failed.
|
||||
*
|
||||
*/
|
||||
int64_t bshuf_bitshuffle(const void* in, void* out, const size_t size,
|
||||
const size_t elem_size, size_t block_size);
|
||||
|
||||
|
||||
/* ---- bshuf_bitunshuffle ----
|
||||
*
|
||||
* Unshuffle bitshuffled data.
|
||||
*
|
||||
* Untranspose the bits within elements, in blocks of *block_size*
|
||||
* elements.
|
||||
*
|
||||
* To properly unshuffle bitshuffled data, *size*, *elem_size* and *block_size*
|
||||
* must match the parameters used to shuffle the data.
|
||||
*
|
||||
* Parameters
|
||||
* ----------
|
||||
* in : input buffer, must be of size * elem_size bytes
|
||||
* out : output buffer, must be of size * elem_size bytes
|
||||
* size : number of elements in input
|
||||
* elem_size : element size of typed data
|
||||
* block_size : Do transpose in blocks of this many elements. Pass 0 to
|
||||
* select automatically (recommended).
|
||||
*
|
||||
* Returns
|
||||
* -------
|
||||
* number of bytes processed, negative error-code if failed.
|
||||
*
|
||||
*/
|
||||
int64_t bshuf_bitunshuffle(const void* in, void* out, const size_t size,
|
||||
const size_t elem_size, size_t block_size);
|
||||
|
||||
#ifdef __cplusplus
|
||||
} // extern "C"
|
||||
#endif
|
||||
|
||||
#endif // BITSHUFFLE_CORE_H
|
||||
@@ -1,75 +0,0 @@
|
||||
/*
|
||||
* Bitshuffle - Filter for improving compression of typed binary data.
|
||||
*
|
||||
* This file is part of Bitshuffle
|
||||
* Author: Kiyoshi Masui <kiyo@physics.ubc.ca>
|
||||
* Website: http://www.github.com/kiyo-masui/bitshuffle
|
||||
* Created: 2014
|
||||
*
|
||||
* See LICENSE file for details about copyright and rights to use.
|
||||
*/
|
||||
|
||||
|
||||
#ifndef BITSHUFFLE_INTERNALS_H
|
||||
#define BITSHUFFLE_INTERNALS_H
|
||||
|
||||
// We assume GNU g++ defining `__cplusplus` has stdint.h
|
||||
//#if (defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199900L) || defined(__cplusplus)
|
||||
#include <stdint.h>
|
||||
//#else
|
||||
// typedef unsigned char uint8_t;
|
||||
// typedef unsigned short uint16_t;
|
||||
// typedef unsigned int uint32_t;
|
||||
// typedef signed int int32_t;
|
||||
// typedef unsigned long long uint64_t;
|
||||
// typedef long long int64_t;
|
||||
//#endif
|
||||
|
||||
#include <stdlib.h>
|
||||
#include "iochain.h"
|
||||
|
||||
|
||||
// Constants.
|
||||
#ifndef BSHUF_MIN_RECOMMEND_BLOCK
|
||||
#define BSHUF_MIN_RECOMMEND_BLOCK 128
|
||||
#define BSHUF_BLOCKED_MULT 8 // Block sizes must be multiple of this.
|
||||
#define BSHUF_TARGET_BLOCK_SIZE_B 8192
|
||||
#endif
|
||||
|
||||
|
||||
// Macros.
|
||||
#define CHECK_ERR_FREE(count, buf) if (count < 0) { free(buf); return count; }
|
||||
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* ---- Utility functions for internal use only ---- */
|
||||
|
||||
int64_t bshuf_trans_bit_elem(const void* in, void* out, const size_t size,
|
||||
const size_t elem_size);
|
||||
|
||||
/* Read a 32 bit unsigned integer from a buffer big endian order. */
|
||||
uint32_t bshuf_read_uint32_BE(const void* buf);
|
||||
|
||||
/* Write a 32 bit unsigned integer to a buffer in big endian order. */
|
||||
void bshuf_write_uint32_BE(void* buf, uint32_t num);
|
||||
|
||||
int64_t bshuf_untrans_bit_elem(const void* in, void* out, const size_t size,
|
||||
const size_t elem_size);
|
||||
|
||||
/* Function definition for worker functions that process a single block. */
|
||||
typedef int64_t (*bshufBlockFunDef)(ioc_chain* C_ptr,
|
||||
const size_t size, const size_t elem_size);
|
||||
|
||||
/* Wrap a function for processing a single block to process an entire buffer in
|
||||
* parallel. */
|
||||
int64_t bshuf_blocked_wrap_fun(bshufBlockFunDef fun, const void* in, void* out,
|
||||
const size_t size, const size_t elem_size, size_t block_size);
|
||||
|
||||
#ifdef __cplusplus
|
||||
} // extern "C"
|
||||
#endif
|
||||
|
||||
#endif // BITSHUFFLE_INTERNALS_H
|
||||
@@ -1,137 +0,0 @@
|
||||
#include "compression.h"
|
||||
|
||||
#include <stdexcept>
|
||||
#include <arpa/inet.h>
|
||||
|
||||
extern "C"{
|
||||
#include "lz4.h"
|
||||
#include "bitshuffle.h"
|
||||
}
|
||||
|
||||
using namespace std;
|
||||
|
||||
#define is_little_endian htonl(1) != 1
|
||||
|
||||
size_t bsread::get_lz4_max_buffer_size(size_t n_elements,
|
||||
size_t element_size)
|
||||
{
|
||||
size_t n_bytes = n_elements * element_size;
|
||||
return static_cast<size_t>(LZ4_compressBound(n_bytes)) + 4;
|
||||
}
|
||||
|
||||
size_t bsread::compress_lz4(const char* data,
|
||||
size_t n_elements,
|
||||
size_t element_size,
|
||||
char* buffer,
|
||||
size_t buffer_size)
|
||||
{
|
||||
size_t data_len = n_elements * element_size;
|
||||
|
||||
// The bytes should be in big endian (network order).
|
||||
if(is_little_endian) {
|
||||
((uint32_t*)buffer)[0] = htonl(data_len);
|
||||
} else {
|
||||
((uint32_t*)buffer)[0] = data_len;
|
||||
}
|
||||
|
||||
size_t compressed_size = LZ4_compress_default(data, &buffer[4], data_len, buffer_size-4);
|
||||
|
||||
if(!compressed_size) {
|
||||
throw runtime_error("Error while compressing [LZ4] channel:");
|
||||
}
|
||||
|
||||
return compressed_size+4;
|
||||
|
||||
}
|
||||
|
||||
size_t bsread::decompress_lz4(const char* compressed_data, size_t compressed_size, char* data) {
|
||||
|
||||
uint32_t expected_data_size;
|
||||
|
||||
if(is_little_endian) {
|
||||
expected_data_size = ntohl(((uint32_t*)compressed_data)[0]);
|
||||
} else {
|
||||
expected_data_size = ((uint32_t*)compressed_data)[0];
|
||||
}
|
||||
|
||||
// 4 bytes of our header.
|
||||
compressed_size -= 4;
|
||||
|
||||
int decompressed_size = LZ4_decompress_safe(&compressed_data[4], data, compressed_size, expected_data_size);
|
||||
|
||||
if (expected_data_size != decompressed_size) {
|
||||
throw runtime_error("Expected and decompressed data len do not match.");
|
||||
}
|
||||
|
||||
// If the value is not positive, we throw an exception anyway.
|
||||
return (size_t) decompressed_size;
|
||||
}
|
||||
|
||||
size_t bsread::get_bitshuffle_max_buffer_size(size_t n_elements,
|
||||
size_t element_size)
|
||||
{
|
||||
return bshuf_compress_lz4_bound(n_elements, element_size, 0) + 12;
|
||||
}
|
||||
|
||||
size_t bsread::compress_bitshuffle(const char* data, size_t n_elements, size_t element_size, char* buffer){
|
||||
|
||||
size_t block_size = bshuf_default_block_size(element_size);
|
||||
|
||||
uint64_t uncompressed_data_len = (uint64_t) n_elements * element_size;
|
||||
// The block size has to be multiplied by the elm_size before inserting it into the binary header.
|
||||
// https://github.com/kiyo-masui/bitshuffle/blob/04e58bd553304ec26e222654f1d9b6ff64e97d10/src/bshuf_h5filter.c#L167
|
||||
uint32_t header_block_size = (uint32_t) block_size * element_size;
|
||||
|
||||
// The system is little endian, convert values to big endian (network order).
|
||||
if (is_little_endian) {
|
||||
uint32_t high_bytes = htonl((uint32_t)(uncompressed_data_len >> 32));
|
||||
uint32_t low_bytes = htonl((uint32_t)(uncompressed_data_len & 0xFFFFFFFFLL));
|
||||
uncompressed_data_len = (((uint64_t)low_bytes) << 32) | high_bytes;
|
||||
|
||||
header_block_size = htonl(header_block_size);
|
||||
}
|
||||
|
||||
((int64_t*)buffer)[0] = uncompressed_data_len;
|
||||
((int32_t*)buffer)[2] = header_block_size;
|
||||
|
||||
auto compressed_size = bshuf_compress_lz4(data, &buffer[12], n_elements, element_size, block_size);
|
||||
|
||||
if (compressed_size <= 0) {
|
||||
throw runtime_error("Error while compressing [LZ4] channel:");
|
||||
}
|
||||
|
||||
// If the value is not positive, we throw an exception anyway.
|
||||
return (size_t)compressed_size+12;
|
||||
}
|
||||
|
||||
size_t bsread::decompress_bitshuffle(const char* compressed_data, size_t compressed_size,
|
||||
size_t n_elements, size_t element_size, char* data) {
|
||||
|
||||
uint64_t header_expected_data_size = ((uint64_t*)compressed_data)[0];
|
||||
uint32_t header_block_size = ((uint32_t*)compressed_data)[2];
|
||||
|
||||
size_t expected_data_size = header_expected_data_size;
|
||||
|
||||
if (is_little_endian) {
|
||||
uint32_t high_bytes = ntohl((uint32_t)(header_expected_data_size & 0xFFFFFFFFLL));
|
||||
uint32_t low_bytes = ntohl((uint32_t)(header_expected_data_size >> 32));
|
||||
expected_data_size = (((uint64_t)high_bytes) << 32) | low_bytes;
|
||||
|
||||
header_block_size = ntohl(header_block_size);
|
||||
}
|
||||
|
||||
// The block size has to be multiplied by the elm_size before inserting it into the binary header.
|
||||
// https://github.com/kiyo-masui/bitshuffle/blob/04e58bd553304ec26e222654f1d9b6ff64e97d10/src/bshuf_h5filter.c#L167
|
||||
size_t block_size = header_block_size / element_size;
|
||||
|
||||
auto n_processed_bytes = bshuf_decompress_lz4(&compressed_data[12], data, n_elements, element_size, block_size);
|
||||
|
||||
// 12 bytes of our header.
|
||||
compressed_size -= 12;
|
||||
if (compressed_size != n_processed_bytes) {
|
||||
throw runtime_error("Compressed and processed data len do not match.");
|
||||
}
|
||||
|
||||
return expected_data_size;
|
||||
}
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
#ifndef COMPRESSION_H
|
||||
#define COMPRESSION_H
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
|
||||
namespace compression {
|
||||
|
||||
size_t get_lz4_max_buffer_size(size_t n_elements, size_t element_size);
|
||||
|
||||
size_t compress_lz4(const char* data,
|
||||
size_t n_elements,
|
||||
size_t element_size,
|
||||
char* buffer,
|
||||
size_t buffer_size);
|
||||
|
||||
size_t decompress_lz4(const char* compressed_data,
|
||||
size_t compressed_size,
|
||||
char* data);
|
||||
|
||||
size_t get_bitshuffle_max_buffer_size(size_t n_elements,
|
||||
size_t element_size);
|
||||
|
||||
size_t compress_bitshuffle(const char* data,
|
||||
size_t n_elements,
|
||||
size_t element_size,
|
||||
char* buffer);
|
||||
|
||||
size_t decompress_bitshuffle(const char* compressed_data,
|
||||
size_t compressed_size,
|
||||
size_t n_elements,
|
||||
size_t element_size,
|
||||
char* data);
|
||||
};
|
||||
|
||||
#endif
|
||||
@@ -1,90 +0,0 @@
|
||||
/*
|
||||
* IOchain - Distribute a chain of dependant IO events amoung threads.
|
||||
*
|
||||
* This file is part of Bitshuffle
|
||||
* Author: Kiyoshi Masui <kiyo@physics.ubc.ca>
|
||||
* Website: http://www.github.com/kiyo-masui/bitshuffle
|
||||
* Created: 2014
|
||||
*
|
||||
* See LICENSE file for details about copyright and rights to use.
|
||||
*
|
||||
*/
|
||||
|
||||
#include <stdlib.h>
|
||||
#include "iochain.h"
|
||||
|
||||
|
||||
void ioc_init(ioc_chain *C, const void *in_ptr_0, void *out_ptr_0) {
|
||||
#ifdef _OPENMP
|
||||
omp_init_lock(&C->next_lock);
|
||||
for (size_t ii = 0; ii < IOC_SIZE; ii ++) {
|
||||
omp_init_lock(&(C->in_pl[ii].lock));
|
||||
omp_init_lock(&(C->out_pl[ii].lock));
|
||||
}
|
||||
#endif
|
||||
C->next = 0;
|
||||
C->in_pl[0].ptr = in_ptr_0;
|
||||
C->out_pl[0].ptr = out_ptr_0;
|
||||
}
|
||||
|
||||
|
||||
void ioc_destroy(ioc_chain *C) {
|
||||
#ifdef _OPENMP
|
||||
omp_destroy_lock(&C->next_lock);
|
||||
for (size_t ii = 0; ii < IOC_SIZE; ii ++) {
|
||||
omp_destroy_lock(&(C->in_pl[ii].lock));
|
||||
omp_destroy_lock(&(C->out_pl[ii].lock));
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
const void * ioc_get_in(ioc_chain *C, size_t *this_iter) {
|
||||
#ifdef _OPENMP
|
||||
omp_set_lock(&C->next_lock);
|
||||
#pragma omp flush
|
||||
#endif
|
||||
*this_iter = C->next;
|
||||
C->next ++;
|
||||
#ifdef _OPENMP
|
||||
omp_set_lock(&(C->in_pl[*this_iter % IOC_SIZE].lock));
|
||||
omp_set_lock(&(C->in_pl[(*this_iter + 1) % IOC_SIZE].lock));
|
||||
omp_set_lock(&(C->out_pl[(*this_iter + 1) % IOC_SIZE].lock));
|
||||
omp_unset_lock(&C->next_lock);
|
||||
#endif
|
||||
return C->in_pl[*this_iter % IOC_SIZE].ptr;
|
||||
}
|
||||
|
||||
|
||||
void ioc_set_next_in(ioc_chain *C, size_t* this_iter, void* in_ptr) {
|
||||
C->in_pl[(*this_iter + 1) % IOC_SIZE].ptr = in_ptr;
|
||||
#ifdef _OPENMP
|
||||
omp_unset_lock(&(C->in_pl[(*this_iter + 1) % IOC_SIZE].lock));
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
void * ioc_get_out(ioc_chain *C, size_t *this_iter) {
|
||||
#ifdef _OPENMP
|
||||
omp_set_lock(&(C->out_pl[(*this_iter) % IOC_SIZE].lock));
|
||||
#pragma omp flush
|
||||
#endif
|
||||
void *out_ptr = C->out_pl[*this_iter % IOC_SIZE].ptr;
|
||||
#ifdef _OPENMP
|
||||
omp_unset_lock(&(C->out_pl[(*this_iter) % IOC_SIZE].lock));
|
||||
#endif
|
||||
return out_ptr;
|
||||
}
|
||||
|
||||
|
||||
void ioc_set_next_out(ioc_chain *C, size_t *this_iter, void* out_ptr) {
|
||||
C->out_pl[(*this_iter + 1) % IOC_SIZE].ptr = out_ptr;
|
||||
#ifdef _OPENMP
|
||||
omp_unset_lock(&(C->out_pl[(*this_iter + 1) % IOC_SIZE].lock));
|
||||
// *in_pl[this_iter]* lock released at the end of the iteration to avoid being
|
||||
// overtaken by previous threads and having *out_pl[this_iter]* corrupted.
|
||||
// Especially worried about thread 0, iteration 0.
|
||||
omp_unset_lock(&(C->in_pl[(*this_iter) % IOC_SIZE].lock));
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -1,94 +0,0 @@
|
||||
/*
|
||||
* IOchain - Distribute a chain of dependant IO events amoung threads.
|
||||
*
|
||||
* This file is part of Bitshuffle
|
||||
* Author: Kiyoshi Masui <kiyo@physics.ubc.ca>
|
||||
* Website: http://www.github.com/kiyo-masui/bitshuffle
|
||||
* Created: 2014
|
||||
*
|
||||
* See LICENSE file for details about copyright and rights to use.
|
||||
*
|
||||
*
|
||||
* Header File
|
||||
*
|
||||
* Similar in concept to a queue. Each task includes reading an input
|
||||
* and writing output, but the location of the input/output (the pointers)
|
||||
* depend on the previous item in the chain.
|
||||
*
|
||||
* This is designed for parallelizing blocked compression/decompression IO,
|
||||
* where the destination of a compressed block depends on the compressed size
|
||||
* of all previous blocks.
|
||||
*
|
||||
* Implemented with OpenMP locks.
|
||||
*
|
||||
*
|
||||
* Usage
|
||||
* -----
|
||||
* - Call `ioc_init` in serial block.
|
||||
* - Each thread should create a local variable *size_t this_iter* and
|
||||
* pass its address to all function calls. Its value will be set
|
||||
* inside the functions and is used to identify the thread.
|
||||
* - Each thread must call each of the `ioc_get*` and `ioc_set*` methods
|
||||
* exactly once per iteration, starting with `ioc_get_in` and ending
|
||||
* with `ioc_set_next_out`.
|
||||
* - The order (`ioc_get_in`, `ioc_set_next_in`, *work*, `ioc_get_out`,
|
||||
* `ioc_set_next_out`, *work*) is most efficient.
|
||||
* - Have each thread call `ioc_end_pop`.
|
||||
* - `ioc_get_in` is blocked until the previous entry's
|
||||
* `ioc_set_next_in` is called.
|
||||
* - `ioc_get_out` is blocked until the previous entry's
|
||||
* `ioc_set_next_out` is called.
|
||||
* - There are no blocks on the very first iteration.
|
||||
* - Call `ioc_destroy` in serial block.
|
||||
* - Safe for num_threads >= IOC_SIZE (but less efficient).
|
||||
*
|
||||
*/
|
||||
|
||||
|
||||
#ifndef IOCHAIN_H
|
||||
#define IOCHAIN_H
|
||||
|
||||
|
||||
#include <stdlib.h>
|
||||
#ifdef _OPENMP
|
||||
#include <omp.h>
|
||||
#endif
|
||||
|
||||
|
||||
#define IOC_SIZE 33
|
||||
|
||||
|
||||
typedef struct ioc_ptr_and_lock {
|
||||
#ifdef _OPENMP
|
||||
omp_lock_t lock;
|
||||
#endif
|
||||
void *ptr;
|
||||
} ptr_and_lock;
|
||||
|
||||
typedef struct ioc_const_ptr_and_lock {
|
||||
#ifdef _OPENMP
|
||||
omp_lock_t lock;
|
||||
#endif
|
||||
const void *ptr;
|
||||
} const_ptr_and_lock;
|
||||
|
||||
|
||||
typedef struct ioc_chain {
|
||||
#ifdef _OPENMP
|
||||
omp_lock_t next_lock;
|
||||
#endif
|
||||
size_t next;
|
||||
const_ptr_and_lock in_pl[IOC_SIZE];
|
||||
ptr_and_lock out_pl[IOC_SIZE];
|
||||
} ioc_chain;
|
||||
|
||||
|
||||
void ioc_init(ioc_chain *C, const void *in_ptr_0, void *out_ptr_0);
|
||||
void ioc_destroy(ioc_chain *C);
|
||||
const void * ioc_get_in(ioc_chain *C, size_t *this_iter);
|
||||
void ioc_set_next_in(ioc_chain *C, size_t* this_iter, void* in_ptr);
|
||||
void * ioc_get_out(ioc_chain *C, size_t *this_iter);
|
||||
void ioc_set_next_out(ioc_chain *C, size_t *this_iter, void* out_ptr);
|
||||
|
||||
#endif // IOCHAIN_H
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,360 +0,0 @@
|
||||
/*
|
||||
LZ4 - Fast LZ compression algorithm
|
||||
Header File
|
||||
Copyright (C) 2011-2015, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- LZ4 source repository : https://github.com/Cyan4973/lz4
|
||||
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
*/
|
||||
#pragma once
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/*
|
||||
* lz4.h provides block compression functions, and gives full buffer control to programmer.
|
||||
* If you need to generate inter-operable compressed data (respecting LZ4 frame specification),
|
||||
* and can let the library handle its own memory, please use lz4frame.h instead.
|
||||
*/
|
||||
|
||||
/**************************************
|
||||
* Version
|
||||
**************************************/
|
||||
#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */
|
||||
#define LZ4_VERSION_MINOR 7 /* for new (non-breaking) interface capabilities */
|
||||
#define LZ4_VERSION_RELEASE 1 /* for tweaks, bug-fixes, or development */
|
||||
#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE)
|
||||
int LZ4_versionNumber (void);
|
||||
|
||||
/**************************************
|
||||
* Tuning parameter
|
||||
**************************************/
|
||||
/*
|
||||
* LZ4_MEMORY_USAGE :
|
||||
* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
|
||||
* Increasing memory usage improves compression ratio
|
||||
* Reduced memory usage can improve speed, due to cache effect
|
||||
* Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache
|
||||
*/
|
||||
#define LZ4_MEMORY_USAGE 14
|
||||
|
||||
|
||||
/**************************************
|
||||
* Simple Functions
|
||||
**************************************/
|
||||
|
||||
int LZ4_compress_default(const char* source, char* dest, int sourceSize, int maxDestSize);
|
||||
int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize);
|
||||
|
||||
/*
|
||||
LZ4_compress_default() :
|
||||
Compresses 'sourceSize' bytes from buffer 'source'
|
||||
into already allocated 'dest' buffer of size 'maxDestSize'.
|
||||
Compression is guaranteed to succeed if 'maxDestSize' >= LZ4_compressBound(sourceSize).
|
||||
It also runs faster, so it's a recommended setting.
|
||||
If the function cannot compress 'source' into a more limited 'dest' budget,
|
||||
compression stops *immediately*, and the function result is zero.
|
||||
As a consequence, 'dest' content is not valid.
|
||||
This function never writes outside 'dest' buffer, nor read outside 'source' buffer.
|
||||
sourceSize : Max supported value is LZ4_MAX_INPUT_VALUE
|
||||
maxDestSize : full or partial size of buffer 'dest' (which must be already allocated)
|
||||
return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize)
|
||||
or 0 if compression fails
|
||||
|
||||
LZ4_decompress_safe() :
|
||||
compressedSize : is the precise full size of the compressed block.
|
||||
maxDecompressedSize : is the size of destination buffer, which must be already allocated.
|
||||
return : the number of bytes decompressed into destination buffer (necessarily <= maxDecompressedSize)
|
||||
If destination buffer is not large enough, decoding will stop and output an error code (<0).
|
||||
If the source stream is detected malformed, the function will stop decoding and return a negative result.
|
||||
This function is protected against buffer overflow exploits, including malicious data packets.
|
||||
It never writes outside output buffer, nor reads outside input buffer.
|
||||
*/
|
||||
|
||||
|
||||
/**************************************
|
||||
* Advanced Functions
|
||||
**************************************/
|
||||
#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */
|
||||
#define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16)
|
||||
|
||||
/*
|
||||
LZ4_compressBound() :
|
||||
Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible)
|
||||
This function is primarily useful for memory allocation purposes (destination buffer size).
|
||||
Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example).
|
||||
Note that LZ4_compress_default() compress faster when dest buffer size is >= LZ4_compressBound(srcSize)
|
||||
inputSize : max supported value is LZ4_MAX_INPUT_SIZE
|
||||
return : maximum output size in a "worst case" scenario
|
||||
or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE)
|
||||
*/
|
||||
int LZ4_compressBound(int inputSize);
|
||||
|
||||
/*
|
||||
LZ4_compress_fast() :
|
||||
Same as LZ4_compress_default(), but allows to select an "acceleration" factor.
|
||||
The larger the acceleration value, the faster the algorithm, but also the lesser the compression.
|
||||
It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed.
|
||||
An acceleration value of "1" is the same as regular LZ4_compress_default()
|
||||
Values <= 0 will be replaced by ACCELERATION_DEFAULT (see lz4.c), which is 1.
|
||||
*/
|
||||
int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, int acceleration);
|
||||
|
||||
|
||||
/*
|
||||
LZ4_compress_fast_extState() :
|
||||
Same compression function, just using an externally allocated memory space to store compression state.
|
||||
Use LZ4_sizeofState() to know how much memory must be allocated,
|
||||
and allocate it on 8-bytes boundaries (using malloc() typically).
|
||||
Then, provide it as 'void* state' to compression function.
|
||||
*/
|
||||
int LZ4_sizeofState(void);
|
||||
int LZ4_compress_fast_extState (void* state, const char* source, char* dest, int inputSize, int maxDestSize, int acceleration);
|
||||
|
||||
|
||||
/*
|
||||
LZ4_compress_destSize() :
|
||||
Reverse the logic, by compressing as much data as possible from 'source' buffer
|
||||
into already allocated buffer 'dest' of size 'targetDestSize'.
|
||||
This function either compresses the entire 'source' content into 'dest' if it's large enough,
|
||||
or fill 'dest' buffer completely with as much data as possible from 'source'.
|
||||
*sourceSizePtr : will be modified to indicate how many bytes where read from 'source' to fill 'dest'.
|
||||
New value is necessarily <= old value.
|
||||
return : Nb bytes written into 'dest' (necessarily <= targetDestSize)
|
||||
or 0 if compression fails
|
||||
*/
|
||||
int LZ4_compress_destSize (const char* source, char* dest, int* sourceSizePtr, int targetDestSize);
|
||||
|
||||
|
||||
/*
|
||||
LZ4_decompress_fast() :
|
||||
originalSize : is the original and therefore uncompressed size
|
||||
return : the number of bytes read from the source buffer (in other words, the compressed size)
|
||||
If the source stream is detected malformed, the function will stop decoding and return a negative result.
|
||||
Destination buffer must be already allocated. Its size must be a minimum of 'originalSize' bytes.
|
||||
note : This function fully respect memory boundaries for properly formed compressed data.
|
||||
It is a bit faster than LZ4_decompress_safe().
|
||||
However, it does not provide any protection against intentionally modified data stream (malicious input).
|
||||
Use this function in trusted environment only (data to decode comes from a trusted source).
|
||||
*/
|
||||
int LZ4_decompress_fast (const char* source, char* dest, int originalSize);
|
||||
|
||||
/*
|
||||
LZ4_decompress_safe_partial() :
|
||||
This function decompress a compressed block of size 'compressedSize' at position 'source'
|
||||
into destination buffer 'dest' of size 'maxDecompressedSize'.
|
||||
The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached,
|
||||
reducing decompression time.
|
||||
return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize)
|
||||
Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller.
|
||||
Always control how many bytes were decoded.
|
||||
If the source stream is detected malformed, the function will stop decoding and return a negative result.
|
||||
This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets
|
||||
*/
|
||||
int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize);
|
||||
|
||||
|
||||
/***********************************************
|
||||
* Streaming Compression Functions
|
||||
***********************************************/
|
||||
#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4)
|
||||
#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(long long))
|
||||
/*
|
||||
* LZ4_stream_t
|
||||
* information structure to track an LZ4 stream.
|
||||
* important : init this structure content before first use !
|
||||
* note : only allocated directly the structure if you are statically linking LZ4
|
||||
* If you are using liblz4 as a DLL, please use below construction methods instead.
|
||||
*/
|
||||
typedef struct { long long table[LZ4_STREAMSIZE_U64]; } LZ4_stream_t;
|
||||
|
||||
/*
|
||||
* LZ4_resetStream
|
||||
* Use this function to init an allocated LZ4_stream_t structure
|
||||
*/
|
||||
void LZ4_resetStream (LZ4_stream_t* streamPtr);
|
||||
|
||||
/*
|
||||
* LZ4_createStream will allocate and initialize an LZ4_stream_t structure
|
||||
* LZ4_freeStream releases its memory.
|
||||
* In the context of a DLL (liblz4), please use these methods rather than the static struct.
|
||||
* They are more future proof, in case of a change of LZ4_stream_t size.
|
||||
*/
|
||||
LZ4_stream_t* LZ4_createStream(void);
|
||||
int LZ4_freeStream (LZ4_stream_t* streamPtr);
|
||||
|
||||
/*
|
||||
* LZ4_loadDict
|
||||
* Use this function to load a static dictionary into LZ4_stream.
|
||||
* Any previous data will be forgotten, only 'dictionary' will remain in memory.
|
||||
* Loading a size of 0 is allowed.
|
||||
* Return : dictionary size, in bytes (necessarily <= 64 KB)
|
||||
*/
|
||||
int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize);
|
||||
|
||||
/*
|
||||
* LZ4_compress_fast_continue
|
||||
* Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio.
|
||||
* Important : Previous data blocks are assumed to still be present and unmodified !
|
||||
* 'dst' buffer must be already allocated.
|
||||
* If maxDstSize >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
|
||||
* If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero.
|
||||
*/
|
||||
int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int maxDstSize, int acceleration);
|
||||
|
||||
/*
|
||||
* LZ4_saveDict
|
||||
* If previously compressed data block is not guaranteed to remain available at its memory location
|
||||
* save it into a safer place (char* safeBuffer)
|
||||
* Note : you don't need to call LZ4_loadDict() afterwards,
|
||||
* dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue()
|
||||
* Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error
|
||||
*/
|
||||
int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dictSize);
|
||||
|
||||
|
||||
/************************************************
|
||||
* Streaming Decompression Functions
|
||||
************************************************/
|
||||
|
||||
#define LZ4_STREAMDECODESIZE_U64 4
|
||||
#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long))
|
||||
typedef struct { unsigned long long table[LZ4_STREAMDECODESIZE_U64]; } LZ4_streamDecode_t;
|
||||
/*
|
||||
* LZ4_streamDecode_t
|
||||
* information structure to track an LZ4 stream.
|
||||
* init this structure content using LZ4_setStreamDecode or memset() before first use !
|
||||
*
|
||||
* In the context of a DLL (liblz4) please prefer usage of construction methods below.
|
||||
* They are more future proof, in case of a change of LZ4_streamDecode_t size in the future.
|
||||
* LZ4_createStreamDecode will allocate and initialize an LZ4_streamDecode_t structure
|
||||
* LZ4_freeStreamDecode releases its memory.
|
||||
*/
|
||||
LZ4_streamDecode_t* LZ4_createStreamDecode(void);
|
||||
int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
|
||||
|
||||
/*
|
||||
* LZ4_setStreamDecode
|
||||
* Use this function to instruct where to find the dictionary.
|
||||
* Setting a size of 0 is allowed (same effect as reset).
|
||||
* Return : 1 if OK, 0 if error
|
||||
*/
|
||||
int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize);
|
||||
|
||||
/*
|
||||
*_continue() :
|
||||
These decoding functions allow decompression of multiple blocks in "streaming" mode.
|
||||
Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB)
|
||||
In the case of a ring buffers, decoding buffer must be either :
|
||||
- Exactly same size as encoding buffer, with same update rule (block boundaries at same positions)
|
||||
In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB).
|
||||
- Larger than encoding buffer, by a minimum of maxBlockSize more bytes.
|
||||
maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block.
|
||||
In which case, encoding and decoding buffers do not need to be synchronized,
|
||||
and encoding ring buffer can have any size, including small ones ( < 64 KB).
|
||||
- _At least_ 64 KB + 8 bytes + maxBlockSize.
|
||||
In which case, encoding and decoding buffers do not need to be synchronized,
|
||||
and encoding ring buffer can have any size, including larger than decoding buffer.
|
||||
Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer,
|
||||
and indicate where it is saved using LZ4_setStreamDecode()
|
||||
*/
|
||||
int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize);
|
||||
int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize);
|
||||
|
||||
|
||||
/*
|
||||
Advanced decoding functions :
|
||||
*_usingDict() :
|
||||
These decoding functions work the same as
|
||||
a combination of LZ4_setStreamDecode() followed by LZ4_decompress_x_continue()
|
||||
They are stand-alone. They don't need nor update an LZ4_streamDecode_t structure.
|
||||
*/
|
||||
int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize);
|
||||
int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize);
|
||||
|
||||
|
||||
|
||||
/**************************************
|
||||
* Obsolete Functions
|
||||
**************************************/
|
||||
/* Deprecate Warnings */
|
||||
/* Should these warnings messages be a problem,
|
||||
it is generally possible to disable them,
|
||||
with -Wno-deprecated-declarations for gcc
|
||||
or _CRT_SECURE_NO_WARNINGS in Visual for example.
|
||||
You can also define LZ4_DEPRECATE_WARNING_DEFBLOCK. */
|
||||
#ifndef LZ4_DEPRECATE_WARNING_DEFBLOCK
|
||||
# define LZ4_DEPRECATE_WARNING_DEFBLOCK
|
||||
# define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
|
||||
# if (LZ4_GCC_VERSION >= 405) || defined(__clang__)
|
||||
# define LZ4_DEPRECATED(message) __attribute__((deprecated(message)))
|
||||
# elif (LZ4_GCC_VERSION >= 301)
|
||||
# define LZ4_DEPRECATED(message) __attribute__((deprecated))
|
||||
# elif defined(_MSC_VER)
|
||||
# define LZ4_DEPRECATED(message) __declspec(deprecated(message))
|
||||
# else
|
||||
# pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler")
|
||||
# define LZ4_DEPRECATED(message)
|
||||
# endif
|
||||
#endif /* LZ4_DEPRECATE_WARNING_DEFBLOCK */
|
||||
|
||||
/* Obsolete compression functions */
|
||||
/* These functions are planned to start generate warnings by r131 approximately */
|
||||
int LZ4_compress (const char* source, char* dest, int sourceSize);
|
||||
int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize);
|
||||
int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize);
|
||||
int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
|
||||
int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize);
|
||||
int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
|
||||
|
||||
/* Obsolete decompression functions */
|
||||
/* These function names are completely deprecated and must no longer be used.
|
||||
They are only provided here for compatibility with older programs.
|
||||
- LZ4_uncompress is the same as LZ4_decompress_fast
|
||||
- LZ4_uncompress_unknownOutputSize is the same as LZ4_decompress_safe
|
||||
These function prototypes are now disabled; uncomment them only if you really need them.
|
||||
It is highly recommended to stop using these prototypes and migrate to maintained ones */
|
||||
/* int LZ4_uncompress (const char* source, char* dest, int outputSize); */
|
||||
/* int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); */
|
||||
|
||||
/* Obsolete streaming functions; use new streaming interface whenever possible */
|
||||
LZ4_DEPRECATED("use LZ4_createStream() instead") void* LZ4_create (char* inputBuffer);
|
||||
LZ4_DEPRECATED("use LZ4_createStream() instead") int LZ4_sizeofStreamState(void);
|
||||
LZ4_DEPRECATED("use LZ4_resetStream() instead") int LZ4_resetStreamState(void* state, char* inputBuffer);
|
||||
LZ4_DEPRECATED("use LZ4_saveDict() instead") char* LZ4_slideInputBuffer (void* state);
|
||||
|
||||
/* Obsolete streaming decoding functions */
|
||||
LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize);
|
||||
LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize);
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
@@ -1,28 +0,0 @@
|
||||
#include "config.hpp"
|
||||
|
||||
namespace config {
|
||||
// Number of receiving threads. Roughly 1 thread / (GB/s)
|
||||
int zmq_n_io_threads = 1;
|
||||
int zmq_receive_timeout = 100;
|
||||
|
||||
// JSON header buffer size - 1MB.
|
||||
int zmq_buffer_size_header = 1024 * 1024 * 1;
|
||||
// Data message buffer size - 10MB.
|
||||
int zmq_buffer_size_data = 1024 * 1024 * 10;
|
||||
|
||||
// Ring buffer config.
|
||||
// Allow for a couple of seconds (file creation might be slow).
|
||||
size_t ring_buffer_n_slots = 1000;
|
||||
// Delay before trying again to get data from the ring buffer.
|
||||
uint32_t ring_buffer_read_retry_interval = 5;
|
||||
|
||||
std::string raw_image_dataset_name = "raw_data";
|
||||
|
||||
// By how much to enlarge a dataset when a resizing is needed.
|
||||
hsize_t dataset_increase_step = 1000;
|
||||
// To which value to initialize a dataset size.
|
||||
hsize_t initial_dataset_size = 1000;
|
||||
|
||||
// Delay in between attempts to see if the requred parameters were passed over the REST api.
|
||||
uint32_t parameters_read_retry_interval = 300;
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
#include <H5Cpp.h>
|
||||
#include <string>
|
||||
|
||||
#ifndef CONFIG_H
|
||||
#define CONFIG_H
|
||||
|
||||
namespace config
|
||||
{
|
||||
extern int zmq_n_io_threads;
|
||||
extern int zmq_receive_timeout;
|
||||
extern int zmq_buffer_size_header;
|
||||
extern int zmq_buffer_size_data;
|
||||
|
||||
extern size_t ring_buffer_n_slots;
|
||||
extern uint32_t ring_buffer_read_retry_interval;
|
||||
|
||||
extern hsize_t dataset_increase_step;
|
||||
extern hsize_t initial_dataset_size;
|
||||
extern std::string raw_image_dataset_name;
|
||||
|
||||
extern uint32_t parameters_read_retry_interval;
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -1,74 +0,0 @@
|
||||
#include "gtest/gtest.h"
|
||||
#include "../src/BufferedWriter.hpp"
|
||||
using namespace std;
|
||||
|
||||
TEST(BufferedWriter, get_buffered_writer)
|
||||
{
|
||||
auto header_values = shared_ptr<unordered_map<string, HeaderDataType>>(new unordered_map<string, HeaderDataType> {
|
||||
{"frame", HeaderDataType("uint64")},
|
||||
{"module_number", HeaderDataType("uint64", 2)}
|
||||
});
|
||||
|
||||
auto metadata_buffer1 = unique_ptr<MetadataBuffer>(new MetadataBuffer(1, header_values));
|
||||
|
||||
auto dummy_writer = get_buffered_writer("/dev/null", 10, move(metadata_buffer1));
|
||||
EXPECT_TRUE(dynamic_cast<DummyBufferedWriter*>(dummy_writer.get()));
|
||||
|
||||
auto metadata_buffer2 = unique_ptr<MetadataBuffer>(new MetadataBuffer(1, header_values));
|
||||
|
||||
auto real_writer = get_buffered_writer("real_file.h5", 10, move(metadata_buffer2));
|
||||
EXPECT_FALSE(dynamic_cast<DummyBufferedWriter*>(real_writer.get()));
|
||||
}
|
||||
|
||||
TEST(BufferedWriter, DummyBufferedWriter)
|
||||
{
|
||||
DummyBufferedWriter dummy_writer;
|
||||
|
||||
EXPECT_FALSE(dummy_writer.is_file_open());
|
||||
|
||||
EXPECT_TRUE(dummy_writer.is_data_for_current_file(12345678));
|
||||
|
||||
EXPECT_NO_THROW(dummy_writer.cache_metadata("test", 10, nullptr));
|
||||
|
||||
EXPECT_NO_THROW(dummy_writer.write_metadata_to_file());
|
||||
|
||||
EXPECT_NO_THROW(dummy_writer.close_file());
|
||||
|
||||
EXPECT_THROW(dummy_writer.get_h5_file(), runtime_error);
|
||||
|
||||
unique_ptr<char> buffer(new char[1]);
|
||||
vector<size_t> shape = {1};
|
||||
|
||||
EXPECT_NO_THROW(dummy_writer.write_data("does not matter", 0, buffer.get(), shape, 0, "nop", "nop"));
|
||||
}
|
||||
|
||||
TEST(BufferedWriter, process_manager_workflow)
|
||||
{
|
||||
auto header_values = shared_ptr<unordered_map<string, HeaderDataType>>(new unordered_map<string, HeaderDataType> {
|
||||
{"frame", HeaderDataType("uint64")},
|
||||
{"module_number", HeaderDataType("uint64", 2)}
|
||||
});
|
||||
|
||||
auto metadata_buffer = unique_ptr<MetadataBuffer>(new MetadataBuffer(1, header_values));
|
||||
auto writer = get_buffered_writer("/dev/null", 10, move(metadata_buffer));
|
||||
|
||||
ASSERT_NO_THROW(writer->create_file());
|
||||
|
||||
EXPECT_TRUE(writer->is_data_for_current_file(0));
|
||||
EXPECT_TRUE(writer->is_data_for_current_file(123456));
|
||||
|
||||
// This 2 should not happen, because is_data_for_current_file should always return true.
|
||||
ASSERT_NO_THROW(writer->write_metadata_to_file());
|
||||
ASSERT_THROW(writer->get_h5_file(), runtime_error);
|
||||
|
||||
unique_ptr<char> buffer(new char[1]);
|
||||
vector<size_t> shape = {1};
|
||||
EXPECT_NO_THROW(writer->write_data("does not matter", 0, buffer.get(), shape, 0, "nop", "nop"));
|
||||
|
||||
EXPECT_NO_THROW(writer->cache_metadata("test", 10, nullptr));
|
||||
|
||||
// This is important to prevent get_h5_file from being called.
|
||||
EXPECT_FALSE(writer->is_file_open());
|
||||
|
||||
EXPECT_NO_THROW(writer->close_file());
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
#include "gtest/gtest.h"
|
||||
#include "../src/H5Writer.hpp"
|
||||
using namespace std;
|
||||
|
||||
TEST(H5Writer, get_h5_writer)
|
||||
{
|
||||
auto dummy_writer = get_h5_writer("/dev/null");
|
||||
EXPECT_TRUE(dynamic_cast<DummyH5Writer*>(dummy_writer.get()));
|
||||
|
||||
auto real_writer = get_h5_writer("real_file.h5");
|
||||
EXPECT_FALSE(dynamic_cast<DummyH5Writer*>(real_writer.get()));
|
||||
}
|
||||
|
||||
TEST(H5Writer, DummyH5Writer)
|
||||
{
|
||||
DummyH5Writer dummy_writer;
|
||||
|
||||
EXPECT_FALSE(dummy_writer.is_file_open());
|
||||
|
||||
EXPECT_NO_THROW(dummy_writer.close_file());
|
||||
|
||||
EXPECT_THROW(dummy_writer.get_h5_file(), runtime_error);
|
||||
|
||||
unique_ptr<char> buffer(new char[1]);
|
||||
vector<size_t> shape = {1};
|
||||
|
||||
EXPECT_NO_THROW(dummy_writer.write_data("does not matter", 0, buffer.get(), shape, 0, "nop", "nop"));
|
||||
}
|
||||
@@ -1,67 +0,0 @@
|
||||
#include "gtest/gtest.h"
|
||||
#include "../src/MetadataBuffer.hpp"
|
||||
|
||||
using namespace std;
|
||||
|
||||
TEST(MetadataBuffer, basic_operations)
|
||||
{
|
||||
int n_frames = 10;
|
||||
int n_modules = 3;
|
||||
|
||||
auto header_values = shared_ptr<unordered_map<string, HeaderDataType>>(new unordered_map<string, HeaderDataType> {
|
||||
{"frame", HeaderDataType("uint64")},
|
||||
{"module_number", HeaderDataType("uint64", n_modules)}
|
||||
});
|
||||
|
||||
MetadataBuffer metadata_buffer(n_frames, header_values);
|
||||
|
||||
uint64_t base_frame_data = 12345678;
|
||||
uint64_t base_module_number = 0;
|
||||
|
||||
for (int i = 0; i<n_frames; i++) {
|
||||
uint64_t frame_index = base_frame_data + i;
|
||||
metadata_buffer.add_metadata_to_buffer("frame", i, (char*)&frame_index);
|
||||
|
||||
uint64_t module_number = base_module_number + i;
|
||||
uint64_t module_number_data[n_modules];
|
||||
|
||||
for (int module_index=0; module_index<n_modules; module_index++) {
|
||||
module_number_data[module_index] = module_number;
|
||||
}
|
||||
|
||||
metadata_buffer.add_metadata_to_buffer("module_number", i, (char*)&module_number_data);
|
||||
}
|
||||
|
||||
uint64_t* frame_data = (uint64_t*) metadata_buffer.get_metadata_values("frame").get();
|
||||
uint64_t* module_number_data = (uint64_t*) metadata_buffer.get_metadata_values("module_number").get();
|
||||
|
||||
for (int i = 0; i<n_frames; i++) {
|
||||
EXPECT_EQ(frame_data[i], base_frame_data+i);
|
||||
|
||||
for (int module_index=0; module_index<n_modules; module_index++) {
|
||||
EXPECT_EQ(module_number_data[n_modules*i + module_index], base_module_number + i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST(MetadataBuffer, missing_keys)
|
||||
{
|
||||
int n_frames = 10;
|
||||
int n_modules = 3;
|
||||
|
||||
auto header_values = shared_ptr<unordered_map<string, HeaderDataType>>(new unordered_map<string, HeaderDataType> {
|
||||
{"frame", HeaderDataType("uint64")},
|
||||
{"module_number", HeaderDataType("uint64", n_modules)}
|
||||
});
|
||||
|
||||
MetadataBuffer metadata_buffer(n_frames, header_values);
|
||||
|
||||
uint64_t data = 0;
|
||||
|
||||
metadata_buffer.add_metadata_to_buffer("frame", 0, (char*)&data);
|
||||
metadata_buffer.get_metadata_values("frame");
|
||||
|
||||
EXPECT_THROW(metadata_buffer.get_metadata_values("non_existant"), runtime_error);
|
||||
EXPECT_THROW(metadata_buffer.add_metadata_to_buffer("non_existant", 0, nullptr), runtime_error);
|
||||
EXPECT_THROW(metadata_buffer.add_metadata_to_buffer("frame", n_frames, nullptr), runtime_error);
|
||||
}
|
||||
@@ -1,149 +0,0 @@
|
||||
#include "gtest/gtest.h"
|
||||
#include "../src/ZmqReceiver.hpp"
|
||||
|
||||
using namespace std;
|
||||
namespace pt = boost::property_tree;
|
||||
|
||||
TEST(ZmqReceiver, get_type_byte_size)
|
||||
{
|
||||
EXPECT_TRUE(get_type_byte_size("uint8") == 1);
|
||||
EXPECT_TRUE(get_type_byte_size("int8") == 1);
|
||||
EXPECT_TRUE(get_type_byte_size("uint16") == 2);
|
||||
EXPECT_TRUE(get_type_byte_size("int16") == 2);
|
||||
EXPECT_TRUE(get_type_byte_size("uint32") == 4);
|
||||
EXPECT_TRUE(get_type_byte_size("int32") == 4);
|
||||
EXPECT_TRUE(get_type_byte_size("float32") == 4);
|
||||
EXPECT_TRUE(get_type_byte_size("uint64") == 8);
|
||||
EXPECT_TRUE(get_type_byte_size("int64") == 8);
|
||||
EXPECT_TRUE(get_type_byte_size("float64") == 8);
|
||||
}
|
||||
|
||||
TEST(ZmqReceiver, HeaderDataType)
|
||||
{
|
||||
HeaderDataType header_data_type("float64", 4);
|
||||
|
||||
ASSERT_TRUE(header_data_type.type == "float64");
|
||||
ASSERT_TRUE(header_data_type.value_bytes_size == 8);
|
||||
ASSERT_TRUE(header_data_type.value_shape == 4);
|
||||
ASSERT_TRUE(header_data_type.endianness == "little");
|
||||
}
|
||||
|
||||
TEST(ZmqReceiver, get_value_from_json)
|
||||
{
|
||||
pt::ptree json_header;
|
||||
|
||||
uint64_t frame_number = 1234567890;
|
||||
json_header.add("frame_number", frame_number);
|
||||
HeaderDataType header_data_type_scalar("uint64");
|
||||
|
||||
auto scalar_buffer = get_value_from_json(json_header, "frame_number", header_data_type_scalar);
|
||||
auto scalar_value = reinterpret_cast<uint64_t*>(scalar_buffer.get());
|
||||
|
||||
ASSERT_TRUE(*scalar_value == frame_number);
|
||||
|
||||
double modules_number[] = {-345.12, 1234567.43, -2323456.32};
|
||||
pt::ptree modulus_number_child;
|
||||
|
||||
for (int i=0; i<3; i++) {
|
||||
pt::ptree value;
|
||||
value.put("", modules_number[i]);
|
||||
modulus_number_child.push_back(make_pair("", value));
|
||||
}
|
||||
|
||||
json_header.add_child("modules_number", modulus_number_child);
|
||||
|
||||
HeaderDataType header_data_type_array("float64", 3);
|
||||
|
||||
auto array_buffer = get_value_from_json(json_header, "modules_number", header_data_type_array);
|
||||
auto array_values = reinterpret_cast<double*>(array_buffer.get());
|
||||
|
||||
for (int i=0; i<3; i++) {
|
||||
ASSERT_TRUE(array_values[i] == modules_number[i]);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(ZmqReceiver, read_json_header)
|
||||
{
|
||||
int n_modules = 1;
|
||||
|
||||
auto header_values = shared_ptr<unordered_map<string, HeaderDataType>>(new unordered_map<string, HeaderDataType> {
|
||||
{"pulse_id", HeaderDataType("uint64")},
|
||||
{"frame", HeaderDataType("uint64")},
|
||||
{"is_good_frame", HeaderDataType("uint64")},
|
||||
{"daq_rec", HeaderDataType("int64")},
|
||||
|
||||
{"pulse_id_diff", HeaderDataType("int64", n_modules)},
|
||||
{"framenum_diff", HeaderDataType("int64", n_modules)},
|
||||
|
||||
{"missing_packets_1", HeaderDataType("uint64", n_modules)},
|
||||
{"missing_packets_2", HeaderDataType("uint64", n_modules)},
|
||||
{"daq_recs", HeaderDataType("uint64", n_modules)},
|
||||
|
||||
{"pulse_ids", HeaderDataType("uint64", n_modules)},
|
||||
{"framenums", HeaderDataType("uint64", n_modules)},
|
||||
|
||||
{"module_number", HeaderDataType("uint64", n_modules)}
|
||||
});
|
||||
|
||||
ZmqReceiver receiver("something", 1, 1, header_values);
|
||||
|
||||
auto header_string = "{\"missing_packets_2\":[2],"
|
||||
"\"missing_packets_1\":[1],"
|
||||
"\"frame\":0,"
|
||||
"\"daq_recs\":[3840],"
|
||||
"\"module_number\":[0],"
|
||||
"\"shape\":[512,1024],"
|
||||
"\"pulse_id\":6021771850,"
|
||||
"\"framenum_diff\":[-2],"
|
||||
"\"pulse_ids\":[6021771850],"
|
||||
"\"is_good_frame\":1,"
|
||||
"\"framenums\":[193],"
|
||||
"\"pulse_id_diff\":[-1],"
|
||||
"\"daq_rec\":-1,"
|
||||
"\"type\":\"uint16\","
|
||||
"\"htype\":\"array-1.0\"}";
|
||||
|
||||
auto metadata = receiver.read_json_header(header_string);
|
||||
|
||||
ASSERT_TRUE(metadata->frame_index == 0);
|
||||
ASSERT_TRUE(metadata->endianness == "little");
|
||||
ASSERT_TRUE(metadata->type == "uint16");
|
||||
ASSERT_TRUE(metadata->frame_shape[0] == 512);
|
||||
ASSERT_TRUE(metadata->frame_shape[1] == 1024);
|
||||
|
||||
auto pulse_id = reinterpret_cast<uint64_t*>(metadata->header_values.at("pulse_id").get());
|
||||
ASSERT_TRUE(pulse_id[0] == 6021771850);
|
||||
|
||||
auto frame = reinterpret_cast<uint64_t*>(metadata->header_values.at("frame").get());
|
||||
ASSERT_TRUE(frame[0] == 0);
|
||||
|
||||
auto is_good_frame = reinterpret_cast<uint64_t*>(metadata->header_values.at("is_good_frame").get());
|
||||
ASSERT_TRUE(is_good_frame[0] == 1);
|
||||
|
||||
auto daq_rec = reinterpret_cast<int64_t*>(metadata->header_values.at("daq_rec").get());
|
||||
ASSERT_TRUE(daq_rec[0] == -1);
|
||||
|
||||
auto pulse_id_diff = reinterpret_cast<int64_t*>(metadata->header_values.at("pulse_id_diff").get());
|
||||
ASSERT_TRUE(pulse_id_diff[0] == -1);
|
||||
|
||||
auto framenum_diff = reinterpret_cast<int64_t*>(metadata->header_values.at("framenum_diff").get());
|
||||
ASSERT_TRUE(framenum_diff[0] == -2);
|
||||
|
||||
auto missing_packets_1 = reinterpret_cast<uint64_t*>(metadata->header_values.at("missing_packets_1").get());
|
||||
ASSERT_TRUE(missing_packets_1[0] == 1);
|
||||
|
||||
auto missing_packets_2 = reinterpret_cast<uint64_t*>(metadata->header_values.at("missing_packets_2").get());
|
||||
ASSERT_TRUE(missing_packets_2[0] == 2);
|
||||
|
||||
auto daq_recs = reinterpret_cast<uint64_t*>(metadata->header_values.at("daq_recs").get());
|
||||
ASSERT_TRUE(daq_recs[0] == 3840);
|
||||
|
||||
auto pulse_ids = reinterpret_cast<uint64_t*>(metadata->header_values.at("pulse_ids").get());
|
||||
ASSERT_TRUE(pulse_ids[0] == 6021771850);
|
||||
|
||||
auto framenums = reinterpret_cast<uint64_t*>(metadata->header_values.at("framenums").get());
|
||||
ASSERT_TRUE(framenums[0] == 193);
|
||||
|
||||
auto module_number = reinterpret_cast<uint64_t*>(metadata->header_values.at("module_number").get());
|
||||
ASSERT_TRUE(module_number[0] == 0);
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
#include "gtest/gtest.h"
|
||||
#include "../src/H5Writer.hpp"
|
||||
using namespace std;
|
||||
|
||||
TEST(H5Writer, get_h5_writer)
|
||||
{
|
||||
auto dummy_writer = get_h5_writer("/dev/null");
|
||||
EXPECT_TRUE(dynamic_cast<DummyH5Writer*>(dummy_writer.get()));
|
||||
|
||||
auto real_writer = get_h5_writer("real_file.h5");
|
||||
EXPECT_FALSE(dynamic_cast<DummyH5Writer*>(real_writer.get()));
|
||||
}
|
||||
|
||||
TEST(H5Writer, DummyH5Writer)
|
||||
{
|
||||
DummyH5Writer dummy_writer;
|
||||
|
||||
EXPECT_FALSE(dummy_writer.is_file_open());
|
||||
|
||||
EXPECT_NO_THROW(dummy_writer.close_file());
|
||||
|
||||
EXPECT_THROW(dummy_writer.get_h5_file(), runtime_error);
|
||||
|
||||
unique_ptr<char> buffer(new char[1]);
|
||||
vector<size_t> shape = {1};
|
||||
|
||||
EXPECT_NO_THROW(dummy_writer.write_data("does not matter", 0, buffer.get(), shape, 0, "nop", "nop"));
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
#include "gtest/gtest.h"
|
||||
#include "test_ZmqReceiver.cpp"
|
||||
#include "test_H5Writer.cpp"
|
||||
#include "test_MetadataBuffer.cpp"
|
||||
#include "test_BufferedWriter.cpp"
|
||||
|
||||
using namespace std;
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
::testing::InitGoogleTest(&argc, argv);
|
||||
return RUN_ALL_TESTS();
|
||||
}
|
||||
Reference in New Issue
Block a user