sf-daq/sf_daq_buffer
0
0
Fork 0
mirror of https://github.com/paulscherrerinstitute/sf_daq_buffer.git synced 2026-04-21 16:44:38 +02:00
Code Issues Packages Projects Releases Wiki Activity

Remove stuff we no longer use

Browse Source
This commit is contained in:
babic_a
2020-04-24 14:43:09 +02:00
parent 742f584220
commit d7dcfec8f6
13 changed files with 0 additions and 678 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CMakeLists.txt
-3
View File
@@ -30,7 +30,4 @@ add_subdirectory(
add_subdirectory("core-buffer")
add_subdirectory("core-writer")
#add_subdirectory("csaxs")
add_subdirectory("sf-writer")
add_subdirectory("sf-buffer")
#add_subdirectory("swmr-writer")
csaxs/CsaxsFormat.cpp
-85
View File
@@ -1,85 +0,0 @@
#include <memory>
#include <unordered_map>
#include <iostream>
#include "config.hpp"
#include "H5Format.hpp"
using namespace std;
using s_ptr = shared_ptr<h5_base>;
class CsaxsFormat : public H5Format
{
shared_ptr<unordered_map<string, DATA_TYPE>> input_value_type = NULL;
shared_ptr<unordered_map<string, boost::any>> default_values = NULL;
shared_ptr<unordered_map<string, std::string>> dataset_move_mapping = NULL;
shared_ptr<h5_parent> file_format = NULL;
public:
~CsaxsFormat(){};
CsaxsFormat(const string& dataset_name)
{
input_value_type.reset(new unordered_map<string, DATA_TYPE>());
default_values.reset(new unordered_map<string, boost::any>());
// After format has been writen, where to move the raw datasets.
dataset_move_mapping.reset(new std::unordered_map<string, string>(
{
{config::raw_image_dataset_name, "detector/" + dataset_name},
{"frame", "detector/frame"},
{"is_good_frame", "detector/is_good_frame"},
{"missing_packets_1", "detector/missing_packets_1"},
{"missing_packets_2", "detector/missing_packets_2"},
{"daq_recs", "detector/daq_recs"},
{"daq_rec", "detector/daq_rec"},
{"framenum_diff", "detector/framenum_diff"},
{"framenums", "detector/framenums"},
{"module_number", "detector/module_number"},
}));
// Definition of the file format.
file_format.reset(
new h5_parent("", EMPTY_ROOT, {
s_ptr(new h5_group("detector", {}))
}));
}
const h5_parent& get_format_definition() const override
{
return *file_format;
}
const unordered_map<string, boost::any>& get_default_values() const override
{
return *default_values;
}
void add_calculated_values(unordered_map<string, boost::any>& values) const override
{
// No calculated values.
}
void add_input_values(unordered_map<string, boost::any>& values,
const unordered_map<string, boost::any>& input_values) const override
{
// Input value mapping is 1:1.
for (const auto& input_value : input_values) {
const auto& name = input_value.first;
const auto& value = input_value.second;
values[name] = value;
}
}
const std::unordered_map<string, DATA_TYPE>& get_input_value_type() const override
{
return *input_value_type;
}
const unordered_map<string, string>& get_dataset_move_mapping() const override {
return *dataset_move_mapping;
}
};
csaxs/Makefile
-34
View File
@@ -1,34 +0,0 @@
SRC_DIR = .
OBJ_DIR = ./obj
BIN_DIR = ./bin
MKDIR = mkdir -p
CC = g++
CFLAGS = -Wall -Wfatal-errors -std=c++11 -I${CONDA_PREFIX}/include -I${CONDA_PREFIX}/include/cpp_h5_writer
LDFLAGS = -L${CONDA_PREFIX}/lib -L/usr/lib64 -lcpp_h5_writer -lzmq -lhdf5 -lhdf5_hl -lhdf5_cpp -lhdf5_hl_cpp -lboost_system -lboost_regex -lboost_thread -lpthread
HEADERS = $(wildcard $(SRC_DIR)/*.hpp)
SRCS = $(wildcard $(SRC_DIR)/*.cpp)
OBJS = $(patsubst $(SRC_DIR)/%.cpp, $(OBJ_DIR)/%.o, $(SRCS))
csaxs_h5_writer: export LD_LIBRARY_PATH=${CONDA_PREFIX}/lib
csaxs_h5_writer: lib build_dirs $(OBJS)
$(CC) $(LDFLAGS) -o $(BIN_DIR)/csaxs_h5_writer $(OBJS) $(LDFLAGS)
lib:
$(MAKE) -C ../lib deploy
debug: CFLAGS += -DDEBUG_OUTPUT -g
debug: csaxs_h5_writer
deploy: csaxs_h5_writer
cp bin/* ${CONDA_PREFIX}/bin
$(OBJ_DIR)/%.o: $(SRC_DIR)/%.cpp
$(CC) $(CFLAGS) $(LDFLAGS) -c -o $@ $<
build_dirs:
$(MKDIR) $(OBJ_DIR) $(BIN_DIR)
clean:
rm -rf $(OBJ_DIR) $(BIN_DIR)
csaxs/conda-recipe/build.sh
-4
View File
@@ -1,4 +0,0 @@
make
cp bin/* ${PREFIX}/bin
cp ../../bin/* ${PREFIX}/lib
csaxs/conda-recipe/meta.yaml
-26
View File
@@ -1,26 +0,0 @@
package:
name: csaxs_h5_writer
version: 0.0.1
build:
number: 0
source:
path: ..
requirements:
build:
- make
- gcc
- cppzmq ==4.2.1
- hdf5 ==1.10.1
- boost ==1.61.0
run:
- cppzmq ==4.2.1
- hdf5 ==1.10.1
- boost ==1.61.0
about:
summary: "cSAXS ZMQ to H5 writer."
csaxs/csaxs_h5_writer.cpp
-69
View File
@@ -1,69 +0,0 @@
#include <iostream>
#include <sstream>
#include <stdexcept>
#include "config.hpp"
#include "WriterManager.hpp"
#include "ZmqReceiver.hpp"
#include "ProcessManager.hpp"
#include "CsaxsFormat.cpp"
int main (int argc, char *argv[])
{
if (argc != 7) {
cout << endl;
cout << "Usage: csaxs_h5_writer [connection_address] [output_file] [n_frames]";
cout << " [rest_port] [user_id] [n_modules]" << endl;
cout << "\tconnection_address: Address to connect to the stream (PULL). Example: tcp://127.0.0.1:40000" << endl;
cout << "\toutput_file: Name of the output file." << endl;
cout << "\tn_frames: Number of images to acquire. 0 for infinity (until /stop is called)." << endl;
cout << "\trest_port: Port to start the REST Api on." << endl;
cout << "\tuser_id: uid under which to run the writer. -1 to leave it as it is." << endl;
cout << "\tn_modules: Number of detector modules to be written." << endl;
cout << endl;
exit(-1);
}
string connect_address = string(argv[1]);
string output_file = string(argv[2]);
int n_frames = atoi(argv[3]);
int rest_port = atoi(argv[4]);
int user_id = atoi(argv[5]);
int n_modules = atoi(argv[6]);
string bsread_rest_address = "http://localhost:9999/";
if (user_id != -1) {
writer_utils::set_process_id(user_id);
}
writer_utils::create_destination_folder(output_file);
auto header_values = shared_ptr<unordered_map<string, HeaderDataType>>(new unordered_map<string, HeaderDataType> {
{"frame", HeaderDataType("uint64")},
{"is_good_frame", HeaderDataType("uint64")},
{"daq_rec", HeaderDataType("int64")},
{"framenum_diff", HeaderDataType("int64", n_modules)},
{"missing_packets_1", HeaderDataType("uint64", n_modules)},
{"missing_packets_2", HeaderDataType("uint64", n_modules)},
{"daq_recs", HeaderDataType("uint64", n_modules)},
{"framenums", HeaderDataType("uint64", n_modules)},
{"module_number", HeaderDataType("uint64", n_modules)}
});
CsaxsFormat format("images");
WriterManager writer_manager(format.get_input_value_type(), output_file, n_frames);
ZmqReceiver receiver(connect_address, config::zmq_n_io_threads, config::zmq_receive_timeout, header_values);
RingBuffer ring_buffer(config::ring_buffer_n_slots);
ProcessManager process_manager(writer_manager, receiver, ring_buffer, format, rest_port, bsread_rest_address);
process_manager.run_writer();
return 0;
}
sf-writer/BufferMultiReader.cpp
-82
View File
@@ -1,82 +0,0 @@
//#include <ZmqRecvModule.hpp>
//#include "BufferMultiReader.hpp"
//#include "BufferUtils.hpp"
//#include <sstream>
//
//using namespace std;
//
//BufferMultiReader::BufferMultiReader(
// const std::string& root_folder) :
// root_folder_(root_folder),
// is_running_(true),
// pulse_id_(0)
//{
// frame_buffer_ = new uint16_t[1000*32*512*1024];
// frame_metadata_buffer_ = new UdpFrameMetadata[32];
//}
//
//BufferMultiReader::~BufferMultiReader()
//{
// delete[] frame_buffer_;
// delete[] frame_metadata_buffer_;
//}
//
//char* BufferMultiReader::get_buffer()
//{
// return (char*) frame_buffer_;
//}
//
//
//UdpFrameMetadata BufferMultiReader::load_frame_to_buffer(
// const uint64_t pulse_id)
//{
// if (pulse_id >= min_cached_pulse_id_ && pulse_id <= max_cached_pulse_id_) {
// auto file_frame_index = BufferUtils::get_file_frame_index(pulse_id);
// return frame_metadata_buffer_[file_frame_index];
// }
//
// for (size_t i_module; i_module<32; i_module++) {
// stringstream name;
// name << "M";
// if (i_module < 10) name << "0";
// name << (int) i_module;
// string device_name = name.str();
//
// auto pulse_filename = BufferUtils::get_filename(
// root_folder_, device_name, pulse_id);
//
// H5::H5File input_file(pulse_filename,
// H5F_ACC_RDONLY | H5F_ACC_SWMR_READ);
//
// auto image_dataset = input_file.openDataSet("image");
// auto pulse_id_dataset = input_file.openDataSet("pulse_id");
// auto frame_id_dataset = input_file.openDataSet("frame_id");
// auto daq_rec_dataset = input_file.openDataSet("daq_rec");
// auto received_packets_dataset =
// input_file.openDataSet("received_packets");
//
// hsize_t buff_dim[3] = {1000, 32*512, 1024};
// H5::DataSpace buffer_space (3, buff_dim);
// hsize_t b_offset[] = {0, i_module*512, 0};
// hsize_t b_stride[] = {n_new_pulses, 512, 1024};
// hsize_t b_block[] = {1, 512, 1024};
// hsize_t b_count[] = {1000, 1, 1};
// buffer_space.selectHyperslab(
// H5S_SELECT_SET, b_count, b_offset, b_stride, b_block);
//
// image_dataset.read(
// image_buffer_, H5::PredType::NATIVE_UINT16,
// buffer_space);
// pulse_id_dataset.read(
// pulse_id_buffer, H5::PredType::NATIVE_UINT64);
// frame_id_dataset.read(
// frame_id_buffer, H5::PredType::NATIVE_UINT64);
// daq_rec_dataset.read(
// daq_rec_buffer, H5::PredType::NATIVE_UINT32);
// received_packets_dataset.read(
// received_packets_buffer, H5::PredType::NATIVE_UINT16);
// }
//
// UdpFrameMetadata metadata = frame_metadata_buffer_[0];
// return metadata;
//}
sf-writer/BufferMultiReader.hpp
-43
View File
@@ -1,43 +0,0 @@
//#ifndef BUFFERMULTIREADER_H
//#define BUFFERMULTIREADER_H
//
//#include <thread>
//#include "RingBuffer.hpp"
//
//
//#pragma pack(push)
//#pragma pack(1)
//struct BufferMetadata {
// uint64_t pulse_id;
// uint64_t frame_id;
// uint32_t daq_rac;
// uint16_t n_recv_frames;
//};
//#pragma pack(pop)
//
//class BufferMultiReader
//{
// const std::string root_folder_;
// std::atomic_bool is_running_;
// uint16_t* frame_buffer_;
// UdpFrameMetadata* frame_metadata_buffer_;
//
// uint64_t min_cached_pulse_id_;
// uint64_t max_cached_pulse_id_;
//
//
//
//protected:
// void read_thread(uint8_t module_number);
//
//public:
// BufferMultiReader(const std::string& root_folder);
//
// virtual ~BufferMultiReader();
//
// char* get_buffer();
//
// UdpFrameMetadata load_frame_to_buffer(const uint64_t pulse_id);
//};
//
//#endif
sf-writer/CMakeLists.txt
-19
View File
@@ -1,19 +0,0 @@
file(GLOB SOURCES
*.cpp)
add_executable(sf-writer
sf_h5_writer.cpp
BufferMultiReader.cpp
SFWriter.cpp)
set_target_properties(sf-writer PROPERTIES OUTPUT_NAME sf_h5_writer)
target_link_libraries(sf-writer
core-buffer
core-writer
external
zmq
hdf5
hdf5_hl
hdf5_cpp
boost_system
pthread)
sf-writer/SfFormat.cpp
-111
View File
@@ -1,111 +0,0 @@
#include <memory>
#include <unordered_map>
#include "config.hpp"
#include "H5Format.hpp"
using namespace std;
using s_ptr = shared_ptr<h5_base>;
class SfFormat : public H5Format
{
shared_ptr<unordered_map<string, DATA_TYPE>> input_value_type = NULL;
shared_ptr<unordered_map<string, boost::any>> default_values = NULL;
shared_ptr<unordered_map<string, std::string>> dataset_move_mapping = NULL;
shared_ptr<h5_parent> file_format = NULL;
public:
~SfFormat(){};
SfFormat(const string& dataset_name, int n_bad_modules)
{
// Input values definition type.
// Which type should be the parameters you receive over the REST api.
input_value_type.reset(
new unordered_map<string, DATA_TYPE>({
{"general/created", NX_DATE_TIME},
{"general/user", NX_CHAR},
{"general/process", NX_CHAR},
{"general/instrument", NX_CHAR}
}));
// Default values used in the file format.
default_values.reset(new std::unordered_map<string, boost::any>(
{
{"general/n_bad_modules", n_bad_modules},
{"general/detector_name", dataset_name}
}));
// After format has been writen, where to move the raw datasets.
dataset_move_mapping.reset(new std::unordered_map<string, string>(
{
{config::raw_image_dataset_name, "data/" + dataset_name + "/data"},
{"pulse_id", "data/" + dataset_name + "/pulse_id"},
{"frame", "data/" + dataset_name + "/frame"},
{"is_good_frame", "data/" + dataset_name + "/is_good_frame"},
{"missing_packets_1", "data/" + dataset_name + "/missing_packets_1"},
{"missing_packets_2", "data/" + dataset_name + "/missing_packets_2"},
{"daq_recs", "data/" + dataset_name + "/daq_recs"},
{"daq_rec", "data/" + dataset_name + "/daq_rec"},
{"framenum_diff", "data/" + dataset_name + "/framenum_diff"},
{"pulse_ids", "data/" + dataset_name + "/pulse_ids"},
{"framenums", "data/" + dataset_name + "/framenums"},
{"pulse_id_diff", "data/" + dataset_name + "/pulse_id_diff"},
{"module_number", "data/" + dataset_name + "/module_number"},
}));
// Definition of the file format.
file_format.reset(
new h5_parent("", EMPTY_ROOT, {
s_ptr(new h5_group("general", {
s_ptr(new h5_dataset("created", "general/created", NX_DATE_TIME)),
s_ptr(new h5_dataset("user", "general/user", NX_CHAR)),
s_ptr(new h5_dataset("process", "general/process", NX_CHAR)),
s_ptr(new h5_dataset("instrument", "general/instrument", NX_CHAR)),
s_ptr(new h5_dataset("detector_name", "general/detector_name", NX_CHAR)),
s_ptr(new h5_dataset("n_bad_modules", "general/n_bad_modules", NX_INT)),
})),
s_ptr(new h5_group("data", {
s_ptr(new h5_group(dataset_name, {}))
}))
}));
}
const h5_parent& get_format_definition() const override
{
return *file_format;
}
const unordered_map<string, boost::any>& get_default_values() const override
{
return *default_values;
}
void add_calculated_values(unordered_map<string, boost::any>& values) const override
{
// No calculated values.
}
void add_input_values(unordered_map<string, boost::any>& values,
const unordered_map<string, boost::any>& input_values) const override
{
// Input value mapping is 1:1.
for (const auto& input_value : input_values) {
const auto& name = input_value.first;
const auto& value = input_value.second;
values[name] = value;
}
}
const std::unordered_map<string, DATA_TYPE>& get_input_value_type() const override
{
return *input_value_type;
}
const unordered_map<string, string>& get_dataset_move_mapping() const override {
return *dataset_move_mapping;
}
};
sf-writer/test/test_sf_writer.py
-70
View File
@@ -1,70 +0,0 @@
import argparse
from mflow import mflow, BIND, PUSH
import numpy
import requests
def main():
parser = argparse.ArgumentParser(description='Generate SF detector test stream.')
parser.add_argument("-o", '--stream_address', default="tcp://127.0.0.1:8888", help="Address to bind the stream.")
parser.add_argument("-i", '--n_images', default=100, help="Number of images to generate.")
parser.add_argument('-m', '--n_modules', default=5, help="Number of modules to simulate.")
arguments = parser.parse_args()
stream_address = arguments.stream_address
n_images = arguments.n_images
n_modules = arguments.n_modules
shape = [n_modules * 512, 1024]
dtype = "uint16"
print("Use writer start command:")
print("\t./sf_h5_writer tcp://127.0.0.1:8888 test%d.h5 100 12000 -1 localhost 5 0 test_sf_writer 10")
writer_parameters_url = "http://localhost:12000/parameters"
writer_parameters = {
"general/created": "right now",
"general/user": "tester",
"general/process": "test_writer.py",
"general/instrument": "test script"
}
header = {
"pulse_id": 0,
"frame": 0,
"is_good_frame": 1,
"daq_rec": 1,
"shape": shape,
"type": dtype,
"pulse_id_diff": [1] * n_modules,
"framenum_diff": [1] * n_modules,
"missing_packets_1": [1] * n_modules,
"missing_packets_2": [1] * n_modules,
"daq_recs": [1] * n_modules,
"pulse_ids": [1] * n_modules,
"framenums": [1] * n_modules,
"module_number": [1] * n_modules,
}
image = numpy.random.rand(shape[0], shape[1]) * (2 ** 16)
image = image.astype(dtype)
stream = mflow.connect(address=stream_address, conn_type=BIND, mode=PUSH)
for pulse_id in range(n_images):
print("Sending pulse_id %d." % pulse_id)
header["pulse_id"] = pulse_id
header["frame"] = pulse_id
stream.send(header, send_more=True, as_json=True)
stream.send(image, send_more=False)
if pulse_id == 0:
requests.post(url=writer_parameters_url, json=writer_parameters)
if __name__ == "__main__":
main()
test/Makefile
-32
View File
@@ -1,32 +0,0 @@
SRC_DIR = .
OBJ_DIR = ./obj
BIN_DIR = ./bin
MKDIR = mkdir -p
CC = g++
CFLAGS = -Wall -Wfatal-errors -std=c++11 -I${CONDA_PREFIX}/include -I${CONDA_PREFIX}/include/cpp_h5_writer
LDFLAGS = -L${CONDA_PREFIX}/lib -L/usr/lib64 -lcpp_h5_writer -lzmq -lhdf5 -lhdf5_hl -lhdf5_cpp -lhdf5_hl_cpp -lboost_system -lboost_regex -lboost_thread -lpthread
HEADERS = $(wildcard $(SRC_DIR)/*.hpp)
SRCS = $(wildcard $(SRC_DIR)/*.cpp)
OBJS = $(patsubst $(SRC_DIR)/%.cpp, $(OBJ_DIR)/%.o, $(SRCS))
h5_write_perf: export LD_LIBRARY_PATH=${CONDA_PREFIX}/lib
h5_write_perf: CFLAGS += -DDEBUG_OUTPUT -g
h5_write_perf: lib build_dirs $(OBJS)
$(CC) $(LDFLAGS) -o $(BIN_DIR)/h5_write_perf $(OBJS) $(LDFLAGS)
lib:
$(MAKE) -C ../lib deploy
deploy: h5_write_perf
cp bin/* ${CONDA_PREFIX}/bin
$(OBJ_DIR)/%.o: $(SRC_DIR)/%.cpp
$(CC) $(CFLAGS) $(LDFLAGS) -c -o $@ $<
build_dirs:
$(MKDIR) $(OBJ_DIR) $(BIN_DIR)
clean:
rm -rf $(OBJ_DIR) $(BIN_DIR)
test/h5_write_perf.cpp
-100
View File
@@ -1,100 +0,0 @@
#include <iostream>
#include <sstream>
#include <stdexcept>
#include <unistd.h>
#include <string>
#include "H5Writer.hpp"
using namespace std;
using namespace std::chrono;
void write_frame(H5Writer& writer, size_t index, char* buffer, size_t buffer_length,
char* metadata_buffer, size_t metadata_buffer_length, size_t n_metadata, size_t n_modules) {
writer.write_data("data",
index,
buffer,
{n_modules * 512, 1024},
buffer_length,
"uint16",
"little");
for (size_t meta_index=0; meta_index < n_metadata; meta_index++) {
writer.write_data(to_string(meta_index),
index,
metadata_buffer,
{n_modules},
metadata_buffer_length,
"uint64",
"little");
}
}
int main (int argc, char *argv[])
{
if (argc != 6) {
cout << endl;
cout << "Usage: h5_write_perf [output_file] [n_frames] [n_modules] [frame_rate] [n_metadata]" << endl;
cout << "\toutput_file: Name of the output file." << endl;
cout << "\tn_frames: Number of images to write." << endl;
cout << "\tn_modules: Numbers of 512*1024 modules." << endl;
cout << "\tframe_rate: Frame rate in Hz." << endl;
cout << "\tn_metadata: Number of metadata datasets to be written." << endl;
cout << endl;
exit(-1);
}
string output_file = string(argv[1]);
int n_frames = atoi(argv[2]);
int n_modules = atoi(argv[3]);
int frame_rate = atoi(argv[4]);
int n_metadata = atoi(argv[5]);
size_t buffer_length = n_modules * 512 * 1024 * sizeof(u_int16_t);
char* buffer = new char[buffer_length];
size_t metadata_buffer_length = 4 * n_modules;
char* metadata_buffer = new char[metadata_buffer_length];
H5Writer writer(output_file, n_frames, n_frames, n_frames);
// Initialize all datasets;
write_frame(writer, 0, buffer, buffer_length, metadata_buffer, metadata_buffer_length, n_metadata, n_modules);
auto total_sleep_time = 0.0;
auto total_write_time = 0.0;
auto missed_frames = 0;
auto start_time_frame = std::chrono::system_clock::now();
for (int index=0; index<n_frames; index++) {
write_frame(writer, index, buffer, buffer_length, metadata_buffer, metadata_buffer_length, n_metadata, n_modules);
auto time_diff = duration<float, milli>(std::chrono::system_clock::now() - start_time_frame).count();
auto sleep_time = (1.0/frame_rate*1000) - time_diff;
if (sleep_time < 0) {
cout << "Not in time for frame " << index << endl;
sleep_time = 0;
missed_frames++;
}
usleep(sleep_time*1000);
total_sleep_time += sleep_time;
total_write_time += time_diff;
start_time_frame = std::chrono::system_clock::now();
}
writer.close_file();
cout << "total sleep: " << total_sleep_time/1000 << " total write: " << total_write_time/1000;
cout << " missed frames: " << missed_frames/float(n_frames)*100 << "%" <<endl;
return 0;
}