mx/Jungfraujoch
3
2 Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases 90 Wiki Activity

ZMQPreviewPublisher: Stream CBOR data

Browse Source
This commit is contained in:
leonarski_f committed 2023-11-08 19:19:32 +01:00
1 parent cc34a9801e
commit 56476e3e5f
7 files changed
+157 -279
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

common/ZMQPreviewPublisher.cpp
+6 -39
View File
@@ -9,56 +9,23 @@ ZMQPreviewPublisher::ZMQPreviewPublisher(ZMQContext& context, const std::string&
socket.Bind(addr);
}
void ZMQPreviewPublisher::Start(const DiffractionExperiment &experiment, const JFCalibration &calibration) {
void ZMQPreviewPublisher::StartDataCollection(const uint8_t *image_data, size_t image_size, int64_t preview_stride) {
{
std::unique_lock<std::mutex> ul(m);
stride = experiment.GetPreviewStride();
stride = preview_stride;
current_part = -1;
}
auto mask = calibration.CalculateNexusMask(experiment);
JFJochProtoBuf::PreviewFrame frame;
frame.set_image_number(-1);
frame.set_width(experiment.GetXPixelsNum());
frame.set_height(experiment.GetYPixelsNum());
frame.set_pixel_depth(4);
frame.set_data(mask.data(), experiment.GetPixelsNum() * sizeof(uint32_t));
socket.Send(grpcToJson(frame));
socket.Send(image_data, image_size);
}
void ZMQPreviewPublisher::Stop(const DiffractionExperiment& experiment) {}
void ZMQPreviewPublisher::Publish(const DiffractionExperiment& experiment, const void* image_data, const DataMessage &message) {
void ZMQPreviewPublisher::SendImage(const uint8_t *image_data, size_t image_size, int64_t image_number) {
{
std::unique_lock<std::mutex> ul(m);
int64_t part = message.number / stride;
int64_t part = image_number / stride;
if (current_part >= part)
return;
else
current_part = part;
}
JFJochProtoBuf::PreviewFrame frame;
frame.set_image_number(message.number);
frame.set_total_images(experiment.GetImageNum());
frame.set_wavelength_a(experiment.GetWavelength_A());
frame.set_beam_x_pxl(experiment.GetBeamX_pxl());
frame.set_beam_y_pxl(experiment.GetBeamY_pxl());
frame.set_saturation_value(experiment.GetOverflow());
frame.set_file_prefix(experiment.GetFilePrefix());
frame.set_detector_distance_mm(experiment.GetDetectorDistance_mm());
frame.set_width(experiment.GetXPixelsNum());
frame.set_height(experiment.GetYPixelsNum());
frame.set_pixel_depth(experiment.GetPixelDepth());
frame.set_data(image_data, experiment.GetPixelsNum() * experiment.GetPixelDepth());
for (const auto &s: message.spots) {
auto fr = frame.add_spots();
fr->set_x(s.x);
fr->set_y(s.y);
fr->set_indexed(s.indexed);
}
socket.Send(grpcToJson(frame));
socket.Send(image_data, image_size);
}
common/ZMQPreviewPublisher.h
+2 -3
View File
@@ -19,9 +19,8 @@ class ZMQPreviewPublisher {
mutable std::mutex m;
public:
ZMQPreviewPublisher(ZMQContext& context, const std::string& addr);
void Start(const DiffractionExperiment& experiment, const JFCalibration &calibration);
void Publish(const DiffractionExperiment& experiment, const void* image_data, const DataMessage &message);
void Stop(const DiffractionExperiment& experiment);
void StartDataCollection(const uint8_t *image_data, size_t image_size, int64_t preview_stride);
void SendImage(const uint8_t *image_data, size_t image_size, int64_t image_number);
};
grpc/jfjoch.proto
-23
View File
@@ -405,29 +405,6 @@ message DataProcessingSettings {
bool preview_indexed_only = 10;
}
message PreviewFrameSpot {
float x = 1;
float y = 2;
bool indexed = 3;
}
message PreviewFrame {
int64 image_number = 1;
int64 total_images = 2;
float wavelength_A = 3;
float beam_x_pxl = 4;
float beam_y_pxl = 5;
float detector_distance_mm = 6;
int64 saturation_value = 7;
string file_prefix = 8;
int64 width = 9;
int64 height = 10;
int64 pixel_depth = 11;
reserved 12;
bytes data = 13;
repeated PreviewFrameSpot spots = 14;
}
// Broker
message ModuleStatistics {
python/jfjoch_pb2.py
+39 -43
View File
@@ -13,25 +13,25 @@ _sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0cjfjoch.proto\x12\x0eJFJochProtoBuf\"\x07\n\x05\x45mpty\"W\n\x08UnitCell\x12\t\n\x01\x61\x18\x01 \x01(\x02\x12\t\n\x01\x62\x18\x02 \x01(\x02\x12\t\n\x01\x63\x18\x03 \x01(\x02\x12\r\n\x05\x61lpha\x18\x04 \x01(\x02\x12\x0c\n\x04\x62\x65ta\x18\x05 \x01(\x02\x12\r\n\x05gamma\x18\x06 \x01(\x02\")\n\x06Vector\x12\t\n\x01x\x18\x01 \x01(\x02\x12\t\n\x01y\x18\x02 \x01(\x02\x12\t\n\x01z\x18\x03 \x01(\x02\"|\n\x10RotationSettings\x12\x17\n\x0fstart_angle_deg\x18\x01 \x01(\x02\x12 \n\x18\x61ngle_incr_per_image_deg\x18\x02 \x01(\x02\x12-\n\rrotation_axis\x18\x03 \x01(\x0b\x32\x16.JFJochProtoBuf.Vector\"\x1c\n\x04Plot\x12\t\n\x01x\x18\x01 \x03(\x02\x12\t\n\x01y\x18\x02 \x03(\x02\"\xc4\x04\n\x0f\x44\x61tasetSettings\x12\x1a\n\x12images_per_trigger\x18\x01 \x01(\x03\x12\x10\n\x08ntrigger\x18\x02 \x01(\x03\x12:\n\x11\x66pga_pixel_output\x18\x03 \x01(\x0e\x32\x1f.JFJochProtoBuf.FPGAPixelOutput\x12\x11\n\tsummation\x18\x04 \x01(\x03\x12\x12\n\nbeam_x_pxl\x18\x05 \x01(\x02\x12\x12\n\nbeam_y_pxl\x18\x06 \x01(\x02\x12\x1c\n\x14\x64\x65tector_distance_mm\x18\x07 \x01(\x02\x12\x19\n\x11photon_energy_keV\x18\x08 \x01(\x02\x12\x13\n\x0b\x66ile_prefix\x18\t \x01(\t\x12\x17\n\x0f\x64\x61ta_file_count\x18\n \x01(\x03\x12\x30\n\x0b\x63ompression\x18\x0b \x01(\x0e\x32\x1b.JFJochProtoBuf.Compression\x12\x13\n\x0bsample_name\x18\x0c \x01(\t\x12\x30\n\tunit_cell\x18\r \x01(\x0b\x32\x18.JFJochProtoBuf.UnitCellH\x00\x88\x01\x01\x12\x1a\n\x12space_group_number\x18\x0e \x01(\x03\x12 \n\x18rad_int_solid_angle_corr\x18\x12 \x01(\x08\x12!\n\x19rad_int_polarization_corr\x18\x13 \x01(\x08\x12#\n\x1brad_int_polarization_factor\x18\x14 \x01(\x02\x12\x18\n\x10save_calibration\x18\x15 \x01(\x08\x42\x0c\n\n_unit_cell\"\x9a\x03\n\x10\x44\x65tectorSettings\x12\x15\n\rframe_time_us\x18\x01 \x01(\x03\x12\x1a\n\rcount_time_us\x18\x02 \x01(\x03H\x00\x88\x01\x01\x12\x1a\n\x12storage_cell_count\x18\x03 \x01(\x03\x12%\n\x1duse_internal_packet_generator\x18\x04 \x01(\x08\x12\x18\n\x10\x63ollect_raw_data\x18\x05 \x01(\x08\x12\x1f\n\x12pedestal_g0_frames\x18\x06 \x01(\x03H\x01\x88\x01\x01\x12\x1f\n\x12pedestal_g1_frames\x18\x07 \x01(\x03H\x02\x88\x01\x01\x12\x1f\n\x12pedestal_g2_frames\x18\x08 \x01(\x03H\x03\x88\x01\x01\x12\"\n\x15storage_cell_delay_ns\x18\n \x01(\x03H\x04\x88\x01\x01\x42\x10\n\x0e_count_time_usB\x15\n\x13_pedestal_g0_framesB\x15\n\x13_pedestal_g1_framesB\x15\n\x13_pedestal_g2_framesB\x18\n\x16_storage_cell_delay_ns\"b\n\x16\x44\x65tectorModuleGeometry\x12\x0e\n\x06pixel0\x18\x01 \x01(\x03\x12\x1b\n\x13\x66\x61st_direction_step\x18\x02 \x01(\x03\x12\x1b\n\x13slow_direction_step\x18\x03 \x01(\x03\"z\n\x10\x44\x65tectorGeometry\x12\x11\n\twidth_pxl\x18\x01 \x01(\x03\x12\x12\n\nheight_pxl\x18\x02 \x01(\x03\x12?\n\x0fmodule_geometry\x18\x03 \x03(\x0b\x32&.JFJochProtoBuf.DetectorModuleGeometry\"\xb2\x01\n\x08\x44\x65tector\x12\x10\n\x08nmodules\x18\x01 \x01(\x03\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x15\n\rpixel_size_mm\x18\x03 \x01(\x02\x12\x17\n\x0fmodule_hostname\x18\x04 \x03(\t\x12\x32\n\x08geometry\x18\x05 \x01(\x0b\x32 .JFJochProtoBuf.DetectorGeometry\x12\x1b\n\x13udp_interface_count\x18\x07 \x01(\x03\"\xdf\x05\n\x10InternalSettings\x12\"\n\x1a\x66rame_time_pedestalG1G2_us\x18\x01 \x01(\x03\x12\x15\n\rframe_time_us\x18\x03 \x01(\x03\x12\x15\n\rcount_time_us\x18\x04 \x01(\x03\x12*\n\x08\x64\x65tector\x18\x05 \x01(\x0b\x32\x18.JFJochProtoBuf.Detector\x12\x14\n\x0cndatastreams\x18\x06 \x01(\x03\x12&\n\x1einternal_fpga_packet_generator\x18\t \x01(\x08\x12\x15\n\rstorage_cells\x18\n \x01(\x03\x12\x1a\n\x12storage_cell_start\x18\x0b \x01(\x03\x12\x1d\n\x15storage_cell_delay_ns\x18\' \x01(\x03\x12\x1a\n\x12pedestal_g0_frames\x18\x0c \x01(\x03\x12\x1a\n\x12pedestal_g1_frames\x18\r \x01(\x03\x12\x1a\n\x12pedestal_g2_frames\x18\x0e \x01(\x03\x12\x19\n\x11preview_period_us\x18\x0f \x01(\x03\x12*\n\x04mode\x18\x13 \x01(\x0e\x32\x1c.JFJochProtoBuf.DetectorMode\x12\x19\n\x11mask_module_edges\x18\x14 \x01(\x08\x12\x17\n\x0fmask_chip_edges\x18\x15 \x01(\x08\x12\x16\n\x0eipv4_base_addr\x18\x16 \x01(\x03\x12\r\n\x05low_q\x18\x1a \x01(\x02\x12\x0e\n\x06high_q\x18\x1b \x01(\x02\x12\x11\n\tq_spacing\x18\x1c \x01(\x02\x12\x10\n\x08git_sha1\x18\x1d \x01(\t\x12\x10\n\x08git_date\x18\x1e \x01(\t\x12\x13\n\x0bsource_name\x18 \x01(\t\x12\x19\n\x11source_name_short\x18! \x01(\t\x12\x17\n\x0finstrument_name\x18\" \x01(\t\x12\x1d\n\x15instrument_name_short\x18# \x01(\t\x12\x18\n\x10\x64\x65\x62ug_pixel_mask\x18& \x01(\x08\"|\n\x14JungfraujochSettings\x12\x30\n\x07\x64\x61taset\x18\x01 \x01(\x0b\x32\x1f.JFJochProtoBuf.DatasetSettings\x12\x32\n\x08internal\x18\x02 \x01(\x0b\x32 .JFJochProtoBuf.InternalSettings\"\x1e\n\nJFPedestal\x12\x10\n\x08pedestal\x18\x01 \x01(\x0c\"-\n\x11JFGainCalibration\x12\x18\n\x10gain_calibration\x18\x01 \x01(\x0c\"\xb2\x01\n\rJFCalibration\x12\x10\n\x08nmodules\x18\x01 \x01(\x04\x12\x16\n\x0enstorage_cells\x18\x02 \x01(\x04\x12,\n\x08pedestal\x18\x03 \x03(\x0b\x32\x1a.JFJoLine truncated
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0cjfjoch.proto\x12\x0eJFJochProtoBuf\"\x07\n\x05\x45mpty\"W\n\x08UnitCell\x12\t\n\x01\x61\x18\x01 \x01(\x02\x12\t\n\x01\x62\x18\x02 \x01(\x02\x12\t\n\x01\x63\x18\x03 \x01(\x02\x12\r\n\x05\x61lpha\x18\x04 \x01(\x02\x12\x0c\n\x04\x62\x65ta\x18\x05 \x01(\x02\x12\r\n\x05gamma\x18\x06 \x01(\x02\")\n\x06Vector\x12\t\n\x01x\x18\x01 \x01(\x02\x12\t\n\x01y\x18\x02 \x01(\x02\x12\t\n\x01z\x18\x03 \x01(\x02\"|\n\x10RotationSettings\x12\x17\n\x0fstart_angle_deg\x18\x01 \x01(\x02\x12 \n\x18\x61ngle_incr_per_image_deg\x18\x02 \x01(\x02\x12-\n\rrotation_axis\x18\x03 \x01(\x0b\x32\x16.JFJochProtoBuf.Vector\"\x1c\n\x04Plot\x12\t\n\x01x\x18\x01 \x03(\x02\x12\t\n\x01y\x18\x02 \x03(\x02\"\xc4\x04\n\x0f\x44\x61tasetSettings\x12\x1a\n\x12images_per_trigger\x18\x01 \x01(\x03\x12\x10\n\x08ntrigger\x18\x02 \x01(\x03\x12:\n\x11\x66pga_pixel_output\x18\x03 \x01(\x0e\x32\x1f.JFJochProtoBuf.FPGAPixelOutput\x12\x11\n\tsummation\x18\x04 \x01(\x03\x12\x12\n\nbeam_x_pxl\x18\x05 \x01(\x02\x12\x12\n\nbeam_y_pxl\x18\x06 \x01(\x02\x12\x1c\n\x14\x64\x65tector_distance_mm\x18\x07 \x01(\x02\x12\x19\n\x11photon_energy_keV\x18\x08 \x01(\x02\x12\x13\n\x0b\x66ile_prefix\x18\t \x01(\t\x12\x17\n\x0f\x64\x61ta_file_count\x18\n \x01(\x03\x12\x30\n\x0b\x63ompression\x18\x0b \x01(\x0e\x32\x1b.JFJochProtoBuf.Compression\x12\x13\n\x0bsample_name\x18\x0c \x01(\t\x12\x30\n\tunit_cell\x18\r \x01(\x0b\x32\x18.JFJochProtoBuf.UnitCellH\x00\x88\x01\x01\x12\x1a\n\x12space_group_number\x18\x0e \x01(\x03\x12 \n\x18rad_int_solid_angle_corr\x18\x12 \x01(\x08\x12!\n\x19rad_int_polarization_corr\x18\x13 \x01(\x08\x12#\n\x1brad_int_polarization_factor\x18\x14 \x01(\x02\x12\x18\n\x10save_calibration\x18\x15 \x01(\x08\x42\x0c\n\n_unit_cell\"\x9a\x03\n\x10\x44\x65tectorSettings\x12\x15\n\rframe_time_us\x18\x01 \x01(\x03\x12\x1a\n\rcount_time_us\x18\x02 \x01(\x03H\x00\x88\x01\x01\x12\x1a\n\x12storage_cell_count\x18\x03 \x01(\x03\x12%\n\x1duse_internal_packet_generator\x18\x04 \x01(\x08\x12\x18\n\x10\x63ollect_raw_data\x18\x05 \x01(\x08\x12\x1f\n\x12pedestal_g0_frames\x18\x06 \x01(\x03H\x01\x88\x01\x01\x12\x1f\n\x12pedestal_g1_frames\x18\x07 \x01(\x03H\x02\x88\x01\x01\x12\x1f\n\x12pedestal_g2_frames\x18\x08 \x01(\x03H\x03\x88\x01\x01\x12\"\n\x15storage_cell_delay_ns\x18\n \x01(\x03H\x04\x88\x01\x01\x42\x10\n\x0e_count_time_usB\x15\n\x13_pedestal_g0_framesB\x15\n\x13_pedestal_g1_framesB\x15\n\x13_pedestal_g2_framesB\x18\n\x16_storage_cell_delay_ns\"b\n\x16\x44\x65tectorModuleGeometry\x12\x0e\n\x06pixel0\x18\x01 \x01(\x03\x12\x1b\n\x13\x66\x61st_direction_step\x18\x02 \x01(\x03\x12\x1b\n\x13slow_direction_step\x18\x03 \x01(\x03\"z\n\x10\x44\x65tectorGeometry\x12\x11\n\twidth_pxl\x18\x01 \x01(\x03\x12\x12\n\nheight_pxl\x18\x02 \x01(\x03\x12?\n\x0fmodule_geometry\x18\x03 \x03(\x0b\x32&.JFJochProtoBuf.DetectorModuleGeometry\"\xb2\x01\n\x08\x44\x65tector\x12\x10\n\x08nmodules\x18\x01 \x01(\x03\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x15\n\rpixel_size_mm\x18\x03 \x01(\x02\x12\x17\n\x0fmodule_hostname\x18\x04 \x03(\t\x12\x32\n\x08geometry\x18\x05 \x01(\x0b\x32 .JFJochProtoBuf.DetectorGeometry\x12\x1b\n\x13udp_interface_count\x18\x07 \x01(\x03\"\xdf\x05\n\x10InternalSettings\x12\"\n\x1a\x66rame_time_pedestalG1G2_us\x18\x01 \x01(\x03\x12\x15\n\rframe_time_us\x18\x03 \x01(\x03\x12\x15\n\rcount_time_us\x18\x04 \x01(\x03\x12*\n\x08\x64\x65tector\x18\x05 \x01(\x0b\x32\x18.JFJochProtoBuf.Detector\x12\x14\n\x0cndatastreams\x18\x06 \x01(\x03\x12&\n\x1einternal_fpga_packet_generator\x18\t \x01(\x08\x12\x15\n\rstorage_cells\x18\n \x01(\x03\x12\x1a\n\x12storage_cell_start\x18\x0b \x01(\x03\x12\x1d\n\x15storage_cell_delay_ns\x18\' \x01(\x03\x12\x1a\n\x12pedestal_g0_frames\x18\x0c \x01(\x03\x12\x1a\n\x12pedestal_g1_frames\x18\r \x01(\x03\x12\x1a\n\x12pedestal_g2_frames\x18\x0e \x01(\x03\x12\x19\n\x11preview_period_us\x18\x0f \x01(\x03\x12*\n\x04mode\x18\x13 \x01(\x0e\x32\x1c.JFJochProtoBuf.DetectorMode\x12\x19\n\x11mask_module_edges\x18\x14 \x01(\x08\x12\x17\n\x0fmask_chip_edges\x18\x15 \x01(\x08\x12\x16\n\x0eipv4_base_addr\x18\x16 \x01(\x03\x12\r\n\x05low_q\x18\x1a \x01(\x02\x12\x0e\n\x06high_q\x18\x1b \x01(\x02\x12\x11\n\tq_spacing\x18\x1c \x01(\x02\x12\x10\n\x08git_sha1\x18\x1d \x01(\t\x12\x10\n\x08git_date\x18\x1e \x01(\t\x12\x13\n\x0bsource_name\x18 \x01(\t\x12\x19\n\x11source_name_short\x18! \x01(\t\x12\x17\n\x0finstrument_name\x18\" \x01(\t\x12\x1d\n\x15instrument_name_short\x18# \x01(\t\x12\x18\n\x10\x64\x65\x62ug_pixel_mask\x18& \x01(\x08\"|\n\x14JungfraujochSettings\x12\x30\n\x07\x64\x61taset\x18\x01 \x01(\x0b\x32\x1f.JFJochProtoBuf.DatasetSettings\x12\x32\n\x08internal\x18\x02 \x01(\x0b\x32 .JFJochProtoBuf.InternalSettings\"\x1e\n\nJFPedestal\x12\x10\n\x08pedestal\x18\x01 \x01(\x0c\"-\n\x11JFGainCalibration\x12\x18\n\x10gain_calibration\x18\x01 \x01(\x0c\"\xb2\x01\n\rJFCalibration\x12\x10\n\x08nmodules\x18\x01 \x01(\x04\x12\x16\n\x0enstorage_cells\x18\x02 \x01(\x04\x12,\n\x08pedestal\x18\x03 \x03(\x0b\x32\x1a.JFJoLine truncated
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'jfjoch_pb2', globals())
if _descriptor._USE_C_DESCRIPTORS == False:
DESCRIPTOR._options = None
_COMPRESSION._serialized_start=8801
_COMPRESSION._serialized_end=8885
_DETECTORMODE._serialized_start=8887
_DETECTORMODE._serialized_end=8977
_FPGAFIFOSTATUSENUM._serialized_start=8979
_FPGAFIFOSTATUSENUM._serialized_end=9033
_STATE._serialized_start=9035
_STATE._serialized_end=9129
_FPGAPIXELOUTPUT._serialized_start=9131
_FPGAPIXELOUTPUT._serialized_end=9204
_PLOTTYPE._serialized_start=9206
_PLOTTYPE._serialized_end=9329
_COMPRESSION._serialized_start=8421
_COMPRESSION._serialized_end=8505
_DETECTORMODE._serialized_start=8507
_DETECTORMODE._serialized_end=8597
_FPGAFIFOSTATUSENUM._serialized_start=8599
_FPGAFIFOSTATUSENUM._serialized_end=8653
_STATE._serialized_start=8655
_STATE._serialized_end=8749
_FPGAPIXELOUTPUT._serialized_start=8751
_FPGAPIXELOUTPUT._serialized_end=8824
_PLOTTYPE._serialized_start=8826
_PLOTTYPE._serialized_end=8949
_EMPTY._serialized_start=32
_EMPTY._serialized_end=39
_UNITCELL._serialized_start=41
@@ -104,34 +104,30 @@ if _descriptor._USE_C_DESCRIPTORS == False:
_FPGASTATUS._serialized_end=6449
_DATAPROCESSINGSETTINGS._serialized_start=6452
_DATAPROCESSINGSETTINGS._serialized_end=6828
_PREVIEWFRAMESPOT._serialized_start=6830
_PREVIEWFRAMESPOT._serialized_end=6887
_PREVIEWFRAME._serialized_start=6890
_PREVIEWFRAME._serialized_end=7208
_MODULESTATISTICS._serialized_start=7211
_MODULESTATISTICS._serialized_end=7448
_IMAGE._serialized_start=7450
_IMAGE._serialized_end=7523
_MASKTOLOAD._serialized_start=7525
_MASKTOLOAD._serialized_end=7571
_MEASUREMENTSTATISTICS._serialized_start=7574
_MEASUREMENTSTATISTICS._serialized_end=8187
_BROKERSTATUS._serialized_start=8190
_BROKERSTATUS._serialized_end=8405
_BROKERFULLSTATUS._serialized_start=8408
_BROKERFULLSTATUS._serialized_end=8572
_DETECTORLISTELEMENT._serialized_start=8574
_DETECTORLISTELEMENT._serialized_end=8646
_DETECTORLIST._serialized_start=8648
_DETECTORLIST._serialized_end=8766
_DETECTORSELECTION._serialized_start=8768
_DETECTORSELECTION._serialized_end=8799
_GRPC_JFJOCHRECEIVER._serialized_start=9332
_GRPC_JFJOCHRECEIVER._serialized_end=10025
_GRPC_JFJOCHWRITER._serialized_start=10028
_GRPC_JFJOCHWRITER._serialized_end=10230
_GRPC_JFJOCHDETECTOR._serialized_start=10233
_GRPC_JFJOCHDETECTOR._serialized_end=10619
_GRPC_JFJOCHBROKER._serialized_start=10622
_GRPC_JFJOCHBROKER._serialized_end=12242
_MODULESTATISTICS._serialized_start=6831
_MODULESTATISTICS._serialized_end=7068
_IMAGE._serialized_start=7070
_IMAGE._serialized_end=7143
_MASKTOLOAD._serialized_start=7145
_MASKTOLOAD._serialized_end=7191
_MEASUREMENTSTATISTICS._serialized_start=7194
_MEASUREMENTSTATISTICS._serialized_end=7807
_BROKERSTATUS._serialized_start=7810
_BROKERSTATUS._serialized_end=8025
_BROKERFULLSTATUS._serialized_start=8028
_BROKERFULLSTATUS._serialized_end=8192
_DETECTORLISTELEMENT._serialized_start=8194
_DETECTORLISTELEMENT._serialized_end=8266
_DETECTORLIST._serialized_start=8268
_DETECTORLIST._serialized_end=8386
_DETECTORSELECTION._serialized_start=8388
_DETECTORSELECTION._serialized_end=8419
_GRPC_JFJOCHRECEIVER._serialized_start=8952
_GRPC_JFJOCHRECEIVER._serialized_end=9645
_GRPC_JFJOCHWRITER._serialized_start=9648
_GRPC_JFJOCHWRITER._serialized_end=9850
_GRPC_JFJOCHDETECTOR._serialized_start=9853
_GRPC_JFJOCHDETECTOR._serialized_end=10239
_GRPC_JFJOCHBROKER._serialized_start=10242
_GRPC_JFJOCHBROKER._serialized_end=11862
# @@protoc_insertion_point(module_scope)
receiver/JFJochReceiver.cpp
+71 -73
View File
@@ -72,9 +72,6 @@ JFJochReceiver::JFJochReceiver(const JFJochProtoBuf::ReceiverInput &settings,
logger.Info("NUMA policy: {}", numa_policy.GetName());
if (experiment.GetDetectorMode() == DetectorMode::Conversion) {
if (preview_publisher != nullptr)
preview_publisher->Start(experiment, calib.value());
rad_int_mapping = std::make_unique<RadialIntegrationMapping>(experiment);
rad_int_profile = std::make_unique<RadialIntegrationProfile>(*rad_int_mapping, experiment);
rad_int_corr = CalcRadIntCorr(experiment);
@@ -136,74 +133,76 @@ JFJochReceiver::JFJochReceiver(const JFJochProtoBuf::ReceiverInput &settings,
if (experiment.GetImageNum() > 0) {
logger.Info("Data file count {}", experiment.GetDataFileCount());
if (push_images_to_writer) {
StartMessage message{};
experiment.FillMessage(message);
message.arm_date = time_UTC(std::chrono::system_clock::now());
StartMessage message{};
experiment.FillMessage(message);
message.arm_date = time_UTC(std::chrono::system_clock::now());
JFJochBitShuffleCompressor compressor(CompressionAlgorithm::BSHUF_LZ4);
std::vector<uint8_t> pixel_mask;
std::vector<std::vector<uint8_t> > pedestal;
JFJochBitShuffleCompressor compressor(CompressionAlgorithm::BSHUF_LZ4);
std::vector<uint8_t> pixel_mask;
std::vector<std::vector<uint8_t> > pedestal;
if (calib) {
size_t xpixel = experiment.GetXPixelsNum();
size_t ypixel = experiment.GetYPixelsNum();
if (calib) {
size_t xpixel = experiment.GetXPixelsNum();
size_t ypixel = experiment.GetYPixelsNum();
pixel_mask = compressor.Compress(calib->CalculateNexusMask(experiment, 0));
message.AddPixelMask(CBORImage{
.data = pixel_mask.data(),
.size = pixel_mask.size(),
.xpixel = (size_t) xpixel,
.ypixel = (size_t) ypixel,
.pixel_depth_bytes = 4,
.pixel_is_signed = false,
.pixel_is_float = false,
.algorithm = CompressionAlgorithm::BSHUF_LZ4,
.channel = "sc0"
});
if (experiment.GetSaveCalibration()) {
for (int sc = 0; sc < experiment.GetStorageCellNumber(); sc++) {
for (int gain = 0; gain < 3; gain++) {
auto tmp = compressor.Compress(calib->GetPedestal(gain, sc));
pedestal.emplace_back(tmp);
std::string channel = "pedestal_G" + std::to_string(gain);
pixel_mask = compressor.Compress(calib->CalculateNexusMask(experiment, 0));
message.AddPixelMask(CBORImage{
.data = pixel_mask.data(),
.size = pixel_mask.size(),
.xpixel = (size_t) xpixel,
.ypixel = (size_t) ypixel,
.pixel_depth_bytes = 4,
.pixel_is_signed = false,
.pixel_is_float = false,
.algorithm = CompressionAlgorithm::BSHUF_LZ4,
.channel = "sc0"
});
if (experiment.GetSaveCalibration()) {
for (int sc = 0; sc < experiment.GetStorageCellNumber(); sc++) {
for (int gain = 0; gain < 3; gain++) {
auto tmp = compressor.Compress(calib->GetPedestal(gain, sc));
pedestal.emplace_back(tmp);
std::string channel = "pedestal_G" + std::to_string(gain);
if (experiment.GetStorageCellNumber() > 1)
channel += "_sc" + std::to_string(sc);
if (experiment.GetStorageCellNumber() > 1)
channel += "_sc" + std::to_string(sc);
CBORImage image{
.data = pedestal.at(pedestal.size() - 1).data(),
.size = pedestal.at(pedestal.size() - 1).size(),
.xpixel = (size_t) xpixel,
.ypixel = (size_t) ypixel,
.pixel_depth_bytes = 2,
.pixel_is_signed = false,
.pixel_is_float = false,
.algorithm = CompressionAlgorithm::BSHUF_LZ4,
.channel = channel
};
CBORImage image{
.data = pedestal.at(pedestal.size() - 1).data(),
.size = pedestal.at(pedestal.size() - 1).size(),
.xpixel = (size_t) xpixel,
.ypixel = (size_t) ypixel,
.pixel_depth_bytes = 2,
.pixel_is_signed = false,
.pixel_is_float = false,
.algorithm = CompressionAlgorithm::BSHUF_LZ4,
.channel = channel
};
message.AddCalibration(image);
}
message.AddCalibration(image);
}
}
}
}
if (rad_int_mapping) {
message.rad_int_bin_number = rad_int_mapping->GetBinNumber();
message.rad_int_bin_to_q = rad_int_mapping->GetBinToQ();
message.rad_int_solid_angle_corr = rad_int_mapping->GetSolidAngleCorr();
} else
message.rad_int_bin_number = 0;
if (rad_int_mapping) {
message.rad_int_bin_number = rad_int_mapping->GetBinNumber();
message.rad_int_bin_to_q = rad_int_mapping->GetBinToQ();
message.rad_int_solid_angle_corr = rad_int_mapping->GetSolidAngleCorr();
} else
message.rad_int_bin_number = 0;
std::vector<uint8_t> serialization_buffer(message.approx_size);
JFJochFrameSerializer serializer(serialization_buffer.data(), serialization_buffer.size());
std::vector<uint8_t> serialization_buffer(message.approx_size);
JFJochFrameSerializer serializer(serialization_buffer.data(), serialization_buffer.size());
serializer.SerializeSequenceStart(message);
serializer.SerializeSequenceStart(message);
if (preview_publisher != nullptr)
preview_publisher->StartDataCollection(serialization_buffer.data(), serializer.GetBufferSize(),
experiment.GetPreviewStride());
if (push_images_to_writer)
image_pusher.StartDataCollection(serialization_buffer.data(), serializer.GetBufferSize(),
experiment.GetDataFileCount());
}
for (int i = 0; i < experiment.GetImageNum(); i++)
images_to_go.Put(i);
@@ -465,26 +464,28 @@ void JFJochReceiver::FrameTransformationThread() {
*rad_int_profile_per_file[image_number % experiment.GetDataFileCount()] += *rad_int_profile_image;
}
message.receiver_available_send_buffers = GetAvailableSendBuffers();
auto send_buffer_handle = send_buffer_avail.GetBlocking();
auto ptr = send_buffer + send_buffer_size * send_buffer_handle;
JFJochFrameSerializer serializer(ptr, send_buffer_size);
PrepareCBORImage(message, experiment, nullptr, 0);
serializer.SerializeImage(message);
if (serializer.GetRemainingBuffer() < experiment.GetMaxCompressedSize())
throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds,
"Not enough memory to save image");
size_t image_size = transformation.SaveCompressedImage(ptr + serializer.GetImageAppendOffset());
serializer.AppendImage(image_size);
if (preview_publisher && (!local_data_processing_settings.preview_indexed_only() || indexed))
preview_publisher->Publish(experiment, transformation.GetPreviewImage(), message);
preview_publisher->SendImage(ptr, serializer.GetBufferSize(), image_number);
if (push_images_to_writer) {
message.receiver_available_send_buffers = GetAvailableSendBuffers();
auto send_buffer_handle = send_buffer_avail.GetBlocking();
auto ptr = send_buffer + send_buffer_size * send_buffer_handle;
JFJochFrameSerializer serializer(ptr, send_buffer_size);
PrepareCBORImage(message, experiment, nullptr, 0);
serializer.SerializeImage(message);
if (serializer.GetRemainingBuffer() < experiment.GetMaxCompressedSize())
throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds,
"Not enough memory to save image");
size_t image_size = transformation.SaveCompressedImage(ptr + serializer.GetImageAppendOffset());
serializer.AppendImage(image_size);
image_pusher.SendImage(ptr, serializer.GetBufferSize(), image_number,
&send_buffer_zero_copy_ret_val[send_buffer_handle]);
compressed_size += image_size;
}
} else
send_buffer_avail.Put(send_buffer_handle);
UpdateMaxImage(image_number);
images_sent++;
@@ -609,9 +610,6 @@ void JFJochReceiver::FinalizeMeasurement() {
logger.Info("Disconnected from writers");
}
if (preview_publisher != nullptr)
preview_publisher->Stop(experiment);
for (int d = 0; d < ndatastreams; d++)
acquisition_device[d]->Cancel();
tests/JFJochFullIntegrationTest.cpp
+21 -47
View File
@@ -648,36 +648,24 @@ TEST_CASE("JFJochIntegrationTest_ZMQ_with_preview", "[JFJochReceiver]") {
std::string s;
JFJochFrameDeserializer deserializer;
// Pixel mask
REQUIRE(rcv_preview_socket.Receive(s, false) > 0);
JFJochProtoBuf::PreviewFrame frame;
REQUIRE_NOTHROW(frame = jsonToGrpc<JFJochProtoBuf::PreviewFrame>(s));
// Check header
REQUIRE(frame.image_number() == -1);
REQUIRE(frame.width() == state_machine.NotThreadSafe_Experiment().GetXPixelsNum());
REQUIRE(frame.height() == state_machine.NotThreadSafe_Experiment().GetYPixelsNum());
REQUIRE(frame.pixel_depth() == 4);
deserializer.Process((uint8_t *) s.data(), s.size());
REQUIRE(deserializer.GetType() == JFJochFrameDeserializer::Type::START);
auto start_msg = deserializer.GetStartMessage();
REQUIRE(start_msg.image_size_x == state_machine.NotThreadSafe_Experiment().GetXPixelsNum());
REQUIRE(start_msg.image_size_y == state_machine.NotThreadSafe_Experiment().GetYPixelsNum());
// First frame
REQUIRE(rcv_preview_socket.Receive(s, false) > 0);
REQUIRE_NOTHROW(frame = jsonToGrpc<JFJochProtoBuf::PreviewFrame>(s));
deserializer.Process((uint8_t *) s.data(), s.size());
REQUIRE(deserializer.GetType() == JFJochFrameDeserializer::Type::IMAGE);
auto image_msg = deserializer.GetDataMessage();
// Check header
REQUIRE(frame.image_number() == 0);
REQUIRE(frame.beam_x_pxl() == Approx(setup.beam_x_pxl()));
REQUIRE(frame.beam_y_pxl() == Approx(setup.beam_y_pxl()));
REQUIRE(frame.width() == state_machine.NotThreadSafe_Experiment().GetXPixelsNum());
REQUIRE(frame.height() == state_machine.NotThreadSafe_Experiment().GetYPixelsNum());
REQUIRE(frame.pixel_depth() == 2);
// Check compressed image
size_t npixel = state_machine.NotThreadSafe_Experiment().GetPixelsNum();
std::vector<char> rcv_image(frame.data().size());
rcv_image = {frame.data().begin(), frame.data().end()};
REQUIRE(rcv_image.size() == state_machine.NotThreadSafe_Experiment().GetPixelsNum() * sizeof(int16_t));
REQUIRE(image_msg.number == 0);
// Check no more frames waiting
REQUIRE(rcv_preview_socket.Receive(s, false) == -1);
@@ -694,7 +682,6 @@ TEST_CASE("JFJochIntegrationTest_ZMQ_with_preview", "[JFJochReceiver]") {
writer_server->Shutdown();
}
TEST_CASE("JFJochIntegrationTest_ZMQ_with_preview_no_writer", "[JFJochReceiver]") {
Logger logger("JFJochIntegrationTest_ZMQ_with_preview_no_writer");
@@ -767,37 +754,24 @@ TEST_CASE("JFJochIntegrationTest_ZMQ_with_preview_no_writer", "[JFJochReceiver]"
logger.Info("Stopped measurement");
std::string s;
JFJochFrameDeserializer deserializer;
// Pixel mask
REQUIRE(rcv_preview_socket.Receive(s, false) > 0);
JFJochProtoBuf::PreviewFrame frame;
REQUIRE_NOTHROW(frame = jsonToGrpc<JFJochProtoBuf::PreviewFrame>(s));
// Check header
REQUIRE(frame.image_number() == -1);
REQUIRE(frame.width() == state_machine.NotThreadSafe_Experiment().GetXPixelsNum());
REQUIRE(frame.height() == state_machine.NotThreadSafe_Experiment().GetYPixelsNum());
REQUIRE(frame.pixel_depth() == 4);
deserializer.Process((uint8_t *) s.data(), s.size());
REQUIRE(deserializer.GetType() == JFJochFrameDeserializer::Type::START);
auto start_msg = deserializer.GetStartMessage();
REQUIRE(start_msg.image_size_x == state_machine.NotThreadSafe_Experiment().GetXPixelsNum());
REQUIRE(start_msg.image_size_y == state_machine.NotThreadSafe_Experiment().GetYPixelsNum());
// First frame
REQUIRE(rcv_preview_socket.Receive(s, false) > 0);
REQUIRE_NOTHROW(frame = jsonToGrpc<JFJochProtoBuf::PreviewFrame>(s));
deserializer.Process((uint8_t *) s.data(), s.size());
REQUIRE(deserializer.GetType() == JFJochFrameDeserializer::Type::IMAGE);
auto image_msg = deserializer.GetDataMessage();
// Check header
REQUIRE(frame.image_number() == 0);
REQUIRE(frame.beam_x_pxl() == Approx(setup.beam_x_pxl()));
REQUIRE(frame.beam_y_pxl() == Approx(setup.beam_y_pxl()));
REQUIRE(frame.width() == state_machine.NotThreadSafe_Experiment().GetXPixelsNum());
REQUIRE(frame.height() == state_machine.NotThreadSafe_Experiment().GetYPixelsNum());
REQUIRE(frame.pixel_depth() == 2);
// Check compressed image
size_t npixel = state_machine.NotThreadSafe_Experiment().GetPixelsNum();
std::vector<char> rcv_image(frame.data().size());
rcv_image = {frame.data().begin(), frame.data().end()};
REQUIRE(rcv_image.size() == state_machine.NotThreadSafe_Experiment().GetPixelsNum() * sizeof(int16_t));
REQUIRE(image_msg.number == 0);
// Check no more frames waiting
REQUIRE(rcv_preview_socket.Receive(s, false) == -1);
tests/ZMQPreviewPublisherTest.cpp
+18 -51
View File
@@ -14,48 +14,23 @@ TEST_CASE("ZMQPreviewPublisher","[ZMQ]") {
DiffractionExperiment experiment(DetectorGeometry(1, 1, 0, 0, false));
std::string z = "Msg 0";
JFCalibration calibration(experiment);
publisher.Start(experiment, calibration);
publisher.StartDataCollection((uint8_t *) z.data(), z.size(), experiment.GetPreviewStride());
std::vector<int16_t> image(experiment.GetPixelsNum());
// Predictable random number generator
std::mt19937 g1(19876);
std::uniform_int_distribution<int16_t> distribution(-200,25000);
for (auto &i: image)
i = distribution(g1);
std::vector<SpotToSave> spots;
spots.push_back(SpotToSave{.x = 7, .y = 8, .intensity = 34, .indexed = false});
spots.push_back(SpotToSave{.x = 37, .y = 48, .intensity = 123, .indexed = true});
DataMessage message{.number = 564, .spots = spots};
publisher.Publish(experiment, image.data(), message);
z = "Msg 1";
publisher.SendImage((uint8_t *) z.data(), z.size(), 0);
std::string s;
// Pixel mask
REQUIRE(socket.Receive(s, false) > 0);
JFJochProtoBuf::PreviewFrame frame;
REQUIRE_NOTHROW(frame = jsonToGrpc<JFJochProtoBuf::PreviewFrame>(s));
REQUIRE(frame.pixel_depth() == 4);
REQUIRE(frame.image_number() == -1);
REQUIRE(s == "Msg 0");
// Frame
REQUIRE(socket.Receive(s, false) > 0);
REQUIRE_NOTHROW(frame = jsonToGrpc<JFJochProtoBuf::PreviewFrame>(s));
REQUIRE(frame.pixel_depth() == 2);
REQUIRE(frame.image_number() == 564);
std::vector<char> image_out = {frame.data().begin(), frame.data().end()};
REQUIRE(memcmp(image.data(), image_out.data(), experiment.GetPixelsNum() * experiment.GetPixelDepth()) == 0);
REQUIRE(frame.spots_size() == 2);
REQUIRE(frame.spots(0).x() == 7);
REQUIRE(!frame.spots(0).indexed());
REQUIRE(frame.spots(1).y() == 48);
REQUIRE(frame.spots(1).indexed());
REQUIRE(s == "Msg 1");
REQUIRE(socket.Receive(s, false) < 0);
}
@@ -73,39 +48,31 @@ TEST_CASE("ZMQPreviewPublisher_FrameNumbers","[ZMQ]") {
REQUIRE(experiment.GetPreviewStride() == 10);
JFCalibration calibration(experiment);
publisher.Start(experiment, calibration);
std::vector<int16_t> image(experiment.GetPixelsNum());
std::string z = "Start";
publisher.Publish(experiment, image.data(), DataMessage{.number = 0});
publisher.Publish(experiment, image.data(), DataMessage{.number = 9});
publisher.Publish(experiment, image.data(), DataMessage{.number = 12});
publisher.Publish(experiment, image.data(), DataMessage{.number = 10});
publisher.Publish(experiment, image.data(), DataMessage{.number = 25});
publisher.Publish(experiment, image.data(), DataMessage{.number = 20});
publisher.Publish(experiment, image.data(), DataMessage{.number = 18});
publisher.StartDataCollection((uint8_t *) z.data(), z.size(), experiment.GetPreviewStride());
for (const auto& i: {0, 9, 12, 10, 25, 20, 18}) {
z = "Msg " + std::to_string(i);
publisher.SendImage((uint8_t *) z.data(), z.size(), i);
}
std::string s;
JFJochProtoBuf::PreviewFrame frame;
// Pixel mask
REQUIRE(socket.Receive(s, false) > 0);
REQUIRE(s == "Start");
// Frame
REQUIRE(socket.Receive(s, false) > 0);
REQUIRE_NOTHROW(frame = jsonToGrpc<JFJochProtoBuf::PreviewFrame>(s));
REQUIRE(frame.image_number() == 0);
std::vector<char> image_out = {frame.data().begin(), frame.data().end()};
REQUIRE(s == "Msg 0");
REQUIRE(socket.Receive(s, false) > 0);
REQUIRE_NOTHROW(frame = jsonToGrpc<JFJochProtoBuf::PreviewFrame>(s));
REQUIRE(frame.image_number() == 12);
REQUIRE(s == "Msg 12");
REQUIRE(socket.Receive(s, false) > 0);
REQUIRE_NOTHROW(frame = jsonToGrpc<JFJochProtoBuf::PreviewFrame>(s));
REQUIRE(frame.image_number() == 25);
REQUIRE(s == "Msg 25");
REQUIRE(socket.Receive(s, false) < 0);
}