HDF5 file writer - per file analysis

This commit is contained in:
leonarski_f committed 2023-04-19 23:09:26 +02:00
1 parent fae3cf5b42
commit f14693fe43
12 files changed
+123 -96

No files matched your search

+2
View File
@@ -322,6 +322,8 @@ void JFJochStateMachine::SetFullMeasurementOutput(JFJochProtoBuf::BrokerFullStat
for (const auto &i: output.writer()) {
writer_perf += i.performance_mbs();
images_written += i.nimages();
for (const auto &f: i.file_statistics())
*tmp.add_file_statistics() = f;
}
tmp.set_writer_performance_mbs(writer_perf);
+8
View File
@@ -286,10 +286,16 @@ message WriterInput {
int64 series_id = 2;
}
message DataFileStatistics {
string name = 1;
int64 nimages = 2;
}
message WriterOutput {
int64 nimages = 1;
float performance_MBs = 2;
float performance_Hz = 3;
repeated DataFileStatistics file_statistics = 4;
}
// Detector
@@ -455,6 +461,8 @@ message MeasurementStatistics {
int64 detector_width = 12;
int64 detector_height = 13;
int64 detector_pixel_depth = 14;
repeated DataFileStatistics file_statistics = 15;
}
message BrokerStatus {
+63 -61
View File
@@ -13,7 +13,7 @@ _sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0cjfjoch.proto\x12\x0eJFJochProtoBuf\"\x07\n\x05\x45mpty\"W\n\x08UnitCell\x12\t\n\x01\x61\x18\x01 \x01(\x02\x12\t\n\x01\x62\x18\x02 \x01(\x02\x12\t\n\x01\x63\x18\x03 \x01(\x02\x12\r\n\x05\x61lpha\x18\x04 \x01(\x02\x12\x0c\n\x04\x62\x65ta\x18\x05 \x01(\x02\x12\r\n\x05gamma\x18\x06 \x01(\x02\")\n\x06Vector\x12\t\n\x01x\x18\x01 \x01(\x02\x12\t\n\x01y\x18\x02 \x01(\x02\x12\t\n\x01z\x18\x03 \x01(\x02\"|\n\x10RotationSettings\x12\x17\n\x0fstart_angle_deg\x18\x01 \x01(\x02\x12 \n\x18\x61ngle_incr_per_image_deg\x18\x02 \x01(\x02\x12-\n\rrotation_axis\x18\x03 \x01(\x0b\x32\x16.JFJochProtoBuf.Vector\"\x1c\n\x04Plot\x12\t\n\x01x\x18\x01 \x03(\x02\x12\t\n\x01y\x18\x02 \x03(\x02\"\xa5\x04\n\x0f\x44\x61tasetSettings\x12\x1a\n\x12images_per_trigger\x18\x01 \x01(\x03\x12\x10\n\x08ntrigger\x18\x02 \x01(\x03\x12\x13\n\tsummation\x18\x03 \x01(\x03H\x00\x12\x17\n\rimage_time_us\x18\x04 \x01(\x03H\x00\x12\x12\n\nbeam_x_pxl\x18\x05 \x01(\x02\x12\x12\n\nbeam_y_pxl\x18\x06 \x01(\x02\x12\x1c\n\x14\x64\x65tector_distance_mm\x18\x07 \x01(\x02\x12\x19\n\x11photon_energy_keV\x18\x08 \x01(\x02\x12\x13\n\x0b\x66ile_prefix\x18\t \x01(\t\x12\x17\n\x0f\x64\x61ta_file_count\x18\n \x01(\x03\x12\x30\n\x0b\x63ompression\x18\x0b \x01(\x0e\x32\x1b.JFJochProtoBuf.Compression\x12\x13\n\x0bsample_name\x18\x0c \x01(\t\x12\x30\n\tunit_cell\x18\r \x01(\x0b\x32\x18.JFJochProtoBuf.UnitCellH\x01\x88\x01\x01\x12\x1a\n\x12space_group_number\x18\x0e \x01(\x03\x12\x36\n\x11scattering_vector\x18\x0f \x01(\x0b\x32\x16.JFJochProtoBuf.VectorH\x02\x88\x01\x01\x12\x18\n\x10\x61pply_pixel_mask\x18\x10 \x01(\x08\x12\x12\n\nbinning2x2\x18\x11 \x01(\x08\x42\x08\n\x06timingB\x0c\n\n_unit_cellB\x14\n\x12_scattering_vector\"\xf6\x02\n\x10\x44\x65tectorSettings\x12\x15\n\rframe_time_us\x18\x01 \x01(\x03\x12\x1a\n\rcount_time_us\x18\x02 \x01(\x03H\x00\x88\x01\x01\x12\x19\n\x11use_storage_cells\x18\x03 \x01(\x08\x12%\n\x1duse_internal_packet_generator\x18\x04 \x01(\x08\x12\x18\n\x10\x63ollect_raw_data\x18\x05 \x01(\x08\x12\x1f\n\x12pedestal_g0_frames\x18\x06 \x01(\x03H\x01\x88\x01\x01\x12\x1f\n\x12pedestal_g1_frames\x18\x07 \x01(\x03H\x02\x88\x01\x01\x12\x1f\n\x12pedestal_g2_frames\x18\x08 \x01(\x03H\x03\x88\x01\x01\x12\x19\n\x11\x63onversion_on_cpu\x18\t \x01(\x08\x42\x10\n\x0e_count_time_usB\x15\n\x13_pedestal_g0_framesB\x15\n\x13_pedestal_g1_framesB\x15\n\x13_pedestal_g2_frames\"b\n\x16\x44\x65tectorModuleGeometry\x12\x0e\n\x06pixel0\x18\x01 \x01(\x03\x12\x1b\n\x13\x66\x61st_direction_step\x18\x02 \x01(\x03\x12\x1b\n\x13slow_direction_step\x18\x03 \x01(\x03\"z\n\x10\x44\x65tectorGeometry\x12\x11\n\twidth_pxl\x18\x01 \x01(\x03\x12\x12\n\nheight_pxl\x18\x02 \x01(\x03\x12?\n\x0fmodule_geometry\x18\x03 \x03(\x0b\x32&.JFJochProtoBuf.DetectorModuleGeometry\"\xc1\x01\n\x08\x44\x65tector\x12\x10\n\x08nmodules\x18\x01 \x01(\x03\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x15\n\rpixel_size_mm\x18\x03 \x01(\x02\x12\x17\n\x0fmodule_hostname\x18\x04 \x03(\t\x12\x32\n\x08geometry\x18\x05 \x01(\x0b\x32 .JFJochProtoBuf.DetectorGeometry\x12*\n\x04type\x18\x06 \x01(\x0e\x32\x1c.JFJochProtoBuf.DetectorType\"\xe1\x05\n\x10InternalSettings\x12\"\n\x1a\x66rame_time_pedestalG1G2_us\x18\x01 \x01(\x03\x12\x15\n\rframe_time_us\x18\x03 \x01(\x03\x12\x15\n\rcount_time_us\x18\x04 \x01(\x03\x12*\n\x08\x64\x65tector\x18\x05 \x01(\x0b\x32\x18.JFJochProtoBuf.Detector\x12\x14\n\x0cndatastreams\x18\x06 \x01(\x03\x12&\n\x1einternal_fpga_packet_generator\x18\t \x01(\x08\x12\x15\n\rstorage_cells\x18\n \x01(\x03\x12\x1a\n\x12storage_cell_start\x18\x0b \x01(\x03\x12\x1a\n\x12pedestal_g0_frames\x18\x0c \x01(\x03\x12\x1a\n\x12pedestal_g1_frames\x18\r \x01(\x03\x12\x1a\n\x12pedestal_g2_frames\x18\x0e \x01(\x03\x12\x19\n\x11preview_period_us\x18\x0f \x01(\x03\x12\x1e\n\x16spot_finding_period_us\x18\x10 \x01(\x03\x12*\n\x04mode\x18\x13 \x01(\x0e\x32\x1c.JFJochProtoBuf.DetectorMode\x12\x19\n\x11mask_module_edges\x18\x14 \x01(\x08\x12\x17\n\x0fmask_chip_edges\x18\x15 \x01(\x08\x12\x16\n\x0eipv4_base_addr\x18\x16 \x01(\x03\x12\r\n\x05low_q\x18\x1a \x01(\x02\x12\x0e\n\x06high_q\x18\x1b \x01(\x02\x12\x11\n\tq_spacing\x18\x1c \x01(\x02\x12\x10\n\x08git_sha1\x18\x1d \x01(\t\x12\x10\n\x08git_date\x18\x1e \x01(\t\x12\x19\n\x11\x63onversion_on_cpu\x18\x1f \x01(\x08\x12\x13\n\x0bsource_name\x18 \x01(\t\x12\x19\n\x11source_name_short\x18! \x01(\t\x12\x17\n\x0finstrument_name\x18\" \x01(\t\x12\x1d\n\x15instrument_name_short\x18# \x01(\t\"|\n\x14JungfraujochSettings\x12\x30\n\x07\x64\x61taset\x18\x01 \x01(\x0b\x32\x1f.JFJochProtoBuf.DatasetSettings\x12\x32\n\x08internal\x18\x02 \x01(\x0b\x32 .JFJochProtoBuf.InternalSettings\"\x1e\n\nJFPedestal\x12\x10\n\x08pedestal\x18\x01 \x01(\x0c\"-\n\x11JFGainCalibration\x12\x18\n\x10gain_calibration\x18\x01 \x01(\x0c\"\xb2\x01\n\rJFCalibration\x12\x10\n\x08nmodules\x18\x01 \x01(\x04\x12\x16\n\x0enstorage_cells\x18\x02 \x01(\x04\x12,\n\x08pedestal\x18\x03 \x03(\x0b\x32\x1a.JFJochProtoBuf.JFPedestal\x12\x0c\n\x04mask\x18\Line truncated
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0cjfjoch.proto\x12\x0eJFJochProtoBuf\"\x07\n\x05\x45mpty\"W\n\x08UnitCell\x12\t\n\x01\x61\x18\x01 \x01(\x02\x12\t\n\x01\x62\x18\x02 \x01(\x02\x12\t\n\x01\x63\x18\x03 \x01(\x02\x12\r\n\x05\x61lpha\x18\x04 \x01(\x02\x12\x0c\n\x04\x62\x65ta\x18\x05 \x01(\x02\x12\r\n\x05gamma\x18\x06 \x01(\x02\")\n\x06Vector\x12\t\n\x01x\x18\x01 \x01(\x02\x12\t\n\x01y\x18\x02 \x01(\x02\x12\t\n\x01z\x18\x03 \x01(\x02\"|\n\x10RotationSettings\x12\x17\n\x0fstart_angle_deg\x18\x01 \x01(\x02\x12 \n\x18\x61ngle_incr_per_image_deg\x18\x02 \x01(\x02\x12-\n\rrotation_axis\x18\x03 \x01(\x0b\x32\x16.JFJochProtoBuf.Vector\"\x1c\n\x04Plot\x12\t\n\x01x\x18\x01 \x03(\x02\x12\t\n\x01y\x18\x02 \x03(\x02\"\xa5\x04\n\x0f\x44\x61tasetSettings\x12\x1a\n\x12images_per_trigger\x18\x01 \x01(\x03\x12\x10\n\x08ntrigger\x18\x02 \x01(\x03\x12\x13\n\tsummation\x18\x03 \x01(\x03H\x00\x12\x17\n\rimage_time_us\x18\x04 \x01(\x03H\x00\x12\x12\n\nbeam_x_pxl\x18\x05 \x01(\x02\x12\x12\n\nbeam_y_pxl\x18\x06 \x01(\x02\x12\x1c\n\x14\x64\x65tector_distance_mm\x18\x07 \x01(\x02\x12\x19\n\x11photon_energy_keV\x18\x08 \x01(\x02\x12\x13\n\x0b\x66ile_prefix\x18\t \x01(\t\x12\x17\n\x0f\x64\x61ta_file_count\x18\n \x01(\x03\x12\x30\n\x0b\x63ompression\x18\x0b \x01(\x0e\x32\x1b.JFJochProtoBuf.Compression\x12\x13\n\x0bsample_name\x18\x0c \x01(\t\x12\x30\n\tunit_cell\x18\r \x01(\x0b\x32\x18.JFJochProtoBuf.UnitCellH\x01\x88\x01\x01\x12\x1a\n\x12space_group_number\x18\x0e \x01(\x03\x12\x36\n\x11scattering_vector\x18\x0f \x01(\x0b\x32\x16.JFJochProtoBuf.VectorH\x02\x88\x01\x01\x12\x18\n\x10\x61pply_pixel_mask\x18\x10 \x01(\x08\x12\x12\n\nbinning2x2\x18\x11 \x01(\x08\x42\x08\n\x06timingB\x0c\n\n_unit_cellB\x14\n\x12_scattering_vector\"\xf6\x02\n\x10\x44\x65tectorSettings\x12\x15\n\rframe_time_us\x18\x01 \x01(\x03\x12\x1a\n\rcount_time_us\x18\x02 \x01(\x03H\x00\x88\x01\x01\x12\x19\n\x11use_storage_cells\x18\x03 \x01(\x08\x12%\n\x1duse_internal_packet_generator\x18\x04 \x01(\x08\x12\x18\n\x10\x63ollect_raw_data\x18\x05 \x01(\x08\x12\x1f\n\x12pedestal_g0_frames\x18\x06 \x01(\x03H\x01\x88\x01\x01\x12\x1f\n\x12pedestal_g1_frames\x18\x07 \x01(\x03H\x02\x88\x01\x01\x12\x1f\n\x12pedestal_g2_frames\x18\x08 \x01(\x03H\x03\x88\x01\x01\x12\x19\n\x11\x63onversion_on_cpu\x18\t \x01(\x08\x42\x10\n\x0e_count_time_usB\x15\n\x13_pedestal_g0_framesB\x15\n\x13_pedestal_g1_framesB\x15\n\x13_pedestal_g2_frames\"b\n\x16\x44\x65tectorModuleGeometry\x12\x0e\n\x06pixel0\x18\x01 \x01(\x03\x12\x1b\n\x13\x66\x61st_direction_step\x18\x02 \x01(\x03\x12\x1b\n\x13slow_direction_step\x18\x03 \x01(\x03\"z\n\x10\x44\x65tectorGeometry\x12\x11\n\twidth_pxl\x18\x01 \x01(\x03\x12\x12\n\nheight_pxl\x18\x02 \x01(\x03\x12?\n\x0fmodule_geometry\x18\x03 \x03(\x0b\x32&.JFJochProtoBuf.DetectorModuleGeometry\"\xc1\x01\n\x08\x44\x65tector\x12\x10\n\x08nmodules\x18\x01 \x01(\x03\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x15\n\rpixel_size_mm\x18\x03 \x01(\x02\x12\x17\n\x0fmodule_hostname\x18\x04 \x03(\t\x12\x32\n\x08geometry\x18\x05 \x01(\x0b\x32 .JFJochProtoBuf.DetectorGeometry\x12*\n\x04type\x18\x06 \x01(\x0e\x32\x1c.JFJochProtoBuf.DetectorType\"\xe1\x05\n\x10InternalSettings\x12\"\n\x1a\x66rame_time_pedestalG1G2_us\x18\x01 \x01(\x03\x12\x15\n\rframe_time_us\x18\x03 \x01(\x03\x12\x15\n\rcount_time_us\x18\x04 \x01(\x03\x12*\n\x08\x64\x65tector\x18\x05 \x01(\x0b\x32\x18.JFJochProtoBuf.Detector\x12\x14\n\x0cndatastreams\x18\x06 \x01(\x03\x12&\n\x1einternal_fpga_packet_generator\x18\t \x01(\x08\x12\x15\n\rstorage_cells\x18\n \x01(\x03\x12\x1a\n\x12storage_cell_start\x18\x0b \x01(\x03\x12\x1a\n\x12pedestal_g0_frames\x18\x0c \x01(\x03\x12\x1a\n\x12pedestal_g1_frames\x18\r \x01(\x03\x12\x1a\n\x12pedestal_g2_frames\x18\x0e \x01(\x03\x12\x19\n\x11preview_period_us\x18\x0f \x01(\x03\x12\x1e\n\x16spot_finding_period_us\x18\x10 \x01(\x03\x12*\n\x04mode\x18\x13 \x01(\x0e\x32\x1c.JFJochProtoBuf.DetectorMode\x12\x19\n\x11mask_module_edges\x18\x14 \x01(\x08\x12\x17\n\x0fmask_chip_edges\x18\x15 \x01(\x08\x12\x16\n\x0eipv4_base_addr\x18\x16 \x01(\x03\x12\r\n\x05low_q\x18\x1a \x01(\x02\x12\x0e\n\x06high_q\x18\x1b \x01(\x02\x12\x11\n\tq_spacing\x18\x1c \x01(\x02\x12\x10\n\x08git_sha1\x18\x1d \x01(\t\x12\x10\n\x08git_date\x18\x1e \x01(\t\x12\x19\n\x11\x63onversion_on_cpu\x18\x1f \x01(\x08\x12\x13\n\x0bsource_name\x18 \x01(\t\x12\x19\n\x11source_name_short\x18! \x01(\t\x12\x17\n\x0finstrument_name\x18\" \x01(\t\x12\x1d\n\x15instrument_name_short\x18# \x01(\t\"|\n\x14JungfraujochSettings\x12\x30\n\x07\x64\x61taset\x18\x01 \x01(\x0b\x32\x1f.JFJochProtoBuf.DatasetSettings\x12\x32\n\x08internal\x18\x02 \x01(\x0b\x32 .JFJochProtoBuf.InternalSettings\"\x1e\n\nJFPedestal\x12\x10\n\x08pedestal\x18\x01 \x01(\x0c\"-\n\x11JFGainCalibration\x12\x18\n\x10gain_calibration\x18\x01 \x01(\x0c\"\xb2\x01\n\rJFCalibration\x12\x10\n\x08nmodules\x18\x01 \x01(\x04\x12\x16\n\x0enstorage_cells\x18\x02 \x01(\x04\x12,\n\x08pedestal\x18\x03 \x03(\x0b\x32\x1a.JFJochProtoBuf.JFPedestal\x12\x0c\n\x04mask\x18\Line truncated
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'jfjoch_pb2', globals())
@@ -22,18 +22,18 @@ if _descriptor._USE_C_DESCRIPTORS == False:
DESCRIPTOR._options = None
_FPGASTATUS_FIFOSTATUSENTRY._options = None
_FPGASTATUS_FIFOSTATUSENTRY._serialized_options = b'8\001'
_COMPRESSION._serialized_start=8399
_COMPRESSION._serialized_end=8483
_DETECTORTYPE._serialized_start=8485
_DETECTORTYPE._serialized_end=8524
_DETECTORMODE._serialized_start=8526
_DETECTORMODE._serialized_end=8616
_FPGAFIFOSTATUS._serialized_start=8618
_FPGAFIFOSTATUS._serialized_end=8668
_STATE._serialized_start=8670
_STATE._serialized_end=8764
_PLOTTYPE._serialized_start=8766
_PLOTTYPE._serialized_end=8842
_COMPRESSION._serialized_start=8575
_COMPRESSION._serialized_end=8659
_DETECTORTYPE._serialized_start=8661
_DETECTORTYPE._serialized_end=8700
_DETECTORMODE._serialized_start=8702
_DETECTORMODE._serialized_end=8792
_FPGAFIFOSTATUS._serialized_start=8794
_FPGAFIFOSTATUS._serialized_end=8844
_STATE._serialized_start=8846
_STATE._serialized_end=8940
_PLOTTYPE._serialized_start=8942
_PLOTTYPE._serialized_end=9018
_EMPTY._serialized_start=32
_EMPTY._serialized_end=39
_UNITCELL._serialized_start=41
@@ -84,52 +84,54 @@ if _descriptor._USE_C_DESCRIPTORS == False:
_RECEIVERDATAPROCESSINGPLOTS._serialized_end=4341
_WRITERINPUT._serialized_start=4343
_WRITERINPUT._serialized_end=4405
_WRITEROUTPUT._serialized_start=4407
_WRITEROUTPUT._serialized_end=4487
_DETECTORMODULECONFIG._serialized_start=4490
_DETECTORMODULECONFIG._serialized_end=4748
_DETECTORCONFIG._serialized_start=4750
_DETECTORCONFIG._serialized_end=4846
_DETECTORINPUT._serialized_start=4849
_DETECTORINPUT._serialized_end=5098
_DETECTOROUTPUT._serialized_start=5100
_DETECTOROUTPUT._serialized_end=5116
_DETECTORSTATUS._serialized_start=5118
_DETECTORSTATUS._serialized_end=5216
_FPGASTATUS._serialized_start=5219
_FPGASTATUS._serialized_end=6289
_FPGASTATUS_FIFOSTATUSENTRY._serialized_start=6208
_FPGASTATUS_FIFOSTATUSENTRY._serialized_end=6289
_INDEXERDATAPROCESSINGPLOTS._serialized_start=6291
_INDEXERDATAPROCESSINGPLOTS._serialized_end=6364
_DATAPROCESSINGSETTINGS._serialized_start=6367
_DATAPROCESSINGSETTINGS._serialized_end=6713
_PREVIEWFRAME._serialized_start=6716
_PREVIEWFRAME._serialized_end=6985
_MODULESTATISTICS._serialized_start=6988
_MODULESTATISTICS._serialized_end=7225
_IMAGE._serialized_start=7227
_IMAGE._serialized_end=7300
_MASKTOLOAD._serialized_start=7302
_MASKTOLOAD._serialized_end=7348
_MEASUREMENTSTATISTICS._serialized_start=7351
_MEASUREMENTSTATISTICS._serialized_end=7859
_BROKERSTATUS._serialized_start=7862
_BROKERSTATUS._serialized_end=8003
_BROKERFULLSTATUS._serialized_start=8006
_BROKERFULLSTATUS._serialized_end=8170
_DETECTORLISTELEMENT._serialized_start=8172
_DETECTORLISTELEMENT._serialized_end=8244
_DETECTORLIST._serialized_start=8246
_DETECTORLIST._serialized_end=8364
_DETECTORSELECTION._serialized_start=8366
_DETECTORSELECTION._serialized_end=8397
_GRPC_JFJOCHRECEIVER._serialized_start=8845
_GRPC_JFJOCHRECEIVER._serialized_end=9512
_GRPC_JFJOCHWRITER._serialized_start=9515
_GRPC_JFJOCHWRITER._serialized_end=9717
_GRPC_JFJOCHDETECTOR._serialized_start=9720
_GRPC_JFJOCHDETECTOR._serialized_end=10106
_GRPC_JFJOCHBROKER._serialized_start=10109
_GRPC_JFJOCHBROKER._serialized_end=11698
_DATAFILESTATISTICS._serialized_start=4407
_DATAFILESTATISTICS._serialized_end=4458
_WRITEROUTPUT._serialized_start=4461
_WRITEROUTPUT._serialized_end=4602
_DETECTORMODULECONFIG._serialized_start=4605
_DETECTORMODULECONFIG._serialized_end=4863
_DETECTORCONFIG._serialized_start=4865
_DETECTORCONFIG._serialized_end=4961
_DETECTORINPUT._serialized_start=4964
_DETECTORINPUT._serialized_end=5213
_DETECTOROUTPUT._serialized_start=5215
_DETECTOROUTPUT._serialized_end=5231
_DETECTORSTATUS._serialized_start=5233
_DETECTORSTATUS._serialized_end=5331
_FPGASTATUS._serialized_start=5334
_FPGASTATUS._serialized_end=6404
_FPGASTATUS_FIFOSTATUSENTRY._serialized_start=6323
_FPGASTATUS_FIFOSTATUSENTRY._serialized_end=6404
_INDEXERDATAPROCESSINGPLOTS._serialized_start=6406
_INDEXERDATAPROCESSINGPLOTS._serialized_end=6479
_DATAPROCESSINGSETTINGS._serialized_start=6482
_DATAPROCESSINGSETTINGS._serialized_end=6828
_PREVIEWFRAME._serialized_start=6831
_PREVIEWFRAME._serialized_end=7100
_MODULESTATISTICS._serialized_start=7103
_MODULESTATISTICS._serialized_end=7340
_IMAGE._serialized_start=7342
_IMAGE._serialized_end=7415
_MASKTOLOAD._serialized_start=7417
_MASKTOLOAD._serialized_end=7463
_MEASUREMENTSTATISTICS._serialized_start=7466
_MEASUREMENTSTATISTICS._serialized_end=8035
_BROKERSTATUS._serialized_start=8038
_BROKERSTATUS._serialized_end=8179
_BROKERFULLSTATUS._serialized_start=8182
_BROKERFULLSTATUS._serialized_end=8346
_DETECTORLISTELEMENT._serialized_start=8348
_DETECTORLISTELEMENT._serialized_end=8420
_DETECTORLIST._serialized_start=8422
_DETECTORLIST._serialized_end=8540
_DETECTORSELECTION._serialized_start=8542
_DETECTORSELECTION._serialized_end=8573
_GRPC_JFJOCHRECEIVER._serialized_start=9021
_GRPC_JFJOCHRECEIVER._serialized_end=9688
_GRPC_JFJOCHWRITER._serialized_start=9691
_GRPC_JFJOCHWRITER._serialized_end=9893
_GRPC_JFJOCHDETECTOR._serialized_start=9896
_GRPC_JFJOCHDETECTOR._serialized_end=10282
_GRPC_JFJOCHBROKER._serialized_start=10285
_GRPC_JFJOCHBROKER._serialized_end=11874
# @@protoc_insertion_point(module_scope)
+2 -1
View File
@@ -105,7 +105,7 @@ TEST_CASE("JFJochIntegrationTest_ZMQ", "[JFJochReceiver]") {
REQUIRE(statistics.detector_width() == 2068);
REQUIRE(statistics.detector_height() == 2164);
REQUIRE(statistics.detector_pixel_depth() == 2);
REQUIRE(statistics.file_statistics_size() == 2);
fpga_receiver_server->Shutdown();
writer_server->Shutdown();
}
@@ -393,6 +393,7 @@ TEST_CASE("JFJochIntegrationTest_ZMQ_3Writers", "[JFJochReceiver]") {
REQUIRE(statistics.collection_efficiency() == 1.0);
REQUIRE(statistics.images_collected() == nimages);
REQUIRE(statistics.images_written() == nimages);
REQUIRE(statistics.file_statistics_size() == 5);
fpga_receiver_server->Shutdown();
writer_server_0->Shutdown();
+6 -6
View File
@@ -101,14 +101,14 @@ void HDF5DataFile::Write(const DataMessage &msg, uint64_t image_number) {
timestamp[image_number] = msg.timestamp;
}
size_t HDF5DataFile::GetMaxImageNumber() const {
return max_image_number;
HDF5DataFileStatistics HDF5DataFile::GetStatistics() const {
HDF5DataFileStatistics ret;
ret.max_image_number = max_image_number;
ret.total_images = nimages;
ret.filename = filename;
return ret;
}
size_t HDF5DataFile::GetNumImages() const {
return nimages;
}
std::string HDF5DataFile::GetFilename() const {
return filename;
}
+7 -3
View File
@@ -12,9 +12,14 @@
#include "../common/SpotToSave.h"
#include "../frame_serialize/ImageMessage.h"
struct HDF5DataFileStatistics {
std::string filename;
uint64_t max_image_number;
uint64_t total_images;
};
class HDF5DataFile {
std::string filename;
std::string master_filename;
std::unique_ptr<HDF5File> data_file = nullptr;
std::unique_ptr<HDF5DataSet> data_set = nullptr;
@@ -47,9 +52,8 @@ public:
size_t max_spots = 0);
~HDF5DataFile();
void Write(const DataMessage& msg, uint64_t image_number);
size_t GetMaxImageNumber() const;
HDF5DataFileStatistics GetStatistics() const;
size_t GetNumImages() const;
std::string GetFilename() const;
};
#endif //HDF5DATAFILE_H
+8
View File
@@ -28,3 +28,11 @@ void HDF5Writer::Write(const DataMessage& message) {
files[file_number]->Write(message, message.number / files.size());
}
void HDF5Writer::GetStatistics(std::vector<HDF5DataFileStatistics> &v) const {
for (const auto &f: files) {
auto tmp = f->GetStatistics();
if (tmp.total_images > 0)
v.push_back(tmp);
}
}
+1
View File
@@ -15,6 +15,7 @@ class HDF5Writer {
public:
explicit HDF5Writer(const StartMessage &request);
void Write(const DataMessage& msg);
void GetStatistics(std::vector<HDF5DataFileStatistics> &v) const;
};
#endif //JUNGFRAUJOCH_HDF5WRITER_H
+9 -4
View File
@@ -34,10 +34,15 @@ grpc::Status JFJochWriterService::Stop(grpc::ServerContext *context, const JFJoc
std::shared_lock<std::shared_mutex> ul(m);
logger.Info("Stopping writer");
if (writer_future.valid()) {
ZMQImagePullerStatistics ret = writer_future.get();
response->set_nimages(ret.processed_images);
response->set_performance_mbs(ret.performance_MBs);
response->set_performance_hz(ret.performance_Hz);
auto ret = writer_future.get();
response->set_nimages(ret.image_puller_stats.processed_images);
response->set_performance_mbs(ret.image_puller_stats.performance_MBs);
response->set_performance_hz(ret.image_puller_stats.performance_Hz);
for (const auto &f: ret.data_file_stats) {
auto *tmp = response->add_file_statistics();
tmp->set_name(f.filename);
tmp->set_nimages(f.max_image_number + 1);
}
} else {
response->set_nimages(0);
response->set_performance_mbs(0);
+1 -1
View File
@@ -13,7 +13,7 @@ class JFJochWriterService final : public JFJochProtoBuf::gRPC_JFJochWriter::Serv
std::unique_ptr<StreamWriter> writer;
Logger &logger;
ZMQContext &zmq_context;
std::future<ZMQImagePullerStatistics> writer_future;
std::future<StreamWriterStatistics> writer_future;
public:
JFJochWriterService(ZMQContext &in_context, Logger &in_logger);
JFJochWriterService &BaseDirectory(const std::string& input);
+8 -17
View File
@@ -26,7 +26,7 @@ void StreamWriter::StartDataCollection() {
MakeDirectory(start_message.file_prefix);
}
void StreamWriter::CollectImages() {
void StreamWriter::CollectImages(std::vector<HDF5DataFileStatistics> &v) {
HDF5Writer writer(start_message);
image_puller.WaitForImage();
while (image_puller.GetFrameType() == JFJochFrameDeserializer::Type::IMAGE) {
@@ -34,6 +34,7 @@ void StreamWriter::CollectImages() {
writer.Write(image_array);
image_puller.WaitForImage();
}
writer.GetStatistics(v);
}
void StreamWriter::EndDataCollection() {
@@ -51,10 +52,11 @@ void StreamWriter::Abort() {
image_puller.Abort();
}
ZMQImagePullerStatistics StreamWriter::Run() {
StreamWriterStatistics StreamWriter::Run() {
StreamWriterStatistics ret;
StartDataCollection();
try {
CollectImages();
CollectImages(ret.data_file_stats);
} catch (JFJochException &e) {
// Error during collecting images will skip to end data collection
// End data collection will consume all images till the end
@@ -62,19 +64,8 @@ ZMQImagePullerStatistics StreamWriter::Run() {
}
EndDataCollection();
auto puller_stats = image_puller.GetStatistics();
ret.image_puller_stats = image_puller.GetStatistics();
logger.Info("Write task done. Images = {} Throughput = {:.0f} MB/s Frame rate = {:.0f} Hz",
puller_stats.processed_images, puller_stats.performance_MBs, puller_stats.performance_Hz);
return puller_stats;
ret.image_puller_stats.processed_images, ret.image_puller_stats.performance_MBs, ret.image_puller_stats.performance_Hz);
return ret;
}
void StreamWriter::BaseDirectory(const std::string &input) {
logger.Info("Setting base directory to " + input);
try {
std::filesystem::current_path(input);
} catch (const std::filesystem::filesystem_error& err) {
logger.Error("Cannot set base directory: " + std::string(err.what()));
throw JFJochException(JFJochExceptionCategory::FileWriteError,
"Cannot set base directory " + std::string(err.what()));
}
}
+8 -3
View File
@@ -5,6 +5,12 @@
#define JUNGFRAUJOCH_STREAMWRITER_H
#include "ZMQImagePuller.h"
#include "HDF5DataFile.h"
struct StreamWriterStatistics {
ZMQImagePullerStatistics image_puller_stats;
std::vector<HDF5DataFileStatistics> data_file_stats;
};
class StreamWriter {
ZMQImagePuller image_puller;
@@ -12,13 +18,12 @@ class StreamWriter {
StartMessage start_message;
void StartDataCollection();
void CollectImages();
void CollectImages(std::vector<HDF5DataFileStatistics> &v);
void EndDataCollection();
public:
StreamWriter(ZMQContext& context, Logger &logger, const std::string& zmq_addr);
ZMQImagePullerStatistics Run();
StreamWriterStatistics Run();
void Abort();
void BaseDirectory(const std::string& path);
};