mx/Jungfraujoch
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 44 Wiki Activity

ADU histogram: Save

Browse Source
This commit is contained in:
leonarski_f
2023-10-21 19:51:21 +02:00
parent 53f4f4acf9
commit 99741ae5c5
13 changed files with 125 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
common/ADUHistogram.cpp Normal file
View File

@@ -0,0 +1,26 @@
// Copyright (2019-2023) Paul Scherrer Institute
#include "ADUHistogram.h"
ADUHistogram::ADUHistogram() : histogram(ADU_HISTO_BIN_COUNT) {
}
void ADUHistogram::Add(const DeviceOutput &output) {
std::unique_lock<std::mutex> ul(m);
for (int i = 0; i < ADU_HISTO_BIN_COUNT; i++)
histogram[i] += output.adu_histogram[i];
}
const std::vector<uint64_t> &ADUHistogram::GetHistogram() const {
return histogram;
}
void ADUHistogram::GetPlot(JFJochProtoBuf::Plot &plot) const {
std::unique_lock<std::mutex> ul(m);
for (int i = 0; i < ADU_HISTO_BIN_COUNT; i++) {
plot.add_x(ADU_HISTO_BIN_WIDTH * i + ADU_HISTO_BIN_WIDTH / 2);
plot.add_y(histogram[i]);
}
}

20
common/ADUHistogram.h Normal file
View File

@@ -0,0 +1,20 @@
// Copyright (2019-2023) Paul Scherrer Institute
#ifndef JUNGFRAUJOCH_ADUHISTOGRAM_H
#define JUNGFRAUJOCH_ADUHISTOGRAM_H
#include <mutex>
#include <jfjoch.pb.h>
#include "../receiver/AcquisitionDevice.h"
class ADUHistogram {
mutable std::mutex m;
std::vector<uint64_t> histogram;
public:
explicit ADUHistogram();
void Add(const DeviceOutput &output);
const std::vector<uint64_t>& GetHistogram() const;
void GetPlot(JFJochProtoBuf::Plot &plot) const;
};
#endif //JUNGFRAUJOCH_ADUHISTOGRAM_H

4
common/CMakeLists.txt
View File

@@ -48,7 +48,9 @@ ADD_LIBRARY( CommonFunctions STATIC
CUDAWrapper.cpp
CUDAWrapper.h
NUMAHWPolicy.cpp
NUMAHWPolicy.h)
NUMAHWPolicy.h
ADUHistogram.cpp
ADUHistogram.h)
TARGET_LINK_LIBRARIES(CommonFunctions Compression FrameSerialize libzmq JFCalibration JFJochProtoBuf -lrt)

2
frame_serialize/CBORMessages.h
View File

@@ -48,6 +48,8 @@ struct DataMessage {
std::string series_unique_id;
uint64_t series_id;
std::vector<uint64_t> adu_histogram;
};
struct GoniometerAxis {

3
frame_serialize/CborUtil.h
View File

@@ -8,6 +8,7 @@
constexpr const CborTag TagMultiDimArray = 40;
constexpr const CborTag TagDECTRISCompression = 56500;
// rfc8746
constexpr const CborTag TagFloatLE = 0b01010101;
constexpr const CborTag TagUnsignedInt8Bit = 0b01000000;
constexpr const CborTag TagSignedInt8Bit = 0b01001000;
@@ -18,4 +19,6 @@ constexpr const CborTag TagSignedInt16BitLE = 0b01001101;
constexpr const CborTag TagUnsignedInt32BitLE = 0b01000110;
constexpr const CborTag TagSignedInt32BitLE = 0b01001110;
constexpr const CborTag TagUnsignedInt64BitLE = 0b01000111;
#endif //JUNGFRAUJOCH_CBORUTIL_H

48
frame_serialize/JFJochFrameDeserializer.cpp
View File

@@ -108,26 +108,6 @@ inline std::pair<uint64_t, uint64_t> GetRational(CborValue &value) {
return ret;
}
inline void GetCBORFloatArray(CborValue &value, std::vector<float> &v) {
if (GetCBORTag(value) != TagFloatLE)
throw JFJochException(JFJochExceptionCategory::CBORError, "Incorrect array type tag");
auto len = GetCBORArrayLen(value);
if (len > 0) {
CborValue array_value;
v.resize(len);
cborErr(cbor_value_enter_container(&value, &array_value));
for (int i = 0; i < len; i++)
v[i] = GetCBORFloat(array_value);
cborErr(cbor_value_leave_container(&value, &array_value));
} else
cbor_value_advance(&value);
}
inline std::pair<uint64_t, uint64_t> GetCBORDimension2DArray(CborValue &value) {
std::pair<uint64_t, uint64_t> ret;
@@ -179,6 +159,32 @@ inline std::pair<const uint8_t *, size_t> GetCBORByteString(CborValue &value) {
return {ptr, len};
}
inline void GetCBORFloatArray(CborValue &value, std::vector<float> &v) {
if (GetCBORTag(value) != TagFloatLE)
throw JFJochException(JFJochExceptionCategory::CBORError, "Incorrect array type tag");
auto [ptr, len] = GetCBORByteString(value);
if (len % sizeof(float))
throw JFJochException(JFJochExceptionCategory::CBORError, "Size mismatch");
v.resize(len / sizeof(float));
memcpy(v.data(), ptr, len);
}
inline void GetCBORUInt64Array(CborValue &value, std::vector<uint64_t> &v) {
if (GetCBORTag(value) != TagUnsignedInt64BitLE)
throw JFJochException(JFJochExceptionCategory::CBORError, "Incorrect array type tag");
auto [ptr, len] = GetCBORByteString(value);
if (len % sizeof(uint64_t))
throw JFJochException(JFJochExceptionCategory::CBORError, "Size mismatch");
v.resize(len / sizeof(uint64_t));
memcpy(v.data(), ptr, len);
}
inline void GetCBORTypedArray(CBORImage &v, CborValue &value) {
CborTag tag = GetCBORTag(value);
@@ -401,6 +407,8 @@ void JFJochFrameDeserializer::ProcessImageMessageUserDataElement(CborValue &valu
data_message.storage_cell = GetCBORUInt(map_value) & UINT32_MAX;
else if (key == "bunch_id")
data_message.bunch_id = GetCBORUInt(map_value);
else if (key == "adu_histogram")
GetCBORUInt64Array(map_value, data_message.adu_histogram);
else
cbor_value_advance(&map_value);
}

16
frame_serialize/JFJochFrameSerializer.cpp
View File

@@ -126,16 +126,16 @@ inline void CBOR_ENC_2D_TYPED_ARRAY(CborEncoder &encoder, const CBORImage& image
}
inline void CBOR_ENC(CborEncoder &encoder, const char* key, const std::vector<float>& v) {
CborEncoder arrayEncoder;
cborErr(cbor_encode_text_stringz(&encoder, key));
cborErr(cbor_encode_tag(&encoder, TagFloatLE));
cborErr(cbor_encoder_create_array(&encoder, &arrayEncoder, v.size()));
cborErr(cbor_encode_byte_string(&encoder, (uint8_t *) v.data(), v.size() * sizeof(float)));
}
for (const auto &i : v)
cborErr(cbor_encode_float(&arrayEncoder, i));
cborErr(cbor_encoder_close_container(&encoder, &arrayEncoder));
inline void CBOR_ENC(CborEncoder &encoder, const char* key, const std::vector<uint64_t>& v) {
cborErr(cbor_encode_text_stringz(&encoder, key));
cborErr(cbor_encode_tag(&encoder, TagUnsignedInt64BitLE));
cborErr(cbor_encode_byte_string(&encoder, (uint8_t *) v.data(), v.size() * sizeof(uint64_t)));
}
inline void CBOR_ENC_RATIONAL(CborEncoder &encoder, const char* key, uint64_t numerator, uint64_t denominator) {
@@ -422,7 +422,7 @@ void JFJochFrameSerializer::SerializeImage(const DataMessage& message) {
message.timestamp_base);
cborErr(cbor_encode_text_stringz(&mapEncoder, "user_data"));
cborErr(cbor_encoder_create_map(&mapEncoder, &userDataMapEncoder, 10));
cborErr(cbor_encoder_create_map(&mapEncoder, &userDataMapEncoder, 11));
CBOR_ENC(userDataMapEncoder, "magic_number", user_data_magic_number);
CBOR_ENC(userDataMapEncoder, "spots", message.spots);
@@ -434,6 +434,8 @@ void JFJochFrameSerializer::SerializeImage(const DataMessage& message) {
CBOR_ENC(userDataMapEncoder, "receiver_available_send_buffers", message.receiver_available_send_buffers);
CBOR_ENC(userDataMapEncoder, "receiver_aq_dev_delay", message.receiver_aq_dev_delay);
CBOR_ENC(userDataMapEncoder, "storage_cell", (uint64_t) message.storage_cell);
CBOR_ENC(userDataMapEncoder, "adu_histogram", message.adu_histogram);
cborErr(cbor_encoder_close_container(&mapEncoder, &userDataMapEncoder));
CBOR_ENC(mapEncoder, "data", message.image);

1
grpc/jfjoch.proto
View File

@@ -285,6 +285,7 @@ enum PlotType {
SPOT_COUNT = 2;
INDEXING_RATE = 3;
INDEXING_RATE_PER_FILE = 4;
ADU_HISTOGRAM = 5;
}
message PlotRequest {

20
python/jfjoch_pb2.py
View File

File diff suppressed because one or more lines are too long

16
receiver/JFJochReceiver.cpp
View File

@@ -8,6 +8,7 @@
#include "../jungfrau/JFPedestalCalc.h"
#include "../image_analysis/IndexerWrapper.h"
#include "../common/DiffractionGeometry.h"
#include "../common/ADUHistogram.h"
inline std::string time_UTC(const std::chrono::time_point<std::chrono::system_clock> &input) {
auto time_ms = std::chrono::duration_cast<std::chrono::milliseconds>(input.time_since_epoch()).count();
@@ -344,7 +345,7 @@ void JFJochReceiver::MeasurePedestalThread(uint16_t data_stream, uint16_t module
void JFJochReceiver::MiniSummationThread(int d, int m, size_t image_number, bool &send_image,
FrameTransformation &transformation, DataMessage &message,
std::vector<JFConversionFixedPoint> &conversion,
RadialIntegrationProfile *profile) {
ADUHistogram& adu_histogram, RadialIntegrationProfile *profile) {
for (int j = 0; j < experiment.GetSummation(); j++) {
size_t frame_number = image_number * experiment.GetSummation() + j;
acquisition_device[d]->Counters().WaitForFrame(frame_number + 2);
@@ -370,6 +371,9 @@ void JFJochReceiver::MiniSummationThread(int d, int m, size_t image_number, bool
if (profile)
profile->Add(*acquisition_device[d]->GetDeviceOutput(frame_number, m));
adu_histogram.Add(*acquisition_device[d]->GetDeviceOutput(frame_number, m));
adu_histogram_total.Add(*acquisition_device[d]->GetDeviceOutput(frame_number, m));
if (experiment.GetConversionOnCPU()) {
auto &conv = conversion.at(experiment.GetFirstModuleOfDataStream(d) + m);
transformation.ProcessModule(conv, src, m, d);
@@ -434,6 +438,7 @@ void JFJochReceiver::FrameTransformationThread() {
message.timestamp_base = 10*1000*1000;
message.exptime_base = 10*1000*1000;
message.indexing_result = 0;
ADUHistogram adu_histogram;
bool send_preview = false;
bool send_bkg_estimate = false;
@@ -463,7 +468,7 @@ void JFJochReceiver::FrameTransformationThread() {
this, d, m, image_number,
std::ref(send_image), std::ref(transformation),
std::ref(message), std::ref(*conversion),
rad_int_profile_image.get()));
std::ref(adu_histogram), rad_int_profile_image.get()));
message.receiver_aq_dev_delay = max_delay;
} else {
for (int j = 0; j < experiment.GetSummation(); j++) {
@@ -497,6 +502,8 @@ void JFJochReceiver::FrameTransformationThread() {
} else
transformation.ProcessModule(src, m, d);
adu_histogram.Add(*acquisition_device[d]->GetDeviceOutput(frame_number, m));
adu_histogram_total.Add(*acquisition_device[d]->GetDeviceOutput(frame_number, m));
if (rad_int_profile_image)
rad_int_profile_image->Add(*acquisition_device[d]->GetDeviceOutput(frame_number, m));
@@ -572,6 +579,8 @@ void JFJochReceiver::FrameTransformationThread() {
*rad_int_profile_per_file[image_number % experiment.GetDataFileCount()] += *rad_int_profile_image;
}
message.adu_histogram = adu_histogram.GetHistogram();
if (send_preview)
preview_publisher->Publish(experiment,
transformation.GetPreview16BitImage(),
@@ -784,6 +793,9 @@ JFJochProtoBuf::Plot JFJochReceiver::GetPlots(const JFJochProtoBuf::PlotRequest
case JFJochProtoBuf::INDEXING_RATE_PER_FILE:
indexing_solution_per_file.GetPlot(ret);
break;
case JFJochProtoBuf::ADU_HISTOGRAM:
adu_histogram_total.GetPlot(ret);
break;
default:
// Do nothing
break;

5
receiver/JFJochReceiver.h
View File

@@ -29,6 +29,7 @@
#include "../jungfrau/JFCalibration.h"
#include "../jungfrau/JFConversionFixedPoint.h"
#include "../common/ADUHistogram.h"
class JFJochReceiver {
DiffractionExperiment experiment;
@@ -42,6 +43,8 @@ class JFJochReceiver {
std::vector<std::future<void>> frame_transformation_futures;
std::vector<std::future<void>> data_acquisition_futures;
ADUHistogram adu_histogram_total;
std::unique_ptr<RadialIntegrationMapping> rad_int_mapping;
std::unique_ptr<RadialIntegrationProfile> rad_int_profile;
std::vector<std::unique_ptr<RadialIntegrationProfile>> rad_int_profile_per_file;
@@ -111,7 +114,7 @@ class JFJochReceiver {
void MiniSummationThread(int d, int m, size_t image_number, bool &send_image,
FrameTransformation &transformation, DataMessage &message,
std::vector<JFConversionFixedPoint> &conversion,
RadialIntegrationProfile *profile);
ADUHistogram& adu_histogram, RadialIntegrationProfile *profile);
void Cancel(const JFJochException &e);
void FinalizeMeasurement();
JFJochProtoBuf::DataProcessingSettings GetDataProcessingSettings();

4
writer/HDF5DataFile.cpp
View File

@@ -61,6 +61,7 @@ void HDF5DataFile::CreateFile() {
result_group->NXClass("NXcollection");
rad_int_group = std::make_unique<HDF5Group>(*data_file, "/entry/result/rad_int");
adu_histo_group = std::make_unique<HDF5Group>(*data_file, "/entry/result/adu_histogram");
HDF5DataSpace data_space({1, ypixel, xpixel},{H5S_UNLIMITED, ypixel, xpixel});
data_set = std::make_unique<HDF5DataSet>(*data_file, "/entry/data/data", data_type, data_space, dcpl);
@@ -154,6 +155,9 @@ void HDF5DataFile::Write(const DataMessage &msg, uint64_t image_number) {
if (!msg.rad_int_profile.empty() && (msg.rad_int_profile.size() == rad_int_bin_to_q.size()))
rad_int_group->SaveVector("img" + std::to_string(image_number), msg.rad_int_profile);
if (!msg.adu_histogram.empty())
adu_histo_group->SaveVector("img" + std::to_string(image_number), msg.adu_histogram);
}
HDF5DataFileStatistics HDF5DataFile::GetStatistics() const {

1
writer/HDF5DataFile.h
View File

@@ -29,6 +29,7 @@ class HDF5DataFile {
std::unique_ptr<HDF5Group> result_group = nullptr;
std::unique_ptr<HDF5Group> rad_int_group = nullptr;
std::unique_ptr<HDF5Group> adu_histo_group = nullptr;
size_t xpixel;
size_t ypixel;