mx/Jungfraujoch
0
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 37 Wiki Activity

File writer and spot finding improvements

Browse Source
This commit is contained in:
leonarski_f
2024-04-08 11:18:50 +02:00
parent 15d99c6162
commit c6d2b5eedf
72 changed files with 690 additions and 893 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
acquisition_device/HLSSimulatedDevice.cpp
View File

@@ -309,8 +309,10 @@ void HLSSimulatedDevice::HLSMainThread() {
hls::stream<ap_axiu<32,1,1,1>> spot_finder_result_0;
hls::stream<ap_axiu<32,1,1,1>> spot_finder_result_1;
hls::stream<ap_uint<256>> spot_finder_conn_0;
hls::stream<ap_uint<512>> spot_finder_result_2;
hls::stream<ap_uint<32>> spot_finder_conn_0;
hls::stream<ap_axiu<32,1,1,1>> spot_finder_result_2;
hls::stream<ap_uint<512>> spot_finder_result_3;
hls::stream<ap_uint<32>> spot_finder_mask_0;
hls::stream<ap_uint<256>> roi_calc_result_0;
@@ -545,6 +547,11 @@ void HLSSimulatedDevice::HLSMainThread() {
logger_hls.Info("spot_finder_merge done");
});
hls_cores.emplace_back([&] {
axis_32_to_512(spot_finder_result_2, spot_finder_result_3);
logger_hls.Info("axis_32_to_512 done");
});
hls_cores.emplace_back([&] {
roi_calc(stream_768_4,
stream_768_5,
@@ -563,7 +570,7 @@ void HLSSimulatedDevice::HLSMainThread() {
// 11. Prepare data to write to host memory
hls_cores.emplace_back([&] {
ap_uint<3> state;
host_writer(data_12, adu_histo_result, integration_result_1, spot_finder_result_2, roi_calc_result_0,
host_writer(data_12, adu_histo_result, integration_result_1, spot_finder_result_3, roi_calc_result_0,
axi_compl[11], datamover_out.GetDataStream(),
datamover_out.GetCtrlStream(), work_request_stream, completion_stream,
dma_address_table.data(), packets_processed, host_writer_idle, cancel_data_collection, state);

2
broker/JFJochBrokerHttp.cpp
View File

@@ -363,7 +363,6 @@ inline DatasetSettings Convert(const org::openapitools::server::model::Dataset_s
ret.PhotonEnergyMultiplayer(input.getPhotonEnergyMultiplier());
ret.FilePrefix(input.getFilePrefix());
ret.DataFileCount(input.getDataFileCount());
if (!input.compressionIsSet())
ret.Compression(CompressionAlgorithm::BSHUF_LZ4);
@@ -410,6 +409,7 @@ inline DatasetSettings Convert(const org::openapitools::server::model::Dataset_s
ret.HeaderAppendix(input.getHeaderAppendix());
ret.ImageAppendix(input.getImageAppendix());
ret.SaveCalibration(input.isSaveCalibration());
ret.ImagesPerFile(input.getImagesPerFile());
return ret;
}

17
broker/JFJochBrokerParser.cpp
View File

@@ -34,6 +34,19 @@ inline int64_t GET_I64(const nlohmann::json &j, const std::string& tag, int64_t
return def;
}
inline int32_t GET_I32(const nlohmann::json &j, const std::string& tag, int32_t def) {
if (j.contains(tag)) {
if (!j[tag].is_number_integer())
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, tag + " must be integer");
try {
return j[tag].get<int32_t>();
} catch (std::exception &e) {
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, tag + ": " + e.what());
}
} else
return def;
}
inline float GET_FLOAT(const nlohmann::json &j, const std::string& tag, float def) {
if (j.contains(tag)) {
if (!j[tag].is_number())
@@ -353,3 +366,7 @@ std::string ParseString(const nlohmann::json &input, const std::string& tag) {
int64_t ParseInt64(const nlohmann::json &input, const std::string& tag, int64_t def) {
return GET_I64(input, tag, def);
}
int32_t ParseInt32(const nlohmann::json &input, const std::string& tag, int32_t def) {
return GET_I32(input, tag, def);
}

1
broker/JFJochBrokerParser.h
View File

@@ -20,5 +20,6 @@ void ParseAcquisitionDeviceGroup(const nlohmann::json &input, const std::string&
std::vector<std::string> ParseStringArray(const nlohmann::json &input, const std::string& tag);
std::string ParseString(const nlohmann::json &input, const std::string& tag);
int64_t ParseInt64(const nlohmann::json &input, const std::string& tag, int64_t def);
int32_t ParseInt32(const nlohmann::json &input, const std::string& tag, int32_t def);
#endif //JUNGFRAUJOCH_JFJOCHBROKERPARSER_H

44
broker/gen/model/Dataset_settings.cpp
View File

@@ -33,8 +33,8 @@ Dataset_settings::Dataset_settings()
m_Photon_energy_keV = 0.0f;
m_File_prefix = "";
m_File_prefixIsSet = false;
m_Data_file_count = 1L;
m_Data_file_countIsSet = false;
m_Images_per_file = 1000L;
m_Images_per_fileIsSet = false;
m_Space_group_number = 0L;
m_Space_group_numberIsSet = false;
m_Sample_name = "";
@@ -154,16 +154,16 @@ bool Dataset_settings::validate(std::stringstream& msg, const std::string& pathP
}
if (dataFileCountIsSet())
if (imagesPerFileIsSet())
{
const int64_t& value = m_Data_file_count;
const std::string currentValuePath = _pathPrefix + ".dataFileCount";
const int64_t& value = m_Images_per_file;
const std::string currentValuePath = _pathPrefix + ".imagesPerFile";
if (value < 1ll)
if (value < 0ll)
{
success = false;
msg << currentValuePath << ": must be greater than or equal to 1;";
msg << currentValuePath << ": must be greater than or equal to 0;";
}
}
@@ -258,7 +258,7 @@ bool Dataset_settings::operator==(const Dataset_settings& rhs) const
((!filePrefixIsSet() && !rhs.filePrefixIsSet()) || (filePrefixIsSet() && rhs.filePrefixIsSet() && getFilePrefix() == rhs.getFilePrefix())) &&
((!dataFileCountIsSet() && !rhs.dataFileCountIsSet()) || (dataFileCountIsSet() && rhs.dataFileCountIsSet() && getDataFileCount() == rhs.getDataFileCount())) &&
((!imagesPerFileIsSet() && !rhs.imagesPerFileIsSet()) || (imagesPerFileIsSet() && rhs.imagesPerFileIsSet() && getImagesPerFile() == rhs.getImagesPerFile())) &&
((!spaceGroupNumberIsSet() && !rhs.spaceGroupNumberIsSet()) || (spaceGroupNumberIsSet() && rhs.spaceGroupNumberIsSet() && getSpaceGroupNumber() == rhs.getSpaceGroupNumber())) &&
@@ -319,8 +319,8 @@ void to_json(nlohmann::json& j, const Dataset_settings& o)
j["photon_energy_keV"] = o.m_Photon_energy_keV;
if(o.filePrefixIsSet())
j["file_prefix"] = o.m_File_prefix;
if(o.dataFileCountIsSet())
j["data_file_count"] = o.m_Data_file_count;
if(o.imagesPerFileIsSet())
j["images_per_file"] = o.m_Images_per_file;
if(o.spaceGroupNumberIsSet())
j["space_group_number"] = o.m_Space_group_number;
j["sample_name"] = o.m_Sample_name;
@@ -373,10 +373,10 @@ void from_json(const nlohmann::json& j, Dataset_settings& o)
j.at("file_prefix").get_to(o.m_File_prefix);
o.m_File_prefixIsSet = true;
}
if(j.find("data_file_count") != j.end())
if(j.find("images_per_file") != j.end())
{
j.at("data_file_count").get_to(o.m_Data_file_count);
o.m_Data_file_countIsSet = true;
j.at("images_per_file").get_to(o.m_Images_per_file);
o.m_Images_per_fileIsSet = true;
}
if(j.find("space_group_number") != j.end())
{
@@ -537,22 +537,22 @@ void Dataset_settings::unsetFile_prefix()
{
m_File_prefixIsSet = false;
}
int64_t Dataset_settings::getDataFileCount() const
int64_t Dataset_settings::getImagesPerFile() const
{
return m_Data_file_count;
return m_Images_per_file;
}
void Dataset_settings::setDataFileCount(int64_t const value)
void Dataset_settings::setImagesPerFile(int64_t const value)
{
m_Data_file_count = value;
m_Data_file_countIsSet = true;
m_Images_per_file = value;
m_Images_per_fileIsSet = true;
}
bool Dataset_settings::dataFileCountIsSet() const
bool Dataset_settings::imagesPerFileIsSet() const
{
return m_Data_file_countIsSet;
return m_Images_per_fileIsSet;
}
void Dataset_settings::unsetData_file_count()
void Dataset_settings::unsetImages_per_file()
{
m_Data_file_countIsSet = false;
m_Images_per_fileIsSet = false;
}
int64_t Dataset_settings::getSpaceGroupNumber() const
{

14
broker/gen/model/Dataset_settings.h
View File

@@ -109,12 +109,12 @@ public:
bool filePrefixIsSet() const;
void unsetFile_prefix();
/// <summary>
/// Number of round-robin data files
/// Number of files in a single HDF5 data file (0 &#x3D; write all images to a single data file).
/// </summary>
int64_t getDataFileCount() const;
void setDataFileCount(int64_t const value);
bool dataFileCountIsSet() const;
void unsetData_file_count();
int64_t getImagesPerFile() const;
void setImagesPerFile(int64_t const value);
bool imagesPerFileIsSet() const;
void unsetImages_per_file();
/// <summary>
///
/// </summary>
@@ -217,8 +217,8 @@ protected:
std::string m_File_prefix;
bool m_File_prefixIsSet;
int64_t m_Data_file_count;
bool m_Data_file_countIsSet;
int64_t m_Images_per_file;
bool m_Images_per_fileIsSet;
int64_t m_Space_group_number;
bool m_Space_group_numberIsSet;
std::string m_Sample_name;

8
broker/jfjoch_api.yaml
View File

@@ -94,12 +94,12 @@ components:
type: string
default: ""
description: Prefix for filenames. If left empty, no file will be saved.
data_file_count:
images_per_file:
type: integer
format: int64
minimum: 1
default: 1
description: Number of round-robin data files
minimum: 0
default: 1000
description: Number of files in a single HDF5 data file (0 = write all images to a single data file).
space_group_number:
type: integer
format: int64

9
broker/jfjoch_broker.cpp
View File

@@ -82,7 +82,14 @@ int main (int argc, char **argv) {
ParseAcquisitionDeviceGroup(input, "receiver", aq_devices);
if (aq_devices.size() > 0) {
experiment.DataStreams(aq_devices.size());
image_pusher = std::make_unique<ZMQStream2PusherGroup>(context, ParseStringArray(input, "zmq_image_addr"));
int32_t zmq_send_watermark = ParseInt32(input, "zmq_send_watermark", 100);
int32_t zmq_send_buffer_size = ParseInt32(input, "zmq_send_buffer_size", -1);
image_pusher = std::make_unique<ZMQStream2PusherGroup>(ParseStringArray(input, "zmq_image_addr"),
zmq_send_watermark,
zmq_send_buffer_size);
receiver = std::make_unique<JFJochReceiverService>(aq_devices, logger, *image_pusher);
std::string zmq_preview_addr = ParseString(input, "zmq_preview_addr");

8
broker/redoc-static.html
View File

File diff suppressed because one or more lines are too long

23
common/DatasetSettings.cpp
View File

@@ -16,7 +16,6 @@ DatasetSettings::DatasetSettings() {
detector_distance_mm = 100;
beam_x_pxl = 0.0;
beam_y_pxl = 0.0;
data_file_count = 1;
file_prefix = "test";
ntrigger = 1;
images_per_trigger = 1;
@@ -27,6 +26,7 @@ DatasetSettings::DatasetSettings() {
save_calibration = false;
photon_energy_multiplier = 1.0f;
omega_start = 0.0f;
images_per_file = 1000;
}
DatasetSettings &DatasetSettings::ImagesPerTrigger(int64_t input) {
@@ -89,13 +89,6 @@ DatasetSettings &DatasetSettings::FilePrefix(std::string input) {
return *this;
}
DatasetSettings &DatasetSettings::DataFileCount(int64_t input) {
check_min("File count", input, 1);
check_max("File count", input, 1000);
data_file_count = input;
return *this;
}
DatasetSettings &DatasetSettings::Compression(CompressionAlgorithm input) {
switch (input) {
case CompressionAlgorithm::NO_COMPRESSION:
@@ -316,10 +309,6 @@ std::string DatasetSettings::GetFilePrefix() const {
return file_prefix;
}
int64_t DatasetSettings::GetDataFileCount() const {
return data_file_count;
}
CompressionAlgorithm DatasetSettings::GetCompressionAlgorithm() const {
return compression;
}
@@ -331,3 +320,13 @@ int64_t DatasetSettings::GetNumTriggers() const {
int64_t DatasetSettings::GetImageNumPerTrigger() const {
return images_per_trigger;
}
DatasetSettings &DatasetSettings::ImagesPerFile(int64_t input) {
check_min("Images per file", input, 0);
images_per_file = input;
return *this;
}
int64_t DatasetSettings::GetImagesPerFile() const {
return images_per_file;
}

6
common/DatasetSettings.h
View File

@@ -30,7 +30,7 @@ class DatasetSettings {
float photon_energy_multiplier;
std::string file_prefix;
int64_t data_file_count;
int64_t images_per_file;
CompressionAlgorithm compression;
@@ -58,7 +58,6 @@ public:
DatasetSettings& BeamY_pxl(float input);
DatasetSettings& DetectorDistance_mm(float input);
DatasetSettings& FilePrefix(std::string input);
DatasetSettings& DataFileCount(int64_t input);
DatasetSettings& Compression(CompressionAlgorithm input);
DatasetSettings& SetUnitCell(const std::optional<UnitCell> &cell);
DatasetSettings& SpaceGroupNumber(int64_t input);
@@ -74,6 +73,7 @@ public:
DatasetSettings& SaveCalibration(bool input);
DatasetSettings& Summation(int64_t input);
DatasetSettings& FPGAOutputMode(FPGAPixelOutput input);
DatasetSettings& ImagesPerFile(int64_t input);
std::optional<float> GetAttenuatorTransmission() const;
std::optional<float> GetTotalFlux() const;
@@ -98,11 +98,11 @@ public:
Coord GetScatteringVector() const;
std::string GetFilePrefix() const;
int64_t GetDataFileCount() const;
CompressionAlgorithm GetCompressionAlgorithm() const;
int64_t GetNumTriggers() const;
int64_t GetImageNumPerTrigger() const;
int64_t GetImagesPerFile() const;
};
#endif //JUNGFRAUJOCH_DATASETSETTINGS_H

43
common/DiffractionExperiment.cpp
View File

@@ -46,8 +46,6 @@ DiffractionExperiment::DiffractionExperiment(const DetectorSetup& det_setup)
q_spacing = 0.05;
ipv4_base_addr = 0x0132010a;
git_sha1 = jfjoch_git_sha1();
git_date = jfjoch_git_date();
storage_cells = 1;
storage_cell_start = 15;
@@ -168,10 +166,6 @@ DiffractionExperiment &DiffractionExperiment::FilePrefix(std::string input) {
return *this;
}
DiffractionExperiment &DiffractionExperiment::DataFileCount(int64_t input) {
dataset.DataFileCount(input);
return *this;
}
DiffractionExperiment &DiffractionExperiment::UseInternalPacketGenerator(bool input) {
internal_fpga_packet_generator = input;
return *this;
@@ -433,11 +427,11 @@ std::string DiffractionExperiment::GetFilePrefix() const {
return dataset.GetFilePrefix();
}
int64_t DiffractionExperiment::GetDataFileCount() const {
if (GetStorageCellNumber() > 1)
return GetStorageCellNumber();
int64_t DiffractionExperiment::GetTimePointNumber() const {
if ((GetNumTriggers() < 5) || (GetImageNumPerTrigger() > 250)) // these are heuristic parameters
return 1;
else
return dataset.GetDataFileCount();
return GetImageNumPerTrigger();
}
CompressionAlgorithm DiffractionExperiment::GetCompressionAlgorithm() const {
@@ -586,19 +580,6 @@ uint32_t DiffractionExperiment::GetSrcIPv4Address(uint32_t data_stream, uint32_t
return ipv4_base_addr + (host << 24);
}
bool DiffractionExperiment::CheckGitSha1Consistent() const {
return (git_sha1 == jfjoch_git_sha1());
}
std::string DiffractionExperiment::CheckGitSha1Msg() const {
if (git_sha1 == jfjoch_git_sha1())
return "";
else {
return "Local component git repo is rev. " + jfjoch_git_sha1().substr(0,6) + " (" + jfjoch_git_date() +") remote component repo is rev. "
+ git_sha1.substr(0,6) + " (" + git_date + ")";
}
}
bool DiffractionExperiment::GetMaskModuleEdges() const {
return mask_module_edges;
}
@@ -695,7 +676,7 @@ SpotFindingSettings DiffractionExperiment::DefaultDataProcessingSettings() {
}
void DiffractionExperiment::FillMessage(StartMessage &message) const {
message.data_file_count = GetDataFileCount();
message.images_per_file = GetImagesPerFile();
message.beam_center_x = GetBeamX_pxl();
message.beam_center_y = GetBeamY_pxl();
message.detector_distance = GetDetectorDistance_mm() * 1e-3f;
@@ -1176,3 +1157,17 @@ void DiffractionExperiment::ExportROIMask(uint16_t *dest, size_t module_number)
dest[i] = UINT16_MAX;
}
}
DiffractionExperiment &DiffractionExperiment::ImagesPerFile(int64_t input) {
dataset.ImagesPerFile(input);
return *this;
}
int64_t DiffractionExperiment::GetImagesPerFile() const {
auto tmp = dataset.GetImagesPerFile();
if (tmp == 0)
return GetImageNum();
else
return tmp;
}

20
common/DiffractionExperiment.h
View File

@@ -76,9 +76,6 @@ class DiffractionExperiment {
float high_q;
float q_spacing;
std::string git_sha1;
std::string git_date;
std::string source_name;
std::string source_name_short;
std::string instrument_name;
@@ -158,7 +155,6 @@ public:
DiffractionExperiment& BeamY_pxl(float input);
DiffractionExperiment& DetectorDistance_mm(float input);
DiffractionExperiment& FilePrefix(std::string input);
DiffractionExperiment& DataFileCount(int64_t input);
DiffractionExperiment& Compression(CompressionAlgorithm input);
DiffractionExperiment& SetUnitCell(const std::optional<UnitCell> &cell);
DiffractionExperiment& SpaceGroupNumber(int64_t input);
@@ -176,6 +172,7 @@ public:
DiffractionExperiment& Summation(int64_t input);
DiffractionExperiment& FPGAOutputMode(FPGAPixelOutput input);
DiffractionExperiment& MaxSpotCount(int64_t input);
DiffractionExperiment& ImagesPerFile(int64_t input);
DiffractionExperiment& ImportDatasetSettings(const DatasetSettings& input);
DatasetSettings GetDatasetSettings() const;
@@ -239,9 +236,6 @@ public:
uint32_t GetSrcIPv4Address(uint32_t data_stream, uint32_t half_module) const;
bool CheckGitSha1Consistent() const;
std::string CheckGitSha1Msg() const;
bool GetMaskModuleEdges() const;
bool GetMaskChipEdges() const;
@@ -319,7 +313,7 @@ public:
Coord GetScatteringVector() const;
std::string GetFilePrefix() const;
int64_t GetDataFileCount() const;
int64_t GetTimePointNumber() const;
CompressionAlgorithm GetCompressionAlgorithm() const;
int64_t GetNumTriggers() const;
@@ -328,15 +322,7 @@ public:
ROIMask& ROI();
const ROIMask& ROI() const;
void ExportROIMask(uint16_t *v, size_t module_number) const;
int64_t GetImagesPerFile() const;
};
inline int64_t CalculateStride(const std::chrono::microseconds &frame_time, const std::chrono::microseconds &preview_time) {
if ((preview_time.count() <= 0) || (frame_time.count() <= 0))
return 0;
else if (preview_time < frame_time)
return 1;
else
return preview_time / frame_time;
}
#endif //DIFFRACTIONEXPERIMENT_H

10
common/DiffractionSpot.h
View File

@@ -9,11 +9,11 @@
// Definition of Bragg spot
class DiffractionSpot {
float x;
float y;
int64_t pixel_count;
int64_t photons; // total photon count
int64_t max_photons; // maximum number of counts per pixel in the spot
float x = 0;
float y = 0;
int64_t pixel_count = 0;
int64_t photons = 0; // total photon count
int64_t max_photons = INT64_MIN; // maximum number of counts per pixel in the spot
public:
DiffractionSpot() = default;
DiffractionSpot(uint32_t col, uint32_t line, int64_t photons);

1
fpga/hls/CMakeLists.txt
View File

@@ -90,6 +90,7 @@ MAKE_HLS_MODULE(integration integration.cpp integration_tb.cpp)
MAKE_HLS_MODULE(spot_finder spot_finder.cpp spot_finder_tb.cpp)
MAKE_HLS_MODULE(axis_broadcast axis_broadcast.cpp "")
MAKE_HLS_MODULE(axis_64_to_512 axis_helpers.cpp "")
MAKE_HLS_MODULE(axis_32_to_512 axis_helpers.cpp "")
MAKE_HLS_MODULE(adu_histo adu_histo.cpp adu_histo_tb.cpp)
MAKE_HLS_MODULE(pedestal pedestal.cpp "")
MAKE_HLS_MODULE(frame_summation frame_summation.cpp frame_summation_tb_2.cpp)

21
fpga/hls/axis_helpers.cpp
View File

@@ -24,3 +24,24 @@ void axis_64_to_512(hls::stream<ap_axiu<64,1,1,1>> &data_in,
}
}
void axis_32_to_512(hls::stream<ap_axiu<32,1,1,1>> &data_in,
hls::stream<ap_uint<512>> &data_out) {
#pragma HLS INTERFACE ap_ctrl_none port=return
#pragma HLS INTERFACE axis register both port=data_in
#pragma HLS INTERFACE axis register both port=data_out
ap_axiu<32,1,1,1> packet_32;
data_in >> packet_32;
while (!packet_32.user) {
#pragma HLS PIPELINE II=16
ap_uint<512> val = 0;
val(31,0) = packet_32.data;
for (int i = 1; i < 16; i++) {
data_in >> packet_32;
val(i * 32 + 31, i * 32) = packet_32.data;
}
data_out << val;
data_in >> packet_32;
}
}

9
fpga/hls/hls_jfjoch.h
View File

@@ -111,6 +111,9 @@ void stream_merge(AXI_STREAM &input_network,
void axis_64_to_512(hls::stream<ap_axiu<64,1,1,1>> &data_in,
hls::stream<ap_uint<512>> &data_out);
void axis_32_to_512(hls::stream<ap_axiu<32,1,1,1>> &data_in,
hls::stream<ap_uint<512>> &data_out);
template<int N> ap_uint<N*32> pack32(ap_int<N> in[32]) {
#pragma HLS INLINE
ap_uint<N*32> out;
@@ -409,11 +412,11 @@ void stream_24bit_conv(STREAM_768 &data_in, STREAM_512 &data_out,
void spot_finder_connectivity(hls::stream<ap_axiu<32,1,1,1>> &data_in,
hls::stream<ap_axiu<32,1,1,1>> &data_out,
hls::stream<ap_uint<256>> &connectivity_out);
hls::stream<ap_uint<32>> &connectivity_out);
void spot_finder_merge(hls::stream<ap_axiu<32,1,1,1>> &data_in,
hls::stream<ap_uint<256>> &connectivity_in,
hls::stream<ap_uint<512>> &data_out,
hls::stream<ap_uint<32>> &connectivity_in,
hls::stream<ap_axiu<32,1,1,1>> &data_out,
volatile ap_uint<32> &in_min_pix_per_spot);
void spot_finder_mask(STREAM_768 &data_in,

6
fpga/hls/host_writer.cpp
View File

@@ -142,7 +142,7 @@ void host_writer(STREAM_512 &data_in,
if (send_images) {
setup_datamover(datamover_out_cmd, req_host_offset, RAW_MODULE_SIZE * pixel_depth);
setup_datamover(datamover_out_cmd, req_host_offset + offsetof(DeviceOutput, spot_finding_result),
(SPOT_FINDER_MAX_STRONG_PIXEL / 16 + 1) * 64);
(RAW_MODULE_SIZE / 8 + 64));
setup_datamover(datamover_out_cmd, req_host_offset + offsetof(DeviceOutput, integration_result),
(FPGA_INTEGRATION_BIN_COUNT / 8) * 64);
setup_datamover(datamover_out_cmd, req_host_offset + offsetof(DeviceOutput, adu_histogram),
@@ -181,10 +181,10 @@ void host_writer(STREAM_512 &data_in,
packet_out.last = 0;
// 256 transfers x 512-bit
for (int i = 0; i < SPOT_FINDER_MAX_STRONG_PIXEL / 16 + 1; i++) {
for (int i = 0; i < RAW_MODULE_SIZE / (8 * 64) + 1; i++) {
#pragma HLS PIPELINE II=1
spot_finder_in >> packet_out.data;
packet_out.last = (i == SPOT_FINDER_MAX_STRONG_PIXEL / 16);
packet_out.last = (i == RAW_MODULE_SIZE / (8 * 64));
if (send_images)
host_memory_out << packet_out;
}

2
fpga/hls/spot_finder.cpp
View File

@@ -158,7 +158,7 @@ ap_uint<32> spot_finder_count_threshold(ap_int<24> val[32], ap_int<32> &count_th
return UINT32_MAX;
ap_uint<32> ret = 0;
for (int j = 0; j < 32; j++) {
if (val[j] >= count_threshold)
if (val[j] > count_threshold)
ret[j] = 1;
else
ret[j] = 0;

40
fpga/hls/spot_finder_connectivity.cpp
View File

@@ -4,7 +4,7 @@
void spot_finder_connectivity(hls::stream<ap_axiu<32,1,1,1>> &data_in,
hls::stream<ap_axiu<32,1,1,1>> &data_out,
hls::stream<ap_uint<256>> &connectivity_out) {
hls::stream<ap_uint<32>> &connectivity_out) {
#pragma HLS INTERFACE ap_ctrl_none port=return
#pragma HLS INTERFACE axis register both port=data_in
#pragma HLS INTERFACE axis register both port=data_out
@@ -31,23 +31,23 @@ void spot_finder_connectivity(hls::stream<ap_axiu<32,1,1,1>> &data_in,
}
if (line != 0) {
for (int col = 0; col < RAW_MODULE_COLS / 32; col++) {
ap_uint<256> output = 0;
ap_uint<32> output = 0;
for (int i = 0; i < 32; i++) {
size_t pos = col * 32 + i;
ap_uint<8> connect_mat = 0;
ap_uint<1> connect_mat = 0;
if (pos != 0) {
connect_mat[0] = top_line[pos - 1];
connect_mat[1] = mid_line[pos - 1];
connect_mat[2] = bottom_line[pos - 1];
connect_mat |= top_line[pos - 1];
connect_mat |= mid_line[pos - 1];
connect_mat |= bottom_line[pos - 1];
}
connect_mat[3] = top_line[pos];
connect_mat[4] = bottom_line[pos];
connect_mat |= top_line[pos];
connect_mat |= bottom_line[pos];
if (pos != RAW_MODULE_COLS - 1) {
connect_mat[5] = top_line[pos + 1];
connect_mat[6] = mid_line[pos + 1];
connect_mat[7] = bottom_line[pos + 1];
connect_mat |= top_line[pos + 1];
connect_mat |= mid_line[pos + 1];
connect_mat |= bottom_line[pos + 1];
}
output(i*8+7, i*8) = connect_mat;
output[i] = connect_mat;
}
connectivity_out << output;
}
@@ -62,20 +62,20 @@ void spot_finder_connectivity(hls::stream<ap_axiu<32,1,1,1>> &data_in,
for (int col = 0; col < RAW_MODULE_COLS / 32; col++) {
#pragma HLS PIPELINE II=1
ap_uint<256> output = 0;
ap_uint<32> output = 0;
for (int i = 0; i < 32; i++) {
size_t pos = col * 32 + i;
ap_uint<8> connect_mat = 0;
ap_uint<1> connect_mat = 0;
if (pos != 0) {
connect_mat[0] = top_line[pos - 1];
connect_mat[1] = mid_line[pos - 1];
connect_mat |= top_line[pos - 1];
connect_mat |= mid_line[pos - 1];
}
connect_mat[3] = top_line[pos];
connect_mat |= top_line[pos];
if (pos != RAW_MODULE_COLS - 1) {
connect_mat[5] = top_line[pos + 1];
connect_mat[6] = mid_line[pos + 1];
connect_mat |= top_line[pos + 1];
connect_mat |= mid_line[pos + 1];
}
output(i*8+7, i*8) = connect_mat;
output[i] = connect_mat;
}
connectivity_out << output;
}

76
fpga/hls/spot_finder_merge.cpp
View File

@@ -11,8 +11,8 @@ inline ap_uint<32> count_pixels(ap_uint<32> &in) {
}
void spot_finder_merge(hls::stream<ap_axiu<32,1,1,1>> &data_in,
hls::stream<ap_uint<256>> &connectivity_in,
hls::stream<ap_uint<512>> &data_out,
hls::stream<ap_uint<32>> &connectivity_in,
hls::stream<ap_axiu<32,1,1,1>> &data_out,
volatile ap_uint<32> &in_min_pix_per_spot) {
#pragma HLS INTERFACE ap_ctrl_none port=return
#pragma HLS INTERFACE axis register both port=data_in
@@ -20,73 +20,31 @@ void spot_finder_merge(hls::stream<ap_axiu<32,1,1,1>> &data_in,
#pragma HLS INTERFACE axis register both port=connectivity_in
#pragma HLS INTERFACE ap_none register port=in_min_pix_per_spot
ap_uint<32> memory[SPOT_FINDER_MAX_STRONG_PIXEL / 32][32];
#pragma HLS ARRAY_PARTITION variable=memory type=complete dim=2
ap_uint<32> memory_index[32];
#pragma HLS ARRAY_PARTITION variable=memory_index type=complete dim=1
for (int i = 0 ; i < SPOT_FINDER_MAX_STRONG_PIXEL / 32; i++) {
#pragma HLS PIPELINE II=1
for (int j = 0; j < 32; j++)
memory[i][j] = UINT32_MAX;
}
for (int j = 0; j < 32; j++) {
#pragma HLS UNROLL
memory_index[j] = 0;
}
ap_axiu<32,1,1,1> val = data_in.read();
ap_uint<32> conn;
while (!val.user) {
ap_uint<32> min_pix_per_spot = in_min_pix_per_spot;
ap_uint<32> strong_pixel_count = 0;
for (int i = 0; i < 16384; i++) {
#pragma HLS PIPELINE II=1
ap_uint<256> conn = connectivity_in.read();
for (int j = 0; j < 32; j++) {
if ((min_pix_per_spot > 1) && (conn(j * 8 + 7, j*8) == 0))
val.data[j] = 0;
if (val.data[j]) {
ap_uint<8> tmp = memory_index[j] % (SPOT_FINDER_MAX_STRONG_PIXEL / 32);
memory[tmp][j] = (i * 32 + j) | (conn(j * 8 + 7, j*8) << 24);
memory_index[j] = tmp + 1;
}
}
connectivity_in >> conn;
if (min_pix_per_spot > 1)
val.data = val.data & conn;
strong_pixel_count += count_pixels(val.data);
val = data_in.read();
}
ap_uint<32> max_memory_index = 0;
for (int i = 0; i < 32; i++) {
#pragma HLS UNROLL
if (max_memory_index < memory_index[i])
max_memory_index = memory_index[i];
memory_index[i] = 0;
data_out << val;
data_in >> val;
}
for (int i = 0; i < SPOT_FINDER_MAX_STRONG_PIXEL / 32; i++) {
#pragma HLS PIPELINE II=2
ap_uint<512> out_val = 0;
for (int j = 0; j < 16; j++) {
out_val(j * 32 + 31, j * 32) = memory[i][j];
memory[i][j] = UINT32_MAX;
}
data_out << out_val;
for (int j = 0; j < 16; j++) {
out_val(j * 32 + 31, j * 32) = memory[i][j + 16];
memory[i][j + 16] = UINT32_MAX;
}
data_out << out_val;
}
{
ap_uint<512> out_val = 0;
for (int i = 0; i < 16; i++) {
out_val(i*32+31, i * 32) = val.data;
val = data_in.read();
}
out_val(95, 64) = strong_pixel_count;
out_val(127, 96) = max_memory_index;
data_out << out_val;
for (int i = 0; i < 16; i++) {
if (i == 2)
val.data = strong_pixel_count;
data_out << val;
data_in >> val;
}
}
data_out << val;
}

7
fpga/include/jfjoch_fpga.h
View File

@@ -27,7 +27,7 @@ typedef __u64 uint64_t;
#define GAIN_G2_MULTIPLIER (-1)
#define JFJOCH_FPGA_MAGIC 0x52324158
#define JFJOCH_FPGA_RELEASE 0x0055
#define JFJOCH_FPGA_RELEASE 0x0056
#define JFJOCH_FPGA_VARIANT_100G 0
#define JFJOCH_FPGA_VARIANT_8x10G 1
@@ -218,12 +218,11 @@ struct IntegrationResult {
};
struct SpotFindingResult {
uint32_t strong_pixel_number[SPOT_FINDER_MAX_STRONG_PIXEL];
char strong_pixel[RAW_MODULE_SIZE / 8];
int32_t count_threshold;
float snr_threshold;
int32_t strong_pixel_count;
uint32_t max_memory_index;
int32_t reserved[12];
int32_t reserved[13];
};
struct ModuleStatistics {

23
fpga/scripts/image_processing.tcl
View File

@@ -123,6 +123,9 @@ proc create_hier_cell_image_processing { parentCell nameHier } {
# Create instance: adu_histo_0, and set properties
set adu_histo_0 [ create_bd_cell -type ip -vlnv psi.ch:hls:adu_histo:1.0 adu_histo_0 ]
# Create instance: axis_32_to_512_0, and set properties
set axis_32_to_512_0 [ create_bd_cell -type ip -vlnv psi.ch:hls:axis_32_to_512:1.0 axis_32_to_512_0 ]
# Create instance: axis_64_to_512_0, and set properties
set axis_64_to_512_0 [ create_bd_cell -type ip -vlnv psi.ch:hls:axis_64_to_512:1.0 axis_64_to_512_0 ]
@@ -309,10 +312,16 @@ proc create_hier_cell_image_processing { parentCell nameHier } {
# Create instance: axis_spot_finder_fifo_2, and set properties
set axis_spot_finder_fifo_2 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_data_fifo:2.0 axis_spot_finder_fifo_2 ]
set_property -dict [ list \
CONFIG.FIFO_DEPTH {2048} \
CONFIG.FIFO_MEMORY_TYPE {ultra} \
CONFIG.FIFO_DEPTH {128} \
] $axis_spot_finder_fifo_2
# Create instance: axis_spot_finder_fifo_3, and set properties
set axis_spot_finder_fifo_3 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_data_fifo:2.0 axis_spot_finder_fifo_3 ]
set_property -dict [ list \
CONFIG.FIFO_DEPTH {4096} \
CONFIG.FIFO_MEMORY_TYPE {ultra} \
] $axis_spot_finder_fifo_3
# Create instance: eiger_reorder_0, and set properties
set eiger_reorder_0 [ create_bd_cell -type ip -vlnv psi.ch:hls:eiger_reorder:1.0 eiger_reorder_0 ]
@@ -371,7 +380,7 @@ proc create_hier_cell_image_processing { parentCell nameHier } {
set stream_24bit_conv_0 [ create_bd_cell -type ip -vlnv psi.ch:hls:stream_24bit_conv:1.0 stream_24bit_conv_0 ]
# Create interface connections
connect_bd_intf_net -intf_net Conn1 [get_bd_intf_pins spot_finder_out] [get_bd_intf_pins axis_spot_finder_fifo_2/M_AXIS]
connect_bd_intf_net -intf_net Conn1 [get_bd_intf_pins spot_finder_out] [get_bd_intf_pins axis_spot_finder_fifo_3/M_AXIS]
connect_bd_intf_net -intf_net Conn2 [get_bd_intf_pins m_axi_d_hbm_p0] [get_bd_intf_pins jf_conversion_0/m_axi_d_hbm_p0]
connect_bd_intf_net -intf_net Conn3 [get_bd_intf_pins integration_result_out] [get_bd_intf_pins axis_integration_result_fifo_1/M_AXIS]
connect_bd_intf_net -intf_net Conn5 [get_bd_intf_pins m_axi_d_hbm_p16] [get_bd_intf_pins integration_0/m_axi_d_hbm_p0]
@@ -394,6 +403,7 @@ proc create_hier_cell_image_processing { parentCell nameHier } {
connect_bd_intf_net -intf_net adu_histo_0_data_out [get_bd_intf_pins adu_histo_0/data_out] [get_bd_intf_pins axis_data_fifo_2/S_AXIS]
connect_bd_intf_net -intf_net adu_histo_0_m_axis_completion [get_bd_intf_pins adu_histo_0/m_axis_completion] [get_bd_intf_pins axis_compl_fifo_1/S_AXIS]
connect_bd_intf_net -intf_net adu_histo_0_result_out [get_bd_intf_pins result_out] [get_bd_intf_pins adu_histo_0/result_out]
connect_bd_intf_net -intf_net axis_32_to_512_0_data_out [get_bd_intf_pins axis_32_to_512_0/data_out] [get_bd_intf_pins axis_spot_finder_fifo_3/S_AXIS]
connect_bd_intf_net -intf_net axis_64_to_512_0_data_out [get_bd_intf_pins axis_64_to_512_0/data_out] [get_bd_intf_pins axis_integration_result_fifo_1/S_AXIS]
connect_bd_intf_net -intf_net axis_compl_fifo_0_M_AXIS [get_bd_intf_pins adu_histo_0/s_axis_completion] [get_bd_intf_pins axis_compl_fifo_0/M_AXIS]
connect_bd_intf_net -intf_net axis_compl_fifo_1_M_AXIS [get_bd_intf_pins axis_compl_fifo_1/M_AXIS] [get_bd_intf_pins mask_missing_0/s_axis_completion]
@@ -423,6 +433,7 @@ proc create_hier_cell_image_processing { parentCell nameHier } {
connect_bd_intf_net -intf_net axis_spot_finder_conn_fifo_0_M_AXIS [get_bd_intf_pins axis_spot_finder_conn_fifo_0/M_AXIS] [get_bd_intf_pins spot_finder_merge_0/connectivity_in]
connect_bd_intf_net -intf_net axis_spot_finder_fifo_0_M_AXIS [get_bd_intf_pins axis_spot_finder_fifo_0/M_AXIS] [get_bd_intf_pins spot_finder_connecti_0/data_in]
connect_bd_intf_net -intf_net axis_spot_finder_fifo_1_M_AXIS [get_bd_intf_pins axis_spot_finder_fifo_1/M_AXIS] [get_bd_intf_pins spot_finder_merge_0/data_in]
connect_bd_intf_net -intf_net axis_spot_finder_fifo_2_M_AXIS [get_bd_intf_pins axis_32_to_512_0/data_in] [get_bd_intf_pins axis_spot_finder_fifo_2/M_AXIS]
connect_bd_intf_net -intf_net data_in_1 [get_bd_intf_pins data_in] [get_bd_intf_pins axis_data_fifo_0/S_AXIS]
connect_bd_intf_net -intf_net eiger_reorder_0_data_out [get_bd_intf_pins axis_data_fifo_4/S_AXIS] [get_bd_intf_pins eiger_reorder_0/data_out]
connect_bd_intf_net -intf_net eiger_reorder_0_m_axis_completion [get_bd_intf_pins axis_compl_fifo_3/S_AXIS] [get_bd_intf_pins eiger_reorder_0/m_axis_completion]
@@ -472,10 +483,10 @@ proc create_hier_cell_image_processing { parentCell nameHier } {
connect_bd_intf_net -intf_net stream_24bit_conv_0_data_out [get_bd_intf_pins axis_register_slice_data_3/S_AXIS] [get_bd_intf_pins stream_24bit_conv_0/data_out]
# Create port connections
connect_bd_net -net ap_rst_n_1 [get_bd_pins ap_rst_n] [get_bd_pins adu_histo_0/ap_rst_n] [get_bd_pins axis_64_to_512_0/ap_rst_n] [get_bd_pins eiger_reorder_0/ap_rst_n] [get_bd_pins frame_summation_0/ap_rst_n] [get_bd_pins integration_0/ap_rst_n] [get_bd_pins jf_conversion_0/ap_rst_n] [get_bd_pins mask_missing_0/ap_rst_n] [get_bd_pins pedestal_0/ap_rst_n] [get_bd_pins roi_calc_0/ap_rst_n] [get_bd_pins spot_finder_0/ap_rst_n] [get_bd_pins spot_finder_connecti_0/ap_rst_n] [get_bd_pins spot_finder_mask_0/ap_rst_n] [get_bd_pins spot_finder_merge_0/ap_rst_n] [get_bd_pins stream_24bit_conv_0/ap_rst_n]
connect_bd_net -net ap_rst_n_1 [get_bd_pins ap_rst_n] [get_bd_pins adu_histo_0/ap_rst_n] [get_bd_pins axis_32_to_512_0/ap_rst_n] [get_bd_pins axis_64_to_512_0/ap_rst_n] [get_bd_pins eiger_reorder_0/ap_rst_n] [get_bd_pins frame_summation_0/ap_rst_n] [get_bd_pins integration_0/ap_rst_n] [get_bd_pins jf_conversion_0/ap_rst_n] [get_bd_pins mask_missing_0/ap_rst_n] [get_bd_pins pedestal_0/ap_rst_n] [get_bd_pins roi_calc_0/ap_rst_n] [get_bd_pins spot_finder_0/ap_rst_n] [get_bd_pins spot_finder_connecti_0/ap_rst_n] [get_bd_pins spot_finder_mask_0/ap_rst_n] [get_bd_pins spot_finder_merge_0/ap_rst_n] [get_bd_pins stream_24bit_conv_0/ap_rst_n]
connect_bd_net -net ap_start_1 [get_bd_pins frame_summation_0/ap_start] [get_bd_pins integration_0/ap_start] [get_bd_pins one/dout] [get_bd_pins spot_finder_0/ap_start] [get_bd_pins stream_24bit_conv_0/ap_start]
connect_bd_net -net axi_clk_1 [get_bd_pins axi_clk] [get_bd_pins adu_histo_0/ap_clk] [get_bd_pins axis_64_to_512_0/ap_clk] [get_bd_pins axis_compl_fifo_0/s_axis_aclk] [get_bd_pins axis_compl_fifo_1/s_axis_aclk] [get_bd_pins axis_compl_fifo_2/s_axis_aclk] [get_bd_pins axis_compl_fifo_3/s_axis_aclk] [get_bd_pins axis_compl_fifo_4/s_axis_aclk] [get_bd_pins axis_compl_fifo_5/s_axis_aclk] [get_bd_pins axis_compl_fifo_6/s_axis_aclk] [get_bd_pins axis_compl_fifo_7/s_axis_aclk] [get_bd_pins axis_data_fifo_0/s_axis_aclk] [get_bd_pins axis_data_fifo_1/s_axis_aclk] [get_bd_pins axis_data_fifo_10/s_axis_aclk] [get_bd_pins axis_data_fifo_2/s_axis_aclk] [get_bd_pins axis_data_fifo_3/s_axis_aclk] [get_bd_pins axis_data_fifo_4/s_axis_aclk] [get_bd_pins axis_data_fifo_5/s_axis_aclk] [get_bd_pins axis_data_fifo_6/s_axis_aclk] [get_bd_pins axis_data_fifo_7/s_axis_aclk] [get_bd_pins axis_data_fifo_8/s_axis_aclk] [get_bd_pins axis_data_fifo_9/s_axis_aclk] [get_bd_pins axis_data_spot_finder_mask_0/s_axis_aclk] [get_bd_pins axis_integration_result_fifo_0/s_axis_aclk] [get_bd_pins axis_integration_result_fifo_1/s_axis_aclk] [get_bd_pins axis_register_slice_data_1/aclk] [get_bd_pins axis_register_slice_data_2/aclk] [get_bd_pins axis_register_slice_data_3/aclk] [get_bd_pins axis_roi_calc_result_fifo_0/s_axis_aclk] [get_bd_pins axis_spot_finder_conn_fifo_0/s_axis_aclk] [get_bd_pins axis_spot_finder_fifo_0/s_axis_aclk] [get_bd_pins axis_spot_finder_fifo_1/s_axis_aclk] [get_bd_pins axis_spot_finder_fifo_2/s_axis_aclk] [get_bd_pins eiger_reorder_0/ap_clk] [get_bd_pins frame_summation_0/ap_clk] [get_bd_pins integration_0/ap_clk] [get_bd_pins jf_conversion_0/ap_clk] [get_bd_pins mask_missing_0/ap_clk] [get_bd_pins pedestal_0/ap_clk] [get_bd_pins roi_calc_0/ap_clk] [get_bd_pins smartconnect_0/aclk] [get_bd_pins smartconnect_1/aclk] [get_bd_pins smartconnect_2/aclk] [get_bd_pins smartconnect_3/aclk] [get_bd_pins smartconnect_4/aclk] [get_bd_pins smartconnect_5/aclk] [get_bd_pins spot_finder_0/ap_clk] [get_bd_pins spot_finder_connecti_0/ap_clk] [get_bd_pins spot_finder_mask_0/ap_clk] [get_bd_pins spot_finder_merge_0/ap_clk] [get_bd_pins stream_24bit_conv_0/ap_clk]
connect_bd_net -net axi_rst_n_1 [get_bd_pins axi_rst_n] [get_bd_pins axis_compl_fifo_0/s_axis_aresetn] [get_bd_pins axis_compl_fifo_1/s_axis_aresetn] [get_bd_pins axis_compl_fifo_2/s_axis_aresetn] [get_bd_pins axis_compl_fifo_3/s_axis_aresetn] [get_bd_pins axis_compl_fifo_4/s_axis_aresetn] [get_bd_pins axis_compl_fifo_5/s_axis_aresetn] [get_bd_pins axis_compl_fifo_6/s_axis_aresetn] [get_bd_pins axis_compl_fifo_7/s_axis_aresetn] [get_bd_pins axis_data_fifo_0/s_axis_aresetn] [get_bd_pins axis_data_fifo_1/s_axis_aresetn] [get_bd_pins axis_data_fifo_10/s_axis_aresetn] [get_bd_pins axis_data_fifo_2/s_axis_aresetn] [get_bd_pins axis_data_fifo_3/s_axis_aresetn] [get_bd_pins axis_data_fifo_4/s_axis_aresetn] [get_bd_pins axis_data_fifo_5/s_axis_aresetn] [get_bd_pins axis_data_fifo_6/s_axis_aresetn] [get_bd_pins axis_data_fifo_7/s_axis_aresetn] [get_bd_pins axis_data_fifo_8/s_axis_aresetn] [get_bd_pins axis_data_fifo_9/s_axis_aresetn] [get_bd_pins axis_data_spot_finder_mask_0/s_axis_aresetn] [get_bd_pins axis_integration_result_fifo_0/s_axis_aresetn] [get_bd_pins axis_integration_result_fifo_1/s_axis_aresetn] [get_bd_pins axis_register_slice_data_1/aresetn] [get_bd_pins axis_register_slice_data_2/aresetn] [get_bd_pins axis_register_slice_data_3/aresetn] [get_bd_pins axis_roi_calc_result_fifo_0/s_axis_aresetn] [get_bd_pins axis_spot_finder_conn_fifo_0/s_axis_aresetn] [get_bd_pins axis_spot_finder_fifo_0/s_axis_aresetn] [get_bd_pins axis_spot_finder_fifo_1/s_axis_aresetn] [get_bd_pins axis_spot_finder_fifo_2/s_axis_aresetn] [get_bd_pins smartconnect_0/aresetn] [get_bd_pins smartconnect_1/aresetn] [get_bd_pins smartconnect_2/aresetn] [get_bd_pins smartconnect_3/aresetn] [get_bd_pins smartconnect_4/aresetn] [get_bd_pins smartconnect_5/aresetn]
connect_bd_net -net axi_clk_1 [get_bd_pins axi_clk] [get_bd_pins adu_histo_0/ap_clk] [get_bd_pins axis_32_to_512_0/ap_clk] [get_bd_pins axis_64_to_512_0/ap_clk] [get_bd_pins axis_compl_fifo_0/s_axis_aclk] [get_bd_pins axis_compl_fifo_1/s_axis_aclk] [get_bd_pins axis_compl_fifo_2/s_axis_aclk] [get_bd_pins axis_compl_fifo_3/s_axis_aclk] [get_bd_pins axis_compl_fifo_4/s_axis_aclk] [get_bd_pins axis_compl_fifo_5/s_axis_aclk] [get_bd_pins axis_compl_fifo_6/s_axis_aclk] [get_bd_pins axis_compl_fifo_7/s_axis_aclk] [get_bd_pins axis_data_fifo_0/s_axis_aclk] [get_bd_pins axis_data_fifo_1/s_axis_aclk] [get_bd_pins axis_data_fifo_10/s_axis_aclk] [get_bd_pins axis_data_fifo_2/s_axis_aclk] [get_bd_pins axis_data_fifo_3/s_axis_aclk] [get_bd_pins axis_data_fifo_4/s_axis_aclk] [get_bd_pins axis_data_fifo_5/s_axis_aclk] [get_bd_pins axis_data_fifo_6/s_axis_aclk] [get_bd_pins axis_data_fifo_7/s_axis_aclk] [get_bd_pins axis_data_fifo_8/s_axis_aclk] [get_bd_pins axis_data_fifo_9/s_axis_aclk] [get_bd_pins axis_data_spot_finder_mask_0/s_axis_aclk] [get_bd_pins axis_integration_result_fifo_0/s_axis_aclk] [get_bd_pins axis_integration_result_fifo_1/s_axis_aclk] [get_bd_pins axis_register_slice_data_1/aclk] [get_bd_pins axis_register_slice_data_2/aclk] [get_bd_pins axis_register_slice_data_3/aclk] [get_bd_pins axis_roi_calc_result_fifo_0/s_axis_aclk] [get_bd_pins axis_spot_finder_conn_fifo_0/s_axis_aclk] [get_bd_pins axis_spot_finder_fifo_0/s_axis_aclk] [get_bd_pins axis_spot_finder_fifo_1/s_axis_aclk] [get_bd_pins axis_spot_finder_fifo_2/s_axis_aclk] [get_bd_pins axis_spot_finder_fifo_3/s_axis_aclk] [get_bd_pins eiger_reorder_0/ap_clk] [get_bd_pins frame_summation_0/ap_clk] [get_bd_pins integration_0/ap_clk] [get_bd_pins jf_conversion_0/ap_clk] [get_bd_pins mask_missing_0/ap_clk] [get_bd_pins pedestal_0/ap_clk] [get_bd_pins roi_calc_0/ap_clk] [get_bd_pins smartconnect_0/aclk] [get_bd_pins smartconnect_1/aclk] [get_bd_pins smartconnect_2/aclk] [get_bd_pins smartconnect_3/aclk] [get_bd_pins smartconnect_4/aclk] [get_bd_pins smartconnect_5/aclk] [get_bd_pins spot_finder_0/ap_clk] [get_bd_pins spot_finder_connecti_0/ap_clk] [get_bd_pins spot_finder_mask_0/ap_clk] [get_bd_pins spot_finder_merge_0/ap_clk] [get_bd_pins stream_24bit_conv_0/ap_clk]
connect_bd_net -net axi_rst_n_1 [get_bd_pins axi_rst_n] [get_bd_pins axis_compl_fifo_0/s_axis_aresetn] [get_bd_pins axis_compl_fifo_1/s_axis_aresetn] [get_bd_pins axis_compl_fifo_2/s_axis_aresetn] [get_bd_pins axis_compl_fifo_3/s_axis_aresetn] [get_bd_pins axis_compl_fifo_4/s_axis_aresetn] [get_bd_pins axis_compl_fifo_5/s_axis_aresetn] [get_bd_pins axis_compl_fifo_6/s_axis_aresetn] [get_bd_pins axis_compl_fifo_7/s_axis_aresetn] [get_bd_pins axis_data_fifo_0/s_axis_aresetn] [get_bd_pins axis_data_fifo_1/s_axis_aresetn] [get_bd_pins axis_data_fifo_10/s_axis_aresetn] [get_bd_pins axis_data_fifo_2/s_axis_aresetn] [get_bd_pins axis_data_fifo_3/s_axis_aresetn] [get_bd_pins axis_data_fifo_4/s_axis_aresetn] [get_bd_pins axis_data_fifo_5/s_axis_aresetn] [get_bd_pins axis_data_fifo_6/s_axis_aresetn] [get_bd_pins axis_data_fifo_7/s_axis_aresetn] [get_bd_pins axis_data_fifo_8/s_axis_aresetn] [get_bd_pins axis_data_fifo_9/s_axis_aresetn] [get_bd_pins axis_data_spot_finder_mask_0/s_axis_aresetn] [get_bd_pins axis_integration_result_fifo_0/s_axis_aresetn] [get_bd_pins axis_integration_result_fifo_1/s_axis_aresetn] [get_bd_pins axis_register_slice_data_1/aresetn] [get_bd_pins axis_register_slice_data_2/aresetn] [get_bd_pins axis_register_slice_data_3/aresetn] [get_bd_pins axis_roi_calc_result_fifo_0/s_axis_aresetn] [get_bd_pins axis_spot_finder_conn_fifo_0/s_axis_aresetn] [get_bd_pins axis_spot_finder_fifo_0/s_axis_aresetn] [get_bd_pins axis_spot_finder_fifo_1/s_axis_aresetn] [get_bd_pins axis_spot_finder_fifo_2/s_axis_aresetn] [get_bd_pins axis_spot_finder_fifo_3/s_axis_aresetn] [get_bd_pins smartconnect_0/aresetn] [get_bd_pins smartconnect_1/aresetn] [get_bd_pins smartconnect_2/aresetn] [get_bd_pins smartconnect_3/aresetn] [get_bd_pins smartconnect_4/aresetn] [get_bd_pins smartconnect_5/aresetn]
connect_bd_net -net axis_data_fifo_6_almost_empty [get_bd_pins proc_fifo_empty] [get_bd_pins axis_data_fifo_7/almost_empty]
connect_bd_net -net axis_data_fifo_6_almost_full [get_bd_pins proc_fifo_full] [get_bd_pins axis_data_fifo_7/almost_full]
connect_bd_net -net frame_summation_0_idle [get_bd_pins frame_summation_idle] [get_bd_pins frame_summation_0/idle]

8
frame_serialize/CBORStream2Deserializer.cpp
View File

@@ -622,8 +622,8 @@ void CBORStream2Deserializer::ProcessStartUserData(CborValue &value) {
return;
if (j.contains("file_prefix"))
start_message.file_prefix = j["file_prefix"];
if (j.contains("data_file_count"))
start_message.data_file_count = j["data_file_count"];
if (j.contains("images_per_file"))
start_message.images_per_file = j["images_per_file"];
if (j.contains("user"))
start_message.user_data = j["user"];
if (j.contains("sample_name"))
@@ -855,9 +855,7 @@ void CBORStream2Deserializer::Process(const uint8_t *msg, size_t msg_size) {
while (ProcessImageMessageElement(map_value));
break;
case Type::START:
start_message = StartMessage{
.data_file_count = 1
};
start_message = StartMessage{};
while (ProcessStartMessageElement(map_value));
break;
case Type::END:

2
frame_serialize/CBORStream2Serializer.cpp
View File

@@ -299,7 +299,7 @@ inline void CBOR_ENC_START_USER_DATA(CborEncoder& encoder, const char* key,
const StartMessage& message) {
nlohmann::json j;
j["file_prefix"] = message.file_prefix;
j["data_file_count"] = message.data_file_count;
j["images_per_file"] = message.images_per_file;
j["source_name"] = message.source_name;
j["source_name_short"] = message.source_name_short;
j["instrument_name"] = message.instrument_name;

6
frame_serialize/JFJochMessages.h
View File

@@ -95,8 +95,6 @@ struct GoniometerAxis {
};
struct StartMessage {
uint64_t data_file_count; // user data
float detector_distance;
float beam_center_x;
float beam_center_y;
@@ -138,6 +136,7 @@ struct StartMessage {
std::string sample_name; // user data
std::string file_prefix; // user data
int64_t images_per_file = 1; // user data
std::vector<std::string> channels;
@@ -167,17 +166,14 @@ struct StartMessage {
std::optional<float> total_flux;
std::optional<float> attenuator_transmission;
size_t approx_size = 1024*1024;
// Use function below to update approx_size
void AddPixelMask(CompressedImage image) {
approx_size += image.size;
image.Save();
pixel_mask.emplace_back(std::move(image));
}
void AddCalibration(CompressedImage image) {
approx_size += image.size;
image.Save();
calibration.emplace_back(std::move(image));
}

8
frame_serialize/ZMQStream2PusherGroup.cpp
View File

@@ -27,16 +27,16 @@ ZMQStream2PusherGroup::ZMQStream2PusherGroup(const std::vector<std::string> &add
bool ZMQStream2PusherGroup::SendImage(const DataMessage& message) {
if (pusher.empty())
return false;
auto socket_number = (message.number % file_count) % pusher.size();
auto socket_number = (message.number / images_per_file) % pusher.size();
return pusher[socket_number]->SendImage(message);
}
void ZMQStream2PusherGroup::StartDataCollection(const StartMessage& message) {
if (message.data_file_count < 1)
if (message.images_per_file < 1)
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"File count cannot be zero or negative");
file_count = message.data_file_count;
"Images per file cannot be zero or negative");
images_per_file = message.images_per_file;
for (auto &p: pusher)
p->StartDataCollection(message);

2
frame_serialize/ZMQStream2PusherGroup.h
View File

@@ -9,7 +9,7 @@
class ZMQStream2PusherGroup : public ImagePusher {
std::vector<std::unique_ptr<ZMQStream2Pusher>> pusher;
int64_t file_count = 1;
int64_t images_per_file = 1;
public:
ZMQStream2PusherGroup(ZMQContext &context, const std::vector<std::string>& addr,
int32_t send_buffer_high_watermark = -1, int32_t send_buffer_size = -1);

45
image_analysis/CPUSpotFinder.cpp
View File

@@ -3,13 +3,15 @@
#include "CPUSpotFinder.h"
template <int N>
void FindSpots(StrongPixelSet& set,
void FindSpots(DeviceOutput &output,
int big_column, int big_row,
const SpotFindingSettings& settings,
const int16_t* image) {
uint64_t sum = 0;
uint64_t sum2 = 0;
uint64_t valid_count = 0;
const float *d_array) {
auto image = (int16_t *) output.pixels;
int64_t sum = 0;
int64_t sum2 = 0;
int64_t valid_count = 0;
for (int y = 0; y < N; y++) {
for (int x = 0; x < N; x++) {
@@ -27,16 +29,30 @@ void FindSpots(StrongPixelSet& set,
for (int y = 0; y < N; y++) {
for (int x = 0; x < N; x++) {
size_t coord = (big_row * N + y) * RAW_MODULE_COLS + big_column * N + x;
size_t line = big_row * N + y;
size_t col = big_column * N + x;
size_t coord = line * RAW_MODULE_COLS + col;
bool strong_pixel = true;
if ((line == 255) || (line == 256)
|| (col == 255) || (col == 256)
|| (col == 511) || (col == 512)
|| (col == 767) || (col == 768))
strong_pixel = false;
if (d_array[coord] != 0) {
if ((d_array[coord] < settings.high_resolution_limit)
|| (d_array[coord] > settings.low_resolution_limit))
strong_pixel = false;
}
if ((settings.photon_count_threshold < 0)
&& (settings.signal_to_noise_threshold <= 0))
strong_pixel = false;
if ((settings.photon_count_threshold >= 0)
&& (image[coord] < settings.photon_count_threshold)) {
&& (image[coord] <= settings.photon_count_threshold)) {
strong_pixel = false;
}
@@ -48,17 +64,20 @@ void FindSpots(StrongPixelSet& set,
strong_pixel = false;
}
if (strong_pixel)
set.AddStrongPixel(big_column * N + x, big_row * N + y, image[coord]);
if (strong_pixel) {
output.spot_finding_result.strong_pixel[coord / 8] |= (1 << (coord % 8));
output.spot_finding_result.strong_pixel_count++;
}
}
}
}
void FindSpots(StrongPixelSet& set,
const SpotFindingSettings& settings,
const int16_t* image) {
void FindSpots(DeviceOutput &output, const SpotFindingSettings& settings, const float *d_array) {
for (auto &i: output.spot_finding_result.strong_pixel)
i = 0;
for (int i = 0; i < RAW_MODULE_LINES / 32; i++) {
for (int j = 0; j < RAW_MODULE_COLS / 32; j++)
FindSpots<32>(set, j, i, settings, image);
FindSpots<32>(output, j, i, settings, d_array);
}
}

4
image_analysis/CPUSpotFinder.h
View File

@@ -7,8 +7,6 @@
#include "SpotFindingSettings.h"
#include "StrongPixelSet.h"
void FindSpots(StrongPixelSet& set,
const SpotFindingSettings& settings,
const int16_t* image);
void FindSpots(DeviceOutput &output, const SpotFindingSettings& settings, const float *d_array);
#endif //JUNGFRAUJOCH_CPUSPOTFINDER_H

16
image_analysis/MXAnalyzer.cpp
View File

@@ -2,6 +2,7 @@
#include "MXAnalyzer.h"
#include "CPUSpotFinder.h"
#include "../common/DiffractionGeometry.h"
MXAnalyzer::MXAnalyzer(const DiffractionExperiment &in_experiment)
: experiment(in_experiment) {
@@ -19,16 +20,23 @@ void MXAnalyzer::ReadFromFPGA(const DeviceOutput *output, const SpotFindingSetti
if (!find_spots)
return;
StrongPixelSet strong_pixel_set;
strong_pixel_set.ReadFPGAOutput(*output);
strong_pixel_set.ReadFPGAOutput(experiment, *output);
strong_pixel_set.FindSpots(experiment, settings, spots, module_number);
}
void MXAnalyzer::ReadFromCPU(const int16_t *image, const SpotFindingSettings &settings, size_t module_number) {
if (!find_spots)
return;
StrongPixelSet strong_pixel_set;
FindSpots(strong_pixel_set, settings, image);
strong_pixel_set.FindSpots(experiment, settings, spots, module_number);
std::vector<float> d_map(RAW_MODULE_SIZE);
DeviceOutput output;
memcpy(output.pixels, image, RAW_MODULE_SIZE * sizeof(int16_t));
CalcSpotFinderResolutionMap(d_map.data(), experiment, module_number);
FindSpots(output, settings, d_map.data());
ReadFromFPGA(&output, settings, module_number);
}
bool MXAnalyzer::Process(DataMessage &message) {

233
image_analysis/StrongPixelSet.cpp
View File

@@ -1,4 +1,10 @@
// Copyright (2019-2023) Paul Scherrer Institute
// Copyright (2019-2024) Paul Scherrer Institute
// SparseCCL code taken from https://github.com/acts-project/traccc/blob/main/core/include/traccc/clusterization/detail/sparse_ccl.hpp
// (c) 2021-2022 CERN for the benefit of the ACTS project
// Mozilla Public License Version 2.0
#include <bitset>
#include "StrongPixelSet.h"
@@ -57,145 +63,142 @@ void FilterSpotsByResolution(const DiffractionExperiment& experiment,
output.push_back(input[res_id_vector[i].id]);
}
StrongPixelSet::StrongPixelSet()
: strong_pixel_vector(RAW_MODULE_SIZE, false),
strong_pixel_count(0) {
StrongPixelSet::StrongPixelSet() : strong_pixel_count(0) {
pixels.reserve(max_strong_pixel_per_module);
}
void StrongPixelSet::AddStrongPixel(uint16_t col, uint16_t line, int32_t photons) {
auto key = strong_pixel_coord(col, line);
strong_pixel_map[key] = photons;
strong_pixel_vector.at(line * xpixel + col) = true;
pixels.push_back(strong_pixel{.col = col, .line = line, .counts = photons});
}
void StrongPixelSet::AddSingleStrongPixel(uint16_t col, uint16_t line, int32_t photons) {
auto key = strong_pixel_coord(col, line);
auto p = std::pair<uint32_t, int32_t>(key, photons);
single_pixels.emplace_back(p);
bool is_far_enough(strong_pixel pixel0, strong_pixel pixel1) {
return (pixel1.line - pixel0.line) > 1;
}
inline void StrongPixelSet::AddNeighbor(DiffractionSpot &spot, uint16_t col, uint16_t line) {
if (strong_pixel_vector[line * xpixel + col]) {
uint64_t coord = strong_pixel_coord(col, line);
auto iter = strong_pixel_map.find(coord);
ExtendSpot(spot, iter);
}
bool is_adjacent(strong_pixel pixel0, strong_pixel pixel1) {
return (fabs(pixel0.line - pixel1.line) <= 1) and
(fabs(pixel0.col - pixel1.col) <= 1);
}
// Creates a continuous spot
// strong pixels are loaded into dictionary (one dictionary per frame)
// and routine checks if neighboring pixels are also in dictionary (likely in log(N) time)
DiffractionSpot StrongPixelSet::BuildSpot(std::unordered_map<uint32_t, int32_t>::iterator &it) {
uint16_t col = col_from_strong_pixel(it->first);
uint16_t line = line_from_strong_pixel(it->first);
DiffractionSpot spot(col, line, it->second);
strong_pixel_vector[line * xpixel + col] = false;
strong_pixel_map.erase(it); // Remove strong pixel from the dictionary, so it is not processed again
if (col+1 < xpixel) {
AddNeighbor(spot, col + 1, line);
if (line < ypixel - 1)
AddNeighbor(spot, col + 1, line + 1);
if (line > 0)
AddNeighbor(spot, col + 1, line - 1);
uint16_t StrongPixelSet::find_root(uint16_t e) {
uint16_t r = e;
//assert(r < L.size());
while (L[r] != r) {
r = L[r];
//assert(r < L.size());
}
if (col != 0) {
AddNeighbor(spot, col - 1, line);
if (line < ypixel - 1)
AddNeighbor(spot, col - 1, line + 1);
if (line > 0)
AddNeighbor(spot, col - 1, line - 1);
}
if (line < ypixel - 1)
AddNeighbor(spot, col, line+1);
if (line > 0)
AddNeighbor(spot, col, line-1);
return spot;
return r;
}
void StrongPixelSet::ExtendSpot(DiffractionSpot &spot, std::unordered_map<uint32_t, int32_t>::iterator &it) {
uint16_t col = col_from_strong_pixel(it->first);
uint16_t line = line_from_strong_pixel(it->first);
spot.AddPixel(col, line, it->second);
strong_pixel_vector[line * xpixel + col] = false;
strong_pixel_map.erase(it); // Remove strong pixel from the dictionary, so it is not processed again
if (col+1 < xpixel) {
AddNeighbor(spot, col + 1, line);
if (line < ypixel - 1)
AddNeighbor(spot, col + 1, line + 1);
if (line > 0)
AddNeighbor(spot, col + 1, line - 1);
uint16_t StrongPixelSet::make_union(uint16_t e1, uint16_t e2) {
uint16_t e;
if (e1 < e2) {
e = e1;
//assert(e2 < L.size());
L[e2] = e;
} else {
e = e2;
//assert(e1 < L.size());
L[e1] = e;
}
if (col != 0) {
AddNeighbor(spot, col - 1, line);
if (line < ypixel - 1)
AddNeighbor(spot, col - 1, line + 1);
if (line > 0)
AddNeighbor(spot, col - 1, line - 1);
}
if (line < ypixel - 1)
AddNeighbor(spot, col, line+1);
if (line > 0)
AddNeighbor(spot, col, line-1);
return e;
}
void StrongPixelSet::FindSpots(const DiffractionExperiment &experiment, const SpotFindingSettings &settings,
std::vector<DiffractionSpot> &spots, uint16_t module_number) {
if (settings.min_pix_per_spot == 1) {
for (const auto &i: single_pixels) {
uint16_t col = col_from_strong_pixel(i.first);
uint16_t line = line_from_strong_pixel(i.first);
std::vector<DiffractionSpot> StrongPixelSet::sparseccl() {
L.resize(pixels.size());
DiffractionSpot spot(col, line, i.second);
spots.emplace_back(spot);
unsigned int labels = 0;
// first scan: pixel association
uint16_t start_j = 0;
for (uint16_t i = 0; i < pixels.size(); ++i) {
L[i] = i;
uint16_t ai = i;
for (uint16_t j = start_j; j < i; ++j) {
if (is_adjacent(pixels[i], pixels[j])) {
ai = make_union(ai, find_root(j));
} else if (is_far_enough(pixels[j], pixels[i])) {
++start_j;
}
}
}
while (!strong_pixel_map.empty()) {
auto iter = strong_pixel_map.begin();
DiffractionSpot spot = BuildSpot(iter);
spot.ConvertToImageCoordinates(experiment, module_number);
// second scan: transitive closure
for (unsigned int i = 0; i < L.size(); ++i) {
if (L[i] == i) {
L[i] = labels++;
} else {
L[i] = L[L[i]];
}
}
if ((spot.PixelCount() <= settings.max_pix_per_spot)
&& (spot.PixelCount() >= settings.min_pix_per_spot))
spots.push_back(spot);
std::vector<DiffractionSpot> spots(labels);
for (unsigned int i = 0; i < L.size(); i++)
spots[L[i]].AddPixel(pixels[i].col, pixels[i].line, pixels[i].counts);
return spots;
}
void StrongPixelSet::FindSpots(const DiffractionExperiment &experiment, const SpotFindingSettings &settings,
std::vector<DiffractionSpot> &spots, uint16_t module_number) {
if (!pixels.empty()) {
for (const auto &spot: sparseccl()) {
if ((spot.PixelCount() <= settings.max_pix_per_spot)
&& (spot.PixelCount() >= settings.min_pix_per_spot)) {
auto s = spot;
s.ConvertToImageCoordinates(experiment, module_number);
spots.push_back(s);
}
}
}
}
size_t StrongPixelSet::Count() const {
return strong_pixel_map.size();
}
size_t StrongPixelSet::Common(const StrongPixelSet &set) const {
size_t ret = 0;
for (const auto& pixel: strong_pixel_map) {
if (set.strong_pixel_map.count(pixel.first) == 1)
ret++;
}
return ret;
}
void StrongPixelSet::ReadFPGAOutput(const DeviceOutput &output) {
void StrongPixelSet::ReadFPGAOutput(const DiffractionExperiment & experiment,
const DeviceOutput &output) {
strong_pixel_count += output.spot_finding_result.strong_pixel_count;
for (int i = 0; i < std::min<uint32_t>(output.spot_finding_result.max_memory_index * 32,
SPOT_FINDER_MAX_STRONG_PIXEL); i++) {
uint32_t val = output.spot_finding_result.strong_pixel_number[i];
if (val != UINT32_MAX) {
uint32_t pixel = (val & ((1LU << 20) - 1));
uint8_t conn = ((val >> 24) & UINT8_MAX);
if (conn == 0)
AddSingleStrongPixel(pixel % RAW_MODULE_COLS, pixel / RAW_MODULE_COLS, output.pixels[pixel]);
else
AddStrongPixel(pixel % RAW_MODULE_COLS, pixel / RAW_MODULE_COLS, output.pixels[pixel]);
// Too many strong pixels will kill performance in data processing, so protection is needed
// Also if there are no strong pixels, there is no point in looking for them
if ((output.spot_finding_result.strong_pixel_count == 0) ||
(output.spot_finding_result.strong_pixel_count > max_strong_pixel_per_module))
return;
auto pixel_depth = experiment.GetPixelDepth();
auto out_ptr = (uint32_t *) output.spot_finding_result.strong_pixel;
if (pixel_depth == 2) {
for (int i = 0; i < RAW_MODULE_SIZE / (8 * sizeof(out_ptr[0])); i++) {
size_t npixel = i * 8 * sizeof(out_ptr[0]);
size_t line = npixel / RAW_MODULE_COLS;
if (out_ptr[i] != 0) {
std::bitset<32> bitset(out_ptr[i]);
for (int j = 0; j < 32; j++) {
if (bitset.test(j)) {
size_t col = (npixel | j) % RAW_MODULE_COLS;
AddStrongPixel(col, line, output.pixels[npixel]);
}
}
}
}
} else {
for (int i = 0; i < RAW_MODULE_SIZE / (8 * sizeof(out_ptr[0])); i++) {
size_t npixel = i * 8 * sizeof(out_ptr[0]);
size_t line = npixel / RAW_MODULE_COLS;
if (out_ptr[i] != 0) {
std::bitset<32> bitset(out_ptr[i]);
for (int j = 0; j < 32; j++) {
if (bitset.test(j)) {
size_t col = (npixel | j) % RAW_MODULE_COLS;
AddStrongPixel(col, line, ((int32_t *) output.pixels)[npixel]);
}
}
}
}
}
}

34
image_analysis/StrongPixelSet.h
View File

@@ -12,17 +12,11 @@
#include "../acquisition_device/AcquisitionDevice.h"
#include "SpotFindingSettings.h"
inline uint32_t strong_pixel_coord(uint16_t col, uint16_t line) {
return col + (static_cast<uint32_t>(line) << 16u);
}
inline uint16_t line_from_strong_pixel(uint32_t strong_pixel) {
return ((strong_pixel & 0xFFFF0000u) >> 16u);
}
inline uint16_t col_from_strong_pixel(uint32_t strong_pixel) {
return (strong_pixel & 0x0000FFFFu);
}
struct strong_pixel {
uint16_t col;
uint16_t line;
int32_t counts;
};
void FilterSpotsByResolution(const DiffractionExperiment& experiment,
const std::vector<DiffractionSpot> &input,
@@ -33,28 +27,26 @@ void FilterSpotsByCount(const DiffractionExperiment& experiment,
std::vector<DiffractionSpot> &output);
class StrongPixelSet {
std::vector<std::pair<uint32_t, int32_t>> single_pixels;
std::unordered_map<uint32_t, int32_t> strong_pixel_map;
std::vector<strong_pixel> pixels;
std::vector<uint16_t> L;
static const constexpr size_t max_strong_pixel_per_module = 4000;
static const constexpr uint32_t xpixel = RAW_MODULE_COLS;
static const constexpr uint32_t ypixel = RAW_MODULE_LINES;
uint32_t strong_pixel_count;
std::vector<bool> strong_pixel_vector;
void AddNeighbor(DiffractionSpot &spot, uint16_t col, uint16_t line);
DiffractionSpot BuildSpot(std::unordered_map<uint32_t, int32_t>::iterator &it_frames);
void ExtendSpot(DiffractionSpot &spot, std::unordered_map<uint32_t, int32_t>::iterator &it_frames);
uint16_t find_root(uint16_t e);
uint16_t make_union(uint16_t e1, uint16_t e2);
std::vector<DiffractionSpot> sparseccl();
public:
void ReadFPGAOutput(const DeviceOutput& output);
void ReadFPGAOutput(const DiffractionExperiment& experiment,
const DeviceOutput& output);
StrongPixelSet();
size_t Count() const;
void AddStrongPixel(uint16_t col, uint16_t line, int32_t photons = 1);
void AddSingleStrongPixel(uint16_t col, uint16_t line, int32_t photons = 1);
void FindSpots(const DiffractionExperiment &experiment, const SpotFindingSettings &settings, std::vector<DiffractionSpot> &spots,
uint16_t module_number);
size_t Common(const StrongPixelSet &set) const;
uint32_t GetStrongPixelCount() const;
};

2
image_analysis/fast-feedback-indexer

Submodule image_analysis/fast-feedback-indexer updated: d860d0d5d6...bf27ff20a5

6
receiver/JFJochReceiver.cpp
View File

@@ -69,8 +69,6 @@ JFJochReceiver::JFJochReceiver(const DiffractionExperiment& in_experiment,
logger.Info("Data acquisition devices ready");
if (experiment.GetImageNum() > 0) {
logger.Info("Data file count {}", experiment.GetDataFileCount());
StartMessage message{};
experiment.FillMessage(message);
message.arm_date = time_UTC(std::chrono::system_clock::now());
@@ -359,7 +357,7 @@ void JFJochReceiver::FrameTransformationThread() {
message.resolution_estimation.reset();
}
}
plots.Add(message, image_number % experiment.GetDataFileCount(), az_int_profile_image);
plots.Add(message, image_number % experiment.GetTimePointNumber(), az_int_profile_image);
message.image = transformation.GetCompressedImage();
compressed_size += message.image.size;
@@ -456,7 +454,7 @@ void JFJochReceiver::FinalizeMeasurement() {
message.series_unique_id = experiment.GetSeriesIDString();
message.az_int_result["dataset"] = plots.GetAzIntProfile();
for (int i = 0; i < experiment.GetDataFileCount(); i++)
for (int i = 0; i < experiment.GetTimePointNumber(); i++)
message.az_int_result["file" + std::to_string(i)] = plots.GetAzIntProfilePerFile(i);
for (int i = 0; i < adu_histogram_module.size(); i++)

6
receiver/JFJochReceiverPlots.cpp
View File

@@ -4,13 +4,13 @@
JFJochReceiverPlots::JFJochReceiverPlots(const DiffractionExperiment &experiment,
const AzimuthalIntegrationMapping &mapping)
: indexing_solution_per_file(experiment.GetDataFileCount()),
: indexing_solution_per_file(experiment.GetTimePointNumber()),
default_binning(experiment.GetDefaultPlotBinning()),
az_int_profile(mapping), resolution_estimation(50, 1.0, 5.0) {
az_int_profile.SetTitle("dataset");
for (int i = 0; i < experiment.GetDataFileCount(); i++) {
for (int i = 0; i < experiment.GetTimePointNumber(); i++) {
az_int_profile_per_file.emplace_back(mapping);
az_int_profile_per_file[i].SetTitle("file" + std::to_string(i));
az_int_profile_per_file[i].SetTitle(std::to_string(i));
}
for (const auto &[x, y]: experiment.ROI().GetROINameMap()) {

11
receiver/LossyFilter.cpp
View File

@@ -14,11 +14,8 @@ bool LossyFilter::RollDice() {
return false;
}
void LossyFilter::ApplyFilter(DataMessage &message) {
if (enabled) {
if (message.indexing_result || RollDice())
message.number = (image_number++);
else
message.number = -1;
}
bool LossyFilter::ApplyFilter(DataMessage &message) {
if (!enabled)
return true;
return (message.indexing_result || RollDice());
}

3
receiver/LossyFilter.h
View File

@@ -15,13 +15,12 @@ class LossyFilter {
std::uniform_real_distribution<float> distr{0.0, 1.0};
mutable std::mutex random_m;
std::atomic<int64_t> image_number = 0;
bool enabled;
float p;
bool RollDice();
public:
LossyFilter(bool enabled, float p);
void ApplyFilter(DataMessage& message);
bool ApplyFilter(DataMessage& message);
};

6
tests/CBORTest.cpp
View File

@@ -13,7 +13,6 @@ TEST_CASE("CBORSerialize_Start", "[CBOR]") {
CBORStream2Serializer serializer(buffer.data(), buffer.size());
StartMessage message {
.data_file_count = 3,
.detector_distance = 0.0005,
.beam_center_x = 456.6,
.beam_center_y = 124.3,
@@ -42,6 +41,7 @@ TEST_CASE("CBORSerialize_Start", "[CBOR]") {
.arm_date = "abc",
.sample_name = "lyso",
.file_prefix = "lyso1/dir/file",
.images_per_file = 12345,
.channels = {"default", "sc2"},
.detector_description = "EIGER 16M",
.detector_serial_number = "123",
@@ -75,7 +75,7 @@ TEST_CASE("CBORSerialize_Start", "[CBOR]") {
StartMessage output_message;
REQUIRE_NOTHROW(output_message = deserializer.GetStartMessage());
CHECK(output_message.data_file_count == message.data_file_count);
CHECK(output_message.images_per_file == message.images_per_file);
CHECK(output_message.detector_distance == Approx(message.detector_distance));
CHECK(output_message.beam_center_x == Approx(message.beam_center_x));
CHECK(output_message.beam_center_y == Approx(message.beam_center_y));
@@ -734,7 +734,6 @@ TEST_CASE("CBORSerialize_Start_stream2", "[CBOR]") {
};
StartMessage message {
.data_file_count = 3,
.detector_distance = 0.0005,
.beam_center_x = 456.6,
.beam_center_y = 124.3,
@@ -760,6 +759,7 @@ TEST_CASE("CBORSerialize_Start_stream2", "[CBOR]") {
.arm_date = "abc",
.sample_name = "lyso",
.file_prefix = "lyso1/dir/file",
.images_per_file = 1,
.channels = {"default", "sc2"},
.detector_description = "EIGER 16M",
.detector_serial_number = "123",

2
tests/FPGAHostWriterTest.cpp
View File

@@ -26,7 +26,7 @@ TEST_CASE("FPGA_FrameWriterTestIgnore") {
for (int i = 0; i < nframes; i++) {
for (int j = 0; j < RAW_MODULE_SIZE * 2 / 64; j++)
data_in.write(packet_512_t{.data = 0, .user = 0, .last = (j == RAW_MODULE_SIZE * 2 / 64 - 1)});
for (int j = 0; j < RAW_MODULE_SIZE * sizeof(uint16_t) / (64 * 16) + 1; j++)
for (int j = 0; j < RAW_MODULE_SIZE / (64 * 8) + 1; j++)
spot_finder_in.write(0);
for (int j = 0; j < FPGA_INTEGRATION_BIN_COUNT / 8; j++)
integration_in.write(0);

111
tests/FPGAIntegrationTest.cpp
View File

@@ -1461,21 +1461,9 @@ TEST_CASE("HLS_C_Simulation_internal_packet_generator_integration", "[FPGA][Full
}
bool spot_finder_output_contains(const SpotFindingResult& output, uint32_t number) {
bool ret = false;
for (const auto &i: output.strong_pixel_number) {
if ((i & (((1<<21UL) - 1))) == number)
ret = true;
}
return ret;
}
bool spot_finder_output_connectivity(const SpotFindingResult& output, uint32_t number, uint8_t bit) {
bool ret = false;
for (const auto &i: output.strong_pixel_number) {
if ((i & (((1<<21UL) - 1))) == number)
ret = i & (1LU << (24 + bit));
}
return ret;
size_t byte = number / 8;
size_t bit = number % 8;
return ((output.strong_pixel[byte] & (1 << bit)) != 0);
}
TEST_CASE("HLS_C_Simulation_internal_packet_generator_spot_finder_count_threshold", "[FPGA][Full]") {
@@ -1501,7 +1489,7 @@ TEST_CASE("HLS_C_Simulation_internal_packet_generator_spot_finder_count_threshol
SpotFindingSettings parameters{
.signal_to_noise_threshold = 0.0,
.photon_count_threshold = 10,
.photon_count_threshold = 9,
.min_pix_per_spot = 1
};
test.SetSpotFinderParameters(parameters);
@@ -1519,94 +1507,13 @@ TEST_CASE("HLS_C_Simulation_internal_packet_generator_spot_finder_count_threshol
auto spot_finder_result = test.GetDeviceOutput(0, 0)->spot_finding_result;
REQUIRE (spot_finder_result.strong_pixel_count == 3);
REQUIRE (spot_finder_result.snr_threshold == 0);
REQUIRE (spot_finder_result.count_threshold == 10);
REQUIRE (spot_finder_result.count_threshold == 9);
CHECK (spot_finder_output_contains(spot_finder_result, 0));
CHECK (spot_finder_output_contains(spot_finder_result, 123*1024 + 578));
CHECK (spot_finder_output_contains(spot_finder_result, 121*1024 + 800));
}
TEST_CASE("HLS_C_Simulation_internal_packet_generator_spot_finder_connectivity", "[FPGA][Full]") {
const uint16_t nmodules = 4;
DiffractionExperiment x((DetectorGeometry(nmodules)));
x.Mode(DetectorMode::Raw);
x.UseInternalPacketGenerator(true).ImagesPerTrigger(1).PedestalG0Frames(0);
HLSSimulatedDevice test(0, 64);
std::vector<uint16_t> frame(RAW_MODULE_SIZE, 0);
frame [ 0] = 11;
frame [ 1] = 10;
frame [ 1024] = 12;
frame [2*1024] = 12;
frame [ 99*1024+200] = 12;
frame [100*1024+200] = 11;
frame [100*1024+201] = 10;
frame [101*1024+200] = 12;
frame [RAW_MODULE_SIZE - 1] = 20;
for (int m = 0; m < x.GetModulesNum(); m++)
test.SetInternalGeneratorFrame(frame.data(), m);
SpotFindingSettings parameters{
.signal_to_noise_threshold = 0.0,
.photon_count_threshold = 10,
.min_pix_per_spot = 1
};
test.SetSpotFinderParameters(parameters);
REQUIRE_NOTHROW(test.StartAction(x));
REQUIRE_NOTHROW(test.WaitForActionComplete());
REQUIRE(test.OutputStream().size() == 1);
REQUIRE(test.GetBytesReceived() == 128 * nmodules * JUNGFRAU_PACKET_SIZE_BYTES);
auto imageBuf = test.GetDeviceOutput(0, 0)->pixels;
REQUIRE(memcmp(imageBuf, frame.data(), RAW_MODULE_SIZE * sizeof(uint16_t)) == 0);
auto spot_finder_result = test.GetDeviceOutput(0, 0)->spot_finding_result;
REQUIRE (spot_finder_result.strong_pixel_count == 9);
REQUIRE (spot_finder_result.snr_threshold == 0);
REQUIRE (spot_finder_result.count_threshold == 10);
CHECK (spot_finder_output_contains(spot_finder_result, 0));
CHECK (spot_finder_output_contains(spot_finder_result, 1));
CHECK (spot_finder_output_contains(spot_finder_result, 1024));
CHECK (spot_finder_output_contains(spot_finder_result, RAW_MODULE_SIZE-1));
CHECK(spot_finder_output_connectivity(spot_finder_result, 0, 4));
CHECK(spot_finder_output_connectivity(spot_finder_result, 0, 6));
CHECK(spot_finder_output_connectivity(spot_finder_result, 1, 1));
CHECK(spot_finder_output_connectivity(spot_finder_result, 1, 2));
CHECK(spot_finder_output_connectivity(spot_finder_result, 1024, 3));
CHECK(spot_finder_output_connectivity(spot_finder_result, 1024, 4));
CHECK(spot_finder_output_connectivity(spot_finder_result, 1024, 5));
CHECK(spot_finder_output_connectivity(spot_finder_result, 2048, 3));
CHECK(spot_finder_output_connectivity(spot_finder_result, 100*1024+201, 0));
CHECK(spot_finder_output_connectivity(spot_finder_result, 100*1024+201, 1));
CHECK(spot_finder_output_connectivity(spot_finder_result, 100*1024+201, 2));
CHECK(!spot_finder_output_connectivity(spot_finder_result, 100*1024+201, 3));
CHECK(!spot_finder_output_connectivity(spot_finder_result, 100*1024+201, 4));
CHECK(!spot_finder_output_connectivity(spot_finder_result, 100*1024+201, 5));
CHECK(!spot_finder_output_connectivity(spot_finder_result, 100*1024+201, 6));
CHECK(!spot_finder_output_connectivity(spot_finder_result, 100*1024+201, 7));
CHECK(spot_finder_output_connectivity(spot_finder_result, 99*1024+200, 7));
CHECK(spot_finder_output_connectivity(spot_finder_result, 99*1024+200, 4));
for (int i = 0; i < 8; i++)
CHECK(!spot_finder_output_connectivity(spot_finder_result, RAW_MODULE_SIZE-1, i));
}
TEST_CASE("HLS_C_Simulation_internal_packet_generator_spot_finder_min_pix_per_spot", "[FPGA][Full]") {
const uint16_t nmodules = 4;
@@ -1636,7 +1543,7 @@ TEST_CASE("HLS_C_Simulation_internal_packet_generator_spot_finder_min_pix_per_sp
SpotFindingSettings parameters{
.signal_to_noise_threshold = 0.0,
.photon_count_threshold = 10,
.photon_count_threshold = 9,
.min_pix_per_spot = 2
};
test.SetSpotFinderParameters(parameters);
@@ -1654,7 +1561,7 @@ TEST_CASE("HLS_C_Simulation_internal_packet_generator_spot_finder_min_pix_per_sp
auto spot_finder_result = test.GetDeviceOutput(0, 0)->spot_finding_result;
REQUIRE (spot_finder_result.strong_pixel_count == 8);
REQUIRE (spot_finder_result.snr_threshold == 0);
REQUIRE (spot_finder_result.count_threshold == 10);
REQUIRE (spot_finder_result.count_threshold == 9);
CHECK (spot_finder_output_contains(spot_finder_result, 0));
CHECK (spot_finder_output_contains(spot_finder_result, 1));
@@ -1692,7 +1599,7 @@ TEST_CASE("HLS_C_Simulation_internal_packet_generator_spot_finder_d_min_max", "[
SpotFindingSettings parameters{
.signal_to_noise_threshold = 0.0,
.photon_count_threshold = 10,
.photon_count_threshold = 9,
.min_pix_per_spot = 1,
.high_resolution_limit = 1.25,
.low_resolution_limit = 3.0
@@ -1712,7 +1619,7 @@ TEST_CASE("HLS_C_Simulation_internal_packet_generator_spot_finder_d_min_max", "[
auto spot_finder_result = test.GetDeviceOutput(0, 0)->spot_finding_result;
REQUIRE (spot_finder_result.strong_pixel_count == 2);
REQUIRE (spot_finder_result.snr_threshold == 0);
REQUIRE (spot_finder_result.count_threshold == 10);
REQUIRE (spot_finder_result.count_threshold == 9);
CHECK (spot_finder_output_contains(spot_finder_result, 0));
CHECK (spot_finder_output_contains(spot_finder_result, 3));

46
tests/HDF5WritingTest.cpp
View File

@@ -15,7 +15,7 @@ using namespace std::literals::chrono_literals;
TEST_CASE("HDF5DataSet_scalar", "[HDF5][Unit]") {
uint16_t tmp_scalar = 16788;
{
HDF5File file("scratch1.h5", true, false);
HDF5File file("scratch1.h5");
file.SaveScalar("scalar", tmp_scalar);
}
{
@@ -38,7 +38,7 @@ TEST_CASE("HDF5DataSet_string", "[HDF5][Unit]") {
std::string tmp_string = "HDF5Content";
{
HDF5File file("scratch2.h5", true, false);
HDF5File file("scratch2.h5");
file.SaveScalar("str", tmp_string);
}
{
@@ -62,7 +62,7 @@ TEST_CASE("HDF5DataSet_vector", "[HDF5][Unit]") {
{
RegisterHDF5Filter();
HDF5File file("scratch3.h5", true, false);
HDF5File file("scratch3.h5");
file.SaveVector("vec", tmp_vector);
}
@@ -97,7 +97,7 @@ TEST_CASE("HDF5DataSet_vector_string", "[HDF5][Unit]") {
{
RegisterHDF5Filter();
HDF5File file("scratch4.h5", true, false);
HDF5File file("scratch4.h5");
REQUIRE_NOTHROW(file.SaveVector("vec", tmp_vector));
}
@@ -115,28 +115,6 @@ TEST_CASE("HDF5DataSet_vector_string", "[HDF5][Unit]") {
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5File_Delete", "[HDF5][Unit]") {
uint16_t tmp_scalar = 16788;
{
HDF5File file("scratch1.h5", true, false);
file.SaveScalar("scalar", tmp_scalar);
}
{
HDF5File file("scratch1.h5", false, false);
std::unique_ptr<HDF5DataSet> scalar_dataset;
REQUIRE_NOTHROW(scalar_dataset = std::make_unique<HDF5DataSet>(file, "scalar"));
file.Delete("/scalar");
}
{
HDF5File file("scratch1.h5", false, false);
std::unique_ptr<HDF5DataSet> scalar_dataset;
REQUIRE_THROWS(scalar_dataset = std::make_unique<HDF5DataSet>(file, "scalar"));
}
remove("scratch1.h5");
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5MasterFile", "[HDF5][Full]") {
{
RegisterHDF5Filter();
@@ -194,7 +172,7 @@ TEST_CASE("HDF5Writer", "[HDF5][Full]") {
DiffractionExperiment x(DetectorGeometry(8, 2, 8, 36));
std::vector<SpotToSave> spots;
x.FilePrefix("test02_1p10").ImagesPerTrigger(5).DataFileCount(2).Compression(CompressionAlgorithm::NO_COMPRESSION);
x.FilePrefix("test02_1p10").ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION);
StartMessage start_message;
x.FillMessage(start_message);
@@ -230,7 +208,7 @@ TEST_CASE("HDF5Writer_Spots", "[HDF5][Full]") {
spots.push_back({20,50,12});
spots.push_back({1000,500,3});
x.FilePrefix("test02_1p10_spots").ImagesPerTrigger(5).DataFileCount(2).Compression(CompressionAlgorithm::NO_COMPRESSION);
x.FilePrefix("test02_1p10_spots").ImagesPerTrigger(5).ImagesPerFile(3).Compression(CompressionAlgorithm::NO_COMPRESSION);
StartMessage start_message;
x.FillMessage(start_message);
@@ -269,7 +247,7 @@ TEST_CASE("HDF5Writer_Rad_Int_Profile", "[HDF5][Full]") {
std::vector<float> rad_int_profile(mapping.GetBinNumber(), 4.0);
std::vector<float> rad_int_avg(mapping.GetBinNumber(), 0.33);
x.FilePrefix("test02_1p10_rad_int").ImagesPerTrigger(5).DataFileCount(2).Compression(CompressionAlgorithm::NO_COMPRESSION);
x.FilePrefix("test02_1p10_rad_int").ImagesPerTrigger(5).ImagesPerFile(3).Compression(CompressionAlgorithm::NO_COMPRESSION);
StartMessage start_message;
x.FillMessage(start_message);
start_message.az_int_bin_to_q = mapping.GetBinToQ();
@@ -299,7 +277,7 @@ TEST_CASE("HDF5Writer_Rad_Int_Profile", "[HDF5][Full]") {
TEST_CASE("HDF5Writer_VDS", "[HDF5][Full]") {
DiffractionExperiment x(DetectorGeometry(1));
x.ImagesPerTrigger(5).DataFileCount(5).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("vds");
x.ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("vds");
{
RegisterHDF5Filter();
@@ -333,7 +311,7 @@ TEST_CASE("HDF5Writer_VDS", "[HDF5][Full]") {
REQUIRE_NOTHROW(HDF5Metadata::NXmx(start_message, end_message));
}
{
HDF5File file("vds_master.h5", false, false);
HDF5ReadOnlyFile file("vds_master.h5");
std::unique_ptr<HDF5DataSet> dataset;
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/data/data"));
HDF5DataSpace file_space(*dataset);
@@ -358,7 +336,7 @@ TEST_CASE("HDF5Writer_VDS", "[HDF5][Full]") {
TEST_CASE("HDF5Writer_VDS_missing", "[HDF5][Full]") {
DiffractionExperiment x(DetectorGeometry(1));
x.ImagesPerTrigger(5).DataFileCount(5).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("vds_missing");
x.ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("vds_missing");
{
RegisterHDF5Filter();
@@ -394,7 +372,7 @@ TEST_CASE("HDF5Writer_VDS_missing", "[HDF5][Full]") {
REQUIRE_NOTHROW(HDF5Metadata::NXmx(start_message, end_message));
}
{
HDF5File file("vds_missing_master.h5", false, false);
HDF5ReadOnlyFile file("vds_missing_master.h5");
std::unique_ptr<HDF5DataSet> dataset;
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/data/data"));
HDF5DataSpace file_space(*dataset);
@@ -419,7 +397,7 @@ TEST_CASE("HDF5Writer_VDS_missing", "[HDF5][Full]") {
TEST_CASE("HDF5Writer_VDS_zero_images", "[HDF5][Full]") {
DiffractionExperiment x(DetectorGeometry(1));
x.ImagesPerTrigger(5).DataFileCount(5).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("vds_zero");
x.ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("vds_zero");
{
RegisterHDF5Filter();

20
tests/JFJochReceiverIntegrationTest.cpp
View File

@@ -16,7 +16,7 @@ TEST_CASE("JFJochReceiverTest_Raw", "[JFJochReceiver]") {
x.Mode(DetectorMode::Raw);
x.PedestalG0Frames(0).NumTriggers(1).UseInternalPacketGenerator(true)
.ImagesPerTrigger(100).DataFileCount(16).PhotonEnergy_keV(12.4).Compression(CompressionAlgorithm::NO_COMPRESSION);
.ImagesPerTrigger(100).ImagesPerFile(10).PhotonEnergy_keV(12.4).Compression(CompressionAlgorithm::NO_COMPRESSION);
AcquisitionDeviceGroup aq_devices;
for (int i = 0; i < x.GetDataStreamsNum(); i++) {
@@ -46,7 +46,7 @@ TEST_CASE("JFJochReceiverTest_Conversion", "[JFJochReceiver]") {
x.Mode(DetectorMode::Conversion);
x.PedestalG0Frames(0).NumTriggers(1).UseInternalPacketGenerator(true)
.ImagesPerTrigger(32).DataFileCount(16).PhotonEnergy_keV(12.4).Compression(CompressionAlgorithm::BSHUF_ZSTD);
.ImagesPerTrigger(32).ImagesPerFile(10).PhotonEnergy_keV(12.4).Compression(CompressionAlgorithm::BSHUF_ZSTD);
AcquisitionDeviceGroup aq_devices;
for (int i = 0; i < x.GetDataStreamsNum(); i++) {
@@ -72,7 +72,7 @@ TEST_CASE("JFJochReceiverTest_Conversion_FixedGainG1", "[JFJochReceiver]") {
x.Mode(DetectorMode::Conversion);
x.PedestalG0Frames(0).NumTriggers(1).UseInternalPacketGenerator(true)
.ImagesPerTrigger(32).DataFileCount(16).PhotonEnergy_keV(12.4).Compression(CompressionAlgorithm::BSHUF_ZSTD)
.ImagesPerTrigger(32).ImagesPerFile(10).PhotonEnergy_keV(12.4).Compression(CompressionAlgorithm::BSHUF_ZSTD)
.FixedGainG1(true);
AcquisitionDeviceGroup aq_devices;
@@ -100,7 +100,7 @@ TEST_CASE("JFJochReceiverTest_Conversion_FixedGainG1_onlyG1", "[JFJochReceiver]"
x.Mode(DetectorMode::Conversion);
x.PedestalG0Frames(0).NumTriggers(1).UseInternalPacketGenerator(true)
.ImagesPerTrigger(32).DataFileCount(16).PhotonEnergy_keV(12.4).Compression(CompressionAlgorithm::BSHUF_ZSTD)
.ImagesPerTrigger(32).ImagesPerFile(10).PhotonEnergy_keV(12.4).Compression(CompressionAlgorithm::BSHUF_ZSTD)
.FixedGainG1(true);
std::vector<uint16_t> input(RAW_MODULE_SIZE*2, 0x4000 + 3000);
@@ -128,7 +128,7 @@ TEST_CASE("JFJochReceiverTest_Conversion_U16", "[JFJochReceiver]") {
x.Mode(DetectorMode::Conversion);
x.PedestalG0Frames(0).NumTriggers(1).UseInternalPacketGenerator(true)
.ImagesPerTrigger(32).DataFileCount(16).PhotonEnergy_keV(12.4).Compression(CompressionAlgorithm::BSHUF_ZSTD).FPGAOutputMode(FPGAPixelOutput::Uint16);
.ImagesPerTrigger(32).ImagesPerFile(10).PhotonEnergy_keV(12.4).Compression(CompressionAlgorithm::BSHUF_ZSTD).FPGAOutputMode(FPGAPixelOutput::Uint16);
REQUIRE(!x.IsPixelSigned());
AcquisitionDeviceGroup aq_devices;
@@ -155,7 +155,7 @@ TEST_CASE("JFJochReceiverTest_Conversion_I32", "[JFJochReceiver]") {
x.Mode(DetectorMode::Conversion);
x.PedestalG0Frames(0).NumTriggers(1).UseInternalPacketGenerator(true)
.ImagesPerTrigger(32).DataFileCount(16).PhotonEnergy_keV(12.4).Compression(CompressionAlgorithm::BSHUF_ZSTD).FPGAOutputMode(FPGAPixelOutput::Int32);
.ImagesPerTrigger(32).ImagesPerFile(10).PhotonEnergy_keV(12.4).Compression(CompressionAlgorithm::BSHUF_ZSTD).FPGAOutputMode(FPGAPixelOutput::Int32);
AcquisitionDeviceGroup aq_devices;
for (int i = 0; i < x.GetDataStreamsNum(); i++) {
@@ -181,7 +181,7 @@ TEST_CASE("JFJochReceiverTest_Conversion_Summation2", "[JFJochReceiver]") {
x.Mode(DetectorMode::Conversion);
x.PedestalG0Frames(0).NumTriggers(1).UseInternalPacketGenerator(true)
.ImagesPerTrigger(32).DataFileCount(16).PhotonEnergy_keV(12.4).Compression(CompressionAlgorithm::BSHUF_ZSTD).Summation(2);
.ImagesPerTrigger(32).ImagesPerFile(10).PhotonEnergy_keV(12.4).Compression(CompressionAlgorithm::BSHUF_ZSTD).Summation(2);
AcquisitionDeviceGroup aq_devices;
for (int i = 0; i < x.GetDataStreamsNum(); i++) {
@@ -207,7 +207,7 @@ TEST_CASE("JFJochReceiverTest_Conversion_StorageCell", "[JFJochReceiver]") {
x.Mode(DetectorMode::Conversion);
x.PedestalG0Frames(0).NumTriggers(1).UseInternalPacketGenerator(true)
.ImagesPerTrigger(32).DataFileCount(16).PhotonEnergy_keV(12.4).Compression(CompressionAlgorithm::BSHUF_ZSTD).StorageCells(16);
.ImagesPerTrigger(32).ImagesPerFile(10).PhotonEnergy_keV(12.4).Compression(CompressionAlgorithm::BSHUF_ZSTD).StorageCells(16);
REQUIRE(x.GetImageNum() == 16);
REQUIRE(x.GetStorageCellNumber() == 16);
@@ -587,7 +587,7 @@ TEST_CASE("JFJochReceiverTest_EIGER", "[JFJochReceiver]") {
x.Mode(DetectorMode::Raw);
x.PedestalG0Frames(0).NumTriggers(1).UseInternalPacketGenerator(true)
.ImagesPerTrigger(32).DataFileCount(16).PhotonEnergy_keV(12.4).Compression(CompressionAlgorithm::BSHUF_ZSTD);
.ImagesPerTrigger(32).ImagesPerFile(10).PhotonEnergy_keV(12.4).Compression(CompressionAlgorithm::BSHUF_ZSTD);
AcquisitionDeviceGroup aq_devices;
for (int i = 0; i < x.GetDataStreamsNum(); i++) {
@@ -613,7 +613,7 @@ TEST_CASE("JFJochReceiverTest_EIGER_conversion", "[JFJochReceiver]") {
x.Mode(DetectorMode::Conversion);
x.PedestalG0Frames(0).NumTriggers(1).UseInternalPacketGenerator(true)
.ImagesPerTrigger(32).DataFileCount(16).PhotonEnergy_keV(12.4).Compression(CompressionAlgorithm::BSHUF_ZSTD);
.ImagesPerTrigger(32).ImagesPerFile(10).PhotonEnergy_keV(12.4).Compression(CompressionAlgorithm::BSHUF_ZSTD);
AcquisitionDeviceGroup aq_devices;
for (int i = 0; i < x.GetDataStreamsNum(); i++) {

10
tests/JFJochReceiverProcessingTest.cpp
View File

@@ -19,7 +19,7 @@ TEST_CASE("JFJochIntegrationTest_ZMQ_lysozyme_spot_and_index", "[JFJochReceiver]
const uint16_t nthreads = 4;
DiffractionExperiment experiment(DetectorGeometry(8,2,8,36));
experiment.ImagesPerTrigger(5).NumTriggers(1).DataFileCount(1).UseInternalPacketGenerator(true)
experiment.ImagesPerTrigger(5).NumTriggers(1).UseInternalPacketGenerator(true)
.FilePrefix("lyso_test").ConversionOnFPGA(false)
.DetectorDistance_mm(75).BeamY_pxl(1136).BeamX_pxl(1090).PhotonEnergy_keV(12.4)
.SetUnitCell(UnitCell{.a = 36.9, .b = 78.95, .c = 78.95, .alpha =90, .beta = 90, .gamma = 90});
@@ -92,7 +92,7 @@ TEST_CASE("JFJochIntegrationTest_ZMQ_lysozyme_spot_and_index_min_pix_2", "[JFJoc
const uint16_t nthreads = 4;
DiffractionExperiment experiment(DetectorGeometry(8,2,8,36));
experiment.ImagesPerTrigger(5).NumTriggers(1).DataFileCount(1).UseInternalPacketGenerator(true)
experiment.ImagesPerTrigger(5).NumTriggers(1).UseInternalPacketGenerator(true)
.FilePrefix("lyso_test_min_pix_2").ConversionOnFPGA(false)
.DetectorDistance_mm(75).BeamY_pxl(1136).BeamX_pxl(1090).PhotonEnergy_keV(12.4)
.SetUnitCell(UnitCell{.a = 36.9, .b = 78.95, .c = 78.95, .alpha =90, .beta = 90, .gamma = 90});
@@ -158,7 +158,7 @@ TEST_CASE("JFJochIntegrationTest_ZMQ_lysozyme_spot_and_index_min_pix_2", "[JFJoc
TEST_CASE("GenerateResolutionMap") {
DiffractionExperiment experiment(DetectorGeometry(8,2,8,36));
experiment.ImagesPerTrigger(5).NumTriggers(1).DataFileCount(1).UseInternalPacketGenerator(true)
experiment.ImagesPerTrigger(5).NumTriggers(1).UseInternalPacketGenerator(true)
.FilePrefix("lyso_test").ConversionOnFPGA(false)
.DetectorDistance_mm(75).BeamY_pxl(1136).BeamX_pxl(1090).PhotonEnergy_keV(12.4)
.SetUnitCell(UnitCell{.a = 36.9, .b = 78.95, .c = 78.95, .alpha =90, .beta = 90, .gamma = 90});
@@ -187,7 +187,7 @@ TEST_CASE("JFJochIntegrationTest_ZMQ_lysozyme_resolution", "[JFJochReceiver]") {
const uint16_t nthreads = 4;
DiffractionExperiment experiment(DetectorGeometry(8,2,8,36));
experiment.ImagesPerTrigger(5).NumTriggers(1).DataFileCount(1).UseInternalPacketGenerator(true)
experiment.ImagesPerTrigger(5).NumTriggers(1).UseInternalPacketGenerator(true)
.FilePrefix("lyso_test_resolution").ConversionOnFPGA(false)
.DetectorDistance_mm(75).BeamY_pxl(1136).BeamX_pxl(1090).PhotonEnergy_keV(12.4)
.SetUnitCell(UnitCell{.a = 36.9, .b = 78.95, .c = 78.95, .alpha =90, .beta = 90, .gamma = 90})
@@ -251,7 +251,7 @@ TEST_CASE("JFJochIntegrationTest_ZMQ_ROI", "[JFJochReceiver]") {
const uint16_t nthreads = 4;
DiffractionExperiment experiment(DetectorGeometry(8,2,8,36));
experiment.ImagesPerTrigger(5).NumTriggers(1).DataFileCount(1).UseInternalPacketGenerator(true)
experiment.ImagesPerTrigger(5).NumTriggers(1).UseInternalPacketGenerator(true)
.FilePrefix("lyso_test").ConversionOnFPGA(false)
.DetectorDistance_mm(75).BeamY_pxl(1136).BeamX_pxl(1090).PhotonEnergy_keV(12.4)
.SetUnitCell(UnitCell{.a = 36.9, .b = 78.95, .c = 78.95, .alpha =90, .beta = 90, .gamma = 90});

4
tests/JPEGTest.cpp
View File

@@ -26,7 +26,7 @@ TEST_CASE("JPEGTest","[JPEG]") {
TEST_CASE("PreviewImage_GenerateJPEG","[JPEG]") {
DiffractionExperiment experiment(DetectorGeometry(8,2,8,36));
experiment.ImagesPerTrigger(5).NumTriggers(1).DataFileCount(1).UseInternalPacketGenerator(true)
experiment.ImagesPerTrigger(5).NumTriggers(1).UseInternalPacketGenerator(true)
.FilePrefix("lyso_test_min_pix_2").ConversionOnFPGA(false)
.DetectorDistance_mm(75).BeamY_pxl(1136).BeamX_pxl(1090).PhotonEnergy_keV(12.4)
.SetUnitCell(UnitCell{.a = 36.9, .b = 78.95, .c = 78.95, .alpha =90, .beta = 90, .gamma = 90});
@@ -74,7 +74,7 @@ TEST_CASE("PreviewImage_GenerateJPEG","[JPEG]") {
TEST_CASE("PreviewImage_GenerateJPEG_ROI","[JPEG]") {
DiffractionExperiment experiment(DetectorGeometry(8,2,8,36));
experiment.ImagesPerTrigger(5).NumTriggers(1).DataFileCount(1).UseInternalPacketGenerator(true)
experiment.ImagesPerTrigger(5).NumTriggers(1).UseInternalPacketGenerator(true)
.FilePrefix("lyso_test_min_pix_2").ConversionOnFPGA(false)
.DetectorDistance_mm(75).BeamY_pxl(1136).BeamX_pxl(1090).PhotonEnergy_keV(12.4)
.SetUnitCell(UnitCell{.a = 36.9, .b = 78.95, .c = 78.95, .alpha =90, .beta = 90, .gamma = 90});

33
tests/LossyFilterTest.cpp
View File

@@ -11,17 +11,9 @@ TEST_CASE("LossyFilterDisabled","[LossyFilter]") {
.indexing_result = 0
};
filter.ApplyFilter(message);
REQUIRE(message.number == 123);
REQUIRE(message.indexing_result == 0);
filter.ApplyFilter(message);
REQUIRE(message.number == 123);
REQUIRE(message.indexing_result == 0);
filter.ApplyFilter(message);
REQUIRE(message.number == 123);
REQUIRE(message.indexing_result == 0);
REQUIRE(filter.ApplyFilter(message));
REQUIRE(filter.ApplyFilter(message));
REQUIRE(filter.ApplyFilter(message));
}
TEST_CASE("LossyFilterEnable","[LossyFilter]") {
@@ -37,17 +29,10 @@ TEST_CASE("LossyFilterEnable","[LossyFilter]") {
.indexing_result = 0
};
filter.ApplyFilter(message_1);
REQUIRE(message_1.number == 0);
filter.ApplyFilter(message_2);
REQUIRE(message_2.number == -1);
filter.ApplyFilter(message_1);
REQUIRE(message_1.number == 1);
filter.ApplyFilter(message_1);
REQUIRE(message_1.number == 2);
REQUIRE(filter.ApplyFilter(message_1));
REQUIRE(!filter.ApplyFilter(message_2));
REQUIRE(filter.ApplyFilter(message_1));
REQUIRE(filter.ApplyFilter(message_1));
}
TEST_CASE("LossyFilterEnable_Prob","[LossyFilter]") {
@@ -60,10 +45,8 @@ TEST_CASE("LossyFilterEnable_Prob","[LossyFilter]") {
.number = 124,
.indexing_result = 0
};
filter.ApplyFilter(message);
accepted += filter.ApplyFilter(message);
count++;
if (message.number < 0)
accepted++;
}
double frac = static_cast<double>(accepted) / static_cast<double>(count);
REQUIRE(frac < 0.51);

7
tests/StrongPixelSetTest.cpp
View File

@@ -18,12 +18,6 @@ TEST_CASE("DiffractionSpot_AddOperator","[StrongPixelSet]") {
REQUIRE(spot1.PixelCount() == 2);
}
TEST_CASE("StrongPixelSet_coord2uint64_t","[StrongPixelSet]") {
uint32_t val = strong_pixel_coord(15,6667);
REQUIRE(col_from_strong_pixel(val) == 15);
REQUIRE(line_from_strong_pixel(val) == 6667);
}
TEST_CASE("StrongPixelSet_BuildSpots","[StrongPixelSet]") {
DiffractionExperiment experiment(DetectorGeometry(1,1,0,0,false));
experiment.Mode(DetectorMode::Raw);
@@ -46,7 +40,6 @@ TEST_CASE("StrongPixelSet_BuildSpots","[StrongPixelSet]") {
strong_pixel_set.AddStrongPixel(8,105);
strong_pixel_set.AddStrongPixel(8,104);
strong_pixel_set.FindSpots(experiment, settings, spots, 0);
REQUIRE(strong_pixel_set.Count() == 0);
REQUIRE(spots.size() == 1);
REQUIRE(spots[0].Count() == 9.0f);

4
tests/TIFFTest.cpp
View File

@@ -34,7 +34,7 @@ TEST_CASE("TIFFTest_File_signed","[TIFF]") {
TEST_CASE("PreviewImage_GenerateTIFF","[TIFF]") {
DiffractionExperiment experiment(DetectorGeometry(8,2,8,36));
experiment.ImagesPerTrigger(5).NumTriggers(1).DataFileCount(1).UseInternalPacketGenerator(true)
experiment.ImagesPerTrigger(5).NumTriggers(1).UseInternalPacketGenerator(true)
.FilePrefix("lyso_test_min_pix_2").ConversionOnFPGA(false)
.DetectorDistance_mm(75).BeamY_pxl(1136).BeamX_pxl(1090).PhotonEnergy_keV(12.4)
.SetUnitCell(UnitCell{.a = 36.9, .b = 78.95, .c = 78.95, .alpha =90, .beta = 90, .gamma = 90});
@@ -70,7 +70,7 @@ TEST_CASE("PreviewImage_GenerateTIFF","[TIFF]") {
TEST_CASE("PreviewImage_GenerateTIFFDioptas","[TIFF]") {
DiffractionExperiment experiment(DetectorGeometry(8,2,8,36));
experiment.ImagesPerTrigger(5).NumTriggers(1).DataFileCount(1).UseInternalPacketGenerator(true)
experiment.ImagesPerTrigger(5).NumTriggers(1).UseInternalPacketGenerator(true)
.FilePrefix("lyso_test_min_pix_2").ConversionOnFPGA(false)
.DetectorDistance_mm(75).BeamY_pxl(1136).BeamX_pxl(1090).PhotonEnergy_keV(12.4)
.SetUnitCell(UnitCell{.a = 36.9, .b = 78.95, .c = 78.95, .alpha =90, .beta = 90, .gamma = 90});

9
tests/ZMQImagePusherTest.cpp
View File

@@ -23,7 +23,7 @@ void test_puller(ZMQImagePuller *puller,
while (puller->GetFrameType() != CBORStream2Deserializer::Type::END) {
if (puller->GetFrameType() == CBORStream2Deserializer::Type::IMAGE) {
auto image = puller->GetDataMessage();
if ((nwriter > 1) && (image.number % nwriter != writer_id))
if ((nwriter > 1) && ((image.number / 16) % nwriter != writer_id))
diff_split[writer_id]++;
@@ -85,7 +85,7 @@ TEST_CASE("ZMQImageCommTest_1Writer","[ZeroMQ]") {
std::thread sender_thread = std::thread([&] {
StartMessage message {
.data_file_count = 16
.images_per_file = 16
};
EndMessage end_message{
.write_master_file = true
@@ -161,7 +161,7 @@ TEST_CASE("ZMQImageCommTest_2Writers","[ZeroMQ]") {
std::thread sender_thread = std::thread([&] {
StartMessage message {
.data_file_count = 16
.images_per_file = 16
};
EndMessage end_message{
.write_master_file = true
@@ -247,7 +247,7 @@ TEST_CASE("ZMQImageCommTest_4Writers","[ZeroMQ]") {
std::thread sender_thread = std::thread([&] {
StartMessage message {
.data_file_count = 16
.images_per_file = 16
};
EndMessage end_message{
.write_master_file = true
@@ -317,7 +317,6 @@ TEST_CASE("ZMQImagePuller_abort","[ZeroMQ]") {
TEST_CASE("ZMQImageCommTest_NoWriter","[ZeroMQ]") {
ZMQStream2PusherGroup pusher({"ipc://*"});
StartMessage msg;
msg.data_file_count = 1;
REQUIRE_THROWS(pusher.StartDataCollection(msg));
std::vector<uint8_t> test(512*1024, 11);

3
tools/HDF5DatasetWriteTest.cpp
View File

@@ -32,7 +32,7 @@ int main(int argc, char **argv) {
std::chrono::microseconds period_us((rate == 0) ? 0 : (int64_t) (1.0e6 / rate));
HDF5File data(argv[1], false, false);
HDF5ReadOnlyFile data(argv[1]);
HDF5DataSet dataset(data, "/entry/data/data");
HDF5DataSpace file_space(dataset);
@@ -48,7 +48,6 @@ int main(int argc, char **argv) {
x.BeamX_pxl(1090).BeamY_pxl(1136).DetectorDistance_mm(75).PhotonEnergy_keV(WVL_1A_IN_KEV);
x.MaskModuleEdges(true);
x.MaskChipEdges(true);
x.DataFileCount(4);
if ((file_space.GetDimensions()[1] == 2164) && (file_space.GetDimensions()[2] == 2068)) {
std::cout << "JF4M with gaps detected (2068 x 2164)" << std::endl;

27
tools/SpotFindingPerformanceTest.cpp
View File

@@ -6,12 +6,17 @@
int main() {
Logger logger("SpotFindingPerformanceTest");
size_t nstrong_pixel = 2048;
size_t nstrong_pixel = 4000;
std::vector<uint8_t> strong_pixel(RAW_MODULE_SIZE, 0);
std::mt19937 g1(67678);
std::uniform_int_distribution<size_t> distribution(0, RAW_MODULE_SIZE - 1);
DeviceOutput dev_output;
dev_output.spot_finding_result.strong_pixel_count = nstrong_pixel;
for (auto &i: dev_output.spot_finding_result.strong_pixel)
i = 0;
// Can think of smarter way to select pixels
int pixels_set = 0;
@@ -19,25 +24,11 @@ int main() {
size_t pixel = distribution(g1);
if (strong_pixel[pixel] == 0) {
strong_pixel[pixel] = 1;
dev_output.spot_finding_result.strong_pixel[pixel / 8] |= (1 << (pixel % 8));
pixels_set++;
}
}
DeviceOutput dev_output;
dev_output.spot_finding_result.strong_pixel_count = nstrong_pixel;
dev_output.spot_finding_result.max_memory_index = nstrong_pixel/32;
for (auto &i: dev_output.spot_finding_result.strong_pixel_number)
i = UINT32_MAX;
size_t curr_id = 0;
for (int i = 0; i < RAW_MODULE_SIZE; i++) {
if (strong_pixel[i]) {
dev_output.spot_finding_result.strong_pixel_number[curr_id] = i;
curr_id++;
}
}
DiffractionExperiment experiment(DetectorGeometry(18, 3, 8, 36));
experiment.DetectorDistance_mm(50.0).BeamX_pxl(1200).BeamY_pxl(1200);
@@ -48,7 +39,7 @@ int main() {
.low_resolution_limit = 100.0
};
int iterations = 2000;
int iterations = 200;
{
int tmp = 0;
auto start_time = std::chrono::system_clock::now();
@@ -56,7 +47,7 @@ int main() {
std::vector<DiffractionSpot> spots, spots_out;
for (int m = 0; m < experiment.GetModulesNum(); m++) {
StrongPixelSet strong_pixel_set;
strong_pixel_set.ReadFPGAOutput(dev_output);
strong_pixel_set.ReadFPGAOutput(experiment, dev_output);
strong_pixel_set.FindSpots(experiment, settings, spots, m);
}
FilterSpotsByCount(experiment, spots, spots_out);

4
tools/jfjoch_writer_test.cpp
View File

@@ -28,7 +28,7 @@ int main(int argc, char **argv) {
DiffractionExperiment x(DetectorGeometry(8, 2, 8, 36));
x.Summation(1).ImagesPerTrigger(nimages_out).Mode(DetectorMode::Conversion);
HDF5File data(argv[1], false, false);
HDF5ReadOnlyFile data(argv[1]);
HDF5DataSet dataset(data, "/entry/data/data");
HDF5DataSpace file_space(dataset);
@@ -54,8 +54,6 @@ int main(int argc, char **argv) {
for (int i = 0; i < nsockets; i++)
zmq_addr.emplace_back("tcp://0.0.0.0:" + std::to_string(BASE_TCP_PORT + i));
x.DataFileCount(nsockets);
ZMQStream2PusherGroup pusher(context, zmq_addr);
FrameTransformation transformation(x);

4
writer/CMakeLists.txt
View File

@@ -22,7 +22,9 @@ ADD_LIBRARY(JFJochWriter STATIC
HDF5DataFilePluginAzInt.cpp
HDF5DataFilePluginAzInt.h
HDF5DataFilePluginJUNGFRAU.cpp
HDF5DataFilePluginJUNGFRAU.h)
HDF5DataFilePluginJUNGFRAU.h
HDF5DataFilePluginResEstimation.cpp
HDF5DataFilePluginResEstimation.h)
TARGET_LINK_LIBRARIES(JFJochWriter JFJochHDF5Wrappers JFJochCommon CBORStream2FrameSerialize)

85
writer/HDF5DataFile.cpp
View File

@@ -4,6 +4,12 @@
#include "HDF5DataFile.h"
#include "../compression/JFJochCompressor.h"
#include "HDF5DataFilePluginAzInt.h"
#include "HDF5DataFilePluginMX.h"
#include "HDF5DataFilePluginXFEL.h"
#include "HDF5DataFilePluginJUNGFRAU.h"
#include "HDF5DataFilePluginResEstimation.h"
HDF5DataFile::HDF5DataFile(const std::string &in_filename,
const std::vector<float>& in_rad_int_bin_to_q,
size_t in_max_spots) {
@@ -16,25 +22,52 @@ HDF5DataFile::HDF5DataFile(const std::string &in_filename,
plugins.emplace_back(std::make_unique<HDF5DataFilePluginJUNGFRAU>());
plugins.emplace_back(std::make_unique<HDF5DataFilePluginAzInt>(in_rad_int_bin_to_q));
plugins.emplace_back(std::make_unique<HDF5DataFilePluginXFEL>());
plugins.emplace_back(std::make_unique<HDF5DataFilePluginResEstimation>());
plugins.emplace_back(std::make_unique<HDF5DataFilePluginMX>(in_max_spots));
}
HDF5DataFile::~HDF5DataFile() {
if (data_file) {
HDF5Group group_exp(*data_file, "/entry/detector");
group_exp.NXClass("NXcollection");
group_exp.SaveVector("timestamp", timestamp);
group_exp.SaveVector("exptime", exptime);
group_exp.SaveVector("number", number);
for (auto &p: plugins)
p->WriteFinal(*data_file);
data_file.reset();
std::string old_filename = filename + ".tmp";
std::rename(old_filename.c_str(), filename.c_str());
}
}
void HDF5DataFile::CreateFile(const DataMessage& msg) {
SetupSWMRFile(msg);
HDF5Dcpl dcpl;
HDF5DataType data_type(msg.image.pixel_depth_bytes, msg.image.pixel_is_signed);
xpixel = msg.image.xpixel;
ypixel = msg.image.ypixel;
dcpl.SetCompression(msg.image.algorithm, msg.image.pixel_depth_bytes, JFJochBitShuffleCompressor::DefaultBlockSize);
dcpl.SetChunking( {1, ypixel, xpixel});
if (msg.image.pixel_is_signed) {
if (msg.image.pixel_depth_bytes == 2)
dcpl.SetFillValue16(INT16_MIN);
else
dcpl.SetFillValue32(INT32_MIN);
}
data_file = std::make_unique<HDF5File>(filename + ".tmp");
chmod(filename.c_str(), S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); // default permissions
HDF5Group(*data_file, "/entry").NXClass("NXentry");
HDF5Group(*data_file, "/entry/data").NXClass("NXdata");
HDF5DataSpace data_space({1, ypixel, xpixel}, {H5S_UNLIMITED, ypixel, xpixel});
data_set = std::make_unique<HDF5DataSet>(*data_file, "/entry/data/data", data_type, data_space, dcpl);
data_file = std::make_unique<HDF5File>(filename, false, true);
data_set = std::make_unique<HDF5DataSet>(*data_file, "/entry/data/data");
for (auto &p: plugins)
p->OpenFile(*data_file, msg);
}
@@ -53,6 +86,7 @@ void HDF5DataFile::Write(const DataMessage &msg, uint64_t image_number) {
data_set->SetExtent({max_image_number+1, ypixel, xpixel});
timestamp.resize(max_image_number + 1);
exptime.resize(max_image_number + 1);
number.resize(max_image_number + 1);
}
nimages++;
@@ -63,17 +97,7 @@ void HDF5DataFile::Write(const DataMessage &msg, uint64_t image_number) {
timestamp[image_number] = msg.timestamp;
exptime[image_number] = msg.exptime;
auto now_time = std::chrono::system_clock::now();
auto time_diff = now_time - (last_flush + swmr_flush_period);
if (time_diff.count() >= 0) {
data_set->Flush();
for (auto &p: plugins)
p->Flush();
last_flush = now_time;
}
number[image_number] = msg.number;
}
HDF5DataFileStatistics HDF5DataFile::GetStatistics() const {
@@ -87,34 +111,3 @@ HDF5DataFileStatistics HDF5DataFile::GetStatistics() const {
size_t HDF5DataFile::GetNumImages() const {
return nimages;
}
void HDF5DataFile::SetupSWMRFile(const DataMessage& msg) {
HDF5Dcpl dcpl;
HDF5DataType data_type(msg.image.pixel_depth_bytes, msg.image.pixel_is_signed);
xpixel = msg.image.xpixel;
ypixel = msg.image.ypixel;
dcpl.SetCompression(msg.image.algorithm, msg.image.pixel_depth_bytes, JFJochBitShuffleCompressor::DefaultBlockSize);
dcpl.SetChunking( {1, ypixel, xpixel});
if (msg.image.pixel_is_signed) {
if (msg.image.pixel_depth_bytes == 2)
dcpl.SetFillValue16(INT16_MIN);
else
dcpl.SetFillValue32(INT32_MIN);
}
HDF5File data_file_local(filename, true, true);
chmod(filename.c_str(), S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); // default permissions
HDF5Group(data_file_local, "/entry").NXClass("NXentry");
HDF5Group(data_file_local, "/entry/data").NXClass("NXdata");
HDF5DataSpace data_space({1, ypixel, xpixel}, {H5S_UNLIMITED, ypixel, xpixel});
HDF5DataSet(data_file_local, "/entry/data/data", data_type, data_space, dcpl);
for (auto &p: plugins)
p->SetupSWMRFile(data_file_local, msg);
last_flush = std::chrono::system_clock::now();
}

12
writer/HDF5DataFile.h
View File

@@ -11,10 +11,7 @@
#include "../common/SpotToSave.h"
#include "../frame_serialize/JFJochMessages.h"
#include "HDF5DataFilePluginAzInt.h"
#include "HDF5DataFilePluginMX.h"
#include "HDF5DataFilePluginXFEL.h"
#include "HDF5DataFilePluginJUNGFRAU.h"
#include "HDF5DataFilePlugin.h"
struct HDF5DataFileStatistics {
std::string filename;
@@ -28,7 +25,7 @@ class HDF5DataFile {
std::unique_ptr<HDF5File> data_file = nullptr;
std::unique_ptr<HDF5DataSet> data_set = nullptr;
std::unique_ptr<HDF5DataSet> data_set_image_number = nullptr;
std::vector<std::unique_ptr<HDF5DataFilePlugin>> plugins;
size_t xpixel;
@@ -40,12 +37,9 @@ class HDF5DataFile {
std::vector<uint64_t> timestamp;
std::vector<uint32_t> exptime;
std::chrono::time_point<std::chrono::system_clock> last_flush;
std::chrono::microseconds swmr_flush_period = std::chrono::seconds(5);
std::vector<uint64_t> number;
void CreateFile(const DataMessage& msg);
void SetupSWMRFile(const DataMessage& msg);
public:
HDF5DataFile(const std::string& name, const std::vector<float>& rad_int_bin_to_q, size_t max_spots = 0);
~HDF5DataFile();

3
writer/HDF5DataFilePlugin.h
View File

@@ -10,10 +10,9 @@ class HDF5DataFilePlugin {
protected:
size_t max_image_number = 0;
public:
virtual void SetupSWMRFile(HDF5File &data_file, const DataMessage& msg) = 0;
virtual void OpenFile(HDF5File &data_file, const DataMessage& msg) = 0;
virtual void Write(const DataMessage& msg) = 0;
virtual void Flush() = 0;
virtual void WriteFinal(HDF5File &data_file) = 0;
virtual ~HDF5DataFilePlugin() = default;
};

17
writer/HDF5DataFilePluginAzInt.cpp
View File

@@ -5,7 +5,7 @@
HDF5DataFilePluginAzInt::HDF5DataFilePluginAzInt(const std::vector<float> &in_rad_int_bin_to_q) :
rad_int_bin_to_q(in_rad_int_bin_to_q) {}
void HDF5DataFilePluginAzInt::SetupSWMRFile(HDF5File &data_file, const DataMessage &msg) {
void HDF5DataFilePluginAzInt::OpenFile(HDF5File &data_file, const DataMessage &msg) {
if (rad_int_bin_to_q.empty())
return;
@@ -16,13 +16,9 @@ void HDF5DataFilePluginAzInt::SetupSWMRFile(HDF5File &data_file, const DataMessa
HDF5DataSpace data_space_rad_int({1, rad_int_bin_to_q.size()}, {H5S_UNLIMITED, rad_int_bin_to_q.size()});
HDF5Dcpl dcpl_rad_int;
dcpl_rad_int.SetChunking({1, rad_int_bin_to_q.size()});
HDF5DataSet(data_file, "/entry/azint/result", HDF5DataType(0.0f), data_space_rad_int, dcpl_rad_int);
}
void HDF5DataFilePluginAzInt::OpenFile(HDF5File &data_file, const DataMessage &msg) {
if (rad_int_bin_to_q.empty())
return;
data_set_az_int = std::make_unique<HDF5DataSet>(data_file, "/entry/azint/result");
data_set_az_int = std::make_unique<HDF5DataSet>(data_file, "/entry/azint/result", HDF5DataType(0.0f),
data_space_rad_int, dcpl_rad_int);
}
void HDF5DataFilePluginAzInt::Write(const DataMessage &msg) {
@@ -40,9 +36,4 @@ void HDF5DataFilePluginAzInt::Write(const DataMessage &msg) {
data_set_az_int->WriteVec(msg.az_int_profile, {image_number, 0}, {1, rad_int_bin_to_q.size()});
}
void HDF5DataFilePluginAzInt::Flush() {
if (rad_int_bin_to_q.empty())
return;
data_set_az_int->Flush();
}
void HDF5DataFilePluginAzInt::WriteFinal(HDF5File &data_file) {}

3
writer/HDF5DataFilePluginAzInt.h
View File

@@ -11,10 +11,9 @@ class HDF5DataFilePluginAzInt : public HDF5DataFilePlugin {
public:
explicit HDF5DataFilePluginAzInt(const std::vector<float> &rad_int_bin_to_q);
void SetupSWMRFile(HDF5File &data_file, const DataMessage& msg) override;
void OpenFile(HDF5File &data_file, const DataMessage& msg) override;
void Write(const DataMessage& msg) override;
void Flush() override;
void WriteFinal(HDF5File &data_file) override;
};
#endif //JUNGFRAUJOCH_HDF5DATAFILEPLUGINAZINT_H

58
writer/HDF5DataFilePluginJUNGFRAU.cpp
View File

@@ -2,57 +2,33 @@
#include "HDF5DataFilePluginJUNGFRAU.h"
void HDF5DataFilePluginJUNGFRAU::SetupSWMRFile(HDF5File &data_file, const DataMessage &msg) {
if (msg.jf_info.has_value() && msg.storage_cell.has_value() && msg.receiver_aq_dev_delay.has_value())
enable = true;
else
return;
HDF5Group(data_file, "/entry/jungfrau").NXClass("NXcollection");
HDF5DataSpace data_space_linear({2}, {H5S_UNLIMITED});
HDF5Dcpl dcpl_linear;
dcpl_linear.SetChunking({1});
HDF5DataSet(data_file, "/entry/jungfrau/debug", HDF5DataType(4, false), data_space_linear, dcpl_linear).SetExtent({1});
HDF5DataSet(data_file, "/entry/jungfrau/storageCell", HDF5DataType(2, false), data_space_linear, dcpl_linear).SetExtent({1});
HDF5DataSet(data_file, "/entry/jungfrau/receiverDelay", HDF5DataType(4, false), data_space_linear, dcpl_linear).SetExtent({1});
}
void HDF5DataFilePluginJUNGFRAU::OpenFile(HDF5File &data_file, const DataMessage &msg) {
if (!enable)
return;
data_set_jf_info = std::make_unique<HDF5DataSet>(data_file, "/entry/jungfrau/debug");
data_set_receiver_aq_dev_delay = std::make_unique<HDF5DataSet>(data_file, "/entry/jungfrau/receiverDelay");
data_set_storage_cell = std::make_unique<HDF5DataSet>(data_file, "/entry/jungfrau/storageCell");
}
void HDF5DataFilePluginJUNGFRAU::OpenFile(HDF5File &in_data_file, const DataMessage &msg) {}
void HDF5DataFilePluginJUNGFRAU::Write(const DataMessage &msg) {
if (!enable
|| !msg.jf_info.has_value()
if (!msg.jf_info.has_value()
|| !msg.receiver_aq_dev_delay.has_value()
|| !msg.storage_cell.has_value())
return;
size_t image_number = msg.number;
if (image_number >= max_image_number) {
max_image_number = image_number;
data_set_jf_info->SetExtent({max_image_number + 1});
data_set_receiver_aq_dev_delay->SetExtent({max_image_number + 1});
data_set_storage_cell->SetExtent({max_image_number + 1});
if (image_number >= jf_info.size()) {
jf_info.resize(image_number + 1);
storage_cell.resize(image_number + 1);
receiver_aq_dev_delay.resize(image_number + 1);
}
data_set_jf_info->WriteScalar(msg.jf_info.value(), {image_number});
data_set_receiver_aq_dev_delay->WriteScalar(msg.receiver_aq_dev_delay.value(), {image_number});
data_set_storage_cell->WriteScalar(msg.storage_cell.value(), {image_number});
receiver_aq_dev_delay[image_number] = msg.receiver_aq_dev_delay.value();
jf_info[image_number] = msg.jf_info.value();
storage_cell[image_number]= msg.storage_cell.value();
}
void HDF5DataFilePluginJUNGFRAU::Flush() {
if (!enable)
return;
data_set_jf_info->Flush();
data_set_receiver_aq_dev_delay->Flush();
data_set_storage_cell->Flush();
void HDF5DataFilePluginJUNGFRAU::WriteFinal(HDF5File &data_file) {
if (!jf_info.empty()) {
HDF5Group group(data_file, "/entry/jungfrau");
group.NXClass("NXcollection");
group.SaveVector("info", jf_info);
group.SaveVector("rcv_delay", receiver_aq_dev_delay);
group.SaveVector("storage_cell", storage_cell);
}
}

12
writer/HDF5DataFilePluginJUNGFRAU.h
View File

@@ -6,17 +6,13 @@
#include "HDF5DataFilePlugin.h"
class HDF5DataFilePluginJUNGFRAU : public HDF5DataFilePlugin {
bool enable = false;
std::unique_ptr<HDF5DataSet> data_set_jf_info = nullptr;
std::unique_ptr<HDF5DataSet> data_set_storage_cell = nullptr;
std::unique_ptr<HDF5DataSet> data_set_receiver_aq_dev_delay = nullptr;
std::vector<uint64_t> jf_info;
std::vector<uint32_t> storage_cell;
std::vector<uint64_t> receiver_aq_dev_delay;
public:
void SetupSWMRFile(HDF5File &data_file, const DataMessage& msg) override;
void OpenFile(HDF5File &data_file, const DataMessage& msg) override;
void Write(const DataMessage& msg) override;
void Flush() override;
void WriteFinal(HDF5File &data_file) override;
};
#endif //JUNGFRAUJOCH_HDF5DATAFILEPLUGINJUNGFRAU_H

103
writer/HDF5DataFilePluginMX.cpp
View File

@@ -2,7 +2,9 @@
#include "HDF5DataFilePluginMX.h"
void HDF5DataFilePluginMX::SetupSWMRFile(HDF5File &data_file, const DataMessage &msg) {
HDF5DataFilePluginMX::HDF5DataFilePluginMX(size_t in_max_spots) : max_spots(in_max_spots) {}
void HDF5DataFilePluginMX::OpenFile(HDF5File &data_file, const DataMessage &msg) {
if (max_spots == 0)
return;
@@ -11,45 +13,14 @@ void HDF5DataFilePluginMX::SetupSWMRFile(HDF5File &data_file, const DataMessage
HDF5DataSpace data_space_spots({1, max_spots}, {H5S_UNLIMITED, max_spots});
HDF5Dcpl dcpl_spots;
dcpl_spots.SetChunking({1, max_spots});
HDF5DataSet(data_file, "/entry/MX/peakXPosRaw", HDF5DataType(0.0f), data_space_spots, dcpl_spots);
HDF5DataSet(data_file, "/entry/MX/peakYPosRaw", HDF5DataType(0.0f), data_space_spots, dcpl_spots);
HDF5DataSet(data_file, "/entry/MX/peakTotalIntensity", HDF5DataType(0.0f), data_space_spots, dcpl_spots);
HDF5DataSet(data_file, "/entry/MX/peakIndexed", HDF5DataType(1, false), data_space_spots, dcpl_spots);
HDF5DataSpace data_space_linear({2}, {H5S_UNLIMITED});
HDF5Dcpl dcpl_linear;
dcpl_linear.SetChunking({1});
HDF5DataSet(data_file, "/entry/MX/nPeaks", HDF5DataType(4, false), data_space_linear, dcpl_linear).SetExtent({1});
HDF5DataSet(data_file, "/entry/MX/strongPixels", HDF5DataType(4, false), data_space_linear, dcpl_linear).SetExtent({1});
HDF5DataSet(data_file, "/entry/MX/imageIndexed", HDF5DataType(1, false), data_space_linear, dcpl_linear).SetExtent({1});
if (msg.resolution_estimation) {
res_estimation = true;
HDF5DataSet(data_file, "/entry/MX/resolutionEstimation", HDF5DataType(0.0f), data_space_linear,
dcpl_linear).SetExtent({1});
}
HDF5DataSpace data_space_lattice({1, 9}, {H5S_UNLIMITED, 9});
HDF5Dcpl dcpl_lattice;
dcpl_lattice.SetChunking({1, 9});
HDF5DataSet(data_file, "/entry/MX/latticeIndexed", HDF5DataType(0.0f), data_space_lattice, dcpl_lattice);
}
void HDF5DataFilePluginMX::OpenFile(HDF5File &data_file, const DataMessage &msg) {
if (max_spots == 0)
return;
data_set_spot_x = std::make_unique<HDF5DataSet>(data_file, "/entry/MX/peakXPosRaw");
data_set_spot_y = std::make_unique<HDF5DataSet>(data_file, "/entry/MX/peakYPosRaw");
data_set_spot_int = std::make_unique<HDF5DataSet>(data_file, "/entry/MX/peakTotalIntensity");
data_set_spot_indexed = std::make_unique<HDF5DataSet>(data_file, "/entry/MX/peakIndexed");
data_set_spot_npeaks = std::make_unique<HDF5DataSet>(data_file, "/entry/MX/nPeaks");
data_set_strong_pixel_count = std::make_unique<HDF5DataSet>(data_file, "/entry/MX/strongPixels");
data_set_indexed = std::make_unique<HDF5DataSet>(data_file, "/entry/MX/imageIndexed");
data_set_indexed_lattice = std::make_unique<HDF5DataSet>(data_file, "/entry/MX/latticeIndexed");
if (res_estimation)
data_set_resolution_estimation = std::make_unique<HDF5DataSet>(data_file, "/entry/MX/resolutionEstimation");
data_set_spot_x = std::make_unique<HDF5DataSet>(data_file, "/entry/MX/peakXPosRaw", HDF5DataType(0.0f),
data_space_spots, dcpl_spots);
data_set_spot_y = std::make_unique<HDF5DataSet>(data_file, "/entry/MX/peakYPosRaw", HDF5DataType(0.0f),
data_space_spots, dcpl_spots);
data_set_spot_int = std::make_unique<HDF5DataSet>(data_file, "/entry/MX/peakTotalIntensity", HDF5DataType(0.0f),
data_space_spots, dcpl_spots);
data_set_spot_indexed = std::make_unique<HDF5DataSet>(data_file, "/entry/MX/peakIndexed", HDF5DataType(1, false),
data_space_spots, dcpl_spots);
}
void HDF5DataFilePluginMX::Write(const DataMessage &msg) {
@@ -64,13 +35,11 @@ void HDF5DataFilePluginMX::Write(const DataMessage &msg) {
data_set_spot_y->SetExtent({max_image_number+1, max_spots});
data_set_spot_int->SetExtent({max_image_number+1, max_spots});
data_set_spot_indexed->SetExtent({max_image_number+1, max_spots});
data_set_spot_npeaks->SetExtent({max_image_number+1});
data_set_strong_pixel_count->SetExtent({max_image_number + 1});
data_set_indexed->SetExtent({max_image_number + 1});
data_set_indexed_lattice->SetExtent({max_image_number + 1, 9});
if (res_estimation)
data_set_resolution_estimation->SetExtent({max_image_number + 1});
npeaks.resize(max_image_number + 1);
strong_pixel_count.resize(max_image_number + 1);
indexed.resize(max_image_number + 1);
indexed_lattice.resize((max_image_number + 1) * 9);
}
std::vector<float> spot_x(max_spots), spot_y(max_spots), spot_intensity(max_spots);
@@ -88,33 +57,23 @@ void HDF5DataFilePluginMX::Write(const DataMessage &msg) {
data_set_spot_y->WriteVec(spot_y, {image_number, 0}, {1, max_spots});
data_set_spot_int->WriteVec(spot_intensity, {image_number, 0}, {1, max_spots});
data_set_spot_indexed->WriteVec(spot_indexed, {image_number, 0}, {1, max_spots});
data_set_spot_npeaks->WriteScalar(spot_cnt, {image_number});
data_set_strong_pixel_count->WriteScalar(static_cast<uint32_t>(msg.strong_pixel_count), {image_number});
data_set_indexed->WriteScalar((uint8_t) msg.indexing_result, {image_number});
if (msg.indexing_lattice.size() == 9)
data_set_indexed_lattice->WriteVec(msg.indexing_lattice, {image_number, 0}, {1, 9});
if (res_estimation)
data_set_resolution_estimation->WriteScalar(msg.resolution_estimation.value(), {image_number});
npeaks[image_number] = spot_cnt;
strong_pixel_count[image_number] = msg.strong_pixel_count;
indexed[image_number] = msg.indexing_result;
if (msg.indexing_lattice.size() == 9) {
for (int i = 0; i < 9; i++)
indexed_lattice[image_number * 9 + i] = msg.indexing_lattice[i];
}
}
void HDF5DataFilePluginMX::Flush() {
if (max_spots == 0)
return;
data_set_spot_x->Flush();
data_set_spot_y->Flush();
data_set_spot_int->Flush();
data_set_spot_indexed->Flush();
data_set_spot_npeaks->Flush();
data_set_strong_pixel_count->Flush();
data_set_indexed->Flush();
data_set_indexed_lattice->Flush();
if (res_estimation)
data_set_resolution_estimation->Flush();
}
HDF5DataFilePluginMX::HDF5DataFilePluginMX(size_t in_max_spots) : max_spots(in_max_spots) {
void HDF5DataFilePluginMX::WriteFinal(HDF5File &data_file) {
if (!npeaks.empty())
data_file.SaveVector("/entry/MX/nPeaks", npeaks);
if (!strong_pixel_count.empty())
data_file.SaveVector("/entry/MX/strongPixels", strong_pixel_count);
if (!indexed.empty())
data_file.SaveVector("/entry/MX/imageIndexed", indexed);
if (!indexed_lattice.empty())
data_file.SaveVector("/entry/MX/latticeIndexed", indexed_lattice, {(hsize_t) (max_image_number + 1), 9});
}

17
writer/HDF5DataFilePluginMX.h
View File

@@ -8,29 +8,22 @@
class HDF5DataFilePluginMX : public HDF5DataFilePlugin {
size_t max_spots;
bool res_estimation = false;
// spots
std::unique_ptr<HDF5DataSet> data_set_spot_x = nullptr;
std::unique_ptr<HDF5DataSet> data_set_spot_y = nullptr;
std::unique_ptr<HDF5DataSet> data_set_spot_int = nullptr;
std::unique_ptr<HDF5DataSet> data_set_spot_indexed = nullptr;
std::unique_ptr<HDF5DataSet> data_set_spot_npeaks = nullptr;
std::unique_ptr<HDF5DataSet> data_set_strong_pixel_count = nullptr;
std::vector<uint32_t> npeaks;
std::vector<uint32_t> strong_pixel_count;
// indexing
std::unique_ptr<HDF5DataSet> data_set_indexed = nullptr;
std::unique_ptr<HDF5DataSet> data_set_indexed_lattice = nullptr;
// res. estimation
std::unique_ptr<HDF5DataSet> data_set_resolution_estimation = nullptr;
std::vector<uint8_t> indexed;
std::vector<float> indexed_lattice;
public:
explicit HDF5DataFilePluginMX(size_t max_spots);
void SetupSWMRFile(HDF5File &data_file, const DataMessage& msg) override;
void OpenFile(HDF5File &data_file, const DataMessage& msg) override;
void Write(const DataMessage& msg) override;
void Flush() override;
void WriteFinal(HDF5File &data_file) override;
};
#endif //JUNGFRAUJOCH_HDF5DATAFILEPLUGINMX_H

20
writer/HDF5DataFilePluginResEstimation.cpp Normal file
View File

@@ -0,0 +1,20 @@
// Copyright (2019-2024) Paul Scherrer Institute
#include "HDF5DataFilePluginResEstimation.h"
void HDF5DataFilePluginResEstimation::OpenFile(HDF5File &data_file, const DataMessage &msg) {}
void HDF5DataFilePluginResEstimation::Write(const DataMessage &msg) {
if (!msg.resolution_estimation)
return;
if (msg.number >= res_estimation.size())
res_estimation.resize(msg.number + 1);
res_estimation[msg.number] = msg.resolution_estimation.value();
}
void HDF5DataFilePluginResEstimation::WriteFinal(HDF5File &data_file) {
if (!res_estimation.empty())
data_file.SaveVector("/entry/res_estimation", res_estimation);
}

17
writer/HDF5DataFilePluginResEstimation.h Normal file
View File

@@ -0,0 +1,17 @@
// Copyright (2019-2024) Paul Scherrer Institute
#ifndef JUNGFRAUJOCH_HDF5DATAFILEPLUGINRESESTIMATION_H
#define JUNGFRAUJOCH_HDF5DATAFILEPLUGINRESESTIMATION_H
#include "HDF5DataFilePlugin.h"
class HDF5DataFilePluginResEstimation : public HDF5DataFilePlugin {
std::vector<float> res_estimation;
public:
void OpenFile(HDF5File &data_file, const DataMessage &msg) override;
void Write(const DataMessage &msg) override;
void WriteFinal(HDF5File &data_file) override;
};
#endif //JUNGFRAUJOCH_HDF5DATAFILEPLUGINRESESTIMATION_H

49
writer/HDF5DataFilePluginXFEL.cpp
View File

@@ -2,51 +2,30 @@
#include "HDF5DataFilePluginXFEL.h"
void HDF5DataFilePluginXFEL::SetupSWMRFile(HDF5File &data_file, const DataMessage &msg) {
if (msg.xfel_pulse_id.has_value() && msg.xfel_event_code.has_value())
enable = true;
else
return;
HDF5Group(data_file, "/entry/xfel").NXClass("NXcollection");
HDF5DataSpace data_space_linear({2}, {H5S_UNLIMITED});
HDF5Dcpl dcpl_linear;
dcpl_linear.SetChunking({1});
HDF5DataSet(data_file, "/entry/xfel/pulseID", HDF5DataType(8, false), data_space_linear, dcpl_linear).SetExtent({1});
HDF5DataSet(data_file, "/entry/xfel/eventCode", HDF5DataType(4, false), data_space_linear, dcpl_linear).SetExtent({1});
}
void HDF5DataFilePluginXFEL::OpenFile(HDF5File &data_file, const DataMessage &msg) {
if (!enable)
return;
data_set_xfel_event_code = std::make_unique<HDF5DataSet>(data_file, "/entry/xfel/eventCode");
data_set_xfel_pulseid = std::make_unique<HDF5DataSet>(data_file, "/entry/xfel/pulseID");
}
void HDF5DataFilePluginXFEL::OpenFile(HDF5File &data_file, const DataMessage &msg) {}
void HDF5DataFilePluginXFEL::Write(const DataMessage &msg) {
if (!enable
|| !msg.xfel_pulse_id.has_value()
if (!msg.xfel_pulse_id.has_value()
|| !msg.xfel_event_code.has_value())
return;
size_t image_number = msg.number;
if (image_number >= max_image_number) {
if (image_number >= pulseid.size()) {
max_image_number = image_number;
data_set_xfel_pulseid->SetExtent({max_image_number + 1});
data_set_xfel_event_code->SetExtent({max_image_number + 1});
pulseid.resize(image_number + 1);
event_code.resize(image_number + 1);
}
data_set_xfel_event_code->WriteScalar(msg.xfel_event_code.value(), {image_number});
data_set_xfel_pulseid->WriteScalar(msg.xfel_pulse_id.value(), {image_number});
pulseid[image_number] = msg.xfel_pulse_id.value();
event_code[image_number] = msg.xfel_event_code.value();
}
void HDF5DataFilePluginXFEL::Flush() {
if (!enable)
return;
data_set_xfel_event_code->Flush();
data_set_xfel_pulseid->Flush();
void HDF5DataFilePluginXFEL::WriteFinal(HDF5File &data_file) {
if (!pulseid.empty()) {
HDF5Group group(data_file, "/entry/xfel");
group.NXClass("NXcollection");
group.SaveVector("pulseID", pulseid);
group.SaveVector("eventCode", event_code);
}
}

9
writer/HDF5DataFilePluginXFEL.h
View File

@@ -6,15 +6,12 @@
#include "HDF5DataFilePlugin.h"
class HDF5DataFilePluginXFEL : public HDF5DataFilePlugin {
bool enable = false;
std::unique_ptr<HDF5DataSet> data_set_xfel_pulseid = nullptr;
std::unique_ptr<HDF5DataSet> data_set_xfel_event_code = nullptr;
std::vector<uint64_t> pulseid;
std::vector<uint32_t> event_code;
public:
void SetupSWMRFile(HDF5File &data_file, const DataMessage& msg) override;
void OpenFile(HDF5File &data_file, const DataMessage& msg) override;
void Write(const DataMessage& msg) override;
void Flush() override;
void WriteFinal(HDF5File &data_file) override;
};

20
writer/HDF5NXmx.cpp
View File

@@ -13,7 +13,7 @@ void HDF5Metadata::NXmx( const StartMessage &start, const EndMessage &end) {
MakeDirectory(filename);
HDF5File hdf5_file(filename, true, false);
HDF5File hdf5_file(filename);
chmod(filename.c_str(), S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); // default permissions
hdf5_file.Attr("file_name", filename);
@@ -249,8 +249,13 @@ void HDF5Metadata::LinkToData(HDF5File *hdf5_file, const StartMessage &start, co
hsize_t total_images = end.max_image_number;
hsize_t width = start.image_size_x;
hsize_t height = start.image_size_y;
hsize_t stride = start.data_file_count;
hsize_t file_count = std::min<hsize_t>(stride, total_images);
hsize_t images_per_file = start.images_per_file;
hsize_t file_count = 0;
if (start.images_per_file > 0) {
file_count = total_images / images_per_file;
if (total_images % images_per_file > 0)
file_count++;
}
if (total_images == 0)
return;
@@ -262,13 +267,14 @@ void HDF5Metadata::LinkToData(HDF5File *hdf5_file, const StartMessage &start, co
HDF5Dcpl dcpl;
for (hsize_t file_id = 0; file_id < file_count; file_id++) {
hsize_t images_in_file = total_images / stride;
if (total_images % stride > file_id)
images_in_file++;
hsize_t images_in_file = images_per_file;
if (file_id == file_count - 1)
images_in_file = total_images - (file_count - 1) * images_per_file;
HDF5DataSpace src_data_space({images_in_file, height, width});
HDF5DataSpace virtual_data_space({total_images, height, width});
virtual_data_space.SelectHyperslabWithStride({file_id, 0, 0},{images_in_file, height, width},{stride,1,1});
virtual_data_space.SelectHyperslab({file_id * images_per_file, 0, 0},{images_in_file, height, width});
dcpl.SetVirtual(DataFileName(start.file_prefix, file_id),
"/entry/data/data",src_data_space, virtual_data_space);
}

14
writer/HDF5Objects.cpp
View File

@@ -397,13 +397,10 @@ HDF5Group::~HDF5Group() {
H5Gclose(id);
}
HDF5File::HDF5File(const std::string& filename, bool create, bool swmr) : HDF5Object() {
HDF5File::HDF5File(const std::string& filename) : HDF5Object() {
HDF5Fapl fapl;
if (create) {
fapl.SetVersionTo1p10orNewer();
id = H5Fcreate(filename.c_str(), H5F_ACC_TRUNC | (swmr ? H5F_ACC_SWMR_WRITE : 0), H5P_DEFAULT, fapl.GetID());
} else
id = H5Fopen(filename.c_str(), H5F_ACC_RDWR | (swmr ? H5F_ACC_SWMR_WRITE : 0), H5P_DEFAULT);
fapl.SetVersionTo1p10orNewer();
id = H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, fapl.GetID());
if (id < 0)
throw JFJochException(JFJochExceptionCategory::HDF5, "Cannot open/create data HDF5 file " + filename);
}
@@ -416,11 +413,6 @@ void HDF5File::Delete(const std::string& path) {
H5Ldelete(id, path.c_str(), H5P_DEFAULT);
}
void HDF5File::StartSWMR() {
if (H5Fstart_swmr_write(id) < 0)
throw JFJochException(JFJochExceptionCategory::HDF5, "Cannot start SWMR mode");
}
HDF5ReadOnlyFile::HDF5ReadOnlyFile(const std::string &filename) {
id = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
if (id < 0)

5
writer/HDF5Objects.h
View File

@@ -124,15 +124,14 @@ public:
class HDF5File : public HDF5Object {
public:
HDF5File(const std::string& filename, bool create, bool swmr);
explicit HDF5File(const std::string& filename);
~HDF5File();
void StartSWMR();
void Delete(const std::string& path);
};
class HDF5ReadOnlyFile : public HDF5Object {
public:
HDF5ReadOnlyFile(const std::string& filename);
explicit HDF5ReadOnlyFile(const std::string& filename);
~HDF5ReadOnlyFile();
};

51
writer/HDF5Writer.cpp
View File

@@ -4,29 +4,48 @@
#include "HDF5NXmx.h"
HDF5Writer::HDF5Writer(const StartMessage &request)
: files(request.data_file_count) {
for (int i = 0; i < request.data_file_count; i++)
files[i] = std::make_unique<HDF5DataFile>(HDF5Metadata::DataFileName(request.file_prefix, i),
request.az_int_bin_to_q, request.max_spot_count);
}
: images_per_file(request.images_per_file),
file_prefix(request.file_prefix),
max_spot_count(request.max_spot_count),
az_int_bin_to_q(request.az_int_bin_to_q) {}
void HDF5Writer::Write(const DataMessage& message) {
if (message.image.size == 0)
return;
if (message.number < 0)
throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, "No support for negative images");
if (files.empty())
throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, "No file to write");
size_t file_number = message.number % files.size();
size_t file_number = 0;
size_t image_number = message.number;
if (images_per_file > 0) {
file_number = message.number / images_per_file;
image_number = message.number % images_per_file;
}
if (files.size() <= file_number)
files.resize(file_number + 1);
if (!files[file_number])
files[file_number] = std::make_unique<HDF5DataFile>(HDF5Metadata::DataFileName(file_prefix, file_number),
az_int_bin_to_q, max_spot_count);
// Ignore zero size images
if (message.image.size > 0)
files[file_number]->Write(message, message.number / files.size());
}
files[file_number]->Write(message, image_number);
void HDF5Writer::GetStatistics(std::vector<HDF5DataFileStatistics> &v) const {
for (const auto &f: files) {
auto tmp = f->GetStatistics();
if (tmp.total_images > 0)
v.push_back(tmp);
if (files[file_number]->GetNumImages() == images_per_file) {
stats.emplace_back(files[file_number]->GetStatistics());
files[file_number].reset();
}
}
std::vector<HDF5DataFileStatistics> HDF5Writer::Finalize() {
for (auto &f: files) {
if (f) {
auto tmp = f->GetStatistics();
if (tmp.total_images > 0)
stats.push_back(tmp);
f.reset();
}
}
return stats;
}

7
writer/HDF5Writer.h
View File

@@ -10,10 +10,15 @@
class HDF5Writer {
std::vector<std::unique_ptr<HDF5DataFile> > files;
int64_t images_per_file;
std::string file_prefix;
uint64_t max_spot_count;
std::vector<float> az_int_bin_to_q;
std::vector<HDF5DataFileStatistics> stats;
public:
explicit HDF5Writer(const StartMessage &request);
void Write(const DataMessage& msg);
void GetStatistics(std::vector<HDF5DataFileStatistics> &v) const;
std::vector<HDF5DataFileStatistics> Finalize();
};
#endif //JUNGFRAUJOCH_HDF5WRITER_H

2
writer/StreamWriter.cpp
View File

@@ -69,7 +69,7 @@ void StreamWriter::CollectImages(std::vector<HDF5DataFileStatistics> &v) {
state = StreamWriterState::Idle;
}
writer.GetStatistics(v);
v = writer.Finalize();
}
void StreamWriter::Cancel() {