mx/Jungfraujoch
1
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases 53 Wiki Activity

DiffractionExperiment: Remove 2x2 binning to simplify transformation code

Browse Source
This commit is contained in:
leonarski_f
2023-11-06 14:16:15 +01:00
parent e6442f6384
commit fcd7612656
15 changed files with 134 additions and 617 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/DiffractionExperiment.cpp
+4 -44
View File
@@ -516,17 +516,11 @@ float DiffractionExperiment::GetWavelength_A() const {
}
float DiffractionExperiment::GetBeamX_pxl() const {
if (GetBinning2x2())
return dataset.beam_x_pxl() / 2.0f;
else
return dataset.beam_x_pxl();
return dataset.beam_x_pxl();
}
float DiffractionExperiment::GetBeamY_pxl() const {
if (GetBinning2x2())
return dataset.beam_y_pxl() / 2.0f;
else
return dataset.beam_y_pxl();
return dataset.beam_y_pxl();
}
float DiffractionExperiment::GetDetectorDistance_mm() const {
@@ -608,32 +602,14 @@ int64_t DiffractionExperiment::GetPixelsNum() const {
return GetXPixelsNum() * GetYPixelsNum();
}
int64_t DiffractionExperiment::GetPixelsNumFullImage() const {
return GetXPixelsNumFullImage() * GetYPixelsNumFullImage();
}
int64_t DiffractionExperiment::GetXPixelsNumFullImage() const {
int64_t DiffractionExperiment::GetXPixelsNum() const {
if (GetDetectorMode() != DetectorMode::Conversion)
return RAW_MODULE_COLS;
else
return internal.detector().geometry().width_pxl();
}
int64_t DiffractionExperiment::GetXPixelsNum() const {
if (GetBinning2x2())
return GetXPixelsNumFullImage() / 2;
else
return GetXPixelsNumFullImage();
}
int64_t DiffractionExperiment::GetYPixelsNum() const {
if (GetBinning2x2())
return GetYPixelsNumFullImage() / 2;
else
return GetYPixelsNumFullImage();
}
int64_t DiffractionExperiment::GetYPixelsNumFullImage() const {
if (GetDetectorMode() != DetectorMode::Conversion)
return RAW_MODULE_LINES * GetModulesNum();
else
@@ -884,7 +860,6 @@ void DiffractionExperiment::LoadDatasetSettings(const JFJochProtoBuf::DatasetSet
SpaceGroupNumber(settings.space_group_number());
SampleName(settings.sample_name());
Compression(settings.compression());
Binning2x2(settings.binning2x2());
SaveCalibration(settings.save_calibration());
if (settings.fpga_summation() == 0)
FPGASummation(1);
@@ -1046,22 +1021,7 @@ bool DiffractionExperiment::GetApplyPixelMaskInFPGA() const {
}
float DiffractionExperiment::GetPixelSize_mm() const {
if (GetBinning2x2())
return internal.detector().pixel_size_mm() * 2;
else
return internal.detector().pixel_size_mm();
}
DiffractionExperiment &DiffractionExperiment::Binning2x2(bool input) {
dataset.set_binning2x2(input);
return *this;
}
bool DiffractionExperiment::GetBinning2x2() const {
if (GetDetectorMode() == DetectorMode::Conversion)
return dataset.binning2x2();
else
return false;
return internal.detector().pixel_size_mm();
}
DiffractionExperiment &DiffractionExperiment::SourceName(std::string input) {
common/DiffractionExperiment.h
+3 -12
View File
@@ -86,7 +86,6 @@ public:
DiffractionExperiment& SourceNameShort(std::string input);
DiffractionExperiment& InstrumentName(std::string input);
DiffractionExperiment& InstrumentNameShort(std::string input);
DiffractionExperiment& Binning2x2(bool input);
DiffractionExperiment& ApplyPixelMaskInFPGA(bool input);
operator JFJochProtoBuf::JungfraujochSettings() const;
@@ -150,17 +149,11 @@ public:
int64_t GetFirstModuleOfDataStream(uint16_t data_stream) const;
int64_t GetPixelsNum() const;
int64_t GetPixelsNumFullImage() const; // before filters, like ROI or binning
int64_t GetYPixelsNum() const;
int64_t GetYPixelsNumFullImage() const; // before filters, like ROI or binning
int64_t GetXPixelsNum() const;
int64_t GetXPixelsNumFullImage() const; // before filters, like ROI or binning
int64_t GetPixel0OfModule(uint16_t module_number) const; // before filters, like ROI or binning
int64_t GetModuleFastDirectionStep(uint16_t module_number) const; // before filters, like ROI or binning
int64_t GetModuleSlowDirectionStep(uint16_t module_number) const; // before filters, like ROI or binning
int64_t GetPixel0OfModule(uint16_t module_number) const;
int64_t GetModuleFastDirectionStep(uint16_t module_number) const;
int64_t GetModuleSlowDirectionStep(uint16_t module_number) const;
std::chrono::microseconds GetPreviewPeriod() const;
int64_t GetPreviewStride() const;
@@ -197,8 +190,6 @@ public:
bool GetApplyPixelMaskInFPGA() const;
float GetPixelSize_mm() const;
bool GetBinning2x2() const;
std::string GetSourceName() const;
std::string GetSourceNameShort() const;
std::string GetInstrumentName() const;
common/FrameTransformation.cpp
+5 -62
View File
@@ -10,8 +10,7 @@
FrameTransformation::FrameTransformation(const DiffractionExperiment &in_experiment) :
experiment(in_experiment), summation(experiment.GetSummation()),
pixel_depth(experiment.GetPixelDepth()), compressor(in_experiment.GetCompressionAlgorithmEnum()),
binning_2x2(experiment.GetBinning2x2()), conversion_buffer(RAW_MODULE_SIZE) {
pixel_depth(experiment.GetPixelDepth()), compressor(in_experiment.GetCompressionAlgorithmEnum()) {
if ((experiment.GetDetectorMode() == DetectorMode::Conversion) && (summation > 1)) {
for (int i = 0; i < experiment.GetModulesNum(); i++)
@@ -19,11 +18,6 @@ FrameTransformation::FrameTransformation(const DiffractionExperiment &in_experim
}
precompression_buffer.resize(experiment.GetPixelsNum() * pixel_depth);
if (pixel_depth == 4)
image16bit.resize(experiment.GetPixelsNum(), 0);
if (binning_2x2)
full_image_buffer.resize(experiment.GetPixelsNumFullImage() * pixel_depth);
if (experiment.GetApplyPixelMaskInFPGA()) {
// Mask gaps
@@ -52,10 +46,8 @@ template <class Td> void AddToFramesSum(Td *destination, const int16_t *source)
void FrameTransformation::PackSummation() {
for (int m = 0; m < experiment.GetModulesNum(); m++) {
int32_t *output;
if (binning_2x2)
output = (int32_t *) full_image_buffer.data();
else
output = (int32_t *) precompression_buffer.data();
output = (int32_t *) precompression_buffer.data();
TransferModuleAdjustMultipixels(output,
(int32_t *) summation_buffer[m].data(),
@@ -71,25 +63,8 @@ void FrameTransformation::PackSummation() {
}
void FrameTransformation::Pack() {
if (summation > 1) {
if (summation > 1)
PackSummation();
if (binning_2x2)
Bin2x2_sum<int32_t>((int32_t *) precompression_buffer.data(),
(int32_t *) full_image_buffer.data(),
experiment.GetXPixelsNumFullImage(),
experiment.GetYPixelsNumFullImage(),
static_cast<int32_t>(experiment.GetUnderflow()),
static_cast<int32_t>(experiment.GetOverflow()));
} else {
if (binning_2x2)
Bin2x2_sum<int16_t>((int16_t *) precompression_buffer.data(),
(int16_t *) full_image_buffer.data(),
experiment.GetXPixelsNumFullImage(),
experiment.GetYPixelsNumFullImage(),
INT16_MIN, INT16_MAX);
}
}
size_t FrameTransformation::SaveCompressedImage(void *output) {
@@ -102,12 +77,7 @@ void FrameTransformation::ProcessModule(const int16_t *input, uint16_t module_nu
size_t module_number_abs = experiment.GetFirstModuleOfDataStream(data_stream) + module_number;
if (summation == 1) {
int16_t *output;
if (binning_2x2)
output = (int16_t *) full_image_buffer.data();
else
output = (int16_t *) precompression_buffer.data();
auto output = (int16_t *) precompression_buffer.data();
if (experiment.GetDetectorMode() != DetectorMode::Conversion)
memcpy(output + RAW_MODULE_SIZE * module_number_abs,
@@ -136,31 +106,4 @@ const void *FrameTransformation::GetPreviewImage() const {
return precompression_buffer.data();
}
void FrameTransformation::ProcessModule(JFConversion &conv, const int16_t *input, uint16_t module_number, int data_stream) {
size_t module_number_abs = experiment.GetFirstModuleOfDataStream(data_stream) + module_number;
if (summation == 1) {
int16_t *output;
if (binning_2x2)
output = (int16_t *) full_image_buffer.data();
else
output = (int16_t *) precompression_buffer.data();
if (experiment.GetDetectorMode() != DetectorMode::Conversion)
memcpy(output + RAW_MODULE_SIZE * module_number_abs, input, RAW_MODULE_SIZE * experiment.GetPixelDepth());
else {
conv.ConvertModule(conversion_buffer.data(), (uint16_t *) input);
TransferModuleAdjustMultipixels(output, conversion_buffer.data(),
experiment.GetModuleSlowDirectionStep(module_number_abs),
static_cast<int16_t>(INT16_MIN),
static_cast<int16_t>(INT16_MAX),
experiment.GetModuleFastDirectionStep(module_number_abs),
experiment.GetPixel0OfModule(module_number_abs));
}
} else {
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"Summation with CPU conversion not supported at the moment");
}
}
common/FrameTransformation.h
+1 -8
View File
@@ -14,20 +14,13 @@ class FrameTransformation {
std::vector<std::vector<int32_t> > summation_buffer;
std::vector<char> precompression_buffer;
std::vector<int16_t> conversion_buffer;
std::vector<int16_t> image16bit;
const size_t summation;
const size_t pixel_depth;
std::vector<char> full_image_buffer;
bool binning_2x2;
void PackSummation(); // transfer summed image to converted coordinates, clear summation buffer, no compression
public:
FrameTransformation(const DiffractionExperiment &experiment);
explicit FrameTransformation(const DiffractionExperiment &experiment);
void ProcessModule(const int16_t *input, uint16_t module_number, int data_stream);
void ProcessModule(JFConversion &conv, const int16_t *input, uint16_t module_number,
int data_stream);
void Pack(); // transfer summed image to converted coordinates, clear summation buffer
size_t SaveCompressedImage(void *output);
void ApplyROI(const ROIFilter &filter);
common/RawToConvertedGeometry.h
+7 -24
View File
@@ -56,30 +56,13 @@ void RawToConvertedGeometryAdjustMultipixels(const DiffractionExperiment &experi
if (min > 0) min = 0;
Ts max = experiment.GetOverflow() - 1;
if (experiment.GetBinning2x2()) {
std::vector<Ts> tmp(experiment.GetPixelsNumFullImage());
for (size_t module_number = 0; module_number < experiment.GetModulesNum(); module_number++) {
TransferModuleAdjustMultipixels<Ts, Ts>(tmp.data(),
source + module_number * RAW_MODULE_SIZE,
experiment.GetModuleSlowDirectionStep(module_number),
min, max,
experiment.GetModuleFastDirectionStep(module_number),
experiment.GetPixel0OfModule(module_number));
Bin2x2_sum(destination,
tmp.data(),
experiment.GetXPixelsNumFullImage(),
experiment.GetYPixelsNumFullImage(),
min, max);
}
} else {
for (size_t module_number = 0; module_number < experiment.GetModulesNum(); module_number++) {
TransferModuleAdjustMultipixels<Ts, Ts>(destination,
source + module_number * RAW_MODULE_SIZE,
experiment.GetModuleSlowDirectionStep(module_number),
min, max,
experiment.GetModuleFastDirectionStep(module_number),
experiment.GetPixel0OfModule(module_number));
}
for (size_t module_number = 0; module_number < experiment.GetModulesNum(); module_number++) {
TransferModuleAdjustMultipixels<Ts, Ts>(destination,
source + module_number * RAW_MODULE_SIZE,
experiment.GetModuleSlowDirectionStep(module_number),
min, max,
experiment.GetModuleFastDirectionStep(module_number),
experiment.GetPixel0OfModule(module_number));
}
}
common/TestImagePusher.cpp
+1 -4
View File
@@ -104,10 +104,7 @@ bool TestImagePusher::CheckImage(const DiffractionExperiment &x, const std::vect
raw_reference_image.data(),
decompressed_image.data(),
storage_cell);
if (x.GetBinning2x2() && (result > 1.5)) {
logger.Error("Mean conversion error ({:.3f}) larger than threshold", result);
no_errors = false;
} else if (!x.GetBinning2x2() && (result > 0.5)) {
if (result > 0.5) {
logger.Error("Mean conversion error ({:.3f}) larger than threshold", result);
no_errors = false;
} else
grpc/jfjoch.proto
-2
View File
@@ -109,8 +109,6 @@ message DatasetSettings {
optional UnitCell unit_cell = 13;
int64 space_group_number = 14;
bool binning2x2 = 17;
bool rad_int_solid_angle_corr = 18;
bool rad_int_polarization_corr = 19;
float rad_int_polarization_factor = 20;
jungfrau/JFCalibration.cpp
+3 -11
View File
@@ -108,17 +108,9 @@ std::vector<uint32_t> JFCalibration::CalculateNexusMask(const DiffractionExperim
auto mask_raw = CalculateMask(experiment, storage_cell);
if (experiment.GetDetectorMode() == DetectorMode::Conversion) {
DiffractionExperiment x(experiment);
x.Binning2x2(false);
std::vector<uint32_t> tmp(x.GetPixelsNum(), 1);
RawToConvertedGeometry<uint32_t, uint32_t>(x, tmp.data(), mask_raw.data());
if (experiment.GetBinning2x2()) {
std::vector<uint32_t> tmp_binned(experiment.GetPixelsNum());
Bin2x2_or(tmp_binned.data(), tmp.data(), x.GetXPixelsNum(), experiment.GetYPixelsNum());
return tmp_binned;
} else
return tmp;
std::vector<uint32_t> tmp(experiment.GetPixelsNum(), 1);
RawToConvertedGeometry<uint32_t, uint32_t>(experiment, tmp.data(), mask_raw.data());
return tmp;
} else
return mask_raw;
}
python/jfjoch_pb2.py
+106 -106
View File
File diff suppressed because one or more lines are too long
receiver/JFJochReceiver.cpp
+2 -2
View File
@@ -568,10 +568,10 @@ float JFJochReceiver::GetIndexingRate() const {
float JFJochReceiver::GetProgress() const {
if (experiment.GetImageNum() > 0)
return static_cast<float>(max_image_number_sent) / static_cast<float>(experiment.GetImageNum()) * 100.0;
return static_cast<float>(max_image_number_sent) / static_cast<float>(experiment.GetImageNum()) * 100.0f;
else if (experiment.GetFrameNum() > 0)
// Pedestal
return static_cast<float>(max_image_number_sent) / static_cast<float>(experiment.GetFrameNum()) * 100.0;
return static_cast<float>(max_image_number_sent) / static_cast<float>(experiment.GetFrameNum()) * 100.0f;
else
return 100.0;
}
tests/DiffractionExperimentTest.cpp
+2 -45
View File
@@ -707,20 +707,18 @@ TEST_CASE("DiffractionExperiment_LoadDatasetSettings", "[DiffractionExperiment]"
settings.set_data_file_count(5);
settings.set_space_group_number(45);
settings.set_sample_name("lyso1");
settings.set_binning2x2(true);
settings.set_fpga_summation(36);
settings.set_fpga_pixel_output(JFJochProtoBuf::INT16);
REQUIRE_NOTHROW(x.LoadDatasetSettings(settings));
REQUIRE(x.GetImageNumPerTrigger() == 234);
REQUIRE(x.GetBeamX_pxl() == Approx(23.4/2));
REQUIRE(x.GetBeamY_pxl() == Approx(123.4/2));
REQUIRE(x.GetBeamX_pxl() == Approx(23.4));
REQUIRE(x.GetBeamY_pxl() == Approx(123.4));
REQUIRE(x.GetSpaceGroupNumber() == 45);
REQUIRE(x.GetCompressionAlgorithm() == JFJochProtoBuf::BSHUF_LZ4);
REQUIRE(x.GetSampleName() == "lyso1");
REQUIRE(x.GetDataFileCount() == 5);
REQUIRE(x.GetDetectorDistance_mm() == Approx(57.6));
REQUIRE(x.GetBinning2x2());
REQUIRE(x.GetFPGASummation() == 36);
REQUIRE(x.GetFPGAOutputMode() == JFJochProtoBuf::INT16);
}
@@ -902,47 +900,6 @@ TEST_CASE("DiffractionExperiment_FPGA_Summation_output","[DiffractionExperiment]
REQUIRE(!x.GetFPGAOutputSigned());
}
TEST_CASE("DiffractionExperiment_Binning","[DiffractionExperiment]") {
DiffractionExperiment x(DetectorGeometry(8, 2, 8, 36, true));
x.Mode(DetectorMode::Conversion).BeamX_pxl(200.0).BeamY_pxl(400.0);
REQUIRE(!x.GetBinning2x2());
REQUIRE(x.GetPixelSize_mm() == Approx(PIXEL_SIZE_IN_MM));
REQUIRE(x.GetXPixelsNum() == 2068);
REQUIRE(x.GetYPixelsNum() == 2164);
REQUIRE(x.GetXPixelsNumFullImage() == 2068);
REQUIRE(x.GetYPixelsNumFullImage() == 2164);
REQUIRE(x.GetPixelsNumFullImage() == 2164*2068);
REQUIRE(x.GetBeamX_pxl() == Approx(200.0));
REQUIRE(x.GetBeamY_pxl() == Approx(400.0));
REQUIRE(x.GetPixel0OfModule(0) == (2164 - 1)*2068);
x.Binning2x2(true);
REQUIRE(x.GetBinning2x2());
REQUIRE(x.GetPixelSize_mm() == Approx(2*PIXEL_SIZE_IN_MM));
REQUIRE(x.GetXPixelsNumFullImage() == 2068);
REQUIRE(x.GetXPixelsNum() == 2068/2);
REQUIRE(x.GetYPixelsNumFullImage() == 2164);
REQUIRE(x.GetYPixelsNum() == 2164/2);
REQUIRE(x.GetPixelsNumFullImage() == 2164*2068);
REQUIRE(x.GetPixelsNum() == (2164/2)*(2068/2));
REQUIRE(x.GetBeamX_pxl() == Approx(100.0));
REQUIRE(x.GetBeamY_pxl() == Approx(200.0));
REQUIRE(x.GetPixel0OfModule(0) == ((2164 - 1))*(2068));
REQUIRE(x.GetPixel0OfModule(1) == ((2164 - 1))*(2068) + (1030+8));
REQUIRE(x.GetPixel0OfModule(6) == ((2164 - 1))*(2068) - 3 * ((514+36))*(2068));
REQUIRE(x.GetPixel0OfModule(7) == ((2164 - 1))*(2068) - 3 * ((514+36))*(2068) + (1030+8));
x.Mode(DetectorMode::Raw);
REQUIRE(!x.GetBinning2x2());
REQUIRE(x.GetXPixelsNum() == RAW_MODULE_COLS);
REQUIRE(x.GetYPixelsNum() == 8 * RAW_MODULE_LINES);
}
TEST_CASE("DiffractionExperiment_DetectorModuleHostname","[DiffractionExperiment]") {
std::vector<std::string> h = {"mx1", "mx2", "mx3"};
DiffractionExperiment x(DetectorSetup(3, "X", h));
tests/FrameTransformationTest.cpp
-98
View File
@@ -343,53 +343,6 @@ TEST_CASE("FrameTransformation_Converted_bshuf_zstd" ,"") {
REQUIRE(input_1[(511+512)*1024 + 800] == output[CONVERTED_MODULE_SIZE * (nmodules - 2) + 1030 + 800 + 6]);
}
TEST_CASE("FrameTransformation_Converted_bshuf_zstd_bin_2x2" ,"") {
const uint16_t nmodules = 4;
const uint16_t ndatastreams = 2;
DiffractionExperiment experiment(DetectorGeometry(ndatastreams * nmodules, 2, 0, 0, false));
experiment.DataStreams(ndatastreams);
experiment.Mode(DetectorMode::Conversion).Compression(JFJochProtoBuf::BSHUF_ZSTD).Binning2x2(true);
FrameTransformation transformation(experiment);
std::mt19937 g1(1987);
std::uniform_int_distribution<int16_t> dist;
std::vector<int16_t> input_0(nmodules*RAW_MODULE_SIZE);
for (int i = 0; i < nmodules*RAW_MODULE_SIZE; i++)
input_0[i] = (i % 1024) + (i / 1024);
std::vector<int16_t> input_1(nmodules*RAW_MODULE_SIZE);
for (int i = 0; i < nmodules*RAW_MODULE_SIZE; i++)
input_1[i] = 10;
std::vector<char> output_compressed(experiment.GetMaxCompressedSize());
for (int i = 0; i < nmodules; i++) {
REQUIRE_NOTHROW(transformation.ProcessModule(input_0.data() + i * RAW_MODULE_SIZE, i, 0));
REQUIRE_NOTHROW(transformation.ProcessModule(input_1.data() + i * RAW_MODULE_SIZE, i, 1));
}
size_t compressed_size;
transformation.Pack();
REQUIRE_NOTHROW(compressed_size = transformation.SaveCompressedImage(output_compressed.data()));
REQUIRE(bshuf_read_uint64_BE(output_compressed.data()) == experiment.GetPixelsNum() * experiment.GetPixelDepth());
REQUIRE(bshuf_read_uint32_BE(output_compressed.data()+8) == JFJochBitShuffleCompressor::DefaultBlockSize * experiment.GetPixelDepth());
output_compressed.resize(compressed_size);
std::vector<int16_t> output;
REQUIRE_NOTHROW(JFJochDecompress(output, experiment.GetCompressionAlgorithmEnum(), output_compressed,
experiment.GetPixelsNum()));
REQUIRE(output[0] == 0 + 1 + 1 + 2);
REQUIRE(output[1] == 2 + 3 + 3 + 4);
REQUIRE(output[(513/2)*(1030/2) * 2] == 510 + 511 + 511 + 512); // line 510 + 511
REQUIRE(output[(514/2)*(1030/2) * 2] == 1024 + 1025 + 1025 + 1026); // line 1024 + 1025
}
TEST_CASE("FrameTransformation_Converted_bshuf_zstd_rle" ,"") {
const uint16_t nmodules = 4;
const uint16_t ndatastreams = 2;
@@ -520,57 +473,6 @@ TEST_CASE("FrameTransformation_Converted_bshuf_zstd_summation" ,"") {
REQUIRE(4 * input_1[(511+512)*1024 + 800] == output[CONVERTED_MODULE_SIZE * (nmodules - 2) + 1030 + 800 + 6]);
}
TEST_CASE("FrameTransformation_Converted_bshuf_zstd_summation_bin_2x2" ,"") {
const uint16_t nmodules = 4;
const uint16_t ndatastreams = 2;
DiffractionExperiment experiment(DetectorGeometry(ndatastreams * nmodules, 2, 0, 0, false));
experiment.DataStreams(ndatastreams);
experiment.Mode(DetectorMode::Conversion).Summation(4).Compression(JFJochProtoBuf::BSHUF_ZSTD).Binning2x2(true);
REQUIRE(experiment.GetSummation() == 4);
FrameTransformation transformation(experiment);
std::mt19937 g1(1987);
std::uniform_int_distribution<int16_t> dist;
std::vector<int16_t> input_0(nmodules*RAW_MODULE_SIZE);
for (int i = 0; i < nmodules*RAW_MODULE_SIZE; i++)
input_0[i] = (i % 1024) + (i / 1024);
std::vector<int16_t> input_1(nmodules*RAW_MODULE_SIZE);
for (int i = 0; i < nmodules*RAW_MODULE_SIZE; i++)
input_1[i] = 10;
std::vector<char> output_compressed(experiment.GetMaxCompressedSize());
for (int frame = 0; frame < experiment.GetSummation(); frame++) {
for (int i = 0; i < nmodules; i++) {
REQUIRE_NOTHROW(transformation.ProcessModule(input_0.data() + i * RAW_MODULE_SIZE, i, 0));
REQUIRE_NOTHROW(transformation.ProcessModule(input_1.data() + i * RAW_MODULE_SIZE, i, 1));
}
}
size_t compressed_size;
transformation.Pack();
REQUIRE_NOTHROW(compressed_size = transformation.SaveCompressedImage(output_compressed.data()));
REQUIRE(bshuf_read_uint64_BE(output_compressed.data()) == experiment.GetPixelsNum() * experiment.GetPixelDepth());
REQUIRE(bshuf_read_uint32_BE(output_compressed.data()+8) == JFJochBitShuffleCompressor::DefaultBlockSize * experiment.GetPixelDepth());
output_compressed.resize(compressed_size);
std::vector<int32_t> output;
REQUIRE_NOTHROW(JFJochDecompress(output, experiment.GetCompressionAlgorithmEnum(), output_compressed,
experiment.GetPixelsNum()));
REQUIRE(output[0] == 4 * (0 + 1 + 1 + 2));
REQUIRE(output[1] == 4 * (2 + 3 + 3 + 4));
REQUIRE(output[(513/2)*(1030/2) * 2] == 4 * (510 + 511 + 511 + 512)); // line 510 + 511
REQUIRE(output[(514/2)*(1030/2) * 2] == 4 * (1024 + 1025 + 1025 + 1026)); // line 1024 + 1025
}
TEST_CASE("FrameTransformation_Converted_bshuf_zstd_summation_2frames" ,"") {
// Ensure that previous frame is properly cleaned with PackStandardOutput()
const uint16_t nmodules = 4;
tests/JFCalibrationTest.cpp
-51
View File
@@ -238,32 +238,6 @@ TEST_CASE("JFCalibration_CalculateNexusMask","[JFCalibration]") {
REQUIRE(mask_v[1030 * 700 + 300] == 0);
}
TEST_CASE("JFCalibration_CalculateNexusMask_Bin2x2","[JFCalibration]") {
DiffractionExperiment experiment(DetectorGeometry(4, 2, 8, 36, false));
experiment.MaskModuleEdges(false).MaskChipEdges(true).Binning2x2(true);
JFCalibration calibration(4, 2);
calibration.Mask(0) = 50;
calibration.Mask(1) = 30;
calibration.Pedestal(0, 0, 0).GetPedestalMask()[2] = 4;
calibration.Pedestal(0, 1, 0).GetPedestalMask()[4] = 8;
calibration.Pedestal(0, 2, 0).GetPedestalMask()[5] = 16;
auto mask_v = calibration.CalculateNexusMask(experiment);
REQUIRE(mask_v.size() == experiment.GetPixelsNum() );
REQUIRE(mask_v[0] == (50 | 30));
REQUIRE(mask_v[1] == 4);
REQUIRE(mask_v[2] == (8 | 16));
REQUIRE(mask_v[3] == 0);
REQUIRE(mask_v[255/2] != 0);
REQUIRE(mask_v[514/2] != 0);
REQUIRE(mask_v[(1030+7)/2] != 0);
REQUIRE(mask_v[(2068/2)*(513/2)] == 0);
REQUIRE(mask_v[(2068/2)*(514/2)] != 0);
}
TEST_CASE("JFCalibration_CalculateOneByteMask","[JFCalibration]") {
DiffractionExperiment experiment(DetectorGeometry(4, 1, 8, 36, false));
experiment.MaskModuleEdges(false).MaskChipEdges(false);
@@ -285,31 +259,6 @@ TEST_CASE("JFCalibration_CalculateOneByteMask","[JFCalibration]") {
REQUIRE(mask_v[518*(experiment.GetXPixelsNum())] == 0); // Y Gap
}
TEST_CASE("JFCalibration_CalculateOneByteMask_Bin2x2","[JFCalibration]") {
DiffractionExperiment experiment(DetectorGeometry(4, 2, 8, 36, false));
experiment.MaskModuleEdges(false).MaskChipEdges(true).Binning2x2(true);
JFCalibration calibration(experiment);
calibration.Mask(0) = 50;
calibration.Mask(1) = 30;
calibration.Pedestal(0, 0, 0).GetPedestalMask()[2] = 4;
calibration.Pedestal(0, 1, 0).GetPedestalMask()[4] = 8;
calibration.Pedestal(0, 2, 0).GetPedestalMask()[5] = 16;
auto mask_v = calibration.CalculateOneByteMask(experiment);
REQUIRE(mask_v.size() == experiment.GetPixelsNum() );
REQUIRE(mask_v[0] == 0);
REQUIRE(mask_v[1] == 0);
REQUIRE(mask_v[2] == 0);
REQUIRE(mask_v[3] == 1);
REQUIRE(mask_v[255/2] == 0);
REQUIRE(mask_v[514/2] == 0);
REQUIRE(mask_v[(1030+7)/2] == 0);
REQUIRE(mask_v[(2068/2)*(513/2)] == 1);
REQUIRE(mask_v[(2068/2)*(514/2)] == 0);
}
TEST_CASE("JFCalibration_LoadMask_Bitwise","[JFCalibration]") {
DiffractionExperiment experiment(DetectorGeometry(8, 2, 8, 36, true));
tests/JFJochFullIntegrationTest.cpp
-120
View File
@@ -1016,126 +1016,6 @@ TEST_CASE("JFJochIntegrationTest_ZMQ_with_preview_no_writer", "[JFJochReceiver]"
writer_server->Shutdown();
}
TEST_CASE("JFJochIntegrationTest_ZMQ_with_preview_no_writer_binning2x2", "[JFJochReceiver]") {
Logger logger("JFJochIntegrationTest_ZMQ_with_preview_no_writer");
RegisterHDF5Filter();
int64_t nimages = 5;
int64_t ndatastream = 2;
int64_t nmodules = 4;
JFJochServices services(logger);
JFJochStateMachine state_machine(services, logger);
state_machine.AddDetectorSetup(DetectorGeometry(ndatastream * nmodules, 2, 8, 36));
state_machine.NotThreadSafe_Experiment().DataStreams(ndatastream);
state_machine.NotThreadSafe_Experiment().PedestalG0Frames(0).PedestalG1Frames(0).PedestalG2Frames(0).PreviewPeriod(
5ms);
services.Writer("unix:writer_test", "inproc://#1").Receiver("unix:fpga_receiver_test");
logger.Verbose(true);
std::vector<uint16_t> image(RAW_MODULE_SIZE);
std::vector<std::unique_ptr<AcquisitionDevice>> aq_devices;
for (int i = 0; i < ndatastream; i++) {
auto *test = new MockAcquisitionDevice(i, 256);
for (int m = 0; m < state_machine.NotThreadSafe_Experiment().GetModulesNum(i); m++) {
for (int image_num = 1; image_num <= nimages; image_num++)
test->AddModule(image_num, m, image.data());
}
test->Terminate();
aq_devices.emplace_back(test);
}
ZMQContext zmq_context;
std::vector<AcquisitionDevice *> tmp_devices;
for (const auto &i: aq_devices)
tmp_devices.emplace_back(i.get());
ZMQImagePusher pusher(zmq_context, {"inproc://#1"});
JFJochReceiverService fpga_receiver(tmp_devices, logger, pusher);
JFJochWriterService writer(zmq_context, logger);
ZMQPreviewPublisher preview(zmq_context, "inproc://#2");
fpga_receiver.PreviewPublisher(&preview).NumThreads(1);
ZMQSocket rcv_preview_socket(zmq_context, ZMQSocketType::Sub);
REQUIRE_NOTHROW(rcv_preview_socket.Connect("inproc://#2"));
rcv_preview_socket.SubscribeAll();
auto fpga_receiver_server = gRPCServer("unix:fpga_receiver_test", fpga_receiver);
auto writer_server = gRPCServer("unix:writer_test", writer);
REQUIRE_NOTHROW(state_machine.Initialize());
logger.Info("Initialized");
JFJochProtoBuf::DatasetSettings setup;
setup.set_ntrigger(1);
setup.set_detector_distance_mm(100);
setup.set_images_per_trigger(5);
setup.set_photon_energy_kev(12.4);
setup.set_beam_x_pxl(123.0);
setup.set_beam_y_pxl(878.0);
setup.set_summation(1);
setup.set_data_file_count(1);
setup.set_binning2x2(true);
REQUIRE_NOTHROW(state_machine.Start(setup));
logger.Info("Started measurement");
REQUIRE_NOTHROW(state_machine.Stop());
logger.Info("Stopped measurement");
std::string s;
// Pixel mask
REQUIRE(rcv_preview_socket.Receive(s, false) > 0);
JFJochProtoBuf::PreviewFrame frame;
REQUIRE_NOTHROW(frame = jsonToGrpc<JFJochProtoBuf::PreviewFrame>(s));
// Check header
REQUIRE(frame.image_number() == -1);
REQUIRE(frame.width() == state_machine.NotThreadSafe_Experiment().GetXPixelsNumFullImage() / 2);
REQUIRE(frame.height() == state_machine.NotThreadSafe_Experiment().GetYPixelsNumFullImage() / 2);
REQUIRE(frame.pixel_depth() == 4);
// First frame
REQUIRE(rcv_preview_socket.Receive(s, false) > 0);
REQUIRE_NOTHROW(frame = jsonToGrpc<JFJochProtoBuf::PreviewFrame>(s));
// Check header
REQUIRE(frame.image_number() == 0);
REQUIRE(frame.beam_x_pxl() == Approx(setup.beam_x_pxl()/2.0f));
REQUIRE(frame.beam_y_pxl() == Approx(setup.beam_y_pxl()/2.0f));
REQUIRE(frame.width() == state_machine.NotThreadSafe_Experiment().GetXPixelsNumFullImage() / 2);
REQUIRE(frame.height() == state_machine.NotThreadSafe_Experiment().GetYPixelsNumFullImage() / 2);
REQUIRE(frame.pixel_depth() == 2);
// Check compressed image
size_t npixel = state_machine.NotThreadSafe_Experiment().GetPixelsNum();
std::vector<char> rcv_image(frame.data().size());
rcv_image = {frame.data().begin(), frame.data().end()};
REQUIRE(rcv_image.size() == state_machine.NotThreadSafe_Experiment().GetPixelsNum() * sizeof(int16_t));
// Check no more frames waiting
REQUIRE(rcv_preview_socket.Receive(s, false) == -1);
auto tmp = state_machine.GetMeasurementStatistics();
REQUIRE(tmp.has_value());
auto statistics = tmp.value();
REQUIRE(statistics.collection_efficiency() == 1.0);
REQUIRE(statistics.images_collected() == 5);
REQUIRE(statistics.images_written() == 0);
fpga_receiver_server->Shutdown();
writer_server->Shutdown();
}
TEST_CASE("JFJochIntegrationTest_ZMQ_lysozyme_spot", "[JFJochReceiver]") {
Logger logger("JFJochIntegrationTest_ZMQ_lysozyme_spot_and_index");
tests/JFJochReceiverIntegrationTest.cpp
-28
View File
@@ -67,34 +67,6 @@ TEST_CASE("JFJochReceiverTest_Conversion", "[JFJochReceiver]") {
REQUIRE(!output.cancelled());
}
TEST_CASE("JFJochReceiverTest_Conversion_Bin2x2", "[JFJochReceiver]") {
Logger logger("JFJochReceiverTest_Conversion_Bin2x2");
DiffractionExperiment x(DetectorGeometry(2));
const uint16_t nthreads = 4;
x.Mode(DetectorMode::Conversion);
x.PedestalG0Frames(0).NumTriggers(1).UseInternalPacketGenerator(true).Binning2x2(true)
.ImagesPerTrigger(32).DataFileCount(16).PhotonEnergy_keV(12.4).Compression(JFJochProtoBuf::BSHUF_ZSTD);
std::vector<std::unique_ptr<AcquisitionDevice>> aq_devices;
for (int i = 0; i < x.GetDataStreamsNum(); i++) {
HLSSimulatedDevice *test;
test = new HLSSimulatedDevice(i, 64);
test->EnableLogging(&logger);
aq_devices.emplace_back(test);
}
JFJochProtoBuf::ReceiverOutput output;
bool ret;
REQUIRE_NOTHROW(ret = JFJochReceiverTest(output, logger, aq_devices, x, nthreads, false));
REQUIRE(ret);
REQUIRE(output.efficiency() == 1.0);
REQUIRE(output.images_sent() == x.GetImageNum());
REQUIRE(!output.cancelled());
}
TEST_CASE("JFJochReceiverTest_Conversion_StorageCell", "[JFJochReceiver]") {
Logger logger("JFJochReceiverTest_Conversion_StorageCell");