mx/Jungfraujoch
1
1
Fork 0
Code Issues Pull Requests 1 Actions 2 Packages Projects Releases 53 Wiki Activity

FrameTransformation: Split packing and compression

Browse Source
This commit is contained in:
leonarski_f
2023-05-01 21:27:08 +02:00
parent dce186f9f9
commit 9d5be2a4e9
7 changed files with 44 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/FrameTransformation.cpp
+6 -9
View File
@@ -40,11 +40,6 @@ FrameTransformation::FrameTransformation(const DiffractionExperiment &in_experim
}
}
FrameTransformation& FrameTransformation::SetOutput(void *output) {
standard_output = (char *) output;
return *this;
}
template <class Td> void AddToFramesSum(Td *destination, const int16_t *source) {
for (int i = 0; i < RAW_MODULE_SIZE; i++) {
if ((source[i] == INT16_MIN) || (destination[i] == INT32_MIN))
@@ -90,9 +85,7 @@ void FrameTransformation::Generate16BitPreview() {
}
}
size_t FrameTransformation::PackStandardOutput() {
if (standard_output == nullptr)
throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, "Default stream output not initialized");
void FrameTransformation::Pack() {
if (summation > 1) {
PackSummation();
@@ -114,7 +107,11 @@ size_t FrameTransformation::PackStandardOutput() {
experiment.GetYPixelsNumFullImage(),
INT16_MIN, INT16_MAX);
}
return compressor.Compress(standard_output, precompression_buffer.data(),
}
size_t FrameTransformation::SaveCompressedImage(void *output) {
return compressor.Compress((char *) output, precompression_buffer.data(),
experiment.GetPixelsNum(), pixel_depth);
}
common/FrameTransformation.h
+2 -5
View File
@@ -16,8 +16,6 @@ class FrameTransformation {
std::vector<char> precompression_buffer;
std::vector<int16_t> image16bit;
char *standard_output = nullptr;
const size_t summation;
const size_t pixel_depth;
@@ -27,12 +25,11 @@ class FrameTransformation {
void Generate16BitPreview();
public:
FrameTransformation(const DiffractionExperiment &experiment);
FrameTransformation& SetOutput(void *output);
void ProcessModule(const int16_t *input, uint16_t module_number, int data_stream);
void ProcessModule(JFConversion &conv, const int16_t *input, uint16_t module_number,
int data_stream);
size_t PackStandardOutput(); // transfer summed image to converted coordinates, clear summation buffer, compress
void Pack(); // transfer summed image to converted coordinates, clear summation buffer
size_t SaveCompressedImage(void *output);
int16_t *GetPreview16BitImage();
};
receiver/JFJochReceiver.cpp
+3 -3
View File
@@ -286,8 +286,6 @@ int64_t JFJochReceiver::FrameTransformationThread() {
int64_t max_thread_delay = 0;
uint64_t image_number;
transformation.SetOutput(writer_buffer.data());
frame_transformation_ready->CountDown();
std::unique_ptr<IndexerWrapper> indexer;
@@ -354,7 +352,7 @@ int64_t JFJochReceiver::FrameTransformationThread() {
}
if (send_image) {
size_t image_size = transformation.PackStandardOutput();
transformation.Pack();
std::vector<DiffractionSpot> spots;
auto local_data_processing_settings = GetDataProcessingSettings();
@@ -410,6 +408,8 @@ int64_t JFJochReceiver::FrameTransformationThread() {
AddRadialIntegrationProfile(message.rad_int_profile);
}
size_t image_size = transformation.SaveCompressedImage(writer_buffer.data());
if (push_images_to_writer) {
PrepareCBORImage(message, experiment, writer_buffer.data(), image_size);
serializer.SerializeImage(message);
tests/FrameTransformationTest.cpp
+25 -38
View File
@@ -39,14 +39,12 @@ TEST_CASE("FrameTransformation_Raw_NoCompression" ,"") {
std::vector<int16_t> output(experiment.GetPixelsNum());
transformation.SetOutput(output.data());
for (int i = 0; i < nmodules; i++) {
REQUIRE_NOTHROW(transformation.ProcessModule(input_0.data() + i * RAW_MODULE_SIZE, i, 0));
REQUIRE_NOTHROW(transformation.ProcessModule(input_1.data() + i * RAW_MODULE_SIZE, i, 1));
}
REQUIRE(transformation.PackStandardOutput() == experiment.GetPixelDepth() * experiment.GetPixelsNum());
transformation.Pack();
REQUIRE(transformation.SaveCompressedImage(output.data()) == experiment.GetPixelDepth() * experiment.GetPixelsNum());
uint32_t diff_0 = 0;
uint32_t diff_1 = 0;
@@ -81,14 +79,13 @@ TEST_CASE("FrameTransformation_Converted_NoCompression" ,"") {
std::vector<int16_t> output(experiment.GetPixelsNum());
transformation.SetOutput(output.data());
for (int i = 0; i < nmodules; i++) {
REQUIRE_NOTHROW(transformation.ProcessModule(input_0.data() + i * RAW_MODULE_SIZE, i, 0));
REQUIRE_NOTHROW(transformation.ProcessModule(input_1.data() + i * RAW_MODULE_SIZE, i, 1));
}
REQUIRE(transformation.PackStandardOutput() == experiment.GetPixelDepth() * experiment.GetPixelsNum());
transformation.Pack();
REQUIRE(transformation.SaveCompressedImage(output.data()) == experiment.GetPixelDepth() * experiment.GetPixelsNum());
REQUIRE(input_0[511*1024] == output[CONVERTED_MODULE_SIZE * (2 * nmodules - 2) + 0]);
REQUIRE(input_0[511*1024+256]/2 == output[CONVERTED_MODULE_SIZE * (2 * nmodules - 2) + 258]);
@@ -134,15 +131,14 @@ TEST_CASE("FrameTransformation_Converted_bshuf_lz4" ,"") {
std::vector<char> output_compressed(experiment.GetMaxCompressedSize());
transformation.SetOutput(output_compressed.data());
for (int i = 0; i < nmodules; i++) {
REQUIRE_NOTHROW(transformation.ProcessModule(input_0.data() + i * RAW_MODULE_SIZE, i, 0));
REQUIRE_NOTHROW(transformation.ProcessModule(input_1.data() + i * RAW_MODULE_SIZE, i, 1));
}
size_t compressed_size;
REQUIRE_NOTHROW(compressed_size = transformation.PackStandardOutput());
transformation.Pack();
REQUIRE_NOTHROW(compressed_size = transformation.SaveCompressedImage(output_compressed.data()));
output_compressed.resize(compressed_size);
REQUIRE(bshuf_read_uint64_BE(output_compressed.data()) == experiment.GetPixelsNum() * experiment.GetPixelDepth());
@@ -195,16 +191,14 @@ TEST_CASE("FrameTransformation_Converted_bshuf_zstd" ,"") {
std::vector<char> output_compressed(experiment.GetMaxCompressedSize());
transformation.SetOutput(output_compressed.data());
for (int i = 0; i < nmodules; i++) {
REQUIRE_NOTHROW(transformation.ProcessModule(input_0.data() + i * RAW_MODULE_SIZE, i, 0));
REQUIRE_NOTHROW(transformation.ProcessModule(input_1.data() + i * RAW_MODULE_SIZE, i, 1));
}
size_t compressed_size;
REQUIRE_NOTHROW(compressed_size = transformation.PackStandardOutput());
transformation.Pack();
REQUIRE_NOTHROW(compressed_size = transformation.SaveCompressedImage(output_compressed.data()));
REQUIRE(bshuf_read_uint64_BE(output_compressed.data()) == experiment.GetPixelsNum() * experiment.GetPixelDepth());
REQUIRE(bshuf_read_uint32_BE(output_compressed.data()+8) == JFJochBitShuffleCompressor::DefaultBlockSize * experiment.GetPixelDepth());
@@ -257,15 +251,14 @@ TEST_CASE("FrameTransformation_Converted_bshuf_zstd_bin_2x2" ,"") {
std::vector<char> output_compressed(experiment.GetMaxCompressedSize());
transformation.SetOutput(output_compressed.data());
for (int i = 0; i < nmodules; i++) {
REQUIRE_NOTHROW(transformation.ProcessModule(input_0.data() + i * RAW_MODULE_SIZE, i, 0));
REQUIRE_NOTHROW(transformation.ProcessModule(input_1.data() + i * RAW_MODULE_SIZE, i, 1));
}
size_t compressed_size;
REQUIRE_NOTHROW(compressed_size = transformation.PackStandardOutput());
transformation.Pack();
REQUIRE_NOTHROW(compressed_size = transformation.SaveCompressedImage(output_compressed.data()));
REQUIRE(bshuf_read_uint64_BE(output_compressed.data()) == experiment.GetPixelsNum() * experiment.GetPixelDepth());
REQUIRE(bshuf_read_uint32_BE(output_compressed.data()+8) == JFJochBitShuffleCompressor::DefaultBlockSize * experiment.GetPixelDepth());
@@ -305,16 +298,14 @@ TEST_CASE("FrameTransformation_Converted_bshuf_zstd_rle" ,"") {
std::vector<char> output_compressed(experiment.GetMaxCompressedSize());
transformation.SetOutput(output_compressed.data());
for (int i = 0; i < nmodules; i++) {
REQUIRE_NOTHROW(transformation.ProcessModule(input_0.data() + i * RAW_MODULE_SIZE, i, 0));
REQUIRE_NOTHROW(transformation.ProcessModule(input_1.data() + i * RAW_MODULE_SIZE, i, 1));
}
size_t compressed_size;
REQUIRE_NOTHROW(compressed_size = transformation.PackStandardOutput());
transformation.Pack();
REQUIRE_NOTHROW(compressed_size = transformation.SaveCompressedImage(output_compressed.data()));
REQUIRE(bshuf_read_uint64_BE(output_compressed.data()) == experiment.GetPixelsNum() * experiment.GetPixelDepth());
REQUIRE(bshuf_read_uint32_BE(output_compressed.data()+8) == JFJochBitShuffleCompressor::DefaultBlockSize * experiment.GetPixelDepth());
@@ -369,8 +360,6 @@ TEST_CASE("FrameTransformation_Converted_bshuf_zstd_summation" ,"") {
std::vector<char> output_compressed(experiment.GetMaxCompressedSize());
transformation.SetOutput(output_compressed.data());
for (int i = 0; i < nmodules; i++) {
REQUIRE_NOTHROW(transformation.ProcessModule(input_0.data() + i * RAW_MODULE_SIZE, i, 0));
REQUIRE_NOTHROW(transformation.ProcessModule(input_1.data() + i * RAW_MODULE_SIZE, i, 1));
@@ -392,7 +381,8 @@ TEST_CASE("FrameTransformation_Converted_bshuf_zstd_summation" ,"") {
}
size_t compressed_size;
REQUIRE_NOTHROW(compressed_size = transformation.PackStandardOutput());
transformation.Pack();
REQUIRE_NOTHROW(compressed_size = transformation.SaveCompressedImage(output_compressed.data()));
REQUIRE(bshuf_read_uint64_BE(output_compressed.data()) == experiment.GetPixelsNum() * experiment.GetPixelDepth());
REQUIRE(bshuf_read_uint32_BE(output_compressed.data()+8) == JFJochBitShuffleCompressor::DefaultBlockSize * experiment.GetPixelDepth());
@@ -440,8 +430,6 @@ TEST_CASE("FrameTransformation_Converted_bshuf_zstd_summation_bin_2x2" ,"") {
std::vector<char> output_compressed(experiment.GetMaxCompressedSize());
transformation.SetOutput(output_compressed.data());
for (int frame = 0; frame < experiment.GetSummation(); frame++) {
for (int i = 0; i < nmodules; i++) {
REQUIRE_NOTHROW(transformation.ProcessModule(input_0.data() + i * RAW_MODULE_SIZE, i, 0));
@@ -450,7 +438,8 @@ TEST_CASE("FrameTransformation_Converted_bshuf_zstd_summation_bin_2x2" ,"") {
}
size_t compressed_size;
REQUIRE_NOTHROW(compressed_size = transformation.PackStandardOutput());
transformation.Pack();
REQUIRE_NOTHROW(compressed_size = transformation.SaveCompressedImage(output_compressed.data()));
REQUIRE(bshuf_read_uint64_BE(output_compressed.data()) == experiment.GetPixelsNum() * experiment.GetPixelDepth());
REQUIRE(bshuf_read_uint32_BE(output_compressed.data()+8) == JFJochBitShuffleCompressor::DefaultBlockSize * experiment.GetPixelDepth());
@@ -493,15 +482,14 @@ TEST_CASE("FrameTransformation_Converted_16bit_preview_summation" ,"") {
std::vector<char> output_compressed(experiment.GetMaxCompressedSize());
transformation.SetOutput(output_compressed.data());
REQUIRE_NOTHROW(transformation.ProcessModule(input_0.data(), 0, 0));
REQUIRE_NOTHROW(transformation.ProcessModule(input_0.data(), 0, 0));
REQUIRE_NOTHROW(transformation.ProcessModule(input_0.data(), 0, 0));
REQUIRE_NOTHROW(transformation.ProcessModule(input_0.data(), 0, 0));
size_t compressed_size;
REQUIRE_NOTHROW(compressed_size = transformation.PackStandardOutput());
transformation.Pack();
REQUIRE_NOTHROW(compressed_size = transformation.SaveCompressedImage(output_compressed.data()));
REQUIRE(bshuf_read_uint64_BE(output_compressed.data()) == experiment.GetPixelsNum() * experiment.GetPixelDepth());
REQUIRE(bshuf_read_uint32_BE(output_compressed.data()+8) == JFJochBitShuffleCompressor::DefaultBlockSize * experiment.GetPixelDepth());
@@ -561,9 +549,7 @@ TEST_CASE("FrameTransformation_Converted_bshuf_zstd_summation_2frames" ,"") {
std::vector<char> output_compressed(experiment.GetMaxCompressedSize());
transformation.SetOutput(output_compressed.data());
for (int i = 0; i < nmodules; i++) {
for (int i = 0; i < nmodules; i++) {
REQUIRE_NOTHROW(transformation.ProcessModule(input_0.data() + i * RAW_MODULE_SIZE, i, 0));
REQUIRE_NOTHROW(transformation.ProcessModule(input_1.data() + i * RAW_MODULE_SIZE, i, 1));
}
@@ -574,7 +560,8 @@ TEST_CASE("FrameTransformation_Converted_bshuf_zstd_summation_2frames" ,"") {
}
size_t compressed_size;
REQUIRE_NOTHROW(compressed_size = transformation.PackStandardOutput());
transformation.Pack();
REQUIRE_NOTHROW(compressed_size = transformation.SaveCompressedImage(output_compressed.data()));
for (int i = 0; i < nmodules; i++) {
REQUIRE_NOTHROW(transformation.ProcessModule(input_0.data() + i * RAW_MODULE_SIZE, i, 0));
@@ -596,7 +583,8 @@ TEST_CASE("FrameTransformation_Converted_bshuf_zstd_summation_2frames" ,"") {
REQUIRE_NOTHROW(transformation.ProcessModule(input_1.data() + i * RAW_MODULE_SIZE, i, 1));
}
REQUIRE_NOTHROW(compressed_size = transformation.PackStandardOutput());
transformation.Pack();
REQUIRE_NOTHROW(compressed_size = transformation.SaveCompressedImage(output_compressed.data()));
REQUIRE(bshuf_read_uint64_BE(output_compressed.data()) == experiment.GetPixelsNum() * experiment.GetPixelDepth());
REQUIRE(bshuf_read_uint32_BE(output_compressed.data()+8) == JFJochBitShuffleCompressor::DefaultBlockSize * experiment.GetPixelDepth());
@@ -644,8 +632,6 @@ TEST_CASE("FrameTransformation_Converted_bshuf_zstd_summation_negatives" ,"") {
std::vector<char> output_compressed(experiment.GetMaxCompressedSize());
transformation.SetOutput(output_compressed.data());
for (int i = 0; i < nmodules; i++) {
REQUIRE_NOTHROW(transformation.ProcessModule(input_0.data() + i * RAW_MODULE_SIZE, i, 0));
REQUIRE_NOTHROW(transformation.ProcessModule(input_1.data() + i * RAW_MODULE_SIZE, i, 1));
@@ -666,7 +652,8 @@ TEST_CASE("FrameTransformation_Converted_bshuf_zstd_summation_negatives" ,"") {
REQUIRE_NOTHROW(transformation.ProcessModule(input_1.data() + i * RAW_MODULE_SIZE, i, 1));
}
size_t compressed_size;
REQUIRE_NOTHROW(compressed_size = transformation.PackStandardOutput());
transformation.Pack();
REQUIRE_NOTHROW(compressed_size = transformation.SaveCompressedImage(output_compressed.data()));
REQUIRE(bshuf_read_uint64_BE(output_compressed.data()) == experiment.GetPixelsNum() * experiment.GetPixelDepth());
REQUIRE(bshuf_read_uint32_BE(output_compressed.data()+8) == JFJochBitShuffleCompressor::DefaultBlockSize * experiment.GetPixelDepth());
tools/CompressionBenchmark.cpp
+4 -4
View File
@@ -21,11 +21,11 @@ std::string CheckCompression(const DiffractionExperiment &x, size_t nimages, con
auto start_time = std::chrono::system_clock::now();
for (int i = 0; i < nimages; i++) {
transformation.SetOutput(output.data() + i * x.GetMaxCompressedSize());
for (int j = 0; j < x.GetModulesNum(); j++ ) {
transformation.ProcessModule(image.data() + (j + i * x.GetModulesNum()) * RAW_MODULE_SIZE, j, 0);
}
compressed_size += transformation.PackStandardOutput();
transformation.Pack();
compressed_size += transformation.SaveCompressedImage(output.data() + i * x.GetMaxCompressedSize());
}
auto end_time = std::chrono::system_clock::now();
@@ -47,11 +47,11 @@ std::string CheckDecompression(const DiffractionExperiment &x, size_t nimages, c
std::vector<size_t> compressed_size(nimages);
for (int i = 0; i < nimages; i++) {
transformation.SetOutput(output[i].data());
for (int j = 0; j < x.GetModulesNum(); j++ ) {
transformation.ProcessModule(image.data() + (j + i * x.GetModulesNum()) * RAW_MODULE_SIZE, j, 0);
}
compressed_size[i] = transformation.PackStandardOutput();
transformation.Pack();
compressed_size[i] = transformation.SaveCompressedImage(output[i].data());
output[i].resize(compressed_size[i]);
}
tools/HDF5DatasetWriteTest.cpp
+2 -2
View File
@@ -87,10 +87,10 @@ int main(int argc, char **argv) {
for (auto &i: output) i.resize(x.GetMaxCompressedSize());
for (int i = 0; i < nimages; i++) {
transformation.SetOutput(output[i].data());
for (int j = 0; j < 8; j++)
transformation.ProcessModule(image + (i * x.GetModulesNum() + j) * RAW_MODULE_SIZE, j, 0);
output_size[i] = transformation.PackStandardOutput();
transformation.Pack();
output_size[i] = transformation.SaveCompressedImage(output[i].data());
}
x.ImagesPerTrigger(nimages_out);
writer/jfjoch_writer_test.cpp
+2 -2
View File
@@ -79,10 +79,10 @@ int main(int argc, char **argv) {
image_tmp_raw.data(),
image_tmp_conv.data());
transformation.SetOutput(output[i].data());
for (int j = 0; j < x.GetModulesNum(); j++)
transformation.ProcessModule(image_tmp_raw.data() + j * RAW_MODULE_SIZE, j, 0);
output_size[i] = transformation.PackStandardOutput();
transformation.Pack();
output_size[i] = transformation.SaveCompressedImage((uint8_t *) output[i].data());
}
logger.Info("Sending {} images", nimages_out);