mx/Jungfraujoch
2
1
Fork 0
Code Issues Pull Requests 1 Actions 3 Packages Projects Releases 55 Wiki Activity

DiffractionExperiment: Clean-up Compression and UnitCell

Browse Source
This commit is contained in:
leonarski_f
2023-11-13 16:56:33 +01:00
parent 2f87b4c1b1
commit e85988dfa2
15 changed files with 172 additions and 164 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/DiffractionExperiment.cpp
+45 -23
View File
@@ -218,8 +218,21 @@ DiffractionExperiment &DiffractionExperiment::MaskModuleEdges(bool input) {
return *this;
}
DiffractionExperiment &DiffractionExperiment::Compression(JFJochProtoBuf::Compression input) {
dataset.set_compression(input);
DiffractionExperiment &DiffractionExperiment::Compression(CompressionAlgorithm input) {
switch (input) {
case CompressionAlgorithm::NO_COMPRESSION:
dataset.set_compression(JFJochProtoBuf::NO_COMPRESSION);
break;
case CompressionAlgorithm::BSHUF_LZ4:
dataset.set_compression(JFJochProtoBuf::BSHUF_LZ4);
break;
case CompressionAlgorithm::BSHUF_ZSTD:
dataset.set_compression(JFJochProtoBuf::BSHUF_ZSTD);
break;
case CompressionAlgorithm::BSHUF_ZSTD_RLE:
dataset.set_compression(JFJochProtoBuf::BSHUF_ZSTD_RLE);
break;
}
return *this;
}
@@ -262,11 +275,6 @@ DiffractionExperiment& DiffractionExperiment::QSpacingForRadialInt_recipA(float
return *this;
}
DiffractionExperiment &DiffractionExperiment::SetUnitCell(const JFJochProtoBuf::UnitCell &cell) {
*dataset.mutable_unit_cell() = cell;
return *this;
}
DiffractionExperiment &DiffractionExperiment::SetUnitCell(const UnitCell &cell) {
JFJochProtoBuf::UnitCell tmp;
tmp.set_a(cell.a);
@@ -275,7 +283,8 @@ DiffractionExperiment &DiffractionExperiment::SetUnitCell(const UnitCell &cell)
tmp.set_alpha(cell.alpha);
tmp.set_beta(cell.beta);
tmp.set_gamma(cell.gamma);
return SetUnitCell(tmp);
*dataset.mutable_unit_cell() = tmp;
return *this;
}
DiffractionExperiment &DiffractionExperiment::SetUnitCell() {
@@ -496,18 +505,12 @@ int64_t DiffractionExperiment::GetDataFileCount() const {
return dataset.data_file_count();
}
JFJochProtoBuf::Compression DiffractionExperiment::GetCompressionAlgorithm() const {
switch (GetDetectorMode()) {
case DetectorMode::Conversion:
return dataset.compression();
default:
// Compression not supported for raw data and pedestal
return JFJochProtoBuf::NO_COMPRESSION;
}
}
CompressionAlgorithm DiffractionExperiment::GetCompressionAlgorithm() const {
if (GetDetectorMode() != DetectorMode::Conversion)
// Compression not supported for raw data and pedestal
return CompressionAlgorithm::NO_COMPRESSION;
CompressionAlgorithm DiffractionExperiment::GetCompressionAlgorithmEnum() const {
switch (GetCompressionAlgorithm()) {
switch (dataset.compression()) {
case JFJochProtoBuf::BSHUF_LZ4:
return CompressionAlgorithm::BSHUF_LZ4;
case JFJochProtoBuf::BSHUF_ZSTD:
@@ -578,7 +581,7 @@ int64_t DiffractionExperiment::GetFirstModuleOfDataStream(uint16_t data_stream)
}
int64_t DiffractionExperiment::GetMaxCompressedSize() const {
return MaxCompressedSize(GetCompressionAlgorithmEnum(),GetPixelsNum(), GetPixelDepth());
return MaxCompressedSize(GetCompressionAlgorithm(), GetPixelsNum(), GetPixelDepth());
}
int64_t DiffractionExperiment::GetPixelsNum() const {
@@ -829,12 +832,31 @@ void DiffractionExperiment::LoadDatasetSettings(const JFJochProtoBuf::DatasetSet
FilePrefix(settings.file_prefix());
DataFileCount(settings.data_file_count());
if (settings.has_unit_cell())
SetUnitCell(settings.unit_cell());
SetUnitCell(UnitCell(settings.unit_cell().a(),
settings.unit_cell().b(),
settings.unit_cell().c(),
settings.unit_cell().alpha(),
settings.unit_cell().beta(),
settings.unit_cell().gamma()));
else
SetUnitCell();
SpaceGroupNumber(settings.space_group_number());
SampleName(settings.sample_name());
Compression(settings.compression());
switch (settings.compression()) {
case JFJochProtoBuf::BSHUF_LZ4:
Compression(CompressionAlgorithm::BSHUF_LZ4);
break;
case JFJochProtoBuf::BSHUF_ZSTD:
Compression(CompressionAlgorithm::BSHUF_ZSTD);
break;
case JFJochProtoBuf::BSHUF_ZSTD_RLE:
Compression(CompressionAlgorithm::BSHUF_ZSTD_RLE);
break;
default:
case JFJochProtoBuf::NO_COMPRESSION:
Compression(CompressionAlgorithm::NO_COMPRESSION);
break;
}
SaveCalibration(settings.save_calibration());
if (settings.summation() == 0)
Summation(1);
@@ -943,7 +965,7 @@ void DiffractionExperiment::FillMessage(StartMessage &message) const {
message.pixel_size_y = GetPixelSize_mm() * 1e-3f;
message.sensor_material = SENSOR_MATERIAL;
message.sensor_thickness = SENSOR_THICKNESS_IN_UM * 1e-6f;
message.compression_algorithm = GetCompressionAlgorithmEnum();
message.compression_algorithm = GetCompressionAlgorithm();
message.compression_block_size = JFJochBitShuffleCompressor::DefaultBlockSize;
message.pixel_bit_depth = GetPixelDepth() * 8;
message.storage_cell_number = GetStorageCellNumber();
common/DiffractionExperiment.h
+2 -4
View File
@@ -33,7 +33,6 @@ class DiffractionExperiment {
JFJochProtoBuf::InternalSettings internal;
constexpr static const int64_t max_spot_count = 100;
DiffractionExperiment& SetUnitCell(const JFJochProtoBuf::UnitCell &input);
public:
// Public methods are atomic
DiffractionExperiment();
@@ -61,7 +60,7 @@ public:
DiffractionExperiment& FilePrefix(std::string input);
DiffractionExperiment& DataFileCount(int64_t input);
DiffractionExperiment& Compression(JFJochProtoBuf::Compression input);
DiffractionExperiment& Compression(CompressionAlgorithm input);
DiffractionExperiment& PreviewPeriod(std::chrono::microseconds input);
DiffractionExperiment& UseInternalPacketGenerator(bool input);
@@ -137,8 +136,7 @@ public:
std::string GetFilePrefix() const;
int64_t GetDataFileCount() const;
JFJochProtoBuf::Compression GetCompressionAlgorithm() const;
CompressionAlgorithm GetCompressionAlgorithmEnum() const;
CompressionAlgorithm GetCompressionAlgorithm() const;
int64_t GetMaxCompressedSize() const;
common/FrameTransformation.cpp
+1 -1
View File
@@ -17,7 +17,7 @@ template <class T> void FillVector(std::vector<char> &v, T fill_value) {
FrameTransformation::FrameTransformation(const DiffractionExperiment &in_experiment) :
experiment(in_experiment),
pixel_depth(experiment.GetPixelDepth()), pixel_signed(experiment.IsPixelSigned()),
compressor(in_experiment.GetCompressionAlgorithmEnum()) {
compressor(in_experiment.GetCompressionAlgorithm()) {
precompression_buffer.resize(experiment.GetPixelsNum() * pixel_depth);
common/ImagePusher.cpp
+1 -1
View File
@@ -12,6 +12,6 @@ void PrepareCBORImage(DataMessage& message,
message.image.pixel_depth_bytes = experiment.GetPixelDepth();
message.image.pixel_is_signed = experiment.IsPixelSigned();
message.image.pixel_is_float = false;
message.image.algorithm = experiment.GetCompressionAlgorithmEnum();
message.image.algorithm = experiment.GetCompressionAlgorithm();
message.image.channel = "default";
}
common/TestImagePusher.cpp
+2 -2
View File
@@ -89,10 +89,10 @@ bool TestImagePusher::CheckImage(const DiffractionExperiment &x, const std::vect
// Image decompression
try {
if (x.GetPixelDepth() == 2)
JFJochDecompress(decompressed_image_16, x.GetCompressionAlgorithmEnum(),
JFJochDecompress(decompressed_image_16, x.GetCompressionAlgorithm(),
receiver_generated_image, x.GetPixelsNum());
else
JFJochDecompress(decompressed_image_32, x.GetCompressionAlgorithmEnum(),
JFJochDecompress(decompressed_image_32, x.GetCompressionAlgorithm(),
receiver_generated_image, x.GetPixelsNum());
} catch (const JFJochException &e) {
grpc/jfjoch.proto
-5
View File
@@ -179,11 +179,6 @@ message InternalSettings {
bool debug_pixel_mask = 38;
}
message JungfraujochSettings {
DatasetSettings dataset = 1;
InternalSettings internal = 2;
}
// Calibration
message ModuleStatistics {
python/jfjoch_pb2.py
+63 -65
View File
File diff suppressed because one or more lines are too long
receiver/jfjoch_action_test.cpp
+1 -1
View File
@@ -101,7 +101,7 @@ int main(int argc, char **argv) {
x.ImagesPerTrigger(nimages).PedestalG0Frames(0).UseInternalPacketGenerator(true).PhotonEnergy_keV(12.4).NumTriggers(1);
x.MaskModuleEdges(false).MaskChipEdges(false);
x.Compression(JFJochProtoBuf::BSHUF_LZ4).DataStreams(nstreams);
x.Compression(CompressionAlgorithm::BSHUF_LZ4).DataStreams(nstreams);
logger.Info("Data streams {} Total modules {} Total images {} Threads {}", nstreams, nmodules, nimages, nthreads);
tests/DiffractionExperimentTest.cpp
+13 -17
View File
@@ -42,8 +42,8 @@ TEST_CASE("DiffractionExperiment_FilePath","[DiffractionExperiment]") {
TEST_CASE("DiffractionExperiment_Compression_Raw","[DiffractionExperiment]") {
DiffractionExperiment x;
for (auto i: {DetectorMode::Raw, DetectorMode::PedestalG0, DetectorMode::PedestalG1, DetectorMode::PedestalG2}) {
x.Mode(i).Compression(JFJochProtoBuf::BSHUF_ZSTD);
REQUIRE(x.GetCompressionAlgorithm() == JFJochProtoBuf::NO_COMPRESSION);
x.Mode(i).Compression(CompressionAlgorithm::BSHUF_ZSTD);
REQUIRE(x.GetCompressionAlgorithm() == CompressionAlgorithm::NO_COMPRESSION);
}
}
@@ -51,21 +51,17 @@ TEST_CASE("DiffractionExperiment_Compression","[DiffractionExperiment]") {
DiffractionExperiment x;
// Compression
x.Compression(JFJochProtoBuf::BSHUF_LZ4);
REQUIRE(x.GetCompressionAlgorithm() == JFJochProtoBuf::BSHUF_LZ4);
REQUIRE(x.GetCompressionAlgorithmEnum() == CompressionAlgorithm::BSHUF_LZ4);
x.Compression(CompressionAlgorithm::BSHUF_LZ4);
REQUIRE(x.GetCompressionAlgorithm() == CompressionAlgorithm::BSHUF_LZ4);
x.Compression(JFJochProtoBuf::BSHUF_ZSTD);
REQUIRE(x.GetCompressionAlgorithm() == JFJochProtoBuf::BSHUF_ZSTD);
REQUIRE(x.GetCompressionAlgorithmEnum() == CompressionAlgorithm::BSHUF_ZSTD);
x.Compression(CompressionAlgorithm::BSHUF_ZSTD);
REQUIRE(x.GetCompressionAlgorithm() == CompressionAlgorithm::BSHUF_ZSTD);
x.Compression(JFJochProtoBuf::BSHUF_ZSTD_RLE);
REQUIRE(x.GetCompressionAlgorithm() == JFJochProtoBuf::BSHUF_ZSTD_RLE);
REQUIRE(x.GetCompressionAlgorithmEnum() == CompressionAlgorithm::BSHUF_ZSTD_RLE);
x.Compression(CompressionAlgorithm::BSHUF_ZSTD_RLE);
REQUIRE(x.GetCompressionAlgorithm() == CompressionAlgorithm::BSHUF_ZSTD_RLE);
x.Compression(JFJochProtoBuf::NO_COMPRESSION);
REQUIRE(x.GetCompressionAlgorithm() == JFJochProtoBuf::NO_COMPRESSION);
REQUIRE(x.GetCompressionAlgorithmEnum() == CompressionAlgorithm::NO_COMPRESSION);
x.Compression(CompressionAlgorithm::NO_COMPRESSION);
REQUIRE(x.GetCompressionAlgorithm() == CompressionAlgorithm::NO_COMPRESSION);
}
//TODO: Update
@@ -643,7 +639,7 @@ TEST_CASE("DiffractionExperiment_DetectorInput_StorageCell","[DiffractionExperim
TEST_CASE("DiffractionExperiment_LoadDatasetSettings", "[DiffractionExperiment]") {
DiffractionExperiment x;
x.ImagesPerTrigger(567).BeamY_pxl(324).Compression(JFJochProtoBuf::BSHUF_ZSTD);
x.ImagesPerTrigger(567).BeamY_pxl(324).Compression(CompressionAlgorithm::BSHUF_ZSTD);
JFJochProtoBuf::DatasetSettings settings;
settings.set_images_per_trigger(234);
settings.set_ntrigger(56);
@@ -662,7 +658,7 @@ TEST_CASE("DiffractionExperiment_LoadDatasetSettings", "[DiffractionExperiment]"
REQUIRE(x.GetBeamX_pxl() == Approx(23.4));
REQUIRE(x.GetBeamY_pxl() == Approx(123.4));
REQUIRE(x.GetSpaceGroupNumber() == 45);
REQUIRE(x.GetCompressionAlgorithm() == JFJochProtoBuf::BSHUF_LZ4);
REQUIRE(x.GetCompressionAlgorithm() == CompressionAlgorithm::BSHUF_LZ4);
REQUIRE(x.GetSampleName() == "lyso1");
REQUIRE(x.GetDataFileCount() == 5);
REQUIRE(x.GetDetectorDistance_mm() == Approx(57.6));
@@ -672,7 +668,7 @@ TEST_CASE("DiffractionExperiment_LoadDatasetSettings", "[DiffractionExperiment]"
TEST_CASE("DiffractionExperiment_LoadDatasetSettings_Invalid", "[DiffractionExperiment]") {
DiffractionExperiment x;
x.ImagesPerTrigger(567).BeamY_pxl(324).Compression(JFJochProtoBuf::BSHUF_ZSTD);
x.ImagesPerTrigger(567).BeamY_pxl(324).Compression(CompressionAlgorithm::BSHUF_ZSTD);
JFJochProtoBuf::DatasetSettings settings;
settings.set_images_per_trigger(-1);
settings.set_ntrigger(56);
tests/FrameTransformationTest.cpp
+12 -12
View File
@@ -21,7 +21,7 @@ TEST_CASE("FrameTransformation_Raw_NoCompression" ,"") {
experiment.DataStreams(ndatastreams);
experiment.Mode(DetectorMode::Raw);
experiment.Compression(JFJochProtoBuf::NO_COMPRESSION);
experiment.Compression(CompressionAlgorithm::NO_COMPRESSION);
FrameTransformation transformation(experiment);
@@ -61,7 +61,7 @@ TEST_CASE("FrameTransformation_Converted_NoCompression" ,"") {
const uint16_t ndatastreams = 2;
DiffractionExperiment experiment(DetectorGeometry(ndatastreams * nmodules, 2));
experiment.DataStreams(ndatastreams);
experiment.Mode(DetectorMode::Conversion).Compression(JFJochProtoBuf::NO_COMPRESSION);
experiment.Mode(DetectorMode::Conversion).Compression(CompressionAlgorithm::NO_COMPRESSION);
FrameTransformation transformation(experiment);
@@ -111,7 +111,7 @@ TEST_CASE("FrameTransformation_Converted_bshuf_lz4" ,"") {
DiffractionExperiment experiment(DetectorGeometry(ndatastreams * nmodules, 2));
experiment.DataStreams(ndatastreams);
experiment.Mode(DetectorMode::Conversion).Compression(JFJochProtoBuf::BSHUF_LZ4);
experiment.Mode(DetectorMode::Conversion).Compression(CompressionAlgorithm::BSHUF_LZ4);
FrameTransformation transformation(experiment);
@@ -142,7 +142,7 @@ TEST_CASE("FrameTransformation_Converted_bshuf_lz4" ,"") {
REQUIRE(bshuf_read_uint32_BE(output_compressed.data()+8) == JFJochBitShuffleCompressor::DefaultBlockSize * experiment.GetPixelDepth());
std::vector<int16_t> output;
REQUIRE_NOTHROW(JFJochDecompress(output, experiment.GetCompressionAlgorithmEnum(), output_compressed,
REQUIRE_NOTHROW(JFJochDecompress(output, experiment.GetCompressionAlgorithm(), output_compressed,
experiment.GetPixelsNum()));
REQUIRE(input_0[511*1024] == output[CONVERTED_MODULE_SIZE * (2 * nmodules - 2) + 0]);
@@ -171,7 +171,7 @@ TEST_CASE("FrameTransformation_Converted_bshuf_zstd" ,"") {
DiffractionExperiment experiment(DetectorGeometry(ndatastreams * nmodules, 2));
experiment.DataStreams(ndatastreams);
experiment.Mode(DetectorMode::Conversion).Compression(JFJochProtoBuf::BSHUF_ZSTD);
experiment.Mode(DetectorMode::Conversion).Compression(CompressionAlgorithm::BSHUF_ZSTD);
FrameTransformation transformation(experiment);
@@ -201,7 +201,7 @@ TEST_CASE("FrameTransformation_Converted_bshuf_zstd" ,"") {
output_compressed.resize(compressed_size);
std::vector<int16_t> output;
REQUIRE_NOTHROW(JFJochDecompress(output, experiment.GetCompressionAlgorithmEnum(), output_compressed,
REQUIRE_NOTHROW(JFJochDecompress(output, experiment.GetCompressionAlgorithm(), output_compressed,
experiment.GetPixelsNum()));
REQUIRE(input_0[511*1024] == output[CONVERTED_MODULE_SIZE * (2 * nmodules - 2) + 0]);
@@ -230,7 +230,7 @@ TEST_CASE("FrameTransformation_Converted_bshuf_zstd_rle" ,"") {
DiffractionExperiment experiment(DetectorGeometry(ndatastreams * nmodules, 2));
experiment.DataStreams(ndatastreams);
experiment.Mode(DetectorMode::Conversion).Compression(JFJochProtoBuf::BSHUF_ZSTD_RLE);
experiment.Mode(DetectorMode::Conversion).Compression(CompressionAlgorithm::BSHUF_ZSTD_RLE);
FrameTransformation transformation(experiment);
@@ -260,7 +260,7 @@ TEST_CASE("FrameTransformation_Converted_bshuf_zstd_rle" ,"") {
output_compressed.resize(compressed_size);
std::vector<int16_t> output;
REQUIRE_NOTHROW(JFJochDecompress(output, experiment.GetCompressionAlgorithmEnum(), output_compressed,
REQUIRE_NOTHROW(JFJochDecompress(output, experiment.GetCompressionAlgorithm(), output_compressed,
experiment.GetPixelsNum()));
REQUIRE(input_0[511*1024] == output[CONVERTED_MODULE_SIZE * (2 * nmodules - 2) + 0]);
@@ -289,7 +289,7 @@ TEST_CASE("FrameTransformation_Converted_bshuf_zstd_32bit" ,"") {
DiffractionExperiment experiment(DetectorGeometry(ndatastreams * nmodules, 2));
experiment.DataStreams(ndatastreams);
experiment.Mode(DetectorMode::Conversion).Compression(JFJochProtoBuf::BSHUF_ZSTD).FPGAOutputMode(JFJochProtoBuf::INT32);
experiment.Mode(DetectorMode::Conversion).Compression(CompressionAlgorithm::BSHUF_ZSTD).FPGAOutputMode(JFJochProtoBuf::INT32);
FrameTransformation transformation(experiment);
@@ -322,7 +322,7 @@ TEST_CASE("FrameTransformation_Converted_bshuf_zstd_32bit" ,"") {
output_compressed.resize(compressed_size);
std::vector<int32_t> output;
REQUIRE_NOTHROW(JFJochDecompress(output, experiment.GetCompressionAlgorithmEnum(), output_compressed,
REQUIRE_NOTHROW(JFJochDecompress(output, experiment.GetCompressionAlgorithm(), output_compressed,
experiment.GetPixelsNum()));
REQUIRE(input_0[511*1024] == output[CONVERTED_MODULE_SIZE * (2 * nmodules - 2) + 0]);
@@ -353,7 +353,7 @@ TEST_CASE("FrameTransformation_Converted_bshuf_zstd_unsigned_16bit" ,"") {
DiffractionExperiment experiment(DetectorGeometry(ndatastreams * nmodules, 2));
experiment.DataStreams(ndatastreams);
experiment.Mode(DetectorMode::Conversion).Compression(JFJochProtoBuf::BSHUF_ZSTD).FPGAOutputMode(JFJochProtoBuf::UINT16);
experiment.Mode(DetectorMode::Conversion).Compression(CompressionAlgorithm::BSHUF_ZSTD).FPGAOutputMode(JFJochProtoBuf::UINT16);
REQUIRE(!experiment.IsPixelSigned());
REQUIRE(experiment.GetPixelDepth() == 2);
@@ -389,7 +389,7 @@ TEST_CASE("FrameTransformation_Converted_bshuf_zstd_unsigned_16bit" ,"") {
output_compressed.resize(compressed_size);
std::vector<uint16_t> output;
REQUIRE_NOTHROW(JFJochDecompress(output, experiment.GetCompressionAlgorithmEnum(), output_compressed,
REQUIRE_NOTHROW(JFJochDecompress(output, experiment.GetCompressionAlgorithm(), output_compressed,
experiment.GetPixelsNum()));
REQUIRE(input_0[511*1024] == output[CONVERTED_MODULE_SIZE * (2 * nmodules - 2) + 0]);
tests/HDF5WritingTest.cpp
+6 -7
View File
@@ -170,7 +170,7 @@ TEST_CASE("HDF5Writer", "[HDF5][Full]") {
DiffractionExperiment x(DetectorGeometry(8, 2, 8, 36));
std::vector<SpotToSave> spots;
x.FilePrefix("test02_1p10").ImagesPerTrigger(5).DataFileCount(2).Compression(JFJochProtoBuf::NO_COMPRESSION);
x.FilePrefix("test02_1p10").ImagesPerTrigger(5).DataFileCount(2).Compression(CompressionAlgorithm::NO_COMPRESSION);
StartMessage start_message;
x.FillMessage(start_message);
@@ -201,7 +201,7 @@ TEST_CASE("HDF5Writer_Spots", "[HDF5][Full]") {
spots.push_back({20,50,12});
spots.push_back({1000,500,3});
x.FilePrefix("test02_1p10_spots").ImagesPerTrigger(5).DataFileCount(2).Compression(JFJochProtoBuf::NO_COMPRESSION);
x.FilePrefix("test02_1p10_spots").ImagesPerTrigger(5).DataFileCount(2).Compression(CompressionAlgorithm::NO_COMPRESSION);
StartMessage start_message;
x.FillMessage(start_message);
@@ -234,7 +234,7 @@ TEST_CASE("HDF5Writer_Rad_Int_Profile", "[HDF5][Full]") {
std::vector<float> rad_int_profile(mapping.GetBinNumber(), 4.0);
std::vector<float> rad_int_avg(mapping.GetBinNumber(), 0.33);
x.FilePrefix("test02_1p10_rad_int").ImagesPerTrigger(5).DataFileCount(2).Compression(JFJochProtoBuf::NO_COMPRESSION);
x.FilePrefix("test02_1p10_rad_int").ImagesPerTrigger(5).DataFileCount(2).Compression(CompressionAlgorithm::NO_COMPRESSION);
StartMessage start_message;
x.FillMessage(start_message);
start_message.rad_int_bin_to_q = mapping.GetBinToQ();
@@ -259,8 +259,7 @@ TEST_CASE("HDF5Writer_Rad_Int_Profile", "[HDF5][Full]") {
TEST_CASE("HDF5Writer_VDS", "[HDF5][Full]") {
DiffractionExperiment x(DetectorGeometry(1));
x.ImagesPerTrigger(5).DataFileCount(5)
.Compression(JFJochProtoBuf::NO_COMPRESSION).FilePrefix("vds");
x.ImagesPerTrigger(5).DataFileCount(5).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("vds");
{
RegisterHDF5Filter();
@@ -314,7 +313,7 @@ TEST_CASE("HDF5Writer_VDS", "[HDF5][Full]") {
TEST_CASE("HDF5Writer_VDS_missing", "[HDF5][Full]") {
DiffractionExperiment x(DetectorGeometry(1));
x.ImagesPerTrigger(5).DataFileCount(5).Compression(JFJochProtoBuf::NO_COMPRESSION).FilePrefix("vds_missing");
x.ImagesPerTrigger(5).DataFileCount(5).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("vds_missing");
{
RegisterHDF5Filter();
@@ -370,7 +369,7 @@ TEST_CASE("HDF5Writer_VDS_missing", "[HDF5][Full]") {
TEST_CASE("HDF5Writer_VDS_zero_images", "[HDF5][Full]") {
DiffractionExperiment x(DetectorGeometry(1));
x.ImagesPerTrigger(5).DataFileCount(5).Compression(JFJochProtoBuf::NO_COMPRESSION).FilePrefix("vds_zero");
x.ImagesPerTrigger(5).DataFileCount(5).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("vds_zero");
{
RegisterHDF5Filter();
tests/JFJochReceiverIntegrationTest.cpp
+6 -6
View File
@@ -15,7 +15,7 @@ TEST_CASE("JFJochReceiverTest_Raw", "[JFJochReceiver]") {
x.Mode(DetectorMode::Raw);
x.PedestalG0Frames(0).NumTriggers(1).UseInternalPacketGenerator(true)
.ImagesPerTrigger(100).DataFileCount(16).PhotonEnergy_keV(12.4).Compression(JFJochProtoBuf::NO_COMPRESSION);
.ImagesPerTrigger(100).DataFileCount(16).PhotonEnergy_keV(12.4).Compression(CompressionAlgorithm::NO_COMPRESSION);
AcquisitionDeviceGroup aq_devices;
for (int i = 0; i < x.GetDataStreamsNum(); i++) {
@@ -46,7 +46,7 @@ TEST_CASE("JFJochReceiverTest_Conversion", "[JFJochReceiver]") {
x.Mode(DetectorMode::Conversion);
x.PedestalG0Frames(0).NumTriggers(1).UseInternalPacketGenerator(true)
.ImagesPerTrigger(32).DataFileCount(16).PhotonEnergy_keV(12.4).Compression(JFJochProtoBuf::BSHUF_ZSTD);
.ImagesPerTrigger(32).DataFileCount(16).PhotonEnergy_keV(12.4).Compression(CompressionAlgorithm::BSHUF_ZSTD);
AcquisitionDeviceGroup aq_devices;
for (int i = 0; i < x.GetDataStreamsNum(); i++) {
@@ -73,7 +73,7 @@ TEST_CASE("JFJochReceiverTest_Conversion_U16", "[JFJochReceiver]") {
x.Mode(DetectorMode::Conversion);
x.PedestalG0Frames(0).NumTriggers(1).UseInternalPacketGenerator(true)
.ImagesPerTrigger(32).DataFileCount(16).PhotonEnergy_keV(12.4).Compression(JFJochProtoBuf::BSHUF_ZSTD).FPGAOutputMode(JFJochProtoBuf::UINT16);
.ImagesPerTrigger(32).DataFileCount(16).PhotonEnergy_keV(12.4).Compression(CompressionAlgorithm::BSHUF_ZSTD).FPGAOutputMode(JFJochProtoBuf::UINT16);
REQUIRE(!x.IsPixelSigned());
AcquisitionDeviceGroup aq_devices;
@@ -101,7 +101,7 @@ TEST_CASE("JFJochReceiverTest_Conversion_I32", "[JFJochReceiver]") {
x.Mode(DetectorMode::Conversion);
x.PedestalG0Frames(0).NumTriggers(1).UseInternalPacketGenerator(true)
.ImagesPerTrigger(32).DataFileCount(16).PhotonEnergy_keV(12.4).Compression(JFJochProtoBuf::BSHUF_ZSTD).FPGAOutputMode(JFJochProtoBuf::INT32);
.ImagesPerTrigger(32).DataFileCount(16).PhotonEnergy_keV(12.4).Compression(CompressionAlgorithm::BSHUF_ZSTD).FPGAOutputMode(JFJochProtoBuf::INT32);
AcquisitionDeviceGroup aq_devices;
for (int i = 0; i < x.GetDataStreamsNum(); i++) {
@@ -128,7 +128,7 @@ TEST_CASE("JFJochReceiverTest_Conversion_Summation2", "[JFJochReceiver]") {
x.Mode(DetectorMode::Conversion);
x.PedestalG0Frames(0).NumTriggers(1).UseInternalPacketGenerator(true)
.ImagesPerTrigger(32).DataFileCount(16).PhotonEnergy_keV(12.4).Compression(JFJochProtoBuf::BSHUF_ZSTD).Summation(2);
.ImagesPerTrigger(32).DataFileCount(16).PhotonEnergy_keV(12.4).Compression(CompressionAlgorithm::BSHUF_ZSTD).Summation(2);
AcquisitionDeviceGroup aq_devices;
for (int i = 0; i < x.GetDataStreamsNum(); i++) {
@@ -155,7 +155,7 @@ TEST_CASE("JFJochReceiverTest_Conversion_StorageCell", "[JFJochReceiver]") {
x.Mode(DetectorMode::Conversion);
x.PedestalG0Frames(0).NumTriggers(1).UseInternalPacketGenerator(true)
.ImagesPerTrigger(32).DataFileCount(16).PhotonEnergy_keV(12.4).Compression(JFJochProtoBuf::BSHUF_ZSTD).StorageCells(16);
.ImagesPerTrigger(32).DataFileCount(16).PhotonEnergy_keV(12.4).Compression(CompressionAlgorithm::BSHUF_ZSTD).StorageCells(16);
REQUIRE(x.GetImageNum() == 16);
REQUIRE(x.GetStorageCellNumber() == 16);
tests/MockAcquisitionDeviceTest.cpp
+2 -2
View File
@@ -48,7 +48,7 @@ TEST_CASE("JFJochReceiverTest_Raw_MockAcquisitionDevice", "[JFJochReceiver]") {
x.Mode(DetectorMode::Raw);
x.PedestalG0Frames(0).NumTriggers(1).UseInternalPacketGenerator(true)
.ImagesPerTrigger(100).DataFileCount(16).PhotonEnergy_keV(12.4).Compression(JFJochProtoBuf::NO_COMPRESSION);
.ImagesPerTrigger(100).DataFileCount(16).PhotonEnergy_keV(12.4).Compression(CompressionAlgorithm::NO_COMPRESSION);
AcquisitionDeviceGroup aq_devices;
for (int i = 0; i < x.GetDataStreamsNum(); i++) {
@@ -82,7 +82,7 @@ TEST_CASE("JFJochReceiverTest_Conversion_MockAcquisitionDevice", "[JFJochReceive
x.Mode(DetectorMode::Conversion);
x.PedestalG0Frames(0).NumTriggers(1).UseInternalPacketGenerator(true).DataStreams(x.GetModulesNum())
.ImagesPerTrigger(32).DataFileCount(16).PhotonEnergy_keV(12.4).Compression(JFJochProtoBuf::BSHUF_ZSTD);
.ImagesPerTrigger(32).DataFileCount(16).PhotonEnergy_keV(12.4).Compression(CompressionAlgorithm::BSHUF_ZSTD);
AcquisitionDeviceGroup aq_devices;
for (int i = 0; i < x.GetDataStreamsNum(); i++) {
tests/ZSTDCompressorTest.cpp
+8 -8
View File
@@ -227,14 +227,14 @@ TEST_CASE("JFjochZstdCompressor_Frame_ones","[ZSTD]") {
TEST_CASE("JFJochCompressor_JFJochDecompressor_ZSTD","[ZSTD]") {
DiffractionExperiment x(DetectorGeometry(8, 2, 8, 36));
x.Compression(JFJochProtoBuf::BSHUF_ZSTD).FPGAOutputMode(JFJochProtoBuf::INT32);
x.Compression(CompressionAlgorithm::BSHUF_ZSTD).FPGAOutputMode(JFJochProtoBuf::INT32);
std::vector<int32_t> image(x.GetPixelsNum());
for (auto &i: image)
i = 345;
JFJochBitShuffleCompressor compressor(x.GetCompressionAlgorithmEnum());
JFJochBitShuffleCompressor compressor(x.GetCompressionAlgorithm());
std::vector<char> tmp(x.GetPixelsNum() * sizeof(int32_t) * 4 + 12);
@@ -242,21 +242,21 @@ TEST_CASE("JFJochCompressor_JFJochDecompressor_ZSTD","[ZSTD]") {
tmp.resize(tmp_size);
std::vector<int32_t> output;
REQUIRE_NOTHROW(JFJochDecompress(output, x.GetCompressionAlgorithmEnum(), tmp, x.GetPixelsNum()));
REQUIRE_NOTHROW(JFJochDecompress(output, x.GetCompressionAlgorithm(), tmp, x.GetPixelsNum()));
REQUIRE(output.size() == x.GetPixelsNum() * x.GetPixelDepth());
REQUIRE(memcmp(image.data(), output.data(), x.GetPixelsNum() * sizeof(int32_t)) == 0);
}
TEST_CASE("JFJochCompressor_JFJochDecompressor_LZ4","[ZSTD]") {
DiffractionExperiment x(DetectorGeometry(8, 2, 8, 36));
x.Compression(JFJochProtoBuf::BSHUF_LZ4).FPGAOutputMode(JFJochProtoBuf::INT32);
x.Compression(CompressionAlgorithm::BSHUF_LZ4).FPGAOutputMode(JFJochProtoBuf::INT32);
std::vector<int32_t> image(x.GetPixelsNum());
for (auto &i: image)
i = 5678;
JFJochBitShuffleCompressor compressor(x.GetCompressionAlgorithmEnum());
JFJochBitShuffleCompressor compressor(x.GetCompressionAlgorithm());
std::vector<char> tmp(x.GetPixelsNum() * sizeof(int32_t) * 4 + 12);
@@ -264,21 +264,21 @@ TEST_CASE("JFJochCompressor_JFJochDecompressor_LZ4","[ZSTD]") {
tmp.resize(tmp_size);
std::vector<int32_t> output;
REQUIRE_NOTHROW(JFJochDecompress(output, x.GetCompressionAlgorithmEnum(), tmp, x.GetPixelsNum()));
REQUIRE_NOTHROW(JFJochDecompress(output, x.GetCompressionAlgorithm(), tmp, x.GetPixelsNum()));
REQUIRE(output.size() == x.GetPixelsNum() * x.GetPixelDepth());
REQUIRE(memcmp(image.data(), output.data(), x.GetPixelsNum() * sizeof(int32_t)) == 0);
}
TEST_CASE("JFJochDecompressor_None","[ZSTD]") {
DiffractionExperiment x(DetectorGeometry(8, 2, 8, 36));
x.Compression(JFJochProtoBuf::NO_COMPRESSION).FPGAOutputMode(JFJochProtoBuf::INT32);
x.Compression(CompressionAlgorithm::NO_COMPRESSION).FPGAOutputMode(JFJochProtoBuf::INT32);
std::vector<int32_t> image(x.GetPixelsNum());
for (auto &i: image)
i = 578;
std::vector<int32_t> output;
REQUIRE_NOTHROW(JFJochDecompress(output, x.GetCompressionAlgorithmEnum(), image, x.GetPixelsNum()));
REQUIRE_NOTHROW(JFJochDecompress(output, x.GetCompressionAlgorithm(), image, x.GetPixelsNum()));
REQUIRE(output.size() == x.GetPixelsNum() * x.GetPixelDepth());
REQUIRE(memcmp(image.data(), output.data(), x.GetPixelsNum() * sizeof(int32_t)) == 0);
}
tools/CompressionBenchmark.cpp
+10 -10
View File
@@ -76,7 +76,7 @@ std::string CheckDecompression(const DiffractionExperiment &x, size_t nimages, c
std::vector<uint16_t> decompress_v;
auto start_time = std::chrono::system_clock::now();
for (int i = 0; i < nimages; i++) {
JFJochDecompress(decompress_v, x.GetCompressionAlgorithmEnum(), output[i], x.GetPixelsNum());
JFJochDecompress(decompress_v, x.GetCompressionAlgorithm(), output[i], x.GetPixelsNum());
}
auto end_time = std::chrono::system_clock::now();
@@ -165,36 +165,36 @@ int main(int argc, char **argv) {
x.MaskChipEdges(false);
x.Compression(JFJochProtoBuf::NO_COMPRESSION);
x.Compression(CompressionAlgorithm::NO_COMPRESSION);
for (int i = 0; i < 3; i++)
CheckCompression(x, nimages, image, nthreads);
for (int i = 0; i <3; i++) {
x.Compression(JFJochProtoBuf::NO_COMPRESSION).Mode(DetectorMode::Raw);
x.Compression(CompressionAlgorithm::NO_COMPRESSION).Mode(DetectorMode::Raw);
std::cout << "None (memcpy) " << CheckCompression(x, nimages, image, nthreads);
x.Compression(JFJochProtoBuf::NO_COMPRESSION).Mode(DetectorMode::Conversion);
x.Compression(CompressionAlgorithm::NO_COMPRESSION).Mode(DetectorMode::Conversion);
std::cout << "None (geom transform) " << CheckCompression(x, nimages, image, nthreads);
x.Compression(JFJochProtoBuf::BSHUF_LZ4);
x.Compression(CompressionAlgorithm::BSHUF_LZ4);
std::cout << "BSHUF/LZ4 " << CheckCompression(x, nimages, image, nthreads);
x.Compression(JFJochProtoBuf::BSHUF_ZSTD);
x.Compression(CompressionAlgorithm::BSHUF_ZSTD);
std::cout << "BSHUF/ZSTD (0) " << CheckCompression(x, nimages, image, nthreads);
x.Compression(JFJochProtoBuf::BSHUF_ZSTD_RLE);
x.Compression(CompressionAlgorithm::BSHUF_ZSTD_RLE);
std::cout << "BSHUF/ZSTD (RLE) " << CheckCompression(x, nimages, image, nthreads);
std::cout << std::endl;
}
std::cout << std::endl << std::endl << "Decompression" << std::endl << std::endl;
x.Compression(JFJochProtoBuf::BSHUF_LZ4);
x.Compression(CompressionAlgorithm::BSHUF_LZ4);
std::cout << "BSHUF/LZ4 " << CheckDecompression(x, nimages, image);
x.Compression(JFJochProtoBuf::BSHUF_ZSTD);
x.Compression(CompressionAlgorithm::BSHUF_ZSTD);
std::cout << "BSHUF/ZSTD " << CheckDecompression(x, nimages, image);
x.Compression(JFJochProtoBuf::BSHUF_ZSTD_RLE);
x.Compression(CompressionAlgorithm::BSHUF_ZSTD_RLE);
std::cout << "BSHUF/ZSTD (RLE) " << CheckDecompression(x, nimages, image);
}