From b5da66baf67ccc6819c69173181f2bc84cea63d4 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Tue, 7 May 2024 14:11:21 +0200 Subject: [PATCH] CompressionBenchmark: Remove --- .gitlab-ci.yml | 3 +- tools/CMakeLists.txt | 5 +- tools/CompressionBenchmark.cpp | 203 --------------------------------- 3 files changed, 2 insertions(+), 209 deletions(-) delete mode 100644 tools/CompressionBenchmark.cpp diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e935caa0..b70d7edb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -118,12 +118,11 @@ test:x86:gcc: - mkdir -p build - cd build - cmake -DCMAKE_BUILD_TYPE=Release .. - - make -j48 CatchTest CompressionBenchmark HDF5DatasetWriteTest + - make -j48 CatchTest HDF5DatasetWriteTest - cd tests - ./CatchTest -r junit -o report.xml - cd ../tools - ./HDF5DatasetWriteTest ../../tests/test_data/compression_benchmark.h5 - - numactl -m 0 -N 0 ./CompressionBenchmark ../../tests/test_data/compression_benchmark.h5 artifacts: expire_in: 1 week reports: diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index a9faae75..10a8fc83 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -2,9 +2,6 @@ ADD_EXECUTABLE(jfjoch_udp_simulator jfjoch_udp_simulator.cpp UDPSimulator.cpp UDPSimulator.h) TARGET_LINK_LIBRARIES(jfjoch_udp_simulator JFJochCommon) -add_executable(CompressionBenchmark CompressionBenchmark.cpp) -target_link_libraries(CompressionBenchmark JFJochWriter JFJochReceiver JFJochCommon) - add_executable(HDF5DatasetWriteTest HDF5DatasetWriteTest.cpp) target_link_libraries(HDF5DatasetWriteTest JFJochWriter JFJochReceiver JFJochCommon) @@ -23,4 +20,4 @@ TARGET_LINK_LIBRARIES(SpotFindingPerformanceTest JFJochImageAnalysis JFJochCommo ADD_EXECUTABLE(AzimIntDataset AzimIntDataset.cpp) TARGET_LINK_LIBRARIES(AzimIntDataset JFJochImageAnalysis JFJochWriter JFJochCommon) -INSTALL(TARGETS jfjoch_udp_simulator CompressionBenchmark HDF5DatasetWriteTest jfjoch_writer_test AzimIntDataset RUNTIME) +INSTALL(TARGETS jfjoch_udp_simulator HDF5DatasetWriteTest jfjoch_writer_test AzimIntDataset RUNTIME) diff --git a/tools/CompressionBenchmark.cpp b/tools/CompressionBenchmark.cpp deleted file mode 100644 index 9f3c8a0b..00000000 --- a/tools/CompressionBenchmark.cpp +++ /dev/null @@ -1,203 +0,0 @@ -// Copyright (2019-2023) Paul Scherrer Institute - -#include -#include - -#include "bitshuffle/bitshuffle_core.h" -#include "zstd/lib/zstd.h" -#include "../writer/HDF5Objects.h" -#include "../receiver/FrameTransformation.h" -#include "../common/RawToConvertedGeometry.h" -#include "JFJochDecompress.h" - -double CheckCompressionThread(const DiffractionExperiment &x, - int32_t nimages, - int32_t image0, - int32_t stride, - const std::vector &image, - std::vector &output) { - double ret = 0; - - FrameTransformation transformation(x); - for (int32_t i = image0; i < nimages; i += stride) { - for (int j = 0; j < x.GetModulesNum(); j++ ) { - transformation.ProcessModule(image.data() + (j + i * x.GetModulesNum()) * RAW_MODULE_SIZE, j, 0); - } - auto image = transformation.GetCompressedImage(); - ret += image.size; - } - return ret; -} - -std::string CheckCompression(const DiffractionExperiment &x, int32_t nimages, const std::vector &image, uint16_t nthreads) { - - double original_size = nimages * x.GetModulesNum() * RAW_MODULE_SIZE * x.GetPixelDepth(); - - std::vector output(nimages * x.GetMaxCompressedSize()); - - double compressed_size = 0; - auto start_time = std::chrono::system_clock::now(); - - std::vector> futures; - - for (int i = 0; i < nthreads; i++) - futures.emplace_back(std::async(std::launch::async, &CheckCompressionThread, std::ref(x), - nimages, i, nthreads, - std::ref(image), std::ref(output))); - - for (int i = 0; i < nthreads; i++) - compressed_size += futures[i].get(); - - auto end_time = std::chrono::system_clock::now(); - auto elapsed = std::chrono::duration_cast(end_time - start_time); - char buf[256]; - - snprintf(buf, 255, "%.2fx compression (%5.2f bits/pxl) Throughput: %5.2f GB/s\n", original_size / compressed_size, compressed_size * 8 / double(nimages * x.GetModulesNum() * RAW_MODULE_SIZE), (original_size / elapsed.count()) / 1000.0); - return std::string(buf); -} - -std::string CheckDecompression(const DiffractionExperiment &x, size_t nimages, const std::vector &image) { - double original_size = nimages * x.GetModulesNum() * RAW_MODULE_SIZE * x.GetPixelDepth(); - - FrameTransformation transformation(x); - std::vector output[nimages]; - for (auto &v: output) - v.resize(x.GetMaxCompressedSize()); - - std::vector compressed_size(nimages); - - for (int i = 0; i < nimages; i++) { - for (int j = 0; j < x.GetModulesNum(); j++ ) { - transformation.ProcessModule(image.data() + (j + i * x.GetModulesNum()) * RAW_MODULE_SIZE, j, 0); - } - auto image = transformation.GetCompressedImage(); - compressed_size[i] = image.size; - output[i].resize(image.size); - memcpy(output[i].data(), image.data, image.size); - } - - std::vector decompress_v; - auto start_time = std::chrono::system_clock::now(); - for (int i = 0; i < nimages; i++) { - JFJochDecompress(decompress_v, x.GetCompressionAlgorithm(), output[i], x.GetPixelsNum()); - } - - auto end_time = std::chrono::system_clock::now(); - auto elapsed = std::chrono::duration_cast(end_time - start_time); - char buf[256]; - - snprintf(buf, 255, " decompression Throughput: %5.2f GB/s\n", (original_size / elapsed.count()) / 1000.0); - return {buf}; -} - -int main(int argc, char **argv) { - - RegisterHDF5Filter(); - - if ((argc != 2) && (argc != 3) && (argc != 4)) { - std::cout << "Usage: ./CompressionBenchmark {}" << std::endl; - exit(EXIT_FAILURE); - } - - uint32_t nthreads = 1; - if (argc >= 3) - nthreads = atol(argv[2]); - - uint64_t nimages = 25; - if (argc == 4) - nimages = atol(argv[3]); - - if ((nthreads <= 0) || (nimages <= 0)) { - std::cerr << "Error in input parameters" << std::endl; - exit(EXIT_FAILURE); - } - - HDF5ReadOnlyFile data(argv[1]); - - HDF5DataSet dataset(data, "/entry/data/data"); - - HDF5DataSpace file_space(dataset); - - if (file_space.GetNumOfDimensions() != 3) { - std::cout << "/entry/data/data must be 3D" << std::endl; - exit(EXIT_FAILURE); - } - - DiffractionExperiment x(DetectorGeometry(8, 2, 8, 36)); - - if ((file_space.GetDimensions()[1] == 2164) && (file_space.GetDimensions()[2] == 2068)) { - std::cout << "JF4M with gaps detected (2068 x 2164)" << std::endl; - } else { - std::cout << "Unknown geometry - exiting" << std::endl; - exit(EXIT_FAILURE); - } - - uint64_t nimages_dataset = file_space.GetDimensions()[0]; - std::cout << "Number of images in the dataset: " << nimages_dataset << std::endl; - if (nimages_dataset > 200) { - nimages_dataset = 200; - std::cout << "Using only " << nimages_dataset << " images" << std::endl; - } - x.Mode(DetectorMode::Conversion); - - std::vector image_conv ( nimages_dataset * file_space.GetDimensions()[1] * file_space.GetDimensions()[2]); - - - std::vector start = {0,0,0}; - std::vector dim = {nimages_dataset, file_space.GetDimensions()[1], file_space.GetDimensions()[2]}; - - auto start_time = std::chrono::system_clock::now(); - dataset.ReadVector(image_conv, start, dim); - auto end_time = std::chrono::system_clock::now(); - auto elapsed = std::chrono::duration_cast(end_time - start_time); - - std::cout << "Images loaded " << elapsed.count() / nimages_dataset << " ms/image " - << (image_conv.size() * sizeof(uint16_t)) / (1000.0 * elapsed.count()) << " MB/s" << std::endl; - - - std::cout << "Images to benchmark " << nimages << std::endl << "Threads: " << nthreads << std::endl << std::endl; - std::vector image( nimages * x.GetModulesNum() * RAW_MODULE_SIZE); - - for (int i = 0; i < nimages; i++) { - ConvertedToRawGeometry(x, - image.data() + i * RAW_MODULE_SIZE * x.GetModulesNum(), - image_conv.data() + (i % nimages_dataset) * file_space.GetDimensions()[1] * file_space.GetDimensions()[2]); - } - - auto image_shuffled = bitshuffle(image, 4096); - - x.MaskChipEdges(false); - - x.Compression(CompressionAlgorithm::NO_COMPRESSION); - for (int i = 0; i < 3; i++) - CheckCompression(x, nimages, image, nthreads); - - for (int i = 0; i <3; i++) { - x.Compression(CompressionAlgorithm::NO_COMPRESSION).Mode(DetectorMode::Raw); - std::cout << "None (memcpy) " << CheckCompression(x, nimages, image, nthreads); - x.Compression(CompressionAlgorithm::NO_COMPRESSION).Mode(DetectorMode::Conversion); - - std::cout << "None (geom transform) " << CheckCompression(x, nimages, image, nthreads); - - x.Compression(CompressionAlgorithm::BSHUF_LZ4); - std::cout << "BSHUF/LZ4 " << CheckCompression(x, nimages, image, nthreads); - - x.Compression(CompressionAlgorithm::BSHUF_ZSTD); - std::cout << "BSHUF/ZSTD (0) " << CheckCompression(x, nimages, image, nthreads); - - x.Compression(CompressionAlgorithm::BSHUF_ZSTD_RLE); - std::cout << "BSHUF/ZSTD (RLE) " << CheckCompression(x, nimages, image, nthreads); - std::cout << std::endl; - } - - std::cout << std::endl << std::endl << "Decompression" << std::endl << std::endl; - - x.Compression(CompressionAlgorithm::BSHUF_LZ4); - std::cout << "BSHUF/LZ4 " << CheckDecompression(x, nimages, image); - - x.Compression(CompressionAlgorithm::BSHUF_ZSTD); - std::cout << "BSHUF/ZSTD " << CheckDecompression(x, nimages, image); - - x.Compression(CompressionAlgorithm::BSHUF_ZSTD_RLE); - std::cout << "BSHUF/ZSTD (RLE) " << CheckDecompression(x, nimages, image); -} \ No newline at end of file