Move all image analysis related code to image_analysis/ directory

.gitmodules

@@ -1,5 +1,5 @@
-[submodule "indexing/fast-feedback-indexer"]
-	path = indexing/fast-feedback-indexer
+[submodule "image_analysis/fast-feedback-indexer"]
+	path = image_analysis/fast-feedback-indexer
 	url = https://github.com/paulscherrerinstitute/fast-feedback-indexer/
 [submodule "frame_serialize/tinycbor"]
 	path = frame_serialize/tinycbor

CMakeLists.txt

@@ -13,7 +13,7 @@ SET(CMAKE_C_FLAGS_RELEASE "-O3 -march=native -mtune=native")
 SET(CMAKE_CUDA_FLAGS_RELEASE "-O3")
 
 SET(JFJOCH_COMPILE_WRITER    ON CACHE BOOL "Compile HDF5 writer")
-SET(JFJOCH_COMPILE_FPGA      ON CACHE BOOL "Compile FPGA part")
+SET(JFJOCH_COMPILE_RECEIVER  ON CACHE BOOL "Compile image receiver")
 SET(JFJOCH_COMPILE_DETECTOR  ON CACHE BOOL "Compile detector control")
 SET(JFJOCH_COMPILE_INDEXER   ON CACHE BOOL "Compile indexer")
 SET(JFJOCH_COMPILE_TESTS     OFF CACHE BOOL "Compile tests")
@@ -35,12 +35,12 @@ ADD_SUBDIRECTORY(compression)
 ADD_SUBDIRECTORY(common)
 ADD_SUBDIRECTORY(broker)
 ADD_SUBDIRECTORY(etc)
-ADD_SUBDIRECTORY(indexing)
 
 SET(jfjoch_executables jfjoch_broker)
 
-IF (JFJOCH_COMPILE_TESTS OR JFJOCH_COMPILE_FPGA)
+IF (JFJOCH_COMPILE_TESTS OR JFJOCH_COMPILE_RECEIVER)
     ADD_SUBDIRECTORY(receiver)
+    ADD_SUBDIRECTORY(image_analysis)
     LIST(APPEND jfjoch_executables jfjoch_receiver)
 ENDIF()
 

README.md

@@ -99,7 +99,7 @@ Automated test routine is then accessible as `tests/CatchTest`. There are also b
 * `CompressionBenchmark` to measure compression bandwidth (single threaded)
 * `HDF5DatasetWriteTest` to measure HDF5 dataset writing speed (single threaded)
 * `DataAnalysisPerfTest` to measure data analysis performance (single threaded)
-* `PedestalPerfTest` to measure pedestal calculation performance
+* `JFCalibrationPerfTest` to measure pedestal calculation and online conversion performance
 
 In addition, tests are executed to verify that datasets written by Jungfraujoch are readable with XDS Durin plugin and CrystFEL. Input files for these programs are placed in `xds_durin` and `crystfel` folders. See `.gitlab-ci.yml` for details.
 

common/CMakeLists.txt

@@ -34,29 +34,19 @@ ADD_LIBRARY( CommonFunctions STATIC
         ThreadSafeFIFO.h
         ZMQPreviewPublisher.cpp ZMQPreviewPublisher.h
         ZMQImagePusher.cpp ZMQImagePusher.h
-        RadialIntegration.cpp RadialIntegration.h
         DiffractionSpot.cpp DiffractionSpot.h
-        StrongPixelSet.cpp StrongPixelSet.h
         Latch.cpp Latch.h
-        RadialIntegrationMapping.cpp RadialIntegrationMapping.h
         StatusVector.h
         ImagePusher.cpp ImagePusher.h
         TestImagePusher.cpp TestImagePusher.h
         SpotToSave.h
         NetworkAddressConvert.h NetworkAddressConvert.cpp
         grpcToJson.h jsonToGrpc.h to_fixed.h
-        GPUImageAnalysis.h GPUImageAnalysis.cu
         DiffractionExperiment.h DiffractionGeometry.cpp)
 
-FIND_LIBRARY(CUDART_LIBRARY cudart_static PATHS ${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES} REQUIRED)
-
-TARGET_LINK_LIBRARIES(CommonFunctions Compression FrameSerialize libzmq JFCalibration JFJochProtoBuf ${CUDART_LIBRARY} -lrt)
+TARGET_LINK_LIBRARIES(CommonFunctions Compression FrameSerialize libzmq JFCalibration JFJochProtoBuf -lrt)
 
 IF(HAS_NUMAIF AND NUMA_LIBRARY)
     TARGET_COMPILE_DEFINITIONS(CommonFunctions PRIVATE -DJFJOCH_USE_NUMA)
     TARGET_LINK_LIBRARIES(CommonFunctions ${NUMA_LIBRARY})
 ENDIF()
-
-FIND_PACKAGE(TIFF REQUIRED)
-FIND_LIBRARY(TIFFXX NAMES tiffxx REQUIRED DOC "Tiff C++ library")
-TARGET_LINK_LIBRARIES(CommonFunctions TIFF::TIFF ${TIFFXX})

image_analysis/CMakeLists.txt

@@ -1,6 +1,10 @@
-ADD_LIBRARY(DataProcessing STATIC
+ADD_LIBRARY(ImageAnalysis STATIC
         CrystalLattice.cpp CrystalLattice.h
         IndexerWrapper.cpp IndexerWrapper.h
+        GPUImageAnalysis.cu GPUImageAnalysis.h
+        RadialIntegration.cpp RadialIntegration.h
+        RadialIntegrationMapping.cpp RadialIntegrationMapping.h
+        StrongPixelSet.cpp StrongPixelSet.h
         fast-feedback-indexer/indexer/src/indexer.cpp
         fast-feedback-indexer/indexer/src/ffbidx/indexer.h
         fast-feedback-indexer/indexer/src/indexer_gpu.cu
@@ -10,8 +14,9 @@ ADD_LIBRARY(DataProcessing STATIC
         fast-feedback-indexer/indexer/src/ffbidx/log.h
         fast-feedback-indexer/indexer/src/ffbidx/exception.h)
 
-TARGET_INCLUDE_DIRECTORIES(DataProcessing PUBLIC
+TARGET_INCLUDE_DIRECTORIES(ImageAnalysis PUBLIC
         fast-feedback-indexer/indexer/src/
         fast-feedback-indexer/eigen)
 
-TARGET_LINK_LIBRARIES(DataProcessing CommonFunctions)
+FIND_LIBRARY(CUDART_LIBRARY cudart_static PATHS ${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES} REQUIRED)
+TARGET_LINK_LIBRARIES(ImageAnalysis CommonFunctions ${CUDART_LIBRARY})

image_analysis/GPUImageAnalysis.cu

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 #include "GPUImageAnalysis.h"
-#include "JFJochException.h"
+#include "../common/JFJochException.h"
 #include <sstream>
 
 // input      X x Y pixels array

image_analysis/RadialIntegration.cpp

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 #include "RadialIntegration.h"
-#include "JFJochException.h"
+#include "../common/JFJochException.h"
 
 RadialIntegration::RadialIntegration(const std::vector<uint16_t>& in_mapping, uint16_t in_nbins) :
         pixel_to_bin(in_mapping), nbins(in_nbins), sum(in_nbins, 0), count(in_nbins, 0)

image_analysis/RadialIntegration.h

@@ -8,7 +8,7 @@
 #include <cstdint>
 #include <cmath>
 
-#include "DiffractionExperiment.h"
+#include "../common/DiffractionExperiment.h"
 #include "RadialIntegrationMapping.h"
 
 class RadialIntegration {

image_analysis/RadialIntegrationMapping.cpp

@@ -4,7 +4,7 @@
 #include <cmath>
 
 #include "RadialIntegrationMapping.h"
-#include "JFJochException.h"
+#include "../common/JFJochException.h"
 
 RadialIntegrationMapping::RadialIntegrationMapping(const DiffractionExperiment& experiment, const uint8_t *one_byte_mask) :
         low_q(experiment.GetLowQForRadialInt_recipA()),

image_analysis/RadialIntegrationMapping.h

@@ -5,7 +5,7 @@
 #define JUNGFRAUJOCH_RADIALINTEGRATIONMAPPING_H
 
 #include <optional>
-#include "DiffractionExperiment.h"
+#include "../common/DiffractionExperiment.h"
 
 class RadialIntegrationMapping {
     const double low_q, high_q, q_spacing;

image_analysis/StrongPixelSet.h

@@ -6,9 +6,9 @@
 
 #include <unordered_map>
 #include <mutex>
-#include "DiffractionExperiment.h"
-#include "Coord.h"
-#include "DiffractionSpot.h"
+#include "../common/DiffractionExperiment.h"
+#include "../common/Coord.h"
+#include "../common/DiffractionSpot.h"
 
 inline uint32_t strong_pixel_coord(uint16_t col, uint16_t line) {
     return col + (static_cast<uint32_t>(line) << 16u);

receiver/CMakeLists.txt

@@ -35,7 +35,7 @@ ADD_LIBRARY(JFJochReceiver STATIC
         JFJochReceiverTest.cpp JFJochReceiverTest.h
         JFJochReceiver.cpp JFJochReceiver.h)
 
-TARGET_LINK_LIBRARIES(JFJochReceiver DataProcessing JungfraujochHost CommonFunctions HLSSimulation)
+TARGET_LINK_LIBRARIES(JFJochReceiver ImageAnalysis JungfraujochHost CommonFunctions HLSSimulation)
 
 IF(HAS_NUMA_H AND NUMA_LIBRARY)
     TARGET_COMPILE_DEFINITIONS(JFJochReceiver PRIVATE -DJFJOCH_USE_NUMA_H)

receiver/JFJochReceiver.cpp

@@ -5,9 +5,9 @@
 
 #include <thread>
 
-#include "../common/GPUImageAnalysis.h"
+#include "../image_analysis/GPUImageAnalysis.h"
 #include "../jungfrau/JFPedestalCalc.h"
-#include "../indexing/IndexerWrapper.h"
+#include "../image_analysis/IndexerWrapper.h"
 
 #ifdef JFJOCH_USE_NUMA
 #include <numa.h>

receiver/JFJochReceiver.h

@@ -12,7 +12,7 @@
 #include "../common/DiffractionExperiment.h"
 #include "../common/JFJochException.h"
 #include "../common/FrameTransformation.h"
-#include "../common/StrongPixelSet.h"
+#include "../image_analysis/StrongPixelSet.h"
 #include "../jungfrau/JFCalibration.h"
 #include "../common/ImagePusher.h"
 
@@ -21,7 +21,7 @@
 #include "../common/ThreadSafeFIFO.h"
 #include "../common/ZMQPreviewPublisher.h"
 
-#include "../common/RadialIntegration.h"
+#include "../image_analysis/RadialIntegration.h"
 #include "../common/Latch.h"
 
 #include "../common/StatusVector.h"

tests/CMakeLists.txt

@@ -1,5 +1,4 @@
-
-add_executable(CatchTest
+ADD_EXECUTABLE(CatchTest
         DiffractionExperimentTest.cpp
         RawToConvertedGeometryTest.cpp
         ../common/RawToConvertedGeometry.h
@@ -27,7 +26,7 @@ add_executable(CatchTest
         StatusVectorTest.cpp ProcessRawPacketTest.cpp
         CBORTest.cpp JFConversionTest.cpp)
 
-target_link_libraries(CatchTest JFJochBroker JFJochReceiver JFJochWriter DataProcessing CommonFunctions HLSSimulation)
+target_link_libraries(CatchTest JFJochBroker JFJochReceiver JFJochWriter ImageAnalysis CommonFunctions HLSSimulation)
 target_include_directories(CatchTest PRIVATE .)
 
-INSTALL(TARGETS CatchTest CompressionBenchmark DataAnalysisPerfTest PreviewTest RUNTIME)
+INSTALL(TARGETS CatchTest RUNTIME)

tests/IndexingUnitTest.cpp

@@ -3,7 +3,7 @@
 
 #include <catch2/catch.hpp>
 #include "../writer/HDF5Objects.h"
-#include "../indexing/IndexerWrapper.h"
+#include "../image_analysis/IndexerWrapper.h"
 
 #define make_unit_cell(a1,a2,a3,a4,a5,a6) UnitCell{.a = a1, .b = a2, .c = a3, .alpha = a4, .beta = a5, .gamma = a6}
 

tests/RadialIntegrationTest.cpp

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 #include <catch2/catch.hpp>
-#include "../common/RadialIntegration.h"
+#include "../image_analysis/RadialIntegration.h"
 
 TEST_CASE("RadialIntegrationMapping_Constructor","[RadialIntegration]") {
     DiffractionExperiment x;
@@ -128,7 +128,7 @@ TEST_CASE("RadialIntegration_GetRangeValue","[RadialIntegration]") {
     REQUIRE(radial.GetRangeValue(15, 15) == 0); // Empty set
 }
 
-#include "../common/GPUImageAnalysis.h"
+#include "../image_analysis/GPUImageAnalysis.h"
 
 TEST_CASE("RadialIntegrationGPU_Process","[RadialIntegration]") {
     std::vector<uint16_t> pixel_to_bin = {0,1,2,4,3,1,2,3};

tests/SpotAnalyzeUnitTest.cpp

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 #include <catch2/catch.hpp>
-#include "../common/StrongPixelSet.h"
+#include "../image_analysis/StrongPixelSet.h"
 
 TEST_CASE("DiffractionSpot_AddOperator","[StrongPixelSet]") {
     DiffractionSpot spot1(4,1,10), spot2(3,4,5);

tests/SpotFinderIntegration.cpp

@@ -4,7 +4,7 @@
 #include <catch2/catch.hpp>
 
 #include <random>
-#include "../common/GPUImageAnalysis.h"
+#include "../image_analysis/GPUImageAnalysis.h"
 #include "FPGAUnitTest.h"
 
 using namespace std::literals::chrono_literals;

tools/CMakeLists.txt

@@ -1,5 +1,5 @@
 ADD_EXECUTABLE(RadialIntDataset RadialIntDataset.cpp)
-TARGET_LINK_LIBRARIES(RadialIntDataset CommonFunctions HDF5Wrappers)
+TARGET_LINK_LIBRARIES(RadialIntDataset ImageAnalysis CommonFunctions HDF5Wrappers)
 
 ADD_EXECUTABLE(jfjoch_udp_simulator jfjoch_udp_simulator.cpp UDPSimulator.cpp UDPSimulator.h)
 TARGET_LINK_LIBRARIES(jfjoch_udp_simulator CommonFunctions)
@@ -11,7 +11,7 @@ add_executable(HDF5DatasetWriteTest HDF5DatasetWriteTest.cpp)
 target_link_libraries(HDF5DatasetWriteTest JFJochWriter CommonFunctions)
 
 ADD_EXECUTABLE(DataAnalysisPerfTest DataAnalysisPerfTest.cpp)
-TARGET_LINK_LIBRARIES(DataAnalysisPerfTest DataProcessing JFJochWriter CommonFunctions)
+TARGET_LINK_LIBRARIES(DataAnalysisPerfTest ImageAnalysis JFJochWriter CommonFunctions)
 
 ADD_EXECUTABLE(PreviewTest PreviewTest.cpp)
 TARGET_LINK_LIBRARIES(PreviewTest JFJochWriter CommonFunctions)

tools/DataAnalysisPerfTest.cpp

@@ -7,11 +7,11 @@
 #include "bitshuffle/bitshuffle_core.h"
 #include "../writer/HDF5Objects.h"
 #include "../common/RawToConvertedGeometry.h"
-#include "../common/GPUImageAnalysis.h"
+#include "../image_analysis/GPUImageAnalysis.h"
 
-#include "../indexing/IndexerWrapper.h"
+#include "../image_analysis/IndexerWrapper.h"
 
-#include "../common/RadialIntegration.h"
+#include "../image_analysis/RadialIntegration.h"
 #include "../common/Logger.h"
 
 #define make_unit_cell(a1,a2,a3,a4,a5,a6) UnitCell{.a = a1, .b = a2, .c = a3, .alpha = a4, .beta = a5, .gamma = a6}

tools/RadialIntDataset.cpp

@@ -8,7 +8,7 @@
 #include <cmath>
 #include <vector>
 #include "../writer/HDF5Objects.h"
-#include "../common/RadialIntegration.h"
+#include "../image_analysis/RadialIntegration.h"
 
 std::vector<uint8_t> GetOneByteMask(DiffractionExperiment &x, HDF5Object &master_file) {
     std::vector<uint8_t> ret(x.GetPixelsNum(), 1);