diff --git a/.gitmodules b/.gitmodules index 9871cde7..03b3076b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,5 +1,5 @@ -[submodule "indexing/fast-feedback-indexer"] - path = indexing/fast-feedback-indexer +[submodule "image_analysis/fast-feedback-indexer"] + path = image_analysis/fast-feedback-indexer url = https://github.com/paulscherrerinstitute/fast-feedback-indexer/ [submodule "frame_serialize/tinycbor"] path = frame_serialize/tinycbor diff --git a/CMakeLists.txt b/CMakeLists.txt index 5da83771..506de6d3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,7 +13,7 @@ SET(CMAKE_C_FLAGS_RELEASE "-O3 -march=native -mtune=native") SET(CMAKE_CUDA_FLAGS_RELEASE "-O3") SET(JFJOCH_COMPILE_WRITER ON CACHE BOOL "Compile HDF5 writer") -SET(JFJOCH_COMPILE_FPGA ON CACHE BOOL "Compile FPGA part") +SET(JFJOCH_COMPILE_RECEIVER ON CACHE BOOL "Compile image receiver") SET(JFJOCH_COMPILE_DETECTOR ON CACHE BOOL "Compile detector control") SET(JFJOCH_COMPILE_INDEXER ON CACHE BOOL "Compile indexer") SET(JFJOCH_COMPILE_TESTS OFF CACHE BOOL "Compile tests") @@ -35,12 +35,12 @@ ADD_SUBDIRECTORY(compression) ADD_SUBDIRECTORY(common) ADD_SUBDIRECTORY(broker) ADD_SUBDIRECTORY(etc) -ADD_SUBDIRECTORY(indexing) SET(jfjoch_executables jfjoch_broker) -IF (JFJOCH_COMPILE_TESTS OR JFJOCH_COMPILE_FPGA) +IF (JFJOCH_COMPILE_TESTS OR JFJOCH_COMPILE_RECEIVER) ADD_SUBDIRECTORY(receiver) + ADD_SUBDIRECTORY(image_analysis) LIST(APPEND jfjoch_executables jfjoch_receiver) ENDIF() diff --git a/README.md b/README.md index 76341568..c17d3fb7 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ Automated test routine is then accessible as `tests/CatchTest`. There are also b * `CompressionBenchmark` to measure compression bandwidth (single threaded) * `HDF5DatasetWriteTest` to measure HDF5 dataset writing speed (single threaded) * `DataAnalysisPerfTest` to measure data analysis performance (single threaded) -* `PedestalPerfTest` to measure pedestal calculation performance +* `JFCalibrationPerfTest` to measure pedestal calculation and online conversion performance In addition, tests are executed to verify that datasets written by Jungfraujoch are readable with XDS Durin plugin and CrystFEL. Input files for these programs are placed in `xds_durin` and `crystfel` folders. See `.gitlab-ci.yml` for details. diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index 61368ddd..c679716c 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -34,29 +34,19 @@ ADD_LIBRARY( CommonFunctions STATIC ThreadSafeFIFO.h ZMQPreviewPublisher.cpp ZMQPreviewPublisher.h ZMQImagePusher.cpp ZMQImagePusher.h - RadialIntegration.cpp RadialIntegration.h DiffractionSpot.cpp DiffractionSpot.h - StrongPixelSet.cpp StrongPixelSet.h Latch.cpp Latch.h - RadialIntegrationMapping.cpp RadialIntegrationMapping.h StatusVector.h ImagePusher.cpp ImagePusher.h TestImagePusher.cpp TestImagePusher.h SpotToSave.h NetworkAddressConvert.h NetworkAddressConvert.cpp grpcToJson.h jsonToGrpc.h to_fixed.h - GPUImageAnalysis.h GPUImageAnalysis.cu DiffractionExperiment.h DiffractionGeometry.cpp) -FIND_LIBRARY(CUDART_LIBRARY cudart_static PATHS ${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES} REQUIRED) - -TARGET_LINK_LIBRARIES(CommonFunctions Compression FrameSerialize libzmq JFCalibration JFJochProtoBuf ${CUDART_LIBRARY} -lrt) +TARGET_LINK_LIBRARIES(CommonFunctions Compression FrameSerialize libzmq JFCalibration JFJochProtoBuf -lrt) IF(HAS_NUMAIF AND NUMA_LIBRARY) TARGET_COMPILE_DEFINITIONS(CommonFunctions PRIVATE -DJFJOCH_USE_NUMA) TARGET_LINK_LIBRARIES(CommonFunctions ${NUMA_LIBRARY}) ENDIF() - -FIND_PACKAGE(TIFF REQUIRED) -FIND_LIBRARY(TIFFXX NAMES tiffxx REQUIRED DOC "Tiff C++ library") -TARGET_LINK_LIBRARIES(CommonFunctions TIFF::TIFF ${TIFFXX}) diff --git a/indexing/CMakeLists.txt b/image_analysis/CMakeLists.txt similarity index 57% rename from indexing/CMakeLists.txt rename to image_analysis/CMakeLists.txt index 3b48641d..e7f8170c 100644 --- a/indexing/CMakeLists.txt +++ b/image_analysis/CMakeLists.txt @@ -1,6 +1,10 @@ -ADD_LIBRARY(DataProcessing STATIC +ADD_LIBRARY(ImageAnalysis STATIC CrystalLattice.cpp CrystalLattice.h IndexerWrapper.cpp IndexerWrapper.h + GPUImageAnalysis.cu GPUImageAnalysis.h + RadialIntegration.cpp RadialIntegration.h + RadialIntegrationMapping.cpp RadialIntegrationMapping.h + StrongPixelSet.cpp StrongPixelSet.h fast-feedback-indexer/indexer/src/indexer.cpp fast-feedback-indexer/indexer/src/ffbidx/indexer.h fast-feedback-indexer/indexer/src/indexer_gpu.cu @@ -10,8 +14,9 @@ ADD_LIBRARY(DataProcessing STATIC fast-feedback-indexer/indexer/src/ffbidx/log.h fast-feedback-indexer/indexer/src/ffbidx/exception.h) -TARGET_INCLUDE_DIRECTORIES(DataProcessing PUBLIC +TARGET_INCLUDE_DIRECTORIES(ImageAnalysis PUBLIC fast-feedback-indexer/indexer/src/ fast-feedback-indexer/eigen) -TARGET_LINK_LIBRARIES(DataProcessing CommonFunctions) +FIND_LIBRARY(CUDART_LIBRARY cudart_static PATHS ${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES} REQUIRED) +TARGET_LINK_LIBRARIES(ImageAnalysis CommonFunctions ${CUDART_LIBRARY}) diff --git a/indexing/CrystalLattice.cpp b/image_analysis/CrystalLattice.cpp similarity index 100% rename from indexing/CrystalLattice.cpp rename to image_analysis/CrystalLattice.cpp diff --git a/indexing/CrystalLattice.h b/image_analysis/CrystalLattice.h similarity index 100% rename from indexing/CrystalLattice.h rename to image_analysis/CrystalLattice.h diff --git a/common/GPUImageAnalysis.cu b/image_analysis/GPUImageAnalysis.cu similarity index 99% rename from common/GPUImageAnalysis.cu rename to image_analysis/GPUImageAnalysis.cu index 0c9fa075..f42e9a01 100644 --- a/common/GPUImageAnalysis.cu +++ b/image_analysis/GPUImageAnalysis.cu @@ -2,7 +2,7 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include "GPUImageAnalysis.h" -#include "JFJochException.h" +#include "../common/JFJochException.h" #include // input X x Y pixels array diff --git a/common/GPUImageAnalysis.h b/image_analysis/GPUImageAnalysis.h similarity index 100% rename from common/GPUImageAnalysis.h rename to image_analysis/GPUImageAnalysis.h diff --git a/indexing/IndexerWrapper.cpp b/image_analysis/IndexerWrapper.cpp similarity index 100% rename from indexing/IndexerWrapper.cpp rename to image_analysis/IndexerWrapper.cpp diff --git a/indexing/IndexerWrapper.h b/image_analysis/IndexerWrapper.h similarity index 100% rename from indexing/IndexerWrapper.h rename to image_analysis/IndexerWrapper.h diff --git a/common/RadialIntegration.cpp b/image_analysis/RadialIntegration.cpp similarity index 98% rename from common/RadialIntegration.cpp rename to image_analysis/RadialIntegration.cpp index fc0a0537..7118af15 100644 --- a/common/RadialIntegration.cpp +++ b/image_analysis/RadialIntegration.cpp @@ -2,7 +2,7 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include "RadialIntegration.h" -#include "JFJochException.h" +#include "../common/JFJochException.h" RadialIntegration::RadialIntegration(const std::vector& in_mapping, uint16_t in_nbins) : pixel_to_bin(in_mapping), nbins(in_nbins), sum(in_nbins, 0), count(in_nbins, 0) diff --git a/common/RadialIntegration.h b/image_analysis/RadialIntegration.h similarity index 95% rename from common/RadialIntegration.h rename to image_analysis/RadialIntegration.h index 77146cae..80f18bbb 100644 --- a/common/RadialIntegration.h +++ b/image_analysis/RadialIntegration.h @@ -8,7 +8,7 @@ #include #include -#include "DiffractionExperiment.h" +#include "../common/DiffractionExperiment.h" #include "RadialIntegrationMapping.h" class RadialIntegration { diff --git a/common/RadialIntegrationMapping.cpp b/image_analysis/RadialIntegrationMapping.cpp similarity index 98% rename from common/RadialIntegrationMapping.cpp rename to image_analysis/RadialIntegrationMapping.cpp index ac315def..e3441c40 100644 --- a/common/RadialIntegrationMapping.cpp +++ b/image_analysis/RadialIntegrationMapping.cpp @@ -4,7 +4,7 @@ #include #include "RadialIntegrationMapping.h" -#include "JFJochException.h" +#include "../common/JFJochException.h" RadialIntegrationMapping::RadialIntegrationMapping(const DiffractionExperiment& experiment, const uint8_t *one_byte_mask) : low_q(experiment.GetLowQForRadialInt_recipA()), diff --git a/common/RadialIntegrationMapping.h b/image_analysis/RadialIntegrationMapping.h similarity index 94% rename from common/RadialIntegrationMapping.h rename to image_analysis/RadialIntegrationMapping.h index 93e38aa2..e3a74332 100644 --- a/common/RadialIntegrationMapping.h +++ b/image_analysis/RadialIntegrationMapping.h @@ -5,7 +5,7 @@ #define JUNGFRAUJOCH_RADIALINTEGRATIONMAPPING_H #include -#include "DiffractionExperiment.h" +#include "../common/DiffractionExperiment.h" class RadialIntegrationMapping { const double low_q, high_q, q_spacing; diff --git a/common/StrongPixelSet.cpp b/image_analysis/StrongPixelSet.cpp similarity index 100% rename from common/StrongPixelSet.cpp rename to image_analysis/StrongPixelSet.cpp diff --git a/common/StrongPixelSet.h b/image_analysis/StrongPixelSet.h similarity index 92% rename from common/StrongPixelSet.h rename to image_analysis/StrongPixelSet.h index dbb245bc..745fa06b 100644 --- a/common/StrongPixelSet.h +++ b/image_analysis/StrongPixelSet.h @@ -6,9 +6,9 @@ #include #include -#include "DiffractionExperiment.h" -#include "Coord.h" -#include "DiffractionSpot.h" +#include "../common/DiffractionExperiment.h" +#include "../common/Coord.h" +#include "../common/DiffractionSpot.h" inline uint32_t strong_pixel_coord(uint16_t col, uint16_t line) { return col + (static_cast(line) << 16u); diff --git a/indexing/fast-feedback-indexer b/image_analysis/fast-feedback-indexer similarity index 100% rename from indexing/fast-feedback-indexer rename to image_analysis/fast-feedback-indexer diff --git a/receiver/CMakeLists.txt b/receiver/CMakeLists.txt index f8a45399..3a619260 100644 --- a/receiver/CMakeLists.txt +++ b/receiver/CMakeLists.txt @@ -35,7 +35,7 @@ ADD_LIBRARY(JFJochReceiver STATIC JFJochReceiverTest.cpp JFJochReceiverTest.h JFJochReceiver.cpp JFJochReceiver.h) -TARGET_LINK_LIBRARIES(JFJochReceiver DataProcessing JungfraujochHost CommonFunctions HLSSimulation) +TARGET_LINK_LIBRARIES(JFJochReceiver ImageAnalysis JungfraujochHost CommonFunctions HLSSimulation) IF(HAS_NUMA_H AND NUMA_LIBRARY) TARGET_COMPILE_DEFINITIONS(JFJochReceiver PRIVATE -DJFJOCH_USE_NUMA_H) diff --git a/receiver/JFJochReceiver.cpp b/receiver/JFJochReceiver.cpp index 824dd796..5adebaa2 100644 --- a/receiver/JFJochReceiver.cpp +++ b/receiver/JFJochReceiver.cpp @@ -5,9 +5,9 @@ #include -#include "../common/GPUImageAnalysis.h" +#include "../image_analysis/GPUImageAnalysis.h" #include "../jungfrau/JFPedestalCalc.h" -#include "../indexing/IndexerWrapper.h" +#include "../image_analysis/IndexerWrapper.h" #ifdef JFJOCH_USE_NUMA #include diff --git a/receiver/JFJochReceiver.h b/receiver/JFJochReceiver.h index b93be4fc..bc4fa34e 100644 --- a/receiver/JFJochReceiver.h +++ b/receiver/JFJochReceiver.h @@ -12,7 +12,7 @@ #include "../common/DiffractionExperiment.h" #include "../common/JFJochException.h" #include "../common/FrameTransformation.h" -#include "../common/StrongPixelSet.h" +#include "../image_analysis/StrongPixelSet.h" #include "../jungfrau/JFCalibration.h" #include "../common/ImagePusher.h" @@ -21,7 +21,7 @@ #include "../common/ThreadSafeFIFO.h" #include "../common/ZMQPreviewPublisher.h" -#include "../common/RadialIntegration.h" +#include "../image_analysis/RadialIntegration.h" #include "../common/Latch.h" #include "../common/StatusVector.h" diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c403a3a1..55cc893e 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,5 +1,4 @@ - -add_executable(CatchTest +ADD_EXECUTABLE(CatchTest DiffractionExperimentTest.cpp RawToConvertedGeometryTest.cpp ../common/RawToConvertedGeometry.h @@ -27,7 +26,7 @@ add_executable(CatchTest StatusVectorTest.cpp ProcessRawPacketTest.cpp CBORTest.cpp JFConversionTest.cpp) -target_link_libraries(CatchTest JFJochBroker JFJochReceiver JFJochWriter DataProcessing CommonFunctions HLSSimulation) +target_link_libraries(CatchTest JFJochBroker JFJochReceiver JFJochWriter ImageAnalysis CommonFunctions HLSSimulation) target_include_directories(CatchTest PRIVATE .) -INSTALL(TARGETS CatchTest CompressionBenchmark DataAnalysisPerfTest PreviewTest RUNTIME) \ No newline at end of file +INSTALL(TARGETS CatchTest RUNTIME) \ No newline at end of file diff --git a/tests/IndexingUnitTest.cpp b/tests/IndexingUnitTest.cpp index deef955d..0450e771 100644 --- a/tests/IndexingUnitTest.cpp +++ b/tests/IndexingUnitTest.cpp @@ -3,7 +3,7 @@ #include #include "../writer/HDF5Objects.h" -#include "../indexing/IndexerWrapper.h" +#include "../image_analysis/IndexerWrapper.h" #define make_unit_cell(a1,a2,a3,a4,a5,a6) UnitCell{.a = a1, .b = a2, .c = a3, .alpha = a4, .beta = a5, .gamma = a6} diff --git a/tests/RadialIntegrationTest.cpp b/tests/RadialIntegrationTest.cpp index 5a83875a..dbb930c8 100644 --- a/tests/RadialIntegrationTest.cpp +++ b/tests/RadialIntegrationTest.cpp @@ -2,7 +2,7 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include -#include "../common/RadialIntegration.h" +#include "../image_analysis/RadialIntegration.h" TEST_CASE("RadialIntegrationMapping_Constructor","[RadialIntegration]") { DiffractionExperiment x; @@ -128,7 +128,7 @@ TEST_CASE("RadialIntegration_GetRangeValue","[RadialIntegration]") { REQUIRE(radial.GetRangeValue(15, 15) == 0); // Empty set } -#include "../common/GPUImageAnalysis.h" +#include "../image_analysis/GPUImageAnalysis.h" TEST_CASE("RadialIntegrationGPU_Process","[RadialIntegration]") { std::vector pixel_to_bin = {0,1,2,4,3,1,2,3}; diff --git a/tests/SpotAnalyzeUnitTest.cpp b/tests/SpotAnalyzeUnitTest.cpp index 6b40fa24..694948e3 100644 --- a/tests/SpotAnalyzeUnitTest.cpp +++ b/tests/SpotAnalyzeUnitTest.cpp @@ -2,7 +2,7 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include -#include "../common/StrongPixelSet.h" +#include "../image_analysis/StrongPixelSet.h" TEST_CASE("DiffractionSpot_AddOperator","[StrongPixelSet]") { DiffractionSpot spot1(4,1,10), spot2(3,4,5); diff --git a/tests/SpotFinderIntegration.cpp b/tests/SpotFinderIntegration.cpp index b6cca138..bd7f4dd3 100644 --- a/tests/SpotFinderIntegration.cpp +++ b/tests/SpotFinderIntegration.cpp @@ -4,7 +4,7 @@ #include #include -#include "../common/GPUImageAnalysis.h" +#include "../image_analysis/GPUImageAnalysis.h" #include "FPGAUnitTest.h" using namespace std::literals::chrono_literals; diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 72e558d6..f57b58b6 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -1,5 +1,5 @@ ADD_EXECUTABLE(RadialIntDataset RadialIntDataset.cpp) -TARGET_LINK_LIBRARIES(RadialIntDataset CommonFunctions HDF5Wrappers) +TARGET_LINK_LIBRARIES(RadialIntDataset ImageAnalysis CommonFunctions HDF5Wrappers) ADD_EXECUTABLE(jfjoch_udp_simulator jfjoch_udp_simulator.cpp UDPSimulator.cpp UDPSimulator.h) TARGET_LINK_LIBRARIES(jfjoch_udp_simulator CommonFunctions) @@ -11,7 +11,7 @@ add_executable(HDF5DatasetWriteTest HDF5DatasetWriteTest.cpp) target_link_libraries(HDF5DatasetWriteTest JFJochWriter CommonFunctions) ADD_EXECUTABLE(DataAnalysisPerfTest DataAnalysisPerfTest.cpp) -TARGET_LINK_LIBRARIES(DataAnalysisPerfTest DataProcessing JFJochWriter CommonFunctions) +TARGET_LINK_LIBRARIES(DataAnalysisPerfTest ImageAnalysis JFJochWriter CommonFunctions) ADD_EXECUTABLE(PreviewTest PreviewTest.cpp) TARGET_LINK_LIBRARIES(PreviewTest JFJochWriter CommonFunctions) diff --git a/tools/DataAnalysisPerfTest.cpp b/tools/DataAnalysisPerfTest.cpp index 17942f05..905903c3 100644 --- a/tools/DataAnalysisPerfTest.cpp +++ b/tools/DataAnalysisPerfTest.cpp @@ -7,11 +7,11 @@ #include "bitshuffle/bitshuffle_core.h" #include "../writer/HDF5Objects.h" #include "../common/RawToConvertedGeometry.h" -#include "../common/GPUImageAnalysis.h" +#include "../image_analysis/GPUImageAnalysis.h" -#include "../indexing/IndexerWrapper.h" +#include "../image_analysis/IndexerWrapper.h" -#include "../common/RadialIntegration.h" +#include "../image_analysis/RadialIntegration.h" #include "../common/Logger.h" #define make_unit_cell(a1,a2,a3,a4,a5,a6) UnitCell{.a = a1, .b = a2, .c = a3, .alpha = a4, .beta = a5, .gamma = a6} diff --git a/tools/RadialIntDataset.cpp b/tools/RadialIntDataset.cpp index e9021e99..7e80f0f3 100644 --- a/tools/RadialIntDataset.cpp +++ b/tools/RadialIntDataset.cpp @@ -8,7 +8,7 @@ #include #include #include "../writer/HDF5Objects.h" -#include "../common/RadialIntegration.h" +#include "../image_analysis/RadialIntegration.h" std::vector GetOneByteMask(DiffractionExperiment &x, HDF5Object &master_file) { std::vector ret(x.GetPixelsNum(), 1);