diff --git a/.gitea/workflows/build_and_test.yml b/.gitea/workflows/build_and_test.yml index ee6cd62e2..58893e991 100644 --- a/.gitea/workflows/build_and_test.yml +++ b/.gitea/workflows/build_and_test.yml @@ -126,6 +126,19 @@ jobs: for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -property installationPath`) do set "VSPATH=%%i" call "%VSPATH%\VC\Auxiliary\Build\vcvars64.bat" cmake --build build + # The quick cross-platform checks (tests/PortableTest.cpp and the [portable] cases), before + # anything is packaged. CPU-only: the runner has no GPU. + - name: Build portable tests + shell: cmd + run: | + for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -property installationPath`) do set "VSPATH=%%i" + call "%VSPATH%\VC\Auxiliary\Build\vcvars64.bat" + cmake --build build --target jfjoch_portable_test + - name: Run portable tests + shell: cmd + run: | + cd build\tests + jfjoch_portable_test.exe "[portable]" - name: Build installer (NSIS) shell: cmd run: | @@ -168,6 +181,13 @@ jobs: # JFJOCH_VIEWER_ONLY is forced on macOS. No -mcpu: the arm64 baseline is what every Apple # Silicon Mac runs, and NEON is part of it. run: cmake -S . -B build-macos -DCMAKE_BUILD_TYPE=Release -DJFJOCH_USE_CUDA=OFF -DCMAKE_PREFIX_PATH="$QT_PREFIX" + # The quick cross-platform checks (tests/PortableTest.cpp and the [portable] cases), before + # anything is packaged. + - name: Portable tests + shell: bash + run: | + cmake --build build-macos --target jfjoch_portable_test -j "$(sysctl -n hw.ncpu)" + cd build-macos/tests && ./jfjoch_portable_test "[portable]" - name: Build viewer and DMG shell: bash run: | @@ -380,6 +400,19 @@ jobs: for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -property installationPath`) do set "VSPATH=%%i" call "%VSPATH%\VC\Auxiliary\Build\vcvars64.bat" cmake --build build-rugnux + # The quick cross-platform checks (tests/PortableTest.cpp and the [portable] cases), before + # anything is packaged. CPU-only: the runner has no GPU. + - name: Build portable tests + shell: cmd + run: | + for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -property installationPath`) do set "VSPATH=%%i" + call "%VSPATH%\VC\Auxiliary\Build\vcvars64.bat" + cmake --build build-rugnux --target jfjoch_portable_test + - name: Run portable tests + shell: cmd + run: | + cd build-rugnux\tests + jfjoch_portable_test.exe "[portable]" - name: Build rugnux zip shell: cmd run: | @@ -415,6 +448,13 @@ jobs: - name: Configure rugnux build shell: bash run: cmake -S . -B build-rugnux-macos -DJFJOCH_RUGNUX_ONLY=ON -DJFJOCH_USE_CUDA=OFF -DCMAKE_BUILD_TYPE=Release + # The quick cross-platform checks (tests/PortableTest.cpp and the [portable] cases), before + # anything is packaged. + - name: Portable tests + shell: bash + run: | + cmake --build build-rugnux-macos --target jfjoch_portable_test -j "$(sysctl -n hw.ncpu)" + cd build-rugnux-macos/tests && ./jfjoch_portable_test "[portable]" - name: Build rugnux tgz shell: bash run: cmake --build build-rugnux-macos --target package -j "$(sysctl -n hw.ncpu)" diff --git a/CMakeLists.txt b/CMakeLists.txt index 33fd421d5..27c88cba7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -365,10 +365,10 @@ SET(CMAKE_POLICY_VERSION_MINIMUM 3.5) # libzmq must be made available BEFORE sls_detector_package for the override above to take effect. FetchContent_MakeAvailable(libzmq) IF (JFJOCH_PORTABLE_ONLY) - # A portable build still needs zstd/hdf5/spdlog/httplib (JFJochReader uses httplib). - # Only sls_detector_package (detector) and catch2 (tests) are not built here -- and sls - # in particular does not configure under MSVC -- so skip just those two. - FetchContent_MakeAvailable(zstd hdf5 spdlog httplib) + # A portable build still needs zstd/hdf5/spdlog/httplib (JFJochReader uses httplib), and catch2 + # for jfjoch_portable_test. Only sls_detector_package (detector) is not built here -- it does + # not configure under MSVC. + FetchContent_MakeAvailable(zstd catch2 hdf5 spdlog httplib) ELSE() FetchContent_MakeAvailable(zstd sls_detector_package catch2 hdf5 spdlog httplib) ENDIF() @@ -555,6 +555,7 @@ IF (JFJOCH_RUGNUX_ONLY) ADD_SUBDIRECTORY(broker) ADD_SUBDIRECTORY(reader) ADD_SUBDIRECTORY(rugnux) + ADD_SUBDIRECTORY(tests) # jfjoch_portable_test only ELSEIF (JFJOCH_VIEWER_ONLY) # Minimal subtree: jfjoch_viewer and only the libraries it transitively links. # (broker here provides JFJochAPI only; its service targets are gated out.) @@ -571,6 +572,7 @@ ELSEIF (JFJOCH_VIEWER_ONLY) ADD_SUBDIRECTORY(rugnux) ADD_SUBDIRECTORY(viewer) ADD_SUBDIRECTORY(tools) # builds only the portable analysis tools (rugnux/extract_hkl) + ADD_SUBDIRECTORY(tests) # jfjoch_portable_test only ELSE() ADD_SUBDIRECTORY(jungfrau) ADD_SUBDIRECTORY(compression) diff --git a/tests/AdaptiveSpotFinderCPUTest.cpp b/tests/AdaptiveSpotFinderCPUTest.cpp index 0db0270a6..14e2cbc61 100644 --- a/tests/AdaptiveSpotFinderCPUTest.cpp +++ b/tests/AdaptiveSpotFinderCPUTest.cpp @@ -28,7 +28,7 @@ SpotFindingSettings AdaptiveSettings() { // Raw (untransformed) geometry: the mapping is built over the raw module layout, which is smaller // than the converted image. The finder has to walk the raw image, so a spot planted at a raw pixel // comes back at that pixel. -TEST_CASE("AdaptiveSpotFinderCPU_RawGeometry", "[AdaptiveSpotFinder]") { +TEST_CASE("AdaptiveSpotFinderCPU_RawGeometry", "[AdaptiveSpotFinder][portable]") { DiffractionExperiment x(DetJF4M()); x.DetectorDistance_mm(80).BeamX_pxl(1030).BeamY_pxl(1080); x.QSpacingForAzimInt_recipA(0.05).QRangeForAzimInt_recipA(0.05, 5.0); diff --git a/tests/AzimuthalIntegrationTest.cpp b/tests/AzimuthalIntegrationTest.cpp index d5adf25fc..5151c4faf 100644 --- a/tests/AzimuthalIntegrationTest.cpp +++ b/tests/AzimuthalIntegrationTest.cpp @@ -5,6 +5,7 @@ #include "../common/AzimuthalIntegrationProfile.h" #include "../common/AzimuthalIntegrationMapping.h" +#include "../common/JFJochMath.h" TEST_CASE("AzimuthalIntegrationMapping_Constructor","[AzimuthalIntegration]") { DiffractionExperiment x(DetJF4M()); @@ -39,7 +40,7 @@ TEST_CASE("AzimuthalIntegrationMapping_GetBinNumber_mask","[AzimuthalIntegration for (int row = 0; row < x.GetYPixelsNum(); row++) { for (int col = 0; col < x.GetXPixelsNum(); col++) { float d = geom.PxlToRes(col, row); - float q = 2 * M_PI / d; + float q = 2 * PI / d; if (q >= 3.1) pixel_mask[row * x.GetXPixelsNum() + col] = 1; } @@ -147,7 +148,7 @@ TEST_CASE("AzimuthalIntegrationMapping_GetMapping","[AzimuthalIntegration]") { CHECK(map[x.GetXPixelsNum() * 1500 + 1500] / 40 == 0); } -TEST_CASE("AzimuthalIntegrationMapping_QToBin","[AzimuthalIntegration]") { +TEST_CASE("AzimuthalIntegrationMapping_QToBin","[AzimuthalIntegration][portable]") { DiffractionExperiment x(DetJF4M()); x.DetectorDistance_mm(50).BeamX_pxl(1000).BeamY_pxl(1000); x.QSpacingForAzimInt_recipA(0.1).QRangeForAzimInt_recipA(0.1, 4); @@ -400,7 +401,7 @@ TEST_CASE("AzimuthalIntegrationProfile_GetMeanValueOfBins","[AzimuthalIntegratio REQUIRE(profile.GetBkgEstimate(x.GetAzimuthalIntegrationSettings()) == Catch::Approx((sum[0] + sum[1] + sum[2]) / double(count[0] + count[1] + count[2]))); } -TEST_CASE("AzimuthalIntegrationProfile_GetResult1D","[AzimuthalIntegration]") { +TEST_CASE("AzimuthalIntegrationProfile_GetResult1D","[AzimuthalIntegration][portable]") { DiffractionExperiment x(DetJF4M()); x.DetectorDistance_mm(50).BeamX_pxl(1000).BeamY_pxl(1000); diff --git a/tests/BraggIntegrationEngineCompressedImageTest.cpp b/tests/BraggIntegrationEngineCompressedImageTest.cpp index 46344bb22..881c4b4b8 100644 --- a/tests/BraggIntegrationEngineCompressedImageTest.cpp +++ b/tests/BraggIntegrationEngineCompressedImageTest.cpp @@ -93,7 +93,7 @@ void RequireIdentical(const std::vector &a, const std::vector buffer(8 * 1024 * 1024); CBORStream2Serializer serializer(buffer.data(), buffer.size()); @@ -491,7 +491,7 @@ TEST_CASE("CBORSerialize_Calibration", "[CBOR]") { CHECK(memcmp(output_message.GetCompressed(), calib1.data(), 256 * sizeof(float)) == 0); } -TEST_CASE("CBORSerialize_End", "[CBOR]") { +TEST_CASE("CBORSerialize_End", "[CBOR][portable]") { std::vector buffer(8 * 1024 * 1024); CBORStream2Serializer serializer(buffer.data(), buffer.size()); @@ -897,7 +897,7 @@ TEST_CASE("CBORSerialize_Image_Rgb", "[CBOR]") { REQUIRE(memcmp(image_array.image.GetCompressed(), test.data(), test.size() * sizeof(rgb)) == 0); } -TEST_CASE("CBORSerialize_Image_Compressed", "[CBOR]") { +TEST_CASE("CBORSerialize_Image_Compressed", "[CBOR][portable]") { std::vector buffer(8 * 1024 * 1024); CBORStream2Serializer serializer(buffer.data(), buffer.size()); @@ -1026,7 +1026,7 @@ TEST_CASE("CBORSerialize_Start_ROIMap", "[CBOR]") { } -TEST_CASE("CBORSerialize_Image_Spots", "[CBOR]") { +TEST_CASE("CBORSerialize_Image_Spots", "[CBOR][portable]") { std::vector buffer(8 * 1024 * 1024); CBORStream2Serializer serializer(buffer.data(), buffer.size()); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 5b386a5a4..81ab48540 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,4 +1,41 @@ +# Quick cross-platform checks: the results that depend on the architecture, the compiler or the +# standard library (bitshuffle/LZ4 SIMD paths, HDF5 read-back, CBF parsing, CPU spot finding, FFTW +# indexing, gemmi MTZ/CIF). Built in every configuration, the portable (viewer/rugnux-only) ones +# included, and linked only against what those build - so it runs on the macOS and Windows jobs. +# Only the cases tagged [portable] are meant to run there (a few seconds in all); the rest of these +# files is compiled in because the tagged cases live next to them. +# EXCLUDE_FROM_ALL: a product build does not pay for it, CI builds the target by name. +ADD_EXECUTABLE(jfjoch_portable_test EXCLUDE_FROM_ALL + PortableTest.cpp + TestData.h + ZSTDCompressorTest.cpp + JFJochReaderTest.cpp + SweepLayoutTest.cpp + CBORTest.cpp + ImageSpotFinderCPUTest.cpp + AdaptiveSpotFinderCPUTest.cpp + AzimuthalIntegrationTest.cpp + CalcBraggPredictionTest.cpp + BraggStencilTest.cpp + BraggIntegrationEngineCompressedImageTest.cpp + WriteReflectionsTest.cpp + LoadReferenceMtzFreeFlagsTest.cpp + LoadReferenceSfCifTest.cpp +) +target_link_libraries(jfjoch_portable_test Catch2WithMain JFJochRugnux JFJochReader JFJochImageAnalysis + JFJochWriter JFJochCommon) +target_include_directories(jfjoch_portable_test PRIVATE .) +# Same device-link step as rugnux (see rugnux/CMakeLists.txt): a portable CUDA build links cuFFT statically. +IF (JFJOCH_PORTABLE_ONLY AND JFJOCH_CUDA_AVAILABLE AND TARGET CUDA::cufft_static) + SET_TARGET_PROPERTIES(jfjoch_portable_test PROPERTIES CUDA_RESOLVE_DEVICE_SYMBOLS ON) +ENDIF() + +IF (JFJOCH_PORTABLE_ONLY) + RETURN() +ENDIF() + ADD_EXECUTABLE(jfjoch_test + PortableTest.cpp DiffractionExperimentTest.cpp RawToConvertedGeometryTest.cpp ../common/RawToConvertedGeometry.h diff --git a/tests/CalcBraggPredictionTest.cpp b/tests/CalcBraggPredictionTest.cpp index 9d94c9a90..48f0c1661 100644 --- a/tests/CalcBraggPredictionTest.cpp +++ b/tests/CalcBraggPredictionTest.cpp @@ -4,6 +4,7 @@ #include #include "../image_analysis/bragg_prediction/BraggPrediction.h" +#include "../common/JFJochMath.h" #include #include "../image_analysis/SensorAbsorption.h" #include "../image_analysis/bragg_prediction/BraggPredictionRot.h" @@ -40,11 +41,11 @@ TEST_CASE("BraggPrediction_FlightPathAttenuation", "[air_path]") { // crossed strictly more of the medium. float prev = 1.0f; for (double alpha_deg : {10.0, 20.0, 30.0, 40.0, 50.0, 55.0}) { - const float f = a18.Factor(static_cast(std::cos(alpha_deg * M_PI / 180.0))); + const float f = a18.Factor(static_cast(std::cos(alpha_deg * PI / 180.0))); CHECK(f > prev); prev = f; } - const float cos55 = static_cast(std::cos(55.0 * M_PI / 180.0)); + const float cos55 = static_cast(std::cos(55.0 * PI / 180.0)); // 18 keV over 160 mm at 55 degrees: +1.5%. CHECK(a18.Factor(cos55) == Catch::Approx(1.0147).epsilon(0.002)); @@ -88,7 +89,7 @@ TEST_CASE("BraggPrediction_FlightPathAttenuation", "[air_path]") { CHECK(vac.WilsonBShift_A2(3.02, 50.0, lambda_of(3.76)) == 0.0); } -TEST_CASE("BraggPrediction_11keV") { +TEST_CASE("BraggPrediction_11keV", "[portable]") { DiffractionExperiment experiment(DetJF4M()); experiment.DetectorDistance_mm(100.0).BeamX_pxl(1500.0).BeamY_pxl(1000.0) .IncidentEnergy_keV(11.0); @@ -160,10 +161,10 @@ TEST_CASE("BraggPrediction_15keV") { } } -TEST_CASE("BraggPrediction_Rot1_Rot2") { +TEST_CASE("BraggPrediction_Rot1_Rot2", "[portable]") { DiffractionExperiment experiment(DetJF4M()); experiment.DetectorDistance_mm(100.0).BeamX_pxl(1500.0).BeamY_pxl(1000.0) - .PoniRot1_rad(2.0/180.0 * M_PI).PoniRot2_rad(3.0/180.0 * M_PI) + .PoniRot1_rad(2.0/180.0 * PI).PoniRot2_rad(3.0/180.0 * PI) .IncidentEnergy_keV(11.0); DiffractionGeometry geom = experiment.GetDiffractionGeometry(); @@ -224,7 +225,7 @@ TEST_CASE("BraggPrediction_backscattering") { } } -TEST_CASE("BraggPrediction_systematic_absences") { +TEST_CASE("BraggPrediction_systematic_absences", "[portable]") { DiffractionExperiment experiment(DetJF4M()); experiment.DetectorDistance_mm(120.0).BeamX_pxl(1500.0).BeamY_pxl(1000.0) .IncidentEnergy_keV(12.0); @@ -404,7 +405,7 @@ TEST_CASE("BraggPredictionGPU") { TEST_CASE("BraggPredictionGPU_Rot1_Rot2") { DiffractionExperiment experiment(DetJF4M()); experiment.DetectorDistance_mm(100.0).BeamX_pxl(1500.0).BeamY_pxl(1000.0) - .PoniRot1_rad(2.0/180.0 * M_PI).PoniRot2_rad(3.0/180.0 * M_PI) + .PoniRot1_rad(2.0/180.0 * PI).PoniRot2_rad(3.0/180.0 * PI) .IncidentEnergy_keV(11.0); DiffractionGeometry geom = experiment.GetDiffractionGeometry(); diff --git a/tests/ImageSpotFinderCPUTest.cpp b/tests/ImageSpotFinderCPUTest.cpp index fe680a6cb..35d06773e 100644 --- a/tests/ImageSpotFinderCPUTest.cpp +++ b/tests/ImageSpotFinderCPUTest.cpp @@ -4,7 +4,7 @@ #include #include "../image_analysis/spot_finding/ImageSpotFinderCPU.h" -TEST_CASE("ImageSpotFinderCPU_SignalToNoise") { +TEST_CASE("ImageSpotFinderCPU_SignalToNoise", "[portable]") { size_t width = 100; size_t height = 100; @@ -108,7 +108,7 @@ TEST_CASE("ImageSpotFinderCPU_CountThreshold_Resolution") { REQUIRE(spots[1].RawCoord().y == 75); } -TEST_CASE("ImageSpotFinderCPU_CountThreshold_Mask") { +TEST_CASE("ImageSpotFinderCPU_CountThreshold_Mask", "[portable]") { size_t width = 100; size_t height = 100; diff --git a/tests/JFJochReaderTest.cpp b/tests/JFJochReaderTest.cpp index ba8d01780..9ade44cb7 100644 --- a/tests/JFJochReaderTest.cpp +++ b/tests/JFJochReaderTest.cpp @@ -608,7 +608,7 @@ TEST_CASE("JFJochReader_GridScan", "[HDF5][Full]") { REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); } -TEST_CASE("JFJochReader_DataI16", "[HDF5][Full]") { +TEST_CASE("JFJochReader_DataI16", "[HDF5][Full][portable]") { DiffractionExperiment x(DetJF(1)); x.FilePrefix("test09").ImagesPerTrigger(4).OverwriteExistingFiles(true); @@ -774,7 +774,7 @@ TEST_CASE("JFJochReader_DataI16_OldMasterFormat", "[HDF5][Full]") { REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); } -TEST_CASE("JFJochReader_DataU16", "[HDF5][Full]") { +TEST_CASE("JFJochReader_DataU16", "[HDF5][Full][portable]") { DiffractionExperiment x(DetJF(1)); x.FilePrefix("test10").ImagesPerTrigger(4).OverwriteExistingFiles(true); @@ -846,7 +846,7 @@ TEST_CASE("JFJochReader_DataU16", "[HDF5][Full]") { REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); } -TEST_CASE("JFJochReader_DataI32", "[HDF5][Full]") { +TEST_CASE("JFJochReader_DataI32", "[HDF5][Full][portable]") { DiffractionExperiment x(DetJF(1)); x.FilePrefix("test11").ImagesPerTrigger(4).OverwriteExistingFiles(true); @@ -913,7 +913,7 @@ TEST_CASE("JFJochReader_DataI32", "[HDF5][Full]") { REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); } -TEST_CASE("JFJochReader_DataU32", "[HDF5][Full]") { +TEST_CASE("JFJochReader_DataU32", "[HDF5][Full][portable]") { DiffractionExperiment x(DetJF(1)); x.FilePrefix("test12").ImagesPerTrigger(4).OverwriteExistingFiles(true); @@ -1786,7 +1786,7 @@ TEST_CASE("JFJochReader_NXmxIntegrated", "[HDF5][Full]") { REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); } -TEST_CASE("JFJochReader_GetRawImage_NXmxLegacy", "[HDF5][Full]") { +TEST_CASE("JFJochReader_GetRawImage_NXmxLegacy", "[HDF5][Full][portable]") { DiffractionExperiment x(DetJF(1)); x.FilePrefix("test_read_raw_image").ImagesPerTrigger(4).OverwriteExistingFiles(true); @@ -1906,7 +1906,7 @@ TEST_CASE("JFJochReader_GetRawImage_Concurrent", "[HDF5][Full]") { REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); } -TEST_CASE("JFJochReader_GetRawImage_VDS", "[HDF5][Full]") { +TEST_CASE("JFJochReader_GetRawImage_VDS", "[HDF5][Full][portable]") { DiffractionExperiment x(DetJF(1)); x.FilePrefix("test_read_raw_image").ImagesPerTrigger(4).OverwriteExistingFiles(true); @@ -1965,7 +1965,7 @@ TEST_CASE("JFJochReader_GetRawImage_VDS", "[HDF5][Full]") { REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); } -TEST_CASE("JFJochReader_GetRawImage_Integrated", "[HDF5][Full]") { +TEST_CASE("JFJochReader_GetRawImage_Integrated", "[HDF5][Full][portable]") { DiffractionExperiment x(DetJF(1)); x.FilePrefix("test_read_raw_image").ImagesPerTrigger(4).OverwriteExistingFiles(true); @@ -2022,7 +2022,7 @@ TEST_CASE("JFJochReader_GetRawImage_Integrated", "[HDF5][Full]") { REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); } -TEST_CASE("JFJochReader_HDF5DataSource_Integrated", "[HDF5][Full]") { +TEST_CASE("JFJochReader_HDF5DataSource_Integrated", "[HDF5][Full][portable]") { DiffractionExperiment x(DetJF(1)); x.FilePrefix("source_integrated").ImagesPerTrigger(5).OverwriteExistingFiles(true); @@ -2070,7 +2070,7 @@ TEST_CASE("JFJochReader_HDF5DataSource_Integrated", "[HDF5][Full]") { REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); } -TEST_CASE("JFJochReader_HDF5DataSource_VDS", "[HDF5][Full]") { +TEST_CASE("JFJochReader_HDF5DataSource_VDS", "[HDF5][Full][portable]") { DiffractionExperiment x(DetJF(1)); x.FilePrefix("source_vds_mapping").ImagesPerTrigger(5).ImagesPerFile(2).OverwriteExistingFiles(true); @@ -2131,7 +2131,7 @@ TEST_CASE("JFJochReader_HDF5DataSource_VDS", "[HDF5][Full]") { REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); } -TEST_CASE("JFJochReader_HDF5DataSource_Legacy", "[HDF5][Full]") { +TEST_CASE("JFJochReader_HDF5DataSource_Legacy", "[HDF5][Full][portable]") { DiffractionExperiment x(DetJF(1)); x.FilePrefix("source_legacy_mapping").ImagesPerTrigger(5).ImagesPerFile(2).OverwriteExistingFiles(true); @@ -3434,7 +3434,7 @@ TEST_CASE("JFJochReader_AxisRecovery", "[HDF5][Full]") { // saturation_value is written inclusive and used exclusive, so a read has to add the count back. It // did not, and the value fell by one on every write-read-write cycle - unbounded, and compounding // whenever a _process.h5 was reprocessed. Nothing caught it: no test asserted the read-back limit. -TEST_CASE("JFJochReader_SaturationSurvivesRoundTrip", "[HDF5][Full]") { +TEST_CASE("JFJochReader_SaturationSurvivesRoundTrip", "[HDF5][Full][portable]") { DiffractionExperiment x(DetJF(1)); x.ImagesPerTrigger(2).OverwriteExistingFiles(true).FilePrefix("test_satrt"); x.BeamX_pxl(100).BeamY_pxl(200).DetectorDistance_mm(150) @@ -3958,7 +3958,7 @@ namespace { } } -TEST_CASE("JFJochCBFReader_AxisTableStatesTheMounting", "[HDF5][Full]") { +TEST_CASE("JFJochCBFReader_AxisTableStatesTheMounting", "[HDF5][Full][portable]") { const int64_t nx = 24, ny = 16; SECTION("a pixel at Count_cutoff is saturated") { @@ -4081,7 +4081,7 @@ namespace { // Three ways a CBF that is not a detector image, or is one with a hole in its head, used to be // opened anyway - each of them silently, which is the failure this project cares most about. -TEST_CASE("JFJochCBFReader_incomplete_header_is_refused_not_misread", "[HDF5][Full]") { +TEST_CASE("JFJochCBFReader_incomplete_header_is_refused_not_misread", "[HDF5][Full][portable]") { const int64_t nx = 24, ny = 16; SECTION("a byte-offset CBF with no PILATUS header is not ours to read") { @@ -4128,7 +4128,7 @@ TEST_CASE("JFJochCBFReader_incomplete_header_is_refused_not_misread", "[HDF5][Fu } } -TEST_CASE("MiniCBF_header_longer_than_the_first_probe", "[HDF5][Full]") { +TEST_CASE("MiniCBF_header_longer_than_the_first_probe", "[HDF5][Full][portable]") { // The header is read with a small first probe, and again with the large one when it does not end // inside it; what stands past the first 16 kB must come out exactly as before. std::string filler; @@ -4261,7 +4261,7 @@ namespace { } } -TEST_CASE("JFJochMarCCDReader_Geometry", "[HDF5][Full]") { +TEST_CASE("JFJochMarCCDReader_Geometry", "[HDF5][Full][portable]") { const int64_t nx = 24, ny = 16; SECTION("the header's geometry reaches the experiment in the units the rest of the code uses") { @@ -4425,7 +4425,7 @@ namespace { } } -TEST_CASE("JFJochSMVReader_Geometry", "[HDF5][Full]") { +TEST_CASE("JFJochSMVReader_Geometry", "[HDF5][Full][portable]") { SECTION("a pixel at CCD_IMAGE_SATURATION is saturated, stated or not") { // The pixels are handed out as 32-bit, so without an explicit limit the fallback would be // INT32_MAX and no 16-bit image could ever reach it. diff --git a/tests/LoadReferenceMtzFreeFlagsTest.cpp b/tests/LoadReferenceMtzFreeFlagsTest.cpp index c2c5b811f..12bc73e43 100644 --- a/tests/LoadReferenceMtzFreeFlagsTest.cpp +++ b/tests/LoadReferenceMtzFreeFlagsTest.cpp @@ -42,7 +42,7 @@ namespace { bool ExpectedFree(int h, int k, int l) { return (h * 7 + k * 13 + l * 17) % 10 == 0; } } -TEST_CASE("LoadReferenceMtz imports FreeR (CCP4 convention: test set = 0)", "[reference_mtz]") { +TEST_CASE("LoadReferenceMtz imports FreeR (CCP4 convention: test set = 0)", "[reference_mtz][portable]") { const auto path = (std::filesystem::temp_directory_path() / "rugnux_freer_ccp4.mtz").string(); WriteMtzWithFreeR(path, /*free_value=*/0); const auto ref = LoadReferenceMtz(path); diff --git a/tests/LoadReferenceSfCifTest.cpp b/tests/LoadReferenceSfCifTest.cpp index b56a2dff6..c8027fb15 100644 --- a/tests/LoadReferenceSfCifTest.cpp +++ b/tests/LoadReferenceSfCifTest.cpp @@ -100,7 +100,7 @@ _refln.intensity_sigma } } -TEST_CASE("LoadReferenceMtz reads a structure-factor mmCIF", "[reference_mtz]") { +TEST_CASE("LoadReferenceMtz reads a structure-factor mmCIF", "[reference_mtz][portable]") { // Named without a .cif extension: the format is recognised by content. const auto path = WriteFile("rugnux_ref_sf.dat", SF_CIF); const auto ref = LoadReferenceMtz(path); diff --git a/tests/PortableTest.cpp b/tests/PortableTest.cpp new file mode 100644 index 000000000..3c67eed15 --- /dev/null +++ b/tests/PortableTest.cpp @@ -0,0 +1,184 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +// Checks that the results which depend on the architecture, the compiler or the standard library are +// the same everywhere: the bitshuffle/LZ4 path (hperf SSE2/AVX2 on x86, NEON on aarch64, SSE2 under +// MSVC) and an end-to-end run of the analysis. Golden values were taken on x86-64 Linux; an arm64 +// or MSVC build that disagrees has a portability defect, not a new reference. + +#include + +#include +#include + +#include "TestData.h" +#include "../compression/JFJochCompressor.h" +#include "../compression/JFJochDecompress.h" +#include "../common/JFJochMath.h" +#include "../writer/HDF5Objects.h" +#include "../image_analysis/indexing/IndexerFactory.h" +#include "../reader/JFJochHDF5Reader.h" +#include "../rugnux/Rugnux.h" + +namespace { + // FNV-1a, spelled out: std::hash is not the same function on every standard library. + uint64_t Fnv1a(const void *data, size_t size) { + auto p = static_cast(data); + uint64_t h = 1469598103934665603ULL; + for (size_t i = 0; i < size; i++) { + h ^= p[i]; + h *= 1099511628211ULL; + } + return h; + } + + uint64_t SplitMix64(uint64_t &state) { + uint64_t z = (state += 0x9e3779b97f4a7c15ULL); + z = (z ^ (z >> 30)) * 0xbf58476d1ce4e5b9ULL; + z = (z ^ (z >> 27)) * 0x94d049bb133111ebULL; + return z ^ (z >> 31); + } +} + +// compression_benchmark.h5 holds 25 int16 frames, bitshuffle/LZ4 (HDF5 filter 32008). One frame is +// read three ways - the raw chunk decoded by the hperf path and by the classic bitshuffle, and +// through the HDF5 filter - and all three must give the frame whose hash was recorded on x86. +TEST_CASE("Portable_BSLZ4_BenchmarkFrameGoldenHash", "[portable][compression]") { + RegisterHDF5Filter(); + HDF5ReadOnlyFile file("../../tests/test_data/compression_benchmark.h5"); + HDF5DataSet dataset(file, "/entry/data/data"); + HDF5DataSpace file_space(dataset); + const auto dims = file_space.GetDimensions(); + REQUIRE(dims.size() == 3); + const size_t npixel = dims[1] * dims[2]; + + const std::pair golden[] = { + {0, 0xaffea29c511b6ec2ULL}, + {24, 0xe46913009c95a1f1ULL} + }; + + for (const auto &[frame, hash]: golden) { + std::vector chunk; + dataset.ReadDirectChunk(chunk, {frame, 0, 0}); + + std::vector hperf, classic, filtered(npixel); + JFJochDecompress(hperf, CompressionAlgorithm::BSHUF_LZ4, chunk, npixel, true); + JFJochDecompress(classic, CompressionAlgorithm::BSHUF_LZ4, chunk, npixel, false); + dataset.ReadVector(filtered, {frame, 0, 0}, {1, dims[1], dims[2]}); + + CHECK(Fnv1a(hperf.data(), hperf.size() * sizeof(int16_t)) == hash); + CHECK(hperf == classic); + CHECK(hperf == filtered); + } +} + +// The encoder must write the same bytes everywhere: a file is decoded on a different machine from +// the one that wrote it, and the header of every block is big-endian regardless of the host. +TEST_CASE("Portable_BSLZ4_EncodeGoldenHash", "[portable][compression]") { + uint64_t state = 42; + std::vector image(1024 * 1024 + 13); // a tail that is not a whole block + for (auto &v: image) + v = (SplitMix64(state) % 16 == 0) ? SplitMix64(state) % 200 : 0; + + JFJochBitShuffleCompressor compressor(CompressionAlgorithm::BSHUF_LZ4); + auto compressed = compressor.Compress(image); + CHECK(compressed.size() == 562349); + CHECK(Fnv1a(compressed.data(), compressed.size()) == 0x874e45eb17a51ed2ULL); + + std::vector decoded; + JFJochDecompress(decoded, CompressionAlgorithm::BSHUF_LZ4, compressed, image.size()); + CHECK(decoded == image); +} + +// One bitshuffle block through the shipped selector (hperf on x86, classic NEON on aarch64) against +// the classic reference transform, over the element sizes and block tails the file format allows. +TEST_CASE("Portable_BitShuffleBlock_MatchesReference", "[portable][compression]") { + uint64_t state = 7; + for (size_t elem: {1, 2, 4, 8}) { + for (size_t size: {8, 16, 24, 136, 1032, 2040, 2048, 8192}) { + for (int pattern = 0; pattern < 3; pattern++) { + std::vector orig(size * elem), ref(size * elem), enc(size * elem), + scratch(size * elem), out(size * elem); + for (size_t i = 0; i < orig.size(); i++) { + if (pattern == 0) orig[i] = static_cast(SplitMix64(state)); // noise + else if (pattern == 1) orig[i] = (i % elem == 0) ? static_cast(i % 7) : 0; // counts + else orig[i] = static_cast((i % elem == elem - 1) ? 0x80 : 0); // sign bit + } + INFO("elem " << elem << " size " << size << " pattern " << pattern); + REQUIRE(bshuf_trans_bit_elem(orig.data(), ref.data(), size, elem) >= 0); + REQUIRE(JFJochBitShuffleBlock(enc.data(), orig.data(), scratch.data(), size, elem) >= 0); + CHECK(enc == ref); + REQUIRE(JFJochBitUnshuffleBlock(out.data(), ref.data(), scratch.data(), size, elem) >= 0); + CHECK(out == orig); + } + } + } +} + +// The CPU indexer (FFTW, NEON codelets on arm64) on a synthetic orthorhombic lattice. Named +// explicitly rather than through Auto, so a CUDA build on a runner without a GPU tests it too. +TEST_CASE("Portable_FFTWIndexer_SyntheticLattice", "[portable][Indexing]") { + const UnitCell uc(39, 45, 78, 90, 90, 90); + const CrystalLattice cl(uc); + + DiffractionExperiment experiment; + IndexingSettings settings; + settings.Algorithm(IndexingAlgorithmEnum::FFTW).FFT_MaxUnitCell_A(250.0).FFT_HighResolution_A(2 * PI / 3.0); + experiment.ImportIndexingSettings(settings).SetUnitCell(uc); + auto indexer = CreateIndexer(experiment); + REQUIRE(indexer); + + std::vector spots; + for (int h = -2; h < 10; h++) + for (int k = -5; k < 10; k++) + for (int l = -3; l < 10; l++) + spots.push_back(h * cl.Astar() + k * cl.Bstar() + l * cl.Cstar()); + + const auto result = indexer->Run(spots); + REQUIRE(result.lattice.size() == 1); + const auto out = result.lattice[0].GetUnitCell(); + std::array lengths = {out.a, out.b, out.c}; + std::sort(lengths.begin(), lengths.end()); + CHECK(lengths[0] == Catch::Approx(39.0)); + CHECK(lengths[1] == Catch::Approx(45.0)); + CHECK(lengths[2] == Catch::Approx(78.0)); + CHECK(out.alpha == Catch::Approx(90.0)); + CHECK(out.beta == Catch::Approx(90.0)); + CHECK(out.gamma == Catch::Approx(90.0)); +} + +// End to end on the git-LFS rotation dataset (tests/data, see README.md there): HDF5 read through +// external links, CPU spot finding, the default indexer (FFTW on a CPU build), refinement and +// integration, on a short wedge. SKIPs when LFS was not pulled. +TEST_CASE("Portable_Rugnux_RotationWedge", "[portable][lfs]") { + const auto master = jfjoch_test::LargeDataFile("rotation_master.h5"); + if (!master) + SKIP("rotation_master.h5 not available (git-lfs data not pulled)"); + + RegisterHDF5Filter(); + JFJochHDF5Reader reader; + REQUIRE_NOTHROW(reader.ReadFile(*master)); + auto dataset = reader.GetDataset(); + REQUIRE(dataset); + + DiffractionExperiment experiment(dataset->experiment); + IndexingSettings indexing; + indexing.Algorithm(IndexingAlgorithmEnum::Auto); + experiment.ImportIndexingSettings(indexing); + + ProcessConfig config; + config.mode = ProcessMode::FullAnalysis; + config.nthreads = std::max(1u, std::thread::hardware_concurrency()); + config.end_image = 4; + config.spot_finding = DiffractionExperiment::DefaultDataProcessingSettings(); + config.spot_finding.indexing = true; + + Rugnux process(reader, experiment, *dataset->pixel_mask, config); + ProcessResult result; + REQUIRE_NOTHROW(result = process.Run()); + + CHECK(result.images_processed == 4); + REQUIRE(result.indexing_rate.has_value()); + CHECK(result.indexing_rate.value() > 0.5f); + reader.Close(); +} diff --git a/tests/SweepLayoutTest.cpp b/tests/SweepLayoutTest.cpp index a3ef530d3..88663dde5 100644 --- a/tests/SweepLayoutTest.cpp +++ b/tests/SweepLayoutTest.cpp @@ -21,7 +21,7 @@ std::vector Series(const std::vector &angles, double incre } // namespace -TEST_CASE("SweepLayout_Contiguous") { +TEST_CASE("SweepLayout_Contiguous", "[portable]") { std::vector angles; for (int i = 0; i < 100; i++) angles.push_back(20.0 + 0.1 * i); @@ -34,7 +34,7 @@ TEST_CASE("SweepLayout_Contiguous") { CHECK(l.files[i] == "f" + std::to_string(i) + ".cbf"); } -TEST_CASE("SweepLayout_Gapped") { +TEST_CASE("SweepLayout_Gapped", "[portable]") { // Frames 3, 4 and 7 of a ten-frame sweep never made it into the archive. The sweep is still ten // steps wide and every frame keeps its own angle. const auto l = sweep::Place(Series({0.0, 0.1, 0.2, 0.5, 0.6, 0.8, 0.9}, 0.1), "test"); diff --git a/tests/WriteReflectionsTest.cpp b/tests/WriteReflectionsTest.cpp index 8609e410e..b8e6d06c4 100644 --- a/tests/WriteReflectionsTest.cpp +++ b/tests/WriteReflectionsTest.cpp @@ -31,7 +31,7 @@ namespace { } } -TEST_CASE("Merged MTZ: the data dataset is id 1 and carries the wavelength", "[write_reflections]") { +TEST_CASE("Merged MTZ: the data dataset is id 1 and carries the wavelength", "[write_reflections][portable]") { jfjoch_test::SyntheticMergeParams params; params.true_space_group = "P 43 21 2"; params.twin_supergroup = "P 43 21 2"; @@ -74,7 +74,7 @@ TEST_CASE("Merged MTZ: the data dataset is id 1 and carries the wavelength", "[w } TEST_CASE("Unmerged MTZ: LP is Lorentz-polarization, QE the sensor efficiency, FLIGHT the flight path", - "[write_reflections]") { + "[write_reflections][portable]") { // The whole point of the split: LP must mean what XDS and DIALS mean by it, and the raw count // sum must still be recoverable from the file alone, as I / LP * QE * FLIGHT. auto experiment = TestExperiment(); diff --git a/tests/ZSTDCompressorTest.cpp b/tests/ZSTDCompressorTest.cpp index 5a497601e..f6f3e3575 100644 --- a/tests/ZSTDCompressorTest.cpp +++ b/tests/ZSTDCompressorTest.cpp @@ -229,7 +229,7 @@ TEST_CASE("JFjochZstdCompressor_Frame_ones","[ZSTD]") { REQUIRE(memcmp(input_shuffled.data(), output.data(), RAW_MODULE_SIZE * sizeof(uint16_t)) == 0); } -TEST_CASE("JFJochCompressor_JFJochDecompressor_ZSTD","[ZSTD]") { +TEST_CASE("JFJochCompressor_JFJochDecompressor_ZSTD","[ZSTD][portable]") { DiffractionExperiment x(DetJF4M()); x.Compression(CompressionAlgorithm::BSHUF_ZSTD).BitDepthImage(32).PixelSigned(true); @@ -269,7 +269,7 @@ TEST_CASE("JFJochCompressor_DestTooSmall_Throws","[ZSTD]") { REQUIRE_NOTHROW(compressor.Compress(big.data(), big.size(), image)); } -TEST_CASE("JFJochCompressor_JFJochDecompressor_LZ4","[ZSTD]") { +TEST_CASE("JFJochCompressor_JFJochDecompressor_LZ4","[ZSTD][portable]") { DiffractionExperiment x(DetJF4M()); x.Compression(CompressionAlgorithm::BSHUF_LZ4).BitDepthImage(32).PixelSigned(true); @@ -396,7 +396,7 @@ TEST_CASE("ZstdHuff_MaskLike", "[ZSTD]") { RequireHuffRoundTrip(image); } -TEST_CASE("ZstdHuff_BenchmarkImage", "[ZSTD]") { +TEST_CASE("ZstdHuff_BenchmarkImage", "[ZSTD][portable]") { RegisterHDF5Filter(); // bitshuffle filter, needed to read the compressed benchmark dataset HDF5ReadOnlyFile data("../../tests/test_data/compression_benchmark.h5"); HDF5DataSet dataset(data, "/entry/data/data"); diff --git a/tests/data/README.md b/tests/data/README.md index 3227cd93b..b1353efcf 100644 --- a/tests/data/README.md +++ b/tests/data/README.md @@ -23,7 +23,8 @@ git lfs pull # or: git lfs pull --include "tests/data/*.h5" | `rotation_master.h5` | protein rotation series (~1800 images) | yes (LFS) | `rotation_master.h5` (plus its `_data_NNNNNN.h5` files) is fetched by `git lfs pull` and -drives `Rugnux_Rotation`. A separate serial dataset is intentionally **not** shipped +drives `Rugnux_Rotation`, and its first four frames `Portable_Rugnux_RotationWedge` (the +`[portable]` end-to-end check, which skips on the macOS/Windows jobs because they do not pull LFS). A separate serial dataset is intentionally **not** shipped to keep the repository small — the rotation series can be run in serial mode (full analysis without rotation indexing) to exercise that path. To add your own dataset, drop the master + its data files here as real files (not symlinks, if you intend to commit them via LFS); the master