Logger: handle situation when git sha1 is empty

This reverts commit dcd6088afe.

.gitignore

@@ -1,3 +0,0 @@
-cmake-build-debug/
-cmake-build-release/
-build*/

.gitlab-ci.yml

@@ -7,6 +7,7 @@ stages:
 build:x86:gcc:
   stage: build
   variables:
+    GIT_SUBMODULE_STRATEGY: recursive
     CC: gcc
     CXX: g++
   tags:
@@ -23,6 +24,7 @@ build:x86:gcc:
 build:x86:gcc_writer:
   stage: build
   variables:
+    GIT_SUBMODULE_STRATEGY: recursive
     CC: gcc
     CXX: g++
   tags:
@@ -34,11 +36,12 @@ build:x86:gcc_writer:
     - cd build
     - source /opt/rh/gcc-toolset-12/enable
     - cmake -DCMAKE_BUILD_TYPE=Release -DJFJOCH_WRITER_ONLY=ON ..
-    - make -j48
+    - make -j48 jfjoch
 
 build:x86:driver:
   stage: build
   variables:
+    GIT_SUBMODULE_STRATEGY: recursive
     CC: gcc
     CXX: g++
   tags:
@@ -57,6 +60,8 @@ build:x86:driver:
 
 build:x86:vitis_hls:
   stage: build
+  variables:
+    GIT_SUBMODULE_STRATEGY: recursive
   tags:
     - x86
   needs: []
@@ -79,43 +84,27 @@ build:x86:vitis_hls:
 
 build:x86:frontend:
   stage: build
+  variables:
+    GIT_SUBMODULE_STRATEGY: recursive
   tags:
     - x86
   needs: []
   script:
-    - mkdir build
+    - cd frontend_ui
+    - npm install
+    - npm run build
     - cd build
-    - /usr/bin/cmake ..
-    - make frontend
-    - cd ../frontend_ui/build
     - tar czf ../../jfjoch_frontend.tar.gz *
   artifacts:
     paths:
       - jfjoch_frontend.tar.gz
     expire_in: 1 week
 
-build:x86:rpm:
-  stage: build
-  tags:
-    - x86
-  needs: []
-  script:
-    - mkdir build
-    - cd build
-    - source /opt/rh/gcc-toolset-12/enable
-    - cmake -DCMAKE_BUILD_TYPE=Release ..
-    - make frontend
-    - make -j48 package
-    - mv *.rpm ..
-  artifacts:
-    paths:
-      - "*.rpm"
-    expire_in: 1 week
-
 test:x86:gcc:
   stage: test
   timeout: 90m
   variables:
+    GIT_SUBMODULE_STRATEGY: recursive
     CTEST_OUTPUT_ON_FAILURE: 1
     CC: gcc
     CXX: g++
@@ -129,9 +118,9 @@ test:x86:gcc:
     - mkdir -p build
     - cd build
     - cmake -DCMAKE_BUILD_TYPE=Release ..
-    - make -j48 jfjoch_test HDF5DatasetWriteTest
+    - make -j48 CatchTest HDF5DatasetWriteTest
     - cd tests
-    - ./jfjoch_test -r junit -o report.xml
+    - ./CatchTest -r junit -o report.xml
     - cd ../tools
     - ./HDF5DatasetWriteTest ../../tests/test_data/compression_benchmark.h5
   artifacts:
@@ -143,6 +132,7 @@ test:x86:crystfel:
   stage: test
   timeout: 90m
   variables:
+    GIT_SUBMODULE_STRATEGY: recursive
     CTEST_OUTPUT_ON_FAILURE: 1
     CC: gcc
     CXX: g++
@@ -166,6 +156,7 @@ test:x86:xds_durin:
   stage: test
   timeout: 90m
   variables:
+    GIT_SUBMODULE_STRATEGY: recursive
     CTEST_OUTPUT_ON_FAILURE: 1
     CC: gcc
     CXX: g++
@@ -189,6 +180,7 @@ test:x86:xds_neggia:
   stage: test
   timeout: 90m
   variables:
+    GIT_SUBMODULE_STRATEGY: recursive
     CTEST_OUTPUT_ON_FAILURE: 1
     CC: gcc
     CXX: g++
@@ -212,6 +204,7 @@ test:x86:xia2.ssx:
   stage: test
   timeout: 90m
   variables:
+    GIT_SUBMODULE_STRATEGY: recursive
     CTEST_OUTPUT_ON_FAILURE: 1
     CC: gcc
     CXX: g++
@@ -237,6 +230,7 @@ synthesis:vivado_pcie_100g:
   stage: synthesis
   dependencies: []
   variables:
+    GIT_SUBMODULE_STRATEGY: recursive
     CC: gcc
     CXX: g++
   rules:
@@ -249,6 +243,8 @@ synthesis:vivado_pcie_100g:
         - fpga/scripts/*
         - fpga/xdc/*
         - fpga/pcie_driver/jfjoch_fpga.h
+    - when: manual
+      allow_failure: true
   tags:
     - vivado
   artifacts:
@@ -270,6 +266,7 @@ synthesis:vivado_pcie_8x10g:
   stage: synthesis
   dependencies: []
   variables:
+    GIT_SUBMODULE_STRATEGY: recursive
     CC: gcc
     CXX: g++
   rules:
@@ -282,6 +279,8 @@ synthesis:vivado_pcie_8x10g:
         - fpga/xdc/*
         - fpga/pcie_driver/jfjoch_fpga.h
       allow_failure: true
+    - when: manual
+      allow_failure: true
   tags:
     - vivado
   artifacts:
@@ -301,9 +300,7 @@ synthesis:vivado_pcie_8x10g:
 
 release:
   stage: release
-  rules:
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-      when: manual
+  when: manual
   tags:
     - x86
   dependencies:
@@ -311,13 +308,9 @@ release:
     - synthesis:vivado_pcie_100g
     - build:x86:frontend
     - build:x86:driver
-    - build:x86:rpm
   script:
     - export PACKAGE_VERSION=`head -n1 VERSION`
     - export PACKAGE_REGISTRY_URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/jungfraujoch/${PACKAGE_VERSION}"
-    - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file jfjoch-driver-dkms-${PACKAGE_VERSION}-1.el8.noarch.rpm "${PACKAGE_REGISTRY_URL}/jfjoch-driver-dkms-${PACKAGE_VERSION}-1.el8.noarch.rpm"'
-    - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file jfjoch-writer-${PACKAGE_VERSION}-1.el8.x86_64.rpm "${PACKAGE_REGISTRY_URL}/jfjoch-writer-${PACKAGE_VERSION}-1.el8.x86_64.rpm"'
-    - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file jfjoch-${PACKAGE_VERSION}-1.el8.x86_64.rpm "${PACKAGE_REGISTRY_URL}/jfjoch-${PACKAGE_VERSION}-1.el8.x86_64.rpm"'
     - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file jfjoch_driver.tar.gz "${PACKAGE_REGISTRY_URL}/jfjoch_driver.tar.gz"'
     - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file jfjoch_frontend.tar.gz "${PACKAGE_REGISTRY_URL}/jfjoch_frontend.tar.gz"'
     - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file jfjoch_fpga_pcie_100g.mcs "${PACKAGE_REGISTRY_URL}/jfjoch_fpga_pcie_100g.mcs"'
@@ -328,6 +321,3 @@ release:
       --assets-link "{\"name\":\"jfjoch_frontend.tar.gz\",\"url\":\"${PACKAGE_REGISTRY_URL}/jfjoch_frontend.tar.gz\"}"
       --assets-link "{\"name\":\"jfjoch_fpga_pcie_8x10g.mcs\",\"url\":\"${PACKAGE_REGISTRY_URL}/jfjoch_fpga_pcie_8x10g.mcs\"}"
       --assets-link "{\"name\":\"jfjoch_fpga_pcie_100g.mcs\",\"url\":\"${PACKAGE_REGISTRY_URL}/jfjoch_fpga_pcie_100g.mcs\"}"
-      --assets-link "{\"name\":\"jfjoch-${PACKAGE_VERSION}-1.el8.x86_64.rpm\",\"url\":\"${PACKAGE_REGISTRY_URL}/jfjoch-${PACKAGE_VERSION}-1.el8.x86_64.rpm\",\"link_type\":\"package\"}"
-      --assets-link "{\"name\":\"jfjoch-writer-${PACKAGE_VERSION}-1.el8.x86_64.rpm\",\"url\":\"${PACKAGE_REGISTRY_URL}/jfjoch-writer-${PACKAGE_VERSION}-1.el8.x86_64.rpm\",\"link_type\":\"package\"}"
-      --assets-link "{\"name\":\"jfjoch-driver-dkms-${PACKAGE_VERSION}-1.el8.noarch.rpm\",\"url\":\"${PACKAGE_REGISTRY_URL}/jfjoch-driver-dkms-${PACKAGE_VERSION}-1.el8.noarch.rpm\",\"link_type\":\"package\"}"

CMakeLists.txt

@@ -1,9 +1,7 @@
 CMAKE_MINIMUM_REQUIRED(VERSION 3.19)
 
-FILE(STRINGS VERSION JFJOCH_VERSION)
-
-PROJECT(jfjoch VERSION 1.0.0 LANGUAGES C CXX)
-SET(CMAKE_POLICY_DEFAULT_CMP0077 NEW)
+PROJECT(Jungfraujoch VERSION 1.0 LANGUAGES C CXX)
+LIST(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
 
 SET(CMAKE_CXX_STANDARD 20)
 SET(CMAKE_CXX_STANDARD_REQUIRED True)
@@ -11,31 +9,8 @@ SET(CMAKE_CXX_STANDARD_REQUIRED True)
 SET(CMAKE_CXX_FLAGS_RELEASE "-O3 -march=native -mtune=native -Wno-deprecated-enum-enum-conversion")
 SET(CMAKE_C_FLAGS_RELEASE "-O3 -march=native -mtune=native")
 
-SET(BUILD_SHARED_LIBS OFF)
-SET(BUILD_TESTING OFF)
-
 SET(ZSTD_LEGACY_SUPPORT OFF)
 SET(ZSTD_MULTITHREAD_SUPPORT OFF)
-SET(ZSTD_BUILD_PROGRAMS OFF)
-SET(ZSTD_BUILD_SHARED OFF)
-
-SET(SLS_USE_RECEIVER OFF)
-SET(SLS_USE_RECEIVER_BINARIES OFF)
-SET(SLS_BUILD_SHARED_LIBRARIES OFF)
-
-SET(BUILD_FAST_INDEXER OFF)
-SET(BUILD_FAST_INDEXER_STATIC ON)
-
-SET(HDF5_ENABLE_SZIP_SUPPORT OFF)
-SET(HDF5_ENABLE_SZIP_ENCODING OFF)
-SET(HDF5_BUILD_EXAMPLES OFF)
-SET(HDF5_BUILD_CPP_LIB OFF)
-SET(HDF5_ENABLE_Z_LIB_SUPPORT OFF)
-SET(HDF5_EXTERNALLY_CONFIGURED 1)
-
-SET(jbig OFF)
-SET(zstd OFF)
-SET(lzma OFF)
 
 INCLUDE(CheckLanguage)
 CHECK_LANGUAGE(CUDA)
@@ -56,28 +31,23 @@ SET(JFJOCH_WRITER_ONLY    OFF CACHE BOOL "Compile HDF5 writer only")
 INCLUDE_DIRECTORIES(include)
 INCLUDE(CheckIncludeFile)
 
+#This is to supress error in TORCH
+
+IF ((NOT EXISTS /usr/bin/python) AND (EXISTS /usr/bin/python3))
+    SET(PYTHON_EXECUTABLE /usr/bin/python3)
+ENDIF()
+
+FIND_PACKAGE(Torch HINTS /opt/libtorch/share/cmake/Torch/)
 FIND_LIBRARY(NUMA_LIBRARY NAMES numa DOC "NUMA Library")
 CHECK_INCLUDE_FILE(numaif.h HAS_NUMAIF)
 CHECK_INCLUDE_FILE(numa.h HAS_NUMA_H)
 
 include(FetchContent)
 
-FetchContent_Declare(tiff
-        GIT_REPOSITORY https://github.com/fleon-psi/libtiff
-        GIT_TAG v4.6.0
-        EXCLUDE_FROM_ALL)
-
-FetchContent_Declare(hdf5
-        GIT_REPOSITORY https://github.com/HDFGroup/hdf5/
-        GIT_TAG hdf5_1.14.4.2
-        GIT_SHALLOW 1
-        EXCLUDE_FROM_ALL)
-
 FetchContent_Declare(
         pistache_http
         GIT_REPOSITORY https://github.com/fleon-psi/pistache
         GIT_TAG       51553b92cc7bb25ac792462722ddd4fae33d14b1
-        EXCLUDE_FROM_ALL
 )
 
 FetchContent_Declare(
@@ -85,7 +55,6 @@ FetchContent_Declare(
         GIT_REPOSITORY https://github.com/facebook/zstd
         GIT_TAG 794ea1b0afca0f020f4e57b6732332231fb23c70
         SOURCE_SUBDIR build/cmake
-        EXCLUDE_FROM_ALL
 )
 
 FetchContent_Declare(
@@ -98,10 +67,9 @@ FetchContent_Declare(
         catch2
         GIT_REPOSITORY https://github.com/catchorg/Catch2
         GIT_TAG 4e8d92b
-        EXCLUDE_FROM_ALL
 )
 
-FetchContent_MakeAvailable(pistache_http zstd sls_detector_package catch2 hdf5 tiff)
+FetchContent_MakeAvailable(pistache_http zstd sls_detector_package catch2)
 
 ADD_SUBDIRECTORY(jungfrau)
 ADD_SUBDIRECTORY(compression)
@@ -112,6 +80,7 @@ ADD_SUBDIRECTORY(frame_serialize)
 ADD_SUBDIRECTORY(detector_control)
 IF (JFJOCH_WRITER_ONLY)
     MESSAGE(STATUS "Compiling HDF5 writer only")
+    SET(jfjoch_executables jfjoch_writer)
 ELSE()
     ADD_SUBDIRECTORY(broker)
     ADD_SUBDIRECTORY(fpga)
@@ -121,52 +90,17 @@ ELSE()
     ADD_SUBDIRECTORY(tests)
     ADD_SUBDIRECTORY(tools)
     ADD_SUBDIRECTORY(preview)
+    ADD_SUBDIRECTORY(resonet)
+    SET(jfjoch_executables jfjoch_broker jfjoch_writer CatchTest CompressionBenchmark HDF5DatasetWriteTest jfjoch_udp_simulator sls_detector_put sls_detector_get)
 ENDIF()
 
-IF (NOT JFJOCH_WRITER_ONLY)
-    ADD_CUSTOM_COMMAND(OUTPUT frontend_ui/build/index.html
-            COMMAND npm install
-            COMMAND npm run build
-            COMMAND npm run redocly
-            WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/frontend_ui)
-    ADD_CUSTOM_TARGET(frontend DEPENDS frontend_ui/build/index.html)
+ADD_CUSTOM_COMMAND(OUTPUT frontend_ui/build/index.html
+        COMMAND npm run build
+        WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/frontend_ui)
+ADD_CUSTOM_TARGET(frontend DEPENDS frontend_ui/build/index.html)
 
-    INSTALL(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/fpga/pcie_driver/
-            DESTINATION /usr/src/jfjoch-1.0.0
-            COMPONENT driver-dkms
-            FILES_MATCHING PATTERN "*.c" PATTERN "*.h" PATTERN "Makefile" PATTERN "dkms.conf")
-
-    FILE(MAKE_DIRECTORY ${CMAKE_SOURCE_DIR}/frontend_ui/build/)
-    INSTALL(DIRECTORY ${CMAKE_SOURCE_DIR}/frontend_ui/build/ DESTINATION share/jfjoch/frontend COMPONENT jfjoch )
-ENDIF()
+ADD_CUSTOM_TARGET(jfjoch DEPENDS ${jfjoch_executables})
 
 IF(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
   SET(CMAKE_INSTALL_PREFIX /opt/jfjoch CACHE PATH "Default directory" FORCE)
 ENDIF(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
-
-# Set Package Name
-set(CPACK_PACKAGE_NAME "jfjoch")
-
-SET(CPACK_COMPONENTS_ALL jfjoch writer driver-dkms)
-SET(CPACK_GENERATOR RPM)
-SET(CPACK_RPM_COMPONENT_INSTALL ON)
-SET(CPACK_RPM_MAIN_COMPONENT jfjoch)
-SET(CPACK_RPM_PACKAGE_RELEASE_DIST ON)
-SET(CPACK_RPM_FILE_NAME "RPM-DEFAULT")
-SET(CPACK_RPM_PACKAGE_VERSION ${JFJOCH_VERSION})
-SET(CPACK_RPM_PACKAGE_RELEASE 1)
-SET(CPACK_RPM_PACKAGE_SUMMARY "Jungfraujoch data acquisition system")
-SET(CPACK_RPM_PACKAGE_DESCRIPTION "Jungfraujoch")
-SET(CPACK_RPM_DRIVER-DKMS_PACKAGE_REQUIRES "dkms, gcc, bash, sed")
-SET(CPACK_RPM_DRIVER-DKMS_PACKAGE_ARCHITECTURE "noarch")
-SET(CPACK_RPM_DRIVER-DKMS_POST_INSTALL_SCRIPT_FILE ${CMAKE_CURRENT_SOURCE_DIR}/fpga/pcie_driver/postinstall.sh)
-SET(CPACK_RPM_DRIVER-DKMS_PRE_UNINSTALL_SCRIPT_FILE ${CMAKE_CURRENT_SOURCE_DIR}/fpga/pcie_driver/preuninstall.sh)
-SET(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION /usr/src)
-
-# Set The Vendor Name
-SET(CPACK_PACKAGE_VENDOR "Paul Scherrer Institut")
-
-# Set The License Information
-SET(CPACK_RPM_PACKAGE_LICENSE "Proprietary")
-
-INCLUDE(CPack)

Dockerfile

@@ -4,7 +4,7 @@ FROM harbor.maxiv.lu.se/dockerhub/library/ubuntu:22.04
 
 RUN set -ex;  \
     apt-get update; \
-    apt-get install -y pkg-config git cmake make g++;\
+    apt-get install -y pkg-config git cmake make g++ libhdf5-dev libczmq-dev;\
     rm -rf /var/lib/apt/lists/*
 
 

README.md

@@ -7,7 +7,6 @@ Citation: F. Leonarski, M. Bruckner, C. Lopez-Cuenca, A. Mozzanica, H.-C. Stadle
 The project is supported by :
 * Innosuisse via Innovation Project "NextGenDCU high data rate acquisition system for X-ray detectors in structural biology applications" (101.535.1 IP-ENG; Apr 2023 - Sep 2025).
 * ETH Domain via Open Research Data Contribute project (Jan - Dec 2023)
-* AMD University Program with donation of licenses of Ethernet IP cores and Vivado software
 
 ## License
 
@@ -34,24 +33,25 @@ Other linux platforms should work, but no tests were done so far.
 ### Dependencies
 Required:
 * C++20 compiler and C++20 standard library; recommended GCC 11+ or clang 14+ (Intel OneAPI, AMD AOCC)
-* CMake version 3.21 or newer + GNU make tool
+* CMake version 3.21 or newer + GNU make tool 
+* HDF5 library version 1.10 or newer
+* TIFF library (with C++ headers)
 * JPEG library (turbo-jpeg is also OK)
 
 Optional:
-* CUDA compiler version 11 or newer - required for MX fast feedback indexer
+* CUDA compiler version 11 or newer - required for MX indexing and ML resolution estimation
 * NUMA library - to pin threads to nodes/CPUs
 * Node.js - to make frontend
+* libtorch - for resolution estimation using model from Stanford - see below
 
-Automatically downloaded by CMake and statically linked: 
+Automatically downloaded by CMake: 
 * SLS Detector Package - see [github.com/slsdetectorgroup/slsDetectorPackage](https://github.com/slsdetectorgroup/slsDetectorPackage)
 * Zstandard (Facebook) - see [github.com/facebook/zstd](https://github.com/facebook/zstd)
 * Pistache webserver  - see [github.com/pistacheio/pistache](https://github.com/pistacheio/pistache)
 * Fast feedback indexer (Hans-Christian Stadler, PSI) - see [github.com/paulscherrerinstitute/fast-feedback-indexer](https://github.com/paulscherrerinstitute/fast-feedback-indexer)
 * Catch2 testing library - see [github.com/catchorg/Catch2](https://github.com/catchorg/Catch2)
-* HDF5 library - see [github.com/HDFGroup/hdf5](https://github.com/HDFGroup/hdf5)
-* TIFF library - see [gitlab.com/libtiff/libtiff](https://gitlab.com/libtiff/libtiff)
 
-Please follow the link provided above to check for LICENSE file. Building code with dependencies above requires access from the build system to github.com.
+Please follow the link provided above to check for LICENSE file
 
 Directly included in the repository:
 * JSON parser/writer from N. Lohmann - see [github.com/nlohmann/json](https://github.com/nlohmann/json)
@@ -76,6 +76,7 @@ For license check LICENSE file in respective directory
 Use the following commands:
 
 ```
+git submodule update --init --recursive
 mkdir build
 cd build
 cmake ..
@@ -87,6 +88,7 @@ In most use cases it is better to have a separate machine, with access to distri
 Such machine needs only a HDF5 writer service with fewer dependencies. For compilation use the following commands:
 
 ```
+git submodule update --init --recursive
 mkdir build
 cd build
 cmake -DJFJOCH_WRITER_ONLY=ON ..
@@ -110,6 +112,7 @@ Frontend is written in TypeScript. For details see [frontend_ui/](frontend_ui) d
 
 Jungfraujoch Cmake scripts have an option to start frontend build with the following command:
 ```
+git submodule update --init --recursive
 mkdir build
 cd build
 cmake ..
@@ -119,7 +122,7 @@ Contrary to standard CMake way, frontend will be built in "source" `frontend_ui/
 
 ## Tests
  
-Automated test routine is then accessible as `tests/jfjoch_test`. There are also benchmark routines:
+Automated test routine is then accessible as `tests/CatchTest`. There are also benchmark routines:
 
 * `HDF5DatasetWriteTest` to measure HDF5 dataset writing speed (single threaded)
 * `jfjoch_spot_finding_test` to apply spot finding and indexing routines in Jungfraujoch to an example dataset - this is equivalent to FPGA spot finding algorithm, but NOT performance equivalent as it is particularly not-efficient
@@ -127,3 +130,16 @@ Automated test routine is then accessible as `tests/jfjoch_test`. There are also
 
 In addition, tests are executed to verify that datasets written by Jungfraujoch are readable with XDS Durin plugin, XDS Neggia plygin and CrystFEL. 
 Input files for these programs are placed in `xds_durin`, `xds_neggia` and `crystfel` folders. See `.gitlab-ci.yml` for details.
+
+## Resolution estimation
+
+Resolution estimation can be done with a recent deep learning model by D. Mendez et al. 
+(see [Acta Cryst D, 80, 26-43](https://doi.org/10.1107/S2059798323010586)), adapted to Jungfraujoch. 
+Model used in the original paper is located in the [resonet/](resonet) directory, after converting to TorchScript format.
+
+To use the feature it is necessary to install [libtorch](https://pytorch.org/) library, preferably in `/opt/libtorch` location. The C++11 ABI version needs to be chosen.
+For RHEL 8 systems, please download older version 2.1.0, as version 2.2.0 requires newer `glibc` library than available with the operating system. 
+Version 2.1.0 can be downloaded with the following command:
+```
+wget https://download.pytorch.org/libtorch/cu121/libtorch-cxx11-abi-shared-with-deps-2.1.0%2Bcu121.zip
+```

VERSION

@@ -1 +1 @@
-1.0.0_rc.8
+1.0.0-test5

broker/CMakeLists.txt

@@ -15,6 +15,4 @@ TARGET_LINK_LIBRARIES(JFJochBroker JFJochReceiver JFJochDetector JFJochCommon JF
 ADD_EXECUTABLE(jfjoch_broker jfjoch_broker.cpp)
 TARGET_LINK_LIBRARIES(jfjoch_broker JFJochBroker)
 
-INSTALL(TARGETS jfjoch_broker RUNTIME COMPONENT jfjoch)
-
-INSTALL(FILES redoc-static.html DESTINATION jfjoch/frontend COMPONENT jfjoch )
+INSTALL(TARGETS jfjoch_broker RUNTIME)

broker/JFJochBrokerHttp.cpp

@@ -4,7 +4,6 @@
 #include <nlohmann/json.hpp>
 #include "JFJochBrokerHttp.h"
 #include "gen/model/Error_message.h"
-#include "../preview/JFJochTIFF.h"
 
 // From https://en.cppreference.com/w/cpp/string/byte/tolower
 inline std::string str_tolower(std::string s) {
@@ -23,10 +22,6 @@ inline SpotFindingSettings Convert(const org::openapitools::server::model::Spot_
     ret.enable = input.isEnable();
     ret.indexing = input.isIndexing();
     ret.indexing_tolerance = input.getIndexingTolerance();
-    if (input.filterPowderRingsIsSet())
-        ret.filter_spots_powder_ring = input.isFilterPowderRings();
-    if (input.minSpotCountPowderRingIsSet())
-        ret.min_spot_count_powder_ring = input.getMinSpotCountPowderRing();
     return ret;
 }
 
@@ -41,8 +36,6 @@ inline org::openapitools::server::model::Spot_finding_settings Convert(const Spo
     ret.setEnable(input.enable);
     ret.setIndexing(input.indexing);
     ret.setIndexingTolerance(input.indexing_tolerance);
-    ret.setFilterPowderRings(input.filter_spots_powder_ring);
-    ret.setMinSpotCountPowderRing(input.min_spot_count_powder_ring);
     return ret;
 }
 
@@ -74,7 +67,6 @@ inline org::openapitools::server::model::Measurement_statistics Convert(const Me
 
     if (input.bkg_estimate)
         ret.setBkgEstimate(input.bkg_estimate.value());
-    ret.setUnitCell(input.unit_cell);
 
     return ret;
 }
@@ -324,7 +316,6 @@ inline PreviewJPEGSettings Convert(const org::openapitools::server::model::Previ
     ret.saturation_value = input.getSaturation();
     ret.show_roi = input.isShowRoi();
     ret.show_indexed = input.isShowIndexed();
-    ret.show_user_mask = input.isShowUserMask();
     if (input.resolutionRingIsSet())
         ret.resolution_ring = input.getResolutionRing();
     return ret;
@@ -660,36 +651,25 @@ void JFJochBrokerHttp::preview_image_tiff_get(Pistache::Http::ResponseWriter &re
         response.send(Pistache::Http::Code::Not_Found);
 }
 
+void JFJochBrokerHttp::plot_resolution_estimate_histogram_get(Pistache::Http::ResponseWriter &response) {
+    GenericPlot(PlotType::ResEstimation, 0, response);
+}
+
 void JFJochBrokerHttp::config_internal_generator_image_put(const Pistache::Rest::Request &request,
                                                            Pistache::Http::ResponseWriter &response) {
     int64_t image_number = 0;
-    auto number_query = request.query().get("id");
+    auto number_query = request.query().get("number");
     if (number_query)
         image_number = std::stoi(number_query.value());
 
     if ((image_number < 0) || (image_number > 127))
-        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "id must be in range 0-127");
+        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "image_number must be in range 0-127");
 
     state_machine.LoadInternalGeneratorImage(request.body().data(), request.body().size(), image_number);
     logger.Info("Internal generator image #{} loaded", image_number);
     response.send(Pistache::Http::Code::Ok);
 }
 
-
-void JFJochBrokerHttp::config_internal_generator_image_tiff_put(const Pistache::Rest::Request &request,
-                                                                Pistache::Http::ResponseWriter &response) {
-    int64_t image_number = 0;
-    auto number_query = request.query().get("id");
-    if (number_query)
-        image_number = std::stoi(number_query.value());
-
-    if ((image_number < 0) || (image_number > 127))
-        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "id must be in range 0-127");
-
-    state_machine.LoadInternalGeneratorImageTIFF(request.body(), image_number);
-    response.send(Pistache::Http::Code::Ok);
-}
-
 void JFJochBrokerHttp::roi_box_get(Pistache::Http::ResponseWriter &response) {
     ProcessOutput(Convert(state_machine.GetBoxROI()), response);
 }
@@ -805,19 +785,3 @@ void JFJochBrokerHttp::plot_strong_pixel_get(const std::optional<int32_t> &binni
                                              Pistache::Http::ResponseWriter &response) {
     GenericPlot(PlotType::StrongPixels, binning, response);
 }
-
-void JFJochBrokerHttp::config_mask_tiff_get(Pistache::Http::ResponseWriter &response) {
-    std::string s = state_machine.GetFullPixelMaskTIFF();
-    response.send(Pistache::Http::Code::Ok, s, Pistache::Http::Mime::MediaType::fromString("image/tiff"));
-}
-
-void JFJochBrokerHttp::config_user_mask_tiff_get(Pistache::Http::ResponseWriter &response) {
-    std::string s = state_machine.GetUserPixelMaskTIFF();
-    response.send(Pistache::Http::Code::Ok, s, Pistache::Http::Mime::MediaType::fromString("image/tiff"));
-}
-
-void JFJochBrokerHttp::config_user_mask_tiff_put(const Pistache::Rest::Request &request,
-                                                  Pistache::Http::ResponseWriter &response) {
-    state_machine.SetUserPixelMask(request.body());
-    response.send(Pistache::Http::Code::Ok);
-}

broker/JFJochBrokerHttp.h

@@ -75,6 +75,8 @@ class JFJochBrokerHttp : public org::openapitools::server::api::DefaultApi {
 
     void plot_rad_int_per_file_get(Pistache::Http::ResponseWriter &response) override;
 
+    void plot_resolution_estimate_histogram_get(Pistache::Http::ResponseWriter &response) override;
+
     void statistics_calibration_get(Pistache::Http::ResponseWriter &response) override;
 
     void statistics_data_collection_get(Pistache::Http::ResponseWriter &response) override;
@@ -118,16 +120,6 @@ class JFJochBrokerHttp : public org::openapitools::server::api::DefaultApi {
     void xfel_event_code_get(Pistache::Http::ResponseWriter &response) override;
     void xfel_pulse_id_get(Pistache::Http::ResponseWriter &response) override;
 
-    void config_mask_tiff_get(Pistache::Http::ResponseWriter &response) override;
-
-    void config_user_mask_tiff_get(Pistache::Http::ResponseWriter &response) override;
-
-    void config_user_mask_tiff_put(const Pistache::Rest::Request &request,
-                                    Pistache::Http::ResponseWriter &response) override;
-
-    void config_internal_generator_image_tiff_put(const Pistache::Rest::Request &request,
-                                                  Pistache::Http::ResponseWriter &response) override;
-
     void GetStaticFile(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
     std::pair<Pistache::Http::Code, std::string> handleOperationException(const std::exception &ex) const noexcept override;
 

broker/JFJochBrokerParser.cpp

@@ -319,6 +319,7 @@ void ParseFacilityConfiguration(const nlohmann::json &input, const std::string&
                                       "omega_axis must be float array of 3");
         }
 
+        experiment.NeuralNetModelPath(GET_STR(j, "neural_net_model", ""));
         if (j.contains("pedestal_g0_frames"))
             experiment.PedestalG0Frames(GET_I64(j, "pedestal_g0_frames"));
         if (j.contains("pedestal_g1_frames"))

broker/JFJochServices.cpp

@@ -5,17 +5,15 @@
 
 JFJochServices::JFJochServices(Logger &in_logger) : logger(in_logger) {}
 
-void JFJochServices::Start(const DiffractionExperiment& experiment,
-                           const PixelMask &pixel_mask,
-                           const JFCalibration &calibration) {
+void JFJochServices::Start(const DiffractionExperiment& experiment, const JFCalibration &calibration) {
     logger.Info("Measurement start for: {}", experiment.GetFilePrefix());
 
     if (receiver != nullptr) {
         logger.Info("   ... receiver start");
         if (experiment.GetDetectorMode() == DetectorMode::Conversion)
-            receiver->Start(experiment, pixel_mask, &calibration);
+            receiver->Start(experiment, &calibration);
         else
-            receiver->Start(experiment, pixel_mask, nullptr);
+            receiver->Start(experiment, nullptr);
 
         if (detector && !experiment.IsUsingInternalPacketGen()) {
             logger.Info("   ... detector start");

broker/JFJochServices.h

@@ -23,9 +23,7 @@ public:
     void On(const DiffractionExperiment& experiment);
     void Off();
     void ConfigureDetector(const DiffractionExperiment& experiment);
-    void Start(const DiffractionExperiment& experiment,
-               const PixelMask &pixel_mask,
-               const JFCalibration &calibration);
+    void Start(const DiffractionExperiment& experiment, const JFCalibration &calibration);
     JFJochServicesOutput Stop();
     void Cancel();
     void Trigger();

broker/JFJochStateMachine.cpp

@@ -3,7 +3,7 @@
 #include <thread>
 
 #include "JFJochStateMachine.h"
-#include "../preview/JFJochTIFF.h"
+#include "../preview/WriteTIFF.h"
 
 void ApplyDetectorSettings(DiffractionExperiment& experiment, const DetectorSettings &settings) {
     auto tmp = experiment;
@@ -66,10 +66,8 @@ void ApplyRadialIntegrationSettings(DiffractionExperiment& experiment, const Rad
 }
 
 JFJochStateMachine::JFJochStateMachine(JFJochServices &in_services, Logger &in_logger)
-        : services(in_services),
-          logger(in_logger),
-          data_processing_settings(DiffractionExperiment::DefaultDataProcessingSettings()),
-          pixel_mask(experiment) {
+: services(in_services), logger(in_logger),
+data_processing_settings(DiffractionExperiment::DefaultDataProcessingSettings()) {
 
 }
 
@@ -128,7 +126,6 @@ void JFJochStateMachine::TakePedestalInternalAll(std::unique_lock<std::mutex> &u
             }
         }
         services.ConfigureDetector(experiment);
-        pixel_mask.LoadDetectorBadPixelMask(calibration->CalculateMask());
     } catch (const std::exception &e) {
         logger.Error("Pedestal sequence error {}", e.what());
         state = JFJochState::Error;
@@ -161,7 +158,7 @@ void JFJochStateMachine::TakePedestalInternalG0(std::unique_lock<std::mutex> &ul
 
     state = JFJochState::Pedestal;
     services.ConfigureDetector(local_experiment);
-    services.Start(local_experiment, pixel_mask, *calibration);
+    services.Start(local_experiment, *calibration);
 
     services.Trigger();
 
@@ -200,7 +197,7 @@ void JFJochStateMachine::TakePedestalInternalG1(std::unique_lock<std::mutex> &ul
 
     state = JFJochState::Pedestal;
     services.ConfigureDetector(local_experiment);
-    services.Start(local_experiment, pixel_mask, *calibration);
+    services.Start(local_experiment, *calibration);
 
     services.Trigger();
 
@@ -239,7 +236,7 @@ void JFJochStateMachine::TakePedestalInternalG2(std::unique_lock<std::mutex> &ul
 
     state = JFJochState::Pedestal;
     services.ConfigureDetector(local_experiment);
-    services.Start(local_experiment, pixel_mask, *calibration);
+    services.Start(local_experiment, *calibration);
 
     services.Trigger();
 
@@ -329,7 +326,7 @@ void JFJochStateMachine::Start(const DatasetSettings& settings) {
     try {
         state = JFJochState::Busy;
         services.SetSpotFindingSettings(GetSpotFindingSettings());
-        services.Start(experiment, pixel_mask, *calibration);
+        services.Start(experiment, *calibration);
 
         state = JFJochState::Measuring;
         measurement = std::async(std::launch::async, &JFJochStateMachine::MeasurementThread, this);
@@ -405,8 +402,6 @@ void JFJochStateMachine::SetFullMeasurementOutput(const JFJochServicesOutput &ou
     tmp.detector_height = experiment.GetYPixelsNum();
     tmp.detector_pixel_depth = experiment.GetPixelDepth();
     tmp.images_expected = experiment.GetImageNum();
-    tmp.unit_cell = experiment.GetUnitCellString();
-
 
     tmp.compression_ratio = output.receiver_output.status.compressed_ratio;
     tmp.collection_efficiency = output.receiver_output.efficiency;
@@ -431,8 +426,6 @@ void JFJochStateMachine::ClearAndSetMeasurementStatistics() {
     tmp.detector_width = experiment.GetYPixelsNum();
     tmp.detector_pixel_depth = experiment.GetPixelDepth();
     tmp.images_expected = experiment.GetImageNum();
-    tmp.unit_cell = experiment.GetUnitCellString();
-
     measurement_statistics = tmp;
 }
 
@@ -452,7 +445,6 @@ std::optional<MeasurementStatistics> JFJochStateMachine::GetMeasurementStatistic
         tmp.detector_height = experiment.GetYPixelsNum();
         tmp.detector_pixel_depth = experiment.GetPixelDepth();
         tmp.images_expected = experiment.GetImageNum();
-        tmp.unit_cell = experiment.GetUnitCellString();
 
         tmp.compression_ratio = rcv_status->compressed_ratio;
         tmp.images_collected = rcv_status->images_collected;
@@ -565,7 +557,6 @@ void JFJochStateMachine::AddDetectorSetup(const DetectorSetup &setup) {
         experiment.Detector(setup);
         gain_calibration = setup.GetGainCalibration();
         current_detector_setup = 0;
-        pixel_mask = PixelMask(setup);
     }
     detector_setup.emplace_back(setup);
 }
@@ -603,7 +594,6 @@ void JFJochStateMachine::SelectDetector(int64_t id) {
     try {
         experiment.Detector(detector_setup[id]);
         gain_calibration = detector_setup[id].GetGainCalibration();
-        pixel_mask = PixelMask(detector_setup[id]);
         state = JFJochState::Inactive;
         current_detector_setup = id;
     } catch (JFJochException &e) {
@@ -710,21 +700,6 @@ void JFJochStateMachine::LoadInternalGeneratorImage(const void *data, size_t siz
     services.LoadInternalGeneratorImage(experiment, image, image_number);
 }
 
-void JFJochStateMachine::LoadInternalGeneratorImageTIFF(const std::string &s, uint64_t image_number) {
-    std::unique_lock<std::mutex> ul(m);
-
-    if (state != JFJochState::Idle)
-        throw WrongDAQStateException ("Can change internal generator image only when detector in Idle state");
-
-    uint32_t cols, lines;
-    auto v = ReadTIFFFromString16(s, cols, lines);
-    if (((cols == experiment.GetXPixelsNum()) && (lines == experiment.GetYPixelsNum()))
-        || ((cols == RAW_MODULE_SIZE) && (lines == RAW_MODULE_LINES * experiment.GetModulesNum())))
-        services.LoadInternalGeneratorImage(experiment, v, image_number);
-    else
-        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Image size doesn't match current detector");
-}
-
 void JFJochStateMachine::SetBoxROI(const std::vector<ROIBox> &input) {
     std::unique_lock<std::mutex> ul(m);
 
@@ -764,33 +739,3 @@ std::vector<uint64_t> JFJochStateMachine::GetXFELEventCode() const {
     services.GetXFELEventCode(ret);
     return ret;
 }
-
-std::string JFJochStateMachine::GetFullPixelMaskTIFF() const {
-    std::unique_lock<std::mutex> ul(m);
-    std::vector v = pixel_mask.GetMask(experiment);
-    return WriteTIFFToString(v.data(), experiment.GetXPixelsNum(), experiment.GetYPixelsNum(),
-                             sizeof(uint32_t), false);
-}
-
-std::string JFJochStateMachine::GetUserPixelMaskTIFF() const {
-    std::unique_lock<std::mutex> ul(m);
-    std::vector v = pixel_mask.GetUserMask(experiment);
-    return WriteTIFFToString(v.data(), experiment.GetXPixelsNum(), experiment.GetYPixelsNum(),
-                             sizeof(uint32_t), false);
-}
-
-void JFJochStateMachine::SetUserPixelMask(const std::string &s) {
-    std::unique_lock<std::mutex> ul(m);
-
-    if (state != JFJochState::Idle)
-        throw WrongDAQStateException ("User mask can be only modified in Idle state");
-
-    try {
-        uint32_t cols, lines;
-        auto v = ReadTIFFFromString32(s, cols, lines);
-        pixel_mask.LoadUserMask(experiment, v);
-    } catch (const JFJochException &e) {
-        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
-                              "Problem handling user mask " + std::string(e.what()));
-    }
-}

broker/JFJochStateMachine.h

@@ -56,8 +56,6 @@ struct MeasurementStatistics {
 
     std::optional<float> bkg_estimate;
     std::optional<std::pair<float, float>> beam_center_drift_pxl;
-
-    std::string unit_cell;
 };
 
 struct DetectorSettings {
@@ -98,8 +96,6 @@ class JFJochStateMachine {
     volatile JFJochState state = JFJochState::Inactive;
     volatile bool cancel_sequence = false;
     std::unique_ptr<JFCalibration> calibration;
-    PixelMask pixel_mask;
-
     std::vector<JFModuleGainCalibration> gain_calibration;
     std::vector<DetectorSetup> detector_setup;
     int64_t current_detector_setup;
@@ -174,7 +170,6 @@ public:
     std::string GetPedestalTIFF(size_t gain_level, size_t sc) const;
 
     void LoadInternalGeneratorImage(const void *data, size_t size, uint64_t image_number);
-    void LoadInternalGeneratorImageTIFF(const std::string &s, uint64_t image_number);
 
     // Not thread safe - only for configuration in serial context
     DiffractionExperiment& NotThreadSafe_Experiment();
@@ -190,10 +185,6 @@ public:
 
     std::vector<uint64_t> GetXFELPulseID() const;
     std::vector<uint64_t> GetXFELEventCode() const;
-
-    std::string GetFullPixelMaskTIFF() const;
-    std::string GetUserPixelMaskTIFF() const;
-    void SetUserPixelMask(const std::string &v);
 };
 
 

broker/README.md

@@ -25,6 +25,7 @@ To run the test:
 
 ### Compile Jungfraujoch with frontend
 ```
+git submodule update --init --recursive
 mkdir build
 cd build
 cmake ..

broker/gen/api/DefaultApi.cpp

@@ -37,16 +37,12 @@ void DefaultApi::setupRoutes() {
     Routes::Get(*router, base + "/config/detector", Routes::bind(&DefaultApi::config_detector_get_handler, this));
     Routes::Put(*router, base + "/config/detector", Routes::bind(&DefaultApi::config_detector_put_handler, this));
     Routes::Put(*router, base + "/config/internal_generator_image", Routes::bind(&DefaultApi::config_internal_generator_image_put_handler, this));
-    Routes::Put(*router, base + "/config/internal_generator_image.tiff", Routes::bind(&DefaultApi::config_internal_generator_image_tiff_put_handler, this));
-    Routes::Get(*router, base + "/config/mask.tiff", Routes::bind(&DefaultApi::config_mask_tiff_get_handler, this));
     Routes::Get(*router, base + "/config/rad_int", Routes::bind(&DefaultApi::config_rad_int_get_handler, this));
     Routes::Put(*router, base + "/config/rad_int", Routes::bind(&DefaultApi::config_rad_int_put_handler, this));
     Routes::Get(*router, base + "/config/select_detector", Routes::bind(&DefaultApi::config_select_detector_get_handler, this));
     Routes::Put(*router, base + "/config/select_detector", Routes::bind(&DefaultApi::config_select_detector_put_handler, this));
     Routes::Get(*router, base + "/config/spot_finding", Routes::bind(&DefaultApi::config_spot_finding_get_handler, this));
     Routes::Put(*router, base + "/config/spot_finding", Routes::bind(&DefaultApi::config_spot_finding_put_handler, this));
-    Routes::Get(*router, base + "/config/user_mask.tiff", Routes::bind(&DefaultApi::config_user_mask_tiff_get_handler, this));
-    Routes::Put(*router, base + "/config/user_mask.tiff", Routes::bind(&DefaultApi::config_user_mask_tiff_put_handler, this));
     Routes::Post(*router, base + "/deactivate", Routes::bind(&DefaultApi::deactivate_post_handler, this));
     Routes::Get(*router, base + "/detector/status", Routes::bind(&DefaultApi::detector_status_get_handler, this));
     Routes::Post(*router, base + "/initialize", Routes::bind(&DefaultApi::initialize_post_handler, this));
@@ -60,6 +56,7 @@ void DefaultApi::setupRoutes() {
     Routes::Get(*router, base + "/plot/rad_int_per_file", Routes::bind(&DefaultApi::plot_rad_int_per_file_get_handler, this));
     Routes::Get(*router, base + "/plot/receiver_delay", Routes::bind(&DefaultApi::plot_receiver_delay_get_handler, this));
     Routes::Get(*router, base + "/plot/receiver_free_send_buffers", Routes::bind(&DefaultApi::plot_receiver_free_send_buffers_get_handler, this));
+    Routes::Get(*router, base + "/plot/resolution_estimate_histogram", Routes::bind(&DefaultApi::plot_resolution_estimate_histogram_get_handler, this));
     Routes::Get(*router, base + "/plot/roi_max_count", Routes::bind(&DefaultApi::plot_roi_max_count_get_handler, this));
     Routes::Get(*router, base + "/plot/roi_sum", Routes::bind(&DefaultApi::plot_roi_sum_get_handler, this));
     Routes::Get(*router, base + "/plot/roi_valid_pixels", Routes::bind(&DefaultApi::plot_roi_valid_pixels_get_handler, this));
@@ -196,45 +193,6 @@ void DefaultApi::config_internal_generator_image_put_handler(const Pistache::Res
         response.send(Pistache::Http::Code::Internal_Server_Error, e.what());
     }
 
-}
-void DefaultApi::config_internal_generator_image_tiff_put_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response) {
-    try {
-
-    try {
-      this->config_internal_generator_image_tiff_put(request, response);
-    } catch (Pistache::Http::HttpError &e) {
-        response.send(static_cast<Pistache::Http::Code>(e.code()), e.what());
-        return;
-    } catch (std::exception &e) {
-        const std::pair<Pistache::Http::Code, std::string> errorInfo = this->handleOperationException(e);
-        response.send(errorInfo.first, errorInfo.second);
-        return;
-    }
-
-    } catch (std::exception &e) {
-        response.send(Pistache::Http::Code::Internal_Server_Error, e.what());
-    }
-
-}
-void DefaultApi::config_mask_tiff_get_handler(const Pistache::Rest::Request &, Pistache::Http::ResponseWriter response) {
-    try {
-
-
-    try {
-        this->config_mask_tiff_get(response);
-    } catch (Pistache::Http::HttpError &e) {
-        response.send(static_cast<Pistache::Http::Code>(e.code()), e.what());
-        return;
-    } catch (std::exception &e) {
-        const std::pair<Pistache::Http::Code, std::string> errorInfo = this->handleOperationException(e);
-        response.send(errorInfo.first, errorInfo.second);
-        return;
-    }
-
-    } catch (std::exception &e) {
-        response.send(Pistache::Http::Code::Internal_Server_Error, e.what());
-    }
-
 }
 void DefaultApi::config_rad_int_get_handler(const Pistache::Rest::Request &, Pistache::Http::ResponseWriter response) {
     try {
@@ -394,45 +352,6 @@ void DefaultApi::config_spot_finding_put_handler(const Pistache::Rest::Request &
         response.send(Pistache::Http::Code::Internal_Server_Error, e.what());
     }
 
-}
-void DefaultApi::config_user_mask_tiff_get_handler(const Pistache::Rest::Request &, Pistache::Http::ResponseWriter response) {
-    try {
-
-
-    try {
-        this->config_user_mask_tiff_get(response);
-    } catch (Pistache::Http::HttpError &e) {
-        response.send(static_cast<Pistache::Http::Code>(e.code()), e.what());
-        return;
-    } catch (std::exception &e) {
-        const std::pair<Pistache::Http::Code, std::string> errorInfo = this->handleOperationException(e);
-        response.send(errorInfo.first, errorInfo.second);
-        return;
-    }
-
-    } catch (std::exception &e) {
-        response.send(Pistache::Http::Code::Internal_Server_Error, e.what());
-    }
-
-}
-void DefaultApi::config_user_mask_tiff_put_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response) {
-    try {
-
-    try {
-      this->config_user_mask_tiff_put(request, response);
-    } catch (Pistache::Http::HttpError &e) {
-        response.send(static_cast<Pistache::Http::Code>(e.code()), e.what());
-        return;
-    } catch (std::exception &e) {
-        const std::pair<Pistache::Http::Code, std::string> errorInfo = this->handleOperationException(e);
-        response.send(errorInfo.first, errorInfo.second);
-        return;
-    }
-
-    } catch (std::exception &e) {
-        response.send(Pistache::Http::Code::Internal_Server_Error, e.what());
-    }
-
 }
 void DefaultApi::deactivate_post_handler(const Pistache::Rest::Request &, Pistache::Http::ResponseWriter response) {
     try {
@@ -753,6 +672,26 @@ void DefaultApi::plot_receiver_free_send_buffers_get_handler(const Pistache::Res
         response.send(Pistache::Http::Code::Internal_Server_Error, e.what());
     }
 
+}
+void DefaultApi::plot_resolution_estimate_histogram_get_handler(const Pistache::Rest::Request &, Pistache::Http::ResponseWriter response) {
+    try {
+
+
+    try {
+        this->plot_resolution_estimate_histogram_get(response);
+    } catch (Pistache::Http::HttpError &e) {
+        response.send(static_cast<Pistache::Http::Code>(e.code()), e.what());
+        return;
+    } catch (std::exception &e) {
+        const std::pair<Pistache::Http::Code, std::string> errorInfo = this->handleOperationException(e);
+        response.send(errorInfo.first, errorInfo.second);
+        return;
+    }
+
+    } catch (std::exception &e) {
+        response.send(Pistache::Http::Code::Internal_Server_Error, e.what());
+    }
+
 }
 void DefaultApi::plot_roi_max_count_get_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response) {
     try {

broker/gen/api/DefaultApi.h

@@ -61,16 +61,12 @@ private:
     void config_detector_get_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
     void config_detector_put_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
     void config_internal_generator_image_put_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
-    void config_internal_generator_image_tiff_put_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
-    void config_mask_tiff_get_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
     void config_rad_int_get_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
     void config_rad_int_put_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
     void config_select_detector_get_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
     void config_select_detector_put_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
     void config_spot_finding_get_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
     void config_spot_finding_put_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
-    void config_user_mask_tiff_get_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
-    void config_user_mask_tiff_put_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
     void deactivate_post_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
     void detector_status_get_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
     void initialize_post_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
@@ -84,6 +80,7 @@ private:
     void plot_rad_int_per_file_get_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
     void plot_receiver_delay_get_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
     void plot_receiver_free_send_buffers_get_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
+    void plot_resolution_estimate_histogram_get_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
     void plot_roi_max_count_get_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
     void plot_roi_sum_get_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
     void plot_roi_valid_pixels_get_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
@@ -154,20 +151,6 @@ private:
     /// </remarks>
     virtual void config_internal_generator_image_put(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter &response) = 0;
     /// <summary>
-    /// Load TIFF image for internal FPGA generator
-    /// </summary>
-    /// <remarks>
-    /// Load image for internal FPGA generator. This can only happen in Idle state of the detector. Requires TIFF with 16-bit integer numbers of size of detector in raw/converted coordinates  (depending on detector settings). 
-    /// </remarks>
-    virtual void config_internal_generator_image_tiff_put(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter &response) = 0;
-    /// <summary>
-    /// Get mask of the detector
-    /// </summary>
-    /// <remarks>
-    /// Get full pixel mask of the detector See NXmx standard for meaning of pixel values 
-    /// </remarks>
-    virtual void config_mask_tiff_get(Pistache::Http::ResponseWriter &response) = 0;
-    /// <summary>
     /// Get radial integration configuration
     /// </summary>
     /// <remarks>
@@ -213,20 +196,6 @@ private:
     /// <param name="spotFindingSettings"> (optional)</param>
     virtual void config_spot_finding_put(const org::openapitools::server::model::Spot_finding_settings &spotFindingSettings, Pistache::Http::ResponseWriter &response) = 0;
     /// <summary>
-    /// Get user mask of the detector
-    /// </summary>
-    /// <remarks>
-    /// Get user pixel mask of the detector in the actual detector coordinates: 0 - good pixel, 1 - masked
-    /// </remarks>
-    virtual void config_user_mask_tiff_get(Pistache::Http::ResponseWriter &response) = 0;
-    /// <summary>
-    /// Upload user mask of the detector
-    /// </summary>
-    /// <remarks>
-    /// Should be in &#x60;Idle&#x60; state. Upload user mask of the detector - this is for example to account for beam stop shadow or misbehaving regions. If detector is conversion mode the mask can be both in raw (1024x512; stacked modules) or converted coordinates. In the latter case - module gaps are ignored and don&#39;t need to be assigned value. Mask is expected as TIFF (4-byte; unsigned). 0 - good pixel, other value - masked User mask is stored in NXmx pixel mask (bit 8), as well as used in spot finding and azimuthal integration.  User mask is not automatically applied - i.e. pixels with user mask will have a valid pixel value in the images. 
-    /// </remarks>
-    virtual void config_user_mask_tiff_put(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter &response) = 0;
-    /// <summary>
     /// Prepare detector to turn off
     /// </summary>
     /// <remarks>
@@ -324,6 +293,13 @@ private:
     /// <param name="binning">Binning of frames for the plot (0 &#x3D; default binning) (optional, default to 0)</param>
     virtual void plot_receiver_free_send_buffers_get(const std::optional<int32_t> &binning, Pistache::Http::ResponseWriter &response) = 0;
     /// <summary>
+    /// Generate resolution estimate histogram
+    /// </summary>
+    /// <remarks>
+    /// Generate histogram of crystal resolutions from 1.0 to 5.0 A based on ML model
+    /// </remarks>
+    virtual void plot_resolution_estimate_histogram_get(Pistache::Http::ResponseWriter &response) = 0;
+    /// <summary>
     /// Generate plot of ROI max count
     /// </summary>
     /// <remarks>

broker/gen/model/Measurement_statistics.cpp

@@ -51,8 +51,6 @@ Measurement_statistics::Measurement_statistics()
     m_Detector_pixel_depthIsSet = false;
     m_Bkg_estimate = 0.0f;
     m_Bkg_estimateIsSet = false;
-    m_Unit_cell = "";
-    m_Unit_cellIsSet = false;
     
 }
 
@@ -108,7 +106,7 @@ bool Measurement_statistics::validate(std::stringstream& msg, const std::string&
         }
 
     }
-                                    
+                                
     return success;
 }
 
@@ -160,10 +158,7 @@ bool Measurement_statistics::operator==(const Measurement_statistics& rhs) const
     ((!detectorPixelDepthIsSet() && !rhs.detectorPixelDepthIsSet()) || (detectorPixelDepthIsSet() && rhs.detectorPixelDepthIsSet() && getDetectorPixelDepth() == rhs.getDetectorPixelDepth())) &&
     
     
-    ((!bkgEstimateIsSet() && !rhs.bkgEstimateIsSet()) || (bkgEstimateIsSet() && rhs.bkgEstimateIsSet() && getBkgEstimate() == rhs.getBkgEstimate())) &&
-    
-    
-    ((!unitCellIsSet() && !rhs.unitCellIsSet()) || (unitCellIsSet() && rhs.unitCellIsSet() && getUnitCell() == rhs.getUnitCell()))
+    ((!bkgEstimateIsSet() && !rhs.bkgEstimateIsSet()) || (bkgEstimateIsSet() && rhs.bkgEstimateIsSet() && getBkgEstimate() == rhs.getBkgEstimate()))
     
     ;
 }
@@ -206,8 +201,6 @@ void to_json(nlohmann::json& j, const Measurement_statistics& o)
         j["detector_pixel_depth"] = o.m_Detector_pixel_depth;
     if(o.bkgEstimateIsSet())
         j["bkg_estimate"] = o.m_Bkg_estimate;
-    if(o.unitCellIsSet())
-        j["unit_cell"] = o.m_Unit_cell;
     
 }
 
@@ -288,11 +281,6 @@ void from_json(const nlohmann::json& j, Measurement_statistics& o)
         j.at("bkg_estimate").get_to(o.m_Bkg_estimate);
         o.m_Bkg_estimateIsSet = true;
     } 
-    if(j.find("unit_cell") != j.end())
-    {
-        j.at("unit_cell").get_to(o.m_Unit_cell);
-        o.m_Unit_cellIsSet = true;
-    } 
     
 }
 
@@ -551,23 +539,6 @@ void Measurement_statistics::unsetBkg_estimate()
 {
     m_Bkg_estimateIsSet = false;
 }
-std::string Measurement_statistics::getUnitCell() const
-{
-    return m_Unit_cell;
-}
-void Measurement_statistics::setUnitCell(std::string const& value)
-{
-    m_Unit_cell = value;
-    m_Unit_cellIsSet = true;
-}
-bool Measurement_statistics::unitCellIsSet() const
-{
-    return m_Unit_cellIsSet;
-}
-void Measurement_statistics::unsetUnit_cell()
-{
-    m_Unit_cellIsSet = false;
-}
 
 
 } // namespace org::openapitools::server::model

broker/gen/model/Measurement_statistics.h

@@ -163,13 +163,6 @@ public:
     void setBkgEstimate(float const value);
     bool bkgEstimateIsSet() const;
     void unsetBkg_estimate();
-    /// <summary>
-    /// 
-    /// </summary>
-    std::string getUnitCell() const;
-    void setUnitCell(std::string const& value);
-    bool unitCellIsSet() const;
-    void unsetUnit_cell();
 
     friend void to_json(nlohmann::json& j, const Measurement_statistics& o);
     friend void from_json(const nlohmann::json& j, Measurement_statistics& o);
@@ -204,8 +197,6 @@ protected:
     bool m_Detector_pixel_depthIsSet;
     float m_Bkg_estimate;
     bool m_Bkg_estimateIsSet;
-    std::string m_Unit_cell;
-    bool m_Unit_cellIsSet;
     
 };
 

broker/gen/model/Preview_settings.cpp

@@ -26,13 +26,11 @@ Preview_settings::Preview_settings()
     m_Show_spotsIsSet = false;
     m_Show_roi = false;
     m_Show_roiIsSet = false;
-    m_Jpeg_quality = 100L;
+    m_Jpeg_quality = 0L;
     m_Jpeg_qualityIsSet = false;
     m_Show_indexed = false;
     m_Show_indexedIsSet = false;
-    m_Show_user_mask = false;
-    m_Show_user_maskIsSet = false;
-    m_Resolution_ring = 0.1f;
+    m_Resolution_ring = 0.0f;
     m_Resolution_ringIsSet = false;
     
 }
@@ -94,7 +92,7 @@ bool Preview_settings::validate(std::stringstream& msg, const std::string& pathP
         }
 
     }
-                 
+             
     if (resolutionRingIsSet())
     {
         const float& value = m_Resolution_ring;
@@ -138,9 +136,6 @@ bool Preview_settings::operator==(const Preview_settings& rhs) const
     ((!showIndexedIsSet() && !rhs.showIndexedIsSet()) || (showIndexedIsSet() && rhs.showIndexedIsSet() && isShowIndexed() == rhs.isShowIndexed())) &&
     
     
-    ((!showUserMaskIsSet() && !rhs.showUserMaskIsSet()) || (showUserMaskIsSet() && rhs.showUserMaskIsSet() && isShowUserMask() == rhs.isShowUserMask())) &&
-    
-    
     ((!resolutionRingIsSet() && !rhs.resolutionRingIsSet()) || (resolutionRingIsSet() && rhs.resolutionRingIsSet() && getResolutionRing() == rhs.getResolutionRing()))
     
     ;
@@ -163,8 +158,6 @@ void to_json(nlohmann::json& j, const Preview_settings& o)
         j["jpeg_quality"] = o.m_Jpeg_quality;
     if(o.showIndexedIsSet())
         j["show_indexed"] = o.m_Show_indexed;
-    if(o.showUserMaskIsSet())
-        j["show_user_mask"] = o.m_Show_user_mask;
     if(o.resolutionRingIsSet())
         j["resolution_ring"] = o.m_Resolution_ring;
     
@@ -193,11 +186,6 @@ void from_json(const nlohmann::json& j, Preview_settings& o)
         j.at("show_indexed").get_to(o.m_Show_indexed);
         o.m_Show_indexedIsSet = true;
     } 
-    if(j.find("show_user_mask") != j.end())
-    {
-        j.at("show_user_mask").get_to(o.m_Show_user_mask);
-        o.m_Show_user_maskIsSet = true;
-    } 
     if(j.find("resolution_ring") != j.end())
     {
         j.at("resolution_ring").get_to(o.m_Resolution_ring);
@@ -282,23 +270,6 @@ void Preview_settings::unsetShow_indexed()
 {
     m_Show_indexedIsSet = false;
 }
-bool Preview_settings::isShowUserMask() const
-{
-    return m_Show_user_mask;
-}
-void Preview_settings::setShowUserMask(bool const value)
-{
-    m_Show_user_mask = value;
-    m_Show_user_maskIsSet = true;
-}
-bool Preview_settings::showUserMaskIsSet() const
-{
-    return m_Show_user_maskIsSet;
-}
-void Preview_settings::unsetShow_user_mask()
-{
-    m_Show_user_maskIsSet = false;
-}
 float Preview_settings::getResolutionRing() const
 {
     return m_Resolution_ring;

broker/gen/model/Preview_settings.h

@@ -91,13 +91,6 @@ public:
     bool showIndexedIsSet() const;
     void unsetShow_indexed();
     /// <summary>
-    /// Show user mask
-    /// </summary>
-    bool isShowUserMask() const;
-    void setShowUserMask(bool const value);
-    bool showUserMaskIsSet() const;
-    void unsetShow_user_mask();
-    /// <summary>
     /// 
     /// </summary>
     float getResolutionRing() const;
@@ -118,8 +111,6 @@ protected:
     bool m_Jpeg_qualityIsSet;
     bool m_Show_indexed;
     bool m_Show_indexedIsSet;
-    bool m_Show_user_mask;
-    bool m_Show_user_maskIsSet;
     float m_Resolution_ring;
     bool m_Resolution_ringIsSet;
     

broker/gen/model/Spot_finding_settings.cpp

@@ -23,10 +23,6 @@ Spot_finding_settings::Spot_finding_settings()
 {
     m_Enable = true;
     m_Indexing = true;
-    m_Filter_powder_rings = false;
-    m_Filter_powder_ringsIsSet = false;
-    m_Min_spot_count_powder_ring = 20L;
-    m_Min_spot_count_powder_ringIsSet = false;
     m_Signal_to_noise_threshold = 0.0f;
     m_Photon_count_threshold = 0L;
     m_Min_pix_per_spot = 0L;
@@ -56,21 +52,7 @@ bool Spot_finding_settings::validate(std::stringstream& msg, const std::string&
     bool success = true;
     const std::string _pathPrefix = pathPrefix.empty() ? "Spot_finding_settings" : pathPrefix;
 
-                     
-    if (minSpotCountPowderRingIsSet())
-    {
-        const int64_t& value = m_Min_spot_count_powder_ring;
-        const std::string currentValuePath = _pathPrefix + ".minSpotCountPowderRing";
-                
-        
-        if (value < 5ll)
-        {
-            success = false;
-            msg << currentValuePath << ": must be greater than or equal to 5;";
-        }
-
-    }
-         
+                 
     
     /* Signal_to_noise_threshold */ {
         const float& value = m_Signal_to_noise_threshold;
@@ -160,12 +142,6 @@ bool Spot_finding_settings::operator==(const Spot_finding_settings& rhs) const
     (isIndexing() == rhs.isIndexing())
      &&
     
-    
-    ((!filterPowderRingsIsSet() && !rhs.filterPowderRingsIsSet()) || (filterPowderRingsIsSet() && rhs.filterPowderRingsIsSet() && isFilterPowderRings() == rhs.isFilterPowderRings())) &&
-    
-    
-    ((!minSpotCountPowderRingIsSet() && !rhs.minSpotCountPowderRingIsSet()) || (minSpotCountPowderRingIsSet() && rhs.minSpotCountPowderRingIsSet() && getMinSpotCountPowderRing() == rhs.getMinSpotCountPowderRing())) &&
-    
     (getSignalToNoiseThreshold() == rhs.getSignalToNoiseThreshold())
      &&
     
@@ -200,10 +176,6 @@ void to_json(nlohmann::json& j, const Spot_finding_settings& o)
     j = nlohmann::json();
     j["enable"] = o.m_Enable;
     j["indexing"] = o.m_Indexing;
-    if(o.filterPowderRingsIsSet())
-        j["filter_powder_rings"] = o.m_Filter_powder_rings;
-    if(o.minSpotCountPowderRingIsSet())
-        j["min_spot_count_powder_ring"] = o.m_Min_spot_count_powder_ring;
     j["signal_to_noise_threshold"] = o.m_Signal_to_noise_threshold;
     j["photon_count_threshold"] = o.m_Photon_count_threshold;
     j["min_pix_per_spot"] = o.m_Min_pix_per_spot;
@@ -218,16 +190,6 @@ void from_json(const nlohmann::json& j, Spot_finding_settings& o)
 {
     j.at("enable").get_to(o.m_Enable);
     j.at("indexing").get_to(o.m_Indexing);
-    if(j.find("filter_powder_rings") != j.end())
-    {
-        j.at("filter_powder_rings").get_to(o.m_Filter_powder_rings);
-        o.m_Filter_powder_ringsIsSet = true;
-    } 
-    if(j.find("min_spot_count_powder_ring") != j.end())
-    {
-        j.at("min_spot_count_powder_ring").get_to(o.m_Min_spot_count_powder_ring);
-        o.m_Min_spot_count_powder_ringIsSet = true;
-    } 
     j.at("signal_to_noise_threshold").get_to(o.m_Signal_to_noise_threshold);
     j.at("photon_count_threshold").get_to(o.m_Photon_count_threshold);
     j.at("min_pix_per_spot").get_to(o.m_Min_pix_per_spot);
@@ -254,40 +216,6 @@ void Spot_finding_settings::setIndexing(bool const value)
 {
     m_Indexing = value;
 }
-bool Spot_finding_settings::isFilterPowderRings() const
-{
-    return m_Filter_powder_rings;
-}
-void Spot_finding_settings::setFilterPowderRings(bool const value)
-{
-    m_Filter_powder_rings = value;
-    m_Filter_powder_ringsIsSet = true;
-}
-bool Spot_finding_settings::filterPowderRingsIsSet() const
-{
-    return m_Filter_powder_ringsIsSet;
-}
-void Spot_finding_settings::unsetFilter_powder_rings()
-{
-    m_Filter_powder_ringsIsSet = false;
-}
-int64_t Spot_finding_settings::getMinSpotCountPowderRing() const
-{
-    return m_Min_spot_count_powder_ring;
-}
-void Spot_finding_settings::setMinSpotCountPowderRing(int64_t const value)
-{
-    m_Min_spot_count_powder_ring = value;
-    m_Min_spot_count_powder_ringIsSet = true;
-}
-bool Spot_finding_settings::minSpotCountPowderRingIsSet() const
-{
-    return m_Min_spot_count_powder_ringIsSet;
-}
-void Spot_finding_settings::unsetMin_spot_count_powder_ring()
-{
-    m_Min_spot_count_powder_ringIsSet = false;
-}
 float Spot_finding_settings::getSignalToNoiseThreshold() const
 {
     return m_Signal_to_noise_threshold;

broker/gen/model/Spot_finding_settings.h

@@ -58,30 +58,16 @@ public:
     /// Spot_finding_settings members
 
     /// <summary>
-    /// Enable spot finding. This is temporary setting, i.e. can be changed anytime during data collection.  Even if disabled spot finding information will still be send and written, though always with zero spots. 
+    /// Enable spot finding
     /// </summary>
     bool isEnable() const;
     void setEnable(bool const value);
     /// <summary>
-    /// Enable indexing. This is temporary setting, i.e. can be changed anytime during data collection. 
+    /// Enable indexing
     /// </summary>
     bool isIndexing() const;
     void setIndexing(bool const value);
     /// <summary>
-    /// Filter spots which form powder rings (e.g., ice rings)
-    /// </summary>
-    bool isFilterPowderRings() const;
-    void setFilterPowderRings(bool const value);
-    bool filterPowderRingsIsSet() const;
-    void unsetFilter_powder_rings();
-    /// <summary>
-    /// Minimum number of spots to consider a thin resolution shell (0.01 A^-1) a powder ring and filter out.
-    /// </summary>
-    int64_t getMinSpotCountPowderRing() const;
-    void setMinSpotCountPowderRing(int64_t const value);
-    bool minSpotCountPowderRingIsSet() const;
-    void unsetMin_spot_count_powder_ring();
-    /// <summary>
     /// 
     /// </summary>
     float getSignalToNoiseThreshold() const;
@@ -124,10 +110,6 @@ protected:
 
     bool m_Indexing;
 
-    bool m_Filter_powder_rings;
-    bool m_Filter_powder_ringsIsSet;
-    int64_t m_Min_spot_count_powder_ring;
-    bool m_Min_spot_count_powder_ringIsSet;
     float m_Signal_to_noise_threshold;
 
     int64_t m_Photon_count_threshold;

broker/jfjoch_api.yaml

@@ -333,23 +333,11 @@ components:
         enable:
           type: boolean
           default: true
-          description: |
-            Enable spot finding. This is temporary setting, i.e. can be changed anytime during data collection. 
-            Even if disabled spot finding information will still be send and written, though always with zero spots.
+          description: Enable spot finding
         indexing:
           type: boolean
           default: true
-          description: |
-            Enable indexing. This is temporary setting, i.e. can be changed anytime during data collection.
-        filter_powder_rings:
-          type: boolean
-          default: false
-          description: Filter spots which form powder rings (e.g., ice rings)
-        min_spot_count_powder_ring:
-          type: integer
-          format: int64
-          minimum: 5
-          description: Minimum number of spots to consider a thin resolution shell (0.01 A^-1) a powder ring and filter out.
+          description: Enable indexing
         signal_to_noise_threshold:
           type: number
           format: float
@@ -512,8 +500,6 @@ components:
         bkg_estimate:
           type: number
           format: float
-        unit_cell:
-          type: string
     broker_status:
       type: object
       required:
@@ -632,21 +618,14 @@ components:
           type: integer
           description: "Quality of JPEG image (100 - highest; 0 - lowest)"
           format: int64
-          default: 100
           minimum: 0
           maximum: 100
         show_indexed:
           type: boolean
           description: "Preview indexed images only"
-          default: false
-        show_user_mask:
-          type: boolean
-          description: "Show user mask"
-          default: false
         resolution_ring:
           type: number
           format: float
-          default: 0.1
           minimum: 0.1
           maximum: 100.0
     error_message:
@@ -986,9 +965,9 @@ paths:
         Requires binary blob with 16-bit integer numbers of size of detector in raw/converted coordinates 
         (depending on detector settings).
       parameters:
-        - name: id
+        - name: number
           in: query
-          description: Image id to upload
+          description: Image number to upload
           required: false
           schema:
             type: integer
@@ -1011,38 +990,6 @@ paths:
               schema:
                 type: string
                 description: Exception error
-  /config/internal_generator_image.tiff:
-    put:
-      summary: Load TIFF image for internal FPGA generator
-      description: |
-        Load image for internal FPGA generator. This can only happen in Idle state of the detector.
-        Requires TIFF with 16-bit integer numbers of size of detector in raw/converted coordinates 
-        (depending on detector settings).
-      parameters:
-        - in: query
-          name: id
-          description: Image ID to upload
-          required: false
-          schema:
-            type: integer
-            minimum: 0
-            maximum: 127
-      requestBody:
-        content:
-          image/tiff:
-            schema:
-              type: string
-              format: binary
-      responses:
-        "200":
-          description: Everything OK
-        "400":
-          description: Input parsing or validation error
-          content:
-            text/plain:
-              schema:
-                type: string
-                description: Exception error
   /config/select_detector:
     put:
       summary: Select detector
@@ -1498,6 +1445,17 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/plots'
+  /plot/resolution_estimate_histogram:
+    get:
+      summary: Generate resolution estimate histogram
+      description: Generate histogram of crystal resolutions from 1.0 to 5.0 A based on ML model
+      responses:
+        "200":
+          description: Everything OK
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/plots'
   /plot/rad_int:
     get:
       summary: Generate radial integration profile
@@ -1615,63 +1573,6 @@ paths:
                 format: binary
         "404":
           description: No preview image recorded so far
-  /config/mask.tiff:
-    get:
-      summary: Get mask of the detector
-      description: |
-        Get full pixel mask of the detector
-        See NXmx standard for meaning of pixel values
-      responses:
-        "200":
-          description: Pixel mask in TIFF format (4 byte; unsigned)
-          content:
-            image/tiff:
-              schema:
-                type: string
-                format: binary
-  /config/user_mask.tiff:
-    get:
-      summary: Get user mask of the detector
-      description: "Get user pixel mask of the detector in the actual detector coordinates: 0 - good pixel, 1 - masked"
-      responses:
-        "200":
-          description: User mask in TIFF format (4 byte; unsigned)
-          content:
-            image/tiff:
-              schema:
-                type: string
-                format: binary
-    put:
-      summary: Upload user mask of the detector
-      description: |
-        Should be in `Idle` state.
-        Upload user mask of the detector - this is for example to account for beam stop shadow or misbehaving regions.
-        If detector is conversion mode the mask can be both in raw (1024x512; stacked modules) or converted coordinates.
-        In the latter case - module gaps are ignored and don't need to be assigned value.
-        Mask is expected as TIFF (4-byte; unsigned).
-        0 - good pixel, other value - masked
-        User mask is stored in NXmx pixel mask (bit 8), as well as used in spot finding and azimuthal integration. 
-        User mask is not automatically applied - i.e. pixels with user mask will have a valid pixel value in the images.
-      requestBody:
-        content:
-          application/octet-stream:
-            schema:
-              type: string
-              format: binary
-      responses:
-        "200":
-          description: All good
-          content:
-            image/tiff:
-              schema:
-                type: string
-                format: binary
-        "500":
-          description: Error within Jungfraujoch code - see output message.
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/error_message'
   /preview/pedestal.tiff:
     get:
       parameters:

broker/jfjoch_broker.cpp

@@ -2,7 +2,7 @@
 // Using OpenAPI licensed with Apache License 2.0
 
 #include <vector>
-#include <csignal>
+#include <signal.h>
 #include <fstream>
 #include <nlohmann/json.hpp>
 
@@ -12,7 +12,7 @@
 #include "JFJochBrokerHttp.h"
 
 #include "JFJochBrokerParser.h"
-#include "../frame_serialize/ZMQStream2Pusher.h"
+#include "../frame_serialize/ZMQStream2PusherGroup.h"
 #include "../frame_serialize/DumpCBORToFilePusher.h"
 
 static Pistache::Http::Endpoint *httpEndpoint;
@@ -73,6 +73,8 @@ int main (int argc, char **argv) {
     std::unique_ptr<JFJochReceiverService> receiver;
     std::unique_ptr<ImagePusher> image_pusher;
 
+    ZMQContext context;
+
     DiffractionExperiment experiment;
     experiment.MaskChipEdges(true).MaskModuleEdges(true);
 
@@ -86,15 +88,9 @@ int main (int argc, char **argv) {
             int32_t zmq_send_watermark = ParseInt32(input, "zmq_send_watermark", 100);
             int32_t zmq_send_buffer_size = ParseInt32(input, "zmq_send_buffer_size", -1);
 
-            auto tmp = std::make_unique<ZMQStream2Pusher>(ParseStringArray(input, "zmq_image_addr"),
+            image_pusher = std::make_unique<ZMQStream2PusherGroup>(ParseStringArray(input, "zmq_image_addr"),
                                                                    zmq_send_watermark,
                                                                    zmq_send_buffer_size);
-
-            std::string preview_addr = ParseString(input, "zmq_preview_addr", "");
-            if (!preview_addr.empty())
-                tmp->PreviewSocket(preview_addr);
-
-            image_pusher = std::move(tmp);
         } else if (pusher_type == "dump_cbor") {
             image_pusher = std::make_unique<DumpCBORToFilePusher>();
         } else

common/CMakeLists.txt

@@ -23,19 +23,14 @@ MESSAGE(STATUS "Jungfraujoch version: ${PACKAGE_VERSION}")
 
 CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/GitInfo.cpp.in" "${CMAKE_CURRENT_BINARY_DIR}/GitInfo.cpp" @ONLY)
 
-ADD_LIBRARY(JFJochLogger STATIC
+ADD_LIBRARY( JFJochCommon STATIC
         Logger.cpp Logger.h
-        ${CMAKE_CURRENT_BINARY_DIR}/GitInfo.cpp GitInfo.h
-)
-
-ADD_LIBRARY(JFJochZMQ STATIC ZMQWrappers.cpp ZMQWrappers.h)
-
-ADD_LIBRARY(JFJochCommon STATIC
         Coord.cpp Coord.h
         DiffractionExperiment.cpp DiffractionExperiment.h
         RawToConvertedGeometry.h
         JFJochException.h
         Definitions.h
+        ${CMAKE_CURRENT_BINARY_DIR}/GitInfo.cpp GitInfo.h
         ThreadSafeFIFO.h
         DiffractionSpot.cpp DiffractionSpot.h
         StatusVector.h
@@ -50,6 +45,7 @@ ADD_LIBRARY(JFJochCommon STATIC
         RawToConvertedGeometryCore.h
         Plot.h
         ../fpga/pcie_driver/jfjoch_fpga.h
+        ZMQWrappers.cpp ZMQWrappers.h
         DatasetSettings.cpp DatasetSettings.h
         ROIMap.cpp ROIMap.h
         ROIElement.cpp ROIElement.h
@@ -61,9 +57,7 @@ ADD_LIBRARY(JFJochCommon STATIC
         PixelMask.cpp PixelMask.h
 )
 
-TARGET_LINK_LIBRARIES(JFJochCommon JFJochLogger Compression JFCalibration -lrt)
-
-TARGET_LINK_LIBRARIES(JFJochZMQ "$<BUILD_INTERFACE:libzmq-static>")
+TARGET_LINK_LIBRARIES(JFJochCommon Compression JFCalibration "$<BUILD_INTERFACE:libzmq-static>" -lrt)
 
 IF (CMAKE_CUDA_COMPILER)
     TARGET_SOURCES(JFJochCommon PRIVATE CUDAWrapper.cu )

common/DiffractionExperiment.cpp

@@ -9,7 +9,6 @@
 #include "JFJochException.h"
 #include "RawToConvertedGeometry.h"
 #include "../fpga/pcie_driver/jfjoch_fpga.h"
-#include "../include/spdlog/fmt/fmt.h"
 
 using namespace std::literals::chrono_literals;
 
@@ -29,6 +28,8 @@ DiffractionExperiment::DiffractionExperiment(const DetectorSetup& det_setup)
     internal_fpga_packet_generator = false;
     internal_fpga_packet_generator_images = 1;
 
+    debug_pixel_mask = false;
+
     ndatastreams = 1;
 
     frame_time = std::chrono::microseconds(MIN_FRAME_TIME_HALF_SPEED_IN_US);
@@ -596,20 +597,6 @@ std::optional<UnitCell> DiffractionExperiment::GetUnitCell() const {
     return dataset.GetUnitCell();
 }
 
-std::string DiffractionExperiment::GetUnitCellString() const {
-    auto uc = dataset.GetUnitCell();
-    if (uc.has_value()) {
-        return fmt::format("{:.1f}, {:.1f}, {:.1f}, {:.1f}, {:.1f} {:.1f}",
-                           uc.value().a,
-                           uc.value().b,
-                           uc.value().c,
-                           uc.value().alpha,
-                           uc.value().beta,
-                           uc.value().gamma);
-    } else
-        return "-";
-}
-
 Coord DiffractionExperiment::LabCoord(float detector_x, float detector_y) const {
     // Assumes planar detector, 90 deg towards beam
     return {(detector_x - GetBeamX_pxl()) * GetPixelSize_mm() ,
@@ -682,7 +669,6 @@ void DiffractionExperiment::CheckDataProcessingSettings(const SpotFindingSetting
         check_min("Spot finding low resolution limit", settings.low_resolution_limit, 1.0);
         check_max("Spot finding low resolution limit", settings.low_resolution_limit, 50.0);
     }
-    check_min("min spot count for powder ring detection", settings.min_spot_count_powder_ring, 5);
     check_min("indexing tolerance", settings.indexing_tolerance, 0.0);
     check_max("indexing tolerance", settings.indexing_tolerance, 1.0);
 }
@@ -751,8 +737,6 @@ void DiffractionExperiment::FillMessage(StartMessage &message) const {
 
     for (const auto &[x, y]: roi_mask.GetROINameMap())
         message.roi_names.emplace_back(x);
-
-    message.data_reduction_factor_serialmx = GetDataReductionFactorSerialMX();
 }
 
 float DiffractionExperiment::GetPixelSize_mm() const {
@@ -1080,6 +1064,15 @@ const DetectorSetup &DiffractionExperiment::GetDetectorSetup() const {
     return detector;
 }
 
+DiffractionExperiment &DiffractionExperiment::NeuralNetModelPath(const std::string &input) {
+    neural_net_model_path = input;
+    return *this;
+}
+
+std::string DiffractionExperiment::GetNeuralNetModelPath() const {
+    return neural_net_model_path;
+}
+
 DiffractionExperiment &DiffractionExperiment::PulsedSource(bool input) {
     pulsed_source = input;
     return *this;

common/DiffractionExperiment.h

@@ -82,12 +82,15 @@ class DiffractionExperiment {
     std::string instrument_name_short;
     bool pulsed_source;
 
+    bool debug_pixel_mask;
     Coord default_omega_axis;
 
     bool conversion_on_fpga;
 
     uint64_t series_id;
 
+    std::string neural_net_model_path;
+
     bool rad_int_solid_angle_corr;
     bool rad_int_polarization_corr;
     float rad_int_polarization_factor{};
@@ -141,6 +144,7 @@ public:
     DiffractionExperiment& FixedGainG1(bool input);
     DiffractionExperiment& IncrementSeriesID();
     DiffractionExperiment& ConversionOnFPGA(bool input);
+    DiffractionExperiment& NeuralNetModelPath(const std::string& input);
     DiffractionExperiment& PulsedSource(bool input);
 
     DiffractionExperiment& ImagesPerTrigger(int64_t input);
@@ -278,6 +282,7 @@ public:
     bool IsConversionOnFPGA() const;
 
     const DetectorSetup& GetDetectorSetup() const;
+    std::string GetNeuralNetModelPath() const;
 
     bool IsPulsedSource() const;
 
@@ -294,7 +299,6 @@ public:
     std::string GetImageAppendix() const;
     float GetPhotonEnergyMultiplier() const;
     std::optional<UnitCell> GetUnitCell() const;
-    std::string GetUnitCellString() const;
     int64_t GetSpaceGroupNumber() const;
     bool GetSaveCalibration() const;
     int64_t GetSummation() const;

common/NUMAHWPolicy.cpp

@@ -46,6 +46,15 @@ NUMAHWPolicy::NUMAHWPolicy(const std::string &policy) : name(policy) {
         throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Unknown NUMA policy");
 }
 
+NUMAHWPolicy::NUMAHWPolicy(const NUMAHWPolicy &other) : bindings(other.bindings), name(other.name), curr_thread(0) {}
+
+NUMAHWPolicy &NUMAHWPolicy::operator=(const NUMAHWPolicy &other) {
+    bindings = other.bindings;
+    name = other.name;
+    curr_thread = 0;
+    return *this;
+}
+
 NUMABinding NUMAHWPolicy::GetBinding(uint32_t thread) {
     if (bindings.empty())
         return NUMABinding{.cpu_node = -1, .mem_node = -1, .gpu = -1};
@@ -53,6 +62,14 @@ NUMABinding NUMAHWPolicy::GetBinding(uint32_t thread) {
         return bindings.at(thread % bindings.size());
 }
 
+NUMABinding NUMAHWPolicy::GetBinding() {
+    return GetBinding(curr_thread++);
+}
+
+void NUMAHWPolicy::Bind() {
+    Bind(GetBinding());
+}
+
 void NUMAHWPolicy::Bind(uint32_t thread) {
     Bind(GetBinding(thread));
 }

common/NUMAHWPolicy.h

@@ -17,14 +17,19 @@ struct NUMABinding {
 class NUMAHWPolicy {
     std::string name;
     std::vector<NUMABinding> bindings;
+    std::atomic<uint32_t> curr_thread = 0;
 public:
     NUMAHWPolicy() = default;
     explicit NUMAHWPolicy(const std::string& policy);
+    NUMAHWPolicy(const NUMAHWPolicy& other);
+    NUMAHWPolicy& operator=(const NUMAHWPolicy& other);
     NUMABinding GetBinding(uint32_t thread);
+    NUMABinding GetBinding(); // round-robin
 
     const std::string &GetName() const;
 
     void Bind(uint32_t thread);
+    void Bind(); // round-robin
     static void Bind(const NUMABinding &binding);
     static void RunOnNode(int32_t cpu_node);
     static void MemOnNode(int32_t mem_node);

common/PixelMask.cpp

@@ -12,7 +12,7 @@ PixelMask::PixelMask(const DetectorSetup &detector)
 PixelMask::PixelMask(const DiffractionExperiment& experiment)
         : PixelMask(experiment.GetDetectorSetup()) {}
 
-void PixelMask::LoadMask(const std::vector<uint32_t> &input_mask, uint8_t bit) {
+void PixelMask::LoadDetectorBadPixelMask(const std::vector<uint32_t> &input_mask, uint8_t bit) {
     if (input_mask.size() != mask.size())
         throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
                               "Input match doesn't fit the detector ");
@@ -25,11 +25,7 @@ void PixelMask::LoadMask(const std::vector<uint32_t> &input_mask, uint8_t bit) {
     }
 }
 
-void PixelMask::LoadDetectorBadPixelMask(const std::vector<uint32_t> &input_mask) {
-    LoadMask(input_mask, ErrorPixelBit);
-}
-
-std::vector<uint32_t> PixelMask::GetMask(const DiffractionExperiment &experiment, bool conv) const {
+std::vector<uint32_t> PixelMask::GetMask(const DiffractionExperiment &experiment) {
     std::vector<uint32_t> mask_out = mask;
 
     // apply edge mask
@@ -43,7 +39,7 @@ std::vector<uint32_t> PixelMask::GetMask(const DiffractionExperiment &experiment
                         || (line == RAW_MODULE_LINES - 1)
                         || (col == 0)
                         || (col == RAW_MODULE_COLS - 1))
-                        mask_out[pixel] |= (1 << ModuleEdgePixelBit);
+                        mask_out[pixel] |= (1 << 30);
                 }
 
                 if (experiment.GetMaskChipEdges()) {
@@ -51,39 +47,33 @@ std::vector<uint32_t> PixelMask::GetMask(const DiffractionExperiment &experiment
                         || (col == 511) || (col == 512)
                         || (col == 767) || (col == 768)
                         || (line == 255) || (line== 256))
-                        mask_out[pixel] |= (1 << ChipGapPixelBit);
+                        mask_out[pixel] |= (1 << 31);
                 }
             }
         }
     }
 
-    if (conv && (experiment.GetDetectorMode() == DetectorMode::Conversion)) {
-        std::vector<uint32_t> tmp(experiment.GetPixelsNum(), 1<<ModuleGapPixelBit); // nonfunctional areas (i.e. gaps) are filled with 1
+    if (experiment.GetDetectorMode() == DetectorMode::Conversion) {
+        std::vector<uint32_t> tmp(experiment.GetPixelsNum(), 1); // nonfuctional areas (i.e. gaps) are filled with 1
         RawToConvertedGeometry<uint32_t, uint32_t>(experiment, tmp.data(), mask_out.data());
         return tmp;
     } else
         return mask_out;
 }
 
-std::vector<uint32_t> PixelMask::GetUserMask(const DiffractionExperiment &experiment, bool conv) const {
-    std::vector<uint32_t> ret = GetMask(experiment, conv);
-    for (auto &i: ret)
-        i = ((i & (1 << UserMaskedPixelBit)) != 0) ? 1 : 0;
-    return ret;
-}
-
 void PixelMask::LoadUserMask(const DiffractionExperiment &experiment,
-                             const std::vector<uint32_t> &in_mask) {
+                             const std::vector<uint32_t> &in_mask,
+                             uint8_t bit) {
     if (experiment.GetModulesNum() != nmodules)
         throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
                               "Mismatch in module size");
 
     if (in_mask.size() == nmodules * RAW_MODULE_SIZE) {
-        LoadMask(in_mask, UserMaskedPixelBit);
+        LoadDetectorBadPixelMask(in_mask, bit);
     } else if (in_mask.size() == experiment.GetPixelsNum()) {
         std::vector<uint32_t> raw_mask(nmodules * RAW_MODULE_SIZE);
         ConvertedToRawGeometry(experiment, raw_mask.data(), in_mask.data());
-        LoadMask(raw_mask, UserMaskedPixelBit);
+        LoadDetectorBadPixelMask(in_mask, bit);
     } else
         throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
                               "Size of input user mask invalid");

common/PixelMask.h

@@ -9,21 +9,12 @@
 class PixelMask {
     size_t nmodules;
     std::vector<uint32_t> mask;
-    void LoadMask(const std::vector<uint32_t>& mask, uint8_t bit);
 public:
-    // NXmx bits
-    constexpr static const uint8_t ModuleGapPixelBit = 0;
-    constexpr static const uint8_t ErrorPixelBit = 1;
-    constexpr static const uint8_t UserMaskedPixelBit = 8;
-    constexpr static const uint8_t ChipGapPixelBit = 31;
-    constexpr static const uint8_t ModuleEdgePixelBit = 30;
-
     PixelMask(const DetectorSetup& detector);
     PixelMask(const DiffractionExperiment& experiment);
-    void LoadUserMask(const DiffractionExperiment& experiment, const std::vector<uint32_t>& mask);
-    void LoadDetectorBadPixelMask(const std::vector<uint32_t>& mask);
-    std::vector<uint32_t> GetMask(const DiffractionExperiment& experiment, bool conv = true) const;
-    std::vector<uint32_t> GetUserMask(const DiffractionExperiment& experiment, bool conv = true) const;
+    void LoadUserMask(const DiffractionExperiment& experiment, const std::vector<uint32_t>& mask, uint8_t bit);
+    void LoadDetectorBadPixelMask(const std::vector<uint32_t>& mask, uint8_t bit);
+    std::vector<uint32_t> GetMask(const DiffractionExperiment& experiment);
 };
 
 

common/Plot.h

@@ -7,12 +7,9 @@
 #include <cstdint>
 #include <string>
 
-enum class PlotType {
-    BkgEstimate, RadInt, RadIntPerTimePoint, SpotCount, SpotCountInRings,
-    IndexingRate, IndexingRatePerTimePoint,
-    ErrorPixels, ImageCollectionEfficiency, ReceiverDelay, ReceiverFreeSendBuf, StrongPixels,
-    ROISum, ROIMaxCount, ROIPixels
-};
+enum class PlotType {BkgEstimate, RadInt, RadIntPerTimePoint, SpotCount, IndexingRate, IndexingRatePerTimePoint,
+                     ErrorPixels, ImageCollectionEfficiency, ReceiverDelay, ReceiverFreeSendBuf, StrongPixels,
+                     ROISum, ROIMaxCount, ROIPixels, ResEstimation};
 
 struct PlotRequest {
     PlotType type;

common/StatusVector.h

@@ -7,7 +7,6 @@
 #include <map>
 #include <mutex>
 #include <cmath>
-#include <optional>
 
 #include "Plot.h"
 
@@ -38,11 +37,6 @@ template <class T> class StatusVector {
         return ret;
     }
 public:
-    void AddElement(uint32_t id, std::optional<T> val) {
-        if (val.has_value())
-            AddElement(id, val.value());
-    }
-
     void AddElement(uint32_t id, T val) {
         std::unique_lock<std::mutex> ul(m);
         if (id >= content.size()) {

common/ZMQWrappers.cpp

@@ -8,7 +8,8 @@ ZMQContext::ZMQContext() {
 
     // Default is to have 2 I/O threads per ZMQ context
     if (zmq_ctx_set(context, ZMQ_IO_THREADS, 2) != 0)
-        throw JFJochException(JFJochExceptionCategory::ZeroMQ, "Cannot set number of I/O threads");
+        throw JFJochException(JFJochExceptionCategory::ZeroMQ,
+                              "Cannot set number of I/O threads");
 }
 
 ZMQContext &ZMQContext::NumThreads(int32_t threads) {
@@ -26,7 +27,7 @@ void *ZMQContext::GetContext() const {
     return context;
 }
 
-ZMQSocket::ZMQSocket(ZMQSocketType in_socket_type) : socket_type(in_socket_type) {
+ZMQSocket::ZMQSocket(ZMQContext &context, ZMQSocketType in_socket_type) : socket_type(in_socket_type) {
     socket = zmq_socket(context.GetContext(), static_cast<int>(socket_type));
 
     if (socket == nullptr)

common/ZMQWrappers.h

@@ -42,15 +42,14 @@ public:
 
 class ZMQSocket {
     std::mutex m;
-    ZMQContext context;
     ZMQSocketType socket_type;
     void *socket;
     void SetSocketOption(int32_t option_name, int32_t value);
 public:
     ZMQSocket(ZMQSocket &socket) = delete;
     const ZMQSocket& operator=(ZMQSocket &socket) = delete;
-    explicit ZMQSocket(ZMQSocketType socket_type);
-     ~ZMQSocket();
+    ZMQSocket(ZMQContext &context, ZMQSocketType socket_type);
+    ~ZMQSocket();
     void Connect(const std::string& addr);
     void Disconnect(const std::string& addr);
     void Bind(const std::string& addr);

detector_control/CMakeLists.txt

@@ -1,2 +1,6 @@
+INSTALL(TARGETS sls_detector_put sls_detector_get RUNTIME)
+
 ADD_LIBRARY(JFJochDetector STATIC DetectorWrapper.cpp DetectorWrapper.h)
-TARGET_LINK_LIBRARIES(JFJochDetector JFJochCommon slsDetectorStatic slsSupportStatic)
+TARGET_LINK_LIBRARIES(JFJochDetector JFJochCommon slsSupportShared slsDetectorShared)
+
+

fpga/hls/spot_finder.cpp

@@ -13,18 +13,13 @@
 struct spot_finder_packet {
     ap_uint<768> data;
     ap_uint<32> mask;
-    ap_uint<32> strong_pixel;
-    ap_int<32> count_threshold;
-    ap_uint<32> snr_threshold;
     ap_uint<1> user;
     ap_uint<1> last;
 };
 
 void spot_finder_in_stream(STREAM_768 &data_in,
                            hls::stream<ap_uint<32>> &mask_in,
-                           hls::stream<spot_finder_packet> &data_out,
-                           volatile ap_int<32> &in_count_threshold,
-                           volatile ap_uint<32> &in_snr_threshold) {
+                           hls::stream<spot_finder_packet> &data_out) {
     ap_uint<32> mask;
     packet_768_t packet_in;
     {
@@ -38,53 +33,12 @@ void spot_finder_in_stream(STREAM_768 &data_in,
     while (!packet_in.user) {
 #pragma HLS PIPELINE II=1
         mask_in >> mask;
-        data_out << spot_finder_packet{
-            .data = packet_in.data,
-                .mask = mask,
-                .strong_pixel = 0,
-                .count_threshold = in_count_threshold,
-                .snr_threshold = in_snr_threshold,
-                .user = packet_in.user,
-                .last = packet_in.last
-        };
+        data_out << spot_finder_packet{.data = packet_in.data, .mask = mask, .user = packet_in.user, .last = packet_in.last};
         data_in >> packet_in;
     }
     data_out << spot_finder_packet{.data = packet_in.data, .mask = 0, .user = packet_in.user, .last = packet_in.last};
 }
 
-void spot_finder_out_stream(hls::stream<spot_finder_packet> &data_in,
-                            STREAM_768 &data_out,
-                            hls::stream<ap_axiu<32,1,1,1>> &strong_pixel_out) {
-    spot_finder_packet packet_in;
-    {
-#pragma HLS PROTOCOL fixed
-        data_in >> packet_in;
-        ap_wait();
-        data_out << packet_768_t{.data = packet_in.data, .user = packet_in.user, .last = packet_in.last};
-        ap_wait();
-    }
-
-    data_in >> packet_in;
-    while (!packet_in.user) {
-        ap_uint<32> count_threshold = packet_in.count_threshold;
-        ap_uint<32> snr_threshold_u32 = packet_in.snr_threshold;
-        for (int i = 0; i < RAW_MODULE_SIZE * sizeof(uint16_t) / 64; i++) {
-#pragma HLS PIPELINE II=1
-            data_out << packet_768_t{.data = packet_in.data, .user = packet_in.user, .last = packet_in.last};
-            strong_pixel_out << ap_axiu<32,1,1,1>{.data = packet_in.strong_pixel, .user = 0};
-            data_in >> packet_in;
-        }
-
-        // Save module statistics
-        strong_pixel_out << ap_axiu<32,1,1,1>{.data = count_threshold, .user = 0};
-        strong_pixel_out << ap_axiu<32,1,1,1>{.data = snr_threshold_u32, .user = 0};
-        for (int i = 0; i < 14;i++)
-            strong_pixel_out << ap_axiu<32,1,1,1>{.data =  0, .user = 0};
-    }
-    strong_pixel_out << ap_axiu<32,1,1,1>{.data = 0, .user = 1};
-    data_out << packet_768_t{.data = packet_in.data, .user = packet_in.user, .last = packet_in.last};
-}
-
 ap_int<SUM_BITWIDTH> calc_sum(ap_int<24> val[32], ap_uint<32> mask) {
 #pragma HLS PIPELINE II=1
     ap_int<SUM_BITWIDTH> ret = 0;
@@ -119,8 +73,6 @@ void spot_finder_prepare(hls::stream<spot_finder_packet> &data_in,
                          hls::stream<ap_int<SUM_BITWIDTH>> &sum_out,
                          hls::stream<ap_int<SUM2_BITWIDTH>> &sum2_out,
                          hls::stream<ap_int<VALID_BITWIDTH>> &valid_out) {
-    ap_uint<32> strong_pixel;
-
     spot_finder_packet packet;
     {
 #pragma HLS PROTOCOL fixed
@@ -143,7 +95,6 @@ void spot_finder_prepare(hls::stream<spot_finder_packet> &data_in,
         for (ap_uint<15> i = 0; i < RAW_MODULE_SIZE * sizeof(uint16_t) / 64; i++) {
 #pragma HLS PIPELINE II=1
             data_out << packet;
-
             ap_int<24> val[32];
             unpack32(packet.data, val);
             ap_uint<32> mask = packet.mask;
@@ -216,28 +167,31 @@ ap_uint<32> spot_finder_count_threshold(ap_int<24> val[32], ap_int<32> &count_th
 }
 
 void spot_finder_apply_threshold(hls::stream<spot_finder_packet> &data_in,
-                                 hls::stream<spot_finder_packet> &data_out,
+                                 STREAM_768 &data_out,
                                  hls::stream<ap_int<SUM_BITWIDTH>> &sum_in,
                                  hls::stream<ap_int<SUM2_BITWIDTH>> &sum2_in,
-                                 hls::stream<ap_int<VALID_BITWIDTH>> &valid_in) {
-    ap_uint<32> strong_pixel_prev;
+                                 hls::stream<ap_int<VALID_BITWIDTH>> &valid_in,
+                                 hls::stream<ap_axiu<32,1,1,1>> &strong_pixel_out,
+                                 volatile ap_int<32> &in_count_threshold,
+                                 volatile ap_uint<32> &in_snr_threshold) {
     spot_finder_packet packet_in;
     {
 #pragma HLS PROTOCOL fixed
         data_in >> packet_in;
         ap_wait();
-        data_out << packet_in;
+        data_out << packet_768_t{.data = packet_in.data, .user = packet_in.user, .last = packet_in.last};
         ap_wait();
     }
 
+
     ap_int<SUM_BITWIDTH> sum[32];
     ap_int<SUM2_BITWIDTH> sum2[32];
     ap_int<VALID_BITWIDTH> valid[32];
 
     data_in >> packet_in;
     while (!packet_in.user) {
-        ap_int<32> count_threshold = packet_in.count_threshold;
-        ap_uint<32> snr_threshold_u32 = packet_in.snr_threshold;
+        ap_int<32> count_threshold = in_count_threshold;
+        ap_uint<32> snr_threshold_u32 = in_snr_threshold;
         float_uint32 thr;
         thr.u = snr_threshold_u32;
         ap_ufixed<10,8, AP_RND_CONV> snr_threshold = thr.f;
@@ -258,28 +212,34 @@ void spot_finder_apply_threshold(hls::stream<spot_finder_packet> &data_in,
                 valid_in >> valid[i % 32];
             }
 
+            data_out << packet_768_t{.data = packet_in.data, .user = packet_in.user, .last = packet_in.last};
             ap_int<24> data_unpacked[32];
             unpack32(packet_in.data, data_unpacked);
 
             ap_uint<32> strong_pixel = spot_finder_count_threshold(data_unpacked, count_threshold) &
-                                       spot_finder_snr_threshold(data_unpacked, snr_threshold_2,
-                                                                 sum[i % 32], sum2[i % 32], valid[i % 32]);
+                    spot_finder_snr_threshold(data_unpacked, snr_threshold_2,
+                                              sum[i % 32], sum2[i % 32], valid[i % 32]);
+
+            strong_pixel = strong_pixel & packet_in.mask;
 
             if ((snr_threshold == 0) && (count_threshold <= 0))
                 strong_pixel = 0;
 
-            strong_pixel = strong_pixel & packet_in.mask;
-
-            packet_in.mask |= strong_pixel; // mask strong pixels
-            packet_in.strong_pixel |= strong_pixel; // add strong pixels to the output
-
-            data_out << packet_in;
+            strong_pixel_out << ap_axiu<32,1,1,1>{.data = strong_pixel, .user = 0};
             data_in >> packet_in;
         }
+
+        // Save module statistics
+       strong_pixel_out << ap_axiu<32,1,1,1>{.data = count_threshold, .user = 0};
+       strong_pixel_out << ap_axiu<32,1,1,1>{.data = snr_threshold_u32, .user = 0};
+       	  for (int i = 0; i < 14;i++)
+              strong_pixel_out << ap_axiu<32,1,1,1>{.data =  0, .user = 0};
     }
-    data_out << packet_in;
+    strong_pixel_out << ap_axiu<32,1,1,1>{.data = 0, .user = 1};
+    data_out << packet_768_t{.data = packet_in.data, .user = packet_in.user, .last = packet_in.last};
 }
 
+
 void spot_finder(STREAM_768 &data_in,
                  hls::stream<ap_uint<32>> &mask_in,
                  STREAM_768 &data_out,
@@ -296,38 +256,22 @@ void spot_finder(STREAM_768 &data_in,
 
     hls::stream<spot_finder_packet, 2> data_0;
     hls::stream<spot_finder_packet, 1080> data_1;
-    hls::stream<spot_finder_packet, 8> data_2;
-    hls::stream<spot_finder_packet, 1080> data_3;
-    hls::stream<spot_finder_packet, 8> data_4;
-
 #pragma HLS BIND_STORAGE variable=data_1 type=fifo impl=bram
-#pragma HLS BIND_STORAGE variable=data_3 type=fifo impl=bram
-
     hls::stream<ap_int<SUM_BITWIDTH>, 24> sum_0;
     hls::stream<ap_int<SUM2_BITWIDTH>, 24> sum2_0;
     hls::stream<ap_int<VALID_BITWIDTH>, 24> valid_0;
-
-    hls::stream<ap_int<SUM_BITWIDTH>, 24> sum_1;
-    hls::stream<ap_int<SUM2_BITWIDTH>, 24> sum2_1;
-    hls::stream<ap_int<VALID_BITWIDTH>, 24> valid_1;
-
 #ifndef JFJOCH_HLS_NOSYNTH
-    spot_finder_in_stream(data_in, mask_in, data_0, in_count_threshold, in_snr_threshold);
+    spot_finder_in_stream(data_in, mask_in, data_0);
     spot_finder_prepare(data_0, data_1, sum_0, sum2_0, valid_0);
-    spot_finder_apply_threshold(data_1, data_2, sum_0, sum2_0, valid_0);
-    spot_finder_prepare(data_2, data_3, sum_1, sum2_1, valid_1);
-    spot_finder_apply_threshold(data_3, data_4, sum_1, sum2_1, valid_1);
-    spot_finder_out_stream(data_4, data_out, strong_pixel_out);
+    spot_finder_apply_threshold(data_1, data_out, sum_0, sum2_0, valid_0, strong_pixel_out,
+                                in_count_threshold, in_snr_threshold);
 #else
     std::vector<std::thread> spot_finder_cores;
-    spot_finder_cores.emplace_back([&] {spot_finder_in_stream(data_in, mask_in, data_0, in_count_threshold,
-                                                              in_snr_threshold);});
+    spot_finder_cores.emplace_back([&] {spot_finder_in_stream(data_in, mask_in, data_0);});
     spot_finder_cores.emplace_back([&] {spot_finder_prepare(data_0, data_1, sum_0, sum2_0, valid_0);});
-    spot_finder_cores.emplace_back([&] {spot_finder_apply_threshold(data_1, data_2, sum_0, sum2_0, valid_0);});
-    spot_finder_cores.emplace_back([&] {spot_finder_prepare(data_2, data_3, sum_1, sum2_1, valid_1);});
-    spot_finder_cores.emplace_back([&] {spot_finder_apply_threshold(data_3, data_4, sum_1, sum2_1, valid_1);});
-    spot_finder_cores.emplace_back([&] {spot_finder_out_stream(data_4, data_out, strong_pixel_out);});
-
+    spot_finder_cores.emplace_back([&] {spot_finder_apply_threshold(data_1, data_out, sum_0, sum2_0, valid_0,
+                                                                    strong_pixel_out, in_count_threshold,
+                                                                    in_snr_threshold);});
     for (auto &i : spot_finder_cores)
         i.join();
 #endif

fpga/hls/spot_finder_mask.cpp

@@ -74,8 +74,8 @@ void spot_finder_mask(STREAM_768 &data_in,
                     || (((col == 0) || (col == 8) || (col == 16) || (col == 24)) && (j ==  0))
                     || (pixel_val[j] == INT24_MIN)
                     || (pixel_val[j] == INT24_MAX)
-                    || (d[j] < min_d_value)
-                    || (d[j] > max_d_value))
+                    || ((d[j] != 0) && (d[j] < min_d_value))
+                    || ((d[j] != 0) && (d[j] > max_d_value)))
                     mask_val[j] = 0;
                 else
                     mask_val[j] = 1;

fpga/host_library/CMakeLists.txt

@@ -4,7 +4,7 @@ TARGET_LINK_LIBRARIES(JFJochDevice JFJochCommon)
 
 ADD_EXECUTABLE(jfjoch_pcie_status jfjoch_pcie_status.cpp)
 TARGET_LINK_LIBRARIES(jfjoch_pcie_status JFJochDevice )
-INSTALL(TARGETS jfjoch_pcie_status RUNTIME COMPONENT jfjoch)
+INSTALL(TARGETS jfjoch_pcie_status RUNTIME)
 
 ADD_EXECUTABLE(jfjoch_pcie_set_network jfjoch_pcie_set_network.cpp)
 TARGET_LINK_LIBRARIES(jfjoch_pcie_set_network JFJochDevice )

fpga/pcie_driver/dkms.conf

@@ -1,5 +1,5 @@
 PACKAGE_NAME=jfjoch
-PACKAGE_VERSION=1.0.0
+PACKAGE_VERSION=0.1
 
 DEST_MODULE_LOCATION=/extra
 BUILT_MODULE_NAME=jfjoch
@@ -7,4 +7,4 @@ BUILT_MODULE_LOCATION=src/
 
 MAKE="'make' -C src/ all"
 CLEAN="'make' -C src/ clean"
-AUTOINSTALL="yes"
+AUTOINSTALL="yes"

fpga/pcie_driver/install_dkms.sh

@@ -1,11 +1,12 @@
 #!/bin/bash
 
 # Copyright (2019-2023) Paul Scherrer Institute
-VERSION=1.0.0
+VERSION=0.1
 
 mkdir -p /usr/src/jfjoch-${VERSION}/src
 cp dkms.conf /usr/src/jfjoch-${VERSION}
-cp *.c *.h Makefile /usr/src/jfjoch-${VERSION}/src
+cp *.c *.h Makefile ../../common/Definitions.h /usr/src/jfjoch-0.1/src
+sed -i "s,../../common/Definitions.h,Definitions.h," /usr/src/jfjoch-0.1/src/jfjoch_drv.h
 
 dkms add -m jfjoch -v ${VERSION}
 dkms install -m jfjoch -v ${VERSION}

fpga/pcie_driver/jfjoch_drv.c

@@ -11,7 +11,7 @@
 MODULE_AUTHOR("Filip Leonarski; Paul Scherrer Institute");
 MODULE_DESCRIPTION("Jungfraujoch device module");
 MODULE_LICENSE("GPL");
-MODULE_VERSION("1.0.0");
+MODULE_VERSION("0.1");
 
 #define XDMA_GEN4_x8        (0x9048)
 #define XDMA_GEN3_x16       (0x903F)

fpga/pcie_driver/jfjoch_pcie_setup.c

@@ -54,13 +54,12 @@ int jfjoch_check_version(struct pci_dev *pdev) {
     release_level = ioread32(drvdata->bar0 + ACTION_CONFIG_OFFSET + ADDR_RELEASE_LEVEL);
 
     if (action_type != JFJOCH_FPGA_MAGIC) {
-        dev_err(dev, "Very likely the FPGA is not flashed with Jungfraujoch design, given that action type register value is wrong (%x).\n", action_type);
+        dev_err(dev, "Mismatch in JFJoch action type (%x)\n", action_type);
         return -EINVAL;
     }
 
     if (release_level != JFJOCH_FPGA_RELEASE) {
-        dev_err(dev, "Jungfraujoch FPGA is flashed with design made with release %x, while driver is compiled with release %x.\n", release_level, JFJOCH_FPGA_RELEASE);
-        dev_err(dev, "Given FPGA releases have braking changes in FPGA-driver interface (register map, data structure size, etc.)\n it is not safe and not supported at the moment to operate multiple releases with one driver.\n");
+        dev_err(dev, "Mismatch in JFJoch release level (%x)\n", release_level);
         return -EINVAL;
     }
     return 0;

fpga/pcie_driver/postinstall.sh

@@ -1,14 +0,0 @@
-#!/bin/bash
-# Taken from https://schneide.blog/2015/08/10/packaging-kernel-modulesdrivers-using-dkms/
-
-VERSION="1.0.0"
-
-occurrences=`/usr/sbin/dkms status | grep jfjoch | grep ${VERSION} | wc -l`
-
-if [ ! occurrences > 0 ]; then
-    /usr/sbin/dkms add -m jfjoch -v ${VERSION}
-fi
-/usr/sbin/dkms build -m jfjoch -v ${VERSION}
-/usr/sbin/dkms install -m jfjoch -v ${VERSION}
-
-exit 0

fpga/pcie_driver/preuninstall.sh

@@ -1,8 +0,0 @@
-#!/bin/bash
-# Taken from https://schneide.blog/2015/08/10/packaging-kernel-modulesdrivers-using-dkms/
-
-VERSION="1.0.0"
-
-/usr/sbin/dkms remove -m jfjoch -v ${VERSION} --all
-
-exit 0

frame_serialize/CBORStream2Deserializer.cpp

@@ -456,8 +456,6 @@ namespace {
                     message.user_data = GetCBORString(value);
                 else if (key == "spots")
                     GetCBORSpots(message, value);
-                else if (key == "spot_count_in_rings")
-                    message.spot_count_in_rings = GetCBORUInt(value);
                 else if (key == "az_int_profile")
                     GetCBORFloatArray(value, message.az_int_profile);
                 else if (key == "indexing_result")
@@ -638,8 +636,6 @@ namespace {
                 message.roi_names = j["roi_names"];
             if (j.contains("write_master_file"))
                 message.write_master_file = j["write_master_file"];
-            if (j.contains("data_reduction_factor_serialmx"))
-                message.data_reduction_factor_serialmx = j["data_reduction_factor_serialmx"];
         } catch (const std::exception &e) {
             throw JFJochException(JFJochExceptionCategory::CBORError,
                                   "Cannot parse user_data as valid JSON " + std::string(e.what()));

frame_serialize/CBORStream2Serializer.cpp

@@ -316,8 +316,6 @@ inline void CBOR_ENC_START_USER_DATA(CborEncoder& encoder, const char* key,
     j["gain_file_names"] = message.gain_file_names;
     if (message.write_master_file)
         j["write_master_file"] = message.write_master_file.value();
-    if (message.data_reduction_factor_serialmx)
-        j["data_reduction_factor_serialmx"] = message.data_reduction_factor_serialmx.value();
 
     auto str = j.dump();
 
@@ -450,8 +448,6 @@ void CBORStream2Serializer::SerializeImage(const DataMessage& message) {
     CBOR_ENC_RATIONAL(mapEncoder, "end_time", message.timestamp + message.exptime, message.timestamp_base);
 
     CBOR_ENC(mapEncoder, "spots", message.spots);
-    CBOR_ENC(mapEncoder, "spot_count_in_rings", message.spot_count_in_rings);
-
     CBOR_ENC(mapEncoder, "az_int_profile", message.az_int_profile);
     CBOR_ENC(mapEncoder, "indexing_result", message.indexing_result);
     if (!message.indexing_lattice.empty())

frame_serialize/CMakeLists.txt

@@ -20,8 +20,10 @@ TARGET_LINK_LIBRARIES(CBORStream2FrameSerialize tinycbor)
 ADD_LIBRARY(ImagePusher STATIC
         ImagePusher.cpp ImagePusher.h
         TestImagePusher.cpp TestImagePusher.h
-        ZMQStream2Pusher.cpp ZMQStream2Pusher.h
+        ZMQStream2PusherGroup.cpp ZMQStream2PusherGroup.h
+        ZMQStream2Pusher.cpp
+        ZMQStream2Pusher.h
         DumpCBORToFilePusher.cpp
         DumpCBORToFilePusher.h)
 
-TARGET_LINK_LIBRARIES(ImagePusher JFJochZMQ CBORStream2FrameSerialize JFJochCommon Compression)
+TARGET_LINK_LIBRARIES(ImagePusher CBORStream2FrameSerialize JFJochCommon Compression)

frame_serialize/JFJochMessages.h

@@ -45,7 +45,6 @@ struct DataMessage {
     float image_collection_efficiency;
 
     std::vector<SpotToSave> spots;
-    std::optional<uint64_t> spot_count_in_rings = 0;
 
     std::vector<float> az_int_profile;
     float bkg_estimate;
@@ -76,6 +75,8 @@ struct DataMessage {
     std::optional<uint64_t> receiver_free_send_buf;
     std::optional<uint64_t> storage_cell;
 
+    std::optional<float> resolution_estimation;
+
     std::optional<uint64_t> xfel_pulse_id;
     std::optional<uint64_t> xfel_event_code;
 
@@ -169,8 +170,6 @@ struct StartMessage {
     }
 
     std::string user_data;
-
-    std::optional<float> data_reduction_factor_serialmx;
 };
 
 struct EndMessage {

frame_serialize/ZMQStream2Pusher.cpp

@@ -1,117 +1,66 @@
 // Copyright (2019-2024) Paul Scherrer Institute
 
 #include "ZMQStream2Pusher.h"
-#include "CBORStream2Serializer.h"
 
-ZMQStream2Pusher::ZMQStream2Pusher(const std::vector<std::string> &addr,
-                                             int32_t send_buffer_high_watermark, int32_t send_buffer_size)
-        : serialization_buffer(256*1024*1024),
-          serializer(serialization_buffer.data(), serialization_buffer.size()),
-          preview_counter(std::chrono::seconds(1)) {
-    if (addr.empty())
-        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "No writer ZMQ address provided");
+ZMQStream2Pusher::ZMQStream2Pusher(ZMQContext &context, const std::string &addr, int32_t send_buffer_high_watermark,
+                                   int32_t send_buffer_size)
+        : socket(context, ZMQSocketType::Push) {
+    Bind(addr, send_buffer_high_watermark, send_buffer_size);
+}
 
-    for (const auto &a : addr) {
-        auto s = std::make_unique<ZMQSocket>(ZMQSocketType::Push);
-        if (send_buffer_size > 0)
-            s->SendBufferSize(send_buffer_size);
-        if (send_buffer_high_watermark > 0)
-            s->SendWaterMark(send_buffer_high_watermark);
-        s->SendTimeout(std::chrono::seconds(5)); // 5 seconds should be more than enough to flush buffers and to still give fast response
-        s->Bind(a);
-        socket.emplace_back(std::move(s));
-    }
+ZMQStream2Pusher::ZMQStream2Pusher(const std::string &addr, int32_t send_buffer_high_watermark,
+                                   int32_t send_buffer_size)
+        : context(std::make_unique<ZMQContext>()),
+          socket(*context, ZMQSocketType::Push) {
+    Bind(addr, send_buffer_high_watermark, send_buffer_size);
+}
+
+void ZMQStream2Pusher::Bind(const std::string &addr, int32_t send_buffer_high_watermark, int32_t send_buffer_size) {
+
+    if (send_buffer_size > 0)
+        socket.SendBufferSize(send_buffer_size);
+    if (send_buffer_high_watermark > 0)
+        socket.SendWaterMark(send_buffer_high_watermark);
+    socket.SendTimeout(std::chrono::seconds(5)); // 5 seconds should be more than enough to flush buffers and to still give fast response
+    socket.Bind(addr);
+}
+
+void ZMQStream2Pusher::StartDataCollection(StartMessage &message) {
+    size_t approx_size = 1024*1024;
+    for (const auto &x : message.pixel_mask)
+        approx_size += x.size;
+
+    std::vector<uint8_t> serialization_buffer(approx_size);
+    CBORStream2Serializer serializer(serialization_buffer.data(), serialization_buffer.size());
+    serializer.SerializeSequenceStart(message);
+    if (!socket.Send(serialization_buffer.data(), serializer.GetBufferSize(), true))
+        throw JFJochException(JFJochExceptionCategory::ZeroMQ, "Timeout on pushing start message on addr " + GetAddress());
 }
 
 bool ZMQStream2Pusher::SendImage(const uint8_t *image_data, size_t image_size, int64_t image_number) {
-    if (preview_socket) {
-        if (preview_counter.GeneratePreview())
-            preview_socket->Send(image_data, image_size, false);
-    }
-
-    if (!socket.empty()) {
-        auto socket_number = (image_number / images_per_file) % socket.size();
-        return socket[socket_number]->Send(image_data, image_size, false);
-    } else
-        return false;
+    return socket.Send(image_data, image_size, false);
 }
 
 void ZMQStream2Pusher::SendImage(const uint8_t *image_data, size_t image_size, int64_t image_number,
-                                      ZeroCopyReturnValue *z) {
-    if (preview_socket) {
-        if (preview_counter.GeneratePreview())
-            preview_socket->Send(image_data, image_size, false);
-    }
-
-    if (!socket.empty()) {
-        auto socket_number = (image_number / images_per_file) % socket.size();
-        socket[socket_number]->SendZeroCopy(image_data, image_size, z);
-    } else
-        z->release();
+                                 ZeroCopyReturnValue *z) {
+    socket.SendZeroCopy(image_data,image_size, z);
 }
 
-void ZMQStream2Pusher::StartDataCollection(StartMessage& message) {
-      if (message.images_per_file < 1)
-        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
-                              "Images per file cannot be zero or negative");
-    images_per_file = message.images_per_file;
+bool ZMQStream2Pusher::EndDataCollection(const EndMessage &message) {
+    std::vector<uint8_t> serialization_buffer(80 * 1024 * 1024);
+    CBORStream2Serializer serializer(serialization_buffer.data(), serialization_buffer.size());
 
-    serializer.SerializeSequenceStart(message);
-
-    for (auto &s: socket) {
-        if (!s->Send(serialization_buffer.data(), serializer.GetBufferSize(), true))
-            throw JFJochException(JFJochExceptionCategory::ZeroMQ, "Timeout on pushing start message on addr "
-                                                                   + s->GetEndpointName());
-        if (message.write_master_file) {
-            message.write_master_file = false;
-            serializer.SerializeSequenceStart(message);
-        }
-    }
-
-    if (preview_socket)
-        preview_socket->Send(serialization_buffer.data(), serializer.GetBufferSize(), true);
+    serializer.SerializeSequenceEnd(message);
+    return socket.Send(serialization_buffer.data(), serializer.GetBufferSize(), true); // Blocking
 }
 
 bool ZMQStream2Pusher::SendCalibration(const CompressedImage &message) {
-    if (socket.empty())
-        return false;
-
+    std::vector<uint8_t> serialization_buffer(80 * 1024 * 1024);
+    CBORStream2Serializer serializer(serialization_buffer.data(), serialization_buffer.size());
     serializer.SerializeCalibration(message);
-
-    return socket[0]->Send(serialization_buffer.data(), serializer.GetBufferSize(), true);
+    return socket.Send(serialization_buffer.data(), serializer.GetBufferSize(), true); // Blocking
 }
 
-bool ZMQStream2Pusher::EndDataCollection(const EndMessage& message) {
-    serializer.SerializeSequenceEnd(message);
-
-    bool ret = true;
-    for (auto &s: socket) {
-        if (!s->Send(serialization_buffer.data(), serializer.GetBufferSize(), true))
-            ret = false;
-    }
-
-    if (preview_socket)
-        preview_socket->Send(serialization_buffer.data(), serializer.GetBufferSize(), true);
-
-    return ret;
-}
-
-std::vector<std::string> ZMQStream2Pusher::GetAddress() {
-    std::vector<std::string> ret;
-    for (auto &p: socket)
-        ret.push_back(p->GetEndpointName());
-    return ret;
-}
-
-ZMQStream2Pusher &ZMQStream2Pusher::PreviewSocket(const std::string &addr) {
-    preview_socket = std::make_unique<ZMQSocket>(ZMQSocketType::Pub);
-    preview_socket->Bind(addr);
-    return *this;
-}
-
-std::string ZMQStream2Pusher::GetPreviewAddress() {
-    if (preview_socket)
-        return preview_socket->GetEndpointName();
-    else
-        return "";
+std::string ZMQStream2Pusher::GetAddress() {
+    return socket.GetEndpointName();
 }

frame_serialize/ZMQStream2Pusher.h

@@ -3,40 +3,30 @@
 #ifndef JUNGFRAUJOCH_ZMQSTREAM2PUSHER_H
 #define JUNGFRAUJOCH_ZMQSTREAM2PUSHER_H
 
-#include <mutex>
-
 #include "ImagePusher.h"
 #include "../common/ZMQWrappers.h"
-#include "../preview/PreviewCounter.h"
 
 class ZMQStream2Pusher : public ImagePusher {
-    std::vector<uint8_t> serialization_buffer;
-    CBORStream2Serializer serializer;
-
-    std::vector<std::unique_ptr<ZMQSocket>> socket;
-
-    std::unique_ptr<ZMQSocket> preview_socket;
-    PreviewCounter preview_counter;
-
-    int64_t images_per_file = 1;
+    std::unique_ptr<ZMQContext> context;
+    ZMQSocket socket;
 public:
-    explicit ZMQStream2Pusher(const std::vector<std::string>& addr,
+    ZMQStream2Pusher(ZMQContext& context,
+                     const std::string& addr,
+                     int32_t send_buffer_high_watermark = -1,
+                     int32_t send_buffer_size = -1);
+
+    explicit ZMQStream2Pusher(const std::string& addr,
                               int32_t send_buffer_high_watermark = -1,
                               int32_t send_buffer_size = -1);
-
-    ZMQStream2Pusher& PreviewSocket(const std::string& addr);
-    std::string GetPreviewAddress();
-
-    std::vector<std::string> GetAddress();
-
-    // Strictly serial, as order of these is important
+    void Bind(const std::string& addr, int32_t send_buffer_high_watermark, int32_t send_buffer_size);
     void StartDataCollection(StartMessage& message) override;
-    bool EndDataCollection(const EndMessage& message) override;
+    bool SendImage(const uint8_t *image_data, size_t image_size, int64_t image_number) override;
+    void SendImage(const uint8_t *image_data, size_t image_size, int64_t image_number, ZeroCopyReturnValue *z) override;
+    bool EndDataCollection(const EndMessage &message) override;
     bool SendCalibration(const CompressedImage& message) override;
 
-    // Thread-safe
-    void SendImage(const uint8_t *image_data, size_t image_size, int64_t image_number, ZeroCopyReturnValue *z) override;
-    bool SendImage(const uint8_t *image_data, size_t image_size, int64_t image_number) override;
+    std::string GetAddress();
 };
 
+
 #endif //JUNGFRAUJOCH_ZMQSTREAM2PUSHER_H

frame_serialize/ZMQStream2PusherGroup.cpp

@@ -0,0 +1,75 @@
+// Copyright (2019-2024) Paul Scherrer Institute
+
+#include "ZMQStream2PusherGroup.h"
+#include "CBORStream2Serializer.h"
+
+ZMQStream2PusherGroup::ZMQStream2PusherGroup(ZMQContext &zmq_context, const std::vector<std::string> &addr,
+                                             int32_t send_buffer_high_watermark, int32_t send_buffer_size) {
+    if (addr.empty())
+        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
+                              "No writer ZMQ address provided");
+
+    for (const auto &a : addr)
+        pusher.emplace_back(std::make_unique<ZMQStream2Pusher>
+                                    (zmq_context, a, send_buffer_high_watermark, send_buffer_size));
+}
+
+ZMQStream2PusherGroup::ZMQStream2PusherGroup(const std::vector<std::string> &addr,
+                                             int32_t send_buffer_high_watermark, int32_t send_buffer_size) {
+    if (addr.empty())
+        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
+                              "No writer ZMQ address provided");
+    for (const auto &a : addr)
+        pusher.emplace_back(std::make_unique<ZMQStream2Pusher>
+                                    (a, send_buffer_high_watermark, send_buffer_size));
+}
+
+bool ZMQStream2PusherGroup::SendImage(const uint8_t *image_data, size_t image_size, int64_t image_number) {
+    if (!pusher.empty()) {
+        auto socket_number = (image_number / images_per_file) % pusher.size();
+        return pusher[socket_number]->SendImage(image_data, image_size, image_number);
+    } else
+        return false;
+}
+
+void ZMQStream2PusherGroup::SendImage(const uint8_t *image_data, size_t image_size, int64_t image_number,
+                                      ZeroCopyReturnValue *z) {
+    if (!pusher.empty()) {
+        auto socket_number = (image_number / images_per_file) % pusher.size();
+        pusher[socket_number]->SendImage(image_data, image_size, image_number, z);
+    }
+}
+
+void ZMQStream2PusherGroup::StartDataCollection(StartMessage& message) {
+    if (message.images_per_file < 1)
+        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
+                              "Images per file cannot be zero or negative");
+    images_per_file = message.images_per_file;
+
+    for (auto &p: pusher) {
+        p->StartDataCollection(message);
+        message.write_master_file = false;
+    }
+}
+
+bool ZMQStream2PusherGroup::SendCalibration(const CompressedImage &message) {
+    if (pusher.empty())
+        return false;
+    return pusher[0]->SendCalibration(message);
+}
+
+bool ZMQStream2PusherGroup::EndDataCollection(const EndMessage& message) {
+    bool ret = true;
+    for (auto &p: pusher) {
+        if (!p->EndDataCollection(message))
+            ret = false;
+    }
+    return ret;
+}
+
+std::vector<std::string> ZMQStream2PusherGroup::GetAddress() {
+    std::vector<std::string> ret;
+    for (auto &p: pusher)
+        ret.push_back(p->GetAddress());
+    return ret;
+}

frame_serialize/ZMQStream2PusherGroup.h

@@ -0,0 +1,29 @@
+// Copyright (2019-2024) Paul Scherrer Institute
+
+#ifndef JUNGFRAUJOCH_ZMQSTREAM2PUSHERGROUP_H
+#define JUNGFRAUJOCH_ZMQSTREAM2PUSHERGROUP_H
+
+#include "ImagePusher.h"
+#include "ZMQStream2Pusher.h"
+#include "../common/ZMQWrappers.h"
+
+class ZMQStream2PusherGroup : public ImagePusher {
+   std::vector<std::unique_ptr<ZMQStream2Pusher>> pusher;
+   int64_t images_per_file = 1;
+public:
+    ZMQStream2PusherGroup(ZMQContext &context, const std::vector<std::string>& addr,
+                          int32_t send_buffer_high_watermark = -1, int32_t send_buffer_size = -1);
+    // High performance implementation, where each socket has dedicated ZMQ context
+    explicit ZMQStream2PusherGroup(const std::vector<std::string>& addr,
+                                   int32_t send_buffer_high_watermark = -1, int32_t send_buffer_size = -1);
+
+    void StartDataCollection(StartMessage& message) override;
+    void SendImage(const uint8_t *image_data, size_t image_size, int64_t image_number, ZeroCopyReturnValue *z) override;
+    bool SendImage(const uint8_t *image_data, size_t image_size, int64_t image_number) override;
+    bool EndDataCollection(const EndMessage& message) override;
+    bool SendCalibration(const CompressedImage& message) override;
+
+    std::vector<std::string> GetAddress();
+};
+
+#endif //JUNGFRAUJOCH_ZMQSTREAM2PUSHERGROUP_H

frontend_ui/package.json

@@ -1,7 +1,7 @@
 {
-  "name": "jungfraujoch-frontend",
-  "version": "1.0.0",
-  "license": "Proprietary",
+  "name": "jungfrau-frontend",
+  "version": "0.1.0",
+  "license": "GPL-3.0",
   "private": true,
   "homepage": "/frontend",
   "dependencies": {
@@ -10,7 +10,6 @@
     "@mui/icons-material": "^5.10.6",
     "@mui/material": "^5.10.6",
     "@mui/x-data-grid": "^5.17.14",
-    "@redocly/cli": "^1.12.2",
     "@types/jest": "^29.2.4",
     "@types/node": "^18.11.13",
     "@types/react": "^18.0.26",
@@ -26,10 +25,9 @@
   "scripts": {
     "start": "REACT_APP_VERSION=$(git rev-parse --short HEAD) PORT=8000 react-scripts start",
     "build": "REACT_APP_VERSION=$(git rev-parse --short HEAD) react-scripts build",
-    "redocly": "redocly build-docs ../broker/jfjoch_api.yaml --output=build/openapi.html",
     "test": "react-scripts test",
     "eject": "react-scripts eject",
-    "openapi": "./node_modules/openapi-typescript-codegen/bin/index.js -i ../broker/jfjoch_api.yaml --output ./src/openapi"
+    "openapi": "openapi --input ../broker/jfjoch_api.yaml --output ./src/openapi"
   },
   "eslintConfig": {
     "extends": "react-app"

frontend_ui/src/App.tsx

@@ -145,8 +145,7 @@ class App extends Component<MyProps, MyState> {
                 href="mailto:filip.leonarski@psi.ch">Filip Leonarski</a> <br/>
                 For more information see <a href="https://doi.org/10.1107/S1600577522010268"><i>J. Synchrotron
                     Rad.</i> (2023). <b>30</b>, 227–234</a> <br/>
-                Build: {process.env.REACT_APP_VERSION}&nbsp;&nbsp;&nbsp;
-                <a href="/frontend/openapi.html">API reference</a></center>
+                Build: {process.env.REACT_APP_VERSION}</center>
             <br/>
         </ThemeProvider>
     }

frontend_ui/src/components/BkgEstimatePlot.tsx

@@ -9,7 +9,7 @@ type MyProps = {};
 class BkgEstimatePlot extends Component<MyProps> {
 
     render() {
-        return <Paper style={{textAlign: 'center'}} sx={{height: 500, width: "100%"}}>
+        return <Paper style={{textAlign: 'center'}} sx={{height: 450, width: "100%"}}>
             <Box sx={{width: "100%", height: 50}}>
                 <br/>
                 <center><strong>Background estimate</strong></center>

frontend_ui/src/components/DataProcessingPlot.tsx

@@ -16,7 +16,9 @@ export enum PlotType {
     STRONG_PIXELS,
     ROI_SUM,
     ROI_MAX_COUNT,
-    RECEIVER_FREE_SEND_BUFS
+    RES_ESTIMATION,
+    RECEIVER_FREE_SEND_BUFS,
+    BEAM_CENTER_DRIFT
 }
 
 type MyProps = {
@@ -143,6 +145,13 @@ class DataProcessingPlot extends Component<MyProps, MyState> {
                         this.setState({connection_error: true});
                     });
                 break;
+            case PlotType.RES_ESTIMATION:
+                DefaultService.getPlotResolutionEstimateHistogram()
+                    .then(data => this.setState({plots: data, connection_error: false}))
+                    .catch(error => {
+                        this.setState({connection_error: true});
+                    });
+                break;
             case PlotType.RAD_INT_PER_FILE:
                 DefaultService.getPlotRadIntPerFile()
                     .then(data => this.setState({plots: data, connection_error: false}))

frontend_ui/src/components/DataProcessingPlots.tsx

@@ -69,6 +69,9 @@ class DataProcessingPlots extends Component<MyProps, MyState> {
             case "11":
                 this.setState({type: PlotType.ROI_SUM, xlabel: "Image number", ylabel: "Photon count"});
                 break;
+            case "12":
+                this.setState({type: PlotType.RES_ESTIMATION, xlabel: "Resolution [&#8491;]", ylabel: "Number of crystals"});
+                break;
             case "13":
                 this.setState({type: PlotType.RECEIVER_FREE_SEND_BUFS, xlabel: "Image number", ylabel: "Number of buffers"});
                 break;
@@ -80,7 +83,7 @@ class DataProcessingPlots extends Component<MyProps, MyState> {
     };
 
     render() {
-        return <Paper style={{textAlign: 'center'}} sx={{ height: 700, width: "100%" }}>
+        return <Paper style={{textAlign: 'center'}} sx={{ height: 630, width: "100%" }}>
             <Toolbar>
 
                 <Grid container sx={{ minWidth: 500 }} >
@@ -97,6 +100,7 @@ class DataProcessingPlots extends Component<MyProps, MyState> {
                             <MenuItem value={3}>Azimuthal integration profile</MenuItem>
                             <MenuItem value={11}>ROI area sum</MenuItem>
                             <MenuItem value={6}>ROI area max count</MenuItem>
+                            <MenuItem value={12}>Crystal resolution histogram</MenuItem>
                             <MenuItem value={4}>Indexing rate (per time point)</MenuItem>
                             <MenuItem value={5}>Azimuthal integration profile (per time point)</MenuItem>
                             <MenuItem value={8}>Error and saturated pixels</MenuItem>

frontend_ui/src/components/DataProcessingSettings.tsx

@@ -17,8 +17,6 @@ class DataProcessingSettings extends Component<MyProps, MyState> {
         s: {
             enable: true,
             indexing: true,
-            filter_powder_rings: false,
-            min_spot_count_powder_ring: 20,
             photon_count_threshold: 8,
             signal_to_noise_threshold: 3.0,
             min_pix_per_spot: 2,
@@ -52,6 +50,10 @@ class DataProcessingSettings extends Component<MyProps, MyState> {
         this.interval = setInterval(() => this.getValues(), 1000);
     }
 
+    componentWillUnmount() {
+        clearInterval(this.interval);
+    }
+
     setPhotonCountThreshold = (event: Event, newValue: number | number[]) => {
         this.setState(prevState => (
             {
@@ -62,7 +64,6 @@ class DataProcessingSettings extends Component<MyProps, MyState> {
             }
         ));
         this.putValues();
-        this.getValues();
     }
 
     setSignalToNoiseThreshold = (event: Event, newValue: number | number[]) => {
@@ -131,28 +132,8 @@ class DataProcessingSettings extends Component<MyProps, MyState> {
         this.getValues();
     }
 
-    enableFilterPowderRingToggle = (event: React.ChangeEvent<HTMLInputElement>) => {
-        let x = this.state;
-        x.s.filter_powder_rings = event.target.checked;
-        this.putValues2(x);
-        this.getValues();
-    }
-
-    setMinSpotCountPowderRing = (event: Event, newValue: number | number[]) => {
-        this.setState(prevState => (
-            {
-                s : {
-                    ...prevState.s,
-                    min_spot_count_powder_ring: newValue as number
-                }
-            }
-        ));
-        this.putValues();
-        this.getValues();
-    }
-
     render() {
-        return <Paper style={{textAlign: 'center'}} sx={{ height: 700, width: '100%' }}>
+        return <Paper style={{textAlign: 'center'}} sx={{ height: 630, width: '100%' }}>
                     <Grid container spacing={0}>
 
                         <Grid item xs={1}/>
@@ -160,53 +141,43 @@ class DataProcessingSettings extends Component<MyProps, MyState> {
                             <br/><strong>Spot finding parameters</strong><br/><br/>
                             <Switch onChange={this.enableSpotFindingToggle} checked={this.state.s.enable}
                                     disabled={this.state.connection_error}/>
-                            Spot finding
-                            <br/><br/>
+                            Enable spot finding &nbsp;
+                            <Switch onChange={this.enableIndexingToggle} checked={this.state.s.indexing}
+                                    disabled={this.state.connection_error || !this.state.s.enable}/>
+                            Enable indexing <br/><br/>
 
                             <Typography gutterBottom> Count threshold </Typography>
                             <Slider disabled={this.state.connection_error || !this.state.s.enable}
                                     value={Number(this.state.s.photon_count_threshold)}
                                     onChange={this.setPhotonCountThreshold}
-                                    min={1} max={50} step={1} valueLabelDisplay="auto"/>
+                                    min={1} max={50} step={1} valueLabelDisplay="auto" />
 
                             <br/><Typography> Signal-to-noise threshold </Typography>
                             <Slider disabled={this.state.connection_error || !this.state.s.enable}
                                     value={Number(this.state.s.signal_to_noise_threshold)}
                                     onChange={this.setSignalToNoiseThreshold}
-                                    min={2} max={10} step={0.5} valueLabelDisplay="auto"/>
+                                    min={2} max={10} step={0.5} valueLabelDisplay="auto" />
 
                             <br/><Typography> Minimum pixel / spot </Typography>
                             <Slider disabled={this.state.connection_error || !this.state.s.enable}
                                     value={Number(this.state.s.min_pix_per_spot)}
                                     onChange={this.setMinPixPerSpot}
-                                    min={1} max={8} step={1} valueLabelDisplay="auto"/>
+                                    min={1} max={8} step={1} valueLabelDisplay="auto"  />
                             <Typography> High resolution limit [&#8491;] </Typography>
                             <Slider disabled={this.state.connection_error || !this.state.s.enable}
                                     value={Number(this.state.s.high_resolution_limit)}
                                     onChange={this.setHighResolutionLimit}
-                                    min={1} max={5} step={0.2} valueLabelDisplay="auto"/>
-                            <br/><br/>
-                            <Switch onChange={this.enableFilterPowderRingToggle}
-                                    checked={this.state.s.filter_powder_rings}
-                                    disabled={this.state.connection_error || !this.state.s.enable}/>
-                            Filter spots in powder rings (<i>e.g.</i>, ice)
-                            <br/><br/>
-                            <Typography> Min spots to filter powder ring </Typography>
-                            <Slider disabled={this.state.connection_error || !this.state.s.enable || !this.state.s.filter_powder_rings}
-                                    value={Number(this.state.s.filter_powder_rings)}
-                                    onChange={this.setMinSpotCountPowderRing}
-                                    min={5} max={50} step={1} valueLabelDisplay="auto"/>
-                            <br/> <br/>
-                            <Switch onChange={this.enableIndexingToggle} checked={this.state.s.indexing}
-                                    disabled={this.state.connection_error || !this.state.s.enable}/>
-                            Indexing <br/><br/>
+                                    min={1} max={5} step={0.2} valueLabelDisplay="auto"  />
                             <Typography> Indexing spot acceptance tolerance </Typography>
-                            <Slider disabled={this.state.connection_error || !this.state.s.enable || !this.state.s.indexing}
+                            <Slider disabled={this.state.connection_error || !this.state.s.enable}
                                     value={Number(this.state.s.indexing_tolerance)}
                                     onChange={this.setIndexingTolerance}
-                                    min={0.0} max={0.3} step={0.01} valueLabelDisplay="auto"/>
+                                    min={0.0} max={0.3} step={0.01} valueLabelDisplay="auto"  />
+                            <br/><br/>
                         </Grid>
                         <Grid item xs={1}/>
+
+
                     </Grid>
                 </Paper>
     }

frontend_ui/src/components/MeasurementStatistics.tsx

@@ -48,7 +48,7 @@ class MeasurementStatistics extends Component<MyProps, MyState> {
     }
 
     render() {
-        return <Paper style={{textAlign: 'center'}} sx={{ height: 500, width: '100%' }}>
+        return <Paper style={{textAlign: 'center'}} sx={{ height: 450, width: '100%' }}>
             <Grid container spacing={0}>
                 <Grid item xs={1}/>
                 <Grid item xs={10}>
@@ -98,10 +98,6 @@ class MeasurementStatistics extends Component<MyProps, MyState> {
                                     <TableCell align="right">{(this.state.s.bkg_estimate !== undefined)
                                         ? this.state.s.bkg_estimate.toPrecision(7) : "-"}</TableCell>
                                 </TableRow>
-                                <TableRow>
-                                    <TableCell component="th" scope="row"> Unit cell: </TableCell>
-                                    <TableCell align="right">{this.state.s.unit_cell}</TableCell>
-                                </TableRow>
                             </TableBody>
                         </Table>
                     </TableContainer>

frontend_ui/src/components/PreviewImage.tsx

@@ -26,10 +26,9 @@ class PreviewImage extends Component<MyProps, MyState> {
             show_spots: true,
             show_roi: false,
             show_indexed: false,
-            show_user_mask: false,
             resolution_ring: 0.5
         },
-        s_url: null,
+        s_url: "",
         update: true,
         connection_error: true
     }
@@ -95,17 +94,6 @@ class PreviewImage extends Component<MyProps, MyState> {
         );
     }
 
-    showUserMaskToggle = (event: React.ChangeEvent<HTMLInputElement>) => {
-        this.setState(prevState => (
-            {
-                settings : {
-                    ...prevState.settings,
-                    show_user_mask: event.target.checked
-                }
-            })
-        );
-    }
-
     getValues() {
         if (!this.state.update)
             return;
@@ -136,12 +124,9 @@ class PreviewImage extends Component<MyProps, MyState> {
         clearInterval(this.interval);
     }
 
-    render() {
-        return <Paper sx={{height: 1250, width: 1000, m: 2}}
-                      component={Stack}
-                      direction="column">
-            <br/>
-            <Stack spacing={2} direction="row" sx={{mb: 1}} alignItems="center">
+    preview() {
+        return <div><br/>
+            <Stack spacing={2} direction="row" sx={{ mb: 1 }} alignItems="center">
                 &nbsp;&nbsp;<strong>Preview image</strong>
                 <Switch disabled={this.state.connection_error} checked={this.state.update}
                         onChange={this.updateToggle} name="Update"/>
@@ -155,16 +140,13 @@ class PreviewImage extends Component<MyProps, MyState> {
                 <Switch disabled={this.state.connection_error} checked={this.state.settings.show_indexed}
                         onChange={this.showIndexedToggle} name="Show ROI"/>
                 Show only indexed images&nbsp;&nbsp;&nbsp;
-                <Switch disabled={this.state.connection_error} checked={this.state.settings.show_user_mask}
-                        onChange={this.showUserMaskToggle} name="Show user mask"/>
-                Show user mask&nbsp;&nbsp;&nbsp;
-                <Box sx={{width: 200}}>
+                <Box sx={{ width: 200 }}>
                     <Slider disabled={this.state.connection_error}
                             value={Number(this.state.settings.saturation)} min={1} max={80}
                             onChange={this.setSaturation} valueLabelDisplay="auto"/> <br/>Saturation value
                 </Box>
                 &nbsp;&nbsp;&nbsp;
-                <Box sx={{width: 200}}>
+                <Box sx={{ width: 200 }}>
                     <Slider disabled={this.state.connection_error}
                             value={(this.state.settings.resolution_ring === undefined) ? 0.5 : Number(this.state.settings.resolution_ring)}
                             min={0.5} max={5.0} step={0.1}
@@ -173,7 +155,8 @@ class PreviewImage extends Component<MyProps, MyState> {
 
             </Stack>
             <br/>
-            {(!this.state.connection_error && (this.state.s_url !== null)) ?
+            {
+                this.state.s_url !== null ?
                     <Stack
                         direction="row"
                         justifyContent="center"
@@ -181,13 +164,20 @@ class PreviewImage extends Component<MyProps, MyState> {
                     >
                         <TransformWrapper>
                             <TransformComponent>
-                                <img src={this.state.s_url} alt="Live preview"
-                                     style={{maxWidth: "100%", maxHeight: 900}}/>
+                                <img src={this.state.s_url} alt="Live preview" style={{maxWidth: "100%", maxHeight: 900}}/>
                             </TransformComponent>
                         </TransformWrapper>
-                    </Stack> : <div>Preview not available</div>
+                    </Stack>: <div/>
             }
             <br/>
+        </div>
+    }
+
+    render() {
+        return <Paper sx={{height: 1050, width: 850, m: 2}}
+                      component={Stack}
+                      direction="column">
+            {(!this.state.connection_error && (this.state.s_url !== null)) ? this.preview() : "Preview not available"}
         </Paper>
     }
 }

frontend_ui/src/openapi/models/measurement_statistics.ts

@@ -31,7 +31,6 @@ export type measurement_statistics = {
     detector_height?: number;
     detector_pixel_depth?: measurement_statistics.detector_pixel_depth;
     bkg_estimate?: number;
-    unit_cell?: string;
 };
 
 export namespace measurement_statistics {

frontend_ui/src/openapi/models/preview_settings.ts

@@ -27,10 +27,6 @@ export type preview_settings = {
      * Preview indexed images only
      */
     show_indexed?: boolean;
-    /**
-     * Show user mask
-     */
-    show_user_mask?: boolean;
     resolution_ring?: number;
 };
 

frontend_ui/src/openapi/models/spot_finding_settings.ts

@@ -5,24 +5,13 @@
 
 export type spot_finding_settings = {
     /**
-     * Enable spot finding. This is temporary setting, i.e. can be changed anytime during data collection.
-     * Even if disabled spot finding information will still be send and written, though always with zero spots.
-     *
+     * Enable spot finding
      */
     enable: boolean;
     /**
-     * Enable indexing. This is temporary setting, i.e. can be changed anytime during data collection.
-     *
+     * Enable indexing
      */
     indexing: boolean;
-    /**
-     * Filter spots which form powder rings (e.g., ice rings)
-     */
-    filter_powder_rings?: boolean;
-    /**
-     * Minimum number of spots to consider a thin resolution shell (0.01 A^-1) a powder ring and filter out.
-     */
-    min_spot_count_powder_ring?: number;
     signal_to_noise_threshold: number;
     photon_count_threshold: number;
     min_pix_per_spot: number;

frontend_ui/src/openapi/services/DefaultService.ts

@@ -280,20 +280,20 @@ export class DefaultService {
      * Requires binary blob with 16-bit integer numbers of size of detector in raw/converted coordinates
      * (depending on detector settings).
      *
-     * @param id Image id to upload
+     * @param number Image number to upload
      * @param requestBody
      * @returns any Everything OK
      * @throws ApiError
      */
     public static putConfigInternalGeneratorImage(
-        id?: number,
+        number?: number,
         requestBody?: Blob,
     ): CancelablePromise<any> {
         return __request(OpenAPI, {
             method: 'PUT',
             url: '/config/internal_generator_image',
             query: {
-                'id': id,
+                'number': number,
             },
             body: requestBody,
             mediaType: 'application/octet-stream',
@@ -303,35 +303,6 @@ export class DefaultService {
         });
     }
 
-    /**
-     * Load TIFF image for internal FPGA generator
-     * Load image for internal FPGA generator. This can only happen in Idle state of the detector.
-     * Requires TIFF with 16-bit integer numbers of size of detector in raw/converted coordinates
-     * (depending on detector settings).
-     *
-     * @param id Image ID to upload
-     * @param requestBody
-     * @returns any Everything OK
-     * @throws ApiError
-     */
-    public static putConfigInternalGeneratorImageTiff(
-        id?: number,
-        requestBody?: Blob,
-    ): CancelablePromise<any> {
-        return __request(OpenAPI, {
-            method: 'PUT',
-            url: '/config/internal_generator_image.tiff',
-            query: {
-                'id': id,
-            },
-            body: requestBody,
-            mediaType: 'image/tiff',
-            errors: {
-                400: `Input parsing or validation error`,
-            },
-        });
-    }
-
     /**
      * Select detector
      * Jungfraujoch allows to control multiple detectors and/or region-of-interests.
@@ -753,6 +724,19 @@ export class DefaultService {
         });
     }
 
+    /**
+     * Generate resolution estimate histogram
+     * Generate histogram of crystal resolutions from 1.0 to 5.0 A based on ML model
+     * @returns plots Everything OK
+     * @throws ApiError
+     */
+    public static getPlotResolutionEstimateHistogram(): CancelablePromise<plots> {
+        return __request(OpenAPI, {
+            method: 'GET',
+            url: '/plot/resolution_estimate_histogram',
+        });
+    }
+
     /**
      * Generate radial integration profile
      * Generate average radial integration profile
@@ -876,63 +860,6 @@ export class DefaultService {
         });
     }
 
-    /**
-     * Get mask of the detector
-     * Get full pixel mask of the detector
-     * See NXmx standard for meaning of pixel values
-     *
-     * @returns binary Pixel mask in TIFF format (4 byte; unsigned)
-     * @throws ApiError
-     */
-    public static getConfigMaskTiff(): CancelablePromise<Blob> {
-        return __request(OpenAPI, {
-            method: 'GET',
-            url: '/config/mask.tiff',
-        });
-    }
-
-    /**
-     * Get user mask of the detector
-     * Get user pixel mask of the detector in the actual detector coordinates: 0 - good pixel, 1 - masked
-     * @returns binary User mask in TIFF format (4 byte; unsigned)
-     * @throws ApiError
-     */
-    public static getConfigUserMaskTiff(): CancelablePromise<Blob> {
-        return __request(OpenAPI, {
-            method: 'GET',
-            url: '/config/user_mask.tiff',
-        });
-    }
-
-    /**
-     * Upload user mask of the detector
-     * Should be in `Idle` state.
-     * Upload user mask of the detector - this is for example to account for beam stop shadow or misbehaving regions.
-     * If detector is conversion mode the mask can be both in raw (1024x512; stacked modules) or converted coordinates.
-     * In the latter case - module gaps are ignored and don't need to be assigned value.
-     * Mask is expected as TIFF (4-byte; unsigned).
-     * 0 - good pixel, other value - masked
-     * User mask is stored in NXmx pixel mask (bit 8), as well as used in spot finding and azimuthal integration.
-     * User mask is not automatically applied - i.e. pixels with user mask will have a valid pixel value in the images.
-     *
-     * @param requestBody
-     * @returns binary All good
-     * @throws ApiError
-     */
-    public static putConfigUserMaskTiff(
-        requestBody?: Blob,
-    ): CancelablePromise<Blob> {
-        return __request(OpenAPI, {
-            method: 'PUT',
-            url: '/config/user_mask.tiff',
-            body: requestBody,
-            mediaType: 'application/octet-stream',
-            errors: {
-                500: `Error within Jungfraujoch code - see output message.`,
-            },
-        });
-    }
-
     /**
      * Get pedestal G0 in TIFF format
      * @param gainLevel Gain level (0, 1, 2)

image_analysis/CMakeLists.txt

@@ -21,8 +21,8 @@ IF (CMAKE_CUDA_COMPILER)
 
     FetchContent_Declare(
             fast-indexer
-            GIT_REPOSITORY https://github.com/fleon-psi/fast-feedback-indexer/
-            GIT_TAG 66c3f44
+            GIT_REPOSITORY https://github.com/paulscherrerinstitute/fast-feedback-indexer/
+            GIT_TAG 2d6fa7511f240eeb4918dd7647edd0642e987317
     )
 
     FetchContent_MakeAvailable(fast-indexer)

image_analysis/CPUSpotFinder.cpp

@@ -1,125 +1,86 @@
 // Copyright (2019-2024) Paul Scherrer Institute
 
 #include "CPUSpotFinder.h"
-#define NUM_PASS 2
 
 template <int N>
 void FindSpots(DeviceOutput &output,
                int big_column, int big_row,
                const SpotFindingSettings& settings,
-               const float *d_array,
-               float *arr_mean,
-               float *arr_stddev,
-               uint32_t *arr_valid_count,
-               uint32_t *arr_strong_pixel) {
+               const float *d_array) {
     auto image = (int16_t *) output.pixels;
 
-    std::vector<uint8_t> mask(N * N, 0);
+    int64_t sum = 0;
+    int64_t sum2 = 0;
+    int64_t valid_count = 0;
 
     for (int y = 0; y < N; y++) {
-        size_t line = big_row * N + y;
         for (int x = 0; x < N; x++) {
+            size_t coord = (big_row * N + y) * RAW_MODULE_COLS + big_column * N + x;
+            if ((image[coord] != INT16_MIN) || (image[coord] != INT16_MAX)) {
+                sum += image[coord];
+                sum2 += image[coord] * image[coord];
+                valid_count++;
+            }
+        }
+    }
+
+    int64_t variance = valid_count * sum2 - sum * sum;
+    float threshold = variance * settings.signal_to_noise_threshold * settings.signal_to_noise_threshold;
+
+    for (int y = 0; y < N; y++) {
+        for (int x = 0; x < N; x++) {
+            size_t line = big_row * N + y;
             size_t col = big_column * N + x;
             size_t coord = line * RAW_MODULE_COLS + col;
 
-            uint8_t bad_pixel = 0;
+            bool strong_pixel = true;
+
             if ((line == 255) || (line == 256)
                 || (col == 255) || (col == 256)
                 || (col == 511) || (col == 512)
                 || (col == 767) || (col == 768))
-                bad_pixel = 1;
+                strong_pixel = false;
 
-            if ((d_array[coord] < settings.high_resolution_limit)
-                || (d_array[coord] > settings.low_resolution_limit))
-                bad_pixel = 1;
-
-            if ((image[coord] == INT16_MIN) || (image[coord] > 32760))
-                bad_pixel = 1;
-
-            mask[y * N + x] = bad_pixel;
-        }
-    }
-
-    for (int i = 0; i < NUM_PASS; i++) {
-        int64_t sum = 0;
-        int64_t sum2 = 0;
-        int64_t valid_count = 0;
-
-        for (int y = 0; y < N; y++) {
-            for (int x = 0; x < N; x++) {
-                size_t coord = (big_row * N + y) * RAW_MODULE_COLS + big_column * N + x;
-                if (mask[y * N + x] == 0) {
-                    sum += image[coord];
-                    sum2 += image[coord] * image[coord];
-                    valid_count++;
-                }
+            if (d_array[coord] != 0) {
+                if ((d_array[coord] < settings.high_resolution_limit)
+                    || (d_array[coord] > settings.low_resolution_limit))
+                    strong_pixel = false;
             }
-        }
 
-        int64_t variance = valid_count * sum2 - sum * sum;
-        float threshold = variance * settings.signal_to_noise_threshold * settings.signal_to_noise_threshold;
+            if ((settings.photon_count_threshold < 0)
+                && (settings.signal_to_noise_threshold <= 0))
+                strong_pixel = false;
 
-        for (int y = 0; y < N; y++) {
-            size_t line = big_row * N + y;
-            for (int x = 0; x < N; x++) {
-                size_t col = big_column * N + x;
-                size_t coord = line * RAW_MODULE_COLS + col;
+            if ((settings.photon_count_threshold >= 0)
+                && (image[coord] <= settings.photon_count_threshold)) {
+                strong_pixel = false;
+            }
 
-                arr_valid_count[coord] = valid_count;
-                if (valid_count > 0) {
-                    arr_mean[coord] = static_cast<float>(sum) / static_cast<float>(valid_count);
-                    arr_stddev[coord] = sqrtf(
-                            static_cast<float>(variance) / static_cast<float>(valid_count * valid_count));
-                } else {
-                    arr_mean[coord] = -1;
-                    arr_stddev[coord] = -1;
-                }
+            if (image[coord] > 32760)
+                strong_pixel = false;
 
-                bool strong_pixel = true;
-
-                if (mask[y * N + x])
+            if (settings.signal_to_noise_threshold > 0) {
+                int64_t in_minus_mean = image[coord] * valid_count - sum;
+                if ((in_minus_mean * in_minus_mean <= threshold)
+                    || (in_minus_mean <= 0)
+                    || (valid_count < N * N / 2))
                     strong_pixel = false;
+            }
 
-                if ((settings.photon_count_threshold < 0)
-                    && (settings.signal_to_noise_threshold <= 0))
-                    strong_pixel = false;
-
-                if ((settings.photon_count_threshold >= 0)
-                    && (image[coord] <= settings.photon_count_threshold)) {
-                    strong_pixel = false;
-                }
-
-                if (settings.signal_to_noise_threshold > 0) {
-                    int64_t in_minus_mean = image[coord] * valid_count - sum;
-                    if ((in_minus_mean * in_minus_mean <= threshold)
-                        || (in_minus_mean <= 0)
-                        || (valid_count <= N * N / 2))
-                        strong_pixel = false;
-                }
-
-                if (strong_pixel) {
-                    mask[y * N + x] = 1;
-                    arr_strong_pixel[coord]++;
-                    output.spot_finding_result.strong_pixel[coord / 8] |= (1 << (coord % 8));
-                    output.spot_finding_result.strong_pixel_count++;
-                }
+            if (strong_pixel) {
+                output.spot_finding_result.strong_pixel[coord / 8] |= (1 << (coord % 8));
+                output.spot_finding_result.strong_pixel_count++;
             }
         }
     }
 }
 
-void FindSpots(DeviceOutput &output,
-               const SpotFindingSettings& settings,
-               const float *d_array,
-               float *arr_mean,
-               float *arr_stddev,
-               uint32_t *arr_valid_count,
-               uint32_t *arr_strong_pixel) {
+void FindSpots(DeviceOutput &output, const SpotFindingSettings& settings, const float *d_array) {
     for (auto &i: output.spot_finding_result.strong_pixel)
         i = 0;
 
     for (int i = 0; i < RAW_MODULE_LINES / 32; i++) {
         for (int j = 0; j < RAW_MODULE_COLS / 32; j++)
-            FindSpots<32>(output, j, i, settings, d_array, arr_mean, arr_stddev, arr_valid_count, arr_strong_pixel);
+            FindSpots<32>(output, j, i, settings, d_array);
     }
 }

image_analysis/CPUSpotFinder.h

@@ -7,13 +7,6 @@
 #include "SpotFindingSettings.h"
 #include "StrongPixelSet.h"
 
-void FindSpots(DeviceOutput &output,
-               const SpotFindingSettings& settings,
-               const float *d_array,
-               float *arr_mean,
-               float *arr_stddev,
-               uint32_t *arr_valid_count,
-               uint32_t *arr_strong_pixel);
-
+void FindSpots(DeviceOutput &output, const SpotFindingSettings& settings, const float *d_array);
 
 #endif //JUNGFRAUJOCH_CPUSPOTFINDER_H

image_analysis/IndexerWrapper.cpp

@@ -8,17 +8,13 @@ void IndexerWrapper::Setup(const UnitCell &cell) {
 #endif
 }
 
-std::vector<IndexingResult> IndexerWrapper::Run(const std::vector<Coord> &coord, float indexing_threshold, int nspots) {
+std::vector<IndexingResult> IndexerWrapper::Run(const std::vector<Coord> &coord, float indexing_threshold) {
 #ifdef JFJOCH_USE_CUDA
     std::vector<IndexingResult> ret;
 
-    if (nspots > coord.size())
-        nspots = coord.size();
-
-    if (nspots <= viable_cell_min_spots)
+    if (coord.size() <= viable_cell_min_spots)
         return ret;
 
-    assert(nspots <= MAX_SPOT_COUNT);
     assert(coord.size() <= MAX_SPOT_COUNT);
 
     for (int i = 0; i < coord.size(); i++) {
@@ -28,9 +24,9 @@ std::vector<IndexingResult> IndexerWrapper::Run(const std::vector<Coord> &coord,
     }
 
     // Index
-    indexer.index(1, nspots);
+    indexer.index(1, coord.size());
 
-    fast_feedback::refine::indexer_ifssr<float>::refine(indexer.spotM().topRows(nspots),
+    fast_feedback::refine::indexer_ifssr<float>::refine(indexer.spotM().topRows(coord.size()),
                                                         indexer.oCellM(),
                                                         indexer.oScoreV(),
                                                         cifssr);
@@ -42,28 +38,30 @@ std::vector<IndexingResult> IndexerWrapper::Run(const std::vector<Coord> &coord,
     // Get best cell
     auto id = fast_feedback::refine::best_cell(indexer.oScoreV());
 
-    auto cell = indexer.oCell(id).colwise().reverse();
-    fast_feedback::refine::make_right_handed(cell);
+    bool indexed = fast_feedback::refine::is_viable_cell(indexer.oCell(id), indexer.Spots(),
+                                                         indexing_threshold, viable_cell_min_spots,
+                                                         false);
 
-    using M3x = Eigen::MatrixX3<float>;
-    M3x resid = indexer.spotM().topRows(coord.size()) * cell.transpose();
-    const M3x miller = round(resid.array());
-    resid -= miller;
+    // Check if result is viable
+    if (indexed) {
+        auto cell = indexer.oCell(id).colwise().reverse();
+        fast_feedback::refine::make_right_handed(cell);
 
-    auto indexed_spots = (resid.rowwise().norm().array() < indexing_threshold);
-    auto indexed_spot_count = indexed_spots.topRows(nspots).count();
+        // get indexed spots
+        using M3x = Eigen::MatrixX3<float>;
+        M3x resid = indexer.spotM().topRows(coord.size()) * cell.transpose();
+        const M3x miller = round(resid.array());
+        const M3x predicted = miller * cell.transpose().inverse();
 
-    // Conditions for indexing:
-    // * There must be 9 AND 20% of all spots indexed
-    // * Decision is only based on spots used for indexing (not all spots)
-    if ((indexed_spot_count > viable_cell_min_spots) && (indexed_spot_count > 0.20 * nspots)) {
         IndexingResult result;
         result.l = CrystalLattice(cell);
 
-        result.indexed_spot.resize(coord.size());
+        result.predicted_spots.resize(coord.size());
 
         for (int i = 0; i < coord.size(); i++)
-            result.indexed_spot[i] = indexed_spots(i);
+            result.predicted_spots[i] = Coord(predicted.coeff(i, 0),
+                                              predicted.coeff(i, 1),
+                                              predicted.coeff(i, 2));
 
         ret.emplace_back(result);
     }

image_analysis/IndexerWrapper.h

@@ -15,7 +15,7 @@
 
 struct IndexingResult {
     CrystalLattice l;
-    std::vector<uint8_t> indexed_spot;
+    std::vector<Coord> predicted_spots;
 };
 
 class IndexerWrapper {
@@ -36,7 +36,7 @@ class IndexerWrapper {
     constexpr const static uint32_t viable_cell_min_spots = 9;
 public:
     void Setup(const UnitCell &cell);
-    std::vector<IndexingResult> Run(const std::vector<Coord> &coord, float indexing_threshold, int nspots = -1);
+    std::vector<IndexingResult> Run(const std::vector<Coord> &coord, float indexing_threshold);
 };
 
 

image_analysis/MXAnalyzer.cpp

@@ -27,16 +27,12 @@ MXAnalyzer::MXAnalyzer(const DiffractionExperiment &in_experiment)
 : experiment(in_experiment) {
     auto uc = experiment.GetUnitCell();
     if (uc) {
-        try {
-            indexer = std::make_unique<IndexerWrapper>();
-            indexer->Setup(uc.value());
-        } catch (const std::exception &e) {
-            throw JFJochException(JFJochExceptionCategory::GPUCUDAError, e.what());
-        }
+        do_indexing = true;
+        indexer.Setup(uc.value());
     }
-
     if (experiment.IsSpotFindingEnabled())
         find_spots = true;
+
 }
 
 void MXAnalyzer::ReadFromFPGA(const DeviceOutput *output, const SpotFindingSettings &settings, size_t module_number) {
@@ -60,114 +56,45 @@ void MXAnalyzer::ReadFromCPU(const int16_t *image, const SpotFindingSettings &se
 
     CalcSpotFinderResolutionMap(d_map.data(), experiment, module_number);
 
-    arr_mean.resize(experiment.GetModulesNum() * RAW_MODULE_SIZE);
-    arr_sttdev.resize(experiment.GetModulesNum() * RAW_MODULE_SIZE);
-    arr_valid_count.resize(experiment.GetModulesNum() * RAW_MODULE_SIZE);
-    arr_strong_pixel.resize(experiment.GetModulesNum() * RAW_MODULE_SIZE);
-
-    FindSpots(output,
-              settings,
-              d_map.data(),
-              arr_mean.data() + module_number * RAW_MODULE_SIZE,
-              arr_sttdev.data() + module_number * RAW_MODULE_SIZE,
-              arr_valid_count.data() + module_number * RAW_MODULE_SIZE,
-              arr_strong_pixel.data() + module_number * RAW_MODULE_SIZE);
+    FindSpots(output, settings, d_map.data());
 
     ReadFromFPGA(&output, settings, module_number);
 }
 
-uint32_t MXAnalyzer::FilterSpotsInPowderRings(const std::vector<DiffractionSpot> &spots_filter,
-                                              std::vector<DiffractionSpot> &spots_out,
-                                              int64_t min_spot_count_ring) {
-    uint32_t ret = 0;
-
-    double high_q = 5.0;
-    double low_q = 0;
-    double q_spacing = 0.01;
-
-    size_t bin_count = (high_q - low_q) / q_spacing + 1;
-
-    std::vector<std::vector<uint32_t> > bins(bin_count);
-
-    for (int i = 0; i < spots_filter.size(); i++) {
-        double q = 2 * M_PI / spots_filter[i].GetResolution(experiment);
-        if ((q >= low_q) && (q < high_q)) {
-            int32_t q_bin = std::floor((q - low_q) / q_spacing);
-            bins[q_bin].push_back(i);
-        } else // spots outside of azim. int. range are not filtered
-            spots_out.push_back(spots_filter[i]);
-    }
-
-    for (auto & bin : bins) {
-        if (bin.size() > min_spot_count_ring)
-            ret += bin.size();
-        else {
-            for (auto &iter: bin)
-                spots_out.push_back(spots_filter[iter]);
-        }
-    }
-    return ret;
-}
-
 void MXAnalyzer::Process(DataMessage &message, const SpotFindingSettings& settings) {
     message.indexing_result = false;
     if (!find_spots)
         return;
 
     std::vector<DiffractionSpot> spots_out;
-    if (settings.filter_spots_powder_ring) {
-        std::vector<DiffractionSpot> spots_no_rings;
-        message.spot_count_in_rings = FilterSpotsInPowderRings(spots,
-                                                               spots_no_rings,
-                                                               settings.min_spot_count_powder_ring);
-        FilterSpotsByCount(experiment, spots_no_rings, spots_out);
-    } else
-        FilterSpotsByCount(experiment, spots, spots_out);
-
+    FilterSpotsByCount(experiment, spots, spots_out);
     spots.clear();
 
     for (const auto &spot: spots_out)
         message.spots.push_back(spot);
 
-    if (indexer && settings.indexing) {
+    if (do_indexing && settings.indexing) {
         std::vector<Coord> recip;
         recip.reserve(spots_out.size());
         for (const auto &i: spots_out)
             recip.push_back(i.ReciprocalCoord(experiment));
 
-        std::vector<IndexingResult> indexer_result;
-
-        // If there is a really large number of spots detected, it is better to start with a smaller subset
-        if (recip.size() > 80)
-            indexer_result = indexer->Run(recip, settings.indexing_tolerance, 50);
-        if (indexer_result.empty())
-            indexer_result = indexer->Run(recip, settings.indexing_tolerance);
+        auto indexer_result = indexer.Run(recip, settings.indexing_tolerance);
 
         if (!indexer_result.empty()) {
             message.indexing_result = true;
-            assert(indexer_result[0].indexed_spot.size() == recip.size());
 
             // identify indexed spots
-            for (int i = 0; i < recip.size(); i++)
-                message.spots[i].indexed = indexer_result[0].indexed_spot[i];
+            for (int i = 0; i < recip.size(); i++) {
+                auto predicted_pos = RecipToDector(experiment, indexer_result[0].predicted_spots[i]);
+                float x_diff = predicted_pos.first - spots_out[i].RawCoord().x;
+                float y_diff = predicted_pos.second - spots_out[i].RawCoord().y;
+                message.spots[i].indexed = (x_diff * x_diff + y_diff * y_diff
+                                            < spot_distance_threshold_pxl * spot_distance_threshold_pxl);
+            }
 
             indexer_result[0].l.Save(message.indexing_lattice);
+            message.indexing_unit_cell = indexer_result[0].l.GetUnitCell();
         }
     }
 }
-
-const std::vector<float> &MXAnalyzer::GetCPUMean() const {
-    return arr_mean;
-}
-
-const std::vector<float> &MXAnalyzer::GetCPUStdDev() const {
-    return arr_sttdev;
-}
-
-const std::vector<uint32_t> &MXAnalyzer::GetCPUValidCount() const {
-    return arr_valid_count;
-}
-
-const std::vector<uint32_t> &MXAnalyzer::GetCPUStrongPixel() const {
-    return arr_strong_pixel;
-}

image_analysis/MXAnalyzer.h

@@ -9,21 +9,12 @@
 
 class MXAnalyzer {
     const DiffractionExperiment &experiment;
-    std::unique_ptr<IndexerWrapper> indexer;
+    IndexerWrapper indexer;
+    bool do_indexing = false;
     bool find_spots = false;
     std::vector<DiffractionSpot> spots;
     constexpr static const float spot_distance_threshold_pxl = 2.0f;
-
-    std::vector<float> arr_mean;
-    std::vector<float> arr_sttdev;
-    std::vector<uint32_t> arr_valid_count;
-    std::vector<uint32_t> arr_strong_pixel;
 public:
-    const std::vector<float> &GetCPUMean() const;
-    const std::vector<float> &GetCPUStdDev() const;
-    const std::vector<uint32_t> &GetCPUValidCount() const;
-    const std::vector<uint32_t> &GetCPUStrongPixel() const;
-
     explicit MXAnalyzer(const DiffractionExperiment& experiment);
 
     void ReadFromFPGA(const DeviceOutput* output,
@@ -36,10 +27,6 @@ public:
 
     void Process(DataMessage &message,
                  const SpotFindingSettings& settings);
-
-    uint32_t FilterSpotsInPowderRings(const std::vector<DiffractionSpot> &spots_filter,
-                                      std::vector<DiffractionSpot> &spots_out,
-                                      int64_t min_spot_count_ring);
 };
 
 

image_analysis/SpotFindingSettings.h

@@ -7,17 +7,14 @@
 
 struct SpotFindingSettings {
     bool enable = true;
+    bool indexing = true;
     float signal_to_noise_threshold = 3;  // STRONG_PIXEL in XDS
     int64_t photon_count_threshold = 10;  // Threshold in photon counts
     int64_t min_pix_per_spot = 2;         // Minimum pixels per spot
     int64_t max_pix_per_spot = 50;        // Maximum pixels per spot
-    float high_resolution_limit = 2.0;
+    float high_resolution_limit = 2.5;
     float low_resolution_limit = 50.0;
 
-    bool filter_spots_powder_ring = false;
-    int64_t min_spot_count_powder_ring = 20;
-
-    bool indexing = true;
     float indexing_tolerance = 0.1;
 };
 

jungfrau/JFCalibration.cpp

@@ -3,7 +3,7 @@
 #include "JFCalibration.h"
 
 #include <cstring>
-#include "../preview/JFJochTIFF.h"
+#include "../preview/WriteTIFF.h"
 
 JFCalibration::JFCalibration(size_t in_nmodules, size_t in_nstorage_cells) :
         nmodules(in_nmodules),

preview/CMakeLists.txt

@@ -1,14 +1,21 @@
+FIND_PACKAGE(TIFF COMPONENTS CXX REQUIRED)
 FIND_PACKAGE(JPEG REQUIRED)
 
 ADD_LIBRARY(JFJochPreview STATIC
-        JFJochTIFF.cpp JFJochTIFF.h
-        JFJochJPEG.cpp JFJochJPEG.h
+        WriteTIFF.cpp WriteTIFF.h
+        WriteJPEG.cpp WriteJPEG.h
         PreviewCounter.cpp PreviewCounter.h
         PreviewImage.cpp PreviewImage.h)
 
 TARGET_LINK_LIBRARIES(JFJochPreview PUBLIC JFJochCommon)
 
-TARGET_LINK_LIBRARIES(JFJochPreview PUBLIC tiff tiffxx)
+IF((EXISTS ${TIFF_INCLUDE_DIR}/tiffio.hxx) AND (EXISTS ${TIFF_INCLUDE_DIR}/tiffio.h))
+    TARGET_INCLUDE_DIRECTORIES(JFJochPreview PRIVATE ${TIFF_INCLUDE_DIR})
+    TARGET_LINK_LIBRARIES(JFJochPreview PUBLIC ${TIFF_LIBRARIES})
+    MESSAGE(STATUS "TIFF headers present and library included")
+ELSE()
+    MESSAGE(FATAL_ERROR "TIFF headers tiffio.h and tiffio.hxx not present")
+ENDIF()
 
 IF (EXISTS ${JPEG_INCLUDE_DIR}/jpeglib.h)
     TARGET_INCLUDE_DIRECTORIES(JFJochPreview PRIVATE ${JPEG_INCLUDE_DIR})

preview/JFJochTIFF.cpp

@@ -1,110 +0,0 @@
-// Copyright (2019-2024) Paul Scherrer Institute
-
-#include <tiffio.h>
-#include <tiffio.hxx>
-#include <sstream>
-
-#include "JFJochTIFF.h"
-#include "../common/JFJochException.h"
-
-
-void WriteTIFF(TIFF *tiff, void *buff, size_t cols, size_t lines, size_t elem_size, bool is_signed) {
-
-    if (tiff == nullptr)
-        throw JFJochException(JFJochExceptionCategory::TIFFGeneratorError, "TIFFStreamOpen error");
-
-    TIFFSetField(tiff, TIFFTAG_IMAGEWIDTH, cols);  // set the width of the image
-    TIFFSetField(tiff, TIFFTAG_IMAGELENGTH, lines);    // set the height of the image
-    TIFFSetField(tiff, TIFFTAG_SAMPLESPERPIXEL, 1);   // set number of channels per pixel
-    TIFFSetField(tiff, TIFFTAG_BITSPERSAMPLE, elem_size * 8);    // set the size of the channels
-    TIFFSetField(tiff, TIFFTAG_COMPRESSION, COMPRESSION_NONE); // setc ompression to LZW
-    TIFFSetField(tiff, TIFFTAG_ROWSPERSTRIP, lines);
-    if (is_signed)
-        TIFFSetField(tiff, TIFFTAG_SAMPLEFORMAT, SAMPLEFORMAT_INT);
-    else
-        TIFFSetField(tiff, TIFFTAG_SAMPLEFORMAT, SAMPLEFORMAT_UINT);
-
-    if (TIFFWriteEncodedStrip(tiff, 0, buff, cols * lines * elem_size) < 0)
-        throw JFJochException(JFJochExceptionCategory::TIFFGeneratorError, "TIFFWriteEncodedStrip error");
-}
-
-std::string WriteTIFFToString(void *buff, size_t cols, size_t lines, size_t elem_size, bool is_signed) {
-    std::stringstream os;
-
-    TIFF *tiff = TIFFStreamOpen("x", (std::ostream *) &os);
-    WriteTIFF(tiff, buff, cols, lines, elem_size, is_signed);
-    TIFFClose(tiff);
-
-    return os.str();
-}
-
-void WriteTIFFToFile(const std::string &filename, void *buff, size_t cols, size_t lines, size_t elem_size,
-                     bool is_signed) {
-    TIFF *tiff = TIFFOpen(filename.c_str(), "w");
-
-    WriteTIFF(tiff, buff, cols, lines, elem_size, is_signed);
-
-    TIFFClose(tiff);
-}
-
-std::vector<uint32_t> ReadTIFFFromString32(const std::string &s, uint32_t &cols, uint32_t &lines) {
-    uint32_t rows_per_string = 0;
-
-    std::vector<uint32_t> ret;
-
-    uint16_t elem_size;
-
-    std::istringstream input_TIFF_stream(s);
-    TIFF* tiff = TIFFStreamOpen("MemTIFF", &input_TIFF_stream);
-    TIFFGetField(tiff, TIFFTAG_IMAGEWIDTH, &cols);  // get the width of the image
-    TIFFGetField(tiff, TIFFTAG_IMAGELENGTH, &lines);    // get the height of the image
-    TIFFGetField(tiff, TIFFTAG_BITSPERSAMPLE, &elem_size);    // get the size of the channels
-    TIFFGetField(tiff, TIFFTAG_ROWSPERSTRIP, &rows_per_string);
-
-    if (elem_size != 32)
-        throw JFJochException(JFJochExceptionCategory::TIFFGeneratorError, "Only 32-bit format supported");
-
-    ret.resize(cols * lines);
-
-    if (cols * sizeof(uint32_t) != TIFFScanlineSize(tiff))
-        throw JFJochException(JFJochExceptionCategory::TIFFGeneratorError, "TIFFScanlineSize mismatch");
-
-    for (int i = 0; i < lines; i++) {
-        if (TIFFReadScanline(tiff, ret.data() + i * cols, i, 0) < 0)
-            throw JFJochException(JFJochExceptionCategory::TIFFGeneratorError, "TIFFReadScanline error");
-    }
-
-    TIFFClose(tiff);
-    return ret;
-}
-
-std::vector<uint16_t> ReadTIFFFromString16(const std::string &s, uint32_t &cols, uint32_t &lines) {
-    uint32_t rows_per_string = 0;
-
-    std::vector<uint16_t> ret;
-
-    uint16_t elem_size;
-
-    std::istringstream input_TIFF_stream(s);
-    TIFF* tiff = TIFFStreamOpen("MemTIFF", &input_TIFF_stream);
-    TIFFGetField(tiff, TIFFTAG_IMAGEWIDTH, &cols);  // get the width of the image
-    TIFFGetField(tiff, TIFFTAG_IMAGELENGTH, &lines);    // get the height of the image
-    TIFFGetField(tiff, TIFFTAG_BITSPERSAMPLE, &elem_size);    // get the size of the channels
-    TIFFGetField(tiff, TIFFTAG_ROWSPERSTRIP, &rows_per_string);
-
-    if (elem_size != 16)
-        throw JFJochException(JFJochExceptionCategory::TIFFGeneratorError, "Only 16-bit format supported");
-
-    ret.resize(cols * lines);
-
-    if (cols * sizeof(uint16_t) != TIFFScanlineSize(tiff))
-        throw JFJochException(JFJochExceptionCategory::TIFFGeneratorError, "TIFFScanlineSize mismatch");
-
-    for (int i = 0; i < lines; i++) {
-        if (TIFFReadScanline(tiff, ret.data() + i * cols, i, 0) < 0)
-            throw JFJochException(JFJochExceptionCategory::TIFFGeneratorError, "TIFFReadScanline error");
-    }
-
-    TIFFClose(tiff);
-    return ret;
-}

preview/PreviewImage.cpp

@@ -4,8 +4,8 @@
 
 #include <cmath>
 
-#include "JFJochJPEG.h"
-#include "JFJochTIFF.h"
+#include "WriteJPEG.h"
+#include "WriteTIFF.h"
 #include "../common/JFJochException.h"
 #include "../common/DiffractionGeometry.h"
 
@@ -31,7 +31,26 @@ constexpr const static rgb plotly[] = {{0x1f, 0x77, 0xb4},
 constexpr const static rgb indigo = {.r = 0x3f, .g = 0x51, .b = 0xb5};
 constexpr const static rgb gray = {.r = 0xbe, .g = 0xbe, .b = 0xbe};
 
-PreviewImage::PreviewImage(std::chrono::microseconds period) : counter(period) {}
+PreviewImage::PreviewImage(const DiffractionExperiment &in_experiment) :
+        experiment(in_experiment),
+        xpixel(experiment.GetXPixelsNum()),
+        ypixel(experiment.GetYPixelsNum()),
+        beam_x(experiment.GetBeamX_pxl()),
+        beam_y(experiment.GetBeamY_pxl()),
+        pixel_depth_bytes(experiment.GetPixelDepth()),
+        pixel_is_signed(experiment.IsPixelSigned()),
+        uncompressed_image(experiment.GetPixelsNum() * experiment.GetPixelDepth()),
+        roi_map(experiment.ROI()),
+        counter(experiment.GetPreviewPeriod()) {}
+
+void PreviewImage::UpdateImage(const void *in_uncompressed_image,
+                               const std::vector<SpotToSave> &in_spots) {
+    if (counter.GeneratePreview()) {
+        std::unique_lock<std::mutex> ul(m);
+        memcpy(uncompressed_image.data(), in_uncompressed_image, xpixel * ypixel * pixel_depth_bytes);
+        spots = in_spots;
+    }
+}
 
 void colormap(std::vector<unsigned char>& ret, float v, size_t pixel) {
     if ((v < 0.0) || (v > 1.0)) {
@@ -85,6 +104,48 @@ std::vector<unsigned char> GenerateRGB(const T* value, size_t npixel, uint32_t s
     return ret;
 }
 
+std::string PreviewImage::GenerateJPEG(const PreviewJPEGSettings &settings) const {
+    std::vector<unsigned char> v;
+    {
+        // JPEG compression is outside the critical loop protected by m
+        std::unique_lock<std::mutex> ul(m);
+        if (!pixel_is_signed) {
+            if (pixel_depth_bytes == 2)
+                v = GenerateRGB<uint16_t>((uint16_t *) uncompressed_image.data(), xpixel * ypixel,
+                                         settings.saturation_value, UINT16_MAX);
+            else
+                v = GenerateRGB<uint32_t>((uint32_t *) uncompressed_image.data(), xpixel * ypixel,
+                                         settings.saturation_value, UINT32_MAX);
+        } else {
+            if (pixel_depth_bytes == 2)
+                v = GenerateRGB<int16_t>((int16_t *) uncompressed_image.data(), xpixel * ypixel,
+                                         settings.saturation_value, INT16_MIN);
+            else
+                v = GenerateRGB<int32_t>((int32_t *) uncompressed_image.data(), xpixel * ypixel,
+                                         settings.saturation_value, INT32_MIN);
+        }
+        if (settings.show_spots)
+            AddSpots(v);
+    }
+
+    if (settings.show_roi)
+        AddROI(v);
+
+    if (settings.resolution_ring)
+        AddResolutionRing(v, settings.resolution_ring.value());
+
+    AddBeamCenter(v);
+
+    return WriteJPEGToMem(v, xpixel, ypixel, settings.jpeg_quality);
+}
+
+std::string PreviewImage::GenerateTIFF() const {
+    std::unique_lock<std::mutex> ul(m);
+    std::string s = WriteTIFFToString(const_cast<uint8_t *>(uncompressed_image.data()),
+                                      xpixel, ypixel, pixel_depth_bytes, pixel_is_signed);
+    return s;
+}
+
 template <class T>
 std::vector<uint16_t> GenerateDioptasPreview(const void* input, size_t xpixel, size_t ypixel, T special_value) {
     auto input_ptr = (T *) input;
@@ -104,6 +165,24 @@ std::vector<uint16_t> GenerateDioptasPreview(const void* input, size_t xpixel, s
     return vec;
 }
 
+std::string PreviewImage::GenerateTIFFDioptas() const {
+    std::unique_lock<std::mutex> ul(m);
+
+    std::vector<uint16_t> vec;
+    if (pixel_is_signed) {
+        if (pixel_depth_bytes == 2)
+            vec = GenerateDioptasPreview<int16_t>(uncompressed_image.data(), xpixel, ypixel, INT16_MIN);
+        else
+            vec = GenerateDioptasPreview<int32_t>(uncompressed_image.data(), xpixel, ypixel, INT32_MIN);
+    } else {
+        if (pixel_depth_bytes == 2)
+            vec = GenerateDioptasPreview<uint16_t>(uncompressed_image.data(), xpixel, ypixel, UINT16_MAX);
+        else
+            vec = GenerateDioptasPreview<uint32_t>(uncompressed_image.data(), xpixel, ypixel, UINT32_MAX);
+    }
+    return WriteTIFFToString(vec.data(), xpixel, ypixel, 2, false);
+}
+
 void PreviewImage::AddBeamCenter(std::vector<uint8_t> &rgb_image) const {
     size_t beam_x_int = std::lround(beam_x);
     size_t beam_y_int = std::lround(beam_y);
@@ -140,7 +219,7 @@ void PreviewImage::AddSpots(std::vector<uint8_t> &rgb_image) const {
 void PreviewImage::AddROI(std::vector<uint8_t> &rgb_image) const {
     int64_t roi_counter = 0;
 
-    for (const auto &box: experiment.ROI().GetROIBox()) {
+    for (const auto &box: roi_map.GetROIBox()) {
         int rectangle_width = 5;
 
         for (auto x = box.GetXMin() - rectangle_width; x <= box.GetXMax() + rectangle_width; x++) {
@@ -159,7 +238,7 @@ void PreviewImage::AddROI(std::vector<uint8_t> &rgb_image) const {
         roi_counter++;
     }
 
-    for (const auto &circle: experiment.ROI().GetROICircle()) {
+    for (const auto &circle: roi_map.GetROICircle()) {
         int width = 5;
 
         for (int64_t y = std::floor(circle.GetY() - circle.GetRadius_pxl() - width);
@@ -191,132 +270,3 @@ void PreviewImage::AddResolutionRing(std::vector<uint8_t> &rgb_image, float d) c
         }
     }
 }
-
-void PreviewImage::AddUserMask(std::vector<uint8_t> &rgb_image) const {
-    for (int y = 0; y < ypixel; y++) {
-        for (int x = 0; x < xpixel; x++) {
-            if (user_mask[y * xpixel + x] != 0)
-                color_pixel(rgb_image, x, y, gray);
-        }
-    }
-}
-
-void PreviewImage::Configure(const DiffractionExperiment &in_experiment, const PixelMask& mask) {
-    std::unique_lock<std::mutex> ul(m);
-
-    experiment = in_experiment;
-    initialized = false;
-    xpixel = experiment.GetXPixelsNum();
-    ypixel = experiment.GetYPixelsNum();
-    beam_x = experiment.GetBeamX_pxl();
-    beam_y = experiment.GetBeamY_pxl();
-    pixel_depth_bytes = experiment.GetPixelDepth();
-    pixel_is_signed = experiment.IsPixelSigned();
-    uncompressed_image.resize(experiment.GetPixelsNum() * experiment.GetPixelDepth());
-    memset(uncompressed_image.data(), 0, experiment.GetPixelsNum() * experiment.GetPixelDepth());
-    user_mask = mask.GetUserMask(experiment);
-    if (user_mask.size() != experiment.GetPixelsNum())
-        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "User pixel mask size incorrect");
-}
-
-void PreviewImage::Configure() {
-    std::unique_lock<std::mutex> ul(m);
-
-    initialized = false;
-    xpixel = 0;
-    ypixel = 0;
-}
-
-void PreviewImage::UpdateImage(const void *in_uncompressed_image,
-                               const std::vector<SpotToSave> &in_spots) {
-    if (counter.GeneratePreview()) {
-        std::unique_lock<std::mutex> ul(m);
-
-        if (xpixel * ypixel == 0)
-            throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Preview not configured");
-
-        initialized = true;
-        memcpy(uncompressed_image.data(), in_uncompressed_image, xpixel * ypixel * pixel_depth_bytes);
-        spots = in_spots;
-    }
-}
-
-std::string PreviewImage::GenerateJPEG(const PreviewJPEGSettings &settings) const {
-    std::vector<unsigned char> v;
-    size_t local_xpixel;
-    size_t local_ypixel;
-
-    {
-        // JPEG compression is outside the critical loop protected by m
-        std::unique_lock<std::mutex> ul(m);
-
-        local_xpixel = xpixel;
-        local_ypixel = ypixel;
-
-        if (!initialized)
-            return {};
-
-        if (!pixel_is_signed) {
-            if (pixel_depth_bytes == 2)
-                v = GenerateRGB<uint16_t>((uint16_t *) uncompressed_image.data(), xpixel * ypixel,
-                                          settings.saturation_value, UINT16_MAX);
-            else
-                v = GenerateRGB<uint32_t>((uint32_t *) uncompressed_image.data(), xpixel * ypixel,
-                                          settings.saturation_value, UINT32_MAX);
-        } else {
-            if (pixel_depth_bytes == 2)
-                v = GenerateRGB<int16_t>((int16_t *) uncompressed_image.data(), xpixel * ypixel,
-                                         settings.saturation_value, INT16_MIN);
-            else
-                v = GenerateRGB<int32_t>((int32_t *) uncompressed_image.data(), xpixel * ypixel,
-                                         settings.saturation_value, INT32_MIN);
-        }
-
-        if (settings.show_user_mask)
-            AddUserMask(v);
-
-        if (settings.show_spots)
-            AddSpots(v);
-
-        if (settings.show_roi)
-            AddROI(v);
-
-        if (settings.resolution_ring)
-            AddResolutionRing(v, settings.resolution_ring.value());
-
-        AddBeamCenter(v);
-    }
-
-    return WriteJPEGToMem(v, local_xpixel, local_ypixel, settings.jpeg_quality);
-}
-
-
-std::string PreviewImage::GenerateTIFF() const {
-    std::unique_lock<std::mutex> ul(m);
-    if (!initialized)
-        return {};
-
-    std::string s = WriteTIFFToString(const_cast<uint8_t *>(uncompressed_image.data()),
-                                      xpixel, ypixel, pixel_depth_bytes, pixel_is_signed);
-    return s;
-}
-
-std::string PreviewImage::GenerateTIFFDioptas() const {
-    std::unique_lock<std::mutex> ul(m);
-    if (!initialized)
-        return {};
-
-    std::vector<uint16_t> vec;
-    if (pixel_is_signed) {
-        if (pixel_depth_bytes == 2)
-            vec = GenerateDioptasPreview<int16_t>(uncompressed_image.data(), xpixel, ypixel, INT16_MIN);
-        else
-            vec = GenerateDioptasPreview<int32_t>(uncompressed_image.data(), xpixel, ypixel, INT32_MIN);
-    } else {
-        if (pixel_depth_bytes == 2)
-            vec = GenerateDioptasPreview<uint16_t>(uncompressed_image.data(), xpixel, ypixel, UINT16_MAX);
-        else
-            vec = GenerateDioptasPreview<uint32_t>(uncompressed_image.data(), xpixel, ypixel, UINT32_MAX);
-    }
-    return WriteTIFFToString(vec.data(), xpixel, ypixel, 2, false);
-}

preview/PreviewImage.h

@@ -8,7 +8,6 @@
 
 #include "../common/SpotToSave.h"
 #include "../common/DiffractionExperiment.h"
-#include "../common/PixelMask.h"
 #include "PreviewCounter.h"
 
 struct PreviewJPEGSettings {
@@ -17,7 +16,6 @@ struct PreviewJPEGSettings {
     bool show_spots = true;
     bool show_roi = false;
     bool show_indexed = false;
-    bool show_user_mask = false;
     std::optional<float> resolution_ring;
 };
 
@@ -31,22 +29,20 @@ class PreviewImage {
     mutable std::mutex m;
     DiffractionExperiment experiment;
 
-    bool initialized = false;
+    const ROIMap roi_map;
     std::vector<uint8_t> uncompressed_image;
-    std::vector<uint32_t> user_mask;
     std::vector<SpotToSave> spots;
-    size_t xpixel = 0;
-    size_t ypixel = 0;
-    size_t pixel_depth_bytes = 2;
-    bool pixel_is_signed = false;
-    float beam_x = 0;
-    float beam_y = 0;
+    size_t xpixel;
+    size_t ypixel;
+    size_t pixel_depth_bytes;
+    bool pixel_is_signed;
+    float beam_x;
+    float beam_y;
 
     void AddResolutionRing(std::vector<uint8_t> &rgb_image, float d) const;
     void AddBeamCenter(std::vector<uint8_t> &rgb_image) const;
     void AddSpots(std::vector<uint8_t> &rgb_image) const;
     void AddROI(std::vector<uint8_t> &rgb_image) const;
-    void AddUserMask(std::vector<uint8_t> &rgb_image) const;
     PreviewCounter counter;
 
     void color_pixel(std::vector<unsigned char>& ret, int64_t xpixel, int64_t ypixel, const rgb &color) const;
@@ -55,9 +51,7 @@ class PreviewImage {
     void beam_center_mark(std::vector<unsigned char>& ret, int64_t xpixel, int64_t ypixel) const;
 
 public:
-    explicit PreviewImage(std::chrono::microseconds period = std::chrono::seconds(1));
-    void Configure(const DiffractionExperiment& experiment, const PixelMask& mask);
-    void Configure();
+    explicit PreviewImage(const DiffractionExperiment& experiment);
     void UpdateImage(const void *uncompressed_image, const std::vector<SpotToSave> &spots);
     [[nodiscard]] std::string GenerateJPEG(const PreviewJPEGSettings& settings) const;
     [[nodiscard]] std::string GenerateTIFF() const;

preview/WriteJPEG.cpp

@@ -7,7 +7,7 @@
 #include <jpeglib.h>
 
 #include "../common/JFJochException.h"
-#include "JFJochJPEG.h"
+#include "WriteJPEG.h"
 
 
 std::string WriteJPEGToMem(const std::vector<uint8_t> &input, size_t width, size_t height, int quality) {

preview/WriteTIFF.cpp

@@ -0,0 +1,48 @@
+// Copyright (2019-2024) Paul Scherrer Institute
+
+#include <tiffio.h>
+#include <tiffio.hxx>
+#include <sstream>
+
+#include "WriteTIFF.h"
+#include "../common/JFJochException.h"
+
+
+void WriteTIFF(TIFF *tiff, void *buff, size_t cols, size_t lines, size_t elem_size, bool is_signed) {
+
+    if (tiff == nullptr)
+        throw JFJochException(JFJochExceptionCategory::TIFFGeneratorError, "TIFFStreamOpen error");
+
+    TIFFSetField(tiff, TIFFTAG_IMAGEWIDTH, cols);  // set the width of the image
+    TIFFSetField(tiff, TIFFTAG_IMAGELENGTH, lines);    // set the height of the image
+    TIFFSetField(tiff, TIFFTAG_SAMPLESPERPIXEL, 1);   // set number of channels per pixel
+    TIFFSetField(tiff, TIFFTAG_BITSPERSAMPLE, elem_size * 8);    // set the size of the channels
+    TIFFSetField(tiff, TIFFTAG_COMPRESSION, COMPRESSION_NONE); // setc ompression to LZW
+    TIFFSetField(tiff, TIFFTAG_ROWSPERSTRIP, lines);
+    if (is_signed)
+        TIFFSetField(tiff, TIFFTAG_SAMPLEFORMAT, SAMPLEFORMAT_INT);
+    else
+        TIFFSetField(tiff, TIFFTAG_SAMPLEFORMAT, SAMPLEFORMAT_UINT);
+
+    if (TIFFWriteEncodedStrip(tiff, 0, buff, cols * lines * elem_size) < 0)
+        throw JFJochException(JFJochExceptionCategory::TIFFGeneratorError, "TIFFWriteEncodedStrip error");
+}
+
+std::string WriteTIFFToString(void *buff, size_t cols, size_t lines, size_t elem_size, bool is_signed) {
+    std::stringstream os;
+
+    TIFF *tiff = TIFFStreamOpen("x", (std::ostream *) &os);
+    WriteTIFF(tiff, buff, cols, lines, elem_size, is_signed);
+    TIFFClose(tiff);
+
+    return os.str();
+}
+
+void WriteTIFFToFile(const std::string &filename, void *buff, size_t cols, size_t lines, size_t elem_size,
+                     bool is_signed) {
+    TIFF *tiff = TIFFOpen(filename.c_str(), "w");
+
+    WriteTIFF(tiff, buff, cols, lines, elem_size, is_signed);
+
+    TIFFClose(tiff);
+}

preview/WriteTIFF.h

@@ -4,12 +4,9 @@
 #define JUNGFRAUJOCH_WRITETIFF_H
 
 #include <string>
-#include <vector>
 
 std::string WriteTIFFToString(void *buff, size_t cols, size_t lines, size_t elem_size, bool is_signed = false);
 void WriteTIFFToFile(const std::string &filename, void *buff, size_t cols, size_t lines, size_t elem_size,
                      bool is_signed = false);
-std::vector<uint32_t> ReadTIFFFromString32(const std::string& s, uint32_t &cols, uint32_t &lines);
-std::vector<uint16_t> ReadTIFFFromString16(const std::string& s, uint32_t &cols, uint32_t &lines);
 
 #endif //JUNGFRAUJOCH_WRITETIFF_H

receiver/CMakeLists.txt

@@ -11,8 +11,8 @@ ADD_LIBRARY(JFJochReceiver STATIC
         LossyFilter.cpp
         LossyFilter.h)
 
-TARGET_LINK_LIBRARIES(JFJochReceiver ImagePusher JFJochImageAnalysis JFJochAcquisitionDevice JFJochCommon JFJochHLSSimulation JFJochPreview)
+TARGET_LINK_LIBRARIES(JFJochReceiver ImagePusher JFJochImageAnalysis JFJochAcquisitionDevice JFJochCommon JFJochHLSSimulation JFJochPreview JFJochResonet)
 
 ADD_EXECUTABLE(jfjoch_action_test jfjoch_action_test.cpp)
 TARGET_LINK_LIBRARIES(jfjoch_action_test JFJochReceiver)
-INSTALL(TARGETS jfjoch_action_test RUNTIME COMPONENT jfjoch)
+INSTALL(TARGETS jfjoch_action_test RUNTIME)

receiver/JFJochReceiver.cpp

@@ -7,19 +7,17 @@
 #include "../image_analysis/MXAnalyzer.h"
 #include "../common/DiffractionGeometry.h"
 #include "ImageMetadata.h"
+#include "../resonet/NeuralNetResPredictor.h"
 #include "../common/time_utc.h"
 #include "../common/PixelMask.h"
 
 JFJochReceiver::JFJochReceiver(const DiffractionExperiment& in_experiment,
-                               const PixelMask &pixel_mask,
                                const JFCalibration *in_calibration,
                                AcquisitionDeviceGroup &in_aq_device,
                                ImagePusher &in_image_sender,
                                Logger &in_logger, int64_t in_forward_and_sum_nthreads,
                                const NUMAHWPolicy &in_numa_policy,
                                const SpotFindingSettings &in_spot_finding_settings,
-                               PreviewImage &in_preview_image,
-                               PreviewImage &in_preview_image_indexed,
                                SendBuffer &buf) :
         experiment(in_experiment),
         calibration(nullptr),
@@ -36,10 +34,9 @@ JFJochReceiver::JFJochReceiver(const DiffractionExperiment& in_experiment,
         az_int_mapping(experiment),
         plots(experiment, az_int_mapping),
         spot_finding_settings(in_spot_finding_settings),
-        preview_image(in_preview_image),
-        preview_image_indexed(in_preview_image_indexed),
-        serialmx_filter(experiment),
-        user_mask_raw_coord(pixel_mask.GetUserMask(experiment, false))
+        preview_image(experiment),
+        preview_image_indexed(experiment),
+        serialmx_filter(experiment)
 {
     if (experiment.GetDetectorSetup().GetDetectorType() == DetectorType::JUNGFRAU)
         calibration = in_calibration;
@@ -64,16 +61,15 @@ JFJochReceiver::JFJochReceiver(const DiffractionExperiment& in_experiment,
     logger.Info("Data acquisition devices ready");
 
     if (experiment.GetImageNum() > 0) {
-        SendStartMessage(pixel_mask);
+        SendStartMessage();
         SendCalibration();
 
         for (int i = 0; i < experiment.GetImageNum(); i++)
             images_to_go.Put(i);
 
         // Setup frames summation and forwarding
-        for (uint32_t i = 0; i < frame_transformation_nthreads; i++) {
-            auto handle = std::async(std::launch::async, &JFJochReceiver::FrameTransformationThread,
-                                     this, i);
+        for (int i = 0; i < frame_transformation_nthreads; i++) {
+            auto handle = std::async(std::launch::async, &JFJochReceiver::FrameTransformationThread, this);
             frame_transformation_futures.emplace_back(std::move(handle));
         }
 
@@ -92,7 +88,7 @@ JFJochReceiver::JFJochReceiver(const DiffractionExperiment& in_experiment,
     logger.Info("Receiving data started");
 }
 
-void JFJochReceiver::SendStartMessage(const PixelMask &pixel_mask) {
+void JFJochReceiver::SendStartMessage() {
     if (!push_images_to_writer)
         return;
 
@@ -103,6 +99,11 @@ void JFJochReceiver::SendStartMessage(const PixelMask &pixel_mask) {
     message.az_int_bin_to_q = az_int_mapping.GetBinToQ();
     message.write_master_file = true;
 
+    PixelMask pixel_mask(experiment);
+
+    if (calibration)
+        pixel_mask.LoadDetectorBadPixelMask(calibration->CalculateMask(), 1);
+
     std::vector<uint32_t> nexus_mask;
     nexus_mask = pixel_mask.GetMask(experiment);
 
@@ -177,10 +178,6 @@ void JFJochReceiver::AcquireThread(uint16_t data_stream) {
         std::vector<float> tmp(RAW_MODULE_SIZE);
         for (int m = 0; m < experiment.GetModulesNum(data_stream); m++) {
             CalcSpotFinderResolutionMap(tmp.data(), experiment, m_offset + m);
-            for (int i = 0; i < RAW_MODULE_SIZE; i++) {
-                if (user_mask_raw_coord[(m_offset + m) * RAW_MODULE_SIZE + i] != 0)
-                    tmp[i] = 0.0f;
-            }
             acquisition_device[data_stream].InitializeSpotFinderResolutionMap(tmp.data(), m);
 
             CalcAzIntCorrRawCoord(tmp.data(), experiment, m_offset + m);
@@ -259,22 +256,21 @@ void JFJochReceiver::RetrievePedestal() {
     }
 }
 
-void JFJochReceiver::FrameTransformationThread(uint32_t threadid) {
-
-    std::unique_ptr<MXAnalyzer> analyzer;
+void JFJochReceiver::FrameTransformationThread() {
 
     try {
-        numa_policy.Bind(threadid);
-        analyzer = std::make_unique<MXAnalyzer>(experiment);
+        numa_policy.Bind();
     } catch (const JFJochException &e) {
         frame_transformation_ready.count_down();
-        logger.Error("Thread setup error {}", e.what());
+        logger.Error("HW bind error {}", e.what());
         Cancel(e);
-        return;
     }
 
+    NeuralNetResPredictor neural_net_predictor(experiment.GetNeuralNetModelPath());
     FrameTransformation transformation(experiment);
 
+    MXAnalyzer analyzer(experiment);
+
     frame_transformation_ready.count_down();
 
     uint16_t az_int_min_bin = std::floor(az_int_mapping.QToBin(experiment.GetLowQForBkgEstimate_recipA()));
@@ -319,7 +315,7 @@ void JFJochReceiver::FrameTransformationThread(uint32_t threadid) {
                         adu_histogram_module[module_abs_number].Add(*output);
                         az_int_profile_image.Add(*output);
 
-                        analyzer->ReadFromFPGA(output, local_spot_finding_settings, module_abs_number);
+                        analyzer.ReadFromFPGA(output, local_spot_finding_settings, module_abs_number);
 
                         transformation.ProcessModule(output, d);
                     } else
@@ -338,11 +334,12 @@ void JFJochReceiver::FrameTransformationThread(uint32_t threadid) {
                 continue;
             }
 
-            analyzer->Process(message, local_spot_finding_settings);
+            analyzer.Process(message, local_spot_finding_settings);
 
             message.receiver_free_send_buf = send_buf_ctrl.GetAvailBufLocations();
             message.az_int_profile = az_int_profile_image.GetResult();
             message.bkg_estimate = az_int_profile_image.GetMeanValueOfBins(az_int_min_bin, az_int_max_bin);
+            message.resolution_estimation = neural_net_predictor.Inference(experiment, transformation.GetImage());
 
             plots.Add(message, image_number % experiment.GetTimePointNumber(), az_int_profile_image);
 
@@ -375,12 +372,12 @@ void JFJochReceiver::FrameTransformationThread(uint32_t threadid) {
             size_t image_size = transformation.CompressImage(writer_buffer + serializer.GetImageAppendOffset());
             serializer.AppendImage(image_size);
             compressed_size += image_size;
-            image_pusher.SendImage(writer_buffer, serializer.GetBufferSize(), message.number, loc);
+            image_pusher.SendImage(writer_buffer, serializer.GetBufferSize(), image_number, loc);
 
             images_sent++; // Handle case when image not sent properly
-            UpdateMaxImage(message.number);
+            UpdateMaxImage(image_number);
 
-            logger.Debug("Frame transformation thread - done sending image {} / {}", image_number, message.number);
+            logger.Debug("Frame transformation thread - done sending image {}", image_number);
         } catch (const JFJochException &e) {
             logger.ErrorException(e);
             Cancel(e);
@@ -558,10 +555,10 @@ JFJochReceiverStatus JFJochReceiver::GetStatus() const {
 
     if ((experiment.GetImageNum() > 0) && (compressed_size > 0)) {
         ret.compressed_ratio = static_cast<double> ((images_sent + images_skipped)
-                                                    * experiment.GetPixelDepth()
-                                                    * experiment.GetModulesNum()
-                                                    * RAW_MODULE_SIZE)
-                               / static_cast<double> (compressed_size);
+                                                * experiment.GetPixelDepth()
+                                                * experiment.GetModulesNum()
+                                                * RAW_MODULE_SIZE)
+                           / static_cast<double> (compressed_size);
     }
 
     ret.progress = GetProgress();
@@ -576,6 +573,20 @@ JFJochReceiverStatus JFJochReceiver::GetStatus() const {
     return ret;
 }
 
+std::string JFJochReceiver::GetTIFF(bool calibration_run) const {
+    if (calibration_run)
+        return preview_image.GenerateTIFFDioptas();
+    else
+        return preview_image.GenerateTIFF();
+}
+
+std::string JFJochReceiver::GetJPEG(const PreviewJPEGSettings &settings) const {
+    if (settings.show_indexed)
+        return preview_image_indexed.GenerateJPEG(settings);
+    else
+        return preview_image.GenerateJPEG(settings);
+}
+
 void JFJochReceiver::GetXFELEventCode(std::vector<uint64_t> &v) const {
     if (experiment.IsPulsedSource())
         plots.GetXFELEventCode(v);

receiver/JFJochReceiver.h

@@ -19,7 +19,6 @@
 #include "../common/ThreadSafeFIFO.h"
 #include "../common/NUMAHWPolicy.h"
 #include "../common/SendBufferControl.h"
-#include "../common/PixelMask.h"
 
 #include "../image_analysis/StrongPixelSet.h"
 #include "../image_analysis/AzimuthalIntegrationMapping.h"
@@ -109,34 +108,29 @@ class JFJochReceiver {
 
     NUMAHWPolicy numa_policy;
 
-    PreviewImage &preview_image;
-    PreviewImage &preview_image_indexed;
+    PreviewImage preview_image;
+    PreviewImage preview_image_indexed;
 
     LossyFilter serialmx_filter;
 
-    std::vector<uint32_t> user_mask_raw_coord;
-
     void AcquireThread(uint16_t data_stream);
-    void FrameTransformationThread(uint32_t threadid);
+    void FrameTransformationThread();
     void Cancel(const JFJochException &e);
     void FinalizeMeasurement();
     void RetrievePedestal();
     SpotFindingSettings GetSpotFindingSettings();
     void UpdateMaxImage(uint64_t image_number);
     void UpdateMaxDelay(uint64_t delay);
-    void SendStartMessage(const PixelMask &pixel_mask);
+    void SendStartMessage();
     void SendCalibration();
 public:
     JFJochReceiver(const DiffractionExperiment& experiment,
-                   const PixelMask &pixel_mask,
                    const JFCalibration *calibration,
                    AcquisitionDeviceGroup &acquisition_devices,
                    ImagePusher &image_pusher,
                    Logger &logger, int64_t forward_and_sum_nthreads,
                    const NUMAHWPolicy &numa_policy,
                    const SpotFindingSettings &spot_finding_settings,
-                   PreviewImage &preview_image,
-                   PreviewImage &preview_image_indexed,
                    SendBuffer &buffer);
     ~JFJochReceiver();
     JFJochReceiver(const JFJochReceiver &other) = delete;
@@ -154,6 +148,9 @@ public:
 
     MultiLinePlot GetPlots(const PlotRequest& request);
 
+    std::string GetTIFF(bool calibration) const;
+    std::string GetJPEG(const PreviewJPEGSettings &settings) const;
+
     void GetXFELEventCode(std::vector<uint64_t> &v) const;
     void GetXFELPulseID(std::vector<uint64_t> &v) const;
 };

receiver/JFJochReceiverPlots.cpp

@@ -6,7 +6,7 @@ JFJochReceiverPlots::JFJochReceiverPlots(const DiffractionExperiment &experiment
                                          const AzimuthalIntegrationMapping &mapping)
         : indexing_solution_per_time_point(experiment.GetTimePointNumber()),
           default_binning(experiment.GetDefaultPlotBinning()),
-          az_int_profile(mapping) {
+          az_int_profile(mapping), resolution_estimation(50, 1.0, 5.0) {
     az_int_profile.SetTitle("dataset");
     for (int i = 0; i < experiment.GetTimePointNumber(); i++) {
         az_int_profile_per_time_point.emplace_back(mapping);
@@ -23,24 +23,30 @@ JFJochReceiverPlots::JFJochReceiverPlots(const DiffractionExperiment &experiment
 void JFJochReceiverPlots::Add(const DataMessage &msg, uint64_t file_number, const AzimuthalIntegrationProfile &profile) {
     bkg_estimate.AddElement(msg.number, msg.bkg_estimate);
     spot_count.AddElement(msg.number, msg.spots.size());
-    spot_count_in_rings.AddElement(msg.number, msg.spot_count_in_rings);
-
     saturated_pixels.AddElement(msg.number, msg.saturated_pixel_count);
     error_pixels.AddElement(msg.number, msg.error_pixel_count);
     strong_pixels.AddElement(msg.number, msg.strong_pixel_count);
 
     image_collection_efficiency.AddElement(msg.number, msg.image_collection_efficiency);
-    receiver_delay.AddElement(msg.number, msg.receiver_aq_dev_delay);
-    receiver_free_send_buf.AddElement(msg.number, msg.receiver_free_send_buf);
+    if (msg.receiver_aq_dev_delay)
+        receiver_delay.AddElement(msg.number, msg.receiver_aq_dev_delay.value());
+    if (msg.receiver_free_send_buf)
+        receiver_free_send_buf.AddElement(msg.number, msg.receiver_free_send_buf.value());
 
     indexing_solution.AddElement(msg.number, msg.indexing_result);
     indexing_solution_per_time_point.Add(file_number, msg.indexing_result);
 
+    if (msg.resolution_estimation)
+        resolution_estimation.Add(msg.resolution_estimation.value());
+
     az_int_profile += profile;
     az_int_profile_per_time_point.at(file_number) += profile;
 
-    xfel_pulse_id.AddElement(msg.number, msg.xfel_pulse_id);
-    xfel_event_code.AddElement(msg.number, msg.xfel_event_code);
+    if (msg.xfel_pulse_id)
+        xfel_pulse_id.AddElement(msg.number, msg.xfel_pulse_id.value());
+
+    if (msg.xfel_event_code)
+        xfel_event_code.AddElement(msg.number, msg.xfel_event_code.value());
 
     for (const auto &[key, value] : msg.roi) {
         if (roi_sum.contains(key) && roi_sum[key])
@@ -66,15 +72,7 @@ MultiLinePlot JFJochReceiverPlots::GetPlots(const PlotRequest &request) {
         case PlotType::RadInt:
             return az_int_profile.GetPlot();
         case PlotType::SpotCount:
-            tmp = spot_count.GetMeanPlot(nbins);
-            tmp.at(0).title = "Spots (crystal)";
-            ret.emplace_back(tmp.at(0));
-
-            tmp = spot_count_in_rings.GetMeanPlot(nbins);
-            tmp.at(0).title = "Spots (rings)";
-            ret.emplace_back(tmp.at(0));
-
-            return ret;
+            return spot_count.GetMeanPlot(nbins);
         case PlotType::IndexingRate:
             return indexing_solution.GetMeanPlot(nbins);
         case PlotType::BkgEstimate:
@@ -123,6 +121,8 @@ MultiLinePlot JFJochReceiverPlots::GetPlots(const PlotRequest &request) {
                 ret.emplace_back(tmp.at(0));
             }
             return ret;
+        case PlotType::ResEstimation:
+            return resolution_estimation.GetPlot();
         case PlotType::RadIntPerTimePoint:
             tmp = az_int_profile.GetPlot();
             ret.emplace_back(tmp.at(0));

receiver/JFJochReceiverPlots.h

@@ -18,7 +18,6 @@ class JFJochReceiverPlots {
 
     StatusVector<float> bkg_estimate;
     StatusVector<uint64_t> spot_count;
-    StatusVector<uint64_t> spot_count_in_rings;
     StatusVector<uint64_t> indexing_solution;
     StatusVector<uint64_t> saturated_pixels;
     StatusVector<uint64_t> error_pixels;
@@ -26,11 +25,13 @@ class JFJochReceiverPlots {
     StatusVector<uint64_t> receiver_delay;
     StatusVector<uint64_t> receiver_free_send_buf;
     StatusVector<float> image_collection_efficiency;
+    StatusVector<float> unit_cell[6];
     SetAverage<uint64_t> indexing_solution_per_time_point;
 
     std::map<std::string, std::unique_ptr<StatusVector<int64_t>>> roi_sum;
     std::map<std::string, std::unique_ptr<StatusVector<int64_t>>> roi_max_count;
     std::map<std::string, std::unique_ptr<StatusVector<uint64_t>>> roi_pixels;
+    FloatHistogram resolution_estimation;
 
     AzimuthalIntegrationProfile az_int_profile;
     std::vector<AzimuthalIntegrationProfile> az_int_profile_per_time_point;

receiver/JFJochReceiverService.cpp

@@ -45,26 +45,18 @@ std::optional<JFJochReceiverStatus> JFJochReceiverService::GetStatus() {
         return {};
 }
 
-void JFJochReceiverService::Start(const DiffractionExperiment &experiment,
-                                  const PixelMask &pixel_mask,
-                                  const JFCalibration *calibration) {
+void JFJochReceiverService::Start(const DiffractionExperiment &experiment, const JFCalibration *calibration) {
     std::unique_lock ul_state(state_mutex); // unique lock, as it will destroy and create receiver object
     if (state != ReceiverState::Idle)
         throw JFJochException(JFJochExceptionCategory::WrongDAQState, "Receiver not idle, cannot start");
 
     try {
-        preview_image.Configure(experiment, pixel_mask);
-        preview_image_indexed.Configure(experiment, pixel_mask);
-
         // Thanks to properties of unique_ptr, starting new measurement will call destructor of JFJochReceiver, which will
         // ensure that everything was rolled back
-        receiver = std::make_unique<JFJochReceiver>(experiment, pixel_mask,
-                                                    calibration,
+        receiver = std::make_unique<JFJochReceiver>(experiment, calibration,
                                                     aq_devices, image_pusher,
                                                     logger, nthreads, numa_policy,
                                                     spot_finding_settings,
-                                                    preview_image,
-                                                    preview_image_indexed,
                                                     buffer);
         try {
             // Don't want to stop
@@ -136,17 +128,19 @@ std::vector<AcquisitionDeviceNetConfig> JFJochReceiverService::GetNetworkConfig(
 }
 
 std::string JFJochReceiverService::GetTIFF(bool calibration) const {
-    if (calibration)
-        return preview_image.GenerateTIFFDioptas();
+    std::unique_lock ul(state_mutex);
+    if (receiver)
+        return receiver->GetTIFF(calibration);
     else
-        return preview_image.GenerateTIFF();
+        return "";
 }
 
 std::string JFJochReceiverService::GetJPEG(const PreviewJPEGSettings &settings) const {
-    if (settings.show_indexed)
-        return preview_image_indexed.GenerateJPEG(settings);
+    std::unique_lock ul(state_mutex);
+    if (receiver)
+        return receiver->GetJPEG(settings);
     else
-        return preview_image.GenerateJPEG(settings);
+        return "";
 }
 
 void JFJochReceiverService::LoadInternalGeneratorImage(const DiffractionExperiment &experiment,

receiver/JFJochReceiverService.h

@@ -25,9 +25,6 @@ class JFJochReceiverService {
     std::future<void> measurement;
     void FinalizeMeasurement();
     SpotFindingSettings spot_finding_settings;
-
-    PreviewImage preview_image;
-    PreviewImage preview_image_indexed;
 public:
     JFJochReceiverService(AcquisitionDeviceGroup &aq_devices,
                           Logger &logger,
@@ -40,9 +37,7 @@ public:
     void LoadInternalGeneratorImage(const DiffractionExperiment& experiment,
                                     const std::vector<uint16_t> &raw_expected_image,
                                     uint64_t image_number);
-    void Start(const DiffractionExperiment &experiment,
-               const PixelMask &pixel_mask,
-               const JFCalibration *calibration);
+    void Start(const DiffractionExperiment &experiment, const JFCalibration *calibration);
     void Cancel(bool silent);
     JFJochReceiverOutput Stop();
     void SetSpotFindingSettings(const SpotFindingSettings& settings);

receiver/JFJochReceiverTest.cpp

@@ -2,7 +2,7 @@
 
 #include "JFJochReceiverTest.h"
 #include "JFJochReceiverService.h"
-#include "../frame_serialize/ZMQStream2Pusher.h"
+#include "../frame_serialize/ZMQStream2PusherGroup.h"
 #include "../frame_serialize/TestImagePusher.h"
 
 #define STORAGE_CELL_FOR_TEST 11
@@ -54,8 +54,6 @@ bool JFJochReceiverTest(JFJochReceiverOutput &output, Logger &logger,
                         size_t send_buf_size_MiB) {
 
     JFCalibration calib = GeneratePedestalCalibration(x);
-    PixelMask mask(x);
-    mask.LoadDetectorBadPixelMask(calib.CalculateMask());
 
     int64_t image_number = x.GetImageNum() - 1;
 
@@ -77,7 +75,7 @@ bool JFJochReceiverTest(JFJochReceiverOutput &output, Logger &logger,
     settings.max_pix_per_spot = 200;
     service.SetSpotFindingSettings(settings);
 
-    service.Start(x, mask, &calib);
+    service.Start(x, &calib);
     output = service.Stop();
 
     bool no_errors = true;

receiver/jfjoch_action_test.cpp

@@ -19,6 +19,7 @@ void print_usage(Logger &logger) {
     logger.Info("   -i<num>  number of images");
     logger.Info("   -N<num>  number of image processing threads");
     logger.Info("   -P<txt>  NUMA Policy (none|n2g2|n8g4|n8g4_hbm), none is default");
+    logger.Info("   -D<path> use resonet deep learning model for resolution estimation - path to TorchScript");
     logger.Info("   -B<num>  size of send buffer in MiB (default 2048)");
 }
 
@@ -36,6 +37,7 @@ int main(int argc, char **argv) {
     std::string numa_policy_name;
     bool raw_data = false;
     bool force_32bit = false;
+    std::string resonet_path;
     DetectorType detector_type = DetectorType::JUNGFRAU;
     bool hls_simulation = false;
     size_t send_buffer_size_MiB = 2048;
@@ -46,7 +48,7 @@ int main(int argc, char **argv) {
     }
 
     int opt;
-    while ((opt = getopt(argc, argv, "s:i:m:N:P:vRIS:EHB:")) != -1) {
+    while ((opt = getopt(argc, argv, "s:i:m:N:P:vRIS:D:EHB:")) != -1) {
         switch (opt) {
             case 'i':
                 nimages = atol(optarg);
@@ -75,6 +77,9 @@ int main(int argc, char **argv) {
             case 'I':
                 force_32bit = true;
                 break;
+            case 'D':
+                resonet_path = std::string(optarg);
+                break;
             case 'E':
                 detector_type = DetectorType::EIGER;
                 break;
@@ -109,6 +114,8 @@ int main(int argc, char **argv) {
 
     if (force_32bit)
         x.FPGAOutputMode(FPGAPixelOutput::Int32);
+    if (!resonet_path.empty())
+        x.NeuralNetModelPath(resonet_path);
 
     logger.Info("Data streams {} Total modules {} Total images {} Threads {}", nstreams, nmodules, nimages, nthreads);
 

resonet/CMakeLists.txt

@@ -0,0 +1,10 @@
+ADD_LIBRARY(JFJochResonet STATIC NeuralNetResPredictor.cpp NeuralNetResPredictor.h)
+TARGET_LINK_LIBRARIES(JFJochResonet JFJochCommon)
+
+IF (${TORCH_FOUND})
+    TARGET_COMPILE_DEFINITIONS(JFJochResonet PUBLIC -DJFJOCH_USE_TORCH)
+    TARGET_LINK_LIBRARIES(JFJochResonet ${TORCH_LIBRARIES})
+    TARGET_INCLUDE_DIRECTORIES(JFJochResonet PUBLIC ${TORCH_INCLUDE_DIRS})
+    ADD_EXECUTABLE(resonet_test resonet_test.cpp)
+    TARGET_LINK_LIBRARIES(resonet_test JFJochResonet JFJochWriter)
+ENDIF()

resonet/LICENSE.resonet

@@ -0,0 +1,28 @@
+BSD 3-Clause License
+
+Copyright (c) 2023, Macromolecular Crystallography at SSRL
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

resonet/NeuralNetResPredictor.cpp

@@ -0,0 +1,101 @@
+// Copyright (2019-2024) Paul Scherrer Institute
+
+#include "NeuralNetResPredictor.h"
+
+#include <cmath>
+#include "../common/JFJochException.h"
+
+NeuralNetResPredictor::NeuralNetResPredictor(const std::string& model_path)
+        : model_input(512*512),
+          enable(!model_path.empty())
+#ifdef JFJOCH_USE_TORCH
+        , device(torch::kCUDA)
+#endif
+{
+#ifdef JFJOCH_USE_TORCH
+    if (enable) {
+        module = torch::jit::load(model_path);
+        module.to(device);
+    }
+#else
+    enable = false;
+#endif
+}
+
+template<class T>
+void NeuralNetResPredictor::PrepareInternal(const DiffractionExperiment& experiment, const T* image) {
+    size_t pool_factor = GetMaxPoolFactor(experiment);
+    size_t xpixel = experiment.GetXPixelsNum();
+    size_t ypixel = experiment.GetYPixelsNum();
+    size_t start_x = std::lround(experiment.GetBeamX_pxl());
+    size_t start_y = std::lround(experiment.GetBeamY_pxl());
+
+    for (size_t y = 0; y < 512; y++) {
+        size_t y0 = y * pool_factor + start_y;
+        size_t min_yp = std::min(y0 + pool_factor, ypixel);
+        for (size_t x = 0; x < 512; x++) {
+            float val = 0.0;
+            size_t x0 = x * pool_factor + start_x;
+            size_t min_xp = std::min(x0 + pool_factor, ypixel);
+
+            for (size_t yp = y0; yp < min_yp; yp++) {
+                for (size_t xp = x0; xp < min_xp; xp++) {
+                    int16_t pxl = image[yp * xpixel + xp];
+                    if (val < pxl)
+                        val = pxl;
+                }
+            }
+            float max_pool = floorf(sqrtf(val));
+            model_input[512 * y + x] = max_pool;
+        }
+    }
+}
+
+void NeuralNetResPredictor::Prepare(const DiffractionExperiment& experiment, const int16_t *image) {
+    PrepareInternal(experiment, image);
+}
+
+void NeuralNetResPredictor::Prepare(const DiffractionExperiment& experiment, const int32_t *image) {
+    PrepareInternal(experiment, image);
+}
+
+size_t NeuralNetResPredictor::GetMaxPoolFactor(const DiffractionExperiment& experiment) const {
+    float max_direction = std::max(experiment.GetXPixelsNum(), experiment.GetYPixelsNum()) / 2.0;
+    size_t pool_factor = std::lround(max_direction / 512.0f);
+    if (pool_factor <= 0)
+        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Detector size is too small");
+
+    if (pool_factor > 8)
+        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Detector size is too large");
+    return pool_factor;
+}
+
+std::optional<float> NeuralNetResPredictor::Inference(const DiffractionExperiment& experiment, const void *image) {
+    if (!enable)
+        return {};
+#ifdef JFJOCH_USE_TORCH
+    if (experiment.GetPixelDepth() == 2)
+        Prepare(experiment, (int16_t *) image);
+    else
+        Prepare(experiment, (int32_t *) image);
+
+    auto options = torch::TensorOptions().dtype(at::kFloat);
+    auto model_input_tensor = torch::from_blob(model_input.data(), {1,1,512,512}, options).to(device);
+
+    std::vector<torch::jit::IValue> pixels;
+    pixels.emplace_back(std::move(model_input_tensor));
+
+    auto output = module.forward(pixels).toTensor();
+    auto tensor_output = output[0].item<float>();
+    float two_theta = atanf(((2.0f * experiment.GetPixelSize_mm() / experiment.GetDetectorDistance_mm()) * tensor_output));
+    float stheta = sinf(two_theta * 0.5f);
+    float resolution = experiment.GetWavelength_A() / (2.0f * stheta);
+    return resolution;
+#else
+    return {};
+#endif
+}
+
+const std::vector<float> &NeuralNetResPredictor::GetModelInput() const {
+    return model_input;
+}

resonet/NeuralNetResPredictor.h

@@ -0,0 +1,36 @@
+// Copyright (2019-2024) Paul Scherrer Institute
+
+#ifndef JUNGFRAUJOCH_NEURALNETRESPREDICTOR_H
+#define JUNGFRAUJOCH_NEURALNETRESPREDICTOR_H
+
+#ifdef JFJOCH_USE_TORCH
+#include <torch/script.h>
+#endif
+
+#include "../common/DiffractionExperiment.h"
+
+// Based on model described in:
+// Mendez, D., Holton, J. M., Lyubimov, A. Y., Hollatz, S., Mathews, I. I., Cichosz, A., Martirosyan, V.,
+// Zeng, T., Stofer, R., Liu, R., Song, J., McPhillips, S., Soltis, M. & Cohen, A. E. (2024).
+// Acta Cryst. D80, 26-43.
+
+class NeuralNetResPredictor {
+    std::vector<float> model_input;
+    bool enable;
+#ifdef JFJOCH_USE_TORCH
+    torch::Device device;
+    torch::jit::script::Module module;
+#endif
+    template<class T>
+    void PrepareInternal(const DiffractionExperiment& experiment, const T* image);
+public:
+    explicit NeuralNetResPredictor(const std::string& model_path);
+    void Prepare(const DiffractionExperiment& experiment, const int16_t* image);
+    void Prepare(const DiffractionExperiment& experiment, const int32_t* image);
+    std::optional<float> Inference(const DiffractionExperiment& experiment, const void* image);
+    size_t GetMaxPoolFactor(const DiffractionExperiment& experiment) const;
+    const std::vector<float> &GetModelInput() const;
+};
+
+
+#endif //JUNGFRAUJOCH_NEURALNETRESPREDICTOR_H

resonet/resonet_test.cpp

@@ -0,0 +1,51 @@
+// Copyright (2019-2024) Paul Scherrer Institute
+
+#include <iostream>
+
+#include "NeuralNetResPredictor.h"
+#include "../writer/HDF5Objects.h"
+
+int main(int argc, char **argv) {
+    if (argc != 2) {
+        std::cerr << "Usage ./resonet_test <.pt file with traced model>" << std::endl;
+        exit(EXIT_FAILURE);
+    }
+
+    RegisterHDF5Filter();
+
+    DiffractionExperiment experiment(DetectorGeometry(8, 2, 8, 36));
+    experiment.DetectorDistance_mm(75).PhotonEnergy_keV(12.4).BeamY_pxl(1136).BeamX_pxl(1090);
+
+    NeuralNetResPredictor predictor(argv[1]);
+
+    HDF5ReadOnlyFile data("../../tests/test_data/compression_benchmark.h5");
+    HDF5DataSet dataset(data, "/entry/data/data");
+    HDF5DataSpace file_space(dataset);
+
+    std::vector<int16_t> image_conv (file_space.GetDimensions()[1] * file_space.GetDimensions()[2]);
+
+    std::vector<hsize_t> start = {4,0,0};
+    std::vector<hsize_t> file_size = {1, file_space.GetDimensions()[1], file_space.GetDimensions()[2]};
+    dataset.ReadVector(image_conv, start, file_size);
+
+    std::cout << "Max pooling " << predictor.GetMaxPoolFactor(experiment) << std::endl;
+
+    float x = 0;
+    size_t niterations = 25;
+    std::optional<float> result;
+
+    auto start_time = std::chrono::system_clock::now();
+    for (int i = 0; i < niterations; i++) {
+        result =   predictor.Inference(experiment, image_conv.data());
+        if (result)
+            x += result.value();
+    }
+    auto end_time = std::chrono::system_clock::now();
+    auto elapsed = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time);
+    int64_t frequency_Hz = (niterations * 1e6) / (double) (elapsed.count());
+
+    std::cout << ((double) elapsed.count()) / (1e3 * niterations) << " ms" << std::endl;
+    std::cout << frequency_Hz << " Hz" << std::endl;
+    if (result)
+        std::cout << "Resolution " << result.value() << std::endl;
+}