Merge branch '2407-minor-fix' into 'main'

1.0.0-rc.12 Minor fixes

See merge request jungfraujoch/nextgendcu!76

.gitlab-ci.yml

@@ -104,6 +104,7 @@ build:x86:rpm:
     - cd build
     - source /opt/rh/gcc-toolset-12/enable
     - cmake -DCMAKE_BUILD_TYPE=Release ..
+    - make update_version
     - make frontend
     - make -j48 package
     - mv *.rpm ..
@@ -251,6 +252,7 @@ synthesis:vivado_pcie_100g:
         - fpga/pcie_driver/jfjoch_fpga.h
   tags:
     - vivado
+  retry: 1
   artifacts:
     paths:
       - "jfjoch_fpga_pcie_100g.mcs"
@@ -284,6 +286,7 @@ synthesis:vivado_pcie_8x10g:
       allow_failure: true
   tags:
     - vivado
+  retry: 1
   artifacts:
     paths:
       - "jfjoch_fpga_pcie_8x10g.mcs"
@@ -313,8 +316,9 @@ release:
     - build:x86:driver
     - build:x86:rpm
   script:
-    - export PACKAGE_VERSION=`head -n1 VERSION`
-    - export PACKAGE_REGISTRY_URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/jungfraujoch/${PACKAGE_VERSION}"
+    - export PACKAGE_VERSION_SEM=`head -n1 VERSION`
+    - export PACKAGE_VERSION=${PACKAGE_VERSION_SEM//-/_}
+    - export PACKAGE_REGISTRY_URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/jungfraujoch/${PACKAGE_VERSION_SEM}"
     - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file jfjoch-driver-dkms-${PACKAGE_VERSION}-1.el8.noarch.rpm "${PACKAGE_REGISTRY_URL}/jfjoch-driver-dkms-${PACKAGE_VERSION}-1.el8.noarch.rpm"'
     - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file jfjoch-writer-${PACKAGE_VERSION}-1.el8.x86_64.rpm "${PACKAGE_REGISTRY_URL}/jfjoch-writer-${PACKAGE_VERSION}-1.el8.x86_64.rpm"'
     - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file jfjoch-${PACKAGE_VERSION}-1.el8.x86_64.rpm "${PACKAGE_REGISTRY_URL}/jfjoch-${PACKAGE_VERSION}-1.el8.x86_64.rpm"'
@@ -323,7 +327,7 @@ release:
     - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file jfjoch_fpga_pcie_100g.mcs "${PACKAGE_REGISTRY_URL}/jfjoch_fpga_pcie_100g.mcs"'
     - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file jfjoch_fpga_pcie_8x10g.mcs "${PACKAGE_REGISTRY_URL}/jfjoch_fpga_pcie_8x10g.mcs"'
     - >
-      release-cli create --name "Release $PACKAGE_VERSION" --tag-name $PACKAGE_VERSION
+      release-cli create --name "Release $PACKAGE_VERSION_SEM" --tag-name $PACKAGE_VERSION_SEM
       --assets-link "{\"name\":\"jfjoch_driver.tar.gz\",\"url\":\"${PACKAGE_REGISTRY_URL}/jfjoch_driver.tar.gz\"}"
       --assets-link "{\"name\":\"jfjoch_frontend.tar.gz\",\"url\":\"${PACKAGE_REGISTRY_URL}/jfjoch_frontend.tar.gz\"}"
       --assets-link "{\"name\":\"jfjoch_fpga_pcie_8x10g.mcs\",\"url\":\"${PACKAGE_REGISTRY_URL}/jfjoch_fpga_pcie_8x10g.mcs\"}"

CMakeLists.txt

@@ -131,10 +131,19 @@ IF (NOT JFJOCH_WRITER_ONLY)
             WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/frontend_ui)
     ADD_CUSTOM_TARGET(frontend DEPENDS frontend_ui/build/index.html)
 
+    ADD_CUSTOM_TARGET(update_version
+            COMMAND bash update_version.sh
+            WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
+
     INSTALL(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/fpga/pcie_driver/
-            DESTINATION /usr/src/jfjoch-1.0.0
+            DESTINATION /usr/src/jfjoch-${JFJOCH_VERSION}
             COMPONENT driver-dkms
-            FILES_MATCHING PATTERN "*.c" PATTERN "*.h" PATTERN "Makefile" PATTERN "dkms.conf")
+            FILES_MATCHING PATTERN "dkms.conf")
+
+    INSTALL(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/fpga/pcie_driver/
+            DESTINATION /usr/src/jfjoch-${JFJOCH_VERSION}/src
+            COMPONENT driver-dkms
+            FILES_MATCHING PATTERN "*.c" PATTERN "*.h" PATTERN "Makefile")
 
     FILE(MAKE_DIRECTORY ${CMAKE_SOURCE_DIR}/frontend_ui/build/)
     INSTALL(DIRECTORY ${CMAKE_SOURCE_DIR}/frontend_ui/build/ DESTINATION share/jfjoch/frontend COMPONENT jfjoch )

VERSION

@@ -1 +1 @@
-1.0.0_rc.8
+1.0.0-rc.12

acquisition_device/FPGAAcquisitionDevice.cpp

@@ -247,6 +247,11 @@ void FPGAAcquisitionDevice::FillActionRegister(const DiffractionExperiment& x, D
 
     if (x.GetPixelDepth() == 4)
         job.mode |= MODE_32BIT;
+
+    if (x.GetLossyCompressionPoisson()) {
+        job.mode |= MODE_SQROOT;
+        job.sqrtmult = x.GetLossyCompressionPoisson().value();
+    }
 }
 
 void FPGAAcquisitionDevice::Start(const DiffractionExperiment &experiment, uint32_t flag) {

acquisition_device/HLSSimulatedDevice.cpp

@@ -2,117 +2,29 @@
 
 #include "HLSSimulatedDevice.h"
 
-#include <bitset>
-#include <arpa/inet.h>
-#include "../fpga/hls/datamover_model.h"
-#include "../fpga/hls/hls_jfjoch.h"
-
-uint16_t checksum(const uint16_t *addr, size_t count) {
-/* Compute Internet Checksum for "count" bytes
- *         beginning at location "addr".
- */
-    long sum = 0;
-
-    for (int i = 0; i < count / 2; i++)
-        sum += addr[i];
-
-/*  Add left-over byte, if any */
-    if (count % 2 == 1)
-        sum += ((uint8_t *) addr)[count / 2];
-
-/*  Fold 32-bit sum to 16 bits */
-    while (sum>>16)
-        sum = (sum & 0xffff) + (sum >> 16);
-
-    return ~sum;
-}
-
 HLSSimulatedDevice::HLSSimulatedDevice(uint16_t data_stream, size_t in_frame_buffer_size_modules, int16_t numa_node)
-        : FPGAAcquisitionDevice(data_stream),
-          datamover_in(Direction::Input, nullptr),
-          datamover_out(Direction::Output, nullptr),
-          datamover_out_hbm_0(Direction::Output, (char *) hbm.data()),
-          datamover_out_hbm_1(Direction::Output, (char *) hbm.data()),
-          datamover_in_hbm_0(Direction::Input, (char *) hbm.data()),
-          datamover_in_hbm_1(Direction::Input, (char *) hbm.data()),
-          idle(true), hbm(hbm_if_size / 32 * hbm_if_count),
-          dma_address_table(65536) {
+        : FPGAAcquisitionDevice(data_stream) {
     mac_addr = 0xCCAA11223344;
     ipv4_addr = 0x0132010A;
 
     max_modules = MAX_MODULES_FPGA;
 
+    if (data_stream != 0)
+        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
+                              "HLS simulation can only work with 1 data_stream, due to use of static variables");
     MapBuffersStandard(in_frame_buffer_size_modules, numa_node);
 
-    auto in_mem_location32 = (uint32_t *) dma_address_table.data();
 
-    for (int i = 0; i < buffer_device.size(); i++) {
-        in_mem_location32[2 * i    ] = ((uint64_t) buffer_device[i]) & UINT32_MAX;
-        in_mem_location32[2 * i + 1] = ((uint64_t) buffer_device[i]) >> 32;
-    }
+    device = std::make_unique<HLSDevice>(buffer_device);
 }
 
 void HLSSimulatedDevice::CreateFinalPacket(const DiffractionExperiment& experiment) {
-    CreateJFPacket(experiment, UINT64_MAX, 0, 0, nullptr, false);
-}
-
-void HLSSimulatedDevice::SendPacket(char *buffer, int len, uint8_t user) {
-    auto obuff = (ap_uint<512> *)buffer;
-
-    for (int i = 0; i < (len + 63) / 64; i++) {
-        packet_512_t packet_in;
-        if (i == (len + 63) / 64 - 1) packet_in.last = 1;
-        else packet_in.last = 0;
-        packet_in.keep = 0xFFFFFFFFFFFFFFFF;
-        packet_in.user = user;
-        packet_in.data = obuff[i];
-        din_eth.write(packet_in);
-    }
+    device->CreateFinalPacket(experiment);
 }
 
 void HLSSimulatedDevice::CreateJFPacket(const DiffractionExperiment& experiment, uint64_t frame_number, uint32_t eth_packet,
                                         uint32_t module_number, const uint16_t *data, int8_t adjust_axis, uint8_t user) {
-    char buff[256*64];
-    memset(buff, 0, 256*64);
-
-    auto packet = (jf_raw_packet *)buff;
-
-    packet->ether_type = htons(0x0800);
-    packet->sour_mac[0] = 0x00; // module 0
-
-    uint64_t tmp_mac = mac_addr;
-    for (int i = 0; i < 6; i++)
-        packet->dest_mac[i] = (tmp_mac >> (8*i)) % 256;
-
-    uint32_t half_module = 2 * module_number | ((eth_packet >= 64) ? 1 : 0);
-
-    packet->ipv4_header_h = htons(0x4500); // Big endian in IP header!
-    packet->ipv4_header_total_length = htons(8268); // Big endian in IP header!
-    packet->ipv4_header_dest_ip = ipv4_addr;
-    packet->ipv4_header_sour_ip = experiment.GetSrcIPv4Address(data_stream, half_module);
-
-    packet->ipv4_header_ttl_protocol = htons(0x0011);
-    packet->ipv4_header_checksum = checksum( (uint16_t *) &packet->ipv4_header_h, 20); // checksum is already in network order
-
-    packet->udp_dest_port = htons(GetUDPPort()); // module number
-    packet->udp_sour_port = htons(0xDFAC);
-    packet->udp_length = htons(8248);
-
-    // JF headers are little endian
-    packet->jf.detectortype = SLS_DETECTOR_TYPE_JUNGFRAU;
-    packet->jf.timestamp = 0xABCDEF0000FEDCBAL;
-    packet->jf.bunchid   = 0x1234567898765431L;
-    packet->jf.row = half_module;
-    packet->jf.column = 0;
-    packet->jf.framenum = frame_number;
-    packet->jf.packetnum = eth_packet % 64;
-    if (data != nullptr) {
-        for (int i = 0; i < 4096; i++)
-            packet->jf.data[i] = data[i];
-    }
-    packet->udp_checksum = htons(checksum( (uint16_t *) (buff+42), 8192+48));
-
-    SendPacket(buff, (130+adjust_axis)*64, user);
+    device->CreateJFPacket(experiment, frame_number, eth_packet, module_number, data, adjust_axis, user);
 }
 
 void HLSSimulatedDevice::CreateJFPackets(const DiffractionExperiment& experiment, uint64_t frame_number_0, uint64_t frames,
@@ -126,635 +38,67 @@ void HLSSimulatedDevice::CreateJFPackets(const DiffractionExperiment& experiment
 void HLSSimulatedDevice::CreateEIGERPacket(const DiffractionExperiment &experiment, uint64_t frame_number,
                                            uint32_t eth_packet, uint32_t module_number, uint32_t col, uint32_t row,
                                            const uint16_t *data) {
-    char buff[256*64];
-    memset(buff, 0, 256*64);
-
-    auto packet = (eiger_raw_packet *)buff;
-
-    packet->ether_type = htons(0x0800);
-    packet->sour_mac[0] = 0x00; // module 0
-
-    uint64_t tmp_mac = mac_addr;
-    for (int i = 0; i < 6; i++)
-        packet->dest_mac[i] = (tmp_mac >> (8*i)) % 256;
-
-    packet->ipv4_header_h = htons(0x4500); // Big endian in IP header!
-    packet->ipv4_header_total_length = htons(4172); // Big endian in IP header!
-    packet->ipv4_header_dest_ip = ipv4_addr;
-    packet->ipv4_header_sour_ip = experiment.GetSrcIPv4Address(data_stream, 0);
-
-    packet->ipv4_header_ttl_protocol = htons(0x0011);
-    packet->ipv4_header_checksum = checksum( (uint16_t *) &packet->ipv4_header_h, 20); // checksum is already in network order
-
-    packet->udp_dest_port = htons(GetUDPPort()); // module number
-    packet->udp_sour_port = htons(0xDFAC);
-    packet->udp_length = htons(4152);
-
-    // JF headers are little endian
-    packet->eiger.detectortype = SLS_DETECTOR_TYPE_EIGER;
-    packet->eiger.timestamp = 0xABCDEF0000FEDCBAL;
-    packet->eiger.bunchid   = 0x1234567898765431L;
-    packet->eiger.row = (2 * module_number) | (row % 2);
-    packet->eiger.column = col % 2;
-    packet->eiger.framenum = frame_number;
-    packet->eiger.packetnum = eth_packet % 64;
-    if (data != nullptr) {
-        for (int i = 0; i < 2048; i++)
-            packet->eiger.data[i] = data[i];
-    }
-    packet->udp_checksum = htons(checksum( (uint16_t *) (buff+42), 4096+48));
-
-    SendPacket(buff, 66*64, 0);
-}
-
-AXI_STREAM & HLSSimulatedDevice::OutputStream() {
-    return dout_eth;
+    device->CreateEIGERPacket(experiment, frame_number, eth_packet, module_number, col, row, data);
 }
 
 void HLSSimulatedDevice::HW_ReadActionRegister(DataCollectionConfig *job) {
-    memcpy(job, &cfg, sizeof(DataCollectionConfig));
+    device->HW_ReadActionRegister(job);
 }
 
 void HLSSimulatedDevice::HW_WriteActionRegister(const DataCollectionConfig *job) {
-    memcpy(&cfg, job, sizeof(DataCollectionConfig));
+    device->HW_WriteActionRegister(job);
 }
 
 void HLSSimulatedDevice::FPGA_StartAction(const DiffractionExperiment &experiment) {
-    if (action_thread.joinable())
-        action_thread.join();
-
-    run_counter += 1;
-    run_data_collection = 1;
-    cancel_data_collection = 0;
-    idle = false;
-
-    while (!din_frame_generator.empty())
-        din_frame_generator.read();
-
-    datamover_out.ClearCompletedDescriptors();
-
-    action_thread = std::thread(&HLSSimulatedDevice::HLSMainThread, this );
-}
-
-void HLSSimulatedDevice::FrameGeneratorFuture(FrameGeneratorConfig config) {
-    frame_generator(din_frame_generator,
-                    hbm.data(),
-                    hbm.data(),
-                    hbm_if_size,
-                    mac_addr,
-                    ipv4_addr,
-                    cancel_data_collection,
-                    config);
+    device->FPGA_StartAction(experiment);
 }
 
 void HLSSimulatedDevice::HW_RunInternalGenerator(const FrameGeneratorConfig &config) {
-    frame_generator_future = std::async(std::launch::async, &HLSSimulatedDevice::FrameGeneratorFuture, this, config);
+    device->HW_RunInternalGenerator(config);
 }
 
 void HLSSimulatedDevice::FPGA_EndAction() {
-    if (action_thread.joinable())
-        action_thread.join();
+    device->FPGA_EndAction();
 }
 
-HLSSimulatedDevice::~HLSSimulatedDevice() {
-    if (action_thread.joinable())
-        action_thread.join();
-}
 
 bool HLSSimulatedDevice::HW_ReadMailbox(uint32_t *values) {
-    std::unique_lock<std::mutex> ul(completion_mutex);
-
-    ap_uint<32> tmp;
-    bool ret = completion_stream.read_nb(tmp);
-    values[0] = tmp;
-    // equivalent to driver functionality
-    if (ret) {
-        uint32_t data_collection_id = (values[0] >> 16) & 0xFFFF;
-        uint32_t handle = values[0] & 0xFFFF;
-        if (handle == HANDLE_START)
-            completion_count = 0;
-        else if ((handle != HANDLE_END) && (data_collection_id != DATA_COLLECTION_ID_PURGE)) {
-            completion_count++;
-            while (completion_count * DMA_DESCRIPTORS_PER_MODULE > datamover_out.GetCompletedDescriptors())
-                std::this_thread::sleep_for(std::chrono::milliseconds(1));
-        }
-    }
-    return ret;
+    return device->HW_ReadMailbox(values);
 }
 
 void HLSSimulatedDevice::Cancel() {
-    cancel_data_collection = 1;
+    device->Cancel();
 }
 
 bool HLSSimulatedDevice::HW_IsIdle() const {
-    return idle && datamover_out.IsIdle();
+    return device->HW_IsIdle();
 }
 
 
 bool HLSSimulatedDevice::HW_SendWorkRequest(uint32_t handle) {
-    work_request_stream.write(handle);
-    return true;
-}
-
-inline uint32_t float2uint(float f) {
-    float_uint32 fu;
-    fu.f = f;
-    return fu.u;
-}
-
-void HLSSimulatedDevice::HLSMainThread() {
-    ap_uint<1> clear_counters = 0;
-
-    uint64_t packets_processed;
-
-    std::vector<std::thread> hls_cores;
-
-    STREAM_512 ip1, udp1, udp2, icmp1, arp1;
-
-    STREAM_512 network0;
-
-    STREAM_768 stream_768_0;
-    STREAM_768 stream_768_1;
-    STREAM_768 stream_768_2;
-    STREAM_768 stream_768_3;
-    STREAM_768 stream_768_4;
-    STREAM_768 stream_768_5;
-
-    STREAM_512 data_0;
-    hls::stream<ap_axiu<512, 1, 1, 1>, 2> data_1;
-    hls::stream<ap_axiu<512, 1, 1, 1>, 2> data_2;
-    STREAM_512 data_3;
-    STREAM_512 data_4;
-    STREAM_512 data_5;
-    STREAM_512 data_6;
-    STREAM_512 data_7;
-    STREAM_512 data_8;
-    STREAM_512 data_9;
-    STREAM_512 data_10;
-    STREAM_512 data_12;
-    STREAM_512 data_13;
-
-    hls::stream<axis_addr> addr0;
-    hls::stream<axis_addr> addr1;
-    hls::stream<axis_addr> addr2;
-    hls::stream<axis_addr> addr3;
-
-    hls::stream<axis_completion> axi_compl[12];
-
-    hls::stream<ap_uint<16>> hbm_handles;
-    hls::stream<ap_uint<512>> adu_histo_result;
-
-    hls::stream<ap_axiu<64,1,1,1>> integration_result_0;
-    hls::stream<ap_uint<512>> integration_result_1;
-
-    hls::stream<ap_axiu<32,1,1,1>> spot_finder_result_0;
-    hls::stream<ap_axiu<32,1,1,1>> spot_finder_result_1;
-    hls::stream<ap_uint<32>> spot_finder_conn_0;
-    hls::stream<ap_axiu<32,1,1,1>> spot_finder_result_2;
-    hls::stream<ap_uint<512>> spot_finder_result_3;
-
-    hls::stream<ap_uint<32>> spot_finder_mask_0;
-
-    hls::stream<ap_uint<256>> roi_calc_result_0;
-
-    hls::stream<ap_uint<UDP_METADATA_STREAM_WIDTH> > udp_metadata;
-    volatile ap_uint<1> idle_data_collection = 1;
-    ap_uint<1> load_to_hbm_idle;
-    ap_uint<1> save_to_hbm_idle;
-    ap_uint<1> integration_idle;
-    ap_uint<1> stream_conv_idle;
-    ap_uint<1> frame_summation_idle;
-
-    volatile bool done = false;
-    volatile bool udp_done = false; // done AND udp_idle
-    volatile bool sls_done = false; // done AND sls_idle
-
-    // Sent gratuitous ARP message
-    arp(arp1, dout_eth, mac_addr, ipv4_addr, 1, 1);
-
-    Logger logger_hls("HLS");
-
-    volatile rcv_state_t state = RCV_INIT;
-    // Start data collection
-    data_collection_fsm(data_0, data_1,
-                        addr0, addr1,
-                        run_data_collection,
-                        cancel_data_collection,
-                        idle_data_collection,
-                        cfg.mode,
-                        float2uint(cfg.energy_kev),
-                        cfg.nframes,
-                        cfg.nmodules,
-                        cfg.nstorage_cells,
-                        cfg.nsummation , state);
-
-    run_data_collection = 0;
-    data_collection_fsm(data_0, data_1,
-                        addr0, addr1,
-                        run_data_collection,
-                        cancel_data_collection,
-                        idle_data_collection,
-                        cfg.mode,
-                        float2uint(cfg.energy_kev),
-                        cfg.nframes,
-                        cfg.nmodules,
-                        cfg.nstorage_cells,
-                        cfg.nsummation, state);
-
-    hls_cores.emplace_back([&] {
-        while (!udp_done) {
-            if (din_eth.empty() && din_frame_generator.empty())
-                std::this_thread::sleep_for(std::chrono::microseconds(10));
-            else
-                stream_merge(din_eth, din_frame_generator, network0, data_source);
-        }
-        logger_hls.Info("Stream_merge done");
-    });
-
-    hls_cores.emplace_back([&] {
-        while (!udp_done) {
-            if (network0.empty())
-                std::this_thread::sleep_for(std::chrono::microseconds(10));
-            else
-                ethernet(network0, ip1, arp1, mac_addr, eth_packets, clear_counters);
-        }
-        logger_hls.Info("ethernet done");
-    });
-
-    hls_cores.emplace_back([&] {
-        while (!udp_done) {
-            if (ip1.empty())
-                std::this_thread::sleep_for(std::chrono::microseconds(10));
-            else
-                ipv4(ip1, udp1, icmp1, ipv4_addr);
-        }
-        logger_hls.Info("ipv4 done");
-    });
-
-    hls_cores.emplace_back([&] {
-        ap_uint<1> udp_idle = 1;
-        while (!done || !udp_idle) {
-            if (udp1.empty())
-                std::this_thread::sleep_for(std::chrono::microseconds(10));
-            else
-                udp(udp1, udp2, udp_metadata, udp_packets, clear_counters, udp_idle);
-        }
-        udp_done = true;
-        logger_hls.Info("udp done");
-    });
-
-    hls_cores.emplace_back([&] {
-        ap_uint<1> sls_idle = 1;
-        while (!done || !sls_idle) {
-            if (udp2.empty())
-                std::this_thread::sleep_for(std::chrono::microseconds (10));
-            else
-                sls_detector(udp2, udp_metadata, data_0, addr0, sls_packets, udp_eth_err, udp_len_err, current_pulse_id,
-                             clear_counters, sls_idle);
-        }
-        sls_done = true;
-        logger_hls.Info("sls_detector done");
-    });
-
-    // 1. Parse incoming UDP packets
-    hls_cores.emplace_back([&] {
-        while ((state != RCV_WAIT_FOR_START) || (idle_data_collection.read() == 0) || (!data_0.empty()))  {
-            data_collection_fsm(data_0, data_1,
-                                addr0, addr1,
-                                run_data_collection,
-                                cancel_data_collection,
-                                idle_data_collection,
-                                cfg.mode,
-                                float2uint(cfg.energy_kev),
-                                cfg.nframes,
-                                cfg.nmodules,
-                                cfg.nstorage_cells,
-                                cfg.nsummation,
-                                state);
-        }
-        done = true;
-
-        logger_hls.Info("data collection done");
-    });
-
-    // 2. Cache images in HBM
-    hls_cores.emplace_back([&] { save_to_hbm(addr1, axi_compl[0], hbm_handles,
-                                             datamover_out_hbm_0.GetCtrlStream(),
-                                             datamover_out_hbm_1.GetCtrlStream(),
-                                             save_to_hbm_idle,
-                                             hbm_if_size);});
-
-    hls_cores.emplace_back([&] { save_to_hbm_data(data_1,
-                                                  data_3,
-                                                  datamover_out_hbm_0.GetDataStream(),
-                                                  datamover_out_hbm_1.GetDataStream());});
-
-    hls_cores.emplace_back([&] {frame_summation_reorder_compl(data_3, data_4, axi_compl[0], axi_compl[1]);});
-
-    hls_cores.emplace_back([&] { load_from_hbm(data_4, data_5, axi_compl[1], axi_compl[2], hbm_handles,
-                                               datamover_in_hbm_0.GetDataStream(), datamover_in_hbm_1.GetDataStream(),
-                                               datamover_in_hbm_0.GetCtrlStream(), datamover_in_hbm_1.GetCtrlStream(),
-                                               load_to_hbm_idle, hbm_if_size);});
-
-    hls_cores.emplace_back([&] { pedestal(data_5, data_6, axi_compl[2], axi_compl[3],
-                                          hbm.data(),
-                                          hbm.data(),
-                                          hbm.data(),
-                                          hbm.data(),
-                                          hbm.data(),
-                                          hbm.data(),
-                                          hbm.data(),
-                                          hbm.data(),
-                                          hbm.data(),
-                                          hbm.data(),
-                                          hbm.data(),
-                                          hbm.data(),
-                                          hbm_if_size);});
-
-
-    // 3. Calculate histogram of ADU values
-    hls_cores.emplace_back([&] { adu_histo(data_6, data_7, adu_histo_result, axi_compl[3], axi_compl[4]);});
-
-    // 4. Mask missing pixels
-    hls_cores.emplace_back([&] { mask_missing(data_7, data_8, axi_compl[4], axi_compl[5]);});
-
-    // 5. Handle EIGER packets properly
-    hls_cores.emplace_back([&] { eiger_reorder(data_8, data_9, axi_compl[5], axi_compl[6]);});
-
-    // 6. Apply pedestal & gain corrections
-    hls_cores.emplace_back([&] { jf_conversion(data_9, stream_768_0,
-                                               axi_compl[6], axi_compl[7],
-                                               hbm.data(),
-                                               hbm.data(),
-                                               hbm.data(),
-                                               hbm.data(),
-                                               hbm.data(),
-                                               hbm.data(),
-                                               hbm.data(),
-                                               hbm.data(),
-                                               hbm.data(),
-                                               hbm.data(),
-                                               hbm.data(),
-                                               hbm.data(),
-                                               hbm_if_size); });
-
-    // 7. Frame summation
-    hls_cores.emplace_back([&] { frame_summation(stream_768_0, stream_768_1, axi_compl[7], axi_compl[8], frame_summation_idle);});
-
-    // 8. Integration of pixels
-    hls_cores.emplace_back([&] { integration(stream_768_1, stream_768_2, integration_result_0, axi_compl[8], axi_compl[9],
-                                             hbm.data(), hbm.data(), hbm.data(), hbm.data(), integration_idle, hbm_if_size);});
-    hls_cores.emplace_back([&] { axis_64_to_512(integration_result_0, integration_result_1);});
-
-    // 9. Spot finding
-    ap_uint<32> tmp_snr_threshold = float2uint(spot_finder_parameters.snr_threshold);
-    ap_uint<32> min_d = float2uint(spot_finder_parameters.min_d);
-    ap_uint<32> max_d = float2uint(spot_finder_parameters.max_d);
-    ap_int<32> tmp_count_threshold = spot_finder_parameters.count_threshold;
-    ap_uint<32> min_pix_per_spot = spot_finder_parameters.min_pix_per_spot;
-
-    hls_cores.emplace_back([&] {
-        spot_finder_mask(stream_768_2,
-                         stream_768_3,
-                         spot_finder_mask_0,
-                         axi_compl[9],
-                         axi_compl[10],
-                         hbm.data(),
-                         hbm.data(),
-                         min_d,
-                         max_d,
-                         hbm_if_size);
-        logger_hls.Info("spot_finder_mask done");
-    });
-
-    hls_cores.emplace_back([&] {
-        spot_finder(stream_768_3, spot_finder_mask_0, stream_768_4, spot_finder_result_0, tmp_count_threshold, tmp_snr_threshold);
-        logger_hls.Info("spot_finder done");
-    });
-
-    hls_cores.emplace_back([&] {
-        spot_finder_connectivity(spot_finder_result_0,
-                                 spot_finder_result_1,
-                                 spot_finder_conn_0);
-        logger_hls.Info("spot_finder_connectivity done");
-    });
-
-    hls_cores.emplace_back([&] {
-        spot_finder_merge(spot_finder_result_1,
-                          spot_finder_conn_0,
-                          spot_finder_result_2,
-                          min_pix_per_spot);
-        logger_hls.Info("spot_finder_merge done");
-    });
-
-    hls_cores.emplace_back([&] {
-        axis_32_to_512(spot_finder_result_2, spot_finder_result_3);
-        logger_hls.Info("axis_32_to_512 done");
-    });
-
-    hls_cores.emplace_back([&] {
-        roi_calc(stream_768_4,
-                 stream_768_5,
-                 roi_calc_result_0,
-                 axi_compl[10],
-                 axi_compl[11],
-                 hbm.data(),
-                 hbm.data(),
-                 hbm_if_size);
-        logger_hls.Info("roi_calc done");
-    });
-
-    // 10. Reduce/extend 24-bit stream
-    hls_cores.emplace_back([&] { stream_24bit_conv(stream_768_5, data_12, stream_conv_idle);});
-
-    // 11. Prepare data to write to host memory
-    hls_cores.emplace_back([&] {
-        ap_uint<3> state;
-        host_writer(data_12, adu_histo_result, integration_result_1, spot_finder_result_3, roi_calc_result_0,
-                    axi_compl[11], datamover_out.GetDataStream(),
-                    datamover_out.GetCtrlStream(), work_request_stream, completion_stream,
-                    dma_address_table.data(), packets_processed, host_writer_idle, cancel_data_collection, state);
-        logger_hls.Info("host_writer done");
-    });
-
-    for (auto &i : hls_cores)
-        i.join();
-
-    if (frame_generator_future.valid())
-        frame_generator_future.get();
-
-    // reset static counter
-    arp(arp1, dout_eth, mac_addr, ipv4_addr, 0, 1);
-    try {
-        while (!din_eth.empty())
-            din_eth.read();
-
-        if (!addr1.empty())
-            throw std::runtime_error("Addr1 queue not empty");
-
-        if (!addr2.empty())
-            throw std::runtime_error("Addr2 queue not empty");
-
-        if (!addr3.empty())
-            throw std::runtime_error("Addr3 queue not empty");
-
-        if (!data_1.empty())
-            throw std::runtime_error("data_1 queue not empty");
-
-        if (!data_2.empty())
-            throw std::runtime_error("data_2 queue not empty");
-
-        if (!data_3.empty())
-            throw std::runtime_error("data_3 queue not empty");
-
-        if (!data_4.empty())
-            throw std::runtime_error("data_4 queue not empty");
-
-        if (!data_5.empty())
-            throw std::runtime_error("data_5 queue not empty");
-
-        if (!data_7.empty())
-            throw std::runtime_error("data_7 queue not empty");
-
-        if (!data_8.empty())
-            throw std::runtime_error("data_8 queue not empty");
-
-        if (!data_9.empty())
-            throw std::runtime_error("data_9 queue not empty");
-
-        if (!data_10.empty())
-            throw std::runtime_error("data_10 queue not empty");
-
-
-
-        if (!data_12.empty())
-            throw std::runtime_error("data_12 queue not empty");
-
-        if (!data_13.empty())
-            throw std::runtime_error("data_13 queue not empty");
-
-        for (auto &c: axi_compl) {
-            if (!c.empty())
-                throw std::runtime_error("Compl queue not empty");
-        }
-
-        if (!stream_768_0.empty())
-            throw std::runtime_error("stream_768_0 queue not empty");
-
-        if (!stream_768_1.empty())
-            throw std::runtime_error("stream_768_1 queue not empty");
-
-        if (!stream_768_2.empty())
-            throw std::runtime_error("stream_768_2 queue not empty");
-
-        if (!stream_768_3.empty())
-            throw std::runtime_error("stream_768_3 queue not empty");
-
-        if (!stream_768_4.empty())
-            throw std::runtime_error("stream_768_4 queue not empty");
-
-        if (!hbm_handles.empty())
-            throw std::runtime_error("Handles queue not empty");
-
-        if (!integration_result_0.empty())
-            throw std::runtime_error("Integration result queue not empty");
-
-        if (!integration_result_1.empty())
-            throw std::runtime_error("Integration result queue not empty");
-
-        if (!datamover_in.GetDataStream().empty())
-            throw std::runtime_error("Datamover queue is not empty");
-
-        while (!datamover_out.IsIdle())
-            std::this_thread::sleep_for(std::chrono::milliseconds(100));
-    } catch (const std::runtime_error &e) {
-        if (logger)
-            logger->ErrorException(e);
-        throw e;
-    }
-    if (logger)
-        logger->Info("Packets Eth {} UDP {} SLS {} Proc {}", eth_packets, udp_packets, sls_packets, packets_processed);
-    idle = true;
+    return device->HW_SendWorkRequest(handle);
 }
 
 DataCollectionStatus HLSSimulatedDevice::GetDataCollectionStatus() const {
-    DataCollectionStatus status{};
-
-    union {
-        uint64_t u64;
-        double d;
-    } pulse_id_conv;
-    pulse_id_conv.u64 = current_pulse_id;
-
-    status.ctrl_reg = ap_uint<1>(host_writer_idle) ? (1 << 4) : 0;
-    status.max_modules = max_modules;
-    status.hbm_size_bytes = hbm_if_size;
-    status.run_counter = run_counter;
-    status.current_pulseid = pulse_id_conv.d;
-    status.packets_udp = udp_packets;
-    status.packets_eth = eth_packets;
-    status.packets_sls = sls_packets;
-
-    return status;
+    return device->GetDataCollectionStatus();
 }
 
 void HLSSimulatedDevice::HW_LoadCalibration(const LoadCalibrationConfig &config) {
-    int ret = load_calibration(hbm.data(), hbm.data(),
-                               config,
-                               hbm_if_size,
-                               datamover_in.GetCtrlStream(),
-                               datamover_in.GetDataStream(),
-                               dma_address_table.data());
-    if (ret)
-        throw JFJochException(JFJochExceptionCategory::AcquisitionDeviceError,
-                              "Error in loading calibration " + std::to_string(ret));
-
-    if (!datamover_in.GetDataStream().empty())
-        throw std::runtime_error("Datamover queue is not empty");
+    device->HW_LoadCalibration(config);
 }
 
 void HLSSimulatedDevice::HW_SetSpotFinderParameters(const SpotFinderParameters &params) {
-    spot_finder_parameters = params;
+    device->HW_SetSpotFinderParameters(params);
 }
 
 void HLSSimulatedDevice::HW_SetDataSource(uint32_t val) {
-    data_source = val;
+    device->HW_SetDataSource(val);
 }
 
 uint32_t HLSSimulatedDevice::HW_GetDataSource() {
-    return data_source;
+    return device->HW_GetDataSource();
 }
 
 void HLSSimulatedDevice::CreateXfelBunchIDPacket(double pulse_id, uint32_t event_code) {
-
-    union {
-        uint64_t u64;
-        double d;
-    } pulse_id_conv;
-    pulse_id_conv.d = pulse_id;
-
-    bunchid_raw_packet packet{};
-
-    packet.ether_type = htons(0x0800);
-    for (int i = 0; i < 6; i++)
-        packet.dest_mac[i] = 0xFF;
-
-    packet.ipv4_header_h = htons(0x4500); // Big endian in IP header!
-    packet.ipv4_header_total_length = htons(sizeof(bunchid_payload) + 8 + 20); // Big endian in IP header!
-    packet.ipv4_header_dest_ip = UINT32_MAX;
-
-    packet.ipv4_header_ttl_protocol = htons(0x0011);
-    packet.ipv4_header_checksum = checksum( (uint16_t *) &packet.ipv4_header_h, 20); // checksum is already in network order
-
-    packet.udp_length = htons(sizeof(bunchid_payload) + 8);
-    packet.payload.magicn[0] = BUNCHID_MAGICN;
-    packet.payload.magicn[1] = BUNCHID_MAGICN;
-    packet.payload.magicn[2] = BUNCHID_MAGICN;
-    packet.payload.bunchid_msb[0] = (pulse_id_conv.u64 >> 32) & UINT32_MAX;
-    packet.payload.bunchid_msb[1] = (pulse_id_conv.u64 >> 32) & UINT32_MAX;
-    packet.payload.bunchid_lsb[0] = pulse_id_conv.u64 & UINT32_MAX;
-    packet.payload.bunchid_lsb[1] = pulse_id_conv.u64 & UINT32_MAX;
-    packet.payload.montrig = event_code;
-
-    SendPacket((char *) &packet, sizeof(bunchid_raw_packet));
+    device->CreateXfelBunchIDPacket(pulse_id, event_code);
 }

acquisition_device/HLSSimulatedDevice.h

@@ -5,61 +5,13 @@
 
 #include <thread>
 
-#include "../fpga/hls/hls_jfjoch.h"
-#include "../fpga/hls/datamover_model.h"
 #include "../common/DiffractionExperiment.h"
-#include "FPGAAcquisitionDevice.h"
-#include "../jungfrau/sls_packet.h"
+#include "../fpga/hls/HLSDevice.h"
 
-uint16_t checksum(const uint16_t *addr, size_t count);
+#include "FPGAAcquisitionDevice.h"
 
 class HLSSimulatedDevice : public FPGAAcquisitionDevice {
-    AXI_STREAM din_eth;
-    AXI_STREAM din_frame_generator;
-    AXI_STREAM dout_eth;
-
-    uint32_t run_counter = 0;
-
-    ap_uint<2>  data_source = STREAM_MERGE_SRC_NETWORK;
-    SpotFinderParameters spot_finder_parameters;
-
-    uint64_t current_pulse_id = 0;
-
-    DataCollectionConfig cfg;
-
-    volatile bool idle;
-
-    constexpr static const size_t hbm_if_count = 32;
-    constexpr static const size_t hbm_if_size  = 32*1024*1024LU;
-    std::vector<ap_uint<256>> hbm;
-
-    hls::stream<ap_uint<32> > work_request_stream;
-    hls::stream<ap_uint<32> > completion_stream;
-    std::mutex completion_mutex;
-    uint32_t completion_count;
-
-    std::thread action_thread;
-    std::future<void> frame_generator_future;
-
-    Datamover<512> datamover_in;
-    Datamover<512> datamover_out;
-    Datamover<256> datamover_in_hbm_0;
-    Datamover<256> datamover_in_hbm_1;
-    Datamover<256, 16> datamover_out_hbm_0;
-    Datamover<256, 16> datamover_out_hbm_1;
-
-    ap_uint<1> run_data_collection;
-    ap_uint<1> cancel_data_collection;
-    volatile ap_uint<1> host_writer_idle;
-
-    std::vector<uint64_t> dma_address_table;
-
-    uint64_t eth_packets = 0;
-    uint64_t icmp_packets = 0;
-    uint64_t udp_packets = 0;
-    uint64_t sls_packets = 0;
-    uint32_t udp_len_err = 0;
-    uint32_t udp_eth_err = 0;
+    std::unique_ptr<HLSDevice> device;
 
     void HW_ReadActionRegister(DataCollectionConfig *job) override;
     void HW_WriteActionRegister(const DataCollectionConfig *job) override;
@@ -74,12 +26,9 @@ class HLSSimulatedDevice : public FPGAAcquisitionDevice {
     uint32_t HW_GetDataSource() override;
     void HW_SetDataSource(uint32_t val) override;
     void HW_RunInternalGenerator(const FrameGeneratorConfig &config) override;
-    void FrameGeneratorFuture(FrameGeneratorConfig config);
-    void HLSMainThread();
 public:
     HLSSimulatedDevice(uint16_t data_stream, size_t in_frame_buffer_size_modules, int16_t numa_node = -1);
-    ~HLSSimulatedDevice() override;
-    void SendPacket(char *buffer, int len, uint8_t user = 0);
+    ~HLSSimulatedDevice() override = default;
     void CreateJFPacket(const DiffractionExperiment& experiment, uint64_t frame_number, uint32_t eth_packet,
                         uint32_t module_number, const uint16_t *data, int8_t adjust_axis = 0, uint8_t user = 0);
     void CreateJFPackets(const DiffractionExperiment& experiment, uint64_t frame_number_0, uint64_t frames,
@@ -90,7 +39,6 @@ public:
     void CreateXfelBunchIDPacket(double bunchid, uint32_t event_code);
     void CreateFinalPacket(const DiffractionExperiment& experiment);
     DataCollectionStatus GetDataCollectionStatus() const override;
-    AXI_STREAM &OutputStream();
     void Cancel() override;
 };
 

broker/JFJochBrokerHttp.cpp

@@ -5,6 +5,7 @@
 #include "JFJochBrokerHttp.h"
 #include "gen/model/Error_message.h"
 #include "../preview/JFJochTIFF.h"
+#include "../common/GitInfo.h"
 
 // From https://en.cppreference.com/w/cpp/string/byte/tolower
 inline std::string str_tolower(std::string s) {
@@ -52,6 +53,7 @@ inline org::openapitools::server::model::Measurement_statistics Convert(const Me
     if (!input.file_prefix.empty())
         ret.setFilePrefix(input.file_prefix);
 
+    ret.setExperimentGroup(input.experiment_group);
     ret.setImagesExpected(input.images_expected);
     ret.setImagesCollected(input.images_collected);
     ret.setImagesSent(input.images_sent);
@@ -75,7 +77,7 @@ inline org::openapitools::server::model::Measurement_statistics Convert(const Me
     if (input.bkg_estimate)
         ret.setBkgEstimate(input.bkg_estimate.value());
     ret.setUnitCell(input.unit_cell);
-
+    ret.setRunNumber(input.run_number);
     return ret;
 }
 
@@ -192,10 +194,10 @@ inline org::openapitools::server::model::Detector_status Convert(const DetectorS
     output.setHighVoltageV(input.high_voltage_V);
     switch (input.power_state) {
         case DetectorPowerState::ON:
-            output.setPowerchip("On");
+            output.setPowerchip("PowerOn");
             break;
         case DetectorPowerState::OFF:
-            output.setPowerchip("Off");
+            output.setPowerchip("PowerOff");
             break;
         case DetectorPowerState::PARTIAL:
             output.setPowerchip("Partial");
@@ -336,6 +338,13 @@ inline DatasetSettings Convert(const org::openapitools::server::model::Dataset_s
     ret.ImagesPerTrigger(input.getImagesPerTrigger());
     ret.NumTriggers(input.getNtrigger());
 
+    if (input.runNumberIsSet())
+        ret.RunNumber(input.getRunNumber());
+    if (input.runNameIsSet())
+        ret.RunName(input.getRunName());
+
+    ret.ExperimentGroup(input.getExperimentGroup());
+
     if (!input.fpgaOutputIsSet())
         ret.FPGAOutputMode(FPGAPixelOutput::Auto);
     else {
@@ -354,13 +363,13 @@ inline DatasetSettings Convert(const org::openapitools::server::model::Dataset_s
             throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Unknown output format");
     }
 
-    ret.Summation(input.getSummation());
+    if (input.imageTimeUsIsSet())
+        ret.ImageTime(std::chrono::microseconds(input.getImageTimeUs()));
     ret.BeamX_pxl(input.getBeamXPxl());
     ret.BeamY_pxl(input.getBeamYPxl());
     ret.DetectorDistance_mm(input.getDetectorDistanceMm());
-    ret.PhotonEnergy_keV(input.getPhotonEnergyKeV());
-
-    ret.PhotonEnergyMultiplayer(input.getPhotonEnergyMultiplier());
+    ret.PhotonEnergy_keV(input.getIncidentEnergyKeV());
+    ret.PhotonEnergyMultiplayer(input.getEnergyMultiplier());
 
     ret.FilePrefix(input.getFilePrefix());
 
@@ -379,6 +388,8 @@ inline DatasetSettings Convert(const org::openapitools::server::model::Dataset_s
         else
             throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Unknown compression");
     }
+    if (input.poissonCompressionIsSet())
+        ret.LossyCompressionPoisson(input.getPoissonCompression());
 
     if (input.unitCellIsSet())
         ret.SetUnitCell(UnitCell{
@@ -395,12 +406,15 @@ inline DatasetSettings Convert(const org::openapitools::server::model::Dataset_s
     if (input.transmissionIsSet())
         ret.AttenuatorTransmission(input.getTransmission());
 
-    if (input.omegaIsSet()) {
-        ret.OmegaStep(input.getOmega().getStep());
-        ret.OmegaStart(input.getOmega().getStart());
-        if (input.getOmega().getVector().size() == 3) {
-            auto v = input.getOmega().getVector();
-            ret.OmegaAxis(Coord(v[0], v[1], v[2]));
+    if (input.goniometerIsSet()) {
+        ret.Goniometer(GoniometerAxis{
+                .name = input.getGoniometer().getName(),
+                .increment = input.getGoniometer().getStep(),
+                .start = input.getGoniometer().getStart()
+        });
+        if (input.getGoniometer().getVector().size() == 3) {
+            auto v = input.getGoniometer().getVector();
+            ret.RotationAxis(Coord(v[0], v[1], v[2]));
         }
     }
 
@@ -411,7 +425,7 @@ inline DatasetSettings Convert(const org::openapitools::server::model::Dataset_s
     ret.ImagesPerFile(input.getImagesPerFile());
 
     if (input.dataReductionFactorSerialmxIsSet())
-        ret.DataReductionFactorSerialMX(input.getDataReductionFactorSerialmx());
+        ret.LossyCompressionSerialMX(input.getDataReductionFactorSerialmx());
 
     return ret;
 }
@@ -463,8 +477,21 @@ void JFJochBrokerHttp::status_get(Pistache::Http::ResponseWriter &response) {
     ProcessOutput(Convert(state_machine.GetStatus()), response);
 }
 
-void JFJochBrokerHttp::wait_till_done_post(Pistache::Http::ResponseWriter &response) {
-    auto state = state_machine.WaitTillMeasurementDone(std::chrono::seconds(5));
+
+
+void JFJochBrokerHttp::wait_till_done_post(const std::optional<int32_t> &timeout,
+                                           Pistache::Http::ResponseWriter &response) {
+    JFJochState state;
+    if (!timeout)
+        state = state_machine.WaitTillMeasurementDone(std::chrono::minutes(1));
+    else if ((timeout.value() > 3600) || (timeout.value() < 0)) {
+        response.send(Pistache::Http::Code::Bad_Request);
+        return;
+    } else if (timeout.value() == 0)
+        state = state_machine.GetState();
+    else
+        state = state_machine.WaitTillMeasurementDone(std::chrono::seconds(timeout.value()));
+
     switch (state) {
         case JFJochState::Idle:
             response.send(Pistache::Http::Code::Ok);
@@ -821,3 +848,7 @@ void JFJochBrokerHttp::config_user_mask_tiff_put(const Pistache::Rest::Request &
     state_machine.SetUserPixelMask(request.body());
     response.send(Pistache::Http::Code::Ok);
 }
+
+void JFJochBrokerHttp::version_get(Pistache::Http::ResponseWriter &response) {
+    response.send(Pistache::Http::Code::Ok, jfjoch_version(), MIME(Text, Plain));
+}

broker/JFJochBrokerHttp.h

@@ -90,7 +90,7 @@ class JFJochBrokerHttp : public org::openapitools::server::api::DefaultApi {
 
     void status_get(Pistache::Http::ResponseWriter &response) override;
 
-    void wait_till_done_post(Pistache::Http::ResponseWriter &response) override;
+    void wait_till_done_post(const std::optional<int32_t> &timeoutMs, Pistache::Http::ResponseWriter &response) override;
     void trigger_post(Pistache::Http::ResponseWriter &response) override;
     void pedestal_post(Pistache::Http::ResponseWriter &response) override;
 
@@ -143,6 +143,9 @@ class JFJochBrokerHttp : public org::openapitools::server::api::DefaultApi {
         to_json(j, output);
         response.send(Pistache::Http::Code::Ok, j.dump(), MIME(Application, Json));
     }
+
+    void version_get(Pistache::Http::ResponseWriter &response) override;
+
 public:
     JFJochBrokerHttp(const DiffractionExperiment& experiment, std::shared_ptr<Pistache::Rest::Router> &rtr);
     void AddDetectorSetup(const DetectorSetup &setup);

broker/JFJochBrokerParser.cpp

@@ -2,6 +2,8 @@
 
 #include "JFJochBrokerParser.h"
 #include "../common/NetworkAddressConvert.h"
+#include "../frame_serialize/ZMQStream2Pusher.h"
+#include "../frame_serialize/DumpCBORToFilePusher.h"
 
 inline bool CHECK_ARRAY(const nlohmann::json &j, const std::string& tag) {
     if (j.contains(tag)) {
@@ -183,6 +185,7 @@ inline int64_t TimeToUs(const std::string &unit) {
 
 inline std::chrono::microseconds GET_TIME(const nlohmann::json &j, const std::string& tag) {
     if (j.contains(tag)) {
+        // If no units provided for time, this is always microsecond
         if (j[tag].is_number())
             return std::chrono::microseconds (std::lround(j[tag].get<double>() * 1000.0 * 1000.0));
         else if (j[tag].is_string()) {
@@ -283,7 +286,8 @@ DetectorSetup ParseDetectorSetup(const nlohmann::json &j) {
     setup.UDPInterfaceCount(GET_I64(j, "udp_interface_count", 2))
             .SensorThickness_um(GET_FLOAT(j, "sensor_thickness_um", 320.0f))
             .PixelSize_um(GET_FLOAT(j, "pixel_size_um", 75.0f))
-            .SensorMaterial(GET_STR(j, "sensor_material", "Si"));
+            .SensorMaterial(GET_STR(j, "sensor_material", "Si"))
+            .SerialNumber(GET_STR(j, "serial_number",""));
 
     if (j.contains("tx_delay"))
         setup.TxDelay(GET_I64_ARR(j, "tx_delay"));
@@ -299,24 +303,55 @@ void ParseDetectorSetup(const nlohmann::json &j, const std::string& tag, JFJochB
         throw JFJochException(JFJochExceptionCategory::JSON, "Detector setup not found");
 }
 
+void ParseImagePusher(const nlohmann::json &input, std::unique_ptr<ImagePusher> &image_pusher) {
+    std::string pusher_type = ParseString(input, "stream_type", "zmq");
+    if (pusher_type == "zmq") {
+        int32_t zmq_send_watermark = ParseInt32(input, "zmq_send_watermark", 100);
+        int32_t zmq_send_buffer_size = ParseInt32(input, "zmq_send_buffer_size", -1);
+
+        auto tmp = std::make_unique<ZMQStream2Pusher>(ParseStringArray(input, "zmq_image_addr"),
+                                                      zmq_send_watermark,
+                                                      zmq_send_buffer_size);
+
+        std::string preview_addr = ParseString(input, "zmq_preview_addr", "");
+        if (!preview_addr.empty())
+            tmp->PreviewSocket(preview_addr);
+
+        if (input.contains("zmq_preview_period"))
+            tmp->PreviewCounterPeriod(GET_TIME(input, "zmq_preview_period"));
+
+        std::string writer_notification_addr = ParseString(input, "zmq_writer_notification_addr", "");
+        if (!writer_notification_addr.empty())
+            tmp->WriterNotificationSocket(writer_notification_addr);
+
+        image_pusher = std::move(tmp);
+    } else if (pusher_type == "dump_cbor") {
+        image_pusher = std::make_unique<DumpCBORToFilePusher>();
+    } else
+        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
+                              "stream_type allowed: zmq (default), dump_cbor");
+}
+
 void ParseFacilityConfiguration(const nlohmann::json &input, const std::string& tag, DiffractionExperiment &experiment) {
     if (CHECK_OBJECT(input, tag)) {
         auto j = input[tag];
         experiment.SourceName(GET_STR(j, "source_name"));
         experiment.SourceNameShort(GET_STR(j, "source_name_short"));
+        experiment.SourceType(GET_STR(j, "source_type", ""));
+
         experiment.InstrumentName(GET_STR(j, "instrument_name"));
         experiment.InstrumentNameShort(GET_STR(j, "instrument_name_short"));
 
         experiment.PulsedSource(GET_BOOL(j, "pulsed_source", false));
 
-        if (j.contains("omega_axis")) {
-            if (j["omega_axis"].is_array() && (j["omega_axis"].size() == 3))
-                experiment.DefaultOmegaAxis(Coord(j["omega_axis"][0].get<float>(),
-                                                  j["omega_axis"][1].get<float>(),
-                                                  j["omega_axis"][2].get<float>()));
+        if (j.contains("rotation_axis")) {
+            if (j["rotation_axis"].is_array() && (j["rotation_axis"].size() == 3))
+                experiment.DefaultRotationAxis(Coord(j["rotation_axis"][0].get<float>(),
+                                                     j["rotation_axis"][1].get<float>(),
+                                                     j["rotation_axis"][2].get<float>()));
             else
                 throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
-                                      "omega_axis must be float array of 3");
+                                      "rotation_axis must be float array of 3");
         }
 
         if (j.contains("pedestal_g0_frames"))
@@ -325,12 +360,11 @@ void ParseFacilityConfiguration(const nlohmann::json &input, const std::string&
             experiment.PedestalG1Frames(GET_I64(j, "pedestal_g1_frames"));
         if (j.contains("pedestal_g2_frames"))
             experiment.PedestalG2Frames(GET_I64(j, "pedestal_g2_frames"));
-        if (j.contains("detector_trigger_delay_us"))
-            experiment.DetectorDelay(GET_TIME(j, "detector_trigger_delay_us"));
+        if (j.contains("detector_trigger_delay"))
+            experiment.DetectorDelay(GET_TIME(j, "detector_trigger_delay"));
+
+        experiment.FrameTime(GET_TIME(j, "frame_time"), GET_TIME(j, "count_time"));
 
-        experiment.FrameTime(GET_TIME(j, "frame_time_us"), GET_TIME(j, "count_time_us"));
-        if (j.contains("preview_period_us"))
-            experiment.PreviewPeriod(GET_TIME(j, "preview_period_us"));
         experiment.UseInternalPacketGenerator(GET_BOOL(j, "internal_frame_generator", false));
         if (experiment.IsUsingInternalPacketGen())
             experiment.ConversionOnFPGA(false);

broker/JFJochBrokerParser.h

@@ -14,6 +14,7 @@ DetectorGeometry ParseDetectorGeometry(const nlohmann::json &j);
 DetectorSetup ParseDetectorSetup(const nlohmann::json &j);
 void ParseDetectorSetup(const nlohmann::json &j, const std::string& tag, JFJochBrokerHttp& broker);
 void ParseFacilityConfiguration(const nlohmann::json &j, const std::string& tag, DiffractionExperiment &experiment);
+void ParseImagePusher(const nlohmann::json &j, std::unique_ptr<ImagePusher> &image_pusher);
 
 void ParseAcquisitionDeviceGroup(const nlohmann::json &input, const std::string& tag, AcquisitionDeviceGroup &aq_devices);
 std::vector<std::string> ParseStringArray(const nlohmann::json &input, const std::string& tag);

broker/JFJochStateMachine.cpp

@@ -324,7 +324,7 @@ void JFJochStateMachine::Start(const DatasetSettings& settings) {
     else
         experiment.StorageCellStart(0);
 
-    experiment.IncrementSeriesID();
+    experiment.IncrementRunNumber();
 
     try {
         state = JFJochState::Busy;
@@ -340,12 +340,6 @@ void JFJochStateMachine::Start(const DatasetSettings& settings) {
     }
 }
 
-void JFJochStateMachine::WaitTillMeasurementDone() {
-std::unique_lock<std::mutex> ul(m);
-
-c.wait(ul, [&] { return !IsRunning(); });
-}
-
 void JFJochStateMachine::MeasurementThread() {
     try {
         auto tmp_output = services.Stop();
@@ -401,6 +395,9 @@ void JFJochStateMachine::SetFullMeasurementOutput(const JFJochServicesOutput &ou
     MeasurementStatistics tmp{}; // reset last measurement statistics
 
     tmp.file_prefix = experiment.GetFilePrefix();
+    tmp.run_number = experiment.GetRunNumber();
+    tmp.experiment_group = experiment.GetExperimentGroup();
+
     tmp.detector_width = experiment.GetXPixelsNum();
     tmp.detector_height = experiment.GetYPixelsNum();
     tmp.detector_pixel_depth = experiment.GetPixelDepth();
@@ -427,6 +424,9 @@ void JFJochStateMachine::ClearAndSetMeasurementStatistics() {
     MeasurementStatistics tmp{};
 
     tmp.file_prefix = experiment.GetFilePrefix();
+    tmp.run_number = experiment.GetRunNumber();
+    tmp.experiment_group = experiment.GetExperimentGroup();
+
     tmp.detector_height = experiment.GetXPixelsNum();
     tmp.detector_width = experiment.GetYPixelsNum();
     tmp.detector_pixel_depth = experiment.GetPixelDepth();
@@ -448,6 +448,9 @@ std::optional<MeasurementStatistics> JFJochStateMachine::GetMeasurementStatistic
         MeasurementStatistics tmp;
 
         tmp.file_prefix = experiment.GetFilePrefix();
+        tmp.run_number = experiment.GetRunNumber();
+        tmp.experiment_group = experiment.GetExperimentGroup();
+
         tmp.detector_width = experiment.GetXPixelsNum();
         tmp.detector_height = experiment.GetYPixelsNum();
         tmp.detector_pixel_depth = experiment.GetPixelDepth();
@@ -650,6 +653,14 @@ bool JFJochStateMachine::IsRunning() const {
     }
 }
 
+JFJochState JFJochStateMachine::WaitTillMeasurementDone() {
+    std::unique_lock<std::mutex> ul(m);
+
+    c.wait(ul, [&] { return !IsRunning(); });
+
+    return state;
+}
+
 JFJochState JFJochStateMachine::WaitTillMeasurementDone(std::chrono::milliseconds timeout) {
     std::unique_lock<std::mutex> ul(m);
 

broker/JFJochStateMachine.h

@@ -37,6 +37,9 @@ struct DetectorList {
 
 struct MeasurementStatistics {
     std::string file_prefix;
+    std::string experiment_group;
+    int64_t run_number;
+
     int64_t images_expected;
     int64_t images_collected;
     int64_t images_sent;
@@ -138,7 +141,7 @@ public:
     void Pedestal();
     void Deactivate();
     void Start(const DatasetSettings& settings);
-    void WaitTillMeasurementDone();
+    JFJochState WaitTillMeasurementDone();
     JFJochState WaitTillMeasurementDone(std::chrono::milliseconds timeout);
     void Trigger();
 

broker/gen/api/ApiBase.h

@@ -0,0 +1,39 @@
+/**
+* Jungfraujoch
+* Jungfraujoch Broker Web API
+*
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
+*
+* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
+* https://openapi-generator.tech
+* Do not edit the class manually.
+*/
+/*
+ * ApiBase.h
+ *
+ * Generalization of the Api classes
+ */
+
+#ifndef ApiBase_H_
+#define ApiBase_H_
+
+#include <pistache/router.h>
+#include <memory>
+
+namespace org::openapitools::server::api
+{
+
+class ApiBase {
+public:
+    explicit ApiBase(const std::shared_ptr<Pistache::Rest::Router>& rtr) : router(rtr) {};
+    virtual ~ApiBase() = default;
+    virtual void init() = 0;
+
+protected:
+    const std::shared_ptr<Pistache::Rest::Router> router;
+};
+
+} // namespace org::openapitools::server::api
+
+#endif /* ApiBase_H_ */

broker/gen/api/DefaultApi.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -22,7 +22,7 @@ using namespace org::openapitools::server::model;
 const std::string DefaultApi::base = "";
 
 DefaultApi::DefaultApi(const std::shared_ptr<Pistache::Rest::Router>& rtr)
-    : router(rtr)
+    : ApiBase(rtr)
 {
 }
 
@@ -79,6 +79,7 @@ void DefaultApi::setupRoutes() {
     Routes::Get(*router, base + "/statistics/data_collection", Routes::bind(&DefaultApi::statistics_data_collection_get_handler, this));
     Routes::Get(*router, base + "/status", Routes::bind(&DefaultApi::status_get_handler, this));
     Routes::Post(*router, base + "/trigger", Routes::bind(&DefaultApi::trigger_post_handler, this));
+    Routes::Get(*router, base + "/version", Routes::bind(&DefaultApi::version_get_handler, this));
     Routes::Post(*router, base + "/wait_till_done", Routes::bind(&DefaultApi::wait_till_done_post_handler, this));
     Routes::Get(*router, base + "/xfel/event_code", Routes::bind(&DefaultApi::xfel_event_code_get_handler, this));
     Routes::Get(*router, base + "/xfel/pulse_id", Routes::bind(&DefaultApi::xfel_pulse_id_get_handler, this));
@@ -1254,12 +1255,42 @@ void DefaultApi::trigger_post_handler(const Pistache::Rest::Request &, Pistache:
     }
 
 }
-void DefaultApi::wait_till_done_post_handler(const Pistache::Rest::Request &, Pistache::Http::ResponseWriter response) {
+void DefaultApi::version_get_handler(const Pistache::Rest::Request &, Pistache::Http::ResponseWriter response) {
     try {
 
 
     try {
-        this->wait_till_done_post(response);
+        this->version_get(response);
+    } catch (Pistache::Http::HttpError &e) {
+        response.send(static_cast<Pistache::Http::Code>(e.code()), e.what());
+        return;
+    } catch (std::exception &e) {
+        const std::pair<Pistache::Http::Code, std::string> errorInfo = this->handleOperationException(e);
+        response.send(errorInfo.first, errorInfo.second);
+        return;
+    }
+
+    } catch (std::exception &e) {
+        response.send(Pistache::Http::Code::Internal_Server_Error, e.what());
+    }
+
+}
+void DefaultApi::wait_till_done_post_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response) {
+    try {
+
+
+    // Getting the query params
+    auto timeoutQuery = request.query().get("timeout");
+    std::optional<int32_t> timeout;
+    if(timeoutQuery.has_value()){
+        int32_t valueQuery_instance;
+        if(fromStringValue(timeoutQuery.value(), valueQuery_instance)){
+            timeout = valueQuery_instance;
+        }
+    }
+    
+    try {
+        this->wait_till_done_post(timeout, response);
     } catch (Pistache::Http::HttpError &e) {
         response.send(static_cast<Pistache::Http::Code>(e.code()), e.what());
         return;

broker/gen/api/DefaultApi.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -19,6 +19,8 @@
 #define DefaultApi_H_
 
 
+#include "ApiBase.h"
+
 #include <pistache/http.h>
 #include <pistache/router.h>
 #include <pistache/http_headers.h>
@@ -42,15 +44,16 @@
 #include "Roi_circle_list.h"
 #include "Spot_finding_settings.h"
 #include <string>
+#include <vector>
 
 namespace org::openapitools::server::api
 {
 
-class  DefaultApi {
+class  DefaultApi : public ApiBase {
 public:
     explicit DefaultApi(const std::shared_ptr<Pistache::Rest::Router>& rtr);
-    virtual ~DefaultApi() = default;
-    void init();
+    ~DefaultApi() override = default;
+    void init() override;
 
     static const std::string base;
 
@@ -103,13 +106,12 @@ private:
     void statistics_data_collection_get_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
     void status_get_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
     void trigger_post_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
+    void version_get_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
     void wait_till_done_post_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
     void xfel_event_code_get_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
     void xfel_pulse_id_get_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
     void default_api_default_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
 
-    const std::shared_ptr<Pistache::Rest::Router> router;
-
     /// <summary>
     /// Helper function to handle unexpected Exceptions during Parameter parsing and validation.
     /// May be overridden to return custom error formats. This is called inside a catch block.
@@ -468,12 +470,20 @@ private:
     /// </remarks>
     virtual void trigger_post(Pistache::Http::ResponseWriter &response) = 0;
     /// <summary>
+    /// 
+    /// </summary>
+    /// <remarks>
+    /// 
+    /// </remarks>
+    virtual void version_get(Pistache::Http::ResponseWriter &response) = 0;
+    /// <summary>
     /// Wait for acquisition done
     /// </summary>
     /// <remarks>
-    /// Block execution of external script till initialization, data collection or pedestal is finished. Running this command does not affect (cancel) running data collection, it is only to ensure synchronous execution of other software.          To not block web server for a long period of time, the procedure is provided with a timeout of 5 seconds. 
+    /// Block execution of external script till initialization, data collection or pedestal is finished. Running this command does not affect (cancel) running data collection, it is only to ensure synchronous execution of other software.          To not block web server for a indefinite period of time, the procedure is provided with a timeout. Extending timeout is possible, but requires to ensure safety that client will not close the connection and retry the connection. 
     /// </remarks>
-    virtual void wait_till_done_post(Pistache::Http::ResponseWriter &response) = 0;
+    /// <param name="timeout">Timeout in seconds (0 &#x3D;&#x3D; immediate response) (optional, default to 60)</param>
+    virtual void wait_till_done_post(const std::optional<int32_t> &timeout, Pistache::Http::ResponseWriter &response) = 0;
     /// <summary>
     /// Return XFEL event codes for the current data acquisition
     /// </summary>

broker/gen/model/Broker_status.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -114,7 +114,7 @@ bool Broker_status::operator!=(const Broker_status& rhs) const
 
 void to_json(nlohmann::json& j, const Broker_status& o)
 {
-    j = nlohmann::json();
+    j = nlohmann::json::object();
     j["state"] = o.m_State;
     if(o.progressIsSet())
         j["progress"] = o.m_Progress;

broker/gen/model/Broker_status.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -78,8 +78,8 @@ public:
     bool indexingRateIsSet() const;
     void unsetIndexing_rate();
 
-    friend void to_json(nlohmann::json& j, const Broker_status& o);
-    friend void from_json(const nlohmann::json& j, Broker_status& o);
+    friend  void to_json(nlohmann::json& j, const Broker_status& o);
+    friend  void from_json(const nlohmann::json& j, Broker_status& o);
 protected:
     std::string m_State;
 

broker/gen/model/Calibration_statistics_inner.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -98,7 +98,7 @@ bool Calibration_statistics_inner::operator!=(const Calibration_statistics_inner
 
 void to_json(nlohmann::json& j, const Calibration_statistics_inner& o)
 {
-    j = nlohmann::json();
+    j = nlohmann::json::object();
     j["module_number"] = o.m_Module_number;
     j["storage_cell_number"] = o.m_Storage_cell_number;
     j["pedestal_g0_mean"] = o.m_Pedestal_g0_mean;

broker/gen/model/Calibration_statistics_inner.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -103,8 +103,8 @@ public:
     int64_t getMaskedPixels() const;
     void setMaskedPixels(int64_t const value);
 
-    friend void to_json(nlohmann::json& j, const Calibration_statistics_inner& o);
-    friend void from_json(const nlohmann::json& j, Calibration_statistics_inner& o);
+    friend  void to_json(nlohmann::json& j, const Calibration_statistics_inner& o);
+    friend  void from_json(const nlohmann::json& j, Calibration_statistics_inner& o);
 protected:
     int64_t m_Module_number;
 

broker/gen/model/Dataset_settings.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -25,12 +25,12 @@ Dataset_settings::Dataset_settings()
     m_Images_per_triggerIsSet = false;
     m_Ntrigger = 1L;
     m_NtriggerIsSet = false;
-    m_Summation = 1L;
-    m_SummationIsSet = false;
+    m_Image_time_us = 0L;
+    m_Image_time_usIsSet = false;
     m_Beam_x_pxl = 0.0f;
     m_Beam_y_pxl = 0.0f;
     m_Detector_distance_mm = 0.0f;
-    m_Photon_energy_keV = 0.0f;
+    m_Incident_energy_keV = 0.0f;
     m_File_prefix = "";
     m_File_prefixIsSet = false;
     m_Images_per_file = 1000L;
@@ -38,6 +38,7 @@ Dataset_settings::Dataset_settings()
     m_Space_group_number = 0L;
     m_Space_group_numberIsSet = false;
     m_Sample_name = "";
+    m_Sample_nameIsSet = false;
     m_Fpga_output = "auto";
     m_Fpga_outputIsSet = false;
     m_Compression = "bslz4";
@@ -46,15 +47,21 @@ Dataset_settings::Dataset_settings()
     m_Total_fluxIsSet = false;
     m_Transmission = 0.0f;
     m_TransmissionIsSet = false;
-    m_OmegaIsSet = false;
-    m_Header_appendix = "";
+    m_GoniometerIsSet = false;
     m_Header_appendixIsSet = false;
-    m_Image_appendix = "";
     m_Image_appendixIsSet = false;
-    m_Photon_energy_multiplier = 1.0f;
-    m_Photon_energy_multiplierIsSet = false;
+    m_Energy_multiplier = 1.0f;
+    m_Energy_multiplierIsSet = false;
     m_Data_reduction_factor_serialmx = 1.0f;
     m_Data_reduction_factor_serialmxIsSet = false;
+    m_Run_number = 0L;
+    m_Run_numberIsSet = false;
+    m_Run_name = "";
+    m_Run_nameIsSet = false;
+    m_Experiment_group = "";
+    m_Experiment_groupIsSet = false;
+    m_Poisson_compression = 0L;
+    m_Poisson_compressionIsSet = false;
     m_Unit_cellIsSet = false;
     
 }
@@ -107,21 +114,16 @@ bool Dataset_settings::validate(std::stringstream& msg, const std::string& pathP
 
     }
          
-    if (summationIsSet())
+    if (imageTimeUsIsSet())
     {
-        const int64_t& value = m_Summation;
-        const std::string currentValuePath = _pathPrefix + ".summation";
+        const int64_t& value = m_Image_time_us;
+        const std::string currentValuePath = _pathPrefix + ".imageTimeUs";
                 
         
-        if (value < 1ll)
+        if (value < 0ll)
         {
             success = false;
-            msg << currentValuePath << ": must be greater than or equal to 1;";
-        }
-        if (value > 256ll)
-        {
-            success = false;
-            msg << currentValuePath << ": must be less than or equal to 256;";
+            msg << currentValuePath << ": must be greater than or equal to 0;";
         }
 
     }
@@ -141,9 +143,9 @@ bool Dataset_settings::validate(std::stringstream& msg, const std::string& pathP
     }
          
     
-    /* Photon_energy_keV */ {
-        const float& value = m_Photon_energy_keV;
-        const std::string currentValuePath = _pathPrefix + ".photonEnergyKeV";
+    /* Incident_energy_keV */ {
+        const float& value = m_Incident_energy_keV;
+        const std::string currentValuePath = _pathPrefix + ".incidentEnergyKeV";
                 
         
         if (value < static_cast<float>(0))
@@ -206,10 +208,10 @@ bool Dataset_settings::validate(std::stringstream& msg, const std::string& pathP
 
     }
                      
-    if (photonEnergyMultiplierIsSet())
+    if (energyMultiplierIsSet())
     {
-        const float& value = m_Photon_energy_multiplier;
-        const std::string currentValuePath = _pathPrefix + ".photonEnergyMultiplier";
+        const float& value = m_Energy_multiplier;
+        const std::string currentValuePath = _pathPrefix + ".energyMultiplier";
                 
         
         if (value < static_cast<float>(0.015625))
@@ -243,6 +245,39 @@ bool Dataset_settings::validate(std::stringstream& msg, const std::string& pathP
         }
 
     }
+         
+    if (runNumberIsSet())
+    {
+        const int64_t& value = m_Run_number;
+        const std::string currentValuePath = _pathPrefix + ".runNumber";
+                
+        
+        if (value < 0ll)
+        {
+            success = false;
+            msg << currentValuePath << ": must be greater than or equal to 0;";
+        }
+
+    }
+                 
+    if (poissonCompressionIsSet())
+    {
+        const int64_t& value = m_Poisson_compression;
+        const std::string currentValuePath = _pathPrefix + ".poissonCompression";
+                
+        
+        if (value < 0ll)
+        {
+            success = false;
+            msg << currentValuePath << ": must be greater than or equal to 0;";
+        }
+        if (value > 16ll)
+        {
+            success = false;
+            msg << currentValuePath << ": must be less than or equal to 16;";
+        }
+
+    }
         
     return success;
 }
@@ -259,7 +294,7 @@ bool Dataset_settings::operator==(const Dataset_settings& rhs) const
     ((!ntriggerIsSet() && !rhs.ntriggerIsSet()) || (ntriggerIsSet() && rhs.ntriggerIsSet() && getNtrigger() == rhs.getNtrigger())) &&
     
     
-    ((!summationIsSet() && !rhs.summationIsSet()) || (summationIsSet() && rhs.summationIsSet() && getSummation() == rhs.getSummation())) &&
+    ((!imageTimeUsIsSet() && !rhs.imageTimeUsIsSet()) || (imageTimeUsIsSet() && rhs.imageTimeUsIsSet() && getImageTimeUs() == rhs.getImageTimeUs())) &&
     
     (getBeamXPxl() == rhs.getBeamXPxl())
      &&
@@ -270,7 +305,7 @@ bool Dataset_settings::operator==(const Dataset_settings& rhs) const
     (getDetectorDistanceMm() == rhs.getDetectorDistanceMm())
      &&
     
-    (getPhotonEnergyKeV() == rhs.getPhotonEnergyKeV())
+    (getIncidentEnergyKeV() == rhs.getIncidentEnergyKeV())
      &&
     
     
@@ -282,8 +317,8 @@ bool Dataset_settings::operator==(const Dataset_settings& rhs) const
     
     ((!spaceGroupNumberIsSet() && !rhs.spaceGroupNumberIsSet()) || (spaceGroupNumberIsSet() && rhs.spaceGroupNumberIsSet() && getSpaceGroupNumber() == rhs.getSpaceGroupNumber())) &&
     
-    (getSampleName() == rhs.getSampleName())
-     &&
+    
+    ((!sampleNameIsSet() && !rhs.sampleNameIsSet()) || (sampleNameIsSet() && rhs.sampleNameIsSet() && getSampleName() == rhs.getSampleName())) &&
     
     
     ((!fpgaOutputIsSet() && !rhs.fpgaOutputIsSet()) || (fpgaOutputIsSet() && rhs.fpgaOutputIsSet() && getFpgaOutput() == rhs.getFpgaOutput())) &&
@@ -298,7 +333,7 @@ bool Dataset_settings::operator==(const Dataset_settings& rhs) const
     ((!transmissionIsSet() && !rhs.transmissionIsSet()) || (transmissionIsSet() && rhs.transmissionIsSet() && getTransmission() == rhs.getTransmission())) &&
     
     
-    ((!omegaIsSet() && !rhs.omegaIsSet()) || (omegaIsSet() && rhs.omegaIsSet() && getOmega() == rhs.getOmega())) &&
+    ((!goniometerIsSet() && !rhs.goniometerIsSet()) || (goniometerIsSet() && rhs.goniometerIsSet() && getGoniometer() == rhs.getGoniometer())) &&
     
     
     ((!headerAppendixIsSet() && !rhs.headerAppendixIsSet()) || (headerAppendixIsSet() && rhs.headerAppendixIsSet() && getHeaderAppendix() == rhs.getHeaderAppendix())) &&
@@ -307,12 +342,24 @@ bool Dataset_settings::operator==(const Dataset_settings& rhs) const
     ((!imageAppendixIsSet() && !rhs.imageAppendixIsSet()) || (imageAppendixIsSet() && rhs.imageAppendixIsSet() && getImageAppendix() == rhs.getImageAppendix())) &&
     
     
-    ((!photonEnergyMultiplierIsSet() && !rhs.photonEnergyMultiplierIsSet()) || (photonEnergyMultiplierIsSet() && rhs.photonEnergyMultiplierIsSet() && getPhotonEnergyMultiplier() == rhs.getPhotonEnergyMultiplier())) &&
+    ((!energyMultiplierIsSet() && !rhs.energyMultiplierIsSet()) || (energyMultiplierIsSet() && rhs.energyMultiplierIsSet() && getEnergyMultiplier() == rhs.getEnergyMultiplier())) &&
     
     
     ((!dataReductionFactorSerialmxIsSet() && !rhs.dataReductionFactorSerialmxIsSet()) || (dataReductionFactorSerialmxIsSet() && rhs.dataReductionFactorSerialmxIsSet() && getDataReductionFactorSerialmx() == rhs.getDataReductionFactorSerialmx())) &&
     
     
+    ((!runNumberIsSet() && !rhs.runNumberIsSet()) || (runNumberIsSet() && rhs.runNumberIsSet() && getRunNumber() == rhs.getRunNumber())) &&
+    
+    
+    ((!runNameIsSet() && !rhs.runNameIsSet()) || (runNameIsSet() && rhs.runNameIsSet() && getRunName() == rhs.getRunName())) &&
+    
+    
+    ((!experimentGroupIsSet() && !rhs.experimentGroupIsSet()) || (experimentGroupIsSet() && rhs.experimentGroupIsSet() && getExperimentGroup() == rhs.getExperimentGroup())) &&
+    
+    
+    ((!poissonCompressionIsSet() && !rhs.poissonCompressionIsSet()) || (poissonCompressionIsSet() && rhs.poissonCompressionIsSet() && getPoissonCompression() == rhs.getPoissonCompression())) &&
+    
+    
     ((!unitCellIsSet() && !rhs.unitCellIsSet()) || (unitCellIsSet() && rhs.unitCellIsSet() && getUnitCell() == rhs.getUnitCell()))
     
     ;
@@ -325,24 +372,25 @@ bool Dataset_settings::operator!=(const Dataset_settings& rhs) const
 
 void to_json(nlohmann::json& j, const Dataset_settings& o)
 {
-    j = nlohmann::json();
+    j = nlohmann::json::object();
     if(o.imagesPerTriggerIsSet())
         j["images_per_trigger"] = o.m_Images_per_trigger;
     if(o.ntriggerIsSet())
         j["ntrigger"] = o.m_Ntrigger;
-    if(o.summationIsSet())
-        j["summation"] = o.m_Summation;
+    if(o.imageTimeUsIsSet())
+        j["image_time_us"] = o.m_Image_time_us;
     j["beam_x_pxl"] = o.m_Beam_x_pxl;
     j["beam_y_pxl"] = o.m_Beam_y_pxl;
     j["detector_distance_mm"] = o.m_Detector_distance_mm;
-    j["photon_energy_keV"] = o.m_Photon_energy_keV;
+    j["incident_energy_keV"] = o.m_Incident_energy_keV;
     if(o.filePrefixIsSet())
         j["file_prefix"] = o.m_File_prefix;
     if(o.imagesPerFileIsSet())
         j["images_per_file"] = o.m_Images_per_file;
     if(o.spaceGroupNumberIsSet())
         j["space_group_number"] = o.m_Space_group_number;
-    j["sample_name"] = o.m_Sample_name;
+    if(o.sampleNameIsSet())
+        j["sample_name"] = o.m_Sample_name;
     if(o.fpgaOutputIsSet())
         j["fpga_output"] = o.m_Fpga_output;
     if(o.compressionIsSet())
@@ -351,16 +399,24 @@ void to_json(nlohmann::json& j, const Dataset_settings& o)
         j["total_flux"] = o.m_Total_flux;
     if(o.transmissionIsSet())
         j["transmission"] = o.m_Transmission;
-    if(o.omegaIsSet())
-        j["omega"] = o.m_Omega;
+    if(o.goniometerIsSet())
+        j["goniometer"] = o.m_Goniometer;
     if(o.headerAppendixIsSet())
         j["header_appendix"] = o.m_Header_appendix;
     if(o.imageAppendixIsSet())
         j["image_appendix"] = o.m_Image_appendix;
-    if(o.photonEnergyMultiplierIsSet())
-        j["photon_energy_multiplier"] = o.m_Photon_energy_multiplier;
+    if(o.energyMultiplierIsSet())
+        j["energy_multiplier"] = o.m_Energy_multiplier;
     if(o.dataReductionFactorSerialmxIsSet())
         j["data_reduction_factor_serialmx"] = o.m_Data_reduction_factor_serialmx;
+    if(o.runNumberIsSet())
+        j["run_number"] = o.m_Run_number;
+    if(o.runNameIsSet())
+        j["run_name"] = o.m_Run_name;
+    if(o.experimentGroupIsSet())
+        j["experiment_group"] = o.m_Experiment_group;
+    if(o.poissonCompressionIsSet())
+        j["poisson_compression"] = o.m_Poisson_compression;
     if(o.unitCellIsSet())
         j["unit_cell"] = o.m_Unit_cell;
     
@@ -378,15 +434,15 @@ void from_json(const nlohmann::json& j, Dataset_settings& o)
         j.at("ntrigger").get_to(o.m_Ntrigger);
         o.m_NtriggerIsSet = true;
     } 
-    if(j.find("summation") != j.end())
+    if(j.find("image_time_us") != j.end())
     {
-        j.at("summation").get_to(o.m_Summation);
-        o.m_SummationIsSet = true;
+        j.at("image_time_us").get_to(o.m_Image_time_us);
+        o.m_Image_time_usIsSet = true;
     } 
     j.at("beam_x_pxl").get_to(o.m_Beam_x_pxl);
     j.at("beam_y_pxl").get_to(o.m_Beam_y_pxl);
     j.at("detector_distance_mm").get_to(o.m_Detector_distance_mm);
-    j.at("photon_energy_keV").get_to(o.m_Photon_energy_keV);
+    j.at("incident_energy_keV").get_to(o.m_Incident_energy_keV);
     if(j.find("file_prefix") != j.end())
     {
         j.at("file_prefix").get_to(o.m_File_prefix);
@@ -402,7 +458,11 @@ void from_json(const nlohmann::json& j, Dataset_settings& o)
         j.at("space_group_number").get_to(o.m_Space_group_number);
         o.m_Space_group_numberIsSet = true;
     } 
-    j.at("sample_name").get_to(o.m_Sample_name);
+    if(j.find("sample_name") != j.end())
+    {
+        j.at("sample_name").get_to(o.m_Sample_name);
+        o.m_Sample_nameIsSet = true;
+    } 
     if(j.find("fpga_output") != j.end())
     {
         j.at("fpga_output").get_to(o.m_Fpga_output);
@@ -423,10 +483,10 @@ void from_json(const nlohmann::json& j, Dataset_settings& o)
         j.at("transmission").get_to(o.m_Transmission);
         o.m_TransmissionIsSet = true;
     } 
-    if(j.find("omega") != j.end())
+    if(j.find("goniometer") != j.end())
     {
-        j.at("omega").get_to(o.m_Omega);
-        o.m_OmegaIsSet = true;
+        j.at("goniometer").get_to(o.m_Goniometer);
+        o.m_GoniometerIsSet = true;
     } 
     if(j.find("header_appendix") != j.end())
     {
@@ -438,16 +498,36 @@ void from_json(const nlohmann::json& j, Dataset_settings& o)
         j.at("image_appendix").get_to(o.m_Image_appendix);
         o.m_Image_appendixIsSet = true;
     } 
-    if(j.find("photon_energy_multiplier") != j.end())
+    if(j.find("energy_multiplier") != j.end())
     {
-        j.at("photon_energy_multiplier").get_to(o.m_Photon_energy_multiplier);
-        o.m_Photon_energy_multiplierIsSet = true;
+        j.at("energy_multiplier").get_to(o.m_Energy_multiplier);
+        o.m_Energy_multiplierIsSet = true;
     } 
     if(j.find("data_reduction_factor_serialmx") != j.end())
     {
         j.at("data_reduction_factor_serialmx").get_to(o.m_Data_reduction_factor_serialmx);
         o.m_Data_reduction_factor_serialmxIsSet = true;
     } 
+    if(j.find("run_number") != j.end())
+    {
+        j.at("run_number").get_to(o.m_Run_number);
+        o.m_Run_numberIsSet = true;
+    } 
+    if(j.find("run_name") != j.end())
+    {
+        j.at("run_name").get_to(o.m_Run_name);
+        o.m_Run_nameIsSet = true;
+    } 
+    if(j.find("experiment_group") != j.end())
+    {
+        j.at("experiment_group").get_to(o.m_Experiment_group);
+        o.m_Experiment_groupIsSet = true;
+    } 
+    if(j.find("poisson_compression") != j.end())
+    {
+        j.at("poisson_compression").get_to(o.m_Poisson_compression);
+        o.m_Poisson_compressionIsSet = true;
+    } 
     if(j.find("unit_cell") != j.end())
     {
         j.at("unit_cell").get_to(o.m_Unit_cell);
@@ -490,22 +570,22 @@ void Dataset_settings::unsetNtrigger()
 {
     m_NtriggerIsSet = false;
 }
-int64_t Dataset_settings::getSummation() const
+int64_t Dataset_settings::getImageTimeUs() const
 {
-    return m_Summation;
+    return m_Image_time_us;
 }
-void Dataset_settings::setSummation(int64_t const value)
+void Dataset_settings::setImageTimeUs(int64_t const value)
 {
-    m_Summation = value;
-    m_SummationIsSet = true;
+    m_Image_time_us = value;
+    m_Image_time_usIsSet = true;
 }
-bool Dataset_settings::summationIsSet() const
+bool Dataset_settings::imageTimeUsIsSet() const
 {
-    return m_SummationIsSet;
+    return m_Image_time_usIsSet;
 }
-void Dataset_settings::unsetSummation()
+void Dataset_settings::unsetImage_time_us()
 {
-    m_SummationIsSet = false;
+    m_Image_time_usIsSet = false;
 }
 float Dataset_settings::getBeamXPxl() const
 {
@@ -531,13 +611,13 @@ void Dataset_settings::setDetectorDistanceMm(float const value)
 {
     m_Detector_distance_mm = value;
 }
-float Dataset_settings::getPhotonEnergyKeV() const
+float Dataset_settings::getIncidentEnergyKeV() const
 {
-    return m_Photon_energy_keV;
+    return m_Incident_energy_keV;
 }
-void Dataset_settings::setPhotonEnergyKeV(float const value)
+void Dataset_settings::setIncidentEnergyKeV(float const value)
 {
-    m_Photon_energy_keV = value;
+    m_Incident_energy_keV = value;
 }
 std::string Dataset_settings::getFilePrefix() const
 {
@@ -597,6 +677,15 @@ std::string Dataset_settings::getSampleName() const
 void Dataset_settings::setSampleName(std::string const& value)
 {
     m_Sample_name = value;
+    m_Sample_nameIsSet = true;
+}
+bool Dataset_settings::sampleNameIsSet() const
+{
+    return m_Sample_nameIsSet;
+}
+void Dataset_settings::unsetSample_name()
+{
+    m_Sample_nameIsSet = false;
 }
 std::string Dataset_settings::getFpgaOutput() const
 {
@@ -666,28 +755,28 @@ void Dataset_settings::unsetTransmission()
 {
     m_TransmissionIsSet = false;
 }
-org::openapitools::server::model::Rotation_axis Dataset_settings::getOmega() const
+org::openapitools::server::model::Rotation_axis Dataset_settings::getGoniometer() const
 {
-    return m_Omega;
+    return m_Goniometer;
 }
-void Dataset_settings::setOmega(org::openapitools::server::model::Rotation_axis const& value)
+void Dataset_settings::setGoniometer(org::openapitools::server::model::Rotation_axis const& value)
 {
-    m_Omega = value;
-    m_OmegaIsSet = true;
+    m_Goniometer = value;
+    m_GoniometerIsSet = true;
 }
-bool Dataset_settings::omegaIsSet() const
+bool Dataset_settings::goniometerIsSet() const
 {
-    return m_OmegaIsSet;
+    return m_GoniometerIsSet;
 }
-void Dataset_settings::unsetOmega()
+void Dataset_settings::unsetGoniometer()
 {
-    m_OmegaIsSet = false;
+    m_GoniometerIsSet = false;
 }
-std::string Dataset_settings::getHeaderAppendix() const
+nlohmann::json Dataset_settings::getHeaderAppendix() const
 {
     return m_Header_appendix;
 }
-void Dataset_settings::setHeaderAppendix(std::string const& value)
+void Dataset_settings::setHeaderAppendix(nlohmann::json const& value)
 {
     m_Header_appendix = value;
     m_Header_appendixIsSet = true;
@@ -700,11 +789,11 @@ void Dataset_settings::unsetHeader_appendix()
 {
     m_Header_appendixIsSet = false;
 }
-std::string Dataset_settings::getImageAppendix() const
+nlohmann::json Dataset_settings::getImageAppendix() const
 {
     return m_Image_appendix;
 }
-void Dataset_settings::setImageAppendix(std::string const& value)
+void Dataset_settings::setImageAppendix(nlohmann::json const& value)
 {
     m_Image_appendix = value;
     m_Image_appendixIsSet = true;
@@ -717,22 +806,22 @@ void Dataset_settings::unsetImage_appendix()
 {
     m_Image_appendixIsSet = false;
 }
-float Dataset_settings::getPhotonEnergyMultiplier() const
+float Dataset_settings::getEnergyMultiplier() const
 {
-    return m_Photon_energy_multiplier;
+    return m_Energy_multiplier;
 }
-void Dataset_settings::setPhotonEnergyMultiplier(float const value)
+void Dataset_settings::setEnergyMultiplier(float const value)
 {
-    m_Photon_energy_multiplier = value;
-    m_Photon_energy_multiplierIsSet = true;
+    m_Energy_multiplier = value;
+    m_Energy_multiplierIsSet = true;
 }
-bool Dataset_settings::photonEnergyMultiplierIsSet() const
+bool Dataset_settings::energyMultiplierIsSet() const
 {
-    return m_Photon_energy_multiplierIsSet;
+    return m_Energy_multiplierIsSet;
 }
-void Dataset_settings::unsetPhoton_energy_multiplier()
+void Dataset_settings::unsetEnergy_multiplier()
 {
-    m_Photon_energy_multiplierIsSet = false;
+    m_Energy_multiplierIsSet = false;
 }
 float Dataset_settings::getDataReductionFactorSerialmx() const
 {
@@ -751,6 +840,74 @@ void Dataset_settings::unsetData_reduction_factor_serialmx()
 {
     m_Data_reduction_factor_serialmxIsSet = false;
 }
+int64_t Dataset_settings::getRunNumber() const
+{
+    return m_Run_number;
+}
+void Dataset_settings::setRunNumber(int64_t const value)
+{
+    m_Run_number = value;
+    m_Run_numberIsSet = true;
+}
+bool Dataset_settings::runNumberIsSet() const
+{
+    return m_Run_numberIsSet;
+}
+void Dataset_settings::unsetRun_number()
+{
+    m_Run_numberIsSet = false;
+}
+std::string Dataset_settings::getRunName() const
+{
+    return m_Run_name;
+}
+void Dataset_settings::setRunName(std::string const& value)
+{
+    m_Run_name = value;
+    m_Run_nameIsSet = true;
+}
+bool Dataset_settings::runNameIsSet() const
+{
+    return m_Run_nameIsSet;
+}
+void Dataset_settings::unsetRun_name()
+{
+    m_Run_nameIsSet = false;
+}
+std::string Dataset_settings::getExperimentGroup() const
+{
+    return m_Experiment_group;
+}
+void Dataset_settings::setExperimentGroup(std::string const& value)
+{
+    m_Experiment_group = value;
+    m_Experiment_groupIsSet = true;
+}
+bool Dataset_settings::experimentGroupIsSet() const
+{
+    return m_Experiment_groupIsSet;
+}
+void Dataset_settings::unsetExperiment_group()
+{
+    m_Experiment_groupIsSet = false;
+}
+int64_t Dataset_settings::getPoissonCompression() const
+{
+    return m_Poisson_compression;
+}
+void Dataset_settings::setPoissonCompression(int64_t const value)
+{
+    m_Poisson_compression = value;
+    m_Poisson_compressionIsSet = true;
+}
+bool Dataset_settings::poissonCompressionIsSet() const
+{
+    return m_Poisson_compressionIsSet;
+}
+void Dataset_settings::unsetPoisson_compression()
+{
+    m_Poisson_compressionIsSet = false;
+}
 org::openapitools::server::model::Dataset_settings_unit_cell Dataset_settings::getUnitCell() const
 {
     return m_Unit_cell;

broker/gen/model/Dataset_settings.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -19,6 +19,7 @@
 #define Dataset_settings_H_
 
 
+#include <nlohmann/json.hpp>
 #include "Rotation_axis.h"
 #include <string>
 #include "Dataset_settings_unit_cell.h"
@@ -75,12 +76,12 @@ public:
     bool ntriggerIsSet() const;
     void unsetNtrigger();
     /// <summary>
-    /// FPGA frame summation. For summation above two 32-bit pixel format will be used, unless explicitly specified. Frame summation factor applies only to conversion mode (assumed as 1 for raw data). In XFEL mode: summation happens for frames collected with multiple triggers. Ignored for storage cells (assumed as 1). 
+    /// Image time.  If not provided (or zero value) the frame time is assumed as default. Image time must be multiple of frame time; max value is 256 * frame_time.             In XFEL mode: summation happens for frames collected with multiple triggers. Ignored for storage cells and if raw data are saved. 
     /// </summary>
-    int64_t getSummation() const;
-    void setSummation(int64_t const value);
-    bool summationIsSet() const;
-    void unsetSummation();
+    int64_t getImageTimeUs() const;
+    void setImageTimeUs(int64_t const value);
+    bool imageTimeUsIsSet() const;
+    void unsetImage_time_us();
     /// <summary>
     /// /entry/detector/beam_center_x in NXmx Beam center in X direction [pixels] 
     /// </summary>
@@ -97,10 +98,10 @@ public:
     float getDetectorDistanceMm() const;
     void setDetectorDistanceMm(float const value);
     /// <summary>
-    /// Used to calculate /entry/beam/incident_wavelength in NXmx Incident photon energy in keV 
+    /// Used to calculate /entry/beam/incident_wavelength in NXmx Incident particle (photon, electron) energy in keV 
     /// </summary>
-    float getPhotonEnergyKeV() const;
-    void setPhotonEnergyKeV(float const value);
+    float getIncidentEnergyKeV() const;
+    void setIncidentEnergyKeV(float const value);
     /// <summary>
     /// Prefix for filenames. If left empty, no file will be saved.
     /// </summary>
@@ -127,6 +128,8 @@ public:
     /// </summary>
     std::string getSampleName() const;
     void setSampleName(std::string const& value);
+    bool sampleNameIsSet() const;
+    void unsetSample_name();
     /// <summary>
     /// FPGA output data type
     /// </summary>
@@ -158,31 +161,31 @@ public:
     /// <summary>
     /// 
     /// </summary>
-    org::openapitools::server::model::Rotation_axis getOmega() const;
-    void setOmega(org::openapitools::server::model::Rotation_axis const& value);
-    bool omegaIsSet() const;
-    void unsetOmega();
+    org::openapitools::server::model::Rotation_axis getGoniometer() const;
+    void setGoniometer(org::openapitools::server::model::Rotation_axis const& value);
+    bool goniometerIsSet() const;
+    void unsetGoniometer();
     /// <summary>
-    /// Header appendix, added as user_data to start message
+    /// Header appendix, added as user_data/user to start message (can be any valid JSON)
     /// </summary>
-    std::string getHeaderAppendix() const;
-    void setHeaderAppendix(std::string const& value);
+    nlohmann::json getHeaderAppendix() const;
+    void setHeaderAppendix(nlohmann::json const& value);
     bool headerAppendixIsSet() const;
     void unsetHeader_appendix();
     /// <summary>
-    /// Image appendix, added as user_data to image message
+    /// Image appendix, added as user_data to image message (can be any valid JSON)
     /// </summary>
-    std::string getImageAppendix() const;
-    void setImageAppendix(std::string const& value);
+    nlohmann::json getImageAppendix() const;
+    void setImageAppendix(nlohmann::json const& value);
     bool imageAppendixIsSet() const;
     void unsetImage_appendix();
     /// <summary>
-    /// For JUNGFRAU conversion it is possible to multiply energy by a given factor to get fractional/multiplied photon counts
+    /// For JUNGFRAU conversion it is possible to multiply incident energy by a given factor to get fractional/multiplied particle counts
     /// </summary>
-    float getPhotonEnergyMultiplier() const;
-    void setPhotonEnergyMultiplier(float const value);
-    bool photonEnergyMultiplierIsSet() const;
-    void unsetPhoton_energy_multiplier();
+    float getEnergyMultiplier() const;
+    void setEnergyMultiplier(float const value);
+    bool energyMultiplierIsSet() const;
+    void unsetEnergy_multiplier();
     /// <summary>
     /// Rate at which non-indexed images are accepted to be forwarded to writer.  Value of 1.0 (default) means that all images are written. Values below zero mean that non-indexed images will be accepted with a given probability. 
     /// </summary>
@@ -191,6 +194,34 @@ public:
     bool dataReductionFactorSerialmxIsSet() const;
     void unsetData_reduction_factor_serialmx();
     /// <summary>
+    /// Number of run within an experimental session.  Transferred over CBOR stream as \&quot;series ID\&quot;, though not saved in HDF5 file. It is highly recommended to keep this number unique for each data collection during experimental series. If not provided, the number will be automatically incremented. 
+    /// </summary>
+    int64_t getRunNumber() const;
+    void setRunNumber(int64_t const value);
+    bool runNumberIsSet() const;
+    void unsetRun_number();
+    /// <summary>
+    /// Unique ID of run. Transferred over CBOR stream as \&quot;unique series ID\&quot;, though not saved in HDF5 file. It is highly recommended to keep this name unique for each data collection during experimental series. If not provided, the name will be automatically generated as number + colon + file_prefix. 
+    /// </summary>
+    std::string getRunName() const;
+    void setRunName(std::string const& value);
+    bool runNameIsSet() const;
+    void unsetRun_name();
+    /// <summary>
+    /// Name of group owning the data (e.g. p-group or proposal number).  Transferred over CBOR stream, though not saved in HDF5 file. 
+    /// </summary>
+    std::string getExperimentGroup() const;
+    void setExperimentGroup(std::string const& value);
+    bool experimentGroupIsSet() const;
+    void unsetExperiment_group();
+    /// <summary>
+    /// Enable lossy compression of pixel values that preserves Poisson statistics.  Requires to provide a numerical factor SQ. Pixel value P will be transformed to round(sqrt(P) * SQ), with rounding to the closest integer. Compression is turned off if the value is missing or it is set to zero. 
+    /// </summary>
+    int64_t getPoissonCompression() const;
+    void setPoissonCompression(int64_t const value);
+    bool poissonCompressionIsSet() const;
+    void unsetPoisson_compression();
+    /// <summary>
     /// 
     /// </summary>
     org::openapitools::server::model::Dataset_settings_unit_cell getUnitCell() const;
@@ -198,22 +229,22 @@ public:
     bool unitCellIsSet() const;
     void unsetUnit_cell();
 
-    friend void to_json(nlohmann::json& j, const Dataset_settings& o);
-    friend void from_json(const nlohmann::json& j, Dataset_settings& o);
+    friend  void to_json(nlohmann::json& j, const Dataset_settings& o);
+    friend  void from_json(const nlohmann::json& j, Dataset_settings& o);
 protected:
     int64_t m_Images_per_trigger;
     bool m_Images_per_triggerIsSet;
     int64_t m_Ntrigger;
     bool m_NtriggerIsSet;
-    int64_t m_Summation;
-    bool m_SummationIsSet;
+    int64_t m_Image_time_us;
+    bool m_Image_time_usIsSet;
     float m_Beam_x_pxl;
 
     float m_Beam_y_pxl;
 
     float m_Detector_distance_mm;
 
-    float m_Photon_energy_keV;
+    float m_Incident_energy_keV;
 
     std::string m_File_prefix;
     bool m_File_prefixIsSet;
@@ -222,7 +253,7 @@ protected:
     int64_t m_Space_group_number;
     bool m_Space_group_numberIsSet;
     std::string m_Sample_name;
-
+    bool m_Sample_nameIsSet;
     std::string m_Fpga_output;
     bool m_Fpga_outputIsSet;
     std::string m_Compression;
@@ -231,16 +262,24 @@ protected:
     bool m_Total_fluxIsSet;
     float m_Transmission;
     bool m_TransmissionIsSet;
-    org::openapitools::server::model::Rotation_axis m_Omega;
-    bool m_OmegaIsSet;
-    std::string m_Header_appendix;
+    org::openapitools::server::model::Rotation_axis m_Goniometer;
+    bool m_GoniometerIsSet;
+    nlohmann::json m_Header_appendix;
     bool m_Header_appendixIsSet;
-    std::string m_Image_appendix;
+    nlohmann::json m_Image_appendix;
     bool m_Image_appendixIsSet;
-    float m_Photon_energy_multiplier;
-    bool m_Photon_energy_multiplierIsSet;
+    float m_Energy_multiplier;
+    bool m_Energy_multiplierIsSet;
     float m_Data_reduction_factor_serialmx;
     bool m_Data_reduction_factor_serialmxIsSet;
+    int64_t m_Run_number;
+    bool m_Run_numberIsSet;
+    std::string m_Run_name;
+    bool m_Run_nameIsSet;
+    std::string m_Experiment_group;
+    bool m_Experiment_groupIsSet;
+    int64_t m_Poisson_compression;
+    bool m_Poisson_compressionIsSet;
     org::openapitools::server::model::Dataset_settings_unit_cell m_Unit_cell;
     bool m_Unit_cellIsSet;
     

broker/gen/model/Dataset_settings_roi_sum_area.cpp

@@ -1,133 +0,0 @@
-/**
-* Jungfraujoch
-* Jungfraujoch Broker Web API
-*
-* The version of the OpenAPI document: 1.0.1
-* 
-*
-* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
-* https://openapi-generator.tech
-* Do not edit the class manually.
-*/
-
-
-#include "Dataset_settings_roi_sum_area.h"
-#include "Helpers.h"
-
-#include <sstream>
-
-namespace org::openapitools::server::model
-{
-
-Dataset_settings_roi_sum_area::Dataset_settings_roi_sum_area()
-{
-    m_X_min = 0L;
-    m_X_max = 0L;
-    m_Y_min = 0L;
-    m_Y_max = 0L;
-    
-}
-
-void Dataset_settings_roi_sum_area::validate() const
-{
-    std::stringstream msg;
-    if (!validate(msg))
-    {
-        throw org::openapitools::server::helpers::ValidationException(msg.str());
-    }
-}
-
-bool Dataset_settings_roi_sum_area::validate(std::stringstream& msg) const
-{
-    return validate(msg, "");
-}
-
-bool Dataset_settings_roi_sum_area::validate(std::stringstream& msg, const std::string& pathPrefix) const
-{
-    bool success = true;
-    const std::string _pathPrefix = pathPrefix.empty() ? "Dataset_settings_roi_sum_area" : pathPrefix;
-
-                    
-    return success;
-}
-
-bool Dataset_settings_roi_sum_area::operator==(const Dataset_settings_roi_sum_area& rhs) const
-{
-    return
-    
-    
-    (getXMin() == rhs.getXMin())
-     &&
-    
-    (getXMax() == rhs.getXMax())
-     &&
-    
-    (getYMin() == rhs.getYMin())
-     &&
-    
-    (getYMax() == rhs.getYMax())
-    
-    
-    ;
-}
-
-bool Dataset_settings_roi_sum_area::operator!=(const Dataset_settings_roi_sum_area& rhs) const
-{
-    return !(*this == rhs);
-}
-
-void to_json(nlohmann::json& j, const Dataset_settings_roi_sum_area& o)
-{
-    j = nlohmann::json();
-    j["x_min"] = o.m_X_min;
-    j["x_max"] = o.m_X_max;
-    j["y_min"] = o.m_Y_min;
-    j["y_max"] = o.m_Y_max;
-    
-}
-
-void from_json(const nlohmann::json& j, Dataset_settings_roi_sum_area& o)
-{
-    j.at("x_min").get_to(o.m_X_min);
-    j.at("x_max").get_to(o.m_X_max);
-    j.at("y_min").get_to(o.m_Y_min);
-    j.at("y_max").get_to(o.m_Y_max);
-    
-}
-
-int64_t Dataset_settings_roi_sum_area::getXMin() const
-{
-    return m_X_min;
-}
-void Dataset_settings_roi_sum_area::setXMin(int64_t const value)
-{
-    m_X_min = value;
-}
-int64_t Dataset_settings_roi_sum_area::getXMax() const
-{
-    return m_X_max;
-}
-void Dataset_settings_roi_sum_area::setXMax(int64_t const value)
-{
-    m_X_max = value;
-}
-int64_t Dataset_settings_roi_sum_area::getYMin() const
-{
-    return m_Y_min;
-}
-void Dataset_settings_roi_sum_area::setYMin(int64_t const value)
-{
-    m_Y_min = value;
-}
-int64_t Dataset_settings_roi_sum_area::getYMax() const
-{
-    return m_Y_max;
-}
-void Dataset_settings_roi_sum_area::setYMax(int64_t const value)
-{
-    m_Y_max = value;
-}
-
-
-} // namespace org::openapitools::server::model
-

broker/gen/model/Dataset_settings_roi_sum_area.h

@@ -1,97 +0,0 @@
-/**
-* Jungfraujoch
-* Jungfraujoch Broker Web API
-*
-* The version of the OpenAPI document: 1.0.1
-* 
-*
-* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
-* https://openapi-generator.tech
-* Do not edit the class manually.
-*/
-/*
- * Dataset_settings_roi_sum_area.h
- *
- * Rectangle for ROI summation
- */
-
-#ifndef Dataset_settings_roi_sum_area_H_
-#define Dataset_settings_roi_sum_area_H_
-
-
-#include <nlohmann/json.hpp>
-
-namespace org::openapitools::server::model
-{
-
-/// <summary>
-/// Rectangle for ROI summation
-/// </summary>
-class  Dataset_settings_roi_sum_area
-{
-public:
-    Dataset_settings_roi_sum_area();
-    virtual ~Dataset_settings_roi_sum_area() = default;
-
-
-    /// <summary>
-    /// Validate the current data in the model. Throws a ValidationException on failure.
-    /// </summary>
-    void validate() const;
-
-    /// <summary>
-    /// Validate the current data in the model. Returns false on error and writes an error
-    /// message into the given stringstream.
-    /// </summary>
-    bool validate(std::stringstream& msg) const;
-
-    /// <summary>
-    /// Helper overload for validate. Used when one model stores another model and calls it's validate.
-    /// Not meant to be called outside that case.
-    /// </summary>
-    bool validate(std::stringstream& msg, const std::string& pathPrefix) const;
-
-    bool operator==(const Dataset_settings_roi_sum_area& rhs) const;
-    bool operator!=(const Dataset_settings_roi_sum_area& rhs) const;
-
-    /////////////////////////////////////////////
-    /// Dataset_settings_roi_sum_area members
-
-    /// <summary>
-    /// 
-    /// </summary>
-    int64_t getXMin() const;
-    void setXMin(int64_t const value);
-    /// <summary>
-    /// 
-    /// </summary>
-    int64_t getXMax() const;
-    void setXMax(int64_t const value);
-    /// <summary>
-    /// 
-    /// </summary>
-    int64_t getYMin() const;
-    void setYMin(int64_t const value);
-    /// <summary>
-    /// 
-    /// </summary>
-    int64_t getYMax() const;
-    void setYMax(int64_t const value);
-
-    friend void to_json(nlohmann::json& j, const Dataset_settings_roi_sum_area& o);
-    friend void from_json(const nlohmann::json& j, Dataset_settings_roi_sum_area& o);
-protected:
-    int64_t m_X_min;
-
-    int64_t m_X_max;
-
-    int64_t m_Y_min;
-
-    int64_t m_Y_max;
-
-    
-};
-
-} // namespace org::openapitools::server::model
-
-#endif /* Dataset_settings_roi_sum_area_H_ */

broker/gen/model/Dataset_settings_unit_cell.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -185,7 +185,7 @@ bool Dataset_settings_unit_cell::operator!=(const Dataset_settings_unit_cell& rh
 
 void to_json(nlohmann::json& j, const Dataset_settings_unit_cell& o)
 {
-    j = nlohmann::json();
+    j = nlohmann::json::object();
     j["a"] = o.m_a;
     j["b"] = o.m_b;
     j["c"] = o.m_c;

broker/gen/model/Dataset_settings_unit_cell.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -88,8 +88,8 @@ public:
     float getGamma() const;
     void setGamma(float const value);
 
-    friend void to_json(nlohmann::json& j, const Dataset_settings_unit_cell& o);
-    friend void from_json(const nlohmann::json& j, Dataset_settings_unit_cell& o);
+    friend  void to_json(nlohmann::json& j, const Dataset_settings_unit_cell& o);
+    friend  void from_json(const nlohmann::json& j, Dataset_settings_unit_cell& o);
 protected:
     float m_a;
 

broker/gen/model/Detector_list.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -90,7 +90,7 @@ bool Detector_list::operator!=(const Detector_list& rhs) const
 
 void to_json(nlohmann::json& j, const Detector_list& o)
 {
-    j = nlohmann::json();
+    j = nlohmann::json::object();
     j["detectors"] = o.m_Detectors;
     j["current_id"] = o.m_Current_id;
     

broker/gen/model/Detector_list.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -70,8 +70,8 @@ public:
     int64_t getCurrentId() const;
     void setCurrentId(int64_t const value);
 
-    friend void to_json(nlohmann::json& j, const Detector_list& o);
-    friend void from_json(const nlohmann::json& j, Detector_list& o);
+    friend  void to_json(nlohmann::json& j, const Detector_list& o);
+    friend  void from_json(const nlohmann::json& j, Detector_list& o);
 protected:
     std::vector<org::openapitools::server::model::Detector_list_detectors_inner> m_Detectors;
 

broker/gen/model/Detector_list_detectors_inner.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -96,7 +96,7 @@ bool Detector_list_detectors_inner::operator!=(const Detector_list_detectors_inn
 
 void to_json(nlohmann::json& j, const Detector_list_detectors_inner& o)
 {
-    j = nlohmann::json();
+    j = nlohmann::json::object();
     j["id"] = o.m_Id;
     j["description"] = o.m_Description;
     j["nmodules"] = o.m_Nmodules;

broker/gen/model/Detector_list_detectors_inner.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -84,8 +84,8 @@ public:
     int64_t getHeight() const;
     void setHeight(int64_t const value);
 
-    friend void to_json(nlohmann::json& j, const Detector_list_detectors_inner& o);
-    friend void from_json(const nlohmann::json& j, Detector_list_detectors_inner& o);
+    friend  void to_json(nlohmann::json& j, const Detector_list_detectors_inner& o);
+    friend  void from_json(const nlohmann::json& j, Detector_list_detectors_inner& o);
 protected:
     int64_t m_Id;
 

broker/gen/model/Detector_selection.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -66,7 +66,7 @@ bool Detector_selection::operator!=(const Detector_selection& rhs) const
 
 void to_json(nlohmann::json& j, const Detector_selection& o)
 {
-    j = nlohmann::json();
+    j = nlohmann::json::object();
     j["id"] = o.m_Id;
     
 }

broker/gen/model/Detector_selection.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -63,8 +63,8 @@ public:
     int64_t getId() const;
     void setId(int64_t const value);
 
-    friend void to_json(nlohmann::json& j, const Detector_selection& o);
-    friend void from_json(const nlohmann::json& j, Detector_selection& o);
+    friend  void to_json(nlohmann::json& j, const Detector_selection& o);
+    friend  void from_json(const nlohmann::json& j, Detector_selection& o);
 protected:
     int64_t m_Id;
 

broker/gen/model/Detector_settings.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -220,7 +220,7 @@ bool Detector_settings::operator!=(const Detector_settings& rhs) const
 
 void to_json(nlohmann::json& j, const Detector_settings& o)
 {
-    j = nlohmann::json();
+    j = nlohmann::json::object();
     j["frame_time_us"] = o.m_Frame_time_us;
     if(o.countTimeUsIsSet())
         j["count_time_us"] = o.m_Count_time_us;

broker/gen/model/Detector_settings.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -147,8 +147,8 @@ public:
     bool useGainHg0IsSet() const;
     void unsetUse_gain_hg0();
 
-    friend void to_json(nlohmann::json& j, const Detector_settings& o);
-    friend void from_json(const nlohmann::json& j, Detector_settings& o);
+    friend  void to_json(nlohmann::json& j, const Detector_settings& o);
+    friend  void from_json(const nlohmann::json& j, Detector_settings& o);
 protected:
     int64_t m_Frame_time_us;
 

broker/gen/model/Detector_status.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -126,7 +126,7 @@ bool Detector_status::operator!=(const Detector_status& rhs) const
 
 void to_json(nlohmann::json& j, const Detector_status& o)
 {
-    j = nlohmann::json();
+    j = nlohmann::json::object();
     j["state"] = o.m_State;
     j["powerchip"] = o.m_Powerchip;
     j["server_version"] = o.m_Server_version;

broker/gen/model/Detector_status.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -90,8 +90,8 @@ public:
     std::vector<int64_t> getHighVoltageV() const;
     void setHighVoltageV(std::vector<int64_t> const value);
 
-    friend void to_json(nlohmann::json& j, const Detector_status& o);
-    friend void from_json(const nlohmann::json& j, Detector_status& o);
+    friend  void to_json(nlohmann::json& j, const Detector_status& o);
+    friend  void from_json(const nlohmann::json& j, Detector_status& o);
 protected:
     std::string m_State;
 

broker/gen/model/Error_message.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -70,7 +70,7 @@ bool Error_message::operator!=(const Error_message& rhs) const
 
 void to_json(nlohmann::json& j, const Error_message& o)
 {
-    j = nlohmann::json();
+    j = nlohmann::json::object();
     j["msg"] = o.m_Msg;
     j["reason"] = o.m_Reason;
     

broker/gen/model/Error_message.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -69,8 +69,8 @@ public:
     std::string getReason() const;
     void setReason(std::string const& value);
 
-    friend void to_json(nlohmann::json& j, const Error_message& o);
-    friend void from_json(const nlohmann::json& j, Error_message& o);
+    friend  void to_json(nlohmann::json& j, const Error_message& o);
+    friend  void from_json(const nlohmann::json& j, Error_message& o);
 protected:
     std::string m_Msg;
 

broker/gen/model/Helpers.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Helpers.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Measurement_statistics.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -23,6 +23,10 @@ Measurement_statistics::Measurement_statistics()
 {
     m_File_prefix = "";
     m_File_prefixIsSet = false;
+    m_Run_number = 0L;
+    m_Run_numberIsSet = false;
+    m_Experiment_group = "";
+    m_Experiment_groupIsSet = false;
     m_Images_expected = 0L;
     m_Images_expectedIsSet = false;
     m_Images_collected = 0L;
@@ -75,7 +79,7 @@ bool Measurement_statistics::validate(std::stringstream& msg, const std::string&
     bool success = true;
     const std::string _pathPrefix = pathPrefix.empty() ? "Measurement_statistics" : pathPrefix;
 
-                                 
+                                         
     if (collectionEfficiencyIsSet())
     {
         const float& value = m_Collection_efficiency;
@@ -121,6 +125,12 @@ bool Measurement_statistics::operator==(const Measurement_statistics& rhs) const
     ((!filePrefixIsSet() && !rhs.filePrefixIsSet()) || (filePrefixIsSet() && rhs.filePrefixIsSet() && getFilePrefix() == rhs.getFilePrefix())) &&
     
     
+    ((!runNumberIsSet() && !rhs.runNumberIsSet()) || (runNumberIsSet() && rhs.runNumberIsSet() && getRunNumber() == rhs.getRunNumber())) &&
+    
+    
+    ((!experimentGroupIsSet() && !rhs.experimentGroupIsSet()) || (experimentGroupIsSet() && rhs.experimentGroupIsSet() && getExperimentGroup() == rhs.getExperimentGroup())) &&
+    
+    
     ((!imagesExpectedIsSet() && !rhs.imagesExpectedIsSet()) || (imagesExpectedIsSet() && rhs.imagesExpectedIsSet() && getImagesExpected() == rhs.getImagesExpected())) &&
     
     
@@ -175,9 +185,13 @@ bool Measurement_statistics::operator!=(const Measurement_statistics& rhs) const
 
 void to_json(nlohmann::json& j, const Measurement_statistics& o)
 {
-    j = nlohmann::json();
+    j = nlohmann::json::object();
     if(o.filePrefixIsSet())
         j["file_prefix"] = o.m_File_prefix;
+    if(o.runNumberIsSet())
+        j["run_number"] = o.m_Run_number;
+    if(o.experimentGroupIsSet())
+        j["experiment_group"] = o.m_Experiment_group;
     if(o.imagesExpectedIsSet())
         j["images_expected"] = o.m_Images_expected;
     if(o.imagesCollectedIsSet())
@@ -218,6 +232,16 @@ void from_json(const nlohmann::json& j, Measurement_statistics& o)
         j.at("file_prefix").get_to(o.m_File_prefix);
         o.m_File_prefixIsSet = true;
     } 
+    if(j.find("run_number") != j.end())
+    {
+        j.at("run_number").get_to(o.m_Run_number);
+        o.m_Run_numberIsSet = true;
+    } 
+    if(j.find("experiment_group") != j.end())
+    {
+        j.at("experiment_group").get_to(o.m_Experiment_group);
+        o.m_Experiment_groupIsSet = true;
+    } 
     if(j.find("images_expected") != j.end())
     {
         j.at("images_expected").get_to(o.m_Images_expected);
@@ -313,6 +337,40 @@ void Measurement_statistics::unsetFile_prefix()
 {
     m_File_prefixIsSet = false;
 }
+int64_t Measurement_statistics::getRunNumber() const
+{
+    return m_Run_number;
+}
+void Measurement_statistics::setRunNumber(int64_t const value)
+{
+    m_Run_number = value;
+    m_Run_numberIsSet = true;
+}
+bool Measurement_statistics::runNumberIsSet() const
+{
+    return m_Run_numberIsSet;
+}
+void Measurement_statistics::unsetRun_number()
+{
+    m_Run_numberIsSet = false;
+}
+std::string Measurement_statistics::getExperimentGroup() const
+{
+    return m_Experiment_group;
+}
+void Measurement_statistics::setExperimentGroup(std::string const& value)
+{
+    m_Experiment_group = value;
+    m_Experiment_groupIsSet = true;
+}
+bool Measurement_statistics::experimentGroupIsSet() const
+{
+    return m_Experiment_groupIsSet;
+}
+void Measurement_statistics::unsetExperiment_group()
+{
+    m_Experiment_groupIsSet = false;
+}
 int64_t Measurement_statistics::getImagesExpected() const
 {
     return m_Images_expected;

broker/gen/model/Measurement_statistics.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -66,6 +66,20 @@ public:
     bool filePrefixIsSet() const;
     void unsetFile_prefix();
     /// <summary>
+    /// Number of data collection run. This can be either automatically incremented or provided externally for each data collection. 
+    /// </summary>
+    int64_t getRunNumber() const;
+    void setRunNumber(int64_t const value);
+    bool runNumberIsSet() const;
+    void unsetRun_number();
+    /// <summary>
+    /// Name of group owning the data (e.g. p-group or proposal number). 
+    /// </summary>
+    std::string getExperimentGroup() const;
+    void setExperimentGroup(std::string const& value);
+    bool experimentGroupIsSet() const;
+    void unsetExperiment_group();
+    /// <summary>
     /// 
     /// </summary>
     int64_t getImagesExpected() const;
@@ -171,11 +185,15 @@ public:
     bool unitCellIsSet() const;
     void unsetUnit_cell();
 
-    friend void to_json(nlohmann::json& j, const Measurement_statistics& o);
-    friend void from_json(const nlohmann::json& j, Measurement_statistics& o);
+    friend  void to_json(nlohmann::json& j, const Measurement_statistics& o);
+    friend  void from_json(const nlohmann::json& j, Measurement_statistics& o);
 protected:
     std::string m_File_prefix;
     bool m_File_prefixIsSet;
+    int64_t m_Run_number;
+    bool m_Run_numberIsSet;
+    std::string m_Experiment_group;
+    bool m_Experiment_groupIsSet;
     int64_t m_Images_expected;
     bool m_Images_expectedIsSet;
     int64_t m_Images_collected;

broker/gen/model/Plot.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -114,7 +114,7 @@ bool Plot::operator!=(const Plot& rhs) const
 
 void to_json(nlohmann::json& j, const Plot& o)
 {
-    j = nlohmann::json();
+    j = nlohmann::json::object();
     j["title"] = o.m_Title;
     j["x"] = o.m_x;
     j["y"] = o.m_y;

broker/gen/model/Plot.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -75,8 +75,8 @@ public:
     std::vector<float> getY() const;
     void setY(std::vector<float> const value);
 
-    friend void to_json(nlohmann::json& j, const Plot& o);
-    friend void from_json(const nlohmann::json& j, Plot& o);
+    friend  void to_json(nlohmann::json& j, const Plot& o);
+    friend  void from_json(const nlohmann::json& j, Plot& o);
 protected:
     std::string m_Title;
 

broker/gen/model/Plot_request.cpp

@@ -1,106 +0,0 @@
-/**
-* Jungfraujoch
-* Jungfraujoch Broker Web API
-*
-* The version of the OpenAPI document: 1.0.1
-* 
-*
-* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
-* https://openapi-generator.tech
-* Do not edit the class manually.
-*/
-
-
-#include "Plot_request.h"
-#include "Helpers.h"
-
-#include <sstream>
-
-namespace org::openapitools::server::model
-{
-
-Plot_request::Plot_request()
-{
-    m_Binning = 0L;
-    m_BinningIsSet = false;
-    
-}
-
-void Plot_request::validate() const
-{
-    std::stringstream msg;
-    if (!validate(msg))
-    {
-        throw org::openapitools::server::helpers::ValidationException(msg.str());
-    }
-}
-
-bool Plot_request::validate(std::stringstream& msg) const
-{
-    return validate(msg, "");
-}
-
-bool Plot_request::validate(std::stringstream& msg, const std::string& pathPrefix) const
-{
-    bool success = true;
-    const std::string _pathPrefix = pathPrefix.empty() ? "Plot_request" : pathPrefix;
-
-        
-    return success;
-}
-
-bool Plot_request::operator==(const Plot_request& rhs) const
-{
-    return
-    
-    
-    
-    ((!binningIsSet() && !rhs.binningIsSet()) || (binningIsSet() && rhs.binningIsSet() && getBinning() == rhs.getBinning()))
-    
-    ;
-}
-
-bool Plot_request::operator!=(const Plot_request& rhs) const
-{
-    return !(*this == rhs);
-}
-
-void to_json(nlohmann::json& j, const Plot_request& o)
-{
-    j = nlohmann::json();
-    if(o.binningIsSet())
-        j["binning"] = o.m_Binning;
-    
-}
-
-void from_json(const nlohmann::json& j, Plot_request& o)
-{
-    if(j.find("binning") != j.end())
-    {
-        j.at("binning").get_to(o.m_Binning);
-        o.m_BinningIsSet = true;
-    } 
-    
-}
-
-int64_t Plot_request::getBinning() const
-{
-    return m_Binning;
-}
-void Plot_request::setBinning(int64_t const value)
-{
-    m_Binning = value;
-    m_BinningIsSet = true;
-}
-bool Plot_request::binningIsSet() const
-{
-    return m_BinningIsSet;
-}
-void Plot_request::unsetBinning()
-{
-    m_BinningIsSet = false;
-}
-
-
-} // namespace org::openapitools::server::model
-

broker/gen/model/Plot_request.h

@@ -1,78 +0,0 @@
-/**
-* Jungfraujoch
-* Jungfraujoch Broker Web API
-*
-* The version of the OpenAPI document: 1.0.1
-* 
-*
-* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
-* https://openapi-generator.tech
-* Do not edit the class manually.
-*/
-/*
- * Plot_request.h
- *
- * 
- */
-
-#ifndef Plot_request_H_
-#define Plot_request_H_
-
-
-#include <nlohmann/json.hpp>
-
-namespace org::openapitools::server::model
-{
-
-/// <summary>
-/// 
-/// </summary>
-class  Plot_request
-{
-public:
-    Plot_request();
-    virtual ~Plot_request() = default;
-
-
-    /// <summary>
-    /// Validate the current data in the model. Throws a ValidationException on failure.
-    /// </summary>
-    void validate() const;
-
-    /// <summary>
-    /// Validate the current data in the model. Returns false on error and writes an error
-    /// message into the given stringstream.
-    /// </summary>
-    bool validate(std::stringstream& msg) const;
-
-    /// <summary>
-    /// Helper overload for validate. Used when one model stores another model and calls it's validate.
-    /// Not meant to be called outside that case.
-    /// </summary>
-    bool validate(std::stringstream& msg, const std::string& pathPrefix) const;
-
-    bool operator==(const Plot_request& rhs) const;
-    bool operator!=(const Plot_request& rhs) const;
-
-    /////////////////////////////////////////////
-    /// Plot_request members
-
-    /// <summary>
-    /// 
-    /// </summary>
-    int64_t getBinning() const;
-    void setBinning(int64_t const value);
-    bool binningIsSet() const;
-    void unsetBinning();
-
-    friend void to_json(nlohmann::json& j, const Plot_request& o);
-    friend void from_json(const nlohmann::json& j, Plot_request& o);
-protected:
-    int64_t m_Binning;
-    bool m_BinningIsSet;
-    
-};
-
-} // namespace org::openapitools::server::model
-
-#endif /* Plot_request_H_ */

broker/gen/model/Plots.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -91,7 +91,7 @@ bool Plots::operator!=(const Plots& rhs) const
 
 void to_json(nlohmann::json& j, const Plots& o)
 {
-    j = nlohmann::json();
+    j = nlohmann::json::object();
     if(o.titleIsSet())
         j["title"] = o.m_Title;
     j["plot"] = o.m_Plot;

broker/gen/model/Plots.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -73,8 +73,8 @@ public:
     std::vector<org::openapitools::server::model::Plot> getPlot() const;
     void setPlot(std::vector<org::openapitools::server::model::Plot> const& value);
 
-    friend void to_json(nlohmann::json& j, const Plots& o);
-    friend void from_json(const nlohmann::json& j, Plots& o);
+    friend  void to_json(nlohmann::json& j, const Plots& o);
+    friend  void from_json(const nlohmann::json& j, Plots& o);
 protected:
     std::string m_Title;
     bool m_TitleIsSet;

broker/gen/model/Preview_settings.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -153,7 +153,7 @@ bool Preview_settings::operator!=(const Preview_settings& rhs) const
 
 void to_json(nlohmann::json& j, const Preview_settings& o)
 {
-    j = nlohmann::json();
+    j = nlohmann::json::object();
     j["saturation"] = o.m_Saturation;
     if(o.showSpotsIsSet())
         j["show_spots"] = o.m_Show_spots;

broker/gen/model/Preview_settings.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -105,8 +105,8 @@ public:
     bool resolutionRingIsSet() const;
     void unsetResolution_ring();
 
-    friend void to_json(nlohmann::json& j, const Preview_settings& o);
-    friend void from_json(const nlohmann::json& j, Preview_settings& o);
+    friend  void to_json(nlohmann::json& j, const Preview_settings& o);
+    friend  void from_json(const nlohmann::json& j, Preview_settings& o);
 protected:
     int64_t m_Saturation;
 

broker/gen/model/Rad_int_settings.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -102,7 +102,7 @@ bool Rad_int_settings::operator!=(const Rad_int_settings& rhs) const
 
 void to_json(nlohmann::json& j, const Rad_int_settings& o)
 {
-    j = nlohmann::json();
+    j = nlohmann::json::object();
     if(o.polarizationFactorIsSet())
         j["polarization_factor"] = o.m_Polarization_factor;
     j["solid_angle_corr"] = o.m_Solid_angle_corr;

broker/gen/model/Rad_int_settings.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -85,8 +85,8 @@ public:
     float getQSpacing() const;
     void setQSpacing(float const value);
 
-    friend void to_json(nlohmann::json& j, const Rad_int_settings& o);
-    friend void from_json(const nlohmann::json& j, Rad_int_settings& o);
+    friend  void to_json(nlohmann::json& j, const Rad_int_settings& o);
+    friend  void from_json(const nlohmann::json& j, Rad_int_settings& o);
 protected:
     float m_Polarization_factor;
     bool m_Polarization_factorIsSet;

broker/gen/model/Roi_box.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -152,7 +152,7 @@ bool Roi_box::operator!=(const Roi_box& rhs) const
 
 void to_json(nlohmann::json& j, const Roi_box& o)
 {
-    j = nlohmann::json();
+    j = nlohmann::json::object();
     j["name"] = o.m_Name;
     j["min_x_pxl"] = o.m_Min_x_pxl;
     j["max_x_pxl"] = o.m_Max_x_pxl;

broker/gen/model/Roi_box.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -84,8 +84,8 @@ public:
     int64_t getMaxYPxl() const;
     void setMaxYPxl(int64_t const value);
 
-    friend void to_json(nlohmann::json& j, const Roi_box& o);
-    friend void from_json(const nlohmann::json& j, Roi_box& o);
+    friend  void to_json(nlohmann::json& j, const Roi_box& o);
+    friend  void from_json(const nlohmann::json& j, Roi_box& o);
 protected:
     std::string m_Name;
 

broker/gen/model/Roi_box_list.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -92,7 +92,7 @@ bool Roi_box_list::operator!=(const Roi_box_list& rhs) const
 
 void to_json(nlohmann::json& j, const Roi_box_list& o)
 {
-    j = nlohmann::json();
+    j = nlohmann::json::object();
     if(o.roisIsSet() || !o.m_Rois.empty())
         j["rois"] = o.m_Rois;
     

broker/gen/model/Roi_box_list.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -67,8 +67,8 @@ public:
     bool roisIsSet() const;
     void unsetRois();
 
-    friend void to_json(nlohmann::json& j, const Roi_box_list& o);
-    friend void from_json(const nlohmann::json& j, Roi_box_list& o);
+    friend  void to_json(nlohmann::json& j, const Roi_box_list& o);
+    friend  void from_json(const nlohmann::json& j, Roi_box_list& o);
 protected:
     std::vector<org::openapitools::server::model::Roi_box> m_Rois;
     bool m_RoisIsSet;

broker/gen/model/Roi_circle.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -106,7 +106,7 @@ bool Roi_circle::operator!=(const Roi_circle& rhs) const
 
 void to_json(nlohmann::json& j, const Roi_circle& o)
 {
-    j = nlohmann::json();
+    j = nlohmann::json::object();
     j["name"] = o.m_Name;
     j["center_x_pxl"] = o.m_Center_x_pxl;
     j["center_y_pxl"] = o.m_Center_y_pxl;

broker/gen/model/Roi_circle.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -79,8 +79,8 @@ public:
     float getRadiusPxl() const;
     void setRadiusPxl(float const value);
 
-    friend void to_json(nlohmann::json& j, const Roi_circle& o);
-    friend void from_json(const nlohmann::json& j, Roi_circle& o);
+    friend  void to_json(nlohmann::json& j, const Roi_circle& o);
+    friend  void from_json(const nlohmann::json& j, Roi_circle& o);
 protected:
     std::string m_Name;
 

broker/gen/model/Roi_circle_list.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -91,7 +91,7 @@ bool Roi_circle_list::operator!=(const Roi_circle_list& rhs) const
 
 void to_json(nlohmann::json& j, const Roi_circle_list& o)
 {
-    j = nlohmann::json();
+    j = nlohmann::json::object();
     j["rois"] = o.m_Rois;
     
 }

broker/gen/model/Roi_circle_list.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -65,8 +65,8 @@ public:
     std::vector<org::openapitools::server::model::Roi_circle> getRois() const;
     void setRois(std::vector<org::openapitools::server::model::Roi_circle> const& value);
 
-    friend void to_json(nlohmann::json& j, const Roi_circle_list& o);
-    friend void from_json(const nlohmann::json& j, Roi_circle_list& o);
+    friend  void to_json(nlohmann::json& j, const Roi_circle_list& o);
+    friend  void from_json(const nlohmann::json& j, Roi_circle_list& o);
 protected:
     std::vector<org::openapitools::server::model::Roi_circle> m_Rois;
 

broker/gen/model/Rotation_axis.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -21,6 +21,8 @@ namespace org::openapitools::server::model
 
 Rotation_axis::Rotation_axis()
 {
+    m_Name = "omega";
+    m_NameIsSet = false;
     m_Step = 0.0f;
     m_Start = 0.0f;
     m_StartIsSet = false;
@@ -47,6 +49,20 @@ bool Rotation_axis::validate(std::stringstream& msg, const std::string& pathPref
     bool success = true;
     const std::string _pathPrefix = pathPrefix.empty() ? "Rotation_axis" : pathPrefix;
 
+         
+    if (nameIsSet())
+    {
+        const std::string& value = m_Name;
+        const std::string currentValuePath = _pathPrefix + ".name";
+                
+        
+        if (value.length() < 1)
+        {
+            success = false;
+            msg << currentValuePath << ": must be at least 1 characters long;";
+        }
+
+    }
                  
     if (vectorIsSet())
     {
@@ -87,6 +103,9 @@ bool Rotation_axis::operator==(const Rotation_axis& rhs) const
     return
     
     
+    
+    ((!nameIsSet() && !rhs.nameIsSet()) || (nameIsSet() && rhs.nameIsSet() && getName() == rhs.getName())) &&
+    
     (getStep() == rhs.getStep())
      &&
     
@@ -106,7 +125,9 @@ bool Rotation_axis::operator!=(const Rotation_axis& rhs) const
 
 void to_json(nlohmann::json& j, const Rotation_axis& o)
 {
-    j = nlohmann::json();
+    j = nlohmann::json::object();
+    if(o.nameIsSet())
+        j["name"] = o.m_Name;
     j["step"] = o.m_Step;
     if(o.startIsSet())
         j["start"] = o.m_Start;
@@ -117,6 +138,11 @@ void to_json(nlohmann::json& j, const Rotation_axis& o)
 
 void from_json(const nlohmann::json& j, Rotation_axis& o)
 {
+    if(j.find("name") != j.end())
+    {
+        j.at("name").get_to(o.m_Name);
+        o.m_NameIsSet = true;
+    } 
     j.at("step").get_to(o.m_Step);
     if(j.find("start") != j.end())
     {
@@ -131,6 +157,23 @@ void from_json(const nlohmann::json& j, Rotation_axis& o)
     
 }
 
+std::string Rotation_axis::getName() const
+{
+    return m_Name;
+}
+void Rotation_axis::setName(std::string const& value)
+{
+    m_Name = value;
+    m_NameIsSet = true;
+}
+bool Rotation_axis::nameIsSet() const
+{
+    return m_NameIsSet;
+}
+void Rotation_axis::unsetName()
+{
+    m_NameIsSet = false;
+}
 float Rotation_axis::getStep() const
 {
     return m_Step;

broker/gen/model/Rotation_axis.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -19,6 +19,7 @@
 #define Rotation_axis_H_
 
 
+#include <string>
 #include <vector>
 #include <nlohmann/json.hpp>
 
@@ -58,6 +59,13 @@ public:
     /////////////////////////////////////////////
     /// Rotation_axis members
 
+    /// <summary>
+    /// Name of rotation axis (e.g., omega, phi)
+    /// </summary>
+    std::string getName() const;
+    void setName(std::string const& value);
+    bool nameIsSet() const;
+    void unsetName();
     /// <summary>
     /// Angle step in degrees
     /// </summary>
@@ -78,9 +86,11 @@ public:
     bool vectorIsSet() const;
     void unsetVector();
 
-    friend void to_json(nlohmann::json& j, const Rotation_axis& o);
-    friend void from_json(const nlohmann::json& j, Rotation_axis& o);
+    friend  void to_json(nlohmann::json& j, const Rotation_axis& o);
+    friend  void from_json(const nlohmann::json& j, Rotation_axis& o);
 protected:
+    std::string m_Name;
+    bool m_NameIsSet;
     float m_Step;
 
     float m_Start;

broker/gen/model/Spot_finding_settings.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -25,7 +25,7 @@ Spot_finding_settings::Spot_finding_settings()
     m_Indexing = true;
     m_Filter_powder_rings = false;
     m_Filter_powder_ringsIsSet = false;
-    m_Min_spot_count_powder_ring = 20L;
+    m_Min_spot_count_powder_ring = 0L;
     m_Min_spot_count_powder_ringIsSet = false;
     m_Signal_to_noise_threshold = 0.0f;
     m_Photon_count_threshold = 0L;
@@ -197,7 +197,7 @@ bool Spot_finding_settings::operator!=(const Spot_finding_settings& rhs) const
 
 void to_json(nlohmann::json& j, const Spot_finding_settings& o)
 {
-    j = nlohmann::json();
+    j = nlohmann::json::object();
     j["enable"] = o.m_Enable;
     j["indexing"] = o.m_Indexing;
     if(o.filterPowderRingsIsSet())

broker/gen/model/Spot_finding_settings.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0-rc.12
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -117,8 +117,8 @@ public:
     float getIndexingTolerance() const;
     void setIndexingTolerance(float const value);
 
-    friend void to_json(nlohmann::json& j, const Spot_finding_settings& o);
-    friend void from_json(const nlohmann::json& j, Spot_finding_settings& o);
+    friend  void to_json(nlohmann::json& j, const Spot_finding_settings& o);
+    friend  void from_json(const nlohmann::json& j, Spot_finding_settings& o);
 protected:
     bool m_Enable;
 

broker/jfjoch_api.yaml

@@ -2,7 +2,9 @@ openapi: 3.0.3
 info:
   title: Jungfraujoch
   description: Jungfraujoch Broker Web API
-  version: 1.0.1
+  version: 1.0.0-rc.12
+  contact:
+    email: filip.leonarski@psi.ch
 components:
   schemas:
     rotation_axis:
@@ -11,6 +13,11 @@ components:
       required:
         - step
       properties:
+        name:
+          type: string
+          default: omega
+          minLength: 1
+          description: Name of rotation axis (e.g., omega, phi)
         step:
           type: number
           format: float
@@ -37,8 +44,7 @@ components:
         - beam_x_pxl
         - beam_y_pxl
         - detector_distance_mm
-        - photon_energy_keV
-        - sample_name
+        - incident_energy_keV
       properties:
         images_per_trigger:
           type: integer
@@ -56,17 +62,16 @@ components:
           minimum: 1
           description: |
             Number of TTL trigger that the detector is expected to receive during data collection
-        summation:
+        image_time_us:
           type: integer
           format: int64
-          minimum: 1
-          maximum: 256
-          default: 1
+          minimum: 0
           description: |
-            FPGA frame summation. For summation above two 32-bit pixel format will be used, unless explicitly specified.
-            Frame summation factor applies only to conversion mode (assumed as 1 for raw data).
+            Image time. 
+            If not provided (or zero value) the frame time is assumed as default.
+            Image time must be multiple of frame time; max value is 256 * frame_time.            
             In XFEL mode: summation happens for frames collected with multiple triggers.
-            Ignored for storage cells (assumed as 1).
+            Ignored for storage cells and if raw data are saved.
         beam_x_pxl:
           type: number
           format: float
@@ -86,13 +91,13 @@ components:
           description:
             /entry/detector/distance in NXmx
             Detector distance [mm]
-        photon_energy_keV:
+        incident_energy_keV:
           type: number
           format: float
           minimum: 0
           description: |
             Used to calculate /entry/beam/incident_wavelength in NXmx
-            Incident photon energy in keV
+            Incident particle (photon, electron) energy in keV
         file_prefix:
           type: string
           default: ""
@@ -111,6 +116,7 @@ components:
           maximum: 194
         sample_name:
           type: string
+          default: ""
           description: |
             /entry/sample/name in NXmx
             Sample name
@@ -148,21 +154,19 @@ components:
           description: |
             /entry/instrument/attenuator/attenuator_transmission
             Transmission of attenuator (filter) [no units]
-        omega:
+        goniometer:
           $ref: "#/components/schemas/rotation_axis"
         header_appendix:
-          type: string
-          description: Header appendix, added as user_data to start message
+          description: Header appendix, added as user_data/user to start message (can be any valid JSON)
         image_appendix:
-          type: string
-          description: Image appendix, added as user_data to image message
-        photon_energy_multiplier:
+          description: Image appendix, added as user_data to image message (can be any valid JSON)
+        energy_multiplier:
           type: number
           format: float
           default: 1.0
           minimum: 0.015625
           maximum: 4.0
-          description: For JUNGFRAU conversion it is possible to multiply energy by a given factor to get fractional/multiplied photon counts
+          description: For JUNGFRAU conversion it is possible to multiply incident energy by a given factor to get fractional/multiplied particle counts
         data_reduction_factor_serialmx:
           type: number
           format: float
@@ -173,6 +177,37 @@ components:
             Rate at which non-indexed images are accepted to be forwarded to writer. 
             Value of 1.0 (default) means that all images are written.
             Values below zero mean that non-indexed images will be accepted with a given probability.
+        run_number:
+          type: integer
+          format: int64
+          minimum: 0
+          description: |
+            Number of run within an experimental session. 
+            Transferred over CBOR stream as "series ID", though not saved in HDF5 file.
+            It is highly recommended to keep this number unique for each data collection during experimental series.
+            If not provided, the number will be automatically incremented.
+        run_name:
+          type: string
+          description: |
+            Unique ID of run.
+            Transferred over CBOR stream as "unique series ID", though not saved in HDF5 file.
+            It is highly recommended to keep this name unique for each data collection during experimental series.
+            If not provided, the name will be automatically generated as number + colon + file_prefix.
+        experiment_group:
+          type: string
+          description: |
+            Name of group owning the data (e.g. p-group or proposal number). 
+            Transferred over CBOR stream, though not saved in HDF5 file.
+        poisson_compression:
+          type: integer
+          format: int64
+          minimum: 0
+          maximum: 16
+          description: |
+            Enable lossy compression of pixel values that preserves Poisson statistics. 
+            Requires to provide a numerical factor SQ.
+            Pixel value P will be transformed to round(sqrt(P) * SQ), with rounding to the closest integer.
+            Compression is turned off if the value is missing or it is set to zero.
         unit_cell:
           type: object
           description: Units of angstrom and degree
@@ -235,7 +270,7 @@ components:
         powerchip:
           type: string
           description: Power on of ASICs
-          enum: ["On", "Off", "Partial"]
+          enum: ["PowerOn", "PowerOff", "Partial"]
         server_version:
           type: string
           description: Detector server (on read-out boards) version
@@ -459,6 +494,15 @@ components:
       properties:
         file_prefix:
           type: string
+        run_number:
+          type: integer
+          format: int64
+          description: |
+            Number of data collection run. This can be either automatically incremented or provided externally for each data collection.
+        experiment_group:
+          type: string
+          description: |
+            Name of group owning the data (e.g. p-group or proposal number).
         images_expected:
           type: integer
           format: int64
@@ -819,22 +863,37 @@ paths:
   /wait_till_done:
     post:
       summary: Wait for acquisition done
+      parameters:
+        - in: query
+          name: timeout
+          required: false
+          schema:
+            type: integer
+            default: 60
+            minimum: 0
+            maximum: 3600
+          description: Timeout in seconds (0 == immediate response)
       description: |
         Block execution of external script till initialization, data collection or pedestal is finished.
         Running this command does not affect (cancel) running data collection, it is only to ensure synchronous execution of other software.
                 
-        To not block web server for a long period of time, the procedure is provided with a timeout of 5 seconds.
+        To not block web server for a indefinite period of time, the procedure is provided with a timeout.
+        Extending timeout is possible, but requires to ensure safety that client will not close the connection and retry the connection.
       responses:
         "200":
           description: Detector in `Idle` state, another data collection can start immediately
+        "400":
+          description: Timeout parameter out of bounds
         "500":
           description: Error within Jungfraujoch code - see output message.
           content:
             application/json:
               schema:
                 $ref: '#/components/schemas/error_message'
+        "502":
+          description: Detector is inactive mode
         "504":
-          description: 5 second timeout reached, need to restart operation
+          description: Timeout reached, need to restart operation
 
   /trigger:
     post:
@@ -1696,4 +1755,14 @@ paths:
                 type: string
                 format: binary
         "404":
-          description: No calibration recorded so far
+          description: No calibration recorded so far
+  /version:
+    get:
+      responses:
+        "200":
+          description: Release number of Jungfraujoch
+          content:
+            text/plain:
+              schema:
+                type: string
+                example: 1.0.0

broker/jfjoch_broker.cpp

@@ -12,8 +12,6 @@
 #include "JFJochBrokerHttp.h"
 
 #include "JFJochBrokerParser.h"
-#include "../frame_serialize/ZMQStream2Pusher.h"
-#include "../frame_serialize/DumpCBORToFilePusher.h"
 
 static Pistache::Http::Endpoint *httpEndpoint;
 
@@ -81,25 +79,7 @@ int main (int argc, char **argv) {
     if (aq_devices.size() > 0) {
         experiment.DataStreams(aq_devices.size());
 
-        std::string pusher_type = ParseString(input, "stream_type", "zmq");
-        if (pusher_type == "zmq") {
-            int32_t zmq_send_watermark = ParseInt32(input, "zmq_send_watermark", 100);
-            int32_t zmq_send_buffer_size = ParseInt32(input, "zmq_send_buffer_size", -1);
-
-            auto tmp = std::make_unique<ZMQStream2Pusher>(ParseStringArray(input, "zmq_image_addr"),
-                                                                   zmq_send_watermark,
-                                                                   zmq_send_buffer_size);
-
-            std::string preview_addr = ParseString(input, "zmq_preview_addr", "");
-            if (!preview_addr.empty())
-                tmp->PreviewSocket(preview_addr);
-
-            image_pusher = std::move(tmp);
-        } else if (pusher_type == "dump_cbor") {
-            image_pusher = std::make_unique<DumpCBORToFilePusher>();
-        } else
-            throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
-                                  "stream_type allowed: zmq (default), dump_cbor");
+        ParseImagePusher(input, image_pusher);
 
         int32_t send_buffer_size_MiB = ParseInt32(input, "send_buffer_size_MiB", 2048);
         receiver = std::make_unique<JFJochReceiverService>(aq_devices, logger, *image_pusher, send_buffer_size_MiB);
@@ -115,9 +95,9 @@ int main (int argc, char **argv) {
 
     logger.Info("Source {} Instrument {} Default rotation axis {:.2f},{:.2f},{:.2f}",
                 experiment.GetSourceName(), experiment.GetInstrumentName(),
-                experiment.GetDefaultOmegaAxis().x,
-                experiment.GetDefaultOmegaAxis().y,
-                experiment.GetDefaultOmegaAxis().z);
+                experiment.GetDefaultRotationAxis().x,
+                experiment.GetDefaultRotationAxis().y,
+                experiment.GetDefaultRotationAxis().z);
 
     Pistache::Address addr(Pistache::Ipv4::any(), Pistache::Port(http_port));
 

common/CUDAWrapper.cu

@@ -17,8 +17,11 @@ int32_t get_gpu_count() {
 void set_gpu(int32_t dev_id) {
     auto dev_count = get_gpu_count();
 
-    if ((dev_id < 0) || (dev_id >= dev_count))
-        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Device ID cannot be negative");
+    // Ignore if no GPU present
+    if (dev_count > 0) {
+        if ((dev_id < 0) || (dev_id >= dev_count))
+            throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Device ID cannot be negative");
 
-    cuda_err(cudaSetDevice(dev_id));
-}
+        cuda_err(cudaSetDevice(dev_id));
+    }
+}

common/DatasetSettings.cpp

@@ -19,12 +19,10 @@ DatasetSettings::DatasetSettings() {
     file_prefix = "test";
     ntrigger = 1;
     images_per_trigger = 1;
-    summation = 1;
     fpga_pixel_output = FPGAPixelOutput::Auto;
     space_group_number = 0; // not set
     compression = CompressionAlgorithm::BSHUF_LZ4;
     photon_energy_multiplier = 1.0f;
-    omega_start = 0.0f;
     images_per_file = 1000;
     data_reduction_factor_serialmx = 1.0;
 }
@@ -151,40 +149,34 @@ DatasetSettings &DatasetSettings::TotalFlux(const std::optional<float> &input) {
     return *this;
 }
 
-DatasetSettings &DatasetSettings::OmegaStep(const std::optional<float> &input) {
+DatasetSettings &DatasetSettings::Goniometer(const std::optional<GoniometerAxis> &input) {
     if (input) {
-        check_finite("Omega step", input.value());
+        check_finite("Rotation angle increment", input->increment);
+        if (input->increment == 0.0f)
+            throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
+                                  "Angle increment cannot be zero");
+        check_finite("Rotation angle start", input->start);
+        goniometer = input;
     }
-
-    if (input && (input == 0.0f))
-        omega_step.reset();
-    else
-        omega_step = input;
     return *this;
 }
 
-DatasetSettings &DatasetSettings::OmegaStart(float input) {
-    check_finite("Omega start", input);
-    omega_start = input;
-    return *this;
-}
-
-DatasetSettings &DatasetSettings::OmegaAxis(const std::optional<Coord> &c) {
+DatasetSettings &DatasetSettings::RotationAxis(const std::optional<Coord> &c) {
     if (c) {
         if (c->Length() == 0.0)
             throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Cannot use empty vector for omega");
-        omega_rotation_axis = c->Normalize();
+        rotation_axis = c->Normalize();
     } else
-        omega_rotation_axis = c;
+        rotation_axis = c;
     return *this;
 }
 
-DatasetSettings &DatasetSettings::HeaderAppendix(const std::string &input) {
+DatasetSettings &DatasetSettings::HeaderAppendix(const nlohmann::json &input) {
     header_appendix = input;
     return *this;
 }
 
-DatasetSettings &DatasetSettings::ImageAppendix(const std::string &input) {
+DatasetSettings &DatasetSettings::ImageAppendix(const nlohmann::json &input) {
     image_appendix = input;
     return *this;
 }
@@ -198,13 +190,6 @@ DatasetSettings &DatasetSettings::PhotonEnergyMultiplayer(float input) {
     return *this;
 }
 
-DatasetSettings &DatasetSettings::Summation(int64_t input) {
-    check_min("Summation", input, 1);
-    check_max("Summation", input, MAX_FPGA_SUMMATION);
-    summation = input;
-    return *this;
-}
-
 DatasetSettings &DatasetSettings::FPGAOutputMode(FPGAPixelOutput input) {
     switch (input) {
         case FPGAPixelOutput::Auto:
@@ -228,23 +213,19 @@ std::optional<float> DatasetSettings::GetTotalFlux() const {
     return total_flux;
 }
 
-std::optional<float> DatasetSettings::GetOmegaStep() const {
-    return omega_step;
+std::optional<GoniometerAxis> DatasetSettings::GetGoniometer() const {
+    return goniometer;
 }
 
-float DatasetSettings::GetOmegaStart() const {
-    return omega_start;
+std::optional<Coord> DatasetSettings::GetRotationAxis() const {
+    return rotation_axis;
 }
 
-std::optional<Coord> DatasetSettings::GetOmegaAxis() const {
-    return omega_rotation_axis;
-}
-
-std::string DatasetSettings::GetHeaderAppendix() const {
+const nlohmann::json& DatasetSettings::GetHeaderAppendix() const {
     return header_appendix;
 }
 
-std::string DatasetSettings::GetImageAppendix() const {
+const nlohmann::json& DatasetSettings::GetImageAppendix() const {
     return image_appendix;
 }
 
@@ -260,10 +241,6 @@ int64_t DatasetSettings::GetSpaceGroupNumber() const {
     return space_group_number;
 }
 
-int64_t DatasetSettings::GetSummation() const {
-    return summation;
-}
-
 FPGAPixelOutput DatasetSettings::GetFPGAOutputMode() const {
     return fpga_pixel_output;
 }
@@ -322,13 +299,74 @@ int64_t DatasetSettings::GetImagesPerFile() const {
     return images_per_file;
 }
 
-DatasetSettings &DatasetSettings::DataReductionFactorSerialMX(float input) {
+DatasetSettings &DatasetSettings::LossyCompressionSerialMX(float input) {
     check_min("Data reduction factor for serial MX", input, 0.0);
     check_max("Data reduction factor for serial MX", input, 1.0);
     data_reduction_factor_serialmx = input;
     return *this;
 }
 
-float DatasetSettings::GetDataReductionFactorSerialMX() const {
+float DatasetSettings::GetLossyCompressionSerialMX() const {
     return data_reduction_factor_serialmx;
 }
+
+DatasetSettings &DatasetSettings::RunNumber(const std::optional<uint64_t> &input) {
+    if (input) {
+        check_min("Run number", input, 0);
+        check_max("Run number", input, INT64_MAX);
+    }
+    run_number = input;
+    return *this;
+}
+
+DatasetSettings & DatasetSettings::RunName(const std::optional<std::string> &input) {
+    if (input && input.value().empty())
+        run_name = {};
+    else
+        run_name = input;
+    return *this;
+}
+
+DatasetSettings &DatasetSettings::ExperimentGroup(const std::string &input) {
+    group = input;
+    return *this;
+}
+
+std::optional<uint64_t> DatasetSettings::GetRunNumber() const {
+    return run_number;
+}
+
+std::optional<std::string> DatasetSettings::GetRunName() const {
+    return run_name;
+}
+
+std::string DatasetSettings::GetExperimentGroup() const {
+    return group;
+}
+
+std::optional<std::chrono::microseconds> DatasetSettings::GetImageTime() const {
+    return image_time;
+}
+
+DatasetSettings &DatasetSettings::ImageTime(const std::optional<std::chrono::microseconds> input) {
+    if (input && (input.value().count() == 0))
+        image_time = {};
+    else
+        image_time = input;
+    return *this;
+}
+
+DatasetSettings &DatasetSettings::LossyCompressionPoisson(std::optional<int64_t> input) {
+    if (!input || (input == 0))
+        compression_poisson_factor = {};
+    else {
+        check_min("Poisson compression factor", input.value(), 1);
+        check_max("Poisson compression factor", input.value(), 16);
+        compression_poisson_factor = input;
+    }
+    return *this;
+}
+
+std::optional<int64_t> DatasetSettings::GetLossyCompressionPoisson() const {
+    return compression_poisson_factor;
+}

common/DatasetSettings.h

@@ -21,7 +21,6 @@ class DatasetSettings {
     int64_t ntrigger;
 
     FPGAPixelOutput fpga_pixel_output;
-    int64_t summation;
 
     float beam_x_pxl;
     float beam_y_pxl;
@@ -40,14 +39,19 @@ class DatasetSettings {
 
     std::optional<float> total_flux;
     std::optional<float> attenuator_transmission;
-    std::optional<float> omega_step;
-    float omega_start;
-    std::string image_appendix;
-    std::string header_appendix;
+    std::optional<GoniometerAxis> goniometer;
+    nlohmann::json image_appendix;
+    nlohmann::json header_appendix;
 
-    std::optional<Coord> omega_rotation_axis;
+    std::optional<Coord> rotation_axis;
 
     float data_reduction_factor_serialmx;
+
+    std::optional<std::chrono::microseconds> image_time;
+    std::optional<uint64_t> run_number;
+    std::optional<std::string> run_name;
+    std::string group;
+    std::optional<int64_t> compression_poisson_factor;
 public:
 
     DatasetSettings();
@@ -65,28 +69,30 @@ public:
     DatasetSettings& SampleName(std::string input);
     DatasetSettings& AttenuatorTransmission(const std::optional<float> &input);
     DatasetSettings& TotalFlux(const std::optional<float> &input);
-    DatasetSettings& OmegaStep(const std::optional<float> &input);
-    DatasetSettings& OmegaStart(float input);
-    DatasetSettings& OmegaAxis(const std::optional<Coord> &c);
-    DatasetSettings& HeaderAppendix(const std::string& input);
-    DatasetSettings& ImageAppendix(const std::string& input);
+    DatasetSettings& Goniometer(const std::optional<GoniometerAxis>& input);
+    DatasetSettings& RotationAxis(const std::optional<Coord> &c);
+    DatasetSettings& HeaderAppendix(const nlohmann::json& input);
+    DatasetSettings& ImageAppendix(const nlohmann::json& input);
     DatasetSettings& PhotonEnergyMultiplayer(float input);
-    DatasetSettings& Summation(int64_t input);
     DatasetSettings& FPGAOutputMode(FPGAPixelOutput input);
     DatasetSettings& ImagesPerFile(int64_t input);
-    DatasetSettings& DataReductionFactorSerialMX(float input);
+    DatasetSettings& RunNumber(const std::optional<uint64_t> &run_number);
+    DatasetSettings& RunName(const std::optional<std::string> &input);
+    DatasetSettings& ExperimentGroup(const std::string &group);
+    DatasetSettings& ImageTime(const std::optional<std::chrono::microseconds> input);
+
+    DatasetSettings& LossyCompressionSerialMX(float input);
+    DatasetSettings& LossyCompressionPoisson(std::optional<int64_t> input);
 
     std::optional<float> GetAttenuatorTransmission() const;
     std::optional<float> GetTotalFlux() const;
-    std::optional<float> GetOmegaStep() const;
-    float GetOmegaStart() const;
-    std::optional<Coord> GetOmegaAxis() const;
-    std::string GetHeaderAppendix() const;
-    std::string GetImageAppendix() const;
+    std::optional<GoniometerAxis> GetGoniometer() const;
+    std::optional<Coord> GetRotationAxis() const;
+    const nlohmann::json& GetHeaderAppendix() const;
+    const nlohmann::json& GetImageAppendix() const;
     float GetPhotonEnergyMultiplier() const;
     std::optional<UnitCell> GetUnitCell() const;
     int64_t GetSpaceGroupNumber() const;
-    int64_t GetSummation() const;
     FPGAPixelOutput GetFPGAOutputMode() const;
     std::string GetSampleName() const;
     float GetPhotonEnergy_keV() const;
@@ -103,7 +109,14 @@ public:
     int64_t GetNumTriggers() const;
     int64_t GetImageNumPerTrigger() const;
     int64_t GetImagesPerFile() const;
-    float GetDataReductionFactorSerialMX() const;
+
+    std::optional<uint64_t> GetRunNumber() const;
+    std::optional<std::string> GetRunName() const;
+    std::string GetExperimentGroup() const;
+    std::optional<std::chrono::microseconds> GetImageTime() const;
+    
+    float GetLossyCompressionSerialMX() const;
+    std::optional<int64_t> GetLossyCompressionPoisson() const;
 };
 
 #endif //JUNGFRAUJOCH_DATASETSETTINGS_H

common/Definitions.h

@@ -12,10 +12,12 @@
 #define CONVERTED_MODULE_SIZE   (CONVERTED_MODULE_LINES * CONVERTED_MODULE_COLS)
 #define JUNGFRAU_PACKET_SIZE_BYTES  (8192)
 
-#define MIN_COUNT_TIME_IN_US 5
 #define MIN_FRAME_TIME_HALF_SPEED_IN_US 1000
 #define MIN_FRAME_TIME_FULL_SPEED_IN_US 470
-#define MAX_FRAME_TIME 2000
+
+#define MAX_COUNT_TIME_IN_US 1980
+#define MIN_COUNT_TIME_IN_US 3
+
 #define MIN_STORAGE_CELL_DELAY_IN_NS 2100
 #define READOUT_TIME_IN_US 20
 

common/DetectorSetup.cpp

@@ -148,3 +148,12 @@ void DetectorSetup::SetTrimFiles(const std::vector<std::string> &filenames) {
 const std::vector<std::string> &DetectorSetup::GetTrimFileNames() const {
     return trim_file_names;
 }
+
+std::string DetectorSetup::GetSerialNumber() const {
+    return serial_number;
+}
+
+DetectorSetup &DetectorSetup::SerialNumber(const std::string &input) {
+    serial_number = input;
+    return *this;
+}

common/DetectorSetup.h

@@ -10,6 +10,7 @@ enum class DetectorType {EIGER, JUNGFRAU};
 
 class DetectorSetup {
     std::string description;
+    std::string serial_number;
     DetectorGeometry geometry;
     std::vector<std::string> det_modules_hostname;
     std::vector<std::string> gain_file_names;
@@ -41,6 +42,7 @@ public:
     DetectorSetup& SensorThickness_um(float input);
     DetectorSetup& PixelSize_um(float input);
     DetectorSetup& HighVoltage(int32_t input);
+    DetectorSetup& SerialNumber(const std::string &input);
 
     [[nodiscard]] DetectorType GetDetectorType() const;
     [[nodiscard]] const DetectorGeometry& GetGeometry() const;
@@ -56,6 +58,7 @@ public:
     [[nodiscard]] const std::vector<std::string> &GetGainFileNames() const;
     [[nodiscard]] const std::vector<std::string> &GetTrimFileNames() const;
     [[nodiscard]] int32_t GetHighVoltage() const;
+    [[nodiscard]] std::string GetSerialNumber() const;
 };
 
 

common/DiffractionExperiment.cpp

@@ -10,6 +10,7 @@
 #include "RawToConvertedGeometry.h"
 #include "../fpga/pcie_driver/jfjoch_fpga.h"
 #include "../include/spdlog/fmt/fmt.h"
+#include "GitInfo.h"
 
 using namespace std::literals::chrono_literals;
 
@@ -66,6 +67,8 @@ DiffractionExperiment::DiffractionExperiment(const DetectorSetup& det_setup)
     mode = DetectorMode::Conversion;
 
     max_spot_count = MAX_SPOT_COUNT;
+
+    summation = 1;
 }
 
 // setter functions
@@ -100,10 +103,13 @@ DiffractionExperiment &DiffractionExperiment::NumTriggers(int64_t input) {
 DiffractionExperiment &DiffractionExperiment::FrameTime(std::chrono::microseconds in_frame_time,
                                                         std::chrono::microseconds in_count_time) {
     check_min("Frame time (us)", in_frame_time.count(), MIN_FRAME_TIME_FULL_SPEED_IN_US);
-    check_max("Frame time (us)", in_frame_time.count(), MAX_FRAME_TIME);
     check_max("Count time (us)", in_count_time.count(), in_frame_time.count() - READOUT_TIME_IN_US);
     if (in_count_time.count() != 0) {
         check_min("Count time (us)", in_count_time.count(), MIN_COUNT_TIME_IN_US);
+        check_max("Count time (us)", in_count_time.count(), MAX_COUNT_TIME_IN_US);
+    } else {
+        check_max("Count time (us)", in_frame_time.count() - READOUT_TIME_IN_US,
+                  MAX_COUNT_TIME_IN_US);
     }
 
     frame_time = in_frame_time;
@@ -729,6 +735,7 @@ void DiffractionExperiment::FillMessage(StartMessage &message) const {
 
     message.source_name = GetSourceName();
     message.source_name_short = GetSourceNameShort();
+    message.source_type = GetSourceType();
 
     message.instrument_name = GetInstrumentName();
     message.instrument_name_short = GetInstrumentNameShort();
@@ -738,21 +745,26 @@ void DiffractionExperiment::FillMessage(StartMessage &message) const {
     message.countrate_correction_enabled = false;
     message.flatfield_enabled = false;
 
-    if (GetOmegaStep())
-        message.omega = GoniometerAxis{.increment = GetOmegaStep().value(), .start = GetOmegaStart(),
-                .axis = {GetOmegaAxis().x, GetOmegaAxis().y, GetOmegaAxis().z}};
-    else
-        message.omega = GoniometerAxis{.increment = 0.0f};
+    auto goniometer = dataset.GetGoniometer();
+    if (goniometer) {
+        message.goniometer = goniometer;
+        message.rotation_axis[0] = GetRotationAxis().x;
+        message.rotation_axis[1] = GetRotationAxis().y;
+        message.rotation_axis[2] = GetRotationAxis().z;
+    }
 
-    message.series_id = GetSeriesID();
-    message.series_unique_id = GetSeriesIDString();
+    message.run_number = GetRunNumber();
+    message.run_name = GetRunName();
 
     message.gain_file_names = detector.GetGainFileNames();
 
     for (const auto &[x, y]: roi_mask.GetROINameMap())
         message.roi_names.emplace_back(x);
 
-    message.data_reduction_factor_serialmx = GetDataReductionFactorSerialMX();
+    message.data_reduction_factor_serialmx = GetLossyCompressionSerialMX();
+    message.experiment_group = dataset.GetExperimentGroup();
+    message.jfjoch_release = jfjoch_version();
+    message.detector_serial_number = detector.GetSerialNumber();
 }
 
 float DiffractionExperiment::GetPixelSize_mm() const {
@@ -769,6 +781,11 @@ DiffractionExperiment &DiffractionExperiment::SourceNameShort(std::string input)
     return *this;
 }
 
+DiffractionExperiment &DiffractionExperiment::SourceType(std::string input) {
+    source_type = input;
+    return *this;
+}
+
 DiffractionExperiment &DiffractionExperiment::InstrumentName(std::string input) {
     instrument_name = input;
     return *this;
@@ -787,6 +804,10 @@ std::string DiffractionExperiment::GetSourceNameShort() const {
     return source_name_short;
 }
 
+std::string DiffractionExperiment::GetSourceType() const {
+    return source_type;
+}
+
 std::string DiffractionExperiment::GetInstrumentName() const {
     return instrument_name;
 }
@@ -864,7 +885,9 @@ std::chrono::nanoseconds DiffractionExperiment::GetStorageCellDelay() const {
 }
 
 DiffractionExperiment &DiffractionExperiment::Summation(int64_t input) {
-    dataset.Summation(input);
+    check_min("Summation factor", input, 1);
+    check_max("Summation factor", input, MAX_FPGA_SUMMATION);
+    summation = input;
     return *this;
 }
 
@@ -880,7 +903,7 @@ int64_t DiffractionExperiment::GetSummation() const {
             if (GetStorageCellNumber() > 1)
                 return 1;
             else
-                return dataset.GetSummation();
+                return summation;
     }
 }
 
@@ -949,22 +972,13 @@ std::optional<float> DiffractionExperiment::GetTotalFlux() const {
     return dataset.GetTotalFlux();
 }
 
-DiffractionExperiment &DiffractionExperiment::OmegaStep(const std::optional<float> &input) {
-    dataset.OmegaStep(input);
+DiffractionExperiment &DiffractionExperiment::Goniometer(const std::optional<GoniometerAxis> &input) {
+    dataset.Goniometer(input);
     return *this;
 }
 
-DiffractionExperiment &DiffractionExperiment::OmegaStart(float input) {
-    dataset.OmegaStart(input);
-    return *this;
-}
-
-std::optional<float> DiffractionExperiment::GetOmegaStep() const {
-    return dataset.GetOmegaStep();
-}
-
-float DiffractionExperiment::GetOmegaStart() const {
-    return dataset.GetOmegaStart();
+std::optional<GoniometerAxis> DiffractionExperiment::GetGoniometer() const {
+    return dataset.GetGoniometer();
 }
 
 DiffractionExperiment &DiffractionExperiment::UsingGainHG0(bool input) {
@@ -985,63 +999,64 @@ bool DiffractionExperiment::IsUsingGainHG0() const {
     return use_gain_hg0;
 }
 
-DiffractionExperiment &DiffractionExperiment::HeaderAppendix(const std::string &input) {
+DiffractionExperiment &DiffractionExperiment::HeaderAppendix(const nlohmann::json &input) {
     dataset.HeaderAppendix(input);
     return *this;
 }
 
-DiffractionExperiment &DiffractionExperiment::ImageAppendix(const std::string &input) {
+DiffractionExperiment &DiffractionExperiment::ImageAppendix(const nlohmann::json &input) {
     dataset.ImageAppendix(input);
     return *this;
 }
 
-std::string DiffractionExperiment::GetHeaderAppendix() const {
+const nlohmann::json& DiffractionExperiment::GetHeaderAppendix() const {
     return dataset.GetHeaderAppendix();
 }
 
-std::string DiffractionExperiment::GetImageAppendix() const {
+const nlohmann::json& DiffractionExperiment::GetImageAppendix() const {
     return dataset.GetImageAppendix();
 }
 
-DiffractionExperiment &DiffractionExperiment::OmegaAxis(const Coord &c) {
-    dataset.OmegaAxis(c);
+DiffractionExperiment &DiffractionExperiment::RotationAxis(const std::optional<Coord> &c) {
+    dataset.RotationAxis(c);
     return *this;
 }
 
-DiffractionExperiment &DiffractionExperiment::OmegaAxis() {
-    dataset.OmegaAxis({});
-    return *this;
-}
-
-Coord DiffractionExperiment::GetOmegaAxis() const {
-    auto tmp = dataset.GetOmegaAxis();
+Coord DiffractionExperiment::GetRotationAxis() const {
+    auto tmp = dataset.GetRotationAxis();
     if (tmp)
         return tmp.value();
     else
         return default_omega_axis;
 }
 
-DiffractionExperiment &DiffractionExperiment::DefaultOmegaAxis(const Coord &c) {
+DiffractionExperiment &DiffractionExperiment::DefaultRotationAxis(const Coord &c) {
     if (c.Length() == 0.0)
-        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Cannot use empty vector for omega axis");
+        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Cannot use empty vector for goniometer axis");
 
     default_omega_axis = c.Normalize();
     return *this;
 }
 
-Coord DiffractionExperiment::GetDefaultOmegaAxis() const {
+Coord DiffractionExperiment::GetDefaultRotationAxis() const {
     return default_omega_axis;
 }
 
-uint64_t DiffractionExperiment::GetSeriesID() const {
+uint64_t DiffractionExperiment::GetRunNumber() const {
+    if (dataset.GetRunNumber())
+        return dataset.GetRunNumber().value();
     return series_id;
 }
 
-std::string DiffractionExperiment::GetSeriesIDString() const {
-    return std::to_string(series_id) + ": " + dataset.GetFilePrefix();
+std::string DiffractionExperiment::GetRunName() const {
+    auto run_name = dataset.GetRunName();
+    if (run_name)
+        return run_name.value();
+    else
+        return std::to_string(series_id) + ":" + dataset.GetFilePrefix();
 }
 
-DiffractionExperiment &DiffractionExperiment::IncrementSeriesID() {
+DiffractionExperiment &DiffractionExperiment::IncrementRunNumber() {
     series_id++;
     return *this;
 }
@@ -1068,6 +1083,7 @@ bool DiffractionExperiment::IsConversionOnFPGA() const {
 }
 
 DiffractionExperiment &DiffractionExperiment::DetectorDelay(std::chrono::nanoseconds input) {
+    check_min("Detector delay (us)", input.count(), 0);
     detector_delay = input;
     return *this;
 }
@@ -1127,6 +1143,19 @@ int64_t DiffractionExperiment::GetInternalPacketGeneratorImages() const {
 DiffractionExperiment &DiffractionExperiment::ImportDatasetSettings(const DatasetSettings &input) {
     auto tmp = dataset;
     dataset = input;
+
+    auto image_time = input.GetImageTime();
+    if (image_time) {
+        if (image_time->count() % GetFrameTime().count() != 0)
+            throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
+                                  "Image time must be multiple of frame time");
+        if (GetFrameTime().count() == 0)
+            throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
+                                  "Frame time cannot be zero");
+        this->Summation(image_time.value() / GetFrameTime());
+    } else
+        summation = 1;
+
     if (GetFrameNum() >= MAX_FRAMES) {
         dataset = tmp;
         throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
@@ -1176,11 +1205,24 @@ int64_t DiffractionExperiment::GetSendBufferLocationSize() const {
     return GetMaxCompressedSize() + 1024 * 1024;
 }
 
-float DiffractionExperiment::GetDataReductionFactorSerialMX() const {
-    return dataset.GetDataReductionFactorSerialMX();
+float DiffractionExperiment::GetLossyCompressionSerialMX() const {
+    return dataset.GetLossyCompressionSerialMX();
 }
 
-DiffractionExperiment &DiffractionExperiment::DataReductionFactorSerialMX(float input) {
-    dataset.DataReductionFactorSerialMX(input);
+DiffractionExperiment &DiffractionExperiment::LossyCompressionSerialMX(float input) {
+    dataset.LossyCompressionSerialMX(input);
     return *this;
 }
+
+std::optional<int64_t> DiffractionExperiment::GetLossyCompressionPoisson() const {
+    return dataset.GetLossyCompressionPoisson();
+}
+
+DiffractionExperiment &DiffractionExperiment::LossyCompressionPoisson(std::optional<int64_t> input) {
+    dataset.LossyCompressionPoisson(input);
+    return *this;
+}
+
+std::string DiffractionExperiment::GetExperimentGroup() const {
+    return dataset.GetExperimentGroup();
+}

common/DiffractionExperiment.h

@@ -78,6 +78,7 @@ class DiffractionExperiment {
 
     std::string source_name;
     std::string source_name_short;
+    std::string source_type;
     std::string instrument_name;
     std::string instrument_name_short;
     bool pulsed_source;
@@ -97,6 +98,10 @@ class DiffractionExperiment {
     ROIMap roi_mask;
 
     int64_t max_spot_count;
+
+    int64_t summation;
+
+    std::string detector_update_zmq_addr;
 public:
     // Public methods are atomic
     DiffractionExperiment();
@@ -132,14 +137,15 @@ public:
     DiffractionExperiment& StorageCellStart(int64_t input = 15);
 
     DiffractionExperiment& SourceName(std::string input);
+    DiffractionExperiment& SourceType(std::string input);
     DiffractionExperiment& SourceNameShort(std::string input);
     DiffractionExperiment& InstrumentName(std::string input);
     DiffractionExperiment& InstrumentNameShort(std::string input);
-    DiffractionExperiment& DefaultOmegaAxis(const Coord &c);
+    DiffractionExperiment& DefaultRotationAxis(const Coord &c);
 
     DiffractionExperiment& UsingGainHG0(bool input);
     DiffractionExperiment& FixedGainG1(bool input);
-    DiffractionExperiment& IncrementSeriesID();
+    DiffractionExperiment& IncrementRunNumber();
     DiffractionExperiment& ConversionOnFPGA(bool input);
     DiffractionExperiment& PulsedSource(bool input);
 
@@ -156,18 +162,17 @@ public:
     DiffractionExperiment& SampleName(std::string input);
     DiffractionExperiment& AttenuatorTransmission(const std::optional<float> &input);
     DiffractionExperiment& TotalFlux(const std::optional<float> &input);
-    DiffractionExperiment& OmegaStep(const std::optional<float> &input);
-    DiffractionExperiment& OmegaStart(float input);
-    DiffractionExperiment& OmegaAxis(const Coord &c);
-    DiffractionExperiment& OmegaAxis();
-    DiffractionExperiment& HeaderAppendix(const std::string& input);
-    DiffractionExperiment& ImageAppendix(const std::string& input);
+    DiffractionExperiment& Goniometer(const std::optional<GoniometerAxis> &input);
+    DiffractionExperiment& RotationAxis(const std::optional<Coord> &c);
+    DiffractionExperiment& HeaderAppendix(const nlohmann::json& input);
+    DiffractionExperiment& ImageAppendix(const nlohmann::json& input);
     DiffractionExperiment& PhotonEnergyMultiplayer(float input);
     DiffractionExperiment& Summation(int64_t input);
     DiffractionExperiment& FPGAOutputMode(FPGAPixelOutput input);
     DiffractionExperiment& MaxSpotCount(int64_t input);
     DiffractionExperiment& ImagesPerFile(int64_t input);
-    DiffractionExperiment& DataReductionFactorSerialMX(float input);
+    DiffractionExperiment& LossyCompressionSerialMX(float input);
+    DiffractionExperiment& LossyCompressionPoisson(std::optional<int64_t> input);
 
     DiffractionExperiment& ImportDatasetSettings(const DatasetSettings& input);
     DatasetSettings GetDatasetSettings() const;
@@ -252,6 +257,7 @@ public:
 
     float GetPixelSize_mm() const;
     std::string GetSourceName() const;
+    std::string GetSourceType() const;
     std::string GetSourceNameShort() const;
     std::string GetInstrumentName() const;
     std::string GetInstrumentNameShort() const;
@@ -268,13 +274,13 @@ public:
 
     int64_t GetUDPInterfaceCount() const;
     std::vector<DetectorModuleConfig> GetDetectorModuleConfig(const std::vector<AcquisitionDeviceNetConfig>& net_config) const;
-    Coord GetDefaultOmegaAxis() const;
+    Coord GetDefaultRotationAxis() const;
 
     bool IsFixedGainG1() const;
     bool IsUsingGainHG0() const;
 
-    uint64_t GetSeriesID() const;
-    std::string GetSeriesIDString() const;
+    uint64_t GetRunNumber() const;
+    std::string GetRunName() const;
     bool IsConversionOnFPGA() const;
 
     const DetectorSetup& GetDetectorSetup() const;
@@ -287,11 +293,11 @@ public:
 
     std::optional<float> GetAttenuatorTransmission() const;
     std::optional<float> GetTotalFlux() const;
-    std::optional<float> GetOmegaStep() const;
-    float GetOmegaStart() const;
-    Coord GetOmegaAxis() const;
-    std::string GetHeaderAppendix() const;
-    std::string GetImageAppendix() const;
+    std::optional<GoniometerAxis> GetGoniometer() const;
+
+    Coord GetRotationAxis() const;
+    const nlohmann::json& GetHeaderAppendix() const;
+    const nlohmann::json& GetImageAppendix() const;
     float GetPhotonEnergyMultiplier() const;
     std::optional<UnitCell> GetUnitCell() const;
     std::string GetUnitCellString() const;
@@ -319,7 +325,9 @@ public:
     void ExportROIMap(uint16_t *v, size_t module_number) const;
     int64_t GetImagesPerFile() const;
 
-    float GetDataReductionFactorSerialMX() const;
+    float GetLossyCompressionSerialMX() const;
+    std::optional<int64_t> GetLossyCompressionPoisson() const;
+    std::string GetExperimentGroup() const;
 };
 
 #endif //DIFFRACTIONEXPERIMENT_H

detector_control/DetectorWrapper.cpp

@@ -65,14 +65,14 @@ void DetectorWrapper::Initialize(const DiffractionExperiment& experiment,
                 det.setSourceUDPIP(sls::IpAddr(cfg.ipv4_src_addr_1), {i});
                 det.setSourceUDPMAC(sls::MacAddr(BASE_DETECTOR_MAC + i * 2), {i});
 
-                det.setDestinationUDPPort(cfg.udp_dest_port_1, i);
+                det.setDestinationUDPPort(cfg.udp_dest_port_1 + 2 * i, i);
                 det.setDestinationUDPIP(sls::IpAddr(cfg.ipv4_dest_addr_1), {i});
                 det.setDestinationUDPMAC(sls::MacAddr(cfg.mac_addr_dest_1), {i});
 
                 if (experiment.GetUDPInterfaceCount() == 2) {
                     det.setSourceUDPIP2(sls::IpAddr(cfg.ipv4_src_addr_2), {i});
                     det.setSourceUDPMAC2(sls::MacAddr(BASE_DETECTOR_MAC + i * 2 + 1), {i});
-                    det.setDestinationUDPPort2(cfg.udp_dest_port_2, i);
+                    det.setDestinationUDPPort2(cfg.udp_dest_port_2 + 2 * i + 1, i);
                     det.setDestinationUDPIP2(sls::IpAddr(cfg.ipv4_dest_addr_2), {i});
                     det.setDestinationUDPMAC2(sls::MacAddr(cfg.mac_addr_dest_2), {i});
                 }
@@ -91,7 +91,7 @@ void DetectorWrapper::Initialize(const DiffractionExperiment& experiment,
                 throw JFJochException(JFJochExceptionCategory::Detector,
                                       "Discrepancy in module number between DAQ and detector");
             }
-
+            det.setDynamicRange(16);
             det.setTenGiga(true);
             auto trim_files = experiment.GetDetectorSetup().GetTrimFileNames();
 
@@ -100,8 +100,8 @@ void DetectorWrapper::Initialize(const DiffractionExperiment& experiment,
 
                 auto &cfg = mod_cfg[i];
 
-                det.setDestinationUDPPort(cfg.udp_dest_port_1, 2 * i);
-                det.setDestinationUDPPort2(cfg.udp_dest_port_1, 2 * i + 1);
+                det.setDestinationUDPPort(cfg.udp_dest_port_1 + 2 * i, 2 * i);
+                det.setDestinationUDPPort2(cfg.udp_dest_port_1 + 2 * i + 1, 2 * i + 1);
                 det.setSourceUDPIP(sls::IpAddr(cfg.ipv4_src_addr_1), {2 * i, 2 * i + 1});
                 det.setDestinationUDPIP(sls::IpAddr(cfg.ipv4_dest_addr_1), {2 * i, 2 * i + 1});
                 det.setDestinationUDPMAC(sls::MacAddr(cfg.mac_addr_dest_1), {2 * i, 2 * i + 1});

etc/broker.json

@@ -8,13 +8,13 @@
     "pedestal_g0_frames": 2000,
     "pedestal_g1_frames": 300,
     "pedestal_g2_frames": 300,
-    "frame_time_us": "500 us",
-    "count_time_us": "480 us",
-    "preview_period_us": "1 s",
+    "frame_time": "500 us",
+    "count_time": "480 us",
     "detector_ipv4": "10.10.85.0"
   },
   "frontend_directory":"/home/jungfrau/nextgendcu/frontend/build",
   "numa_policy": "n2g2",
+  "zmq_preview_period": "1 s",
   "zmq_image_addr": ["tcp://0.0.0.0:5500", "tcp://0.0.0.0:5501", "tcp://0.0.0.0:5502", "tcp://0.0.0.0:5503"],
   "receiver_threads": 64,
   "receiver": {

etc/broker_crmx.json

@@ -8,15 +8,16 @@
     "pedestal_g0_frames": 2000,
     "pedestal_g1_frames": 300,
     "pedestal_g2_frames": 300,
-    "frame_time_us": "10 ms",
-    "count_time_us": "10 us",
-    "preview_period_us": "1 s",
+    "frame_time": "10 ms",
+    "count_time": "10 us",
     "detector_ipv4": "10.3.30.155",
     "pulsed_source": true
   },
   "frontend_directory":"/home/jungfrau/nextgendcu/frontend/build",
   "numa_policy": "n2g4",
   "zmq_image_addr": ["tcp://0.0.0.0:5500"],
+  "zmq_preview_period": "1 s",
+
   "receiver_threads": 64,
   "receiver": {
     "type": "pcie",

etc/broker_local.json

@@ -8,9 +8,8 @@
     "pedestal_g0_frames": 0,
     "pedestal_g1_frames": 0,
     "pedestal_g2_frames": 0,
-    "frame_time_us": "1000 us",
-    "count_time_us": "980 us",
-    "preview_period_us": "1 s",
+    "frame_time": "1000 us",
+    "count_time": "980 us",
     "detector_ipv4": "10.10.85.20",
     "internal_frame_generator": true
   },
@@ -20,6 +19,9 @@
   },
   "frontend_directory": "../../frontend_ui/build/",
   "zmq_image_addr": ["tcp://0.0.0.0:5500"],
+  "zmq_preview_addr": "tcp://0.0.0.0:5501",
+  "zmq_preview_period": "1 s",
+  "zmq_writer_notification_addr": "ipc://*",
   "detectors": [
     {
       "standard_geometry": {

fpga/CMakeLists.txt

@@ -12,8 +12,6 @@ ELSE()
     MESSAGE(STATUS "Xilinx HLS compiler not found")
 ENDIF()
 
-INCLUDE_DIRECTORIES(include)
-
 ADD_SUBDIRECTORY(hls)
 ADD_SUBDIRECTORY(pcie_driver)
 ADD_SUBDIRECTORY(host_library)

fpga/hdl/action_config.v

@@ -75,7 +75,9 @@
 `define ADDR_NSTORAGE_CELLS      16'h021C
 
 `define ADDR_NSUMMATION          16'h0220
-`define ADDR_DATA_SOURCE         16'h0224
+`define ADDR_SQRTMULT            16'h0224
+
+`define ADDR_DATA_SOURCE         16'h0300
 
 module action_config
 #(parameter C_S_AXI_ADDR_WIDTH = 16,
@@ -114,6 +116,7 @@ module action_config
     output reg [4:0]                       nmodules                 ,
     output reg [3:0]                       nstorage_cells           ,
     output reg [7:0]                       nsummation               ,
+    output reg [7:0]                       sqrtmult                 ,
     output reg [1:0]                       data_source              ,
     output wire [31:0]                     hbm_size_bytes           ,
 
@@ -366,6 +369,9 @@ always @(posedge clk) begin
             `ADDR_NSUMMATION: begin
                 rdata <= nsummation;
             end
+            `ADDR_SQRTMULT: begin
+                rdata <= sqrtmult;
+            end
             `ADDR_DATA_SOURCE: begin
                 rdata <= data_source;
             end
@@ -639,6 +645,15 @@ always @(posedge clk) begin
     end
 end
 
+always @(posedge clk) begin
+    if (!resetn)
+        sqrtmult <= 0;
+    else if (reg_data_collection_idle) begin
+        if (w_hs && waddr == `ADDR_SQRTMULT)
+            sqrtmult <= (s_axi_WDATA[7:0] & wmask[7:0]) | (sqrtmult & !wmask[7:0]);
+    end
+end
+
 always @(posedge clk) begin
     if (!resetn)
         data_source <= 0;

fpga/hls/CMakeLists.txt

@@ -31,11 +31,15 @@ ADD_LIBRARY( JFJochHLSSimulation STATIC
         spot_finder_mask.cpp
         save_to_hbm_data.cpp
         eiger_reorder.cpp
-        roi_calc.cpp)
+        roi_calc.cpp
+        pixel_sqrt.cpp
+        HLSDevice.cpp
+        HLSDevice.h)
+
 
-TARGET_INCLUDE_DIRECTORIES(JFJochHLSSimulation PUBLIC ../include)
 TARGET_LINK_LIBRARIES(JFJochHLSSimulation JFJochCommon)
 TARGET_COMPILE_DEFINITIONS(JFJochHLSSimulation PUBLIC -DJFJOCH_HLS_NOSYNTH)
+TARGET_INCLUDE_DIRECTORIES(JFJochHLSSimulation PUBLIC ../include)
 
 ADD_EXECUTABLE(frame_summation_tb frame_summation_tb.cpp)
 TARGET_LINK_LIBRARIES(frame_summation_tb JFJochHLSSimulation)
@@ -54,55 +58,69 @@ TARGET_LINK_LIBRARIES(adu_histo_tb JFJochHLSSimulation)
 
 IF(VIVADO_HLS)
 
-IF(NOT HLS_SOLUTION_NAME)
-SET(HLS_SOLUTION_NAME base1)
-ENDIF()
+    GET_FILENAME_COMPONENT(VITIS_HLS_DIR ${VIVADO_HLS} DIRECTORY)
+    FIND_FILE(HLS_MATH_H hls_math.h HINT ${VITIS_HLS_DIR}/../include)
+    IF (HLS_MATH_H)
+        GET_FILENAME_COMPONENT(VITIS_HLS_INCLUDE_DIR ${HLS_MATH_H} DIRECTORY)
+        MESSAGE(STATUS "Xilinx HLS headers included")
 
-SET (HLS_IPS "")
+        #TARGET_INCLUDE_DIRECTORIES(JFJochHLSSimulation PUBLIC ${VITIS_HLS_INCLUDE_DIR})
+        #TARGET_COMPILE_DEFINITIONS(JFJochHLSSimulation PUBLIC JFJOCH_USE_HLS_HEADERS)
+    ELSE()
+        MESSAGE(WARNING "Xilinx HLS headers missing")
+        #TARGET_INCLUDE_DIRECTORIES(JFJochHLSSimulation PUBLIC ../include)
+    ENDIF()
 
-FUNCTION( MAKE_HLS_MODULE FUNCTION_NAME SRC_FILE TB_FILE)
-    ADD_CUSTOM_COMMAND(OUTPUT psi_ch_hls_${FUNCTION_NAME}_1_0.zip
-            COMMAND ${CMAKE_COMMAND} -E env SRC_DIR=${CMAKE_CURRENT_SOURCE_DIR} HLS_FILE=${SRC_FILE} HLS_TOP_FUNCTION=${FUNCTION_NAME} HLS_TB_FILE=${TB_FILE} ${VIVADO_HLS} -f ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/synth_hls_function.tcl > hls_${FUNCTION_NAME}.log
-            COMMAND ${CMAKE_COMMAND} -E env HLS_DIR=${CMAKE_CURRENT_BINARY_DIR}/${FUNCTION_NAME}/solution1 CURR_DIR=${CMAKE_CURRENT_BINARY_DIR} bash ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/check_hls.sh ${FUNCTION_NAME}
-            COMMAND ${CMAKE_COMMAND} -E copy ${FUNCTION_NAME}/solution1/impl/ip/psi_ch_hls_${FUNCTION_NAME}_1_0.zip .
-            DEPENDS ${SRC_FILE} hls_jfjoch.h ../pcie_driver/jfjoch_fpga.h)
-    SET (HLS_IPS ${HLS_IPS} psi_ch_hls_${FUNCTION_NAME}_1_0.zip PARENT_SCOPE)
-ENDFUNCTION(MAKE_HLS_MODULE)
+    IF(NOT HLS_SOLUTION_NAME)
+        SET(HLS_SOLUTION_NAME base1)
+    ENDIF()
 
-MAKE_HLS_MODULE(data_collection_fsm data_collection_fsm.cpp "")
-MAKE_HLS_MODULE(timer_host timer.cpp "")
-MAKE_HLS_MODULE(jf_conversion jf_conversion.cpp "")
-MAKE_HLS_MODULE(load_calibration load_calibration.cpp "")
-MAKE_HLS_MODULE(host_writer host_writer.cpp "")
-MAKE_HLS_MODULE(icmp icmp.cpp "")
-MAKE_HLS_MODULE(ipv4 ipv4.cpp "")
-MAKE_HLS_MODULE(ethernet ethernet.cpp "")
-MAKE_HLS_MODULE(arp arp.cpp "")
-MAKE_HLS_MODULE(udp udp.cpp "")
-MAKE_HLS_MODULE(sls_detector sls_detector.cpp "")
-MAKE_HLS_MODULE(frame_generator frame_generator.cpp "")
-MAKE_HLS_MODULE(stream_merge stream_merge.cpp "")
-MAKE_HLS_MODULE(load_from_hbm load_from_hbm.cpp "")
-MAKE_HLS_MODULE(save_to_hbm save_to_hbm.cpp "")
-MAKE_HLS_MODULE(save_to_hbm_data save_to_hbm_data.cpp "")
-MAKE_HLS_MODULE(mask_missing mask_missing.cpp "")
-MAKE_HLS_MODULE(integration integration.cpp integration_tb.cpp)
-MAKE_HLS_MODULE(spot_finder spot_finder.cpp spot_finder_tb.cpp)
-MAKE_HLS_MODULE(axis_broadcast axis_broadcast.cpp "")
-MAKE_HLS_MODULE(axis_64_to_512 axis_helpers.cpp "")
-MAKE_HLS_MODULE(axis_32_to_512 axis_helpers.cpp "")
-MAKE_HLS_MODULE(adu_histo adu_histo.cpp adu_histo_tb.cpp)
-MAKE_HLS_MODULE(pedestal pedestal.cpp "")
-MAKE_HLS_MODULE(frame_summation frame_summation.cpp frame_summation_tb_2.cpp)
-MAKE_HLS_MODULE(frame_summation_reorder_compl frame_summation_reorder_compl.cpp "")
-MAKE_HLS_MODULE(stream_24bit_conv stream_24bit_conv.cpp stream_24bit_conv_tb.cpp)
-MAKE_HLS_MODULE(spot_finder_mask spot_finder_mask.cpp "")
-MAKE_HLS_MODULE(spot_finder_merge spot_finder_merge.cpp "")
-MAKE_HLS_MODULE(spot_finder_connectivity spot_finder_connectivity.cpp "")
-MAKE_HLS_MODULE(eiger_reorder eiger_reorder.cpp "")
-MAKE_HLS_MODULE(roi_calc roi_calc.cpp "")
+    SET (HLS_IPS "")
 
-SET (HLS_IPS ${HLS_IPS} PARENT_SCOPE)
-ADD_CUSTOM_TARGET(hls DEPENDS ${HLS_IPS})
+    FUNCTION( MAKE_HLS_MODULE FUNCTION_NAME SRC_FILE TB_FILE)
+        ADD_CUSTOM_COMMAND(OUTPUT psi_ch_hls_${FUNCTION_NAME}_1_0.zip
+                COMMAND ${CMAKE_COMMAND} -E env SRC_DIR=${CMAKE_CURRENT_SOURCE_DIR} HLS_FILE=${SRC_FILE} HLS_TOP_FUNCTION=${FUNCTION_NAME} HLS_TB_FILE=${TB_FILE} ${VIVADO_HLS} -f ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/synth_hls_function.tcl > hls_${FUNCTION_NAME}.log
+                COMMAND ${CMAKE_COMMAND} -E env HLS_DIR=${CMAKE_CURRENT_BINARY_DIR}/${FUNCTION_NAME}/solution1 CURR_DIR=${CMAKE_CURRENT_BINARY_DIR} bash ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/check_hls.sh ${FUNCTION_NAME}
+                COMMAND ${CMAKE_COMMAND} -E copy ${FUNCTION_NAME}/solution1/impl/ip/psi_ch_hls_${FUNCTION_NAME}_1_0.zip .
+                DEPENDS ${SRC_FILE} hls_jfjoch.h ../pcie_driver/jfjoch_fpga.h)
+        SET (HLS_IPS ${HLS_IPS} psi_ch_hls_${FUNCTION_NAME}_1_0.zip PARENT_SCOPE)
+    ENDFUNCTION(MAKE_HLS_MODULE)
+
+    MAKE_HLS_MODULE(data_collection_fsm data_collection_fsm.cpp "")
+    MAKE_HLS_MODULE(timer_host timer.cpp "")
+    MAKE_HLS_MODULE(jf_conversion jf_conversion.cpp "")
+    MAKE_HLS_MODULE(load_calibration load_calibration.cpp "")
+    MAKE_HLS_MODULE(host_writer host_writer.cpp "")
+    MAKE_HLS_MODULE(pixel_sqrt pixel_sqrt.cpp pixel_sqrt_tb.cpp)
+    MAKE_HLS_MODULE(icmp icmp.cpp "")
+    MAKE_HLS_MODULE(ipv4 ipv4.cpp "")
+    MAKE_HLS_MODULE(ethernet ethernet.cpp "")
+    MAKE_HLS_MODULE(arp arp.cpp "")
+    MAKE_HLS_MODULE(udp udp.cpp "")
+    MAKE_HLS_MODULE(sls_detector sls_detector.cpp "")
+    MAKE_HLS_MODULE(frame_generator frame_generator.cpp "")
+    MAKE_HLS_MODULE(stream_merge stream_merge.cpp "")
+    MAKE_HLS_MODULE(load_from_hbm load_from_hbm.cpp "")
+    MAKE_HLS_MODULE(save_to_hbm save_to_hbm.cpp "")
+    MAKE_HLS_MODULE(save_to_hbm_data save_to_hbm_data.cpp "")
+    MAKE_HLS_MODULE(mask_missing mask_missing.cpp "")
+    MAKE_HLS_MODULE(integration integration.cpp integration_tb.cpp)
+    MAKE_HLS_MODULE(spot_finder spot_finder.cpp spot_finder_tb.cpp)
+    MAKE_HLS_MODULE(axis_broadcast axis_broadcast.cpp "")
+    MAKE_HLS_MODULE(axis_64_to_512 axis_helpers.cpp "")
+    MAKE_HLS_MODULE(axis_32_to_512 axis_helpers.cpp "")
+    MAKE_HLS_MODULE(adu_histo adu_histo.cpp adu_histo_tb.cpp)
+    MAKE_HLS_MODULE(pedestal pedestal.cpp "")
+    MAKE_HLS_MODULE(frame_summation frame_summation.cpp frame_summation_tb_2.cpp)
+    MAKE_HLS_MODULE(frame_summation_reorder_compl frame_summation_reorder_compl.cpp "")
+    MAKE_HLS_MODULE(stream_24bit_conv stream_24bit_conv.cpp stream_24bit_conv_tb.cpp)
+    MAKE_HLS_MODULE(spot_finder_mask spot_finder_mask.cpp "")
+    MAKE_HLS_MODULE(spot_finder_merge spot_finder_merge.cpp "")
+    MAKE_HLS_MODULE(spot_finder_connectivity spot_finder_connectivity.cpp "")
+    MAKE_HLS_MODULE(eiger_reorder eiger_reorder.cpp "")
+    MAKE_HLS_MODULE(roi_calc roi_calc.cpp "")
+
+    SET (HLS_IPS ${HLS_IPS} PARENT_SCOPE)
+    ADD_CUSTOM_TARGET(hls DEPENDS ${HLS_IPS})
 
 ENDIF()

fpga/hls/HLSDevice.cpp

@@ -0,0 +1,791 @@
+// Copyright (2019-2024) Paul Scherrer Institute
+
+#include <future>
+#include <bitset>
+#include <arpa/inet.h>
+
+#include "HLSDevice.h"
+#include "hls_jfjoch.h"
+#include "datamover_model.h"
+#include "../../jungfrau/sls_packet.h"
+#include "../../common/Logger.h"
+#include "../../common/JFJochException.h"
+
+struct HLSDeviceImpl {
+    AXI_STREAM din_eth;
+    AXI_STREAM din_frame_generator;
+    AXI_STREAM dout_eth;
+
+    constexpr static const size_t hbm_if_count = 32;
+    constexpr static const size_t hbm_if_size = 32 * 1024 * 1024LU;
+
+    ap_uint<2> data_source = STREAM_MERGE_SRC_NETWORK;
+
+    std::vector<ap_uint<256>> hbm;
+
+    hls::stream<ap_uint<32> > work_request_stream;
+    hls::stream<ap_uint<32> > completion_stream;
+
+    Datamover<512> datamover_in;
+    Datamover<512> datamover_out;
+    Datamover<256> datamover_in_hbm_0;
+    Datamover<256> datamover_in_hbm_1;
+    Datamover<256, 16> datamover_out_hbm_0;
+    Datamover<256, 16> datamover_out_hbm_1;
+
+    ap_uint<1> run_data_collection;
+    ap_uint<1> cancel_data_collection;
+    volatile ap_uint<1> host_writer_idle;
+
+    HLSDeviceImpl()
+            : hbm(hbm_if_size / 32 * hbm_if_count),
+              datamover_in(Direction::Input, nullptr),
+              datamover_out(Direction::Output, nullptr),
+              datamover_out_hbm_0(Direction::Output, (char *) hbm.data()),
+              datamover_out_hbm_1(Direction::Output, (char *) hbm.data()),
+              datamover_in_hbm_0(Direction::Input, (char *) hbm.data()),
+              datamover_in_hbm_1(Direction::Input, (char *) hbm.data()) {}
+};
+
+uint16_t checksum(const uint16_t *addr, size_t count) {
+/* Compute Internet Checksum for "count" bytes
+ *         beginning at location "addr".
+ */
+    long sum = 0;
+
+    for (int i = 0; i < count / 2; i++)
+        sum += addr[i];
+
+/*  Add left-over byte, if any */
+    if (count % 2 == 1)
+        sum += ((uint8_t *) addr)[count / 2];
+
+/*  Fold 32-bit sum to 16 bits */
+    while (sum>>16)
+        sum = (sum & 0xffff) + (sum >> 16);
+
+    return ~sum;
+}
+
+HLSDevice::HLSDevice(std::vector<DeviceOutput *> &buffer_device) : idle(true), dma_address_table(65536) {
+    impl_ = std::make_unique<HLSDeviceImpl>();
+
+    auto in_mem_location32 = (uint32_t *) dma_address_table.data();
+
+    for (int i = 0; i < buffer_device.size(); i++) {
+        in_mem_location32[2 * i    ] = ((uint64_t) buffer_device[i]) & UINT32_MAX;
+        in_mem_location32[2 * i + 1] = ((uint64_t) buffer_device[i]) >> 32;
+    }
+}
+
+HLSDevice::~HLSDevice() {
+    if (action_thread.joinable())
+        action_thread.join();
+    impl_.reset();
+}
+
+void HLSDevice::CreateFinalPacket(const DiffractionExperiment& experiment) {
+    CreateJFPacket(experiment, UINT64_MAX, 0, 0, nullptr, false);
+}
+
+void HLSDevice::SendPacket(char *buffer, int len, uint8_t user) {
+    auto obuff = (ap_uint<512> *)buffer;
+
+    for (int i = 0; i < (len + 63) / 64; i++) {
+        packet_512_t packet_in;
+        if (i == (len + 63) / 64 - 1) packet_in.last = 1;
+        else packet_in.last = 0;
+        packet_in.keep = 0xFFFFFFFFFFFFFFFF;
+        packet_in.user = user;
+        packet_in.data = obuff[i];
+        impl_->din_eth.write(packet_in);
+    }
+}
+
+void HLSDevice::CreateJFPacket(const DiffractionExperiment& experiment, uint64_t frame_number, uint32_t eth_packet,
+                                        uint32_t module_number, const uint16_t *data, int8_t adjust_axis, uint8_t user) {
+    char buff[256*64];
+    memset(buff, 0, 256*64);
+
+    auto packet = (jf_raw_packet *)buff;
+
+    packet->ether_type = htons(0x0800);
+    packet->sour_mac[0] = 0x00; // module 0
+
+    uint64_t tmp_mac = mac_addr;
+    for (int i = 0; i < 6; i++)
+        packet->dest_mac[i] = (tmp_mac >> (8*i)) % 256;
+
+    uint32_t half_module = 2 * module_number | ((eth_packet >= 64) ? 1 : 0);
+
+    packet->ipv4_header_h = htons(0x4500); // Big endian in IP header!
+    packet->ipv4_header_total_length = htons(8268); // Big endian in IP header!
+    packet->ipv4_header_dest_ip = ipv4_addr;
+    packet->ipv4_header_sour_ip = experiment.GetSrcIPv4Address(0, half_module);
+
+    packet->ipv4_header_ttl_protocol = htons(0x0011);
+    packet->ipv4_header_checksum = checksum( (uint16_t *) &packet->ipv4_header_h, 20); // checksum is already in network order
+
+    packet->udp_dest_port = htons(234); // port doesn't matter
+    packet->udp_sour_port = htons(0xDFAC);
+    packet->udp_length = htons(8248);
+
+    // JF headers are little endian
+    packet->jf.detectortype = SLS_DETECTOR_TYPE_JUNGFRAU;
+    packet->jf.timestamp = 0xABCDEF0000FEDCBAL;
+    packet->jf.bunchid   = 0x1234567898765431L;
+    packet->jf.row = half_module;
+    packet->jf.column = 0;
+    packet->jf.framenum = frame_number;
+    packet->jf.packetnum = eth_packet % 64;
+    if (data != nullptr) {
+        for (int i = 0; i < 4096; i++)
+            packet->jf.data[i] = data[i];
+    }
+    packet->udp_checksum = htons(checksum( (uint16_t *) (buff+42), 8192+48));
+
+    SendPacket(buff, (130+adjust_axis)*64, user);
+}
+
+void HLSDevice::CreateJFPackets(const DiffractionExperiment& experiment, uint64_t frame_number_0, uint64_t frames,
+                                         uint32_t module_number, const uint16_t *data) {
+    for (uint64_t i = 0; i < frames; i++) {
+        for (int j = 0; j < 128; j++)
+            CreateJFPacket(experiment, frame_number_0 + i, j, module_number, data + (i * 128 + j) * 4096, 0, 0);
+    }
+}
+
+void HLSDevice::CreateEIGERPacket(const DiffractionExperiment &experiment, uint64_t frame_number,
+                                           uint32_t eth_packet, uint32_t module_number, uint32_t col, uint32_t row,
+                                           const uint16_t *data) {
+    char buff[256*64];
+    memset(buff, 0, 256*64);
+
+    auto packet = (eiger_raw_packet *)buff;
+
+    packet->ether_type = htons(0x0800);
+    packet->sour_mac[0] = 0x00; // module 0
+
+    uint64_t tmp_mac = mac_addr;
+    for (int i = 0; i < 6; i++)
+        packet->dest_mac[i] = (tmp_mac >> (8*i)) % 256;
+
+    packet->ipv4_header_h = htons(0x4500); // Big endian in IP header!
+    packet->ipv4_header_total_length = htons(4172); // Big endian in IP header!
+    packet->ipv4_header_dest_ip = ipv4_addr;
+    packet->ipv4_header_sour_ip = experiment.GetSrcIPv4Address(0, 0);
+
+    packet->ipv4_header_ttl_protocol = htons(0x0011);
+    packet->ipv4_header_checksum = checksum( (uint16_t *) &packet->ipv4_header_h, 20); // checksum is already in network order
+
+    packet->udp_dest_port = htons(234); // ignored
+    packet->udp_sour_port = htons(0xDFAC);
+    packet->udp_length = htons(4152);
+
+    // JF headers are little endian
+    packet->eiger.detectortype = SLS_DETECTOR_TYPE_EIGER;
+    packet->eiger.timestamp = 0xABCDEF0000FEDCBAL;
+    packet->eiger.bunchid   = 0x1234567898765431L;
+    packet->eiger.row = (2 * module_number) | (row % 2);
+    packet->eiger.column = col % 2;
+    packet->eiger.framenum = frame_number;
+    packet->eiger.packetnum = eth_packet % 64;
+    if (data != nullptr) {
+        for (int i = 0; i < 2048; i++)
+            packet->eiger.data[i] = data[i];
+    }
+    packet->udp_checksum = htons(checksum( (uint16_t *) (buff+42), 4096+48));
+
+    SendPacket(buff, 66*64, 0);
+}
+
+void HLSDevice::HW_ReadActionRegister(DataCollectionConfig *job) {
+    memcpy(job, &cfg, sizeof(DataCollectionConfig));
+}
+
+void HLSDevice::HW_WriteActionRegister(const DataCollectionConfig *job) {
+    memcpy(&cfg, job, sizeof(DataCollectionConfig));
+}
+
+void HLSDevice::FPGA_StartAction(const DiffractionExperiment &experiment) {
+    if (action_thread.joinable())
+        action_thread.join();
+
+    run_counter += 1;
+    impl_->run_data_collection = 1;
+    impl_->cancel_data_collection = 0;
+    idle = false;
+
+    while (!impl_->din_frame_generator.empty())
+        impl_->din_frame_generator.read();
+
+    impl_->datamover_out.ClearCompletedDescriptors();
+
+    action_thread = std::thread(&HLSDevice::HLSMainThread, this );
+}
+
+void HLSDevice::FrameGeneratorFuture(FrameGeneratorConfig config) {
+    frame_generator(impl_->din_frame_generator,
+                    impl_->hbm.data(),
+                    impl_->hbm.data(),
+                    impl_->hbm_if_size,
+                    mac_addr,
+                    ipv4_addr,
+                    impl_->cancel_data_collection,
+                    config);
+}
+
+void HLSDevice::HW_RunInternalGenerator(const FrameGeneratorConfig &config) {
+    frame_generator_future = std::async(std::launch::async, &HLSDevice::FrameGeneratorFuture, this, config);
+}
+
+void HLSDevice::FPGA_EndAction() {
+    if (action_thread.joinable())
+        action_thread.join();
+}
+
+bool HLSDevice::HW_ReadMailbox(uint32_t *values) {
+    std::unique_lock<std::mutex> ul(completion_mutex);
+
+    ap_uint<32> tmp;
+    bool ret = impl_->completion_stream.read_nb(tmp);
+    values[0] = tmp;
+    // equivalent to driver functionality
+    if (ret) {
+        uint32_t data_collection_id = (values[0] >> 16) & 0xFFFF;
+        uint32_t handle = values[0] & 0xFFFF;
+        if (handle == HANDLE_START)
+            completion_count = 0;
+        else if ((handle != HANDLE_END) && (data_collection_id != DATA_COLLECTION_ID_PURGE)) {
+            completion_count++;
+            while (completion_count * DMA_DESCRIPTORS_PER_MODULE > impl_->datamover_out.GetCompletedDescriptors())
+                std::this_thread::sleep_for(std::chrono::milliseconds(1));
+        }
+    }
+    return ret;
+}
+
+void HLSDevice::Cancel() {
+    impl_->cancel_data_collection = 1;
+}
+
+bool HLSDevice::HW_IsIdle() const {
+    return idle && impl_->datamover_out.IsIdle();
+}
+
+
+bool HLSDevice::HW_SendWorkRequest(uint32_t handle) {
+    impl_->work_request_stream.write(handle);
+    return true;
+}
+
+inline uint32_t float2uint(float f) {
+    float_uint32 fu;
+    fu.f = f;
+    return fu.u;
+}
+
+void HLSDevice::HLSMainThread() {
+    ap_uint<1> clear_counters = 0;
+
+    uint64_t packets_processed;
+
+    std::vector<std::thread> hls_cores;
+
+    STREAM_512 ip1, udp1, udp2, icmp1, arp1;
+
+    STREAM_512 network0;
+
+    STREAM_768 stream_768_0;
+    STREAM_768 stream_768_1;
+    STREAM_768 stream_768_2;
+    STREAM_768 stream_768_3;
+    STREAM_768 stream_768_4;
+    STREAM_768 stream_768_5;
+    STREAM_768 stream_768_6;
+
+    STREAM_512 data_0;
+    hls::stream<ap_axiu<512, 1, 1, 1>, 2> data_1;
+    hls::stream<ap_axiu<512, 1, 1, 1>, 2> data_2;
+    STREAM_512 data_3;
+    STREAM_512 data_4;
+    STREAM_512 data_5;
+    STREAM_512 data_6;
+    STREAM_512 data_7;
+    STREAM_512 data_8;
+    STREAM_512 data_9;
+    STREAM_512 data_10;
+    STREAM_512 data_12;
+    STREAM_512 data_13;
+
+    hls::stream<axis_addr> addr0;
+    hls::stream<axis_addr> addr1;
+    hls::stream<axis_addr> addr2;
+    hls::stream<axis_addr> addr3;
+
+    hls::stream<axis_completion> axi_compl[12];
+
+    hls::stream<ap_uint<16>> hbm_handles;
+    hls::stream<ap_uint<512>> adu_histo_result;
+
+    hls::stream<ap_axiu<64,1,1,1>> integration_result_0;
+    hls::stream<ap_uint<512>> integration_result_1;
+
+    hls::stream<ap_axiu<32,1,1,1>> spot_finder_result_0;
+    hls::stream<ap_axiu<32,1,1,1>> spot_finder_result_1;
+    hls::stream<ap_uint<32>> spot_finder_conn_0;
+    hls::stream<ap_axiu<32,1,1,1>> spot_finder_result_2;
+    hls::stream<ap_uint<512>> spot_finder_result_3;
+
+    hls::stream<ap_uint<32>> spot_finder_mask_0;
+
+    hls::stream<ap_uint<256>> roi_calc_result_0;
+
+    hls::stream<ap_uint<UDP_METADATA_STREAM_WIDTH> > udp_metadata;
+    volatile ap_uint<1> idle_data_collection = 1;
+    ap_uint<1> load_to_hbm_idle;
+    ap_uint<1> save_to_hbm_idle;
+    ap_uint<1> integration_idle;
+    ap_uint<1> stream_conv_idle;
+    ap_uint<1> frame_summation_idle;
+
+    volatile bool done = false;
+    volatile bool udp_done = false; // done AND udp_idle
+    volatile bool sls_done = false; // done AND sls_idle
+
+    // Sent gratuitous ARP message
+    arp(arp1, impl_->dout_eth, mac_addr, ipv4_addr, 1, 1);
+
+    Logger logger_hls("HLS");
+
+    volatile rcv_state_t state = RCV_INIT;
+    // Start data collection
+    data_collection_fsm(data_0, data_1,
+                        addr0, addr1,
+                        impl_->run_data_collection,
+                        impl_->cancel_data_collection,
+                        idle_data_collection,
+                        cfg.mode,
+                        float2uint(cfg.energy_kev),
+                        cfg.nframes,
+                        cfg.nmodules,
+                        cfg.nstorage_cells,
+                        cfg.nsummation ,
+                        cfg.sqrtmult, state);
+
+    impl_->run_data_collection = 0;
+    data_collection_fsm(data_0, data_1,
+                        addr0, addr1,
+                        impl_->run_data_collection,
+                        impl_->cancel_data_collection,
+                        idle_data_collection,
+                        cfg.mode,
+                        float2uint(cfg.energy_kev),
+                        cfg.nframes,
+                        cfg.nmodules,
+                        cfg.nstorage_cells,
+                        cfg.nsummation,
+                        cfg.sqrtmult,
+                        state);
+
+    hls_cores.emplace_back([&] {
+        while (!udp_done) {
+            if (impl_->din_eth.empty() && impl_->din_frame_generator.empty())
+                std::this_thread::sleep_for(std::chrono::microseconds(10));
+            else
+                stream_merge(impl_->din_eth, impl_->din_frame_generator, network0, impl_->data_source);
+        }
+        logger_hls.Info("Stream_merge done");
+    });
+
+    hls_cores.emplace_back([&] {
+        while (!udp_done) {
+            if (network0.empty())
+                std::this_thread::sleep_for(std::chrono::microseconds(10));
+            else
+                ethernet(network0, ip1, arp1, mac_addr, eth_packets, clear_counters);
+        }
+        logger_hls.Info("ethernet done");
+    });
+
+    hls_cores.emplace_back([&] {
+        while (!udp_done) {
+            if (ip1.empty())
+                std::this_thread::sleep_for(std::chrono::microseconds(10));
+            else
+                ipv4(ip1, udp1, icmp1, ipv4_addr);
+        }
+        logger_hls.Info("ipv4 done");
+    });
+
+    hls_cores.emplace_back([&] {
+        ap_uint<1> udp_idle = 1;
+        while (!done || !udp_idle) {
+            if (udp1.empty())
+                std::this_thread::sleep_for(std::chrono::microseconds(10));
+            else
+                udp(udp1, udp2, udp_metadata, udp_packets, clear_counters, udp_idle);
+        }
+        udp_done = true;
+        logger_hls.Info("udp done");
+    });
+
+    hls_cores.emplace_back([&] {
+        ap_uint<1> sls_idle = 1;
+        while (!done || !sls_idle) {
+            if (udp2.empty())
+                std::this_thread::sleep_for(std::chrono::microseconds (10));
+            else
+                sls_detector(udp2, udp_metadata, data_0, addr0, sls_packets, udp_eth_err, udp_len_err, current_pulse_id,
+                             clear_counters, sls_idle);
+        }
+        sls_done = true;
+        logger_hls.Info("sls_detector done");
+    });
+
+    // 1. Parse incoming UDP packets
+    hls_cores.emplace_back([&] {
+        while ((state != RCV_WAIT_FOR_START) || (idle_data_collection.read() == 0) || (!data_0.empty()))  {
+            data_collection_fsm(data_0, data_1,
+                                addr0, addr1,
+                                impl_->run_data_collection,
+                                impl_->cancel_data_collection,
+                                idle_data_collection,
+                                cfg.mode,
+                                float2uint(cfg.energy_kev),
+                                cfg.nframes,
+                                cfg.nmodules,
+                                cfg.nstorage_cells,
+                                cfg.nsummation,
+                                cfg.sqrtmult,
+                                state);
+        }
+        done = true;
+
+        logger_hls.Info("data collection done");
+    });
+
+    // 2. Cache images in HBM
+    hls_cores.emplace_back([&] { save_to_hbm(addr1, axi_compl[0], hbm_handles,
+                                             impl_->datamover_out_hbm_0.GetCtrlStream(),
+                                             impl_->datamover_out_hbm_1.GetCtrlStream(),
+                                             save_to_hbm_idle,
+                                             impl_->hbm_if_size);});
+
+    hls_cores.emplace_back([&] { save_to_hbm_data(data_1,
+                                                  data_3,
+                                                  impl_->datamover_out_hbm_0.GetDataStream(),
+                                                  impl_->datamover_out_hbm_1.GetDataStream());});
+
+    hls_cores.emplace_back([&] {frame_summation_reorder_compl(data_3, data_4, axi_compl[0], axi_compl[1]);});
+
+    hls_cores.emplace_back([&] { load_from_hbm(data_4, data_5, axi_compl[1], axi_compl[2], hbm_handles,
+                                               impl_->datamover_in_hbm_0.GetDataStream(), impl_->datamover_in_hbm_1.GetDataStream(),
+                                               impl_->datamover_in_hbm_0.GetCtrlStream(), impl_->datamover_in_hbm_1.GetCtrlStream(),
+                                               load_to_hbm_idle, impl_->hbm_if_size);});
+
+    hls_cores.emplace_back([&] { pedestal(data_5, data_6, axi_compl[2], axi_compl[3],
+                                          impl_->hbm.data(),
+                                          impl_->hbm.data(),
+                                          impl_->hbm.data(),
+                                          impl_->hbm.data(),
+                                          impl_->hbm.data(),
+                                          impl_->hbm.data(),
+                                          impl_->hbm.data(),
+                                          impl_->hbm.data(),
+                                          impl_->hbm.data(),
+                                          impl_->hbm.data(),
+                                          impl_->hbm.data(),
+                                          impl_->hbm.data(),
+                                          impl_->hbm_if_size);});
+
+
+    // 3. Calculate histogram of ADU values
+    hls_cores.emplace_back([&] { adu_histo(data_6, data_7, adu_histo_result, axi_compl[3], axi_compl[4]);});
+
+    // 4. Mask missing pixels
+    hls_cores.emplace_back([&] { mask_missing(data_7, data_8, axi_compl[4], axi_compl[5]);});
+
+    // 5. Handle EIGER packets properly
+    hls_cores.emplace_back([&] { eiger_reorder(data_8, data_9, axi_compl[5], axi_compl[6]);});
+
+    // 6. Apply pedestal & gain corrections
+    hls_cores.emplace_back([&] { jf_conversion(data_9, stream_768_0,
+                                               axi_compl[6], axi_compl[7],
+                                               impl_->hbm.data(),
+                                               impl_->hbm.data(),
+                                               impl_->hbm.data(),
+                                               impl_->hbm.data(),
+                                               impl_->hbm.data(),
+                                               impl_->hbm.data(),
+                                               impl_->hbm.data(),
+                                               impl_->hbm.data(),
+                                               impl_->hbm.data(),
+                                               impl_->hbm.data(),
+                                               impl_->hbm.data(),
+                                               impl_->hbm.data(),
+                                               impl_->hbm_if_size); });
+
+    // 7. Frame summation
+    hls_cores.emplace_back([&] { frame_summation(stream_768_0, stream_768_1, axi_compl[7], axi_compl[8], frame_summation_idle);});
+
+    // 8. Integration of pixels
+    hls_cores.emplace_back([&] { integration(stream_768_1, stream_768_2, integration_result_0, axi_compl[8], axi_compl[9],
+                                             impl_->hbm.data(), impl_->hbm.data(), impl_->hbm.data(), impl_->hbm.data(), integration_idle, impl_->hbm_if_size);});
+    hls_cores.emplace_back([&] { axis_64_to_512(integration_result_0, integration_result_1);});
+
+    // 9. Spot finding
+    ap_uint<32> tmp_snr_threshold = float2uint(spot_finder_parameters.snr_threshold);
+    ap_uint<32> min_d = float2uint(spot_finder_parameters.min_d);
+    ap_uint<32> max_d = float2uint(spot_finder_parameters.max_d);
+    ap_int<32> tmp_count_threshold = spot_finder_parameters.count_threshold;
+    ap_uint<32> min_pix_per_spot = spot_finder_parameters.min_pix_per_spot;
+
+    hls_cores.emplace_back([&] {
+        spot_finder_mask(stream_768_2,
+                         stream_768_3,
+                         spot_finder_mask_0,
+                         axi_compl[9],
+                         axi_compl[10],
+                         impl_->hbm.data(),
+                         impl_->hbm.data(),
+                         min_d,
+                         max_d,
+                         impl_->hbm_if_size);
+        logger_hls.Info("spot_finder_mask done");
+    });
+
+    hls_cores.emplace_back([&] {
+        spot_finder(stream_768_3, spot_finder_mask_0, stream_768_4, spot_finder_result_0, tmp_count_threshold, tmp_snr_threshold);
+        logger_hls.Info("spot_finder done");
+    });
+
+    hls_cores.emplace_back([&] {
+        spot_finder_connectivity(spot_finder_result_0,
+                                 spot_finder_result_1,
+                                 spot_finder_conn_0);
+        logger_hls.Info("spot_finder_connectivity done");
+    });
+
+    hls_cores.emplace_back([&] {
+        spot_finder_merge(spot_finder_result_1,
+                          spot_finder_conn_0,
+                          spot_finder_result_2,
+                          min_pix_per_spot);
+        logger_hls.Info("spot_finder_merge done");
+    });
+
+    hls_cores.emplace_back([&] {
+        axis_32_to_512(spot_finder_result_2, spot_finder_result_3);
+        logger_hls.Info("axis_32_to_512 done");
+    });
+
+    // 10. ROI calculation
+    hls_cores.emplace_back([&] {
+        roi_calc(stream_768_4,
+                 stream_768_5,
+                 roi_calc_result_0,
+                 axi_compl[10],
+                 axi_compl[11],
+                 impl_->hbm.data(),
+                 impl_->hbm.data(),
+                 impl_->hbm_if_size);
+        logger_hls.Info("roi_calc done");
+    });
+
+    // 11. Pixel sq root
+    hls_cores.emplace_back([&] {
+        pixel_sqrt(stream_768_5,
+                   stream_768_6);
+        logger_hls.Info("pixel_sqrt done");
+    });
+
+    // 12. Reduce/extend 24-bit stream
+    hls_cores.emplace_back([&] { stream_24bit_conv(stream_768_6, data_12, stream_conv_idle);});
+
+    // 13. Prepare data to write to host memory
+    hls_cores.emplace_back([&] {
+        ap_uint<3> state;
+        host_writer(data_12, adu_histo_result, integration_result_1, spot_finder_result_3, roi_calc_result_0,
+                    axi_compl[11], impl_->datamover_out.GetDataStream(),
+                    impl_->datamover_out.GetCtrlStream(), impl_->work_request_stream, impl_->completion_stream,
+                    dma_address_table.data(), packets_processed, impl_->host_writer_idle, impl_->cancel_data_collection, state);
+        logger_hls.Info("host_writer done");
+    });
+
+    for (auto &i : hls_cores)
+        i.join();
+
+    if (frame_generator_future.valid())
+        frame_generator_future.get();
+
+    // reset static counter
+    arp(arp1, impl_->dout_eth, mac_addr, ipv4_addr, 0, 1);
+    try {
+        while (!impl_->din_eth.empty())
+            impl_->din_eth.read();
+
+        if (!addr1.empty())
+            throw std::runtime_error("Addr1 queue not empty");
+
+        if (!addr2.empty())
+            throw std::runtime_error("Addr2 queue not empty");
+
+        if (!addr3.empty())
+            throw std::runtime_error("Addr3 queue not empty");
+
+        if (!data_1.empty())
+            throw std::runtime_error("data_1 queue not empty");
+
+        if (!data_2.empty())
+            throw std::runtime_error("data_2 queue not empty");
+
+        if (!data_3.empty())
+            throw std::runtime_error("data_3 queue not empty");
+
+        if (!data_4.empty())
+            throw std::runtime_error("data_4 queue not empty");
+
+        if (!data_5.empty())
+            throw std::runtime_error("data_5 queue not empty");
+
+        if (!data_7.empty())
+            throw std::runtime_error("data_7 queue not empty");
+
+        if (!data_8.empty())
+            throw std::runtime_error("data_8 queue not empty");
+
+        if (!data_9.empty())
+            throw std::runtime_error("data_9 queue not empty");
+
+        if (!data_10.empty())
+            throw std::runtime_error("data_10 queue not empty");
+
+        if (!data_12.empty())
+            throw std::runtime_error("data_12 queue not empty");
+
+        if (!data_13.empty())
+            throw std::runtime_error("data_13 queue not empty");
+
+        for (auto &c: axi_compl) {
+            if (!c.empty())
+                throw std::runtime_error("Compl queue not empty");
+        }
+
+        if (!stream_768_0.empty())
+            throw std::runtime_error("stream_768_0 queue not empty");
+
+        if (!stream_768_1.empty())
+            throw std::runtime_error("stream_768_1 queue not empty");
+
+        if (!stream_768_2.empty())
+            throw std::runtime_error("stream_768_2 queue not empty");
+
+        if (!stream_768_3.empty())
+            throw std::runtime_error("stream_768_3 queue not empty");
+
+        if (!stream_768_4.empty())
+            throw std::runtime_error("stream_768_4 queue not empty");
+
+        if (!hbm_handles.empty())
+            throw std::runtime_error("Handles queue not empty");
+
+        if (!integration_result_0.empty())
+            throw std::runtime_error("Integration result queue not empty");
+
+        if (!integration_result_1.empty())
+            throw std::runtime_error("Integration result queue not empty");
+
+        if (!impl_->datamover_in.GetDataStream().empty())
+            throw std::runtime_error("Datamover queue is not empty");
+
+        while (!impl_->datamover_out.IsIdle())
+            std::this_thread::sleep_for(std::chrono::milliseconds(100));
+    } catch (const std::runtime_error &e) {
+        logger_hls.ErrorException(e);
+        throw e;
+    }
+    idle = true;
+}
+
+DataCollectionStatus HLSDevice::GetDataCollectionStatus() const {
+    DataCollectionStatus status{};
+
+    union {
+        uint64_t u64;
+        double d;
+    } pulse_id_conv;
+    pulse_id_conv.u64 = current_pulse_id;
+
+    status.ctrl_reg = ap_uint<1>(impl_->host_writer_idle) ? (1 << 4) : 0;
+    status.max_modules = MAX_MODULES_FPGA;
+    status.hbm_size_bytes = impl_->hbm_if_size;
+    status.run_counter = run_counter;
+    status.current_pulseid = pulse_id_conv.d;
+    status.packets_udp = udp_packets;
+    status.packets_eth = eth_packets;
+    status.packets_sls = sls_packets;
+
+    return status;
+}
+
+void HLSDevice::HW_LoadCalibration(const LoadCalibrationConfig &config) {
+    int ret = load_calibration(impl_->hbm.data(), impl_->hbm.data(),
+                               config,
+                               impl_->hbm_if_size,
+                               impl_->datamover_in.GetCtrlStream(),
+                               impl_->datamover_in.GetDataStream(),
+                               dma_address_table.data());
+    if (ret)
+        throw JFJochException(JFJochExceptionCategory::AcquisitionDeviceError,
+                              "Error in loading calibration " + std::to_string(ret));
+
+    if (!impl_->datamover_in.GetDataStream().empty())
+        throw std::runtime_error("Datamover queue is not empty");
+}
+
+void HLSDevice::HW_SetSpotFinderParameters(const SpotFinderParameters &params) {
+    spot_finder_parameters = params;
+}
+
+void HLSDevice::HW_SetDataSource(uint32_t val) {
+    impl_->data_source = val;
+}
+
+uint32_t HLSDevice::HW_GetDataSource() {
+    return impl_->data_source;
+}
+
+void HLSDevice::CreateXfelBunchIDPacket(double pulse_id, uint32_t event_code) {
+
+    union {
+        uint64_t u64;
+        double d;
+    } pulse_id_conv;
+    pulse_id_conv.d = pulse_id;
+
+    bunchid_raw_packet packet{};
+
+    packet.ether_type = htons(0x0800);
+    for (int i = 0; i < 6; i++)
+        packet.dest_mac[i] = 0xFF;
+
+    packet.ipv4_header_h = htons(0x4500); // Big endian in IP header!
+    packet.ipv4_header_total_length = htons(sizeof(bunchid_payload) + 8 + 20); // Big endian in IP header!
+    packet.ipv4_header_dest_ip = UINT32_MAX;
+
+    packet.ipv4_header_ttl_protocol = htons(0x0011);
+    packet.ipv4_header_checksum = checksum( (uint16_t *) &packet.ipv4_header_h, 20); // checksum is already in network order
+
+    packet.udp_length = htons(sizeof(bunchid_payload) + 8);
+    packet.payload.magicn[0] = BUNCHID_MAGICN;
+    packet.payload.magicn[1] = BUNCHID_MAGICN;
+    packet.payload.magicn[2] = BUNCHID_MAGICN;
+    packet.payload.bunchid_msb[0] = (pulse_id_conv.u64 >> 32) & UINT32_MAX;
+    packet.payload.bunchid_msb[1] = (pulse_id_conv.u64 >> 32) & UINT32_MAX;
+    packet.payload.bunchid_lsb[0] = pulse_id_conv.u64 & UINT32_MAX;
+    packet.payload.bunchid_lsb[1] = pulse_id_conv.u64 & UINT32_MAX;
+    packet.payload.montrig = event_code;
+
+    SendPacket((char *) &packet, sizeof(bunchid_raw_packet));
+}

fpga/hls/HLSDevice.h

@@ -0,0 +1,75 @@
+// Copyright (2019-2024) Paul Scherrer Institute
+
+#ifndef JFJOCH_HLSDEVICE_H
+#define JFJOCH_HLSDEVICE_H
+
+#include <future>
+
+#include "../../common/DiffractionExperiment.h"
+#include "../pcie_driver/jfjoch_fpga.h"
+
+class HLSDeviceImpl;
+
+class HLSDevice {
+    // Hide all HLS simulation structures here
+    std::unique_ptr<HLSDeviceImpl> impl_;
+
+    std::mutex completion_mutex;
+    uint32_t completion_count;
+
+    std::thread action_thread;
+    std::future<void> frame_generator_future;
+    std::vector<uint64_t> dma_address_table;
+    uint64_t eth_packets = 0;
+    uint64_t icmp_packets = 0;
+    uint64_t udp_packets = 0;
+    uint64_t sls_packets = 0;
+    uint32_t udp_len_err = 0;
+    uint32_t udp_eth_err = 0;
+
+    uint64_t current_pulse_id = 0;
+    uint32_t run_counter = 0;
+
+    DataCollectionConfig cfg;
+    SpotFinderParameters spot_finder_parameters;
+
+    const uint64_t mac_addr = 0xCCAA11223344;
+    const uint32_t ipv4_addr = 0x0132010A;
+    volatile bool idle;
+
+    void FrameGeneratorFuture(FrameGeneratorConfig config);
+    void HLSMainThread();
+public:
+    explicit HLSDevice(std::vector<DeviceOutput *> &device_buffer);
+    ~HLSDevice();
+    void SendPacket(char *buffer, int len, uint8_t user = 0);
+    void CreateJFPacket(const DiffractionExperiment& experiment, uint64_t frame_number, uint32_t eth_packet,
+                        uint32_t module_number, const uint16_t *data, int8_t adjust_axis = 0, uint8_t user = 0);
+    void CreateJFPackets(const DiffractionExperiment& experiment, uint64_t frame_number_0, uint64_t frames,
+                         uint32_t module_number, const uint16_t *data);
+    void CreateEIGERPacket(const DiffractionExperiment& experiment, uint64_t frame_number, uint32_t eth_packet,
+                           uint32_t module_number, uint32_t col, uint32_t row,
+                           const uint16_t *data);
+    void CreateFinalPacket(const DiffractionExperiment& experiment);
+    void CreateXfelBunchIDPacket(double bunchid, uint32_t event_code);
+    
+    DataCollectionStatus GetDataCollectionStatus() const ;
+    void Cancel() ;
+
+    void HW_ReadActionRegister(DataCollectionConfig *job);
+    void HW_WriteActionRegister(const DataCollectionConfig *job);
+
+    void FPGA_StartAction(const DiffractionExperiment &experiment);
+    void FPGA_EndAction();
+    bool HW_IsIdle() const;
+    bool HW_ReadMailbox(uint32_t *values);
+    bool HW_SendWorkRequest(uint32_t handle);
+    void HW_LoadCalibration(const LoadCalibrationConfig &config);
+    void HW_SetSpotFinderParameters(const SpotFinderParameters &params);
+    uint32_t HW_GetDataSource();
+    void HW_SetDataSource(uint32_t val);
+    void HW_RunInternalGenerator(const FrameGeneratorConfig &config) ;
+};
+
+
+#endif //JFJOCH_HLSDEVICE_H

fpga/hls/adu_histo_tb.cpp

@@ -36,9 +36,9 @@ int main() {
     for (int i = 0; i < nframes * RAW_MODULE_SIZE * sizeof(uint16_t) / 64 + 2; i++)
         output.read();
 
-    if (adu_result.size() != nframes * (ADU_HISTO_BIN_COUNT / 16))
+    if (adu_result.size() != nframes * (ADU_HISTO_BIN_COUNT / 16 + 1))
         ret = 1;
-    for (int i = 0; i < nframes * (ADU_HISTO_BIN_COUNT / 16); i++)
+    for (int i = 0; i < nframes * (ADU_HISTO_BIN_COUNT / 16 + 1); i++)
         adu_result.read();
 
     if (compl_out.size() != nframes + 1)

fpga/hls/data_collection_fsm.cpp

@@ -16,6 +16,7 @@ void data_collection_fsm(AXI_STREAM &eth_in,
                          ap_uint<5>  nmodules,
                          ap_uint<4>  nstorage_cells,
                          ap_uint<8> nsummation,
+                         ap_uint<8> sqrtmult,
                          volatile rcv_state_t  &data_collection_state) {
 #pragma HLS INTERFACE ap_ctrl_none port=return
 
@@ -33,6 +34,7 @@ void data_collection_fsm(AXI_STREAM &eth_in,
 #pragma HLS INTERFACE ap_none register port=nmodules
 #pragma HLS INTERFACE ap_none register port=nstorage_cells
 #pragma HLS INTERFACE ap_none register port=nsummation
+#pragma HLS INTERFACE ap_none register port=sqrtmult
 #pragma HLS INTERFACE ap_none register port=data_collection_state
 
 #pragma HLS PIPELINE II=1 style=flp
@@ -78,6 +80,7 @@ void data_collection_fsm(AXI_STREAM &eth_in,
             ACT_REG_NMODULES(packet_out.data) = nmodules;
             ACT_REG_NSTORAGE_CELLS(packet_out.data) = nstorage_cells + 1;
             ACT_REG_NSUMMATION(packet_out.data) = nsummation;
+            ACT_REG_SQRTMULT(packet_out.data) = sqrtmult;
 
             packet_out.user = 0;
             packet_out.last = 1;

fpga/hls/hls_jfjoch.h

@@ -10,17 +10,23 @@
 #include <cstring>
 #include <hls_burst_maxi.h>
 
+#ifndef __SYNTHESIS__
+#define ap_wait()
+#endif
+
 #ifndef JFJOCH_HLS_NOSYNTH
 #include <ap_axi_sdata.h>
 #include <hls_stream.h>
 #include <hls_math.h>
 #else
-#define ap_wait()
+
 #include "parallel_stream.h"
 
 #include <cmath>
 namespace hls {
     inline bool isfinite(float f) {return std::isfinite(f);}
+    // inline float sqrt(float f) {return std::sqrt(f); }
+    template <class T> float sqrt(const T &f) {return std::round(std::sqrt(f.to_float())); }
 }
 #endif
 
@@ -61,7 +67,8 @@ typedef hls::stream<packet_768_t> STREAM_768;
 #define ACT_REG_NFRAMES(x)                ((x)(95 ,  64)) // 32 bit
 #define ACT_REG_NMODULES(x)               ((x)(132, 128)) // 5 bit (0..31)
 #define ACT_REG_NSTORAGE_CELLS(x)         ((x)(148, 144)) // 5 bit
-#define ACT_REG_NSUMMATION(x)             ((x)(167, 160)) // 7 bit (0..255)
+#define ACT_REG_NSUMMATION(x)             ((x)(167, 160)) // 8 bit (0..255)
+#define ACT_REG_SQRTMULT(x)               ((x)(175, 168)) // 8 bit (0..255)
 
 struct axis_datamover_ctrl {
     ap_uint<40+64> data;
@@ -330,6 +337,7 @@ void data_collection_fsm(AXI_STREAM &eth_in,
                          ap_uint<5>  nmodules,
                          ap_uint<4>  nstorage_cells,
                          ap_uint<8> nsummation,
+                         ap_uint<8> sqrootmult,
                          volatile rcv_state_t &state);
 
 void host_writer(STREAM_512 &data_in,
@@ -444,4 +452,7 @@ void roi_calc(STREAM_768 &data_in,
               ap_uint<256> *d_hbm_p1,
               ap_uint<32> hbm_size_bytes);
 
+void pixel_sqrt(STREAM_768 &data_in,
+                STREAM_768 &data_out);
+
 #endif

fpga/hls/host_writer.cpp

@@ -75,6 +75,7 @@ void host_writer(STREAM_512 &data_in,
     state = 0;
 
     ap_uint<16> req_handle;
+    ap_uint<16> req_handle_send = HANDLE_START;
     ap_uint<64> req_host_offset;
 
      while (data_in.empty()) {
@@ -240,7 +241,9 @@ void host_writer(STREAM_512 &data_in,
             host_memory_out << packet_out;
 
         if (send_images) {
-            write_completion(m_axis_completion, req_handle, data_collection_id);
+            if (req_handle_send != HANDLE_START)
+                write_completion(m_axis_completion, req_handle_send, data_collection_id);
+            req_handle_send = req_handle;
             internal_packets_processed += cmpl.packet_count;
             packets_processed = internal_packets_processed;
         }
@@ -252,6 +255,21 @@ void host_writer(STREAM_512 &data_in,
 
     data_in >> packet;
 
+#ifdef JFJOCH_HLS_NOSYNTH
+    while (!host_memory_out.empty())
+        std::this_thread::sleep_for(std::chrono::milliseconds(1));
+#else
+    {
+        // wait for 5 ms (ensure that PCIe buffer is clean)
+#pragma HLS PROTOCOL
+        for (int i = 0; i < 5 * 200 * 1000; i++)
+                ap_wait();
+    }
+#endif
+
+    if (req_handle_send != HANDLE_START)
+        write_completion(m_axis_completion, req_handle_send, data_collection_id);
+
     write_completion(m_axis_completion, HANDLE_END, data_collection_id);
 
     idle = 1;

fpga/hls/pixel_sqrt.cpp

@@ -0,0 +1,47 @@
+// Copyright (2019-2023) Paul Scherrer Institute
+
+#include "hls_jfjoch.h"
+
+void pixel_sqrt(STREAM_768 &data_in,
+                STREAM_768 &data_out) {
+#pragma HLS INTERFACE register both axis port=data_in
+#pragma HLS INTERFACE register both axis port=data_out
+#pragma HLS INTERFACE ap_ctrl_none port=return
+
+    packet_768_t packet_in;
+    {
+#pragma HLS PROTOCOL fixed
+        data_in >> packet_in;
+        ap_wait();
+        data_out << packet_in;
+        ap_wait();
+    }
+
+    ap_uint<32> data_collection_mode = ACT_REG_MODE(packet_in.data);
+    ap_uint<1> apply_sqroot = ((data_collection_mode & MODE_SQROOT) ? 1 : 0);
+    ap_uint<9> sqrtmult = ACT_REG_SQRTMULT(packet_in.data) * ACT_REG_SQRTMULT(packet_in.data);
+
+    data_in >> packet_in;
+
+    while (!packet_in.user) {
+#pragma HLS PIPELINE II=1
+        if (apply_sqroot) {
+            ap_int<24> pixel[32];
+            unpack32(packet_in.data, pixel);
+            for (int i = 0; i < 32; i++) {
+                ap_uint<32> tmp = pixel[i] * sqrtmult;
+                if ((pixel[i] == INT24_MAX) || (pixel[i] == INT24_MIN)) {
+                    pixel[i] = pixel[i];
+                } else if (pixel[i] < 0)
+                    pixel[i] = 0;
+                else
+                    pixel[i] = hls::sqrt(tmp);
+            }
+            packet_in.data = pack32(pixel);
+        }
+
+        data_out << packet_in;
+        data_in >> packet_in;
+    }
+    data_out << packet_in;
+}

fpga/hls/pixel_sqrt_tb.cpp

@@ -0,0 +1,65 @@
+// Copyright (2019-2024) Paul Scherrer Institute
+
+#include "hls_jfjoch.h"
+
+
+int main() {
+    bool all_good = true;
+
+    std::vector<uint16_t> sqrtmult_values = {1,2,4,8,16};
+
+    for (int i = 0; i < sqrtmult_values.size(); i++) {
+        uint16_t sqrtmult = sqrtmult_values[i];
+
+        STREAM_768 input;
+        STREAM_768 output;
+
+        packet_768_t packet_in;
+        packet_in.user = 0;
+        packet_in.data = 0;
+
+        ACT_REG_SQRTMULT(packet_in.data) = sqrtmult;
+        ACT_REG_MODE(packet_in.data) = MODE_SQROOT;
+
+        input << packet_in;
+
+        ap_int<24> values_in[32], values_out[32];
+        for (int i = 0; i < 30; i++)
+            values_in[i] = i;
+
+        values_in[30] = INT24_MAX;
+        values_in[31] = INT24_MIN;
+
+
+        packet_in.data = pack32(values_in);
+        input << packet_in;
+
+        input << packet_768_t{.user = 1};
+
+
+        pixel_sqrt(input, output);
+
+        output.read();
+        packet_768_t packet_out = output.read();
+        output.read();
+
+        unpack32(packet_out.data, values_out);
+
+        for (int i = 0; i < 32; i++) {
+            double value_expected = std::round(std::sqrt(values_in[i].to_float()) * sqrtmult);
+            if (values_in[i] == INT24_MAX)
+                value_expected = INT24_MAX;
+            if (values_in[i] == INT24_MIN)
+                value_expected = INT24_MIN;
+
+            if (values_out[i] != value_expected) {
+                all_good = false;
+                std::cerr << "Value " << i << " expected: " << value_expected << "  calculated " << values_out[i] << " sqrtmult " << sqrtmult << std::endl;
+            }
+        }
+    }
+    if (all_good)
+        return 0;
+    else
+        return 1;
+}

fpga/hls/spot_finder_tb.cpp

@@ -45,9 +45,10 @@ int main() {
 	for (int i = 0; i < nframes * RAW_MODULE_SIZE * sizeof(uint16_t) / 64 + 2; i++)
 		output.read();
 
-	if (strong_pixel.size() != nframes * (RAW_MODULE_SIZE * sizeof(uint16_t) / 64 + 2) + 1)
+	if (strong_pixel.size() != nframes * (RAW_MODULE_SIZE * sizeof(uint16_t) / 64 + 16) + 1) {
 		ret = 1;
-	for (int i = 0; i < nframes * (RAW_MODULE_SIZE * sizeof(uint16_t) / 64 + 2) + 1; i++)
+	}
+	for (int i = 0; i < nframes * (RAW_MODULE_SIZE * sizeof(uint16_t) / 64 + 16) + 1; i++)
 		strong_pixel.read();
 
 	if (ret != 0) {

fpga/hls/stream_24bit_conv_tb.cpp

@@ -3,5 +3,66 @@
 #include "hls_jfjoch.h"
 
 int main() {
+    bool all_good = true;
 
+    STREAM_768 input;
+    STREAM_512 output;
+
+    packet_768_t packet_in;
+    packet_in.user = 0;
+    packet_in.data = 0;
+
+    ACT_REG_MODE(packet_in.data) = MODE_32BIT;
+
+    input << packet_in;
+
+    ap_int<24> values_in[32];
+    ap_int<32> values_out[32];
+
+    for (int i = 0; i < 30; i++)
+        values_in[i] = i;
+
+    values_in[30] = INT24_MAX;
+    values_in[31] = INT24_MIN;
+
+    packet_in.data = pack32(values_in);
+    input << packet_in;
+
+    input << packet_768_t{.user = 1};
+
+    volatile ap_uint<1> idle;
+
+    stream_24bit_conv(input, output, idle);
+
+    if (output.size() != 4) {
+        std::cerr << "Wrong output size" << std::endl;
+        return 1;
+    }
+
+    output.read();
+    packet_512_t packet_out_1 = output.read();
+    packet_512_t packet_out_2 = output.read();
+    output.read();
+
+    ap_uint<1024> packet_out = (packet_out_2.data, packet_out_1.data);
+
+    unpack32(packet_out, values_out);
+
+    for (int i = 0; i < 32; i++) {
+        int32_t value_expected = values_in[i];
+        if (values_in[i] == INT24_MAX)
+            value_expected = INT32_MAX;
+        if (values_in[i] == INT24_MIN)
+            value_expected = INT32_MIN;
+
+        if (values_out[i] != value_expected) {
+            all_good = false;
+            std::cerr << "Value " << i << " expected: " << value_expected << "  calculated " << values_out[i] << std::endl;
+        }
+    }
+
+    if (all_good)
+        return 0;
+    else
+        return 1;
 }

fpga/pcie_driver/dkms.conf

@@ -1,5 +1,5 @@
 PACKAGE_NAME=jfjoch
-PACKAGE_VERSION=1.0.0
+PACKAGE_VERSION=1.0.0-rc.12
 
 DEST_MODULE_LOCATION=/extra
 BUILT_MODULE_NAME=jfjoch

fpga/pcie_driver/jfjoch_drv.c

@@ -11,7 +11,7 @@
 MODULE_AUTHOR("Filip Leonarski; Paul Scherrer Institute");
 MODULE_DESCRIPTION("Jungfraujoch device module");
 MODULE_LICENSE("GPL");
-MODULE_VERSION("1.0.0");
+MODULE_VERSION("1.0.0-rc.12");
 
 #define XDMA_GEN4_x8        (0x9048)
 #define XDMA_GEN3_x16       (0x903F)

fpga/pcie_driver/jfjoch_drv.h

@@ -95,7 +95,7 @@
 #define ADDR_ONE_OVER_ENERGY     0x0214
 #define ADDR_NFRAMES             0x0218
 #define ADDR_NSTORAGE_CELLS      0x021C
-#define ADDR_DATA_SOURCE         0x0224
+#define ADDR_DATA_SOURCE         0x0300
 
 #define ADDR_SPOT_FINDER_THRESHOLD  0x0100
 #define ADDR_SPOT_FINDER_SNR        0x0104
@@ -177,11 +177,9 @@ struct jfjoch_drvdata {
     // however getting things from the queue happens in system call and can be executed in parallel
     // therefore this part is protected by mutex (it is assumed that waiting for interrupts can take seconds,
     // process has to be able to sleep while holding the lock => spinlock would not work)
-    // the mutex is also protecting work_compl_count variable
     struct mutex          work_compl_read_mutex;
     DECLARE_KFIFO(work_compl, u32, MAX_FPGA_BUFFER); // protected by work_compl_read_mutex
     wait_queue_head_t     work_compl_wait_queue; // used for read completion queue method to wait for interrupt, ISR has wake-up call
-    u32                   work_compl_count; // protected by work_compl_read_mutex
     atomic_t              active_handles;
 };
 
@@ -220,7 +218,6 @@ void jfjoch_set_ipv4_addr(struct jfjoch_drvdata *drvdata, const u32 *addr);
 void jfjoch_get_ipv4_addr(struct jfjoch_drvdata *drvdata, u32 *addr);
 int jfjoch_load_calibration(struct jfjoch_drvdata *drvdata, struct LoadCalibrationConfig *config);
 int jfjoch_run_frame_gen(struct jfjoch_drvdata *drvdata, struct FrameGeneratorConfig *config);
-u32 jfjoch_get_c2h_descriptors(struct jfjoch_drvdata *drvdata);
 void jfjoch_set_spot_finder_parameters(struct jfjoch_drvdata *drvdata, struct SpotFinderParameters *params);
 void jfjoch_get_spot_finder_parameters(struct jfjoch_drvdata *drvdata, struct SpotFinderParameters *params);
 u32 jfjoch_get_data_source(struct jfjoch_drvdata *drvdata);
@@ -236,7 +233,6 @@ void jfjoch_get_env_data(struct jfjoch_drvdata *drvdata, struct DeviceStatus *en
 
 void jfjoch_reset(struct jfjoch_drvdata *drvdata);
 
-void jfjoch_write_register(struct jfjoch_drvdata *drvdata, uint32_t addr, uint32_t val);
 uint32_t jfjoch_read_register(struct jfjoch_drvdata *drvdata, uint32_t addr);
 
 int jfjoch_register_sysfs(struct jfjoch_drvdata *drvdata);

fpga/pcie_driver/jfjoch_fpga.h

@@ -27,7 +27,7 @@ typedef __u64 uint64_t;
 #define GAIN_G2_MULTIPLIER (-1)
 
 #define JFJOCH_FPGA_MAGIC 0x52324158
-#define JFJOCH_FPGA_RELEASE 0x0056
+#define JFJOCH_FPGA_RELEASE 0x0057
 
 #define JFJOCH_FPGA_VARIANT_100G      0
 #define JFJOCH_FPGA_VARIANT_8x10G     1
@@ -35,6 +35,7 @@ typedef __u64 uint64_t;
 #define MODE_CONV                    0x0001L
 #define MODE_32BIT                   0x0002L
 #define MODE_UNSIGNED                0x0004L
+#define MODE_SQROOT                  0x0008L
 #define MODE_PEDESTAL_G0             0x0010L
 #define MODE_PEDESTAL_G1             0x0020L
 #define MODE_PEDESTAL_G2             0x0040L
@@ -75,7 +76,7 @@ typedef __u64 uint64_t;
 #define INT24_MIN (-8388608)
 
 // INT32_MAX is written explicitly, as the constant is not present in kernel
-#define MAX_FRAMES (2147483647/(MAX_MODULES_FPGA*DMA_DESCRIPTORS_PER_MODULE))
+#define MAX_FRAMES (2147483647)
 
 #pragma pack(push)
 #pragma pack(4)
@@ -106,6 +107,7 @@ struct DataCollectionConfig {
     uint32_t nframes; // Number of frames for data collection
     uint32_t nstorage_cells; // Number of storage cells minus one (0 = 1SC, 1 = 2SC, ..., 15 = 16SC)
     uint32_t nsummation; // Summation of frames minus one (0 = no summation, 1 = 2 frames, 2 = 3 frames, ..., 255 = 256 frames)
+    uint32_t sqrtmult; // Multiplication factor (minus one) of pixel value BEFORE taking square root function - should be square of the coeff used in other work
 };
 
 struct DataCollectionStatus {

fpga/pcie_driver/jfjoch_function.c

@@ -16,13 +16,8 @@ void jfjoch_start(struct jfjoch_drvdata *drvdata) {
     mutex_lock(&drvdata->work_compl_read_mutex);
     while(!kfifo_is_empty(&drvdata->work_compl))
         tmp2 = kfifo_get(&drvdata->work_compl, &tmp1);
-
-    // Reset work completion queue
-    drvdata->work_compl_count = 0;
-
     mutex_unlock(&drvdata->work_compl_read_mutex);
 
-
     // Set PCIe beats counters
     iowrite32((1 << 1), drvdata->bar0 + PCIE_OFFSET + (1<<12) + 0xC0);
     iowrite32((1 << 2), drvdata->bar0 + PCIE_OFFSET + (1<<12) + 0xC0);
@@ -69,28 +64,14 @@ int jfjoch_send_wr(struct jfjoch_drvdata *drvdata, u32 handle) {
 }
 
 int jfjoch_read_wc(struct jfjoch_drvdata *drvdata, u32 *output) {
-    struct device *const dev = &drvdata->pdev->dev;
-
-    int ret, tmp, i;
-    u32 curr_compl_count, handle;
-    u32 data_collection_id;
+    int ret, tmp;
 
     mutex_lock(&drvdata->work_compl_read_mutex);
 
     ret = wait_event_interruptible_timeout(drvdata->work_compl_wait_queue, !kfifo_is_empty(&drvdata->work_compl), HZ);
-    if (ret > 0) {
+    if (ret > 0)
         tmp = kfifo_get(&drvdata->work_compl, output);
-        handle = (*output) & 0xFFFF;
-        data_collection_id = (*output >> 16) & 0xFFFF;
 
-        if (handle == HANDLE_START) {
-            drvdata->work_compl_count = 0;
-            curr_compl_count = 0;
-        } else if ((handle != HANDLE_END) && (data_collection_id != DATA_COLLECTION_ID_PURGE)) {
-            curr_compl_count = drvdata->work_compl_count;
-            drvdata->work_compl_count++;
-        }
-    }
     mutex_unlock(&drvdata->work_compl_read_mutex);
 
     if (ret < 0)
@@ -98,27 +79,9 @@ int jfjoch_read_wc(struct jfjoch_drvdata *drvdata, u32 *output) {
     else if ((ret == 0) || (tmp == 0))
         return -EAGAIN;
 
-    // Guarantee that data are in host memory
-    if ((handle != HANDLE_START) && (handle != HANDLE_END)) {
-        u32 descriptors = ioread32(drvdata->bar0 + PCIE_OFFSET + (1 << 12) + 0x48);
-        i = 0;
-        while ((descriptors < curr_compl_count * DMA_DESCRIPTORS_PER_MODULE) && (i < 100)) {
-            udelay(10);
-            descriptors = ioread32(drvdata->bar0 + PCIE_OFFSET + (1 << 12) + 0x48);
-            i++;
-        }
-
-        if (descriptors < DMA_DESCRIPTORS_PER_MODULE * curr_compl_count) {
-            dev_err(dev, "Late completion count SW %u, HW %u HANDLE %x", curr_compl_count, descriptors, handle);
-            return -EIO;
-        } else if (i > 0) {
-            dev_warn(dev, "Late completion count SW %u, HW %u HANDLE %x delay %d", curr_compl_count, descriptors, handle, i);
-        }
-    }
     return 0;
 }
 
-
 int jfjoch_set_config(struct jfjoch_drvdata *drvdata, const struct DataCollectionConfig *config) {
     if (config->nframes > MAX_FRAMES)
         return -EINVAL;
@@ -409,10 +372,6 @@ int jfjoch_run_frame_gen(struct jfjoch_drvdata *drvdata, struct FrameGeneratorCo
     return 0;
 }
 
-u32 jfjoch_get_c2h_descriptors(struct jfjoch_drvdata *drvdata) {
-    return ioread32(drvdata->bar0 + PCIE_OFFSET + (1<<12) + 0x48);
-}
-
 void jfjoch_set_spot_finder_parameters(struct jfjoch_drvdata *drvdata, struct SpotFinderParameters *params) {
     memcpy_toio((drvdata->bar0) + ACTION_CONFIG_OFFSET + ADDR_SPOT_FINDER_THRESHOLD,
                 params,

fpga/pcie_driver/jfjoch_ioctl.c

@@ -154,11 +154,6 @@ long jfjoch_cdev_ioctl(struct file *file, unsigned int cmd, unsigned long arg) {
             if (copy_from_user(&frame_generator_config, (char *) arg, sizeof(struct FrameGeneratorConfig)) != 0)
                 return -EFAULT;
             return jfjoch_run_frame_gen(drvdata, &frame_generator_config);
-       case IOCTL_JFJOCH_C2H_DMA_DESC:
-            exchange[0] = jfjoch_get_c2h_descriptors(drvdata);
-            if (copy_to_user((char *) arg, exchange, sizeof(u32)) != 0)
-                return -EFAULT;
-            return 0;
         case IOCTL_JFJOCH_READ_REGISTER:
             // This is the most powerful option from security point of view and can only be performed by root
             if (!capable(CAP_SYS_ADMIN))

fpga/pcie_driver/jfjoch_ioctl.h

@@ -34,8 +34,7 @@
 #define IOCTL_JFJOCH_GET_IPV4        _IOR(IOCTL_JFJOCH_MAGIC, 18, uint32_t)
 // 19-21 are reserved
 #define IOCTL_JFJOCH_RUN_FRAME_GEN   _IOW(IOCTL_JFJOCH_MAGIC, 23, struct FrameGeneratorConfig)
-#define IOCTL_JFJOCH_C2H_DMA_DESC    _IOR(IOCTL_JFJOCH_MAGIC, 24, uint32_t)
-// 25 is reserved
+// 24-25 are reserved
 #define IOCTL_JFJOCH_READ_REGISTER   _IOWR(IOCTL_JFJOCH_MAGIC, 26, struct RegisterConfig )
 #define IOCTL_JFJOCH_SET_SPOTFIN_PAR _IOW(IOCTL_JFJOCH_MAGIC, 27, struct SpotFinderParameters)
 #define IOCTL_JFJOCH_SET_DATA_SOURCE _IOW(IOCTL_JFJOCH_MAGIC, 28, uint32_t)

fpga/pcie_driver/postinstall.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 # Taken from https://schneide.blog/2015/08/10/packaging-kernel-modulesdrivers-using-dkms/
 
-VERSION="1.0.0"
+VERSION=1.0.0-rc.12
 
 occurrences=`/usr/sbin/dkms status | grep jfjoch | grep ${VERSION} | wc -l`
 

fpga/pcie_driver/preuninstall.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 # Taken from https://schneide.blog/2015/08/10/packaging-kernel-modulesdrivers-using-dkms/
 
-VERSION="1.0.0"
+VERSION=1.0.0-rc.12
 
 /usr/sbin/dkms remove -m jfjoch -v ${VERSION} --all