Merge branch '2406-more-minor-stuff' into 'main'

v1.0.0_rc.11

See merge request jungfraujoch/nextgendcu!73

.gitlab-ci.yml

@@ -251,6 +251,7 @@ synthesis:vivado_pcie_100g:
         - fpga/pcie_driver/jfjoch_fpga.h
   tags:
     - vivado
+  retry: 1
   artifacts:
     paths:
       - "jfjoch_fpga_pcie_100g.mcs"
@@ -284,6 +285,7 @@ synthesis:vivado_pcie_8x10g:
       allow_failure: true
   tags:
     - vivado
+  retry: 1
   artifacts:
     paths:
       - "jfjoch_fpga_pcie_8x10g.mcs"

CMakeLists.txt

@@ -134,7 +134,12 @@ IF (NOT JFJOCH_WRITER_ONLY)
     INSTALL(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/fpga/pcie_driver/
             DESTINATION /usr/src/jfjoch-1.0.0
             COMPONENT driver-dkms
-            FILES_MATCHING PATTERN "*.c" PATTERN "*.h" PATTERN "Makefile" PATTERN "dkms.conf")
+            FILES_MATCHING PATTERN "dkms.conf")
+
+    INSTALL(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/fpga/pcie_driver/
+            DESTINATION /usr/src/jfjoch-1.0.0/src
+            COMPONENT driver-dkms
+            FILES_MATCHING PATTERN "*.c" PATTERN "*.h" PATTERN "Makefile")
 
     FILE(MAKE_DIRECTORY ${CMAKE_SOURCE_DIR}/frontend_ui/build/)
     INSTALL(DIRECTORY ${CMAKE_SOURCE_DIR}/frontend_ui/build/ DESTINATION share/jfjoch/frontend COMPONENT jfjoch )

VERSION

@@ -1 +1 @@
-1.0.0_rc.9
+1.0.0_rc.11

broker/JFJochBrokerHttp.cpp

@@ -5,6 +5,7 @@
 #include "JFJochBrokerHttp.h"
 #include "gen/model/Error_message.h"
 #include "../preview/JFJochTIFF.h"
+#include "../common/GitInfo.h"
 
 // From https://en.cppreference.com/w/cpp/string/byte/tolower
 inline std::string str_tolower(std::string s) {
@@ -339,6 +340,9 @@ inline DatasetSettings Convert(const org::openapitools::server::model::Dataset_s
 
     if (input.runNumberIsSet())
         ret.RunNumber(input.getRunNumber());
+    if (input.runNameIsSet())
+        ret.RunName(input.getRunName());
+
     ret.ExperimentGroup(input.getExperimentGroup());
 
     if (!input.fpgaOutputIsSet())
@@ -359,7 +363,8 @@ inline DatasetSettings Convert(const org::openapitools::server::model::Dataset_s
             throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Unknown output format");
     }
 
-    ret.Summation(input.getSummation());
+    if (input.imageTimeUsIsSet())
+        ret.ImageTime(std::chrono::microseconds(input.getImageTimeUs()));
     ret.BeamX_pxl(input.getBeamXPxl());
     ret.BeamY_pxl(input.getBeamYPxl());
     ret.DetectorDistance_mm(input.getDetectorDistanceMm());
@@ -470,8 +475,21 @@ void JFJochBrokerHttp::status_get(Pistache::Http::ResponseWriter &response) {
     ProcessOutput(Convert(state_machine.GetStatus()), response);
 }
 
-void JFJochBrokerHttp::wait_till_done_post(Pistache::Http::ResponseWriter &response) {
-    auto state = state_machine.WaitTillMeasurementDone(std::chrono::seconds(5));
+
+
+void JFJochBrokerHttp::wait_till_done_post(const std::optional<int32_t> &timeout,
+                                           Pistache::Http::ResponseWriter &response) {
+    JFJochState state;
+    if (!timeout)
+        state = state_machine.WaitTillMeasurementDone(std::chrono::minutes(1));
+    else if ((timeout.value() > 3600) || (timeout.value() < 0)) {
+        response.send(Pistache::Http::Code::Bad_Request);
+        return;
+    } else if (timeout.value() == 0)
+        state = state_machine.GetState();
+    else
+        state = state_machine.WaitTillMeasurementDone(std::chrono::seconds(timeout.value()));
+
     switch (state) {
         case JFJochState::Idle:
             response.send(Pistache::Http::Code::Ok);
@@ -828,3 +846,7 @@ void JFJochBrokerHttp::config_user_mask_tiff_put(const Pistache::Rest::Request &
     state_machine.SetUserPixelMask(request.body());
     response.send(Pistache::Http::Code::Ok);
 }
+
+void JFJochBrokerHttp::version_get(Pistache::Http::ResponseWriter &response) {
+    response.send(Pistache::Http::Code::Ok, jfjoch_version(), MIME(Text, Plain));
+}

broker/JFJochBrokerHttp.h

@@ -90,7 +90,7 @@ class JFJochBrokerHttp : public org::openapitools::server::api::DefaultApi {
 
     void status_get(Pistache::Http::ResponseWriter &response) override;
 
-    void wait_till_done_post(Pistache::Http::ResponseWriter &response) override;
+    void wait_till_done_post(const std::optional<int32_t> &timeoutMs, Pistache::Http::ResponseWriter &response) override;
     void trigger_post(Pistache::Http::ResponseWriter &response) override;
     void pedestal_post(Pistache::Http::ResponseWriter &response) override;
 
@@ -143,6 +143,9 @@ class JFJochBrokerHttp : public org::openapitools::server::api::DefaultApi {
         to_json(j, output);
         response.send(Pistache::Http::Code::Ok, j.dump(), MIME(Application, Json));
     }
+
+    void version_get(Pistache::Http::ResponseWriter &response) override;
+
 public:
     JFJochBrokerHttp(const DiffractionExperiment& experiment, std::shared_ptr<Pistache::Rest::Router> &rtr);
     void AddDetectorSetup(const DetectorSetup &setup);

broker/JFJochBrokerParser.cpp

@@ -2,6 +2,8 @@
 
 #include "JFJochBrokerParser.h"
 #include "../common/NetworkAddressConvert.h"
+#include "../frame_serialize/ZMQStream2Pusher.h"
+#include "../frame_serialize/DumpCBORToFilePusher.h"
 
 inline bool CHECK_ARRAY(const nlohmann::json &j, const std::string& tag) {
     if (j.contains(tag)) {
@@ -183,6 +185,7 @@ inline int64_t TimeToUs(const std::string &unit) {
 
 inline std::chrono::microseconds GET_TIME(const nlohmann::json &j, const std::string& tag) {
     if (j.contains(tag)) {
+        // If no units provided for time, this is always microsecond
         if (j[tag].is_number())
             return std::chrono::microseconds (std::lround(j[tag].get<double>() * 1000.0 * 1000.0));
         else if (j[tag].is_string()) {
@@ -283,7 +286,8 @@ DetectorSetup ParseDetectorSetup(const nlohmann::json &j) {
     setup.UDPInterfaceCount(GET_I64(j, "udp_interface_count", 2))
             .SensorThickness_um(GET_FLOAT(j, "sensor_thickness_um", 320.0f))
             .PixelSize_um(GET_FLOAT(j, "pixel_size_um", 75.0f))
-            .SensorMaterial(GET_STR(j, "sensor_material", "Si"));
+            .SensorMaterial(GET_STR(j, "sensor_material", "Si"))
+            .SerialNumber(GET_STR(j, "serial_number",""));
 
     if (j.contains("tx_delay"))
         setup.TxDelay(GET_I64_ARR(j, "tx_delay"));
@@ -299,6 +303,35 @@ void ParseDetectorSetup(const nlohmann::json &j, const std::string& tag, JFJochB
         throw JFJochException(JFJochExceptionCategory::JSON, "Detector setup not found");
 }
 
+void ParseImagePusher(const nlohmann::json &input, std::unique_ptr<ImagePusher> &image_pusher) {
+    std::string pusher_type = ParseString(input, "stream_type", "zmq");
+    if (pusher_type == "zmq") {
+        int32_t zmq_send_watermark = ParseInt32(input, "zmq_send_watermark", 100);
+        int32_t zmq_send_buffer_size = ParseInt32(input, "zmq_send_buffer_size", -1);
+
+        auto tmp = std::make_unique<ZMQStream2Pusher>(ParseStringArray(input, "zmq_image_addr"),
+                                                      zmq_send_watermark,
+                                                      zmq_send_buffer_size);
+
+        std::string preview_addr = ParseString(input, "zmq_preview_addr", "");
+        if (!preview_addr.empty())
+            tmp->PreviewSocket(preview_addr);
+
+        if (input.contains("zmq_preview_period"))
+            tmp->PreviewCounterPeriod(GET_TIME(input, "zmq_preview_period"));
+
+        std::string writer_notification_addr = ParseString(input, "zmq_writer_notification_addr", "");
+        if (!writer_notification_addr.empty())
+            tmp->WriterNotificationSocket(writer_notification_addr);
+
+        image_pusher = std::move(tmp);
+    } else if (pusher_type == "dump_cbor") {
+        image_pusher = std::make_unique<DumpCBORToFilePusher>();
+    } else
+        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
+                              "stream_type allowed: zmq (default), dump_cbor");
+}
+
 void ParseFacilityConfiguration(const nlohmann::json &input, const std::string& tag, DiffractionExperiment &experiment) {
     if (CHECK_OBJECT(input, tag)) {
         auto j = input[tag];
@@ -327,12 +360,11 @@ void ParseFacilityConfiguration(const nlohmann::json &input, const std::string&
             experiment.PedestalG1Frames(GET_I64(j, "pedestal_g1_frames"));
         if (j.contains("pedestal_g2_frames"))
             experiment.PedestalG2Frames(GET_I64(j, "pedestal_g2_frames"));
-        if (j.contains("detector_trigger_delay_us"))
-            experiment.DetectorDelay(GET_TIME(j, "detector_trigger_delay_us"));
+        if (j.contains("detector_trigger_delay"))
+            experiment.DetectorDelay(GET_TIME(j, "detector_trigger_delay"));
+
+        experiment.FrameTime(GET_TIME(j, "frame_time"), GET_TIME(j, "count_time"));
 
-        experiment.FrameTime(GET_TIME(j, "frame_time_us"), GET_TIME(j, "count_time_us"));
-        if (j.contains("preview_period_us"))
-            experiment.PreviewPeriod(GET_TIME(j, "preview_period_us"));
         experiment.UseInternalPacketGenerator(GET_BOOL(j, "internal_frame_generator", false));
         if (experiment.IsUsingInternalPacketGen())
             experiment.ConversionOnFPGA(false);

broker/JFJochBrokerParser.h

@@ -14,6 +14,7 @@ DetectorGeometry ParseDetectorGeometry(const nlohmann::json &j);
 DetectorSetup ParseDetectorSetup(const nlohmann::json &j);
 void ParseDetectorSetup(const nlohmann::json &j, const std::string& tag, JFJochBrokerHttp& broker);
 void ParseFacilityConfiguration(const nlohmann::json &j, const std::string& tag, DiffractionExperiment &experiment);
+void ParseImagePusher(const nlohmann::json &j, std::unique_ptr<ImagePusher> &image_pusher);
 
 void ParseAcquisitionDeviceGroup(const nlohmann::json &input, const std::string& tag, AcquisitionDeviceGroup &aq_devices);
 std::vector<std::string> ParseStringArray(const nlohmann::json &input, const std::string& tag);

broker/JFJochStateMachine.cpp

@@ -340,12 +340,6 @@ void JFJochStateMachine::Start(const DatasetSettings& settings) {
     }
 }
 
-void JFJochStateMachine::WaitTillMeasurementDone() {
-std::unique_lock<std::mutex> ul(m);
-
-c.wait(ul, [&] { return !IsRunning(); });
-}
-
 void JFJochStateMachine::MeasurementThread() {
     try {
         auto tmp_output = services.Stop();
@@ -659,6 +653,14 @@ bool JFJochStateMachine::IsRunning() const {
     }
 }
 
+JFJochState JFJochStateMachine::WaitTillMeasurementDone() {
+    std::unique_lock<std::mutex> ul(m);
+
+    c.wait(ul, [&] { return !IsRunning(); });
+
+    return state;
+}
+
 JFJochState JFJochStateMachine::WaitTillMeasurementDone(std::chrono::milliseconds timeout) {
     std::unique_lock<std::mutex> ul(m);
 

broker/JFJochStateMachine.h

@@ -141,7 +141,7 @@ public:
     void Pedestal();
     void Deactivate();
     void Start(const DatasetSettings& settings);
-    void WaitTillMeasurementDone();
+    JFJochState WaitTillMeasurementDone();
     JFJochState WaitTillMeasurementDone(std::chrono::milliseconds timeout);
     void Trigger();
 

broker/gen/api/DefaultApi.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -79,6 +79,7 @@ void DefaultApi::setupRoutes() {
     Routes::Get(*router, base + "/statistics/data_collection", Routes::bind(&DefaultApi::statistics_data_collection_get_handler, this));
     Routes::Get(*router, base + "/status", Routes::bind(&DefaultApi::status_get_handler, this));
     Routes::Post(*router, base + "/trigger", Routes::bind(&DefaultApi::trigger_post_handler, this));
+    Routes::Get(*router, base + "/version", Routes::bind(&DefaultApi::version_get_handler, this));
     Routes::Post(*router, base + "/wait_till_done", Routes::bind(&DefaultApi::wait_till_done_post_handler, this));
     Routes::Get(*router, base + "/xfel/event_code", Routes::bind(&DefaultApi::xfel_event_code_get_handler, this));
     Routes::Get(*router, base + "/xfel/pulse_id", Routes::bind(&DefaultApi::xfel_pulse_id_get_handler, this));
@@ -1254,12 +1255,42 @@ void DefaultApi::trigger_post_handler(const Pistache::Rest::Request &, Pistache:
     }
 
 }
-void DefaultApi::wait_till_done_post_handler(const Pistache::Rest::Request &, Pistache::Http::ResponseWriter response) {
+void DefaultApi::version_get_handler(const Pistache::Rest::Request &, Pistache::Http::ResponseWriter response) {
     try {
 
 
     try {
-        this->wait_till_done_post(response);
+        this->version_get(response);
+    } catch (Pistache::Http::HttpError &e) {
+        response.send(static_cast<Pistache::Http::Code>(e.code()), e.what());
+        return;
+    } catch (std::exception &e) {
+        const std::pair<Pistache::Http::Code, std::string> errorInfo = this->handleOperationException(e);
+        response.send(errorInfo.first, errorInfo.second);
+        return;
+    }
+
+    } catch (std::exception &e) {
+        response.send(Pistache::Http::Code::Internal_Server_Error, e.what());
+    }
+
+}
+void DefaultApi::wait_till_done_post_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response) {
+    try {
+
+
+    // Getting the query params
+    auto timeoutQuery = request.query().get("timeout");
+    std::optional<int32_t> timeout;
+    if(timeoutQuery.has_value()){
+        int32_t valueQuery_instance;
+        if(fromStringValue(timeoutQuery.value(), valueQuery_instance)){
+            timeout = valueQuery_instance;
+        }
+    }
+    
+    try {
+        this->wait_till_done_post(timeout, response);
     } catch (Pistache::Http::HttpError &e) {
         response.send(static_cast<Pistache::Http::Code>(e.code()), e.what());
         return;

broker/gen/api/DefaultApi.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -103,6 +103,7 @@ private:
     void statistics_data_collection_get_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
     void status_get_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
     void trigger_post_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
+    void version_get_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
     void wait_till_done_post_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
     void xfel_event_code_get_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
     void xfel_pulse_id_get_handler(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
@@ -468,12 +469,20 @@ private:
     /// </remarks>
     virtual void trigger_post(Pistache::Http::ResponseWriter &response) = 0;
     /// <summary>
+    /// 
+    /// </summary>
+    /// <remarks>
+    /// 
+    /// </remarks>
+    virtual void version_get(Pistache::Http::ResponseWriter &response) = 0;
+    /// <summary>
     /// Wait for acquisition done
     /// </summary>
     /// <remarks>
-    /// Block execution of external script till initialization, data collection or pedestal is finished. Running this command does not affect (cancel) running data collection, it is only to ensure synchronous execution of other software.          To not block web server for a long period of time, the procedure is provided with a timeout of 5 seconds. 
+    /// Block execution of external script till initialization, data collection or pedestal is finished. Running this command does not affect (cancel) running data collection, it is only to ensure synchronous execution of other software.          To not block web server for a indefinite period of time, the procedure is provided with a timeout. Extending timeout is possible, but requires to ensure safety that client will not close the connection and retry the connection. 
     /// </remarks>
-    virtual void wait_till_done_post(Pistache::Http::ResponseWriter &response) = 0;
+    /// <param name="timeout">Timeout in seconds (0 &#x3D;&#x3D; immediate response) (optional, default to 60)</param>
+    virtual void wait_till_done_post(const std::optional<int32_t> &timeout, Pistache::Http::ResponseWriter &response) = 0;
     /// <summary>
     /// Return XFEL event codes for the current data acquisition
     /// </summary>

broker/gen/model/Broker_status.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Broker_status.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Calibration_statistics_inner.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Calibration_statistics_inner.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Dataset_settings.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -25,8 +25,8 @@ Dataset_settings::Dataset_settings()
     m_Images_per_triggerIsSet = false;
     m_Ntrigger = 1L;
     m_NtriggerIsSet = false;
-    m_Summation = 1L;
-    m_SummationIsSet = false;
+    m_Image_time_us = 0L;
+    m_Image_time_usIsSet = false;
     m_Beam_x_pxl = 0.0f;
     m_Beam_y_pxl = 0.0f;
     m_Detector_distance_mm = 0.0f;
@@ -38,6 +38,7 @@ Dataset_settings::Dataset_settings()
     m_Space_group_number = 0L;
     m_Space_group_numberIsSet = false;
     m_Sample_name = "";
+    m_Sample_nameIsSet = false;
     m_Fpga_output = "auto";
     m_Fpga_outputIsSet = false;
     m_Compression = "bslz4";
@@ -47,9 +48,7 @@ Dataset_settings::Dataset_settings()
     m_Transmission = 0.0f;
     m_TransmissionIsSet = false;
     m_GoniometerIsSet = false;
-    m_Header_appendix = "";
     m_Header_appendixIsSet = false;
-    m_Image_appendix = "";
     m_Image_appendixIsSet = false;
     m_Energy_multiplier = 1.0f;
     m_Energy_multiplierIsSet = false;
@@ -57,6 +56,8 @@ Dataset_settings::Dataset_settings()
     m_Data_reduction_factor_serialmxIsSet = false;
     m_Run_number = 0L;
     m_Run_numberIsSet = false;
+    m_Run_name = "";
+    m_Run_nameIsSet = false;
     m_Experiment_group = "";
     m_Experiment_groupIsSet = false;
     m_Unit_cellIsSet = false;
@@ -111,21 +112,16 @@ bool Dataset_settings::validate(std::stringstream& msg, const std::string& pathP
 
     }
          
-    if (summationIsSet())
+    if (imageTimeUsIsSet())
     {
-        const int64_t& value = m_Summation;
-        const std::string currentValuePath = _pathPrefix + ".summation";
+        const int64_t& value = m_Image_time_us;
+        const std::string currentValuePath = _pathPrefix + ".imageTimeUs";
                 
         
-        if (value < 1ll)
+        if (value < 0ll)
         {
             success = false;
-            msg << currentValuePath << ": must be greater than or equal to 1;";
-        }
-        if (value > 256ll)
-        {
-            success = false;
-            msg << currentValuePath << ": must be less than or equal to 256;";
+            msg << currentValuePath << ": must be greater than or equal to 0;";
         }
 
     }
@@ -261,7 +257,7 @@ bool Dataset_settings::validate(std::stringstream& msg, const std::string& pathP
         }
 
     }
-            
+                
     return success;
 }
 
@@ -277,7 +273,7 @@ bool Dataset_settings::operator==(const Dataset_settings& rhs) const
     ((!ntriggerIsSet() && !rhs.ntriggerIsSet()) || (ntriggerIsSet() && rhs.ntriggerIsSet() && getNtrigger() == rhs.getNtrigger())) &&
     
     
-    ((!summationIsSet() && !rhs.summationIsSet()) || (summationIsSet() && rhs.summationIsSet() && getSummation() == rhs.getSummation())) &&
+    ((!imageTimeUsIsSet() && !rhs.imageTimeUsIsSet()) || (imageTimeUsIsSet() && rhs.imageTimeUsIsSet() && getImageTimeUs() == rhs.getImageTimeUs())) &&
     
     (getBeamXPxl() == rhs.getBeamXPxl())
      &&
@@ -300,8 +296,8 @@ bool Dataset_settings::operator==(const Dataset_settings& rhs) const
     
     ((!spaceGroupNumberIsSet() && !rhs.spaceGroupNumberIsSet()) || (spaceGroupNumberIsSet() && rhs.spaceGroupNumberIsSet() && getSpaceGroupNumber() == rhs.getSpaceGroupNumber())) &&
     
-    (getSampleName() == rhs.getSampleName())
-     &&
+    
+    ((!sampleNameIsSet() && !rhs.sampleNameIsSet()) || (sampleNameIsSet() && rhs.sampleNameIsSet() && getSampleName() == rhs.getSampleName())) &&
     
     
     ((!fpgaOutputIsSet() && !rhs.fpgaOutputIsSet()) || (fpgaOutputIsSet() && rhs.fpgaOutputIsSet() && getFpgaOutput() == rhs.getFpgaOutput())) &&
@@ -334,6 +330,9 @@ bool Dataset_settings::operator==(const Dataset_settings& rhs) const
     ((!runNumberIsSet() && !rhs.runNumberIsSet()) || (runNumberIsSet() && rhs.runNumberIsSet() && getRunNumber() == rhs.getRunNumber())) &&
     
     
+    ((!runNameIsSet() && !rhs.runNameIsSet()) || (runNameIsSet() && rhs.runNameIsSet() && getRunName() == rhs.getRunName())) &&
+    
+    
     ((!experimentGroupIsSet() && !rhs.experimentGroupIsSet()) || (experimentGroupIsSet() && rhs.experimentGroupIsSet() && getExperimentGroup() == rhs.getExperimentGroup())) &&
     
     
@@ -354,8 +353,8 @@ void to_json(nlohmann::json& j, const Dataset_settings& o)
         j["images_per_trigger"] = o.m_Images_per_trigger;
     if(o.ntriggerIsSet())
         j["ntrigger"] = o.m_Ntrigger;
-    if(o.summationIsSet())
-        j["summation"] = o.m_Summation;
+    if(o.imageTimeUsIsSet())
+        j["image_time_us"] = o.m_Image_time_us;
     j["beam_x_pxl"] = o.m_Beam_x_pxl;
     j["beam_y_pxl"] = o.m_Beam_y_pxl;
     j["detector_distance_mm"] = o.m_Detector_distance_mm;
@@ -366,7 +365,8 @@ void to_json(nlohmann::json& j, const Dataset_settings& o)
         j["images_per_file"] = o.m_Images_per_file;
     if(o.spaceGroupNumberIsSet())
         j["space_group_number"] = o.m_Space_group_number;
-    j["sample_name"] = o.m_Sample_name;
+    if(o.sampleNameIsSet())
+        j["sample_name"] = o.m_Sample_name;
     if(o.fpgaOutputIsSet())
         j["fpga_output"] = o.m_Fpga_output;
     if(o.compressionIsSet())
@@ -387,6 +387,8 @@ void to_json(nlohmann::json& j, const Dataset_settings& o)
         j["data_reduction_factor_serialmx"] = o.m_Data_reduction_factor_serialmx;
     if(o.runNumberIsSet())
         j["run_number"] = o.m_Run_number;
+    if(o.runNameIsSet())
+        j["run_name"] = o.m_Run_name;
     if(o.experimentGroupIsSet())
         j["experiment_group"] = o.m_Experiment_group;
     if(o.unitCellIsSet())
@@ -406,10 +408,10 @@ void from_json(const nlohmann::json& j, Dataset_settings& o)
         j.at("ntrigger").get_to(o.m_Ntrigger);
         o.m_NtriggerIsSet = true;
     } 
-    if(j.find("summation") != j.end())
+    if(j.find("image_time_us") != j.end())
     {
-        j.at("summation").get_to(o.m_Summation);
-        o.m_SummationIsSet = true;
+        j.at("image_time_us").get_to(o.m_Image_time_us);
+        o.m_Image_time_usIsSet = true;
     } 
     j.at("beam_x_pxl").get_to(o.m_Beam_x_pxl);
     j.at("beam_y_pxl").get_to(o.m_Beam_y_pxl);
@@ -430,7 +432,11 @@ void from_json(const nlohmann::json& j, Dataset_settings& o)
         j.at("space_group_number").get_to(o.m_Space_group_number);
         o.m_Space_group_numberIsSet = true;
     } 
-    j.at("sample_name").get_to(o.m_Sample_name);
+    if(j.find("sample_name") != j.end())
+    {
+        j.at("sample_name").get_to(o.m_Sample_name);
+        o.m_Sample_nameIsSet = true;
+    } 
     if(j.find("fpga_output") != j.end())
     {
         j.at("fpga_output").get_to(o.m_Fpga_output);
@@ -481,6 +487,11 @@ void from_json(const nlohmann::json& j, Dataset_settings& o)
         j.at("run_number").get_to(o.m_Run_number);
         o.m_Run_numberIsSet = true;
     } 
+    if(j.find("run_name") != j.end())
+    {
+        j.at("run_name").get_to(o.m_Run_name);
+        o.m_Run_nameIsSet = true;
+    } 
     if(j.find("experiment_group") != j.end())
     {
         j.at("experiment_group").get_to(o.m_Experiment_group);
@@ -528,22 +539,22 @@ void Dataset_settings::unsetNtrigger()
 {
     m_NtriggerIsSet = false;
 }
-int64_t Dataset_settings::getSummation() const
+int64_t Dataset_settings::getImageTimeUs() const
 {
-    return m_Summation;
+    return m_Image_time_us;
 }
-void Dataset_settings::setSummation(int64_t const value)
+void Dataset_settings::setImageTimeUs(int64_t const value)
 {
-    m_Summation = value;
-    m_SummationIsSet = true;
+    m_Image_time_us = value;
+    m_Image_time_usIsSet = true;
 }
-bool Dataset_settings::summationIsSet() const
+bool Dataset_settings::imageTimeUsIsSet() const
 {
-    return m_SummationIsSet;
+    return m_Image_time_usIsSet;
 }
-void Dataset_settings::unsetSummation()
+void Dataset_settings::unsetImage_time_us()
 {
-    m_SummationIsSet = false;
+    m_Image_time_usIsSet = false;
 }
 float Dataset_settings::getBeamXPxl() const
 {
@@ -635,6 +646,15 @@ std::string Dataset_settings::getSampleName() const
 void Dataset_settings::setSampleName(std::string const& value)
 {
     m_Sample_name = value;
+    m_Sample_nameIsSet = true;
+}
+bool Dataset_settings::sampleNameIsSet() const
+{
+    return m_Sample_nameIsSet;
+}
+void Dataset_settings::unsetSample_name()
+{
+    m_Sample_nameIsSet = false;
 }
 std::string Dataset_settings::getFpgaOutput() const
 {
@@ -721,11 +741,11 @@ void Dataset_settings::unsetGoniometer()
 {
     m_GoniometerIsSet = false;
 }
-std::string Dataset_settings::getHeaderAppendix() const
+nlohmann::json Dataset_settings::getHeaderAppendix() const
 {
     return m_Header_appendix;
 }
-void Dataset_settings::setHeaderAppendix(std::string const& value)
+void Dataset_settings::setHeaderAppendix(nlohmann::json const& value)
 {
     m_Header_appendix = value;
     m_Header_appendixIsSet = true;
@@ -738,11 +758,11 @@ void Dataset_settings::unsetHeader_appendix()
 {
     m_Header_appendixIsSet = false;
 }
-std::string Dataset_settings::getImageAppendix() const
+nlohmann::json Dataset_settings::getImageAppendix() const
 {
     return m_Image_appendix;
 }
-void Dataset_settings::setImageAppendix(std::string const& value)
+void Dataset_settings::setImageAppendix(nlohmann::json const& value)
 {
     m_Image_appendix = value;
     m_Image_appendixIsSet = true;
@@ -806,6 +826,23 @@ void Dataset_settings::unsetRun_number()
 {
     m_Run_numberIsSet = false;
 }
+std::string Dataset_settings::getRunName() const
+{
+    return m_Run_name;
+}
+void Dataset_settings::setRunName(std::string const& value)
+{
+    m_Run_name = value;
+    m_Run_nameIsSet = true;
+}
+bool Dataset_settings::runNameIsSet() const
+{
+    return m_Run_nameIsSet;
+}
+void Dataset_settings::unsetRun_name()
+{
+    m_Run_nameIsSet = false;
+}
 std::string Dataset_settings::getExperimentGroup() const
 {
     return m_Experiment_group;

broker/gen/model/Dataset_settings.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech
@@ -19,6 +19,7 @@
 #define Dataset_settings_H_
 
 
+#include <nlohmann/json.hpp>
 #include "Rotation_axis.h"
 #include <string>
 #include "Dataset_settings_unit_cell.h"
@@ -75,12 +76,12 @@ public:
     bool ntriggerIsSet() const;
     void unsetNtrigger();
     /// <summary>
-    /// FPGA frame summation. For summation above two 32-bit pixel format will be used, unless explicitly specified. Frame summation factor applies only to conversion mode (assumed as 1 for raw data). In XFEL mode: summation happens for frames collected with multiple triggers. Ignored for storage cells (assumed as 1). 
+    /// Image time.  If not provided (or zero value) the frame time is assumed as default. Image time must be multiple of frame time; max value is 256 * frame_time.             In XFEL mode: summation happens for frames collected with multiple triggers. Ignored for storage cells and if raw data are saved. 
     /// </summary>
-    int64_t getSummation() const;
-    void setSummation(int64_t const value);
-    bool summationIsSet() const;
-    void unsetSummation();
+    int64_t getImageTimeUs() const;
+    void setImageTimeUs(int64_t const value);
+    bool imageTimeUsIsSet() const;
+    void unsetImage_time_us();
     /// <summary>
     /// /entry/detector/beam_center_x in NXmx Beam center in X direction [pixels] 
     /// </summary>
@@ -127,6 +128,8 @@ public:
     /// </summary>
     std::string getSampleName() const;
     void setSampleName(std::string const& value);
+    bool sampleNameIsSet() const;
+    void unsetSample_name();
     /// <summary>
     /// FPGA output data type
     /// </summary>
@@ -163,17 +166,17 @@ public:
     bool goniometerIsSet() const;
     void unsetGoniometer();
     /// <summary>
-    /// Header appendix, added as user_data to start message
+    /// Header appendix, added as user_data/user to start message (can be any valid JSON)
     /// </summary>
-    std::string getHeaderAppendix() const;
-    void setHeaderAppendix(std::string const& value);
+    nlohmann::json getHeaderAppendix() const;
+    void setHeaderAppendix(nlohmann::json const& value);
     bool headerAppendixIsSet() const;
     void unsetHeader_appendix();
     /// <summary>
-    /// Image appendix, added as user_data to image message
+    /// Image appendix, added as user_data to image message (can be any valid JSON)
     /// </summary>
-    std::string getImageAppendix() const;
-    void setImageAppendix(std::string const& value);
+    nlohmann::json getImageAppendix() const;
+    void setImageAppendix(nlohmann::json const& value);
     bool imageAppendixIsSet() const;
     void unsetImage_appendix();
     /// <summary>
@@ -198,6 +201,13 @@ public:
     bool runNumberIsSet() const;
     void unsetRun_number();
     /// <summary>
+    /// Unique ID of run. Transferred over CBOR stream as \&quot;unique series ID\&quot;, though not saved in HDF5 file. It is highly recommended to keep this name unique for each data collection during experimental series. If not provided, the name will be automatically generated as number + colon + file_prefix. 
+    /// </summary>
+    std::string getRunName() const;
+    void setRunName(std::string const& value);
+    bool runNameIsSet() const;
+    void unsetRun_name();
+    /// <summary>
     /// Name of group owning the data (e.g. p-group or proposal number).  Transferred over CBOR stream, though not saved in HDF5 file. 
     /// </summary>
     std::string getExperimentGroup() const;
@@ -219,8 +229,8 @@ protected:
     bool m_Images_per_triggerIsSet;
     int64_t m_Ntrigger;
     bool m_NtriggerIsSet;
-    int64_t m_Summation;
-    bool m_SummationIsSet;
+    int64_t m_Image_time_us;
+    bool m_Image_time_usIsSet;
     float m_Beam_x_pxl;
 
     float m_Beam_y_pxl;
@@ -236,7 +246,7 @@ protected:
     int64_t m_Space_group_number;
     bool m_Space_group_numberIsSet;
     std::string m_Sample_name;
-
+    bool m_Sample_nameIsSet;
     std::string m_Fpga_output;
     bool m_Fpga_outputIsSet;
     std::string m_Compression;
@@ -247,9 +257,9 @@ protected:
     bool m_TransmissionIsSet;
     org::openapitools::server::model::Rotation_axis m_Goniometer;
     bool m_GoniometerIsSet;
-    std::string m_Header_appendix;
+    nlohmann::json m_Header_appendix;
     bool m_Header_appendixIsSet;
-    std::string m_Image_appendix;
+    nlohmann::json m_Image_appendix;
     bool m_Image_appendixIsSet;
     float m_Energy_multiplier;
     bool m_Energy_multiplierIsSet;
@@ -257,6 +267,8 @@ protected:
     bool m_Data_reduction_factor_serialmxIsSet;
     int64_t m_Run_number;
     bool m_Run_numberIsSet;
+    std::string m_Run_name;
+    bool m_Run_nameIsSet;
     std::string m_Experiment_group;
     bool m_Experiment_groupIsSet;
     org::openapitools::server::model::Dataset_settings_unit_cell m_Unit_cell;

broker/gen/model/Dataset_settings_roi_sum_area.cpp

@@ -1,133 +0,0 @@
-/**
-* Jungfraujoch
-* Jungfraujoch Broker Web API
-*
-* The version of the OpenAPI document: 1.0.1
-* 
-*
-* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
-* https://openapi-generator.tech
-* Do not edit the class manually.
-*/
-
-
-#include "Dataset_settings_roi_sum_area.h"
-#include "Helpers.h"
-
-#include <sstream>
-
-namespace org::openapitools::server::model
-{
-
-Dataset_settings_roi_sum_area::Dataset_settings_roi_sum_area()
-{
-    m_X_min = 0L;
-    m_X_max = 0L;
-    m_Y_min = 0L;
-    m_Y_max = 0L;
-    
-}
-
-void Dataset_settings_roi_sum_area::validate() const
-{
-    std::stringstream msg;
-    if (!validate(msg))
-    {
-        throw org::openapitools::server::helpers::ValidationException(msg.str());
-    }
-}
-
-bool Dataset_settings_roi_sum_area::validate(std::stringstream& msg) const
-{
-    return validate(msg, "");
-}
-
-bool Dataset_settings_roi_sum_area::validate(std::stringstream& msg, const std::string& pathPrefix) const
-{
-    bool success = true;
-    const std::string _pathPrefix = pathPrefix.empty() ? "Dataset_settings_roi_sum_area" : pathPrefix;
-
-                    
-    return success;
-}
-
-bool Dataset_settings_roi_sum_area::operator==(const Dataset_settings_roi_sum_area& rhs) const
-{
-    return
-    
-    
-    (getXMin() == rhs.getXMin())
-     &&
-    
-    (getXMax() == rhs.getXMax())
-     &&
-    
-    (getYMin() == rhs.getYMin())
-     &&
-    
-    (getYMax() == rhs.getYMax())
-    
-    
-    ;
-}
-
-bool Dataset_settings_roi_sum_area::operator!=(const Dataset_settings_roi_sum_area& rhs) const
-{
-    return !(*this == rhs);
-}
-
-void to_json(nlohmann::json& j, const Dataset_settings_roi_sum_area& o)
-{
-    j = nlohmann::json();
-    j["x_min"] = o.m_X_min;
-    j["x_max"] = o.m_X_max;
-    j["y_min"] = o.m_Y_min;
-    j["y_max"] = o.m_Y_max;
-    
-}
-
-void from_json(const nlohmann::json& j, Dataset_settings_roi_sum_area& o)
-{
-    j.at("x_min").get_to(o.m_X_min);
-    j.at("x_max").get_to(o.m_X_max);
-    j.at("y_min").get_to(o.m_Y_min);
-    j.at("y_max").get_to(o.m_Y_max);
-    
-}
-
-int64_t Dataset_settings_roi_sum_area::getXMin() const
-{
-    return m_X_min;
-}
-void Dataset_settings_roi_sum_area::setXMin(int64_t const value)
-{
-    m_X_min = value;
-}
-int64_t Dataset_settings_roi_sum_area::getXMax() const
-{
-    return m_X_max;
-}
-void Dataset_settings_roi_sum_area::setXMax(int64_t const value)
-{
-    m_X_max = value;
-}
-int64_t Dataset_settings_roi_sum_area::getYMin() const
-{
-    return m_Y_min;
-}
-void Dataset_settings_roi_sum_area::setYMin(int64_t const value)
-{
-    m_Y_min = value;
-}
-int64_t Dataset_settings_roi_sum_area::getYMax() const
-{
-    return m_Y_max;
-}
-void Dataset_settings_roi_sum_area::setYMax(int64_t const value)
-{
-    m_Y_max = value;
-}
-
-
-} // namespace org::openapitools::server::model
-

broker/gen/model/Dataset_settings_roi_sum_area.h

@@ -1,97 +0,0 @@
-/**
-* Jungfraujoch
-* Jungfraujoch Broker Web API
-*
-* The version of the OpenAPI document: 1.0.1
-* 
-*
-* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
-* https://openapi-generator.tech
-* Do not edit the class manually.
-*/
-/*
- * Dataset_settings_roi_sum_area.h
- *
- * Rectangle for ROI summation
- */
-
-#ifndef Dataset_settings_roi_sum_area_H_
-#define Dataset_settings_roi_sum_area_H_
-
-
-#include <nlohmann/json.hpp>
-
-namespace org::openapitools::server::model
-{
-
-/// <summary>
-/// Rectangle for ROI summation
-/// </summary>
-class  Dataset_settings_roi_sum_area
-{
-public:
-    Dataset_settings_roi_sum_area();
-    virtual ~Dataset_settings_roi_sum_area() = default;
-
-
-    /// <summary>
-    /// Validate the current data in the model. Throws a ValidationException on failure.
-    /// </summary>
-    void validate() const;
-
-    /// <summary>
-    /// Validate the current data in the model. Returns false on error and writes an error
-    /// message into the given stringstream.
-    /// </summary>
-    bool validate(std::stringstream& msg) const;
-
-    /// <summary>
-    /// Helper overload for validate. Used when one model stores another model and calls it's validate.
-    /// Not meant to be called outside that case.
-    /// </summary>
-    bool validate(std::stringstream& msg, const std::string& pathPrefix) const;
-
-    bool operator==(const Dataset_settings_roi_sum_area& rhs) const;
-    bool operator!=(const Dataset_settings_roi_sum_area& rhs) const;
-
-    /////////////////////////////////////////////
-    /// Dataset_settings_roi_sum_area members
-
-    /// <summary>
-    /// 
-    /// </summary>
-    int64_t getXMin() const;
-    void setXMin(int64_t const value);
-    /// <summary>
-    /// 
-    /// </summary>
-    int64_t getXMax() const;
-    void setXMax(int64_t const value);
-    /// <summary>
-    /// 
-    /// </summary>
-    int64_t getYMin() const;
-    void setYMin(int64_t const value);
-    /// <summary>
-    /// 
-    /// </summary>
-    int64_t getYMax() const;
-    void setYMax(int64_t const value);
-
-    friend void to_json(nlohmann::json& j, const Dataset_settings_roi_sum_area& o);
-    friend void from_json(const nlohmann::json& j, Dataset_settings_roi_sum_area& o);
-protected:
-    int64_t m_X_min;
-
-    int64_t m_X_max;
-
-    int64_t m_Y_min;
-
-    int64_t m_Y_max;
-
-    
-};
-
-} // namespace org::openapitools::server::model
-
-#endif /* Dataset_settings_roi_sum_area_H_ */

broker/gen/model/Dataset_settings_unit_cell.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Dataset_settings_unit_cell.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Detector_list.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Detector_list.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Detector_list_detectors_inner.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Detector_list_detectors_inner.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Detector_selection.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Detector_selection.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Detector_settings.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Detector_settings.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Detector_status.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Detector_status.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Error_message.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Error_message.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Helpers.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Helpers.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Measurement_statistics.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Measurement_statistics.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Plot.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Plot.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Plot_request.cpp

@@ -1,106 +0,0 @@
-/**
-* Jungfraujoch
-* Jungfraujoch Broker Web API
-*
-* The version of the OpenAPI document: 1.0.1
-* 
-*
-* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
-* https://openapi-generator.tech
-* Do not edit the class manually.
-*/
-
-
-#include "Plot_request.h"
-#include "Helpers.h"
-
-#include <sstream>
-
-namespace org::openapitools::server::model
-{
-
-Plot_request::Plot_request()
-{
-    m_Binning = 0L;
-    m_BinningIsSet = false;
-    
-}
-
-void Plot_request::validate() const
-{
-    std::stringstream msg;
-    if (!validate(msg))
-    {
-        throw org::openapitools::server::helpers::ValidationException(msg.str());
-    }
-}
-
-bool Plot_request::validate(std::stringstream& msg) const
-{
-    return validate(msg, "");
-}
-
-bool Plot_request::validate(std::stringstream& msg, const std::string& pathPrefix) const
-{
-    bool success = true;
-    const std::string _pathPrefix = pathPrefix.empty() ? "Plot_request" : pathPrefix;
-
-        
-    return success;
-}
-
-bool Plot_request::operator==(const Plot_request& rhs) const
-{
-    return
-    
-    
-    
-    ((!binningIsSet() && !rhs.binningIsSet()) || (binningIsSet() && rhs.binningIsSet() && getBinning() == rhs.getBinning()))
-    
-    ;
-}
-
-bool Plot_request::operator!=(const Plot_request& rhs) const
-{
-    return !(*this == rhs);
-}
-
-void to_json(nlohmann::json& j, const Plot_request& o)
-{
-    j = nlohmann::json();
-    if(o.binningIsSet())
-        j["binning"] = o.m_Binning;
-    
-}
-
-void from_json(const nlohmann::json& j, Plot_request& o)
-{
-    if(j.find("binning") != j.end())
-    {
-        j.at("binning").get_to(o.m_Binning);
-        o.m_BinningIsSet = true;
-    } 
-    
-}
-
-int64_t Plot_request::getBinning() const
-{
-    return m_Binning;
-}
-void Plot_request::setBinning(int64_t const value)
-{
-    m_Binning = value;
-    m_BinningIsSet = true;
-}
-bool Plot_request::binningIsSet() const
-{
-    return m_BinningIsSet;
-}
-void Plot_request::unsetBinning()
-{
-    m_BinningIsSet = false;
-}
-
-
-} // namespace org::openapitools::server::model
-

broker/gen/model/Plot_request.h

@@ -1,78 +0,0 @@
-/**
-* Jungfraujoch
-* Jungfraujoch Broker Web API
-*
-* The version of the OpenAPI document: 1.0.1
-* 
-*
-* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
-* https://openapi-generator.tech
-* Do not edit the class manually.
-*/
-/*
- * Plot_request.h
- *
- * 
- */
-
-#ifndef Plot_request_H_
-#define Plot_request_H_
-
-
-#include <nlohmann/json.hpp>
-
-namespace org::openapitools::server::model
-{
-
-/// <summary>
-/// 
-/// </summary>
-class  Plot_request
-{
-public:
-    Plot_request();
-    virtual ~Plot_request() = default;
-
-
-    /// <summary>
-    /// Validate the current data in the model. Throws a ValidationException on failure.
-    /// </summary>
-    void validate() const;
-
-    /// <summary>
-    /// Validate the current data in the model. Returns false on error and writes an error
-    /// message into the given stringstream.
-    /// </summary>
-    bool validate(std::stringstream& msg) const;
-
-    /// <summary>
-    /// Helper overload for validate. Used when one model stores another model and calls it's validate.
-    /// Not meant to be called outside that case.
-    /// </summary>
-    bool validate(std::stringstream& msg, const std::string& pathPrefix) const;
-
-    bool operator==(const Plot_request& rhs) const;
-    bool operator!=(const Plot_request& rhs) const;
-
-    /////////////////////////////////////////////
-    /// Plot_request members
-
-    /// <summary>
-    /// 
-    /// </summary>
-    int64_t getBinning() const;
-    void setBinning(int64_t const value);
-    bool binningIsSet() const;
-    void unsetBinning();
-
-    friend void to_json(nlohmann::json& j, const Plot_request& o);
-    friend void from_json(const nlohmann::json& j, Plot_request& o);
-protected:
-    int64_t m_Binning;
-    bool m_BinningIsSet;
-    
-};
-
-} // namespace org::openapitools::server::model
-
-#endif /* Plot_request_H_ */

broker/gen/model/Plots.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Plots.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Preview_settings.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Preview_settings.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Rad_int_settings.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Rad_int_settings.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Roi_box.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Roi_box.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Roi_box_list.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Roi_box_list.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Roi_circle.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Roi_circle.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Roi_circle_list.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Roi_circle_list.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Rotation_axis.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Rotation_axis.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Spot_finding_settings.cpp

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/gen/model/Spot_finding_settings.h

@@ -2,8 +2,8 @@
 * Jungfraujoch
 * Jungfraujoch Broker Web API
 *
-* The version of the OpenAPI document: 1.0.1
-* 
+* The version of the OpenAPI document: 1.0.0_rc.11
+* Contact: filip.leonarski@psi.ch
 *
 * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
 * https://openapi-generator.tech

broker/jfjoch_api.yaml

@@ -2,7 +2,9 @@ openapi: 3.0.3
 info:
   title: Jungfraujoch
   description: Jungfraujoch Broker Web API
-  version: 1.0.1
+  version: 1.0.0_rc.11
+  contact:
+    email: filip.leonarski@psi.ch
 components:
   schemas:
     rotation_axis:
@@ -43,7 +45,6 @@ components:
         - beam_y_pxl
         - detector_distance_mm
         - incident_energy_keV
-        - sample_name
       properties:
         images_per_trigger:
           type: integer
@@ -61,17 +62,16 @@ components:
           minimum: 1
           description: |
             Number of TTL trigger that the detector is expected to receive during data collection
-        summation:
+        image_time_us:
           type: integer
           format: int64
-          minimum: 1
-          maximum: 256
-          default: 1
+          minimum: 0
           description: |
-            FPGA frame summation. For summation above two 32-bit pixel format will be used, unless explicitly specified.
-            Frame summation factor applies only to conversion mode (assumed as 1 for raw data).
+            Image time. 
+            If not provided (or zero value) the frame time is assumed as default.
+            Image time must be multiple of frame time; max value is 256 * frame_time.            
             In XFEL mode: summation happens for frames collected with multiple triggers.
-            Ignored for storage cells (assumed as 1).
+            Ignored for storage cells and if raw data are saved.
         beam_x_pxl:
           type: number
           format: float
@@ -116,6 +116,7 @@ components:
           maximum: 194
         sample_name:
           type: string
+          default: ""
           description: |
             /entry/sample/name in NXmx
             Sample name
@@ -156,11 +157,9 @@ components:
         goniometer:
           $ref: "#/components/schemas/rotation_axis"
         header_appendix:
-          type: string
-          description: Header appendix, added as user_data to start message
+          description: Header appendix, added as user_data/user to start message (can be any valid JSON)
         image_appendix:
-          type: string
-          description: Image appendix, added as user_data to image message
+          description: Image appendix, added as user_data to image message (can be any valid JSON)
         energy_multiplier:
           type: number
           format: float
@@ -187,6 +186,13 @@ components:
             Transferred over CBOR stream as "series ID", though not saved in HDF5 file.
             It is highly recommended to keep this number unique for each data collection during experimental series.
             If not provided, the number will be automatically incremented.
+        run_name:
+          type: string
+          description: |
+            Unique ID of run.
+            Transferred over CBOR stream as "unique series ID", though not saved in HDF5 file.
+            It is highly recommended to keep this name unique for each data collection during experimental series.
+            If not provided, the name will be automatically generated as number + colon + file_prefix.
         experiment_group:
           type: string
           description: |
@@ -847,22 +853,37 @@ paths:
   /wait_till_done:
     post:
       summary: Wait for acquisition done
+      parameters:
+        - in: query
+          name: timeout
+          required: false
+          schema:
+            type: integer
+            default: 60
+            minimum: 0
+            maximum: 3600
+          description: Timeout in seconds (0 == immediate response)
       description: |
         Block execution of external script till initialization, data collection or pedestal is finished.
         Running this command does not affect (cancel) running data collection, it is only to ensure synchronous execution of other software.
                 
-        To not block web server for a long period of time, the procedure is provided with a timeout of 5 seconds.
+        To not block web server for a indefinite period of time, the procedure is provided with a timeout.
+        Extending timeout is possible, but requires to ensure safety that client will not close the connection and retry the connection.
       responses:
         "200":
           description: Detector in `Idle` state, another data collection can start immediately
+        "400":
+          description: Timeout parameter out of bounds
         "500":
           description: Error within Jungfraujoch code - see output message.
           content:
             application/json:
               schema:
                 $ref: '#/components/schemas/error_message'
+        "502":
+          description: Detector is inactive mode
         "504":
-          description: 5 second timeout reached, need to restart operation
+          description: Timeout reached, need to restart operation
 
   /trigger:
     post:
@@ -1724,4 +1745,14 @@ paths:
                 type: string
                 format: binary
         "404":
-          description: No calibration recorded so far
+          description: No calibration recorded so far
+  /version:
+    get:
+      responses:
+        "200":
+          description: Release number of Jungfraujoch
+          content:
+            text/plain:
+              schema:
+                type: string
+                example: 1.0.0

broker/jfjoch_broker.cpp

@@ -12,8 +12,6 @@
 #include "JFJochBrokerHttp.h"
 
 #include "JFJochBrokerParser.h"
-#include "../frame_serialize/ZMQStream2Pusher.h"
-#include "../frame_serialize/DumpCBORToFilePusher.h"
 
 static Pistache::Http::Endpoint *httpEndpoint;
 
@@ -81,25 +79,7 @@ int main (int argc, char **argv) {
     if (aq_devices.size() > 0) {
         experiment.DataStreams(aq_devices.size());
 
-        std::string pusher_type = ParseString(input, "stream_type", "zmq");
-        if (pusher_type == "zmq") {
-            int32_t zmq_send_watermark = ParseInt32(input, "zmq_send_watermark", 100);
-            int32_t zmq_send_buffer_size = ParseInt32(input, "zmq_send_buffer_size", -1);
-
-            auto tmp = std::make_unique<ZMQStream2Pusher>(ParseStringArray(input, "zmq_image_addr"),
-                                                                   zmq_send_watermark,
-                                                                   zmq_send_buffer_size);
-
-            std::string preview_addr = ParseString(input, "zmq_preview_addr", "");
-            if (!preview_addr.empty())
-                tmp->PreviewSocket(preview_addr);
-
-            image_pusher = std::move(tmp);
-        } else if (pusher_type == "dump_cbor") {
-            image_pusher = std::make_unique<DumpCBORToFilePusher>();
-        } else
-            throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
-                                  "stream_type allowed: zmq (default), dump_cbor");
+        ParseImagePusher(input, image_pusher);
 
         int32_t send_buffer_size_MiB = ParseInt32(input, "send_buffer_size_MiB", 2048);
         receiver = std::make_unique<JFJochReceiverService>(aq_devices, logger, *image_pusher, send_buffer_size_MiB);

common/CUDAWrapper.cu

@@ -17,8 +17,11 @@ int32_t get_gpu_count() {
 void set_gpu(int32_t dev_id) {
     auto dev_count = get_gpu_count();
 
-    if ((dev_id < 0) || (dev_id >= dev_count))
-        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Device ID cannot be negative");
+    // Ignore if no GPU present
+    if (dev_count > 0) {
+        if ((dev_id < 0) || (dev_id >= dev_count))
+            throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Device ID cannot be negative");
 
-    cuda_err(cudaSetDevice(dev_id));
-}
+        cuda_err(cudaSetDevice(dev_id));
+    }
+}

common/DatasetSettings.cpp

@@ -19,7 +19,6 @@ DatasetSettings::DatasetSettings() {
     file_prefix = "test";
     ntrigger = 1;
     images_per_trigger = 1;
-    summation = 1;
     fpga_pixel_output = FPGAPixelOutput::Auto;
     space_group_number = 0; // not set
     compression = CompressionAlgorithm::BSHUF_LZ4;
@@ -172,12 +171,12 @@ DatasetSettings &DatasetSettings::RotationAxis(const std::optional<Coord> &c) {
     return *this;
 }
 
-DatasetSettings &DatasetSettings::HeaderAppendix(const std::string &input) {
+DatasetSettings &DatasetSettings::HeaderAppendix(const nlohmann::json &input) {
     header_appendix = input;
     return *this;
 }
 
-DatasetSettings &DatasetSettings::ImageAppendix(const std::string &input) {
+DatasetSettings &DatasetSettings::ImageAppendix(const nlohmann::json &input) {
     image_appendix = input;
     return *this;
 }
@@ -191,13 +190,6 @@ DatasetSettings &DatasetSettings::PhotonEnergyMultiplayer(float input) {
     return *this;
 }
 
-DatasetSettings &DatasetSettings::Summation(int64_t input) {
-    check_min("Summation", input, 1);
-    check_max("Summation", input, MAX_FPGA_SUMMATION);
-    summation = input;
-    return *this;
-}
-
 DatasetSettings &DatasetSettings::FPGAOutputMode(FPGAPixelOutput input) {
     switch (input) {
         case FPGAPixelOutput::Auto:
@@ -229,11 +221,11 @@ std::optional<Coord> DatasetSettings::GetRotationAxis() const {
     return rotation_axis;
 }
 
-std::string DatasetSettings::GetHeaderAppendix() const {
+const nlohmann::json& DatasetSettings::GetHeaderAppendix() const {
     return header_appendix;
 }
 
-std::string DatasetSettings::GetImageAppendix() const {
+const nlohmann::json& DatasetSettings::GetImageAppendix() const {
     return image_appendix;
 }
 
@@ -249,10 +241,6 @@ int64_t DatasetSettings::GetSpaceGroupNumber() const {
     return space_group_number;
 }
 
-int64_t DatasetSettings::GetSummation() const {
-    return summation;
-}
-
 FPGAPixelOutput DatasetSettings::GetFPGAOutputMode() const {
     return fpga_pixel_output;
 }
@@ -331,6 +319,14 @@ DatasetSettings &DatasetSettings::RunNumber(const std::optional<uint64_t> &input
     return *this;
 }
 
+DatasetSettings & DatasetSettings::RunName(const std::optional<std::string> &input) {
+    if (input && input.value().empty())
+        run_name = {};
+    else
+        run_name = input;
+    return *this;
+}
+
 DatasetSettings &DatasetSettings::ExperimentGroup(const std::string &input) {
     group = input;
     return *this;
@@ -340,6 +336,22 @@ std::optional<uint64_t> DatasetSettings::GetRunNumber() const {
     return run_number;
 }
 
+std::optional<std::string> DatasetSettings::GetRunName() const {
+    return run_name;
+}
+
 std::string DatasetSettings::GetExperimentGroup() const {
     return group;
 }
+
+std::optional<std::chrono::microseconds> DatasetSettings::GetImageTime() const {
+    return image_time;
+}
+
+DatasetSettings &DatasetSettings::ImageTime(const std::optional<std::chrono::microseconds> input) {
+    if (input && (input.value().count() == 0))
+        image_time = {};
+    else
+        image_time = input;
+    return *this;
+}

common/DatasetSettings.h

@@ -21,7 +21,6 @@ class DatasetSettings {
     int64_t ntrigger;
 
     FPGAPixelOutput fpga_pixel_output;
-    int64_t summation;
 
     float beam_x_pxl;
     float beam_y_pxl;
@@ -41,14 +40,16 @@ class DatasetSettings {
     std::optional<float> total_flux;
     std::optional<float> attenuator_transmission;
     std::optional<GoniometerAxis> goniometer;
-    std::string image_appendix;
-    std::string header_appendix;
+    nlohmann::json image_appendix;
+    nlohmann::json header_appendix;
 
     std::optional<Coord> rotation_axis;
 
     float data_reduction_factor_serialmx;
 
+    std::optional<std::chrono::microseconds> image_time;
     std::optional<uint64_t> run_number;
+    std::optional<std::string> run_name;
     std::string group;
 public:
 
@@ -69,26 +70,26 @@ public:
     DatasetSettings& TotalFlux(const std::optional<float> &input);
     DatasetSettings& Goniometer(const std::optional<GoniometerAxis>& input);
     DatasetSettings& RotationAxis(const std::optional<Coord> &c);
-    DatasetSettings& HeaderAppendix(const std::string& input);
-    DatasetSettings& ImageAppendix(const std::string& input);
+    DatasetSettings& HeaderAppendix(const nlohmann::json& input);
+    DatasetSettings& ImageAppendix(const nlohmann::json& input);
     DatasetSettings& PhotonEnergyMultiplayer(float input);
-    DatasetSettings& Summation(int64_t input);
     DatasetSettings& FPGAOutputMode(FPGAPixelOutput input);
     DatasetSettings& ImagesPerFile(int64_t input);
     DatasetSettings& DataReductionFactorSerialMX(float input);
     DatasetSettings& RunNumber(const std::optional<uint64_t> &run_number);
+    DatasetSettings& RunName(const std::optional<std::string> &input);
     DatasetSettings& ExperimentGroup(const std::string &group);
+    DatasetSettings& ImageTime(const std::optional<std::chrono::microseconds> input);
 
     std::optional<float> GetAttenuatorTransmission() const;
     std::optional<float> GetTotalFlux() const;
     std::optional<GoniometerAxis> GetGoniometer() const;
     std::optional<Coord> GetRotationAxis() const;
-    std::string GetHeaderAppendix() const;
-    std::string GetImageAppendix() const;
+    const nlohmann::json& GetHeaderAppendix() const;
+    const nlohmann::json& GetImageAppendix() const;
     float GetPhotonEnergyMultiplier() const;
     std::optional<UnitCell> GetUnitCell() const;
     int64_t GetSpaceGroupNumber() const;
-    int64_t GetSummation() const;
     FPGAPixelOutput GetFPGAOutputMode() const;
     std::string GetSampleName() const;
     float GetPhotonEnergy_keV() const;
@@ -108,7 +109,9 @@ public:
     float GetDataReductionFactorSerialMX() const;
 
     std::optional<uint64_t> GetRunNumber() const;
+    std::optional<std::string> GetRunName() const;
     std::string GetExperimentGroup() const;
+    std::optional<std::chrono::microseconds> GetImageTime() const;
 };
 
 #endif //JUNGFRAUJOCH_DATASETSETTINGS_H

common/DetectorSetup.cpp

@@ -148,3 +148,12 @@ void DetectorSetup::SetTrimFiles(const std::vector<std::string> &filenames) {
 const std::vector<std::string> &DetectorSetup::GetTrimFileNames() const {
     return trim_file_names;
 }
+
+std::string DetectorSetup::GetSerialNumber() const {
+    return serial_number;
+}
+
+DetectorSetup &DetectorSetup::SerialNumber(const std::string &input) {
+    serial_number = input;
+    return *this;
+}

common/DetectorSetup.h

@@ -10,6 +10,7 @@ enum class DetectorType {EIGER, JUNGFRAU};
 
 class DetectorSetup {
     std::string description;
+    std::string serial_number;
     DetectorGeometry geometry;
     std::vector<std::string> det_modules_hostname;
     std::vector<std::string> gain_file_names;
@@ -41,6 +42,7 @@ public:
     DetectorSetup& SensorThickness_um(float input);
     DetectorSetup& PixelSize_um(float input);
     DetectorSetup& HighVoltage(int32_t input);
+    DetectorSetup& SerialNumber(const std::string &input);
 
     [[nodiscard]] DetectorType GetDetectorType() const;
     [[nodiscard]] const DetectorGeometry& GetGeometry() const;
@@ -56,6 +58,7 @@ public:
     [[nodiscard]] const std::vector<std::string> &GetGainFileNames() const;
     [[nodiscard]] const std::vector<std::string> &GetTrimFileNames() const;
     [[nodiscard]] int32_t GetHighVoltage() const;
+    [[nodiscard]] std::string GetSerialNumber() const;
 };
 
 

common/DiffractionExperiment.cpp

@@ -67,6 +67,8 @@ DiffractionExperiment::DiffractionExperiment(const DetectorSetup& det_setup)
     mode = DetectorMode::Conversion;
 
     max_spot_count = MAX_SPOT_COUNT;
+
+    summation = 1;
 }
 
 // setter functions
@@ -751,8 +753,8 @@ void DiffractionExperiment::FillMessage(StartMessage &message) const {
         message.rotation_axis[2] = GetRotationAxis().z;
     }
 
-    message.series_id = GetRunNumber();
-    message.series_unique_id = GetRunName();
+    message.run_number = GetRunNumber();
+    message.run_name = GetRunName();
 
     message.gain_file_names = detector.GetGainFileNames();
 
@@ -762,6 +764,7 @@ void DiffractionExperiment::FillMessage(StartMessage &message) const {
     message.data_reduction_factor_serialmx = GetDataReductionFactorSerialMX();
     message.experiment_group = dataset.GetExperimentGroup();
     message.jfjoch_release = jfjoch_version();
+    message.detector_serial_number = detector.GetSerialNumber();
 }
 
 float DiffractionExperiment::GetPixelSize_mm() const {
@@ -882,7 +885,9 @@ std::chrono::nanoseconds DiffractionExperiment::GetStorageCellDelay() const {
 }
 
 DiffractionExperiment &DiffractionExperiment::Summation(int64_t input) {
-    dataset.Summation(input);
+    check_min("Summation factor", input, 1);
+    check_max("Summation factor", input, MAX_FPGA_SUMMATION);
+    summation = input;
     return *this;
 }
 
@@ -898,7 +903,7 @@ int64_t DiffractionExperiment::GetSummation() const {
             if (GetStorageCellNumber() > 1)
                 return 1;
             else
-                return dataset.GetSummation();
+                return summation;
     }
 }
 
@@ -994,21 +999,21 @@ bool DiffractionExperiment::IsUsingGainHG0() const {
     return use_gain_hg0;
 }
 
-DiffractionExperiment &DiffractionExperiment::HeaderAppendix(const std::string &input) {
+DiffractionExperiment &DiffractionExperiment::HeaderAppendix(const nlohmann::json &input) {
     dataset.HeaderAppendix(input);
     return *this;
 }
 
-DiffractionExperiment &DiffractionExperiment::ImageAppendix(const std::string &input) {
+DiffractionExperiment &DiffractionExperiment::ImageAppendix(const nlohmann::json &input) {
     dataset.ImageAppendix(input);
     return *this;
 }
 
-std::string DiffractionExperiment::GetHeaderAppendix() const {
+const nlohmann::json& DiffractionExperiment::GetHeaderAppendix() const {
     return dataset.GetHeaderAppendix();
 }
 
-std::string DiffractionExperiment::GetImageAppendix() const {
+const nlohmann::json& DiffractionExperiment::GetImageAppendix() const {
     return dataset.GetImageAppendix();
 }
 
@@ -1044,7 +1049,11 @@ uint64_t DiffractionExperiment::GetRunNumber() const {
 }
 
 std::string DiffractionExperiment::GetRunName() const {
-    return std::to_string(series_id) + ": " + dataset.GetFilePrefix();
+    auto run_name = dataset.GetRunName();
+    if (run_name)
+        return run_name.value();
+    else
+        return std::to_string(series_id) + ":" + dataset.GetFilePrefix();
 }
 
 DiffractionExperiment &DiffractionExperiment::IncrementRunNumber() {
@@ -1134,6 +1143,19 @@ int64_t DiffractionExperiment::GetInternalPacketGeneratorImages() const {
 DiffractionExperiment &DiffractionExperiment::ImportDatasetSettings(const DatasetSettings &input) {
     auto tmp = dataset;
     dataset = input;
+
+    auto image_time = input.GetImageTime();
+    if (image_time) {
+        if (image_time->count() % GetFrameTime().count() != 0)
+            throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
+                                  "Image time must be multiple of frame time");
+        if (GetFrameTime().count() == 0)
+            throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
+                                  "Frame time cannot be zero");
+        this->Summation(image_time.value() / GetFrameTime());
+    } else
+        summation = 1;
+
     if (GetFrameNum() >= MAX_FRAMES) {
         dataset = tmp;
         throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,

common/DiffractionExperiment.h

@@ -98,6 +98,10 @@ class DiffractionExperiment {
     ROIMap roi_mask;
 
     int64_t max_spot_count;
+
+    int64_t summation;
+
+    std::string detector_update_zmq_addr;
 public:
     // Public methods are atomic
     DiffractionExperiment();
@@ -160,8 +164,8 @@ public:
     DiffractionExperiment& TotalFlux(const std::optional<float> &input);
     DiffractionExperiment& Goniometer(const std::optional<GoniometerAxis> &input);
     DiffractionExperiment& RotationAxis(const std::optional<Coord> &c);
-    DiffractionExperiment& HeaderAppendix(const std::string& input);
-    DiffractionExperiment& ImageAppendix(const std::string& input);
+    DiffractionExperiment& HeaderAppendix(const nlohmann::json& input);
+    DiffractionExperiment& ImageAppendix(const nlohmann::json& input);
     DiffractionExperiment& PhotonEnergyMultiplayer(float input);
     DiffractionExperiment& Summation(int64_t input);
     DiffractionExperiment& FPGAOutputMode(FPGAPixelOutput input);
@@ -291,8 +295,8 @@ public:
     std::optional<GoniometerAxis> GetGoniometer() const;
 
     Coord GetRotationAxis() const;
-    std::string GetHeaderAppendix() const;
-    std::string GetImageAppendix() const;
+    const nlohmann::json& GetHeaderAppendix() const;
+    const nlohmann::json& GetImageAppendix() const;
     float GetPhotonEnergyMultiplier() const;
     std::optional<UnitCell> GetUnitCell() const;
     std::string GetUnitCellString() const;

detector_control/DetectorWrapper.cpp

@@ -65,14 +65,14 @@ void DetectorWrapper::Initialize(const DiffractionExperiment& experiment,
                 det.setSourceUDPIP(sls::IpAddr(cfg.ipv4_src_addr_1), {i});
                 det.setSourceUDPMAC(sls::MacAddr(BASE_DETECTOR_MAC + i * 2), {i});
 
-                det.setDestinationUDPPort(cfg.udp_dest_port_1, i);
+                det.setDestinationUDPPort(cfg.udp_dest_port_1 + 2 * i, i);
                 det.setDestinationUDPIP(sls::IpAddr(cfg.ipv4_dest_addr_1), {i});
                 det.setDestinationUDPMAC(sls::MacAddr(cfg.mac_addr_dest_1), {i});
 
                 if (experiment.GetUDPInterfaceCount() == 2) {
                     det.setSourceUDPIP2(sls::IpAddr(cfg.ipv4_src_addr_2), {i});
                     det.setSourceUDPMAC2(sls::MacAddr(BASE_DETECTOR_MAC + i * 2 + 1), {i});
-                    det.setDestinationUDPPort2(cfg.udp_dest_port_2, i);
+                    det.setDestinationUDPPort2(cfg.udp_dest_port_2 + 2 * i + 1, i);
                     det.setDestinationUDPIP2(sls::IpAddr(cfg.ipv4_dest_addr_2), {i});
                     det.setDestinationUDPMAC2(sls::MacAddr(cfg.mac_addr_dest_2), {i});
                 }
@@ -91,7 +91,7 @@ void DetectorWrapper::Initialize(const DiffractionExperiment& experiment,
                 throw JFJochException(JFJochExceptionCategory::Detector,
                                       "Discrepancy in module number between DAQ and detector");
             }
-
+            det.setDynamicRange(16);
             det.setTenGiga(true);
             auto trim_files = experiment.GetDetectorSetup().GetTrimFileNames();
 
@@ -100,8 +100,8 @@ void DetectorWrapper::Initialize(const DiffractionExperiment& experiment,
 
                 auto &cfg = mod_cfg[i];
 
-                det.setDestinationUDPPort(cfg.udp_dest_port_1, 2 * i);
-                det.setDestinationUDPPort2(cfg.udp_dest_port_1, 2 * i + 1);
+                det.setDestinationUDPPort(cfg.udp_dest_port_1 + 2 * i, 2 * i);
+                det.setDestinationUDPPort2(cfg.udp_dest_port_1 + 2 * i + 1, 2 * i + 1);
                 det.setSourceUDPIP(sls::IpAddr(cfg.ipv4_src_addr_1), {2 * i, 2 * i + 1});
                 det.setDestinationUDPIP(sls::IpAddr(cfg.ipv4_dest_addr_1), {2 * i, 2 * i + 1});
                 det.setDestinationUDPMAC(sls::MacAddr(cfg.mac_addr_dest_1), {2 * i, 2 * i + 1});

etc/broker.json

@@ -8,13 +8,13 @@
     "pedestal_g0_frames": 2000,
     "pedestal_g1_frames": 300,
     "pedestal_g2_frames": 300,
-    "frame_time_us": "500 us",
-    "count_time_us": "480 us",
-    "preview_period_us": "1 s",
+    "frame_time": "500 us",
+    "count_time": "480 us",
     "detector_ipv4": "10.10.85.0"
   },
   "frontend_directory":"/home/jungfrau/nextgendcu/frontend/build",
   "numa_policy": "n2g2",
+  "zmq_preview_period": "1 s",
   "zmq_image_addr": ["tcp://0.0.0.0:5500", "tcp://0.0.0.0:5501", "tcp://0.0.0.0:5502", "tcp://0.0.0.0:5503"],
   "receiver_threads": 64,
   "receiver": {

etc/broker_crmx.json

@@ -8,15 +8,16 @@
     "pedestal_g0_frames": 2000,
     "pedestal_g1_frames": 300,
     "pedestal_g2_frames": 300,
-    "frame_time_us": "10 ms",
-    "count_time_us": "10 us",
-    "preview_period_us": "1 s",
+    "frame_time": "10 ms",
+    "count_time": "10 us",
     "detector_ipv4": "10.3.30.155",
     "pulsed_source": true
   },
   "frontend_directory":"/home/jungfrau/nextgendcu/frontend/build",
   "numa_policy": "n2g4",
   "zmq_image_addr": ["tcp://0.0.0.0:5500"],
+  "zmq_preview_period": "1 s",
+
   "receiver_threads": 64,
   "receiver": {
     "type": "pcie",

etc/broker_local.json

@@ -8,9 +8,8 @@
     "pedestal_g0_frames": 0,
     "pedestal_g1_frames": 0,
     "pedestal_g2_frames": 0,
-    "frame_time_us": "1000 us",
-    "count_time_us": "980 us",
-    "preview_period_us": "1 s",
+    "frame_time": "1000 us",
+    "count_time": "980 us",
     "detector_ipv4": "10.10.85.20",
     "internal_frame_generator": true
   },
@@ -20,6 +19,9 @@
   },
   "frontend_directory": "../../frontend_ui/build/",
   "zmq_image_addr": ["tcp://0.0.0.0:5500"],
+  "zmq_preview_addr": "tcp://0.0.0.0:5501",
+  "zmq_preview_period": "1 s",
+  "zmq_writer_notification_addr": "ipc://*",
   "detectors": [
     {
       "standard_geometry": {

fpga/hls/host_writer.cpp

@@ -75,6 +75,7 @@ void host_writer(STREAM_512 &data_in,
     state = 0;
 
     ap_uint<16> req_handle;
+    ap_uint<16> req_handle_send = HANDLE_START;
     ap_uint<64> req_host_offset;
 
      while (data_in.empty()) {
@@ -240,7 +241,9 @@ void host_writer(STREAM_512 &data_in,
             host_memory_out << packet_out;
 
         if (send_images) {
-            write_completion(m_axis_completion, req_handle, data_collection_id);
+            if (req_handle_send != HANDLE_START)
+                write_completion(m_axis_completion, req_handle_send, data_collection_id);
+            req_handle_send = req_handle;
             internal_packets_processed += cmpl.packet_count;
             packets_processed = internal_packets_processed;
         }
@@ -252,6 +255,21 @@ void host_writer(STREAM_512 &data_in,
 
     data_in >> packet;
 
+#ifdef JFJOCH_HLS_NOSYNTH
+    while (!host_memory_out.empty())
+        std::this_thread::sleep_for(std::chrono::milliseconds(1));
+#else
+    {
+        // wait for 5 ms (ensure that PCIe buffer is clean)
+#pragma HLS PROTOCOL
+        for (int i = 0; i < 5 * 200 * 1000; i++)
+                ap_wait();
+    }
+#endif
+
+    if (req_handle_send != HANDLE_START)
+        write_completion(m_axis_completion, req_handle_send, data_collection_id);
+
     write_completion(m_axis_completion, HANDLE_END, data_collection_id);
 
     idle = 1;

fpga/pcie_driver/jfjoch_drv.h

@@ -177,11 +177,9 @@ struct jfjoch_drvdata {
     // however getting things from the queue happens in system call and can be executed in parallel
     // therefore this part is protected by mutex (it is assumed that waiting for interrupts can take seconds,
     // process has to be able to sleep while holding the lock => spinlock would not work)
-    // the mutex is also protecting work_compl_count variable
     struct mutex          work_compl_read_mutex;
     DECLARE_KFIFO(work_compl, u32, MAX_FPGA_BUFFER); // protected by work_compl_read_mutex
     wait_queue_head_t     work_compl_wait_queue; // used for read completion queue method to wait for interrupt, ISR has wake-up call
-    u32                   work_compl_count; // protected by work_compl_read_mutex
     atomic_t              active_handles;
 };
 
@@ -220,7 +218,6 @@ void jfjoch_set_ipv4_addr(struct jfjoch_drvdata *drvdata, const u32 *addr);
 void jfjoch_get_ipv4_addr(struct jfjoch_drvdata *drvdata, u32 *addr);
 int jfjoch_load_calibration(struct jfjoch_drvdata *drvdata, struct LoadCalibrationConfig *config);
 int jfjoch_run_frame_gen(struct jfjoch_drvdata *drvdata, struct FrameGeneratorConfig *config);
-u32 jfjoch_get_c2h_descriptors(struct jfjoch_drvdata *drvdata);
 void jfjoch_set_spot_finder_parameters(struct jfjoch_drvdata *drvdata, struct SpotFinderParameters *params);
 void jfjoch_get_spot_finder_parameters(struct jfjoch_drvdata *drvdata, struct SpotFinderParameters *params);
 u32 jfjoch_get_data_source(struct jfjoch_drvdata *drvdata);
@@ -236,7 +233,6 @@ void jfjoch_get_env_data(struct jfjoch_drvdata *drvdata, struct DeviceStatus *en
 
 void jfjoch_reset(struct jfjoch_drvdata *drvdata);
 
-void jfjoch_write_register(struct jfjoch_drvdata *drvdata, uint32_t addr, uint32_t val);
 uint32_t jfjoch_read_register(struct jfjoch_drvdata *drvdata, uint32_t addr);
 
 int jfjoch_register_sysfs(struct jfjoch_drvdata *drvdata);

fpga/pcie_driver/jfjoch_fpga.h

@@ -75,7 +75,7 @@ typedef __u64 uint64_t;
 #define INT24_MIN (-8388608)
 
 // INT32_MAX is written explicitly, as the constant is not present in kernel
-#define MAX_FRAMES (2147483647/(MAX_MODULES_FPGA*DMA_DESCRIPTORS_PER_MODULE))
+#define MAX_FRAMES (2147483647)
 
 #pragma pack(push)
 #pragma pack(4)

fpga/pcie_driver/jfjoch_function.c

@@ -16,13 +16,8 @@ void jfjoch_start(struct jfjoch_drvdata *drvdata) {
     mutex_lock(&drvdata->work_compl_read_mutex);
     while(!kfifo_is_empty(&drvdata->work_compl))
         tmp2 = kfifo_get(&drvdata->work_compl, &tmp1);
-
-    // Reset work completion queue
-    drvdata->work_compl_count = 0;
-
     mutex_unlock(&drvdata->work_compl_read_mutex);
 
-
     // Set PCIe beats counters
     iowrite32((1 << 1), drvdata->bar0 + PCIE_OFFSET + (1<<12) + 0xC0);
     iowrite32((1 << 2), drvdata->bar0 + PCIE_OFFSET + (1<<12) + 0xC0);
@@ -69,28 +64,14 @@ int jfjoch_send_wr(struct jfjoch_drvdata *drvdata, u32 handle) {
 }
 
 int jfjoch_read_wc(struct jfjoch_drvdata *drvdata, u32 *output) {
-    struct device *const dev = &drvdata->pdev->dev;
-
-    int ret, tmp, i;
-    u32 curr_compl_count, handle;
-    u32 data_collection_id;
+    int ret, tmp;
 
     mutex_lock(&drvdata->work_compl_read_mutex);
 
     ret = wait_event_interruptible_timeout(drvdata->work_compl_wait_queue, !kfifo_is_empty(&drvdata->work_compl), HZ);
-    if (ret > 0) {
+    if (ret > 0)
         tmp = kfifo_get(&drvdata->work_compl, output);
-        handle = (*output) & 0xFFFF;
-        data_collection_id = (*output >> 16) & 0xFFFF;
 
-        if (handle == HANDLE_START) {
-            drvdata->work_compl_count = 0;
-            curr_compl_count = 0;
-        } else if ((handle != HANDLE_END) && (data_collection_id != DATA_COLLECTION_ID_PURGE)) {
-            curr_compl_count = drvdata->work_compl_count;
-            drvdata->work_compl_count++;
-        }
-    }
     mutex_unlock(&drvdata->work_compl_read_mutex);
 
     if (ret < 0)
@@ -98,27 +79,9 @@ int jfjoch_read_wc(struct jfjoch_drvdata *drvdata, u32 *output) {
     else if ((ret == 0) || (tmp == 0))
         return -EAGAIN;
 
-    // Guarantee that data are in host memory
-    if ((handle != HANDLE_START) && (handle != HANDLE_END)) {
-        u32 descriptors = ioread32(drvdata->bar0 + PCIE_OFFSET + (1 << 12) + 0x48);
-        i = 0;
-        while ((descriptors < curr_compl_count * DMA_DESCRIPTORS_PER_MODULE) && (i < 100)) {
-            udelay(10);
-            descriptors = ioread32(drvdata->bar0 + PCIE_OFFSET + (1 << 12) + 0x48);
-            i++;
-        }
-
-        if (descriptors < DMA_DESCRIPTORS_PER_MODULE * curr_compl_count) {
-            dev_err(dev, "Late completion count SW %u, HW %u HANDLE %x", curr_compl_count, descriptors, handle);
-            return -EIO;
-        } else if (i > 0) {
-            dev_warn(dev, "Late completion count SW %u, HW %u HANDLE %x delay %d", curr_compl_count, descriptors, handle, i);
-        }
-    }
     return 0;
 }
 
-
 int jfjoch_set_config(struct jfjoch_drvdata *drvdata, const struct DataCollectionConfig *config) {
     if (config->nframes > MAX_FRAMES)
         return -EINVAL;
@@ -409,10 +372,6 @@ int jfjoch_run_frame_gen(struct jfjoch_drvdata *drvdata, struct FrameGeneratorCo
     return 0;
 }
 
-u32 jfjoch_get_c2h_descriptors(struct jfjoch_drvdata *drvdata) {
-    return ioread32(drvdata->bar0 + PCIE_OFFSET + (1<<12) + 0x48);
-}
-
 void jfjoch_set_spot_finder_parameters(struct jfjoch_drvdata *drvdata, struct SpotFinderParameters *params) {
     memcpy_toio((drvdata->bar0) + ACTION_CONFIG_OFFSET + ADDR_SPOT_FINDER_THRESHOLD,
                 params,

fpga/pcie_driver/jfjoch_ioctl.c

@@ -154,11 +154,6 @@ long jfjoch_cdev_ioctl(struct file *file, unsigned int cmd, unsigned long arg) {
             if (copy_from_user(&frame_generator_config, (char *) arg, sizeof(struct FrameGeneratorConfig)) != 0)
                 return -EFAULT;
             return jfjoch_run_frame_gen(drvdata, &frame_generator_config);
-       case IOCTL_JFJOCH_C2H_DMA_DESC:
-            exchange[0] = jfjoch_get_c2h_descriptors(drvdata);
-            if (copy_to_user((char *) arg, exchange, sizeof(u32)) != 0)
-                return -EFAULT;
-            return 0;
         case IOCTL_JFJOCH_READ_REGISTER:
             // This is the most powerful option from security point of view and can only be performed by root
             if (!capable(CAP_SYS_ADMIN))

fpga/pcie_driver/jfjoch_ioctl.h

@@ -34,8 +34,7 @@
 #define IOCTL_JFJOCH_GET_IPV4        _IOR(IOCTL_JFJOCH_MAGIC, 18, uint32_t)
 // 19-21 are reserved
 #define IOCTL_JFJOCH_RUN_FRAME_GEN   _IOW(IOCTL_JFJOCH_MAGIC, 23, struct FrameGeneratorConfig)
-#define IOCTL_JFJOCH_C2H_DMA_DESC    _IOR(IOCTL_JFJOCH_MAGIC, 24, uint32_t)
-// 25 is reserved
+// 24-25 are reserved
 #define IOCTL_JFJOCH_READ_REGISTER   _IOWR(IOCTL_JFJOCH_MAGIC, 26, struct RegisterConfig )
 #define IOCTL_JFJOCH_SET_SPOTFIN_PAR _IOW(IOCTL_JFJOCH_MAGIC, 27, struct SpotFinderParameters)
 #define IOCTL_JFJOCH_SET_DATA_SOURCE _IOW(IOCTL_JFJOCH_MAGIC, 28, uint32_t)

frame_serialize/CBORStream2Deserializer.cpp

@@ -441,9 +441,9 @@ namespace {
                 else if (key == "data")
                     ProcessImageData(value, message.image);
                 else if (key == "series_unique_id")
-                    message.series_unique_id = GetCBORString(value);
+                    message.run_name = GetCBORString(value);
                 else if (key == "series_id")
-                    message.series_id = GetCBORUInt(value);
+                    message.run_number = GetCBORUInt(value);
                 else if (key == "real_time") {
                     auto r = GetRational(value);
                     message.exptime = r.first;
@@ -452,9 +452,14 @@ namespace {
                     auto r = GetRational(value);
                     message.timestamp = r.first;
                     message.timestamp_base = r.second;
-                } else if (key == "user_data")
-                    message.user_data = GetCBORString(value);
-                else if (key == "spots")
+                } else if (key == "user_data") {
+                    const std::string s = GetCBORString(value);
+                    try {
+                        message.user_data = nlohmann::json::parse(s);
+                    } catch (...) {
+                        message.user_data = s;
+                    }
+                } else if (key == "spots")
                     GetCBORSpots(message, value);
                 else if (key == "spot_count_in_rings")
                     message.spot_count_in_rings = GetCBORUInt(value);
@@ -654,7 +659,10 @@ namespace {
                 message.experiment_group = j["experiment_group"];
             if (j.contains("jfjoch_release"))
                 message.jfjoch_release = j["jfjoch_release"];
-
+            if (j.contains("socket_number"))
+                message.socket_number = j["socket_number"];
+            if (j.contains("writer_notification_zmq_addr"))
+                message.writer_notification_zmq_addr = j["writer_notification_zmq_addr"];
         } catch (const std::exception &e) {
             throw JFJochException(JFJochExceptionCategory::CBORError,
                                   "Cannot parse user_data as valid JSON " + std::string(e.what()));
@@ -710,9 +718,9 @@ namespace {
                 else if (key == "detector_serial_number")
                     message.detector_serial_number = GetCBORString(value);
                 else if (key == "series_unique_id")
-                    message.series_unique_id = GetCBORString(value);
+                    message.run_name = GetCBORString(value);
                 else if (key == "series_id")
-                    message.series_id = GetCBORUInt(value);
+                    message.run_number = GetCBORUInt(value);
                 else if (key == "pixel_mask")
                     ProcessPixelMaskElement(message, value);
                 else if (key == "channels")
@@ -832,9 +840,9 @@ namespace {
                 if (key == "end_date")
                     message.end_date = GetCBORString(value);
                 else if (key == "series_unique_id")
-                    message.series_unique_id = GetCBORString(value);
+                    message.run_name = GetCBORString(value);
                 else if (key == "series_id")
-                    message.series_id = GetCBORUInt(value);
+                    message.run_number = GetCBORUInt(value);
                 else if (key == "max_image_number")
                     message.max_image_number = GetCBORUInt(value);
                 else if (key == "images_collected")

frame_serialize/CBORStream2Serializer.cpp

@@ -321,9 +321,13 @@ inline void CBOR_ENC_START_USER_DATA(CborEncoder& encoder, const char* key,
         j["write_master_file"] = message.write_master_file.value();
     if (message.data_reduction_factor_serialmx)
         j["data_reduction_factor_serialmx"] = message.data_reduction_factor_serialmx.value();
-    if (message.experiment_group)
-        j["experiment_group"] = message.experiment_group.value();
+    j["experiment_group"] = message.experiment_group;
     j["jfjoch_release"] = message.jfjoch_release;
+    if (message.socket_number)
+        j["socket_number"] = message.socket_number.value();
+
+    j["writer_notification_zmq_addr"] = message.writer_notification_zmq_addr;
+
     auto str = j.dump();
 
     CBOR_ENC(encoder, key, str);
@@ -372,8 +376,8 @@ void CBORStream2Serializer::SerializeSequenceStart(const StartMessage& message)
     CBOR_ENC(mapEncoder, "pixel_mask_enabled", message.pixel_mask_enabled);
     CBOR_ENC(mapEncoder, "detector_description", message.detector_description);
     CBOR_ENC(mapEncoder, "detector_serial_number", message.detector_serial_number);
-    CBOR_ENC(mapEncoder, "series_unique_id", message.series_unique_id);
-    CBOR_ENC(mapEncoder, "series_id", message.series_id);
+    CBOR_ENC(mapEncoder, "series_unique_id", message.run_name);
+    CBOR_ENC(mapEncoder, "series_id", message.run_number);
 
     CBOR_ENC_AXIS(mapEncoder, "detector_translation", message.detector_translation);
     CBOR_ENC_GONIOMETER_MAP(mapEncoder, "goniometer", message);
@@ -418,8 +422,8 @@ void CBORStream2Serializer::SerializeSequenceEnd(const EndMessage& message) {
     CBOR_ENC(mapEncoder, "type", "end");
     CBOR_ENC(mapEncoder, "magic_number", user_data_magic_number);
 
-    CBOR_ENC(mapEncoder, "series_unique_id", message.series_unique_id);
-    CBOR_ENC(mapEncoder, "series_id", message.series_id);
+    CBOR_ENC(mapEncoder, "series_unique_id", message.run_name);
+    CBOR_ENC(mapEncoder, "series_id", message.run_number);
     CBOR_ENC(mapEncoder, "end_date", message.end_date);
 
     CBOR_ENC(mapEncoder, "max_image_number", message.max_image_number);
@@ -444,8 +448,8 @@ void CBORStream2Serializer::SerializeImage(const DataMessage& message) {
 
     CBOR_ENC(mapEncoder, "type", "image");
     CBOR_ENC(mapEncoder, "magic_number", user_data_magic_number);
-    CBOR_ENC(mapEncoder, "series_unique_id", message.series_unique_id);
-    CBOR_ENC(mapEncoder, "series_id", message.series_id);
+    CBOR_ENC(mapEncoder, "series_unique_id", message.run_name);
+    CBOR_ENC(mapEncoder, "series_id", message.run_number);
     CBOR_ENC(mapEncoder, "image_id", message.number);
 
     CBOR_ENC(mapEncoder, "original_image_id", message.original_number);
@@ -474,7 +478,7 @@ void CBORStream2Serializer::SerializeImage(const DataMessage& message) {
     CBOR_ENC(mapEncoder, "bkg_estimate", message.bkg_estimate);
     CBOR_ENC(mapEncoder, "adu_histogram", message.adu_histogram);
     CBOR_ENC(mapEncoder, "roi_integrals", message.roi);
-    CBOR_ENC(mapEncoder, "user_data", message.user_data);
+    CBOR_ENC(mapEncoder, "user_data", message.user_data.dump());
     CBOR_ENC(mapEncoder, "data", message.image);
     cborErr(cbor_encoder_close_container(&encoder, &mapEncoder));
 

frame_serialize/CMakeLists.txt

@@ -21,6 +21,7 @@ ADD_LIBRARY(ImagePusher STATIC
         ImagePusher.cpp ImagePusher.h
         TestImagePusher.cpp TestImagePusher.h
         ZMQStream2Pusher.cpp ZMQStream2Pusher.h
+        ZMQWriterNotificationPuller.cpp ZMQWriterNotificationPuller.h
         DumpCBORToFilePusher.cpp
         DumpCBORToFilePusher.h)
 

frame_serialize/ImagePusher.cpp

@@ -15,3 +15,9 @@ void PrepareCBORImage(DataMessage& message,
     message.image.algorithm = experiment.GetCompressionAlgorithm();
     message.image.channel = "default";
 }
+
+void ImagePusher::Finalize() {}
+
+std::string ImagePusher::GetWriterNotificationSocketAddress() const {
+    return "socket1234";
+}

frame_serialize/ImagePusher.h

@@ -11,6 +11,7 @@
 #include "CBORStream2Serializer.h"
 #include "JFJochMessages.h"
 #include "../common/ZeroCopyReturnValue.h"
+#include "../common/Logger.h"
 
 void PrepareCBORImage(DataMessage& message,
                       const DiffractionExperiment &experiment,
@@ -24,6 +25,9 @@ public:
     virtual void SendImage(const uint8_t *image_data, size_t image_size,  int64_t image_number,
                            ZeroCopyReturnValue *z) = 0;
     virtual bool SendCalibration(const CompressedImage& message) = 0;
+    virtual void Finalize(); // Ensure that all streams are closed, can throw exception
+    virtual std::string GetWriterNotificationSocketAddress() const;
+    virtual ~ImagePusher() = default;
 };
 
 

frame_serialize/JFJochMessages.h

@@ -10,6 +10,8 @@
 #include <optional>
 #include <cstring>
 #include <stdexcept>
+#include <nlohmann/json.hpp>
+
 #include "../compression/CompressionAlgorithmEnum.h"
 #include "../common/SpotToSave.h"
 #include "../common/UnitCell.h"
@@ -62,14 +64,14 @@ struct DataMessage {
     uint32_t exptime;
     uint32_t exptime_base;
 
-    std::string series_unique_id;
-    uint64_t series_id;
+    std::string run_name;
+    uint64_t run_number;
 
     uint64_t saturated_pixel_count;
     uint64_t error_pixel_count;
     uint64_t strong_pixel_count;
 
-    std::string user_data;
+    nlohmann::json user_data;
 
     std::optional<uint64_t> jf_info;
     std::optional<uint64_t> receiver_aq_dev_delay;
@@ -138,8 +140,9 @@ struct StartMessage {
 
     std::string detector_description;
     std::string detector_serial_number;
-    std::string series_unique_id;
-    uint64_t series_id;
+    std::string run_name;
+    uint64_t run_number;
+
     std::vector<std::string> gain_file_names;
 
     std::vector<std::string> roi_names;
@@ -170,12 +173,15 @@ struct StartMessage {
         pixel_mask.emplace_back(std::move(image));
     }
 
-    std::string user_data;
+    nlohmann::json user_data;
 
     std::optional<float> data_reduction_factor_serialmx;
-    std::optional<std::string> experiment_group;
+    std::string experiment_group;
 
     std::string jfjoch_release;
+
+    std::optional<uint64_t> socket_number; // This is number of socket in ZeroMQ
+    std::string writer_notification_zmq_addr; // Socket to inform detector on writer done
 };
 
 struct EndMessage {
@@ -187,8 +193,8 @@ struct EndMessage {
 
     std::optional<std::string> end_date;
 
-    std::string series_unique_id;
-    uint64_t series_id;
+    std::string run_name;
+    uint64_t run_number;
 
     std::map<std::string, std::vector<float>> az_int_result;
     std::map<std::string, std::vector<uint64_t>> adu_histogram;

frame_serialize/README.md

@@ -13,98 +13,101 @@ There are minor differences at the moment:
 
 ## Start message
 
-| Field name                | Type                        | Description                                                                                                                       | Present in DECTRIS format |
-|---------------------------|-----------------------------|-----------------------------------------------------------------------------------------------------------------------------------|:-------------------------:|
-| type                      | String                      | value "start"                                                                                                                     |             X             |
-| magic_number              | uint64                      | Number used to describe version of the Jungfraujoch data interface - to allow to detect inconsistency between sender and receiver |                           |
-| detector_distance         | float                       | Detector distance \[m\]                                                                                                           |                           |
-| detector_translation      | float                       | Detector translation vector \[m\]                                                                                                 |             X             |
-| beam_center_x             | float                       | Beam center in X direction \[pixels\]                                                                                             |             X             |
-| beam_center_y             | float                       | Beam center in Y direction \[pixels\]                                                                                             |             X             |
-| number_of_images          | uint64                      | Number of images in the series                                                                                                    |             X             |
-| image_size_x              | uint64                      | Image width \[pixels\]                                                                                                            |             X             |
-| image_size_y              | uint64                      | Image height \[pixels\]                                                                                                           |             X             |
-| incident_energy           | float                       | X-ray energy \[eV\]                                                                                                               |             X             |
-| incident_wavelength       | float                       | X-ray wavelength \[Angstrom\]                                                                                                     |             X             |
-| frame_time                | float                       | Frame time, if multiple frames per trigger \[s\]                                                                                  |             X             |
-| count_time                | float                       | Exposure time \[s\]                                                                                                               |             X             |
-| saturation_value          | int64                       | Maximum valid sample value                                                                                                        |             X             |
-| error_value               | int64                       | Value used in images to describe pixels that are in error state or missing                                                        |                           |
-| pixel_size_x              | float                       | Pixel width \[m\]                                                                                                                 |             X             |
-| pixel_size_y              | float                       | Pixel height \[m\]                                                                                                                |             X             |
-| sensor_thickness          | float                       | Sensor thickness \[m\]                                                                                                            |             X             |
-| sensor_material           | string                      | Sensor material                                                                                                                   |             X             |
-| arm_data                  | data                        | Approximate date of arming                                                                                                        |             X             |
-| pixel_mask_enabled        | bool                        | Pixel mask applied on images                                                                                                      |             X             |
-| detector_description      | string                      | Name of the detector                                                                                                              |             X             |
-| series_unique_id          | string                      | Unique text ID of the series                                                                                                      |             X             |
-| series_id                 | uint64                      | Unique numeric ID of the series                                                                                                   |             X             |
-| goniometer                | Map                         | See [DECTRIS documentation](https://github.com/dectris/documentation/tree/main/stream_v2) - only one axis is supported            |             X             |
-| pixel_mask                | Map(string -> Image)        | Pixel mask - multiple in case of storage cells                                                                                    |             X             |
-| channels                  | Array(string)               | List of image channels                                                                                                            |             X             |
-| max_spot_count            | uint64                      | Maximum number of spots identified in spot finding                                                                                |                           |
-| storage_cell_number       | uint64                      | Number of storage cells used by JUNGFRAU                                                                                          |                           |
-| image_dtype               | string                      | Pixel bit type (e.g. uint16)                                                                                                      |             X             |
-| unit_cell                 | Array(6 * float) (optional) | Unit cell of the system: a, b, c \[angstrom\] and alpha, beta, gamma \[degree\]                                                   |                           |
-| az_int_bin_number         | uint64                      | Number of azimuthal integration bins                                                                                              |                           |
-| az_int_bin_to_q           | Array(float)                | Q value for each azimuthal integration bin \[angstrom^-1\]                                                                        |                           |
-| summation                 | uint64                      | Factor of frame summation                                                                                                         |                           |
-| user_data                 | string                      | JSON string that can contain the following fields (among others):                                                                 |             X             |
-| - source_name             | string                      | Facility name                                                                                                                     |                           |
-| - source_name_short       | string                      | Facility name (short)                                                                                                             |                           |
-| - source_type             | string (optional)           | Type of X-ray source (use NXsource/type values, for example "Synchrotron X-ray Source" or "Free-Electron Laser")                  |                           |
-| - instrument_name         | string                      | Instrument name                                                                                                                   |                           |
-| - instrument_name_short   | string                      | Instrument name (short)                                                                                                           |                           |
-| - attenuator_transmission | float (optional)            | Attenuator transmission \[\]                                                                                                      |                           |
-| - total_flux              | float (optional)            | Total flux \[ph/s\]                                                                                                               |                           |
-| - data_file_count         | uint64                      | Number of used data files                                                                                                         |                           |
-| - file_prefix             | string                      | File prefix                                                                                                                       |                           |
-| - sample_name             | string                      | Name of the sample                                                                                                                |                           |
-| - rotation_axis           | Array(float)                | Rotation axis vector (array of size 3)                                                                                            |                           |
-| - space_group_number      | uint64                      | Space group number                                                                                                                |                           |
-| - write_master_file       | bool                        | With multiple sockets, it selects which socket will provide master file                                                           |                           |
-| - experiment_group        | string                      | ID of instrument user, e.g., p-group (SLS/SwissFEL) or proposal number                                                            |                           |
+| Field name                     | Type                        | Description                                                                                                                       | Present in DECTRIS format |
+|--------------------------------|-----------------------------|-----------------------------------------------------------------------------------------------------------------------------------|:-------------------------:|
+| type                           | String                      | value "start"                                                                                                                     |             X             |
+| magic_number                   | uint64                      | Number used to describe version of the Jungfraujoch data interface - to allow to detect inconsistency between sender and receiver |                           |
+| detector_distance              | float                       | Detector distance \[m\]                                                                                                           |                           |
+| detector_translation           | float                       | Detector translation vector \[m\]                                                                                                 |             X             |
+| beam_center_x                  | float                       | Beam center in X direction \[pixels\]                                                                                             |             X             |
+| beam_center_y                  | float                       | Beam center in Y direction \[pixels\]                                                                                             |             X             |
+| number_of_images               | uint64                      | Number of images in the series                                                                                                    |             X             |
+| image_size_x                   | uint64                      | Image width \[pixels\]                                                                                                            |             X             |
+| image_size_y                   | uint64                      | Image height \[pixels\]                                                                                                           |             X             |
+| incident_energy                | float                       | X-ray energy \[eV\]                                                                                                               |             X             |
+| incident_wavelength            | float                       | X-ray wavelength \[Angstrom\]                                                                                                     |             X             |
+| frame_time                     | float                       | Frame time, if multiple frames per trigger \[s\]                                                                                  |             X             |
+| count_time                     | float                       | Exposure time \[s\]                                                                                                               |             X             |
+| saturation_value               | int64                       | Maximum valid sample value                                                                                                        |             X             |
+| error_value                    | int64                       | Value used in images to describe pixels that are in error state or missing                                                        |                           |
+| pixel_size_x                   | float                       | Pixel width \[m\]                                                                                                                 |             X             |
+| pixel_size_y                   | float                       | Pixel height \[m\]                                                                                                                |             X             |
+| sensor_thickness               | float                       | Sensor thickness \[m\]                                                                                                            |             X             |
+| sensor_material                | string                      | Sensor material                                                                                                                   |             X             |
+| arm_data                       | data                        | Approximate date of arming                                                                                                        |             X             |
+| pixel_mask_enabled             | bool                        | Pixel mask applied on images                                                                                                      |             X             |
+| detector_description           | string                      | Name of the detector                                                                                                              |             X             |
+| series_unique_id               | string                      | Unique text ID of the series (run_name parameter)                                                                                 |             X             |
+| series_id                      | uint64                      | Unique numeric ID of the series (run_number parameter)                                                                            |             X             |
+| goniometer                     | Map                         | See [DECTRIS documentation](https://github.com/dectris/documentation/tree/main/stream_v2) - only one axis is supported            |             X             |
+| pixel_mask                     | Map(string -> Image)        | Pixel mask - multiple in case of storage cells                                                                                    |             X             |
+| channels                       | Array(string)               | List of image channels                                                                                                            |             X             |
+| max_spot_count                 | uint64                      | Maximum number of spots identified in spot finding                                                                                |                           |
+| storage_cell_number            | uint64                      | Number of storage cells used by JUNGFRAU                                                                                          |                           |
+| image_dtype                    | string                      | Pixel bit type (e.g. uint16)                                                                                                      |             X             |
+| unit_cell                      | Array(6 * float) (optional) | Unit cell of the system: a, b, c \[angstrom\] and alpha, beta, gamma \[degree\]                                                   |                           |
+| az_int_bin_number              | uint64                      | Number of azimuthal integration bins                                                                                              |                           |
+| az_int_bin_to_q                | Array(float)                | Q value for each azimuthal integration bin \[angstrom^-1\]                                                                        |                           |
+| summation                      | uint64                      | Factor of frame summation                                                                                                         |                           |
+| user_data                      | string                      | JSON serialized to string that can contain the following fields:                                                                  |             X             |
+| - user                         | any valid JSON              | Value of header_appendix provided at collection start to Jungfraujoch                                                             |                           |
+| - source_name                  | string                      | Facility name                                                                                                                     |                           |
+| - source_name_short            | string                      | Facility name (short)                                                                                                             |                           |
+| - source_type                  | string (optional)           | Type of X-ray source (use NXsource/type values, for example "Synchrotron X-ray Source" or "Free-Electron Laser")                  |                           |
+| - instrument_name              | string                      | Instrument name                                                                                                                   |                           |
+| - instrument_name_short        | string                      | Instrument name (short)                                                                                                           |                           |
+| - attenuator_transmission      | float (optional)            | Attenuator transmission \[\]                                                                                                      |                           |
+| - total_flux                   | float (optional)            | Total flux \[ph/s\]                                                                                                               |                           |
+| - data_file_count              | uint64                      | Number of used data files                                                                                                         |                           |
+| - file_prefix                  | string                      | File prefix                                                                                                                       |                           |
+| - sample_name                  | string                      | Name of the sample                                                                                                                |                           |
+| - rotation_axis                | Array(float)                | Rotation axis vector (array of size 3)                                                                                            |                           |
+| - space_group_number           | uint64                      | Space group number                                                                                                                |                           |
+| - write_master_file            | bool                        | With multiple sockets, it selects which socket will provide master file                                                           |                           |
+| - writer_notification_zmq_addr | string                      | ZeroMQ address to inform `jfjoch_broker` about writers that finished operation                                                    |                           |
+| - socket_number                | uint64                      | Number of ZeroMQ socket (on `jfjoch_broker` side) used for transmission                                                           |                           |
+| - experiment_group             | string                      | ID of instrument user, e.g., p-group (SLS/SwissFEL) or proposal number                                                            |                           |
 
 See [DECTRIS documentation](https://github.com/dectris/documentation/tree/main/stream_v2) for definition of Image as MultiDimArray with optional compression.
 
 ## Image message
 
-| Field name                 | Type                        | Description                                                                                                                    | Present in DECTRIS format | Optional |
-|----------------------------|-----------------------------|--------------------------------------------------------------------------------------------------------------------------------|:-------------------------:|:--------:|
-| type                       | String                      | value "image"                                                                                                                  |             X             |          |
+| Field name                 | Type                        | Description                                                                                                                       | Present in DECTRIS format | Optional |
+|----------------------------|-----------------------------|-----------------------------------------------------------------------------------------------------------------------------------|:-------------------------:|:--------:|
+| type                       | String                      | value "image"                                                                                                                     |             X             |          |
 | magic_number               | uint64                      | Number used to describe version of the Jungfraujoch data interface - to allow to detect inconsistency between sender and receiver |                           |          |
-| series_unique_id           | string                      | Unique text ID of the series                                                                                                   |             X             |          |
-| series_id                  | uint64                      | Unique numeric ID of the series                                                                                                |             X             |          |
-| image_id                   | uint64                      | Number of image within the series                                                                                              |             X             |          |
-| real_time                  | Rational                    | Exposure time                                                                                                                  |             X             |          |
-| start_time                 | Rational                    | Exposure start time (highly approximate)                                                                                       |             X             |          |
-| end_time                   | Rational                    | Exposure end time (highly approximate)                                                                                         |             X             |          |
-| spots                      | Array(object)               | Spots:                                                                                                                         |                           |          |
-| - x                        | float                       | position in x (pixels)                                                                                                         |                           |          |
-| - y                        | float                       | position in y (pixels)                                                                                                         |                           |          |
-| - I                        | float                       | intensity (photons)                                                                                                            |                           |          |
-| - indexed                  | bool                        | indexed solution                                                                                                               |                           |          |
-| az_int_profile             | Array(float)                | Azimuthal integration results, use az_int_bin_to_q from start message for legend                                               |                           |          |
-| indexing_result            | bool                        | Indexing successful                                                                                                            |                           |          |
-| indexing_lattice           | Array(float * 9) (optional) | Indexing result real lattice; present only if indexed                                                                          |                           |          |
-| xfel_pulse_id              | uint64                      | Bunch ID (for pulsed source, e.g., SwissFEL)                                                                                   |                           |    X     |
-| xfel_event_code            | uint64                      | Event code (for pulsed source, e.g., SwissFEL)                                                                                 |                           |    X     |
-| jf_info                    | uint64                      | Detector info field                                                                                                            |                           |          |
-| receiver_aq_dev_delay      | uint64                      | Receiver internal delay                                                                                                        |                           |          |
-| receiver_free_send_buf     | uint64                      | Receiver internal number of available send buffers                                                                             |                           |          |
-| storage_cell               | uint64                      | Storage cell number                                                                                                            |                           |          |
-| saturated_pixel_count      | uint64                      | Saturated pixel count                                                                                                          |                           |          |
-| error_pixel_count          | uint64                      | Error pixel count                                                                                                              |                           |          |
-| strong_pixel_count         | uint64                      | Strong pixel count (first stage of spot finding)                                                                               |                           |          |
-| data_collection_efficiency | float                       | Image collection efficiency \[\]                                                                                               |                           |          |
-| bkg_estimate               | float                       | Mean value for pixels in resolution range from 3.0 to 5.0 A [\photons\]                                                        |                           |          |
-| adu_histogram              | Array(uint64)               | ADU histogram                                                                                                                  |                           |          |
-| corr_beam_x_pxl            | float                       | Difference between provided and calculated beam center X (pixels) - requires indexing                                          |                           |    X     |
-| corr_beam_y_pxl            | float                       | Difference between provided and calculated beam center Y (pixels) - requires indexing                                          |                           |    X     |
-| corr_det_dist_mm           | float                       | Difference between provided and calculated detector distance (mm) - requires indexing                                          |                           |    X     |
-| user_data                  | string                      | Optional user defined text information                                                                                         |                           |          |
-| roi_sum                    | int64                       | Sum of ROI rectangle \[photons\]                                                                                               |                           |    X     |
-| image                      | Map(string -> Image)        | Image                                                                                                                          |                           |          |
+| series_unique_id           | string                      | Unique text ID of the series (run_name parameter)                                                                                 |             X             |          |
+| series_id                  | uint64                      | Unique numeric ID of the series (run_number parameter)                                                                            |             X             |          |
+| image_id                   | uint64                      | Number of image within the series                                                                                                 |             X             |          |
+| real_time                  | Rational                    | Exposure time                                                                                                                     |             X             |          |
+| start_time                 | Rational                    | Exposure start time (highly approximate)                                                                                          |             X             |          |
+| end_time                   | Rational                    | Exposure end time (highly approximate)                                                                                            |             X             |          |
+| spots                      | Array(object)               | Spots:                                                                                                                            |                           |          |
+| - x                        | float                       | position in x (pixels)                                                                                                            |                           |          |
+| - y                        | float                       | position in y (pixels)                                                                                                            |                           |          |
+| - I                        | float                       | intensity (photons)                                                                                                               |                           |          |
+| - indexed                  | bool                        | indexed solution                                                                                                                  |                           |          |
+| az_int_profile             | Array(float)                | Azimuthal integration results, use az_int_bin_to_q from start message for legend                                                  |                           |          |
+| indexing_result            | bool                        | Indexing successful                                                                                                               |                           |          |
+| indexing_lattice           | Array(float * 9) (optional) | Indexing result real lattice; present only if indexed                                                                             |                           |          |
+| xfel_pulse_id              | uint64                      | Bunch ID (for pulsed source, e.g., SwissFEL)                                                                                      |                           |    X     |
+| xfel_event_code            | uint64                      | Event code (for pulsed source, e.g., SwissFEL)                                                                                    |                           |    X     |
+| jf_info                    | uint64                      | Detector info field                                                                                                               |                           |          |
+| receiver_aq_dev_delay      | uint64                      | Receiver internal delay                                                                                                           |                           |          |
+| receiver_free_send_buf     | uint64                      | Receiver internal number of available send buffers                                                                                |                           |          |
+| storage_cell               | uint64                      | Storage cell number                                                                                                               |                           |          |
+| saturated_pixel_count      | uint64                      | Saturated pixel count                                                                                                             |                           |          |
+| error_pixel_count          | uint64                      | Error pixel count                                                                                                                 |                           |          |
+| strong_pixel_count         | uint64                      | Strong pixel count (first stage of spot finding)                                                                                  |                           |          |
+| data_collection_efficiency | float                       | Image collection efficiency \[\]                                                                                                  |                           |          |
+| bkg_estimate               | float                       | Mean value for pixels in resolution range from 3.0 to 5.0 A [\photons\]                                                           |                           |          |
+| adu_histogram              | Array(uint64)               | ADU histogram                                                                                                                     |                           |          |
+| corr_beam_x_pxl            | float                       | Difference between provided and calculated beam center X (pixels) - requires indexing                                             |                           |    X     |
+| corr_beam_y_pxl            | float                       | Difference between provided and calculated beam center Y (pixels) - requires indexing                                             |                           |    X     |
+| corr_det_dist_mm           | float                       | Difference between provided and calculated detector distance (mm) - requires indexing                                             |                           |    X     |
+| user_data                  | string                      | Optional user defined text information - this is image_appendix serialized to JSON format                                         |                           |          |
+| roi_sum                    | int64                       | Sum of ROI rectangle \[photons\]                                                                                                  |                           |    X     |
+| image                      | Map(string -> Image)        | Image                                                                                                                             |                           |          |
 
 ## End message
 
@@ -112,8 +115,8 @@ See [DECTRIS documentation](https://github.com/dectris/documentation/tree/main/s
 |----------------------------|--------------------------|-----------------------------------------------------------------------------------------------------------------------------------|:-------------------------:|
 | type                       | String                   | value "end"                                                                                                                       |             X             |
 | magic_number               | uint64                   | Number used to describe version of the Jungfraujoch data interface - to allow to detect inconsistency between sender and receiver |                           |
-| series_unique_id           | string                   | Unique text ID of the series                                                                                                      |             X             |
-| series_id                  | uint64                   | Unique numeric ID of the series                                                                                                   |             X             |
+| series_unique_id           | string                   | Unique text ID of the series (run_name parameter)                                                                                 |             X             |
+| series_id                  | uint64                   | Unique numeric ID of the series (run_number parameter)                                                                            |             X             |
 | end_date                   | date                     | Approximate end date                                                                                                              |                           |
 | max_image_number           | uint64                   | Number of image with the highest number (this is counted from 1 - to distinguish zero images and one image)                       |                           |
 | images_collected           | uint64                   | Number of image collected                                                                                                         |                           |
@@ -132,20 +135,21 @@ See [DECTRIS documentation](https://github.com/dectris/documentation/tree/main/s
 | magic_number | uint64               | Number used to describe version of the Jungfraujoch data interface - to allow to detect inconsistency between sender and receiver |                           |
 | data         | Map(string -> Image) | Calibration map (only single pedestal array per message)                                                                          |                           |
 
+# User data
+In many cases there is an interest from facilities to forward more metadata, than available explicitly in the Jungfraujoch. 
+For this reason two fields can be provided: `header_appendix` (sent with start message) and `image_appendix` (send with image message).
+To increase flexibility, both appendices can contain any valid JSON message.
+These appendices are serialized into string and stored in CBOR messages as `user_data`. 
+
+Notably for start message, `user_data` contain more information (non-DECTRIS compliant metadata). 
+Therefore `user_data` is serialized by Jungfraujoch as CBOR object. There is member `user` which contains `header_appendix`.
+
 # Image transfer
 
-Images can be forwarded by Jungraujoch through few interfaces. All of them are ZeroMQ based at the moment, though they differ with serialization layer.
-All interfaces are subclasses of `ImagePusher` class. The following options are available:
+Images can be forwarded by Jungraujoch through ZeroMQ interfaces with serialization compatible with DECTRIS Stream2 serialization. 
+This is using PUSH ZeroMQ socket(s). Multiple sockets can be used, in this case images will be split according to file number to which they belong. All sockets will forward start and end messages. Only first socket will forward calibration messages.
 
-## ZMQStream2Pusher
-Pushes images using Stream2 serialization. This is using PUSH ZeroMQ sockets. Behavior is as following:
-* Start message is sent with timeout of 5s. If within the time the message cannot be put in the outgoing queue or there is no connected puller exception is thrown - this is to stop data collection in absence of writer.
-* Images are sent in non-blocking way and without timeout. If message cannot be sent `false` is returned for the send call - need to be reported or handled in calling function.
-* End message is sent with timeout of 5s. If message cannot be sent `false` is returned for the send call - need to be reported or handled in calling function.
-
-## ZMQStream2PusherGroup
-Same as above, but multiple ZeroMQ sockets are used and images are distributed on a round-robin basis between the sockets based on the image number.
-Behavior for errors is the same as for `ZMQStream2Pusher`.
-
-## TestImagePusher
-This interface is used for automated tests. It reads incoming images and compares one image with the reference data provided. 
+Behavior is as following:
+* Start message is sent with timeout of 5s. If within the time the message cannot be put in the outgoing queue or there is no connected puller exception is thrown - stop data collection with error due to absence of a writer.
+* Images are sent in non-blocking way and without timeout. 
+* End message is sent with timeout of 5s. No error is reported 

frame_serialize/ZMQStream2Pusher.cpp

@@ -55,17 +55,17 @@ void ZMQStream2Pusher::StartDataCollection(StartMessage& message) {
         throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
                               "Images per file cannot be zero or negative");
     images_per_file = message.images_per_file;
+    run_number = message.run_number;
+    run_name = message.run_name;
 
-    serializer.SerializeSequenceStart(message);
-
-    for (auto &s: socket) {
-        if (!s->Send(serialization_buffer.data(), serializer.GetBufferSize(), true))
-            throw JFJochException(JFJochExceptionCategory::ZeroMQ, "Timeout on pushing start message on addr "
-                                                                   + s->GetEndpointName());
-        if (message.write_master_file) {
+    for (int i = 0; i < socket.size(); i++) {
+        message.socket_number = i;
+        if (i > 0)
             message.write_master_file = false;
-            serializer.SerializeSequenceStart(message);
-        }
+        serializer.SerializeSequenceStart(message);
+        if (!socket[i]->Send(serialization_buffer.data(), serializer.GetBufferSize(), true))
+            throw JFJochException(JFJochExceptionCategory::ZeroMQ, "Timeout on pushing start message on addr "
+                                                                   + socket[i]->GetEndpointName());
     }
 
     if (preview_socket)
@@ -115,3 +115,36 @@ std::string ZMQStream2Pusher::GetPreviewAddress() {
     else
         return "";
 }
+
+void ZMQStream2Pusher::Finalize() {
+    if (writer_notification_socket) {
+        for (int i = 0; i < socket.size(); i++) {
+            auto n = writer_notification_socket->Receive(run_number, run_name);
+            if (!n)
+                throw JFJochException(JFJochExceptionCategory::FileWriteError, "No notification received from writer");
+            else if (n->socket_number >= socket.size())
+                throw JFJochException(JFJochExceptionCategory::FileWriteError, "Wrong socket number provided in the message");
+            else if (!n->ok)
+                throw JFJochException(JFJochExceptionCategory::FileWriteError, "Writer (socket "
+                                                                               + socket[n->socket_number]->GetEndpointName()
+                                                                               + ") finished with error");
+        }
+    }
+}
+
+std::string ZMQStream2Pusher::GetWriterNotificationSocketAddress() const {
+    if (writer_notification_socket)
+        return writer_notification_socket->GetEndpointName();
+    else
+        return "";
+}
+
+ZMQStream2Pusher &ZMQStream2Pusher::WriterNotificationSocket(const std::string &addr) {
+    writer_notification_socket = std::make_unique<ZMQWriterNotificationPuller>(addr);
+    return *this;
+}
+
+ZMQStream2Pusher &ZMQStream2Pusher::PreviewCounterPeriod(std::chrono::microseconds input) {
+    preview_counter.Period(input);
+    return *this;
+}

frame_serialize/ZMQStream2Pusher.h

@@ -8,6 +8,7 @@
 #include "ImagePusher.h"
 #include "../common/ZMQWrappers.h"
 #include "../preview/PreviewCounter.h"
+#include "ZMQWriterNotificationPuller.h"
 
 class ZMQStream2Pusher : public ImagePusher {
     std::vector<uint8_t> serialization_buffer;
@@ -18,13 +19,19 @@ class ZMQStream2Pusher : public ImagePusher {
     std::unique_ptr<ZMQSocket> preview_socket;
     PreviewCounter preview_counter;
 
+    std::unique_ptr<ZMQWriterNotificationPuller> writer_notification_socket;
+
     int64_t images_per_file = 1;
+    uint64_t run_number = 0;
+    std::string run_name;
 public:
     explicit ZMQStream2Pusher(const std::vector<std::string>& addr,
                               int32_t send_buffer_high_watermark = -1,
                               int32_t send_buffer_size = -1);
 
     ZMQStream2Pusher& PreviewSocket(const std::string& addr);
+    ZMQStream2Pusher& WriterNotificationSocket(const std::string& addr);
+    ZMQStream2Pusher& PreviewCounterPeriod(std::chrono::microseconds input);
     std::string GetPreviewAddress();
 
     std::vector<std::string> GetAddress();
@@ -37,6 +44,9 @@ public:
     // Thread-safe
     void SendImage(const uint8_t *image_data, size_t image_size, int64_t image_number, ZeroCopyReturnValue *z) override;
     bool SendImage(const uint8_t *image_data, size_t image_size, int64_t image_number) override;
+
+    void Finalize() override;
+    std::string GetWriterNotificationSocketAddress() const override;
 };
 
 #endif //JUNGFRAUJOCH_ZMQSTREAM2PUSHER_H

frame_serialize/ZMQWriterNotificationPuller.cpp

@@ -0,0 +1,34 @@
+// Copyright (2019-2024) Paul Scherrer Institute
+
+#include "ZMQWriterNotificationPuller.h"
+#include "nlohmann/json.hpp"
+
+ZMQWriterNotificationPuller::ZMQWriterNotificationPuller(const std::string &addr, std::chrono::milliseconds timeout)
+        : socket(ZMQSocketType::Pull) {
+    socket.ReceiveTimeout(timeout);
+    socket.Bind(addr);
+}
+
+std::optional<ZMQWriterNotificationOutput> ZMQWriterNotificationPuller::Receive(uint64_t run_number, const std::string &run_name) {
+    ZMQMessage msg;
+    // Loop to ensure that messages with wrong run_number or run_name are filtered
+    while (socket.Receive(msg)) {
+        try {
+            nlohmann::json j = nlohmann::json::parse(std::string((char *) msg.data(), msg.size()));
+            if ((j["run_number"] == run_number) && (j["run_name"] == run_name)) {
+                ZMQWriterNotificationOutput ret{};
+                ret.ok = j["ok"];
+                ret.processed_images = j["processed_images"];
+                ret.socket_number = j["socket_number"];
+                return ret;
+            }
+        } catch (const std::exception &e) {
+            // Cannot parse properly
+        }
+    }
+    return {};
+}
+
+std::string ZMQWriterNotificationPuller::GetEndpointName() {
+    return socket.GetEndpointName();
+}

frame_serialize/ZMQWriterNotificationPuller.h

@@ -0,0 +1,28 @@
+// Copyright (2019-2024) Paul Scherrer Institute
+
+// Copyright (2019-2024) Paul Scherrer Institute
+
+#ifndef JFJOCH_ZMQWRITERNOTIFICATIONPULLER_H
+#define JFJOCH_ZMQWRITERNOTIFICATIONPULLER_H
+
+#include <optional>
+
+#include "../common/ZMQWrappers.h"
+
+struct ZMQWriterNotificationOutput {
+    bool ok;
+    uint64_t socket_number;
+    uint64_t processed_images;
+};
+
+class ZMQWriterNotificationPuller {
+    ZMQSocket socket;
+
+public:
+    ZMQWriterNotificationPuller(const std::string& addr, std::chrono::milliseconds timeout = std::chrono::seconds(15));
+    std::string GetEndpointName();
+    std::optional<ZMQWriterNotificationOutput> Receive(uint64_t run_number, const std::string &run_name);
+};
+
+
+#endif //JFJOCH_ZMQWRITERNOTIFICATIONPULLER_H

image_analysis/MXAnalyzer.cpp

@@ -4,6 +4,7 @@
 #include "CPUSpotFinder.h"
 #include "../common/DiffractionGeometry.h"
 #include "Regression.h"
+#include "../common/CUDAWrapper.h"
 
 double stddev(const std::vector<float> &v) {
     if (v.size() <= 1)
@@ -26,7 +27,7 @@ double stddev(const std::vector<float> &v) {
 MXAnalyzer::MXAnalyzer(const DiffractionExperiment &in_experiment)
 : experiment(in_experiment) {
     auto uc = experiment.GetUnitCell();
-    if (uc) {
+    if (uc && (get_gpu_count() > 0)) {
         try {
             indexer = std::make_unique<IndexerWrapper>();
             indexer->Setup(uc.value());

preview/PreviewCounter.cpp

@@ -18,3 +18,8 @@ bool PreviewCounter::GeneratePreview() {
     } else
         return false;
 }
+
+PreviewCounter &PreviewCounter::Period(std::chrono::microseconds in_period) {
+    period = in_period;
+    return *this;
+}

preview/PreviewCounter.h

@@ -14,6 +14,7 @@ class PreviewCounter {
 public:
     PreviewCounter(std::chrono::microseconds period);
     bool GeneratePreview();
+    PreviewCounter& Period(std::chrono::microseconds period);
 };
 
 #endif //JUNGFRAUJOCH_PREVIEWCOUNTER_H

receiver/JFJochReceiver.cpp

@@ -9,6 +9,7 @@
 #include "ImageMetadata.h"
 #include "../common/time_utc.h"
 #include "../common/PixelMask.h"
+#include "../common/CUDAWrapper.h"
 
 JFJochReceiver::JFJochReceiver(const DiffractionExperiment& in_experiment,
                                const PixelMask &pixel_mask,
@@ -55,6 +56,11 @@ JFJochReceiver::JFJochReceiver(const DiffractionExperiment& in_experiment,
 
     logger.Info("NUMA policy: {}", numa_policy.GetName());
 
+#ifdef JFJOCH_USE_CUDA
+    if (get_gpu_count() == 0)
+        logger.Warning("GPU absent, indexing is not possible");
+#endif
+
     for (int d = 0; d < ndatastreams; d++) {
         acquisition_device[d].PrepareAction(experiment);
         acquisition_device[d].SetSpotFinderParameters(spot_finding_settings);
@@ -102,6 +108,7 @@ void JFJochReceiver::SendStartMessage(const PixelMask &pixel_mask) {
     message.az_int_bin_number = az_int_mapping.GetBinNumber();
     message.az_int_bin_to_q = az_int_mapping.GetBinToQ();
     message.write_master_file = true;
+    message.writer_notification_zmq_addr = image_pusher.GetWriterNotificationSocketAddress();
 
     std::vector<uint32_t> nexus_mask;
     nexus_mask = pixel_mask.GetMask(experiment);
@@ -297,8 +304,8 @@ void JFJochReceiver::FrameTransformationThread(uint32_t threadid) {
             message.number = image_number;
             message.original_number = image_number;
             message.user_data = experiment.GetImageAppendix();
-            message.series_id = experiment.GetRunNumber();
-            message.series_unique_id = experiment.GetRunName();
+            message.run_number = experiment.GetRunNumber();
+            message.run_name = experiment.GetRunName();
 
             ImageMetadata metadata(experiment);
 
@@ -452,8 +459,8 @@ void JFJochReceiver::FinalizeMeasurement() {
         message.max_receiver_delay = max_delay;
         message.efficiency = GetEfficiency();
         message.end_date = time_UTC(std::chrono::system_clock::now());
-        message.series_id = experiment.GetRunNumber();
-        message.series_unique_id = experiment.GetRunName();
+        message.run_number = experiment.GetRunNumber();
+        message.run_name = experiment.GetRunName();
 
         message.az_int_result["dataset"] = plots.GetAzIntProfile();
         for (int i = 0; i < experiment.GetTimePointNumber(); i++)
@@ -487,6 +494,10 @@ void JFJochReceiver::FinalizeMeasurement() {
         throw JFJochException(JFJochExceptionCategory::ZeroMQ, "Send commands not finalized in 10 seconds");
     }
     logger.Info("Receiving data done");
+
+    image_pusher.Finalize();
+
+    logger.Info("Writing process finalized");
 }
 
 void JFJochReceiver::SetSpotFindingSettings(const SpotFindingSettings &in_spot_finding_settings) {

tests/CBORTest.cpp

@@ -44,8 +44,8 @@ TEST_CASE("CBORSerialize_Start", "[CBOR]") {
             .channels = {"default", "sc2"},
             .detector_description = "EIGER 16M",
             .detector_serial_number = "123",
-            .series_unique_id = "bla",
-            .series_id = 4567,
+            .run_name = "bla",
+            .run_number = 4567,
             .gain_file_names = {"abc" , "def", "/dsadasdsa/dadsadas/dsadsa/M056.bin"},
             .roi_names = {"roi0", "roi1"},
             .goniometer = GoniometerAxis{
@@ -66,10 +66,12 @@ TEST_CASE("CBORSerialize_Start", "[CBOR]") {
             .total_flux = 123,
             .attenuator_transmission = 0.345,
             .write_master_file = true,
-            .user_data = "Some random string 12345",
+            .user_data = R"({"pi":3.1415, "z":"string"})"_json,
             .data_reduction_factor_serialmx = 0.75,
             .experiment_group = "p10001",
-            .jfjoch_release = "1.4.98"
+            .jfjoch_release = "1.4.98",
+            .socket_number = 3,
+            .writer_notification_zmq_addr = "tcp://1.2.3.4:5678"
     };
 
     REQUIRE_NOTHROW(serializer.SerializeSequenceStart(message));
@@ -110,8 +112,8 @@ TEST_CASE("CBORSerialize_Start", "[CBOR]") {
     CHECK(output_message.channels == message.channels);
     CHECK(output_message.detector_description == message.detector_description);
     CHECK(output_message.detector_serial_number == message.detector_serial_number);
-    CHECK(output_message.series_unique_id == message.series_unique_id);
-    CHECK(output_message.series_id == message.series_id);
+    CHECK(output_message.run_name == message.run_name);
+    CHECK(output_message.run_number == message.run_number);
     CHECK(output_message.source_type == message.source_type);
     CHECK(output_message.source_name == message.source_name);
     CHECK(output_message.source_name_short == message.source_name_short);
@@ -143,6 +145,8 @@ TEST_CASE("CBORSerialize_Start", "[CBOR]") {
     REQUIRE (output_message.attenuator_transmission);
     CHECK(output_message.attenuator_transmission.value() == message.attenuator_transmission.value());
     CHECK(output_message.user_data == message.user_data);
+    REQUIRE(output_message.user_data.is_object());
+    CHECK(output_message.user_data.size() == 2);
     CHECK(output_message.gain_file_names == message.gain_file_names);
     CHECK(output_message.roi_names == message.roi_names);
     CHECK(output_message.countrate_correction_enabled == message.countrate_correction_enabled);
@@ -151,6 +155,8 @@ TEST_CASE("CBORSerialize_Start", "[CBOR]") {
     CHECK(output_message.data_reduction_factor_serialmx == message.data_reduction_factor_serialmx);
     CHECK(output_message.experiment_group == message.experiment_group);
     CHECK(output_message.jfjoch_release == message.jfjoch_release);
+    CHECK(output_message.socket_number == message.socket_number);
+    CHECK(output_message.writer_notification_zmq_addr == message.writer_notification_zmq_addr);
 }
 
 TEST_CASE("CBORSerialize_Start_PixelMask", "[CBOR]") {
@@ -239,8 +245,8 @@ TEST_CASE("CBORSerialize_End", "[CBOR]") {
             .max_receiver_delay = 3456,
             .efficiency = 0.99,
             .end_date = "ccc",
-            .series_unique_id = "bla5",
-            .series_id = 45676782
+            .run_name = "bla5",
+            .run_number = 45676782
     };
 
     REQUIRE_NOTHROW(serializer.SerializeSequenceEnd(message));
@@ -258,8 +264,8 @@ TEST_CASE("CBORSerialize_End", "[CBOR]") {
     REQUIRE(output_message.efficiency);
     REQUIRE(output_message.efficiency == Catch::Approx(message.efficiency.value()));
     REQUIRE(output_message.end_date == message.end_date);
-    REQUIRE(output_message.series_id == message.series_id);
-    REQUIRE(output_message.series_unique_id == message.series_unique_id);
+    REQUIRE(output_message.run_number == message.run_number);
+    REQUIRE(output_message.run_name == message.run_name);
     REQUIRE(output_message.az_int_result.empty());
 }
 
@@ -272,8 +278,8 @@ TEST_CASE("CBORSerialize_End_RadIntResult", "[CBOR]") {
             .max_receiver_delay = 3456,
             .efficiency = 0.99,
             .end_date = "ccc",
-            .series_unique_id = "bla5",
-            .series_id = 45676782
+            .run_name = "bla5",
+            .run_number = 45676782
     };
 
     message.az_int_result["avg"] = {11.0, 12.0, 13.0};
@@ -302,8 +308,8 @@ TEST_CASE("CBORSerialize_End_ADUHistogram", "[CBOR]") {
             .max_receiver_delay = 3456,
             .efficiency = 0.99,
             .end_date = "ccc",
-            .series_unique_id = "bla5",
-            .series_id = 45676782,
+            .run_name = "bla5",
+            .run_number = 45676782,
             .adu_histogram_bin_width = 55
     };
 
@@ -359,12 +365,12 @@ TEST_CASE("CBORSerialize_Image", "[CBOR]") {
             .adu_histogram = {3, 4, 5, 8},
             .timestamp = 1ul<<27 | 1ul <<35,
             .exptime = 1000,
-            .series_unique_id = "bla2",
-            .series_id = 4567678,
+            .run_name = "bla2",
+            .run_number = 4567678,
             .saturated_pixel_count = 378,
             .error_pixel_count = 123,
             .strong_pixel_count = 1234,
-            .user_data = "Some random string ABCDEF",
+            .user_data = R"({"pi":3.1415, "z":"string"})"_json,
             .jf_info = UINT32_MAX,
             .receiver_aq_dev_delay = 2323,
             .storage_cell = 0xF,
@@ -391,8 +397,8 @@ TEST_CASE("CBORSerialize_Image", "[CBOR]") {
     REQUIRE(image_array.image.size == test.size());
     REQUIRE(image_array.indexing_result == message.indexing_result);
     REQUIRE(image_array.number == 456);
-    REQUIRE(image_array.series_id == message.series_id);
-    REQUIRE(image_array.series_unique_id == message.series_unique_id);
+    REQUIRE(image_array.run_number == message.run_number);
+    REQUIRE(image_array.run_name == message.run_name);
     REQUIRE(memcmp(image_array.image.data, test.data(), test.size()) == 0);
 
     REQUIRE(image_array.xfel_pulse_id == message.xfel_pulse_id);

tests/DiffractionExperimentTest.cpp

@@ -661,6 +661,23 @@ TEST_CASE("DiffractionExperiment_StorageCells_Pedestal_FixedG1","[DiffractionExp
     REQUIRE(x.GetFrameNum() == 323 * 15);
 }
 
+TEST_CASE("DiffractionExperiment_ImageTime","[DiffractionExperiment]") {
+    DiffractionExperiment x(DetectorSetup(3, DetectorType::JUNGFRAU));
+    DatasetSettings s;
+    x.FrameTime(583us);
+
+    s.ImageTime(583us * 4);
+    x.ImportDatasetSettings(s);
+    REQUIRE(x.GetSummation() == 4);
+
+    s.ImageTime({});
+    x.ImportDatasetSettings(s);
+    REQUIRE(x.GetSummation() == 1);
+
+    s.ImageTime(585us);
+    REQUIRE_THROWS(x.ImportDatasetSettings(s));
+}
+
 TEST_CASE("DiffractionExperiment_PulsedSource","[DiffractionExperiment]") {
     DiffractionExperiment x;
     REQUIRE(!x.IsPulsedSource()); // default must be off
@@ -770,9 +787,9 @@ TEST_CASE("DiffractionExperiment_Appendix","") {
     StartMessage message;
 
     x.FillMessage(message);
-    REQUIRE(message.user_data == "");
-    REQUIRE(x.GetImageAppendix()  == "");
-    REQUIRE(x.GetHeaderAppendix() == "");
+    REQUIRE(message.user_data == nlohmann::json());
+    REQUIRE(x.GetImageAppendix()  == nlohmann::json());
+    REQUIRE(x.GetHeaderAppendix() == nlohmann::json());
 
     x.ImageAppendix("ImageAppendix").HeaderAppendix("HeaderAppendix");
 
@@ -885,13 +902,15 @@ TEST_CASE("DiffractionExperiment_SeriesID","[DiffractionExperiment]") {
     REQUIRE(x.GetRunNumber() == 1);
 
     DatasetSettings d;
-    d.RunNumber(25);
+    d.RunNumber(25).RunName("run1");
     x.ImportDatasetSettings(d);
     REQUIRE(x.GetRunNumber() == 25);
+    REQUIRE(x.GetRunName() == "run1");
 
-    d.RunNumber({});
+    d.RunNumber({}).RunName({}).FilePrefix("file1");
     x.ImportDatasetSettings(d);
     REQUIRE(x.GetRunNumber() == 1);
+    REQUIRE(x.GetRunName() == "1:file1");
 }
 
 TEST_CASE("DiffractionExperiment_GetDefaultPlotBinning", "[DiffractionExperiment]") {

tests/HDF5WritingTest.cpp

@@ -227,9 +227,9 @@ TEST_CASE("HDF5Writer_Socket", "[HDF5][Full]") {
 
         DatasetSettings d;
         d.FilePrefix("run0345_lysozyme_acq").ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION)
-                .HeaderAppendix("{\"z\":567}").DetectorDistance_mm(155).BeamX_pxl(1606.62).BeamY_pxl(1669.59)
+                .HeaderAppendix(R"({"z":567})"_json).DetectorDistance_mm(155).BeamX_pxl(1606.62).BeamY_pxl(1669.59)
                 .PhotonEnergy_keV(12.07).SetUnitCell(UnitCell{.a = 97, .b = 97, .c = 38, .alpha= 90, .beta = 90, .gamma = 90})
-                .SpaceGroupNumber(96).RunNumber(345).ExperimentGroup("p12345").SampleName("lysozyme");
+                .SpaceGroupNumber(96).RunNumber(345).ExperimentGroup("p12345").SampleName("lysozyme").RunName("run1");
 
         x.ImportDatasetSettings(d);
         std::vector<SpotToSave> spots;
@@ -271,10 +271,11 @@ TEST_CASE("HDF5Writer_Socket", "[HDF5][Full]") {
         REQUIRE(j["filename"] == "run0345_lysozyme_acq_data_000001.h5");
         REQUIRE(j["file_number"] == 1);
         REQUIRE(j["nimages"] == 2);
-        REQUIRE(j["photon_energy_eV"] == Catch::Approx(x.GetPhotonEnergy_keV() * 1000.0));
+        REQUIRE(j["incident_energy_eV"] == Catch::Approx(x.GetPhotonEnergy_keV() * 1000.0));
         REQUIRE(j["space_group_number"] == 96);
         REQUIRE(j["experiment_group"] == "p12345");
         REQUIRE(j["run_number"] == 345);
+        REQUIRE(j["run_name"] == "run1");
         REQUIRE(j.contains("user_data"));
         REQUIRE(j["user_data"]["z"] == 567);
         std::cout << j.dump(4) << std::endl;
@@ -383,10 +384,26 @@ TEST_CASE("HDF5Writer_Rad_Int_Profile", "[HDF5][Full]") {
 }
 
 TEST_CASE("HDF5NXmx_DataFileName", "[HDF5]") {
-    REQUIRE(HDF5Metadata::DataFileName("z/x", 34) == "z/x_data_000035.h5");
-    REQUIRE(HDF5Metadata::DataFileName("z/x", 0) == "z/x_data_000001.h5");
-    REQUIRE_THROWS(HDF5Metadata::DataFileName("z/x", 1000000));
-    REQUIRE_THROWS(HDF5Metadata::DataFileName("z/x", -1));
+    StartMessage message;
+    message.file_prefix = "z/x";
+    REQUIRE(HDF5Metadata::DataFileName(message, 34) == "z/x_data_000035.h5");
+    REQUIRE(HDF5Metadata::DataFileName(message, 0) == "z/x_data_000001.h5");
+    REQUIRE_THROWS(HDF5Metadata::DataFileName(message, 1000000));
+    REQUIRE_THROWS(HDF5Metadata::DataFileName(message, -1));
+}
+
+TEST_CASE("HDF5NXmx_DataFileName_SwissFEL", "[HDF5]") {
+    StartMessage message;
+    message.file_prefix = "acq";
+    message.source_name = "SwissFEL";
+    message.detector_serial_number = "JF17T16V01";
+    REQUIRE(HDF5Metadata::DataFileName(message, 34) == "acq0035.JF17T16V01.h5");
+    REQUIRE(HDF5Metadata::DataFileName(message, 0) == "acq0001.JF17T16V01.h5");
+    REQUIRE_THROWS(HDF5Metadata::DataFileName(message, 10000));
+    REQUIRE_THROWS(HDF5Metadata::DataFileName(message, -1));
+
+    message.detector_serial_number = "";
+    REQUIRE(HDF5Metadata::DataFileName(message, 34) == "acq0035.JF.h5");
 }
 
 TEST_CASE("HDF5Objects_ExtractFilename", "[HDF5]") {

tests/JFJochBrokerParserTest.cpp

@@ -101,6 +101,7 @@ TEST_CASE("JFJochBrokerParser_DetectorSetup") {
             "description": "PSI JUNGFRAU 2M",
             "udp_interface_count": 1,
             "module_hostname": ["mx1", "mx2", "mx3", "mx4"],
+            "serial_number": "JF1",
             "gain_files": [
                   "../../tests/test_data/gainMaps_M049.bin",
                   "../../tests/test_data/gainMaps_M049.bin",
@@ -121,6 +122,7 @@ TEST_CASE("JFJochBrokerParser_DetectorSetup") {
     REQUIRE(detector.GetDetectorModuleHostname()[2] == "mx3");
     REQUIRE(detector.GetGainCalibration().size() == 4);
     REQUIRE(detector.GetUDPInterfaceCount() == 1);
+    REQUIRE(detector.GetSerialNumber() == "JF1");
 }
 
 TEST_CASE("JFJochBrokerParser_DetectorType_implicit") {
@@ -180,16 +182,15 @@ TEST_CASE("JFJochBrokerParser_ParseFacilityConfiguration") {
 {
   "cfg": {
     "source_name": "Swiss Light Source",
+    "source_type": "XFEL",
     "source_name_short": "SLS",
     "instrument_name": "X06SA VESPA",
     "instrument_name_short": "VESPA",
     "pedestal_g0_frames": 5000,
     "pedestal_g1_frames": 333,
     "pedestal_g2_frames": 333,
-    "frame_time_us": 0.0012,
-    "count_time_us": "950 us",
-    "spot_finding_period_us": " 2 ms",
-    "preview_period_us": "1 s",
+    "frame_time": 0.0012,
+    "count_time": "950 us",
     "detector_ipv4": "10.10.25.0"
   }
 }
@@ -198,6 +199,7 @@ TEST_CASE("JFJochBrokerParser_ParseFacilityConfiguration") {
     DiffractionExperiment x(DetectorGeometry(8));
     ParseFacilityConfiguration(j, "cfg", x);
     REQUIRE(x.GetSourceName() == "Swiss Light Source");
+    REQUIRE(x.GetSourceType() == "XFEL");
     REQUIRE(x.GetSourceNameShort() == "SLS");
     REQUIRE(x.GetInstrumentName() == "X06SA VESPA");
     REQUIRE(x.GetInstrumentNameShort() == "VESPA");

tests/JFJochReceiverProcessingTest.cpp

@@ -54,6 +54,8 @@ TEST_CASE("JFJochIntegrationTest_ZMQ_lysozyme_spot_and_index", "[JFJochReceiver]
     aq_devices.Add(std::move(test));
 
     ZMQStream2Pusher pusher({"ipc://*"});
+    pusher.WriterNotificationSocket("ipc://*");
+
     StreamWriter writer(logger, pusher.GetAddress()[0]);
     auto writer_future = std::async(std::launch::async, &StreamWriter::Run, &writer);
 

tests/StreamWriterTest.cpp

@@ -6,6 +6,7 @@
 #include "../writer/StreamWriter.h"
 #include "../frame_serialize/ZMQStream2Pusher.h"
 #include "../receiver/JFJochReceiverService.h"
+#include "../frame_serialize/ZMQWriterNotificationPuller.h"
 
 TEST_CASE("StreamWriterTest_ZMQ","[JFJochWriter]") {
     RegisterHDF5Filter();
@@ -61,3 +62,59 @@ TEST_CASE("StreamWriterTest_ZMQ","[JFJochWriter]") {
     REQUIRE(std::filesystem::remove("subdir/JFJochWriterTest_data_000001.h5"));
     REQUIRE(std::filesystem::remove("subdir"));
 }
+
+TEST_CASE("StreamWriterTest_ZMQ_Update","[JFJochWriter]") {
+    RegisterHDF5Filter();
+
+    Logger logger("test");
+
+    DatasetSettings d;
+    d.FilePrefix("subdir/JFJochWriterTest").NumTriggers(1).ImagesPerTrigger(5).RunName("run1").RunNumber(256);
+    DiffractionExperiment x(DetectorGeometry(2));
+    x.UseInternalPacketGenerator(true).Mode(DetectorMode::Raw).PedestalG0Frames(0).ImportDatasetSettings(d);
+    PixelMask pixel_mask(x);
+
+    JFModuleGainCalibration gain;
+    AcquisitionDeviceGroup aq_devices;
+    for (int i = 0; i < x.GetDataStreamsNum(); i++)
+        aq_devices.AddHLSDevice(64);
+
+    ZMQStream2Pusher pusher ({"ipc://*"});
+    pusher.WriterNotificationSocket("ipc://*");
+    JFJochReceiverService fpga_receiver_service(aq_devices, logger, pusher);
+
+    JFJochReceiverOutput receiver_output;
+
+    std::unique_ptr<StreamWriter> writer;
+    REQUIRE(x.GetImageNum() == 5);
+    auto pusher_addr = pusher.GetAddress();
+    REQUIRE(pusher_addr.size() == 1);
+    REQUIRE_NOTHROW(writer = std::make_unique<StreamWriter>(logger, pusher_addr[0]));
+    CHECK (writer->GetStatistics().state == StreamWriterState::Idle);
+    REQUIRE_NOTHROW(fpga_receiver_service.Start(x, pixel_mask, nullptr));
+
+    REQUIRE_NOTHROW(writer->Run());
+
+    REQUIRE_NOTHROW(receiver_output = fpga_receiver_service.Stop());
+    CHECK(receiver_output.status.images_sent == 5);
+    CHECK(writer->GetStatistics().state == StreamWriterState::Idle);
+    CHECK(writer->GetStatistics().processed_images == 5);
+    CHECK(writer->GetStatistics().file_prefix == x.GetFilePrefix());
+
+    // HDF5 file can be opened
+    std::unique_ptr<HDF5ReadOnlyFile> file;
+    REQUIRE_NOTHROW(file = std::make_unique<HDF5ReadOnlyFile>("subdir/JFJochWriterTest_data_000001.h5"));
+    std::unique_ptr<HDF5DataSet> dataset;
+    REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(*file, "/entry/data/data"));
+    std::unique_ptr<HDF5DataSpace> dataspace;
+    REQUIRE_NOTHROW(dataspace = std::make_unique<HDF5DataSpace>(*dataset));
+
+    REQUIRE(dataspace->GetNumOfDimensions() == 3);
+    REQUIRE(dataspace->GetDimensions()[0] == 5);
+    REQUIRE(dataspace->GetDimensions()[1] == RAW_MODULE_COLS);
+    REQUIRE(dataspace->GetDimensions()[2] == 2*RAW_MODULE_LINES);
+
+    REQUIRE(std::filesystem::remove("subdir/JFJochWriterTest_master.h5"));
+    REQUIRE(std::filesystem::remove("subdir/JFJochWriterTest_data_000001.h5"));
+    REQUIRE(std::filesystem::remove("subdir"));
+}