Implement raw TCP/IP in jfjoch_broker and jfjoch_writer

broker/JFJochBrokerParser.cpp

@@ -9,6 +9,7 @@
 #include "OpenAPIConvert.h"
 #include "Detector_type.h"
 #include "../image_pusher/NonePusher.h"
+#include "../image_pusher/TcpStreamPusher.h"
 
 DetectorGeometryModular ParseStandardDetectorGeometry(const org::openapitools::server::model::Detector &j) {
     auto s = j.getStandardGeometry();
@@ -190,10 +191,27 @@ std::unique_ptr<ImagePusher> ParseZMQImagePusher(const org::openapitools::server
     return std::move(tmp);
 }
 
+std::unique_ptr<ImagePusher> ParseTCPImagePusher(const org::openapitools::server::model::Jfjoch_settings &j) {
+    if (!j.zeromqIsSet())
+        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Socket settings must be provided");
+
+    std::optional<int32_t> send_buffer_size;
+    if (j.getZeromq().sendBufferSizeIsSet())
+        send_buffer_size = j.getZeromq().getSendBufferSize();
+
+    auto tmp = std::make_unique<TCPStreamPusher>(j.getZeromq().getImageSocket(), send_buffer_size);
+
+    if (j.getZeromq().writerNotificationSocketIsSet())
+        tmp->WriterNotificationSocket(j.getZeromq().getWriterNotificationSocket());
+    return std::move(tmp);
+}
+
 std::unique_ptr<ImagePusher> ParseImagePusher(const org::openapitools::server::model::Jfjoch_settings &j) {
     switch (j.getImagePusher().getValue()) {
         case org::openapitools::server::model::Image_pusher_type::eImage_pusher_type::ZEROMQ:
             return ParseZMQImagePusher(j);
+        case org::openapitools::server::model::Image_pusher_type::eImage_pusher_type::TCP:
+            return ParseTCPImagePusher(j);
         case org::openapitools::server::model::Image_pusher_type::eImage_pusher_type::HDF5:
             return std::make_unique<HDF5FilePusher>();
         case org::openapitools::server::model::Image_pusher_type::eImage_pusher_type::NONE:

broker/gen/model/Image_pusher_type.cpp

@@ -84,6 +84,9 @@ void to_json(nlohmann::json& j, const Image_pusher_type& o)
         case Image_pusher_type::eImage_pusher_type::CBOR:
             j = "CBOR";
             break;
+        case Image_pusher_type::eImage_pusher_type::TCP:
+            j = "TCP";
+            break;
         case Image_pusher_type::eImage_pusher_type::NONE:
             j = "None";
             break;
@@ -103,6 +106,9 @@ void from_json(const nlohmann::json& j, Image_pusher_type& o)
     else if (s == "CBOR") {
      o.setValue(Image_pusher_type::eImage_pusher_type::CBOR);
     } 
+    else if (s == "TCP") {
+     o.setValue(Image_pusher_type::eImage_pusher_type::TCP);
+    } 
     else if (s == "None") {
      o.setValue(Image_pusher_type::eImage_pusher_type::NONE);
     }  else {

broker/gen/model/Image_pusher_type.h

@@ -41,6 +41,7 @@ public:
     ZEROMQ, 
     HDF5, 
     CBOR, 
+    TCP, 
     NONE
     };
 

broker/jfjoch_api.yaml

@@ -1908,7 +1908,7 @@ components:
     image_pusher_type:
       type: string
       default: None
-      enum: [ZeroMQ, HDF5, CBOR, None]
+      enum: [ZeroMQ, HDF5, CBOR, TCP, None]
     standard_detector_geometry:
       type: object
       description: Regular rectangular geometry, first module is in the bottom left corner of the detector

broker/redoc-static.html

@@ -910,7 +910,7 @@ then image might be replaced in the buffer between calling /images and /image.cb
 </div></button></div></div></div><div class="sc-iGgWBj sc-gsFSXq ibAZSi eDeKjd"><div class="sc-ejfMa-d bNUPQg"><button class="sc-EgOXT cFEcOl"><span type="get" class="sc-eZYNyq qGfV http-verb get">get</span><span class="sc-iEXKAA gUNSDu">/version</span><svg class="sc-cwHptR iVFMLx" style="margin-right:-25px" version="1.1" viewBox="0 0 24 24" x="0" xmlns="http://www.w3.org/2000/svg" y="0" aria-hidden="true"><polygon points="17.3 8.3 12 13.6 6.7 8.3 5.3 9.7 12 16.4 18.7 9.7 "></polygon></svg></button><div aria-hidden="true" class="sc-dlWCHZ kWacYr"><div class="sc-hHOBiw cndTNO"><div class="sc-eeDRCY sc-eBMEME gTGgei fbXBig"><p>Test Jungfraujoch system</p>
 </div><div tabindex="0" role="button"><div class="sc-kWtpeL gEncxT"><span>http://localhost:5232</span>/version</div></div></div></div></div></div></div></div></div><div class="sc-dExYaf gZbfaQ"></div></div></div>
       <script>
-      const __redoc_state = {"menu":{"activeItemIdx":-1},"spec":{"data":{"openapi":"3.0.3","info":{"title":"Jungfraujoch","description":"API to control Jungfraujoch developed by the Paul Scherrer Institute (Switzerland).\nJungfraujoch is a data acquisition and analysis system for pixel array detectors, primarly PSI JUNGFRAU.\nJungfraujoch uses FPGA boards to acquire data at high data rates.\n\n# License Clarification\n\nWhile this API definition is licensed under GPL-3.0, **the GPL copyleft provisions do not apply**\nwhen this file is used solely to generate OpenAPI clients or when implementing applications that\ninteract with the API. Generated client code and applications using this API definition are not\nsubject to the GPL license requirements and may be distributed under terms of your choosing.\n\nThis exception is similar in spirit to the Linux Kernel's approach to userspace API headers and\nthe GCC Runtime Library Exception. The Linux Kernel developers have explicitly stated that\nuser programs that merely use the kernel interfaces (syscalls, ioctl definitions, etc.) are not\nderivative works of the kernel and are not subject to the terms of the GPL.\n\nThis exception is intended to allow wider use of this API specification without imposing GPL\nrequirements on applications that merely interact with the API, regardless of whether they\ncommunicate through network calls or other mechanisms.\n","version":"1.0.0-rc.127","contact":{"name":"Filip Leonarski (Paul Scherrer Institute)","email":"filip.leonarski@psi.ch"},"license":{"name":"GPL-3.0","url":"https://www.gnu.org/licenses/gpl-3.0.html"}},"servers":[{"url":"http://localhost:5232","description":"Test Jungfraujoch system"}],"components":{"parameters":{"binning":{"in":"query","name":"binning","required":false,"schema":{"default":1,"type":"integer"},"description":"Binning of frames for the plot (0 = default binning)"},"compression":{"in":"query","name":"compression","required":false,"schema":{"type":"boolean","default":false},"description":"Enable DEFLATE compression of output data."},"experimental_coord":{"in":"query","name":"experimental_coord","required":false,"schema":{"type":"boolean","default":false},"description":"If measurement has goniometer axis defined, plot X-axis will represent rotation angle\nIf measurement has grid scan defined, plot X-axis and Y-axis will represent grid position, Z will be used as the final value\nFor still measurement the number is ignored\n"},"azint_unit":{"in":"query","name":"azint_unit","required":false,"schema":{"type":"string","enum":["Q_recipA","d_A","two_theta_deg"],"default":"Q_recipA"},"description":"Unit used for azim int.\n"},"fill_value":{"in":"query","name":"fill","required":false,"schema":{"type":"number","format":"float"},"description":"Fill value for elements that were missed during data collection\n"},"plot_type":{"in":"query","name":"type","required":true,"description":"Type of requested plot","schema":{"type":"string","enum":["bkg_estimate","azint","azint_1d","spot_count","spot_count_low_res","spot_count_indexed","spot_count_ice","indexing_rate","indexing_time","indexing_unit_cell_length","indexing_unit_cell_angle","profile_radius","mosaicity","b_factor","error_pixels","saturated_pixels","image_collection_efficiency","receiver_delay","receiver_free_send_buf","strong_pixels","roi_sum","roi_mean","roi_max_count","roi_pixels","roi_weighted_x","roi_weighted_y","packets_received","max_pixel_value","resolution_estimate","pixel_sum","processing_time","beam_center_x","beam_center_y"]}},"roi":{"in":"query","name":"roi","required":false,"schema":{"type":"string","minLength":1},"description":"Name of ROI for which plot is requested"},"image_id":{"in":"query","name":"id","required":false,"schema":{"type":"integer","format":"int64","default":-1,"minimum":-2},"description":"Image ID in the image buffer. Special values: -1 - last image in the buffer, -2: last indexed image in the buffer"},"saturation":{"in":"query","name":"saturation","required":false,"schema":{"type":"number","format":"float","minimum":-32767,"maximum":32767},"description":"Saturation value to set contrast in the preview image; if not provided, then autocontrast procedure is used"},"background":{"in":"query","name":"background","required":false,"schema":{"type":"number","format":"float","default":0,"minimum":-32767,"maximum":32767},"description":"Background value to set contrast in the preview image"},"show_spots":{"in":"query","name":"show_spots","schema":{"type":"boolean","default":true},"description":"Show spot finding results on the image"},"show_beam_center":{"in":"query","name":"show_beam_center","schema":{"type":"boolean","default":true},"description":"Show beam center on the image"},"show_roi":{"in":"query","name":"show_roi","schema":{"type":"boolean","default":false},"description":"Show ROI areas on the image"},"jpeg_quality":{"in":"query","name":"jpeg_quality","description":"Quality of JPEG image (100 - highest; 0 - lowest)","schema":{"tLine truncated
+      const __redoc_state = {"menu":{"activeItemIdx":-1},"spec":{"data":{"openapi":"3.0.3","info":{"title":"Jungfraujoch","description":"API to control Jungfraujoch developed by the Paul Scherrer Institute (Switzerland).\nJungfraujoch is a data acquisition and analysis system for pixel array detectors, primarly PSI JUNGFRAU.\nJungfraujoch uses FPGA boards to acquire data at high data rates.\n\n# License Clarification\n\nWhile this API definition is licensed under GPL-3.0, **the GPL copyleft provisions do not apply**\nwhen this file is used solely to generate OpenAPI clients or when implementing applications that\ninteract with the API. Generated client code and applications using this API definition are not\nsubject to the GPL license requirements and may be distributed under terms of your choosing.\n\nThis exception is similar in spirit to the Linux Kernel's approach to userspace API headers and\nthe GCC Runtime Library Exception. The Linux Kernel developers have explicitly stated that\nuser programs that merely use the kernel interfaces (syscalls, ioctl definitions, etc.) are not\nderivative works of the kernel and are not subject to the terms of the GPL.\n\nThis exception is intended to allow wider use of this API specification without imposing GPL\nrequirements on applications that merely interact with the API, regardless of whether they\ncommunicate through network calls or other mechanisms.\n","version":"1.0.0-rc.127","contact":{"name":"Filip Leonarski (Paul Scherrer Institute)","email":"filip.leonarski@psi.ch"},"license":{"name":"GPL-3.0","url":"https://www.gnu.org/licenses/gpl-3.0.html"}},"servers":[{"url":"http://localhost:5232","description":"Test Jungfraujoch system"}],"components":{"parameters":{"binning":{"in":"query","name":"binning","required":false,"schema":{"default":1,"type":"integer"},"description":"Binning of frames for the plot (0 = default binning)"},"compression":{"in":"query","name":"compression","required":false,"schema":{"type":"boolean","default":false},"description":"Enable DEFLATE compression of output data."},"experimental_coord":{"in":"query","name":"experimental_coord","required":false,"schema":{"type":"boolean","default":false},"description":"If measurement has goniometer axis defined, plot X-axis will represent rotation angle\nIf measurement has grid scan defined, plot X-axis and Y-axis will represent grid position, Z will be used as the final value\nFor still measurement the number is ignored\n"},"azint_unit":{"in":"query","name":"azint_unit","required":false,"schema":{"type":"string","enum":["Q_recipA","d_A","two_theta_deg"],"default":"Q_recipA"},"description":"Unit used for azim int.\n"},"fill_value":{"in":"query","name":"fill","required":false,"schema":{"type":"number","format":"float"},"description":"Fill value for elements that were missed during data collection\n"},"plot_type":{"in":"query","name":"type","required":true,"description":"Type of requested plot","schema":{"type":"string","enum":["bkg_estimate","azint","azint_1d","spot_count","spot_count_low_res","spot_count_indexed","spot_count_ice","indexing_rate","indexing_time","indexing_unit_cell_length","indexing_unit_cell_angle","profile_radius","mosaicity","b_factor","error_pixels","saturated_pixels","image_collection_efficiency","receiver_delay","receiver_free_send_buf","strong_pixels","roi_sum","roi_mean","roi_max_count","roi_pixels","roi_weighted_x","roi_weighted_y","packets_received","max_pixel_value","resolution_estimate","pixel_sum","processing_time","beam_center_x","beam_center_y"]}},"roi":{"in":"query","name":"roi","required":false,"schema":{"type":"string","minLength":1},"description":"Name of ROI for which plot is requested"},"image_id":{"in":"query","name":"id","required":false,"schema":{"type":"integer","format":"int64","default":-1,"minimum":-2},"description":"Image ID in the image buffer. Special values: -1 - last image in the buffer, -2: last indexed image in the buffer"},"saturation":{"in":"query","name":"saturation","required":false,"schema":{"type":"number","format":"float","minimum":-32767,"maximum":32767},"description":"Saturation value to set contrast in the preview image; if not provided, then autocontrast procedure is used"},"background":{"in":"query","name":"background","required":false,"schema":{"type":"number","format":"float","default":0,"minimum":-32767,"maximum":32767},"description":"Background value to set contrast in the preview image"},"show_spots":{"in":"query","name":"show_spots","schema":{"type":"boolean","default":true},"description":"Show spot finding results on the image"},"show_beam_center":{"in":"query","name":"show_beam_center","schema":{"type":"boolean","default":true},"description":"Show beam center on the image"},"show_roi":{"in":"query","name":"show_roi","schema":{"type":"boolean","default":false},"description":"Show ROI areas on the image"},"jpeg_quality":{"in":"query","name":"jpeg_quality","description":"Quality of JPEG image (100 - highest; 0 - lowest)","schema":{"tLine truncated
 
       var container = document.getElementById('redoc');
       Redoc.hydrate(__redoc_state, container);

docs/python_client/docs/ImagePusherType.md

@@ -9,6 +9,8 @@
 
 * `CBOR` (value: `'CBOR'`)
 
+* `TCP` (value: `'TCP'`)
+
 * `NONE` (value: `'None'`)
 
 [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)

frontend/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "jungfraujoch-frontend",
-  "version": "1.0.0-rc.126",
+  "version": "1.0.0-rc.127",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "jungfraujoch-frontend",
-      "version": "1.0.0-rc.126",
+      "version": "1.0.0-rc.127",
       "license": "GPL-3.0",
       "dependencies": {
         "@emotion/react": "^11.10.4",

frontend/src/openapi/models/image_pusher_type.ts

@@ -7,5 +7,6 @@ export enum image_pusher_type {
     ZERO_MQ = 'ZeroMQ',
     HDF5 = 'HDF5',
     CBOR = 'CBOR',
+    TCP = 'TCP',
     NONE = 'None',
 }

image_pusher/TcpStreamPusher.cpp

@@ -90,6 +90,34 @@ bool TCPStreamPusher::EndDataCollection(const EndMessage &message) {
     return ret;
 }
 
+std::string TCPStreamPusher::Finalize() {
+    std::string ret;
+    if (writer_notification_socket) {
+        for (size_t i = 0; i < socket.size(); i++) {
+            auto n = writer_notification_socket->Receive(run_number, run_name);
+            if (!n)
+                ret += "Writer " + std::to_string(i) + ": no end notification received within 1 minute from collection end";
+            else if (static_cast<size_t>(n->socket_number) >= socket.size())
+                ret += "Writer " + std::to_string(i) + ": mismatch in socket number";
+            else if (!n->ok)
+                ret += "Writer " + std::to_string(i) + ": " + n->error;
+        }
+    }
+    return ret;
+}
+
+std::string TCPStreamPusher::GetWriterNotificationSocketAddress() const {
+    if (writer_notification_socket)
+        return writer_notification_socket->GetEndpointName();
+    else
+        return "";
+}
+
+TCPStreamPusher &TCPStreamPusher::WriterNotificationSocket(const std::string &addr) {
+    writer_notification_socket = std::make_unique<ZMQWriterNotificationPuller>(addr, std::chrono::minutes(1));
+    return *this;
+}
+
 std::string TCPStreamPusher::PrintSetup() const {
     std::string output = "TCPStream2Pusher: Sending images to sockets: ";
     for (const auto &s : socket)

image_pusher/TcpStreamPusher.h

@@ -4,12 +4,15 @@
 #pragma once
 
 #include "TcpStreamPusherSocket.h"
+#include "ZMQWriterNotificationPuller.h"
 
 class TCPStreamPusher : public ImagePusher {
     std::vector<uint8_t> serialization_buffer;
     CBORStream2Serializer serializer;
     std::vector<std::unique_ptr<TCPStreamPusherSocket>> socket;
 
+    std::unique_ptr<ZMQWriterNotificationPuller> writer_notification_socket;
+
     int64_t images_per_file = 1;
     uint64_t run_number = 0;
     std::string run_name;
@@ -20,11 +23,15 @@ public:
                               std::optional<size_t> zerocopy_threshold = {},
                               size_t send_queue_size = 4096);
 
+    TCPStreamPusher& WriterNotificationSocket(const std::string& addr);
+    std::string GetWriterNotificationSocketAddress() const override;
+
     void StartDataCollection(StartMessage& message) override;
     bool EndDataCollection(const EndMessage& message) override;
     bool SendImage(const uint8_t *image_data, size_t image_size, int64_t image_number) override;
     void SendImage(ZeroCopyReturnValue &z) override;
     bool SendCalibration(const CompressedImage& message) override;
 
+    std::string Finalize() override;
     std::string PrintSetup() const override;
 };

image_pusher/ZMQStream2Pusher.h

@@ -27,6 +27,7 @@ public:
                               std::optional<int32_t> send_buffer_size = {});
 
     ZMQStream2Pusher& WriterNotificationSocket(const std::string& addr);
+    std::string GetWriterNotificationSocketAddress() const override;
 
     std::vector<std::string> GetAddress();
 
@@ -40,7 +41,7 @@ public:
     bool SendImage(const uint8_t *image_data, size_t image_size, int64_t image_number) override;
 
     std::string Finalize() override;
-    std::string GetWriterNotificationSocketAddress() const override;
+
 
     std::string PrintSetup() const override;
 };

writer/jfjoch_writer.cpp

@@ -8,6 +8,7 @@
 #include "StreamWriter.h"
 #include "../common/print_license.h"
 #include "../image_puller/ZMQImagePuller.h"
+#include "../image_puller/TcpImagePuller.h"
 
 static Logger logger("jfjoch_writer");
 
@@ -18,6 +19,7 @@ void print_usage() {
     logger.Info("Usage ./jfjoch_writer {options} <address of the ZeroMQ data source>");
     logger.Info("");
     logger.Info("Available options:");
+    logger.Info("-T                                   Use raw TCP/IP instead of ZeroMQ");
     logger.Info("-d<int> | --root_dir=<int>           Root directory for file writing");
     logger.Info("-r<int> | --zmq_repub_port=<int>     ZeroMQ port for PUSH socket to republish images");
     logger.Info("-f<int> | --zmq_file_port=<int>      ZeroMQ port for PUB socket to inform about finalized files");
@@ -80,8 +82,12 @@ int main(int argc, char **argv) {
 
     int option_index = 0;
     int opt;
-    while ((opt = getopt_long(argc, argv, "?hH:r:f:R:d:W:w:v",long_options, &option_index)) != -1 ) {
+    bool raw_tcp = false;
+    while ((opt = getopt_long(argc, argv, "?hH:r:f:R:d:W:w:vT",long_options, &option_index)) != -1 ) {
         switch (opt) {
+            case 'T':
+                raw_tcp = true;
+                break;
             case 'v':
                 verbose = true;
                 break;
@@ -149,14 +155,24 @@ int main(int argc, char **argv) {
     }
 
     if ((zmq_repub_port < UINT16_MAX) && (zmq_repub_port > 0)) {
+        if (raw_tcp) {
+            logger.Error("TCP republishing not supported at the moment");
+            exit(EXIT_FAILURE);
+        }
         repub_zmq_addr = fmt::format("tcp://0.0.0.0:{:d}", zmq_repub_port);
         logger.Info("Images are republished via ZeroMQ PUSH socket {:s}", repub_zmq_addr);
     }
 
     ZMQContext context;
 
-    ZMQImagePuller puller(argv[first_argc], repub_zmq_addr, rcv_watermark, repub_watermark);
-    writer = new StreamWriter(logger,puller,file_done_zmq_addr, verbose);
+    std::unique_ptr<ImagePuller> puller;
+    if (raw_tcp)
+        puller = std::make_unique<TCPImagePuller>(argv[first_argc]);
+    else
+        puller = std::make_unique<ZMQImagePuller>(argv[first_argc], repub_zmq_addr, rcv_watermark, repub_watermark);
+
+
+    writer = new StreamWriter(logger,*puller,file_done_zmq_addr, verbose);
 
     std::vector<int> sigs{SIGQUIT, SIGINT, SIGTERM, SIGHUP};
     setUpUnixSignals(sigs);