diff --git a/docs/JFJOCH_BROKER.md b/docs/JFJOCH_BROKER.md index 01848418f..cf97154d3 100644 --- a/docs/JFJOCH_BROKER.md +++ b/docs/JFJOCH_BROKER.md @@ -190,6 +190,12 @@ back to the broker, so with it the route answers 409. ## Online analysis +![jfjoch_broker data paths: a PSI JUNGFRAU or EIGER reaches the broker through the FPGA cards, a DECTRIS detector through its stream; one receiver per run - pedestal, FPGA, dark mask, Lite, rugnux rotation or rugnux stills - makes the images and their results, which go through the image buffer and the pusher to jfjoch_writer, and to the preview stream, the metadata stream and the REST status; the configuration keys that choose each path are shown in monospace](images/broker_pipeline.svg) + +*What drives what in `jfjoch_broker`: the inputs on the left, the receiver chosen for each run in the middle, the +outputs on the right, with the configuration keys that choose each path (`docs/images/broker_pipeline_fig.py` +draws it).* + Without rugnux live mode, and always in the FPGA workflow, the broker analyses each image on its own: masking, summation, azimuthal and ROI integration, spot finding, and - with indexing on - indexing of the image and refinement of its geometry (`geom_refinement_algorithm`). A rotation collection is indexed image by image, as a serial diff --git a/docs/images/broker_pipeline.svg b/docs/images/broker_pipeline.svg new file mode 100644 index 000000000..ec1b9f3cc --- /dev/null +++ b/docs/images/broker_pipeline.svg @@ -0,0 +1,137 @@ + + + + +PSI detector +JUNGFRAU, EIGER + +FPGA cards +UDP packets off the wire; +conversion, summation, +spot finder, azimuthal, ROI + +FPGA + +detector.type +JUNGFRAU | EIGER + +DECTRIS detector +SIMPLON API + +Stream puller +ZeroMQ Stream2, CBOR: +images as sent by the detector + +detector.type +DECTRIS + +Receiver(one per run, chosen at start) +receiver_threads + +JFJochReceiverPedestal +pedestal run (G0, G1, G2) +raw frames into the pedestal and its RMS; +no images, nothing on any stream +detector_settings.jungfrau_pedestal_* + +JFJochReceiverFPGA +standard run, dark-mask sequence +image from the card's output, summation on the host +above the card's; each image indexed +image_format.summation · azim_int.force_cpu + +CUDA + +FPGA + +JFJochReceiverDarkMask +dark-mask sequence +dark frames into the mask; +nothing on any stream +dark_mask + +JFJochReceiverLite +rugnux live off, or a rotation run the image buffer cannot hold +each image on the GPU: masking, azimuthal and +ROI integration, spot finding, indexing +indexing · spot_finding · azim_int + +CUDA + +JFJochReceiverRugnuxRotation +rugnux live on, rotation that fits the image buffer +images forwarded as they come; rugnux reads them +back from the image buffer: the whole pipeline +rugnux.enable · reserved_cpus · deadline_s + +rugnux + +CUDA + +JFJochReceiverRugnuxStills +rugnux live on, serial (experimental) +rugnux pre-pass on the first frames; then every frame +analysed and integrated on the GPU, merged live +rugnux.enable · bragg_integration + +rugnux + +CUDA + + + + + + + +Image buffer +compressed images, one slot each; +rugnux rotation reads them back +image_buffer_MiB + +Image pusher +ZeroMQ PUSH or TCP/IP stream; +HDF5 in-process, CBOR files +image_pusher + +jfjoch_writer +NXmx HDF5: master and data files; +rugnux results beside the master +file_writer.format + +Preview stream +ZeroMQ PUB, 1 image/s +zeromq_preview + +Metadata stream +ZeroMQ PUB, bundles per period +zeromq_metadata + +HTTP/REST, web frontend +status, plots, scan result, +image buffer, /rugnux/status + + +START, images, END + + + + + + + + + +images with their per-image results +fed by every receiver; the pedestal and the mask arrive only here + +FPGA +computed on the FPGA card + +CUDA +GPU analysis where a card is present + +rugnux +the full rugnux pipeline, in the broker's process +monospace: keys of the broker configuration (jfjoch_settings) and of the dataset settings + \ No newline at end of file diff --git a/docs/images/broker_pipeline_fig.py b/docs/images/broker_pipeline_fig.py new file mode 100644 index 000000000..d3a4c22b7 --- /dev/null +++ b/docs/images/broker_pipeline_fig.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +"""jfjoch_broker data paths: inputs, the receiver chosen per run, outputs, and the settings that switch them.""" + +# Same style as rugnux_pipeline_fig.py; written next to this script as broker_pipeline.svg. + +import os + +W, H = 1600, 1080 +BLUE, TEAL, ORANGE, INK = "#2F6FDE", "#1A8A7D", "#E8740C", "#222222" +PANEL_FILL, PANEL_STROKE = "#EEF3FC", "#B9CBEF" +GPU_BG, FPGA_BG, RUGNUX_BG = "#2F6FDE", "#E8740C", "#1A8A7D" +GREY = "#6B6B6B" + +out = [] +def a(s): out.append(s) + +def start(): + out.clear() + a(f'') + a('' + '' + '' + '') + a(f'') + +def finish(name): + a('') + open(os.path.join(os.path.dirname(os.path.abspath(__file__)), name), "w").write("\n".join(out)) + +def badge(x, y, w, bg, text): + a(f'') + a(f'{text}') + +BADGES = [("rugnux", 62, RUGNUX_BG), ("CUDA", 52, GPU_BG), ("FPGA", 52, FPGA_BG)] + +def box(x, y, w, h, title, body, color, rule=None, keys=None, gpu=False, fpga=False, rugnux=False, ts=20, bs=15): + a(f'') + a(f'{title}') + by = y + 58 + if rule: # when the run gets this box + a(f'{rule}') + by = y + 76 + if body: + lines = body if isinstance(body, list) else [body] + for i, ln in enumerate(lines): + a(f'{ln}') + if keys: # the settings that steer the box + a(f'{keys}') + bx = x + w - 10 + for (text, bw, bg), on in zip(BADGES, (rugnux, gpu, fpga)): + if on: + bx -= bw + badge(bx, y+h-28, bw, bg, text) + bx -= 8 + +def path(d, dashed=False): + dash = ' stroke-dasharray="8,6"' if dashed else '' + a(f'') + +def label(x, y, text, anchor="middle", size=14): + a(f'{text}') + +def key(x, y, text, anchor="middle"): + a(f'{text}') + +def panel(x0, x1, y0, y1, tx): + a(f'') + a(f'Receiver(one per run, chosen at start)') + +def legend(): + LY = H-58 + lx = 60 + for text, bw, bg, desc, step in [("FPGA", 52, FPGA_BG, "computed on the FPGA card", 290), + ("CUDA", 52, GPU_BG, "GPU analysis where a card is present", 370), + ("rugnux", 62, RUGNUX_BG, "the full rugnux pipeline, in the broker's process", 0)]: + badge(lx, LY-14, bw, bg, text) + a(f'{desc}') + lx += step + key(60, LY+26, "monospace: keys of the broker configuration (jfjoch_settings) and of the dataset settings", anchor="start") + +# ---------- geometry +IX, IW, IH = 40, 170, 80 # inputs +TX, TW = 325, 225 # transport (cards, stream puller) +PX0, PX1 = 570, 1110 # receiver panel +RX, RW, RH = 600, 480, 125 # receiver boxes +R = [125 + i*(RH+18) for i in range(6)] # receiver rows +PY0, PY1 = 70, R[-1] + RH + 25 +BUSL, BUSR = 560, 1150 # arrow buses left and right of the panel +OX, OW = 1200, 350 # outputs + + +def broker(): + start() + + # ---------- inputs and transport + CY, SY = 170, 625 # cards and stream puller rows + box(IX, CY+28, IW, IH, "PSI detector", "JUNGFRAU, EIGER", INK, ts=17, bs=14) + box(TX, CY, TW, 135, "FPGA cards", ["UDP packets off the wire;", "conversion, summation,", "spot finder, azimuthal, ROI"], + TEAL, fpga=True, bs=14) + path(f"M{IX+IW},{CY+68} L{TX-2},{CY+68}") + key((IX+IW+TX)/2, CY+58, "detector.type") + label((IX+IW+TX)/2, CY+86, "JUNGFRAU | EIGER", size=13) + + box(IX, SY+20, IW, IH, "DECTRIS detector", "SIMPLON API", INK, ts=17, bs=14) + box(TX, SY, TW, 120, "Stream puller", ["ZeroMQ Stream2, CBOR:", "images as sent by the detector"], TEAL, bs=14) + path(f"M{IX+IW},{SY+60} L{TX-2},{SY+60}") + key((IX+IW+TX)/2, SY+50, "detector.type") + label((IX+IW+TX)/2, SY+78, "DECTRIS", size=13) + + # ---------- receiver panel + panel(PX0, PX1, PY0, PY1, RX) + key(PX1-20, PY0+32, "receiver_threads", anchor="end") + box(RX, R[0], RW, RH, "JFJochReceiverPedestal", ["raw frames into the pedestal and its RMS;", "no images, nothing on any stream"], BLUE, + rule="pedestal run (G0, G1, G2)", keys="detector_settings.jungfrau_pedestal_*", bs=14) + box(RX, R[1], RW, RH, "JFJochReceiverFPGA", ["image from the card's output, summation on the host", "above the card's; each image indexed"], BLUE, + rule="standard run, dark-mask sequence", keys="image_format.summation · azim_int.force_cpu", fpga=True, gpu=True, bs=14) + box(RX, R[2], RW, RH, "JFJochReceiverDarkMask", ["dark frames into the mask;", "nothing on any stream"], BLUE, + rule="dark-mask sequence", keys="dark_mask", bs=14) + box(RX, R[3], RW, RH, "JFJochReceiverLite", ["each image on the GPU: masking, azimuthal and", "ROI integration, spot finding, indexing"], BLUE, + rule="rugnux live off, or a rotation run the image buffer cannot hold", keys="indexing · spot_finding · azim_int", gpu=True, bs=14) + box(RX, R[4], RW, RH, "JFJochReceiverRugnuxRotation", ["images forwarded as they come; rugnux reads them", "back from the image buffer: the whole pipeline"], BLUE, + rule="rugnux live on, rotation that fits the image buffer", keys="rugnux.enable · reserved_cpus · deadline_s", rugnux=True, gpu=True, bs=14) + box(RX, R[5], RW, RH, "JFJochReceiverRugnuxStills", ["rugnux pre-pass on the first frames; then every frame", "analysed and integrated on the GPU, merged live"], BLUE, + rule="rugnux live on, serial (experimental)", keys="rugnux.enable · bragg_integration", rugnux=True, gpu=True, bs=14) + + # cards feed the first two receivers, the stream puller the other four + for r in R[:2]: + path(f"M{TX+TW},{CY+68} L{BUSL},{CY+68} L{BUSL},{r+RH/2} L{RX-2},{r+RH/2}") + for r in R[2:]: + path(f"M{TX+TW},{SY+60} L{BUSL},{SY+60} L{BUSL},{r+RH/2} L{RX-2},{r+RH/2}") + + # ---------- outputs + O = {"buffer": 125, "pusher": 280, "writer": 435, "preview": 620, "metadata": 750, "rest": 880} + box(OX, O["buffer"], OW, 105, "Image buffer", ["compressed images, one slot each;", "rugnux rotation reads them back"], ORANGE, bs=14) + key(OX+12, O["buffer"]+105+18, "image_buffer_MiB", anchor="start") + box(OX, O["pusher"], OW, 105, "Image pusher", ["ZeroMQ PUSH or TCP/IP stream;", "HDF5 in-process, CBOR files"], ORANGE, bs=14) + key(OX+12, O["pusher"]+105+18, "image_pusher", anchor="start") + box(OX, O["writer"], OW, 105, "jfjoch_writer", ["NXmx HDF5: master and data files;", "rugnux results beside the master"], ORANGE, bs=14) + key(OX+12, O["writer"]+105+18, "file_writer.format", anchor="start") + box(OX, O["preview"], OW, 95, "Preview stream", "ZeroMQ PUB, 1 image/s", ORANGE, bs=14) + key(OX+12, O["preview"]+95+18, "zeromq_preview", anchor="start") + box(OX, O["metadata"], OW, 95, "Metadata stream", "ZeroMQ PUB, bundles per period", ORANGE, bs=14) + key(OX+12, O["metadata"]+95+18, "zeromq_metadata", anchor="start") + box(OX, O["rest"], OW, 105, "HTTP/REST, web frontend", ["status, plots, scan result,", "image buffer, /rugnux/status"], ORANGE, bs=14) + + path(f"M{OX+OW/2},{O['buffer']+105} L{OX+OW/2},{O['pusher']-2}") + path(f"M{OX+OW/2},{O['pusher']+105} L{OX+OW/2},{O['writer']-2}") + label(OX+OW-12, O["pusher"]-18, "START, images, END", anchor="end", size=13) + + # the four receivers that make images feed the right bus; from it the buffer, the two PUB streams and the status + for r in (R[1], R[3], R[4], R[5]): + path(f"M{RX+RW},{r+RH/2} L{BUSR-2},{r+RH/2}") + a(f'') + for name, h in (("buffer", 105), ("preview", 95), ("metadata", 95), ("rest", 105)): + path(f"M{BUSR},{O[name]+h/2} L{OX-2},{O[name]+h/2}") + label(OX+OW/2, O["buffer"]-12, "images with their per-image results", size=13) + label(OX+OW/2, O["rest"]+105+22, "fed by every receiver; the pedestal and the mask arrive only here", size=13) + + legend() + finish("broker_pipeline.svg") + + +broker() +print("ok")