From a3028ff45c10ab78d0530d3f4f1d7f3840da8c9b Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Thu, 15 Oct 2026 20:57:50 +0200 Subject: [PATCH 1/2] Docs: rugnux pipeline figure (Extended Data Fig. 1) on the overview page The owner's generator and its SVG, under docs/images/, shown on RUGNUX_OVERVIEW.md. Corrections against the code, with the owner's header and style kept: - Input: marCCD added; "stream" is not a ZeroMQ stream but the broker's live collection (closed data files read as they are reported, rotation only). - Pre-scan: also masks defective pixels; it MEASURES the beam centre and commits none - the first pass indexes at both centres - so the centre now goes to Index as its own arrow ("measured beam centre"), and the arrow into the per-image panel carries "masks, spot widths". - Index: a first pass on a frame sample (validation frames, rotated into one crystal frame), not a per-image indexer. - Refine: the per-image fit is the in-tree LM solver adapted from Ceres (LMSolver.h), not Ceres Solver itself. - Input arrow: pass 1 integrates only the first half of the sweep (>= 120 deg). - Post-refine: beam centre added (distance, beam, cell, axis; tilt is held); the hypotheses run between the passes (beam-centre arm, metric-symmetry arm, geometry walk, rotation-scale walk) noted; pass 2 is repeated while a round realises a gain. - Whole dataset: the P1 merge is where competing passes are judged (quality guard, beam-centre arms), so it carries "tested"; Symmetry adds the doubled-cell hypothesis; the in-group merge names corrections and error model; structure is drawn dashed with a bypass to the report (no model); report lists unmerged MTZ and the report file. - Legend: a CUDA step takes its CPU path without a GPU. - Taller boxes where a second text line needed room; the SVG is written next to the script. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PXc1zD1yxHqaR1DGdCTHL4 --- docs/RUGNUX_OVERVIEW.md | 2 + docs/images/rugnux_pipeline.svg | 136 +++++++++++++++++++++++++ docs/images/rugnux_pipeline_fig.py | 154 +++++++++++++++++++++++++++++ 3 files changed, 292 insertions(+) create mode 100644 docs/images/rugnux_pipeline.svg create mode 100644 docs/images/rugnux_pipeline_fig.py diff --git a/docs/RUGNUX_OVERVIEW.md b/docs/RUGNUX_OVERVIEW.md index 69cbdd48f..705eb060a 100644 --- a/docs/RUGNUX_OVERVIEW.md +++ b/docs/RUGNUX_OVERVIEW.md @@ -4,6 +4,8 @@ The map of a run, in the order it happens — one paragraph per stage, each link [data-analysis reference](CPU_DATA_ANALYSIS.md) where the algorithm lives. The walk-through is a rotation run with the defaults; stills differences are at the end. +![The rotation pipeline: pass 1 integrates the first half of the sweep at the file's geometry, pass 2 the whole sweep at the refined one, and only pass 2 goes on to the space group and the written merge](images/rugnux_pipeline.svg) + **Open the dataset.** The geometry, wavelength and goniometer come from the file ([What Rugnux reads](RUGNUX_FORMATS.md)). A goniometer axis makes it a rotation run, none makes it serial stills — nothing is asked of the user. diff --git a/docs/images/rugnux_pipeline.svg b/docs/images/rugnux_pipeline.svg new file mode 100644 index 000000000..84346280c --- /dev/null +++ b/docs/images/rugnux_pipeline.svg @@ -0,0 +1,136 @@ + + + + +Per image(independent) + +Decode +bitshuffle/LZ4, CBF + +CUDA + +Azimuthal profile +noise, ice rings + +CUDA + +Spot finding +self-calibrating threshold + +CUDA + +Refine +beam centre, orientation +(Ceres-derived LM) + +Predict +Ewald crossing, partiality + +CUDA + +Integrate +2D profile fit + +Metal + +CUDA + + + + + + +Input +HDF5, CBF, marCCD, SMV; +live from the broker + +Pre-scan +beam-stop shadow, bad pixels, +beam centre, spot widths + +tested + +CUDA + +frame sample + +every frame (pass 1: first half) + +masks, spot widths + +Index +frame sample, 1D FFT + +tested + +CUDA + +Post-refine +distance, beam, cell, axis; +rival hypotheses re-run + +tested + +measured beam centre + +spots + +lattice + + +pass 1 + +pass 2, repeated while it gains +Whole dataset + +Scale & merge (P1) +3D combine of partials; +rival passes judged + +tested + +CUDA + +Symmetry +space group, twinning, +tNCS, doubled cell + +tested + +Scale & merge +in space group: corrections, +error model, resolution cut + +tested + +CUDA + +Structure & map +placement or MR, +σA and anomalous maps + +tested + +CUDA + +Report +MTZ (merged, unmerged), +mmCIF, report, maps + + + + + +without a model + +pass 2 + +CUDA +NVIDIA CUDA kernel when a GPU is present, else CPU + +Metal +also has an Apple Metal kernel + +tested +decision accepted only after a test against the data + \ No newline at end of file diff --git a/docs/images/rugnux_pipeline_fig.py b/docs/images/rugnux_pipeline_fig.py new file mode 100644 index 000000000..7ed96dbf8 --- /dev/null +++ b/docs/images/rugnux_pipeline_fig.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""Extended Data Fig. 1 — Rugnux pipeline, per-image loop vs whole-dataset steps (minimal-text version).""" + +W, H = 1600, 990 +BLUE, TEAL, PURPLE, ORANGE, INK = "#2F6FDE", "#1A8A7D", "#8E44AD", "#E8740C", "#222222" +PANEL_FILL, PANEL_STROKE = "#EEF3FC", "#B9CBEF" +GPU_BG, TEST_BG, METAL_BG = "#2F6FDE", "#1DB38B", "#555555" +GREY = "#6B6B6B" + +out = [] +def a(s): out.append(s) + +a(f'') +a('' + '' + '' + '') +a(f'') + +def box(x, y, w, h, title, body, color, gpu=False, tested=False, metal=False, dashed=False, ts=20, bs=15): + dash = ' stroke-dasharray="9,6"' if dashed else '' + a(f'') + a(f'{title}') + if body: + lines = body if isinstance(body, list) else [body] + for i, ln in enumerate(lines): + a(f'{ln}') + bx = x + w - 10 + if tested: + bx -= 62 + a(f'') + a(f'tested') + bx -= 8 + if metal: + bx -= 52 + a(f'') + a(f'Metal') + bx -= 8 + if gpu: + bx -= 52 + a(f'') + a(f'CUDA') + +def path(d, dashed=False): + dash = ' stroke-dasharray="8,6"' if dashed else '' + a(f'') + +def label(x, y, text, anchor="middle", size=14): + a(f'{text}') + +# ---------- geometry +BW, BH = 220, 105 +PX0, PX1, PY0, PY1 = 300, 1090, 300, 660 # loop panel +R1Y, R2Y = 350, 520 # rows inside panel +C = [360, 610, 860] # columns inside panel +LX = 322 # loop-return / pass-2 column + +# ---------- loop panel +a(f'') +a(f'Per image(independent)') + +box(C[0], R1Y, BW, BH, "Decode", "bitshuffle/LZ4, CBF", BLUE, gpu=True) +box(C[1], R1Y, BW, BH, "Azimuthal profile", "noise, ice rings", BLUE, gpu=True) +box(C[2], R1Y, BW, BH, "Spot finding", "self-calibrating threshold", BLUE, gpu=True) +box(C[0], R2Y, BW, BH, "Refine", ["beam centre, orientation", "(Ceres-derived LM)"], BLUE) +box(C[1], R2Y, BW, BH, "Predict", "Ewald crossing, partiality", BLUE, gpu=True) +box(C[2], R2Y, BW, BH, "Integrate", "2D profile fit", BLUE, gpu=True, metal=True) + +for r in (R1Y, R2Y): + path(f"M{C[0]+BW},{r+52} L{C[1]-2},{r+52}") + path(f"M{C[1]+BW},{r+52} L{C[2]-2},{r+52}") + +# loop return: Integrate bottom -> panel bottom bus -> left column -> Decode left edge +path(f"M{C[2]+90},{R2Y+BH} L{C[2]+90},{PY1-22} L{LX},{PY1-22} L{LX},{R1Y+52} L{C[0]-2},{R1Y+52}") + +# ---------- Input +IX, IY, IW, IH = 60, 140, 200, 100 +box(IX, IY, IW, IH, "Input", ["HDF5, CBF, marCCD, SMV;", "live from the broker"], INK, bs=14) + +# ---------- Pre-scan +SX, SY = 560, 120 +box(SX, SY, 240, BH+22, "Pre-scan", ["beam-stop shadow, bad pixels,", "beam centre, spot widths"], TEAL, gpu=True, tested=True) + +path(f"M{IX+IW},{IY+50} L{SX-2},{IY+50}") +label((IX+IW+SX)/2, IY+42, "frame sample", size=13) +path(f"M{IX+IW/2},{IY+IH} L{IX+IW/2},{IY+IH+40} L{C[0]+110},{IY+IH+40} L{C[0]+110},{PY0-2}") +label(LX+12, IY+IH+32, "every frame (pass 1: first half)", anchor="start", size=13) +path(f"M{SX+120},{SY+BH+22} L{SX+120},{PY0-2}") +label(SX+132, SY+BH+22+28, "masks, spot widths", anchor="start", size=13) + +# ---------- Index and Post-refine +DX, DW = 1200, 250 +box(DX, R1Y, DW, BH, "Index", "frame sample, 1D FFT", TEAL, gpu=True, tested=True) +box(DX, R2Y, DW, BH+22, "Post-refine", ["distance, beam, cell, axis;", "rival hypotheses re-run"], TEAL, tested=True) + +# the beam centre the pre-scan measured is a hypothesis: the first pass indexes at both centres +path(f"M{SX+240},{SY+60} L{DX+DW/2},{SY+60} L{DX+DW/2},{R1Y-2}") +label((SX+240+DX+DW/2)/2, SY+52, "measured beam centre", size=13) + +path(f"M{C[2]+BW},{R1Y+40} L{DX-2},{R1Y+40}") +label((C[2]+BW+DX)/2, R1Y+32, "spots", size=13) +BUSY = 480 +path(f"M{DX+40},{R1Y+BH} L{DX+40},{BUSY} L{C[0]+110},{BUSY} L{C[0]+110},{R2Y-2}") +label(DX+52, BUSY-8, "lattice", anchor="start", size=13) +path(f"M{C[2]+110},{R1Y+BH} L{C[2]+110},{BUSY-2}") + +path(f"M{C[2]+BW},{R2Y+52} L{DX-2},{R2Y+52}") +label((C[2]+BW+DX)/2, R2Y+44, "pass 1", size=13) + +TOPY = 80 +path(f"M{DX+DW},{R2Y+52} L{DX+DW+40},{R2Y+52} L{DX+DW+40},{TOPY} L{LX},{TOPY} L{LX},{R1Y+50}", dashed=True) +label(900, TOPY-8, "pass 2, repeated while it gains", size=14) + +# ---------- Whole-dataset chain +QY = 770 +QBW = 230 +QX = [300, 555, 810, 1065, 1320] +a(f'Whole dataset') +QBH = BH + 22 +box(QX[0], QY, QBW, QBH, "Scale & merge (P1)", ["3D combine of partials;", "rival passes judged"], TEAL, gpu=True, tested=True) +box(QX[1], QY, QBW, QBH, "Symmetry", ["space group, twinning,", "tNCS, doubled cell"], TEAL, tested=True, bs=14) +box(QX[2], QY, QBW, QBH, "Scale & merge", ["in space group: corrections,", "error model, resolution cut"], TEAL, gpu=True, tested=True, bs=14) +box(QX[3], QY, QBW, QBH, "Structure & map", ["placement or MR,", "\u03c3A and anomalous maps"], PURPLE, gpu=True, tested=True, dashed=True, bs=14) +box(QX[4], QY, QBW, QBH, "Report", ["MTZ (merged, unmerged),", "mmCIF, report, maps"], ORANGE, bs=14) +for i in range(4): + path(f"M{QX[i]+QBW},{QY+52} L{QX[i+1]-2},{QY+52}") +# without a model the merge goes straight to the report +path(f"M{QX[2]+QBW-40},{QY} L{QX[2]+QBW-40},{QY-26} L{QX[4]+40},{QY-26} L{QX[4]+40},{QY-2}") +label(QX[3]+QBW/2, QY-34, "without a model", size=13) + +OUTY = 710 +path(f"M{C[2]+160},{R2Y+BH} L{C[2]+160},{OUTY} L{QX[0]+120},{OUTY} L{QX[0]+120},{QY-2}") +label(QX[0]+135, OUTY-8, "pass 2", anchor="start", size=13) + +# ---------- legend +LY = H-40 +lx = PX0 +a(f'') +a(f'CUDA') +a(f'NVIDIA CUDA kernel when a GPU is present, else CPU') +lx += 450 +a(f'') +a(f'Metal') +a(f'also has an Apple Metal kernel') +lx += 320 +a(f'') +a(f'tested') +a(f'decision accepted only after a test against the data') + +a('') +import os +open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "rugnux_pipeline.svg"), "w").write("\n".join(out)) +print("ok") From f9fa106688aa743898eea2e543a28075cff2c925 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Thu, 15 Oct 2026 21:06:43 +0200 Subject: [PATCH 2/2] Docs: pipeline figure trimmed, autodiff badge, and a stills figure beside it Owner's review of the rotation figure: - The in-group merge names the correction surfaces it fits (decay, absorption, modulation; RotationScaleMerge) instead of the word "corrections". - New "autodiff" badge, on the fits whose derivatives are automatic: Refine and the rotation indexer's lattice refinement (XtalOptimizer/XtalRefine, forward- mode Dual numbers), Post-refine (ceres::AutoDiffCostFunction in PostRefine.cpp) and the stills geometry bundle adjustment (GeometryRefiner). Not on the rigid body (analytic plus forward differences), the stills partiality refinement (central differences), the beam-centre fit or the stills indexer's candidate refinement. The Ceres text on Refine is gone. - Text back to about the owner's budget and box sizes: the meaning moved to arrows ("pass 2, or pass 1 if it merges better" into the P1 merge, "beam centre" from the pre-scan to Index) and badges; "rival ..." phrases removed. New: the stills figure (rugnux_pipeline_stills.svg) from the same script, same style and legend. Stills = no goniometer axis or a stationary one (RugnuxRotationData). Geometry bundle adjustment first, only with a known cell (dashed); pre-scan masks the beam stop and bad pixels; per image: decode, profile, spots, index (1D FFT, or ffbidx with a cell; min-pix chosen per image by what indexes: tested), refine beam/orientation/cell, excitation-error prediction, profile fit; whole dataset: consensus cell and per-image scale (CPU), symmetry, merge with partiality post-refinement and error model, structure, report. Both figures are on RUGNUX_OVERVIEW.md, the stills one under "Stills instead". Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PXc1zD1yxHqaR1DGdCTHL4 --- docs/RUGNUX_OVERVIEW.md | 4 +- docs/images/rugnux_pipeline.svg | 114 +++++----- docs/images/rugnux_pipeline_fig.py | 300 +++++++++++++++---------- docs/images/rugnux_pipeline_stills.svg | 120 ++++++++++ 4 files changed, 363 insertions(+), 175 deletions(-) create mode 100644 docs/images/rugnux_pipeline_stills.svg diff --git a/docs/RUGNUX_OVERVIEW.md b/docs/RUGNUX_OVERVIEW.md index 705eb060a..ac2bf8aa5 100644 --- a/docs/RUGNUX_OVERVIEW.md +++ b/docs/RUGNUX_OVERVIEW.md @@ -4,7 +4,7 @@ The map of a run, in the order it happens — one paragraph per stage, each link [data-analysis reference](CPU_DATA_ANALYSIS.md) where the algorithm lives. The walk-through is a rotation run with the defaults; stills differences are at the end. -![The rotation pipeline: pass 1 integrates the first half of the sweep at the file's geometry, pass 2 the whole sweep at the refined one, and only pass 2 goes on to the space group and the written merge](images/rugnux_pipeline.svg) +![The rotation pipeline: pass 1 integrates half the sweep at the file's geometry, pass 2 the whole sweep at the refined one, and the better of the two goes on to the space group and the written merge](images/rugnux_pipeline.svg) **Open the dataset.** The geometry, wavelength and goniometer come from the file ([What Rugnux reads](RUGNUX_FORMATS.md)). A goniometer axis makes it a rotation run, none makes it @@ -76,3 +76,5 @@ subgroup ([§14](CPU_DATA_ANALYSIS_DECISIONS.md)). orientation-tilt post-refinement rather than a rocking curve, and a merohedral indexing ambiguity has to be broken per image, at integration time, against a reference or a model ([Advanced ▸ the indexing ambiguity](RUGNUX_ADVANCED.md#the-indexing-ambiguity)). + +![The stills pipeline: every image indexed on its own, the geometry refined first where the cell is known, and the crystals scaled and merged together](images/rugnux_pipeline_stills.svg) diff --git a/docs/images/rugnux_pipeline.svg b/docs/images/rugnux_pipeline.svg index 84346280c..3f41c54df 100644 --- a/docs/images/rugnux_pipeline.svg +++ b/docs/images/rugnux_pipeline.svg @@ -7,71 +7,74 @@ Decode bitshuffle/LZ4, CBF -CUDA +CUDA Azimuthal profile noise, ice rings -CUDA +CUDA Spot finding self-calibrating threshold -CUDA +CUDA Refine beam centre, orientation -(Ceres-derived LM) + +autodiff Predict Ewald crossing, partiality -CUDA +CUDA Integrate 2D profile fit -Metal +Metal -CUDA +CUDA - -Input -HDF5, CBF, marCCD, SMV; -live from the broker + +Input +live, HDF5, CBF, SMV, marCCD Pre-scan -beam-stop shadow, bad pixels, -beam centre, spot widths +beam-stop shadow, beam centre, +spot widths -tested +tested -CUDA - -frame sample - -every frame (pass 1: first half) +CUDA + +frame sample + +every frame masks, spot widths Index frame sample, 1D FFT -tested - -CUDA - +tested + +autodiff + +CUDA + Post-refine -distance, beam, cell, axis; -rival hypotheses re-run - -tested +distance, beam, cell, axis + +tested + +autodiff -measured beam centre +beam centre spots @@ -84,38 +87,32 @@ Whole dataset Scale & merge (P1) -3D combine of partials; -rival passes judged - -tested - -CUDA +3D combine of partials + +CUDA Symmetry -space group, twinning, -tNCS, doubled cell +space group, twinning, tNCS -tested +tested Scale & merge -in space group: corrections, -error model, resolution cut +in space group, resolution cut +decay, absorption, modulation -tested +tested -CUDA +CUDA Structure & map -placement or MR, -σA and anomalous maps +placement or MR, σA maps -tested +tested -CUDA +CUDA Report -MTZ (merged, unmerged), -mmCIF, report, maps +MTZ, mmCIF, maps @@ -123,14 +120,17 @@ without a model -pass 2 - -CUDA -NVIDIA CUDA kernel when a GPU is present, else CPU - -Metal -also has an Apple Metal kernel - -tested -decision accepted only after a test against the data +pass 2, or pass 1 if it merges better + +CUDA +CUDA kernel when a GPU is present + +Metal +also an Apple Metal kernel + +autodiff +derivatives by automatic differentiation + +tested +accepted only after a test against the data \ No newline at end of file diff --git a/docs/images/rugnux_pipeline_fig.py b/docs/images/rugnux_pipeline_fig.py index 7ed96dbf8..875628fe8 100644 --- a/docs/images/rugnux_pipeline_fig.py +++ b/docs/images/rugnux_pipeline_fig.py @@ -1,24 +1,41 @@ #!/usr/bin/env python3 """Extended Data Fig. 1 — Rugnux pipeline, per-image loop vs whole-dataset steps (minimal-text version).""" +# Two figures in one style: the rotation path (rugnux_pipeline.svg) and the stills path +# (rugnux_pipeline_stills.svg), both written next to this script. + +import os + W, H = 1600, 990 BLUE, TEAL, PURPLE, ORANGE, INK = "#2F6FDE", "#1A8A7D", "#8E44AD", "#E8740C", "#222222" PANEL_FILL, PANEL_STROKE = "#EEF3FC", "#B9CBEF" -GPU_BG, TEST_BG, METAL_BG = "#2F6FDE", "#1DB38B", "#555555" +GPU_BG, TEST_BG, METAL_BG, AD_BG = "#2F6FDE", "#1DB38B", "#555555", "#B7791F" GREY = "#6B6B6B" out = [] def a(s): out.append(s) -a(f'') -a('' - '' - '' - '') -a(f'') +def start(): + out.clear() + a(f'') + a('' + '' + '' + '') + a(f'') -def box(x, y, w, h, title, body, color, gpu=False, tested=False, metal=False, dashed=False, ts=20, bs=15): +def finish(name): + a('') + open(os.path.join(os.path.dirname(os.path.abspath(__file__)), name), "w").write("\n".join(out)) + +def badge(x, y, w, bg, text): + a(f'') + a(f'{text}') + +BADGES = [("tested", 62, TEST_BG), ("autodiff", 68, AD_BG), ("Metal", 52, METAL_BG), ("CUDA", 52, GPU_BG)] + +def box(x, y, w, h, title, body, color, gpu=False, tested=False, metal=False, autodiff=False, dashed=False, ts=20, bs=15): dash = ' stroke-dasharray="9,6"' if dashed else '' a(f'') a(f'{title}') @@ -27,20 +44,11 @@ def box(x, y, w, h, title, body, color, gpu=False, tested=False, metal=False, da for i, ln in enumerate(lines): a(f'{ln}') bx = x + w - 10 - if tested: - bx -= 62 - a(f'') - a(f'tested') - bx -= 8 - if metal: - bx -= 52 - a(f'') - a(f'Metal') - bx -= 8 - if gpu: - bx -= 52 - a(f'') - a(f'CUDA') + for (text, bw, bg), on in zip(BADGES, (tested, autodiff, metal, gpu)): + if on: + bx -= bw + badge(bx, y+h-28, bw, bg, text) + bx -= 8 def path(d, dashed=False): dash = ' stroke-dasharray="8,6"' if dashed else '' @@ -49,106 +57,164 @@ def path(d, dashed=False): def label(x, y, text, anchor="middle", size=14): a(f'{text}') +def panel(x0, x1, y0, y1, tx): + a(f'') + a(f'Per image(independent)') + +def legend(): + LY = H-40 + lx = 60 + for text, bw, bg, desc, step in [("CUDA", 52, GPU_BG, "CUDA kernel when a GPU is present", 330), + ("Metal", 52, METAL_BG, "also an Apple Metal kernel", 280), + ("autodiff", 68, AD_BG, "derivatives by automatic differentiation", 370), + ("tested", 62, TEST_BG, "accepted only after a test against the data", 0)]: + badge(lx, LY-14, bw, bg, text) + a(f'{desc}') + lx += step + # ---------- geometry BW, BH = 220, 105 -PX0, PX1, PY0, PY1 = 300, 1090, 300, 660 # loop panel +PY0, PY1 = 300, 660 # loop panel R1Y, R2Y = 350, 520 # rows inside panel -C = [360, 610, 860] # columns inside panel LX = 322 # loop-return / pass-2 column - -# ---------- loop panel -a(f'') -a(f'Per image(independent)') - -box(C[0], R1Y, BW, BH, "Decode", "bitshuffle/LZ4, CBF", BLUE, gpu=True) -box(C[1], R1Y, BW, BH, "Azimuthal profile", "noise, ice rings", BLUE, gpu=True) -box(C[2], R1Y, BW, BH, "Spot finding", "self-calibrating threshold", BLUE, gpu=True) -box(C[0], R2Y, BW, BH, "Refine", ["beam centre, orientation", "(Ceres-derived LM)"], BLUE) -box(C[1], R2Y, BW, BH, "Predict", "Ewald crossing, partiality", BLUE, gpu=True) -box(C[2], R2Y, BW, BH, "Integrate", "2D profile fit", BLUE, gpu=True, metal=True) - -for r in (R1Y, R2Y): - path(f"M{C[0]+BW},{r+52} L{C[1]-2},{r+52}") - path(f"M{C[1]+BW},{r+52} L{C[2]-2},{r+52}") - -# loop return: Integrate bottom -> panel bottom bus -> left column -> Decode left edge -path(f"M{C[2]+90},{R2Y+BH} L{C[2]+90},{PY1-22} L{LX},{PY1-22} L{LX},{R1Y+52} L{C[0]-2},{R1Y+52}") - -# ---------- Input -IX, IY, IW, IH = 60, 140, 200, 100 -box(IX, IY, IW, IH, "Input", ["HDF5, CBF, marCCD, SMV;", "live from the broker"], INK, bs=14) - -# ---------- Pre-scan -SX, SY = 560, 120 -box(SX, SY, 240, BH+22, "Pre-scan", ["beam-stop shadow, bad pixels,", "beam centre, spot widths"], TEAL, gpu=True, tested=True) - -path(f"M{IX+IW},{IY+50} L{SX-2},{IY+50}") -label((IX+IW+SX)/2, IY+42, "frame sample", size=13) -path(f"M{IX+IW/2},{IY+IH} L{IX+IW/2},{IY+IH+40} L{C[0]+110},{IY+IH+40} L{C[0]+110},{PY0-2}") -label(LX+12, IY+IH+32, "every frame (pass 1: first half)", anchor="start", size=13) -path(f"M{SX+120},{SY+BH+22} L{SX+120},{PY0-2}") -label(SX+132, SY+BH+22+28, "masks, spot widths", anchor="start", size=13) - -# ---------- Index and Post-refine -DX, DW = 1200, 250 -box(DX, R1Y, DW, BH, "Index", "frame sample, 1D FFT", TEAL, gpu=True, tested=True) -box(DX, R2Y, DW, BH+22, "Post-refine", ["distance, beam, cell, axis;", "rival hypotheses re-run"], TEAL, tested=True) - -# the beam centre the pre-scan measured is a hypothesis: the first pass indexes at both centres -path(f"M{SX+240},{SY+60} L{DX+DW/2},{SY+60} L{DX+DW/2},{R1Y-2}") -label((SX+240+DX+DW/2)/2, SY+52, "measured beam centre", size=13) - -path(f"M{C[2]+BW},{R1Y+40} L{DX-2},{R1Y+40}") -label((C[2]+BW+DX)/2, R1Y+32, "spots", size=13) -BUSY = 480 -path(f"M{DX+40},{R1Y+BH} L{DX+40},{BUSY} L{C[0]+110},{BUSY} L{C[0]+110},{R2Y-2}") -label(DX+52, BUSY-8, "lattice", anchor="start", size=13) -path(f"M{C[2]+110},{R1Y+BH} L{C[2]+110},{BUSY-2}") - -path(f"M{C[2]+BW},{R2Y+52} L{DX-2},{R2Y+52}") -label((C[2]+BW+DX)/2, R2Y+44, "pass 1", size=13) - -TOPY = 80 -path(f"M{DX+DW},{R2Y+52} L{DX+DW+40},{R2Y+52} L{DX+DW+40},{TOPY} L{LX},{TOPY} L{LX},{R1Y+50}", dashed=True) -label(900, TOPY-8, "pass 2, repeated while it gains", size=14) - -# ---------- Whole-dataset chain -QY = 770 -QBW = 230 +IX, IY, IW, IH = 60, 150, 220, 80 # input +SY = 120 # pre-scan row +QY, QBW, QBH = 770, 230, BH + 22 # whole-dataset row QX = [300, 555, 810, 1065, 1320] -a(f'Whole dataset') -QBH = BH + 22 -box(QX[0], QY, QBW, QBH, "Scale & merge (P1)", ["3D combine of partials;", "rival passes judged"], TEAL, gpu=True, tested=True) -box(QX[1], QY, QBW, QBH, "Symmetry", ["space group, twinning,", "tNCS, doubled cell"], TEAL, tested=True, bs=14) -box(QX[2], QY, QBW, QBH, "Scale & merge", ["in space group: corrections,", "error model, resolution cut"], TEAL, gpu=True, tested=True, bs=14) -box(QX[3], QY, QBW, QBH, "Structure & map", ["placement or MR,", "\u03c3A and anomalous maps"], PURPLE, gpu=True, tested=True, dashed=True, bs=14) -box(QX[4], QY, QBW, QBH, "Report", ["MTZ (merged, unmerged),", "mmCIF, report, maps"], ORANGE, bs=14) -for i in range(4): - path(f"M{QX[i]+QBW},{QY+52} L{QX[i+1]-2},{QY+52}") -# without a model the merge goes straight to the report -path(f"M{QX[2]+QBW-40},{QY} L{QX[2]+QBW-40},{QY-26} L{QX[4]+40},{QY-26} L{QX[4]+40},{QY-2}") -label(QX[3]+QBW/2, QY-34, "without a model", size=13) - OUTY = 710 -path(f"M{C[2]+160},{R2Y+BH} L{C[2]+160},{OUTY} L{QX[0]+120},{OUTY} L{QX[0]+120},{QY-2}") -label(QX[0]+135, OUTY-8, "pass 2", anchor="start", size=13) -# ---------- legend -LY = H-40 -lx = PX0 -a(f'') -a(f'CUDA') -a(f'NVIDIA CUDA kernel when a GPU is present, else CPU') -lx += 450 -a(f'') -a(f'Metal') -a(f'also has an Apple Metal kernel') -lx += 320 -a(f'') -a(f'tested') -a(f'decision accepted only after a test against the data') +def dataset_row(merge_p1, symmetry, merge, gpu): + a(f'Whole dataset') + box(QX[0], QY, QBW, QBH, "Scale & merge (P1)", merge_p1, TEAL, gpu=gpu) + box(QX[1], QY, QBW, QBH, "Symmetry", symmetry, TEAL, tested=True, bs=14) + box(QX[2], QY, QBW, QBH, "Scale & merge", merge, TEAL, gpu=gpu, tested=gpu, bs=14) + box(QX[3], QY, QBW, QBH, "Structure & map", "placement or MR, σA maps", PURPLE, gpu=True, tested=True, dashed=True, bs=14) + box(QX[4], QY, QBW, QBH, "Report", "MTZ, mmCIF, maps", ORANGE) + for i in range(4): + path(f"M{QX[i]+QBW},{QY+52} L{QX[i+1]-2},{QY+52}") + # without a model the merge goes straight to the report + path(f"M{QX[2]+QBW-40},{QY} L{QX[2]+QBW-40},{QY-26} L{QX[4]+40},{QY-26} L{QX[4]+40},{QY-2}") + label(QX[3]+QBW/2, QY-34, "without a model", size=13) -a('') -import os -open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "rugnux_pipeline.svg"), "w").write("\n".join(out)) + +def rotation(): + start() + PX0, PX1 = 300, 1090 + C = [360, 610, 860] # columns inside panel + + # ---------- loop panel + panel(PX0, PX1, PY0, PY1, C[0]) + box(C[0], R1Y, BW, BH, "Decode", "bitshuffle/LZ4, CBF", BLUE, gpu=True) + box(C[1], R1Y, BW, BH, "Azimuthal profile", "noise, ice rings", BLUE, gpu=True) + box(C[2], R1Y, BW, BH, "Spot finding", "self-calibrating threshold", BLUE, gpu=True) + box(C[0], R2Y, BW, BH, "Refine", "beam centre, orientation", BLUE, autodiff=True) + box(C[1], R2Y, BW, BH, "Predict", "Ewald crossing, partiality", BLUE, gpu=True) + box(C[2], R2Y, BW, BH, "Integrate", "2D profile fit", BLUE, gpu=True, metal=True) + + for r in (R1Y, R2Y): + path(f"M{C[0]+BW},{r+52} L{C[1]-2},{r+52}") + path(f"M{C[1]+BW},{r+52} L{C[2]-2},{r+52}") + + # loop return: Integrate bottom -> panel bottom bus -> left column -> Decode left edge + path(f"M{C[2]+90},{R2Y+BH} L{C[2]+90},{PY1-22} L{LX},{PY1-22} L{LX},{R1Y+52} L{C[0]-2},{R1Y+52}") + + # ---------- Input + box(IX, IY, IW, IH, "Input", "live, HDF5, CBF, SMV, marCCD", INK, bs=14) + + # ---------- Pre-scan + SX = 560 + box(SX, SY, 240, BH+22, "Pre-scan", ["beam-stop shadow, beam centre,", "spot widths"], TEAL, gpu=True, tested=True) + + path(f"M{IX+IW},{IY+40} L{SX-2},{IY+40}") + label((IX+IW+SX)/2, IY+32, "frame sample", size=13) + path(f"M{IX+IW/2},{IY+IH} L{IX+IW/2},{IY+IH+40} L{C[0]+110},{IY+IH+40} L{C[0]+110},{PY0-2}") + label(LX+12, IY+IH+32, "every frame", anchor="start", size=13) + path(f"M{SX+120},{SY+BH+22} L{SX+120},{PY0-2}") + label(SX+132, SY+BH+22+28, "masks, spot widths", anchor="start", size=13) + + # ---------- Index and Post-refine + DX, DW = 1200, 250 + box(DX, R1Y, DW, BH, "Index", "frame sample, 1D FFT", TEAL, gpu=True, autodiff=True, tested=True) + box(DX, R2Y, DW, BH, "Post-refine", "distance, beam, cell, axis", TEAL, autodiff=True, tested=True) + + # the beam centre the pre-scan measured is a hypothesis: the first pass indexes at both centres + path(f"M{SX+240},{SY+60} L{DX+DW/2},{SY+60} L{DX+DW/2},{R1Y-2}") + label((SX+240+DX+DW/2)/2, SY+52, "beam centre", size=13) + + path(f"M{C[2]+BW},{R1Y+40} L{DX-2},{R1Y+40}") + label((C[2]+BW+DX)/2, R1Y+32, "spots", size=13) + BUSY = 480 + path(f"M{DX+40},{R1Y+BH} L{DX+40},{BUSY} L{C[0]+110},{BUSY} L{C[0]+110},{R2Y-2}") + label(DX+52, BUSY-8, "lattice", anchor="start", size=13) + path(f"M{C[2]+110},{R1Y+BH} L{C[2]+110},{BUSY-2}") + + path(f"M{C[2]+BW},{R2Y+52} L{DX-2},{R2Y+52}") + label((C[2]+BW+DX)/2, R2Y+44, "pass 1", size=13) + + TOPY = 80 + path(f"M{DX+DW},{R2Y+52} L{DX+DW+40},{R2Y+52} L{DX+DW+40},{TOPY} L{LX},{TOPY} L{LX},{R1Y+50}", dashed=True) + label(900, TOPY-8, "pass 2, repeated while it gains", size=14) + + # ---------- Whole-dataset chain + dataset_row("3D combine of partials", "space group, twinning, tNCS", + ["in space group, resolution cut", "decay, absorption, modulation"], gpu=True) + path(f"M{C[2]+160},{R2Y+BH} L{C[2]+160},{OUTY} L{QX[0]+120},{OUTY} L{QX[0]+120},{QY-2}") + label(QX[0]+135, OUTY-8, "pass 2, or pass 1 if it merges better", anchor="start", size=13) + + legend() + finish("rugnux_pipeline.svg") + + +def stills(): + start() + PX0, PX1 = 300, 1340 + C = [360, 610, 860, 1110] # columns inside panel + + # ---------- loop panel: the first row left to right, the second back right to left + panel(PX0, PX1, PY0, PY1, C[0]) + box(C[0], R1Y, BW, BH, "Decode", "bitshuffle/LZ4, CBF", BLUE, gpu=True) + box(C[1], R1Y, BW, BH, "Azimuthal profile", "noise per ring", BLUE, gpu=True) + box(C[2], R1Y, BW, BH, "Spot finding", "self-calibrating threshold", BLUE, gpu=True) + box(C[3], R1Y, BW, BH, "Index", "1D FFT, or ffbidx with a cell", BLUE, gpu=True, tested=True, bs=14) + box(C[3], R2Y, BW, BH, "Refine", "beam, orientation, cell", BLUE, autodiff=True) + box(C[2], R2Y, BW, BH, "Predict", "excitation-error cutoff", BLUE, gpu=True) + box(C[1], R2Y, BW, BH, "Integrate", "2D profile fit", BLUE, gpu=True, metal=True) + + for i in range(3): + path(f"M{C[i]+BW},{R1Y+52} L{C[i+1]-2},{R1Y+52}") + path(f"M{C[3]+BW/2},{R1Y+BH} L{C[3]+BW/2},{R2Y-2}") + path(f"M{C[3]},{R2Y+52} L{C[2]+BW+2},{R2Y+52}") + path(f"M{C[2]},{R2Y+52} L{C[1]+BW+2},{R2Y+52}") + + # loop return: Integrate left edge -> left column -> Decode left edge + path(f"M{C[1]},{R2Y+52} L{LX},{R2Y+52} L{LX},{R1Y+52} L{C[0]-2},{R1Y+52}") + + # ---------- Input, geometry refinement, pre-scan + box(IX, IY, IW, IH, "Input", "HDF5, CBF, SMV, marCCD", INK, bs=14) + GX, SX = 560, 900 + box(GX, SY, 240, BH+22, "Geometry", ["with a known cell:", "beam, distance, cell"], TEAL, autodiff=True, dashed=True) + box(SX, SY, 240, BH+22, "Pre-scan", ["beam-stop shadow,", "bad pixels"], TEAL, gpu=True) + + path(f"M{IX+IW},{IY+40} L{GX-2},{IY+40}") + label((IX+IW+GX)/2, IY+32, "strong frames", size=13) + path(f"M{GX+240},{IY+40} L{SX-2},{IY+40}") + path(f"M{IX+IW/2},{IY+IH} L{IX+IW/2},{IY+IH+40} L{C[0]+110},{IY+IH+40} L{C[0]+110},{PY0-2}") + label(LX+12, IY+IH+32, "every frame", anchor="start", size=13) + path(f"M{GX+120},{SY+BH+22} L{GX+120},{PY0-2}") + label(GX+132, SY+BH+22+28, "geometry", anchor="start", size=13) + path(f"M{SX+120},{SY+BH+22} L{SX+120},{PY0-2}") + label(SX+132, SY+BH+22+28, "masks", anchor="start", size=13) + + # ---------- Whole-dataset chain + dataset_row(["consensus cell,", "per-image scale"], "space group, twinning, tNCS", + ["in space group, resolution cut", "partiality, error model"], gpu=False) + path(f"M{C[1]+110},{R2Y+BH} L{C[1]+110},{OUTY} L{QX[0]+120},{OUTY} L{QX[0]+120},{QY-2}") + label(QX[0]+135, OUTY-8, "every crystal", anchor="start", size=13) + + legend() + finish("rugnux_pipeline_stills.svg") + + +rotation() +stills() print("ok") diff --git a/docs/images/rugnux_pipeline_stills.svg b/docs/images/rugnux_pipeline_stills.svg new file mode 100644 index 000000000..87b93a83c --- /dev/null +++ b/docs/images/rugnux_pipeline_stills.svg @@ -0,0 +1,120 @@ + + + + +Per image(independent) + +Decode +bitshuffle/LZ4, CBF + +CUDA + +Azimuthal profile +noise per ring + +CUDA + +Spot finding +self-calibrating threshold + +CUDA + +Index +1D FFT, or ffbidx with a cell + +tested + +CUDA + +Refine +beam, orientation, cell + +autodiff + +Predict +excitation-error cutoff + +CUDA + +Integrate +2D profile fit + +Metal + +CUDA + + + + + + + + +Input +HDF5, CBF, SMV, marCCD + +Geometry +with a known cell: +beam, distance, cell + +autodiff + +Pre-scan +beam-stop shadow, +bad pixels + +CUDA + +strong frames + + +every frame + +geometry + +masks +Whole dataset + +Scale & merge (P1) +consensus cell, +per-image scale + +Symmetry +space group, twinning, tNCS + +tested + +Scale & merge +in space group, resolution cut +partiality, error model + +Structure & map +placement or MR, σA maps + +tested + +CUDA + +Report +MTZ, mmCIF, maps + + + + + +without a model + +every crystal + +CUDA +CUDA kernel when a GPU is present + +Metal +also an Apple Metal kernel + +autodiff +derivatives by automatic differentiation + +tested +accepted only after a test against the data + \ No newline at end of file