xie_x1andClaude Opus 5.5
0f9a3f496f
Code, configs, charge LUTs, best.pth and evaluation records for each training run, the knife-edge / Siemens-star / XRF notebooks with outputs, and the overnight run reports. Epoch checkpoints, inference results and HTML exports stay local (see .gitignore). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
112 lines
4.4 KiB
Bash
112 lines
4.4 KiB
Bash
#!/usr/bin/env bash
|
|
# Overnight run (2026-09-27/28), part B: the measured runs and the notebooks.
|
|
# Started by overnight_poly2to20keV_Moench040.sh once part A (training) is done
|
|
# and state/partB.ready exists; can also be run by hand:
|
|
#
|
|
# setsid nohup bash scripts/overnight_poly2to20keV_Moench040_partB.sh \
|
|
# Results/<experiment> >> Results/overnight_poly2to20keV_Moench040/driver.log 2>&1 < /dev/null &
|
|
#
|
|
# Order: dark runs (fake rate, 1 min) -> knife edge 3/2/1/0 filters, edge + flat
|
|
# (63 M frames of 101x101) -> knife-edge notebook -> Siemens star, four parts of
|
|
# 500 files -> Siemens-star notebook -> HTML copies of both notebooks.
|
|
# A failed run is logged and skipped; the notebooks report what is missing.
|
|
|
|
set -u
|
|
cd /home/xie_x1/MLXID/GridBased
|
|
ENV_BIN=/home/xie_x1/software/miniforge3/envs/pytorch_nightly/bin
|
|
PY=$ENV_BIN/python
|
|
BASE=Results/overnight_poly2to20keV_Moench040
|
|
LOG=$BASE/logs
|
|
STATE=$BASE/state
|
|
STATUS=$BASE/status.md
|
|
REPORT=$BASE/report
|
|
KNIFE=Configs/infer_measurement_poly_Moench040_knife.yaml
|
|
STAR=Configs/infer_measurement_poly_Moench040_star.yaml
|
|
TAG=poly2to20
|
|
mkdir -p "$LOG" "$STATE" "$REPORT"
|
|
|
|
experiment=${1:-$(cat "$STATE/experiment" 2>/dev/null)}
|
|
checkpoint=$experiment/Models/best.pth
|
|
if [ ! -f "$checkpoint" ]; then
|
|
echo "- $(date '+%m-%d %H:%M:%S') part B: no checkpoint at '$checkpoint'" >> "$STATUS"
|
|
exit 1
|
|
fi
|
|
# the CPU side of the inference is small per-batch work; the default thread
|
|
# count (16) thrashes as soon as anything else runs (32 s instead of 0.1 s per
|
|
# decode batch next to the frame generator)
|
|
export OMP_NUM_THREADS=4 MKL_NUM_THREADS=4
|
|
export PATH=$ENV_BIN:$PATH # the notebooks' kernel runs `python`
|
|
export GRID_POLY_EXPERIMENT=$experiment GRID_POLY_TAG=$TAG
|
|
|
|
note() {
|
|
echo "- $(date '+%m-%d %H:%M:%S') $*" >> "$STATUS"
|
|
echo "$(date '+%m-%d %H:%M:%S') $*"
|
|
}
|
|
|
|
run() {
|
|
local name=$1; shift
|
|
if [ -f "$STATE/$name.done" ]; then
|
|
note "$name: already done, skipped"; return 0
|
|
fi
|
|
note "$name: started"
|
|
local started=$(date +%s)
|
|
"$@" > "$LOG/$name.log" 2>&1
|
|
local status=$?
|
|
local minutes=$(( ($(date +%s) - started) / 60 ))
|
|
if [ $status -eq 0 ]; then
|
|
touch "$STATE/$name.done"
|
|
note "$name: done (${minutes} min) $(grep -o '([0-9]* frame/s)' "$LOG/$name.log" | tail -1)"
|
|
else
|
|
note "$name: **FAILED** (exit $status after ${minutes} min), see logs/$name.log"
|
|
fi
|
|
return $status
|
|
}
|
|
|
|
notebook() { # notebook name -> executed in place + HTML copy in report/
|
|
local name=$1
|
|
jupyter nbconvert --to notebook --execute --inplace \
|
|
--ExecutePreprocessor.timeout=3600 --ExecutePreprocessor.kernel_name=python3 \
|
|
"Notebooks/$name.ipynb" && \
|
|
jupyter nbconvert --to html --output-dir "$REPORT" "Notebooks/$name.ipynb"
|
|
}
|
|
|
|
note "=== part B started (pid $$), model $checkpoint ==="
|
|
|
|
# Two lanes in parallel: the knife edge is bound by the GPU and the Python
|
|
# loop (101x101 frames, ~10^4 frame/s), the Siemens star by reading the NFS
|
|
# store and remapping ~820 photons per frame (~3000 frame/s), so they mostly
|
|
# use different resources.
|
|
|
|
knife_lane() {
|
|
# dark runs first: fake photons on 1000 frames each (a minute)
|
|
for run_name in Pedestal_1filters_12keV Pedestal_2filters_12keV Pedestal_3filters_pos0_12keV; do
|
|
run infer_$run_name $PY scripts/05_infer_measurement.py --config $KNIFE \
|
|
--run $run_name --checkpoint "$checkpoint" --tag $TAG
|
|
done
|
|
# the four flux points, edge and flat (63 M frames, ~1.5-2 h)
|
|
for filters in 3filters_pos0 2filters 1filters 0filters; do
|
|
for kind in Edge Flat; do
|
|
run infer_${kind}_${filters} $PY scripts/05_infer_measurement.py --config $KNIFE \
|
|
--run ${kind}_${filters}_12keV --checkpoint "$checkpoint" --tag $TAG
|
|
done
|
|
done
|
|
rm -f "$STATE/notebook_knife.done" # always re-run on what exists
|
|
run notebook_knife notebook KnifeEdge_poly2to20keV_Moench040
|
|
}
|
|
|
|
star_lane() {
|
|
# lower left + lower right in one crop, four parts of 500 files (~2 h)
|
|
for part in 0 1 2 3; do
|
|
run infer_star_part$part $PY scripts/05_infer_measurement.py --config $STAR \
|
|
--run SiemensStar_15keV_part$part --checkpoint "$checkpoint" --tag $TAG
|
|
done
|
|
rm -f "$STATE/notebook_star.done"
|
|
run notebook_star notebook SiemensStar_poly2to20keV_15keV
|
|
}
|
|
|
|
knife_lane &
|
|
star_lane &
|
|
wait
|
|
|
|
note "=== part B finished; notebooks in Notebooks/, HTML copies in $REPORT ==="
|