diff --git a/tools/battery/README.md b/tools/battery/README.md index 9e5a161b2..3dc260c3d 100644 --- a/tools/battery/README.md +++ b/tools/battery/README.md @@ -267,10 +267,11 @@ R-free have a reference. Every crashed set is listed again under the table (`CRA `$B table RUN` prints it again for any finished run. **Crashes.** A run is a `crash` - its own verdict, cause `crash` - when rugnux was killed by a -signal, or its log or terminal output has `Processing failed`, a GPU error (`CUDA (GPU) error`, also -one rugnux handled and went on from: on a validation run that is a defect, not a property of the -data), `terminate called`, `Segmentation fault`, or a compute-sanitizer `ERROR SUMMARY` that is not -zero (`score.crash_note`). It is decided before anything else: a crash fails the set whatever its +signal, or its log or terminal output has `Processing failed`, a device fault (`unspecified launch +failure`, `illegal memory access` and the like - also one rugnux handled and went on from: that is a +defect, not a property of the data; a planned fallback such as a buffer that did not fit is not one), +`terminate called`, `Segmentation fault`, or a compute-sanitizer `ERROR SUMMARY` that is not zero +(`score.crash_note`). It is decided before anything else: a crash fails the set whatever its report says, a no-crystal control included (their normal `No crystal lattice found`, exit 1, is not a crash). A crash counts as a failure in every pass rate, gets its own `CRASHED` section at the top of the report, and is read again from the log when an older run is reported or compared, so an old diff --git a/tools/battery/battery.py b/tools/battery/battery.py index b9e37f464..9d6fb6a5c 100644 --- a/tools/battery/battery.py +++ b/tools/battery/battery.py @@ -42,7 +42,7 @@ manifest.json (what was run, on what, with which references), results.json (one report.md + report.html and work/// (log, report, cif). It is made read-only when done. A full-arm run (no --only / --tier) takes the site lock and refuses to start if another holds it. A run exits with status 1 when any set crashed (score.crash_note: killed by a signal, "Processing -failed", a GPU error even where rugnux went on) or a set of the ci tier it ran failed (lattice, +failed", a device fault even where rugnux went on) or a set of the ci tier it ran failed (lattice, symmetry, no input). `stress` runs sets repeatedly, side by side or under compute-sanitizer and only asks whether any of them crashed. A run that did not finish is never compared or reported on unless asked (--allow-incomplete). diff --git a/tools/battery/score.py b/tools/battery/score.py index c120c1ee6..0dd4dec7f 100644 --- a/tools/battery/score.py +++ b/tools/battery/score.py @@ -55,10 +55,15 @@ def read_report(path): return out -# A line in rugnux's log or terminal output that makes a run a crash. A GPU error counts even where -# rugnux handled it and went on: on a validation run it is a defect, not a property of the data. -CRASH_LINE = re.compile(r"Processing failed|CUDA \(GPU\) error|terminate called|Segmentation fault" - r"|AddressSanitizer|ERROR SUMMARY: [1-9]") +# A line in rugnux's log or terminal output that makes a run a crash. A device fault (a kernel or copy +# that touched memory it must not, which takes the CUDA context down for every thread) counts even +# where rugnux handled it and went on: it is a defect, not a property of the data. A GPU failure that +# rugnux plans for - a buffer that does not fit, answered on the CPU - is not one. +DEVICE_FAULT = (r"unspecified launch failure|illegal memory access|illegal address|misaligned address" + r"|illegal instruction|launch timed out|hardware stack error|device-side assert" + r"|CUDA device unusable") +CRASH_LINE = re.compile(r"Processing failed|terminate called|Segmentation fault|AddressSanitizer" + r"|ERROR SUMMARY: [1-9]|" + DEVICE_FAULT) def crash_note(exit_code, wd): diff --git a/tools/battery/test_score.py b/tools/battery/test_score.py index 8eb28d298..81829b2bb 100644 --- a/tools/battery/test_score.py +++ b/tools/battery/test_score.py @@ -212,9 +212,12 @@ class Crashes(unittest.TestCase): self.assertIn("unspecified launch failure", self.log("[t] [rugnux] [error] Processing failed: CUDA (GPU) error " "(RotationScaleMergeGPU: upload: unspecified launch failure)\n", 1)) - # a GPU error rugnux handled still counts, exit 0 or not - self.assertTrue(self.log("[t] [Rugnux] [warning] Beam stop: GPU accumulate failed " - "(CUDA (GPU) error (out of memory)), falling back to the host\n", 0)) + # a device fault rugnux handled still counts, exit 0 or not; a planned capacity fallback does not + self.assertTrue(self.log("[t] [error] Indexer thread 1 failed: CUDA (GPU) error " + "(an illegal memory access was encountered)\n", 0)) + self.assertEqual(self.log("[t] [Rugnux] [warning] Model validation: the rigid body failed on the " + "GPU (CUDA (GPU) error (rigid body: zone over the engine's reserve)); " + "validating again on the CPU\n", 0), "") self.assertTrue(self.log("", -11)) # rugnux's own refusal, a CUDA build's banner and a timeout are not crashes self.assertEqual(self.log("[t] [rugnux] [info] rugnux Version 1 - CUDA build\n"