"""Finding a dataset's input file, and reading an XDS CORRECT.LP as a reference. Both run only when a manifest is (re)built - `battery.py discover` and `battery.py refs`. A battery run itself never searches: it runs the input the manifest names, so two runs of one manifest always process the same files. """ import glob import os import re # XDS writes calibration images beside the frames; none of them is a sweep. XDS_AUX = ("-CORRECTIONS", "BKGINIT", "BKGPIX", "BLANK", "GAIN", "DECAY", "ABS.", "MODPIX", "FRAME.") def master_frames(path): """Number of frames a NXmx master describes. A sweep is chosen by frame count, never by file size: a master with a few embedded screening frames can outweigh the master of a real sweep whose frames sit in external data files.""" import h5py try: with h5py.File(path, "r") as h: det = "entry/instrument/detector/detectorSpecific" if f"{det}/nimages" in h: n = int(h[f"{det}/nimages"][()]) if f"{det}/ntrigger" in h: n *= max(1, int(h[f"{det}/ntrigger"][()])) return n omega = "entry/sample/goniometer/omega" if omega in h and h[omega].shape: return int(h[omega].shape[0]) n = 0 for k in h.get("entry/data", {}): try: n += int(h["entry/data"][k].shape[0]) except (KeyError, OSError): # an external link whose file is missing pass return n except OSError: return 0 def detector_frame(path): """'marCCD' or 'SMV' if the file's header says so, else None. These formats have no reliable extension (.mccd, .img, .001, ...; .img is SMV at one site and marCCD at the next).""" try: with open(path, "rb") as fh: head = fh.read(64) if head[:2] in (b"II", b"MM"): fh.seek(1028) return "marCCD" if fh.read(3) == b"MMX" else None if head[:1] == b"{" and b"HEADER_BYTES" in head: return "SMV" except OSError: pass return None def find_input(d): """(input file, format, note) for the largest sweep under directory d, or (None, None, why). Frames are grouped per directory by template (the last run of digits in the name), and the biggest group by frame count wins; a tie goes to the alphabetically first. Groups are never merged across directories - two crystals can have identically named frames side by side.""" masters = [m for m in glob.glob(f"{d}/**/*master*.h5", recursive=True) if not os.path.basename(m).startswith("._")] if not masters: # Diamond writes a NeXus .nxs master beside _NNNNNN.h5 data files masters = [m for m in glob.glob(f"{d}/**/*.nxs", recursive=True) if not os.path.basename(m).startswith("._") and "meta" not in os.path.basename(m)] if masters: best = min(masters, key=lambda m: (-master_frames(m), m)) return best, "nxs" if best.endswith(".nxs") else "h5", \ f"master, {master_frames(best)} frames, {len(masters)} master(s)" groups = {} for p in glob.glob(f"{d}/**/*", recursive=True): name = os.path.basename(p) if not os.path.isfile(p) or name.startswith("._") or any(a in name.upper() for a in XDS_AUX): continue key = (os.path.dirname(p), re.sub(r"\d+(?=\D*$)", "#", name)) groups.setdefault(key, []).append(p) for key, files in sorted(groups.items(), key=lambda kv: (-len(kv[1]), kv[0])): if len(files) < 10: break files.sort() if files[0].endswith((".cbf", ".cbf.gz")): fmt = "cbf" else: fmt = detector_frame(files[0]) if fmt: return files[0], fmt, f'{fmt} "{key[1]}" x{len(files)} of {len(groups)} group(s)' return None, None, "no readable sweep found" def input_format(path): """Format tag of an input file named in a manifest.""" if path.endswith(".h5"): return "h5" if path.endswith(".nxs"): return "nxs" if path.endswith((".cbf", ".cbf.gz")): return "cbf" return detector_frame(path) or "other" def parse_correct_lp(path): """Reference values from an XDS CORRECT.LP: the space group and cell CORRECT refined, the resolution range it merged (dmin_xds, dmax), the reference d_min read off its statistics (dmin, dmin_rule - see reference_dmin) and its overall merging statistics. CORRECT.LP states the group and cell twice - first as the INPUT to INTEGRATE (routinely P1), then as CORRECT's own answer. The last SPACE_GROUP_NUMBER and the refined UNIT CELL PARAMETERS are the ones the statistics were computed in.""" txt = open(path, errors="replace").read() ref = {} sgs = re.findall(r"SPACE_GROUP_NUMBER=\s*(\d+)", txt) ref["sgno"] = int(sgs[-1]) if sgs else None cells = re.findall(r"UNIT CELL PARAMETERS\s+([\d.\s]+)", txt) if cells: ref["cell"] = [float(x) for x in cells[-1].split()[:6]] else: m = re.findall(r"UNIT_CELL_CONSTANTS=\s*([\d.\s]+)", txt) ref["cell"] = [float(x) for x in m[-1].split()[:6]] if m else None m = re.search(r"FRIEDEL'S_LAW=\s*(TRUE|FALSE)", txt) ref["anomalous"] = bool(m and m.group(1) == "FALSE") m = re.search(r"^\s*a\s+b\s+ISa\s*\n\s*[\d.Ee+-]+\s+[\d.Ee+-]+\s+([\d.]+)", txt, re.M) ref["isa"] = float(m.group(1)) if m else None # the last "... AS FUNCTION OF RESOLUTION" table; shells from low to high resolution, then total seg = txt[txt.rfind("AS FUNCTION OF RESOLUTION"):] num = lambda x: float(x.rstrip("*%")) shells, total = [], None for line in seg.splitlines(): f = line.split() if f and f[0] == "total" and len(f) >= 11: total = f break if f and re.match(r"^\d+\.\d+$", f[0]) and len(f) >= 11: shells.append(f) if total: ref["completeness"] = num(total[4]) ref["r_meas"] = round(num(total[9]) / 100, 4) ref["cc_half"] = round(num(total[10]) / 100, 4) ref["multiplicity"] = round(int(total[1]) / int(total[2]), 2) if int(total[2]) else None # The range XDS merged: the explicit INCLUDE_RESOLUTION_RANGE, or - when XDS.INP left the high # limit at 0.0, "the whole detector" - the finest shell actually tabulated. The pooled statistics # above are over this range. m = re.search(r"INCLUDE_RESOLUTION_RANGE=\s*([\d.]+)\s+([\d.]+)", txt) include_low, include_high = (float(m.group(1)), float(m.group(2))) if m else (0.0, 0.0) table_high = float(shells[-1][0]) if shells else 0.0 dmin_xds = (include_high if include_high > 0 else table_high) or None ref["dmin"], ref["dmin_rule"] = reference_dmin( [(float(f[0]), num(f[10]) / 100, f[10].endswith("*")) for f in shells], include_low, dmin_xds) ref["dmin_xds"] = dmin_xds ref["dmax"] = include_low or None return ref # The CC1/2 target rugnux's own resolution cutoff uses (--resolution-cc-target, default 0.30) CC_TARGET = 0.30 def reference_dmin(shells, dmax, dmin_xds): """(reference d_min, rule) from XDS's shell table [(shell high-resolution limit, CC1/2, CC1/2 significant)], low to high resolution. XDS's merged range is a resolution only when someone chose it. Left at "the whole detector" (INCLUDE_RESOLUTION_RANGE 0.0, or a limit at the detector edge) it is the detector's reach, not the crystal's: scored against, it is a meaningless number, and forced on rugnux it makes rugnux merge shells of pure noise. XDS itself says which case it is - it marks each shell's CC1/2 with '*' when it is significant at the 0.1% level: xds_range the finest shell's CC1/2 is significant: XDS's limit stands. cc_half_0.30 it is not - XDS merged past its own signal. The reference is then where CC1/2 falls through the target rugnux's own cutoff uses, interpolated in 1/d^2 between the centres of the last shell above it and the first below it.""" if not shells or shells[-1][2]: return dmin_xds, "xds_range" below = next((i for i, (_, cc, _) in enumerate(shells) if cc < CC_TARGET), None) if not below: # None, or already the first shell: no fall-off to read return dmin_xds, "xds_range" s = [1 / d ** 2 for d, _, _ in shells] edges = [1 / dmax ** 2 if dmax else 0.0] + s centre = [(edges[i] + edges[i + 1]) / 2 for i in range(len(s))] cc_hi, cc_lo = shells[below - 1][1], shells[below][1] frac = (cc_hi - CC_TARGET) / (cc_hi - cc_lo) s_cross = centre[below - 1] + frac * (centre[below] - centre[below - 1]) return round(1 / s_cross ** 0.5, 3), f"cc_half_{CC_TARGET:.2f}"