Add from_results_dict / from_results_h5 model loading from saved h5 files

Adds two classmethods to CDIModel and its ptychography subclasses that
allow models to be reconstructed from saved .h5 result files without
needing the original dataset.

- CDIModel base class: add model_class and cdtools_version to
  save_results(); add _load_results_dict() helper (restores state_dict
  + training metadata); add from_results_dict() interface method and a
  concrete from_results_h5() that reads a file and delegates to
  from_results_dict() — subclasses inherit this for free.

- SimplePtycho: make save_results(dataset=None) optional (dataset was
  accepted but never used); add from_results_dict() which reconstructs
  from probe, obj, wavelength, probe_basis, and min_translation stored
  in the state_dict.

- FancyPtycho: add optional translations parameter to __init__,
  registered as original_translations buffer; update from_dataset to
  pass translations; make corrected_translations(dataset=None) fall back
  to self.original_translations when no dataset is provided; make
  save_results(dataset=None) use stored translations when no dataset is
  provided; add from_results_dict() that detects all optional features
  (mask, translation_offsets, weights, near-field propagators, etc.)
  from the state_dict and reconstructs the full model. Also fix a latent
  bug where background was passed to t.nn.Parameter() without
  t.as_tensor(), which now fails when given a numpy array.

All existing tests pass; new tests added for both models verifying that
state_dict, training metadata, and forward pass output are all restored
exactly after a round-trip through from_results_dict and from_results_h5.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
allevitanandClaude Sonnet 4.6 committed 2026-04-13 15:47:07 +02:00
1 parent 3f028e2fbf
commit 6aa5df081b
5 files changed
+291 -11

No files matched your search

+68 -2
View File
@@ -36,7 +36,8 @@ from matplotlib import ticker
import numpy as np
import time
from contextlib import contextmanager
from cdtools.tools.data import nested_dict_to_h5, nested_dict_to_numpy, nested_dict_to_torch
import cdtools
from cdtools.tools.data import nested_dict_to_h5, nested_dict_to_numpy, nested_dict_to_torch, h5_to_nested_dict
from cdtools.reconstructors import AdamReconstructor, LBFGSReconstructor, SGDReconstructor
from cdtools.datasets import CDataset
from typing import List, Union, Tuple
@@ -193,6 +194,8 @@ class CDIModel(t.nn.Module):
'epoch': self.epoch,
'training_history': self.training_history,
'loss_function': self.loss.__name__,
'model_class': type(self).__name__,
'cdtools_version': cdtools.__version__,
}
@@ -207,7 +210,70 @@ class CDIModel(t.nn.Module):
Accepts any additional args that model.save_results needs, for this model
"""
return nested_dict_to_h5(filename, self.save_results(*args))
def _load_results_dict(self, results_dict):
"""Restores model state and training metadata from a results dictionary.
This is the kernel used by from_results_dict implementations. It loads
the state_dict (all parameters and buffers) and restores loss history,
epoch count, and training history.
Parameters
----------
results_dict : dict
The dictionary returned by save_results(), as loaded from an h5 file
or produced directly in memory.
"""
state_dict = nested_dict_to_torch(results_dict['state_dict'])
self.load_state_dict(state_dict)
self.loss_history = list(results_dict['loss_history'])
self.epoch = int(results_dict['epoch'])
self.training_history = str(results_dict['training_history'])
@classmethod
def from_results_dict(cls, results_dict):
"""Reconstructs a model from the dictionary returned by save_results().
Must be implemented by each subclass. The base class raises
NotImplementedError.
Parameters
----------
results_dict : dict
The dictionary returned by save_results(), as loaded from an h5 file
or produced directly in memory.
Returns
-------
model : CDIModel subclass
A fully reconstructed model with all parameters, buffers, and
training metadata restored.
"""
raise NotImplementedError()
@classmethod
def from_results_h5(cls, filename):
"""Reconstructs a model directly from a saved .h5 result file.
Reads the file into a dictionary and delegates to cls.from_results_dict.
Subclasses inherit this method and only need to implement from_results_dict.
Parameters
----------
filename : str or Path
Path to the .h5 file saved by save_to_h5.
Returns
-------
model : CDIModel subclass
A fully reconstructed model with all parameters, buffers, and
training metadata restored.
"""
return cls.from_results_dict(h5_to_nested_dict(filename))
@contextmanager
def save_on_exit(self, filename, *args, exception_filename=None):
+87 -7
View File
@@ -45,6 +45,7 @@ class FancyPtycho(CDIModel):
inv_angular_spectrum_propagator=None,
panel_plot_mode=True,
plot_level=2,
translations=None,
):
super(FancyPtycho, self).__init__(panel_plot_mode=panel_plot_mode,
@@ -155,7 +156,7 @@ class FancyPtycho(CDIModel):
shape = [s//oversampling for s in self.probe[0]]
background = 1e-6 * t.ones(shape, dtype=t.float32)
self.background = t.nn.Parameter(background)
self.background = t.nn.Parameter(t.as_tensor(background, dtype=dtype))
if weights is None:
self.weights = None
@@ -226,6 +227,10 @@ class FancyPtycho(CDIModel):
else:
raise KeyError('Specified loss function not supported')
if translations is not None:
self.register_buffer('original_translations',
t.as_tensor(translations, dtype=dtype))
@classmethod
def from_dataset(cls,
@@ -523,6 +528,7 @@ class FancyPtycho(CDIModel):
inv_angular_spectrum_propagator=inv_angular_spectrum_propagator,
panel_plot_mode=panel_plot_mode,
plot_level=plot_level,
translations=translations,
)
@@ -710,9 +716,18 @@ class FancyPtycho(CDIModel):
mask=mask)
def corrected_translations(self, dataset):
translations = dataset.translations.to(
dtype=t.float32, device=self.probe.device)
def corrected_translations(self, dataset=None):
if dataset is not None:
translations = dataset.translations.to(
dtype=t.float32, device=self.probe.device)
elif (hasattr(self, 'original_translations') and
self.original_translations is not None):
translations = self.original_translations.to(
dtype=t.float32, device=self.probe.device)
else:
raise ValueError(
'Must provide a dataset or have original_translations stored '
'internally (via from_dataset or from_results_dict).')
if (hasattr(self, 'translation_offsets') and
self.translation_offsets is not None):
t_offset = tools.interactions.pixel_to_translations(
@@ -1169,7 +1184,7 @@ class FancyPtycho(CDIModel):
]
def save_results(self, dataset):
def save_results(self, dataset=None):
# This will save out everything needed to recreate the object
# in the same state, but it's not the best formatted. For example,
# "background" stores the square root of the background, etc.
@@ -1178,8 +1193,11 @@ class FancyPtycho(CDIModel):
# We also save out the main results in a more readable format
obj_basis = self.obj_basis.detach().cpu().numpy()
probe_basis = self.probe_basis.detach().cpu().numpy()
translations=self.corrected_translations(dataset).detach().cpu().numpy()
original_translations = dataset.translations.detach().cpu().numpy()
translations = self.corrected_translations(dataset).detach().cpu().numpy()
if dataset is not None:
original_translations = dataset.translations.detach().cpu().numpy()
else:
original_translations = self.original_translations.detach().cpu().numpy()
probe = self.probe.detach().cpu().numpy()
probe = probe * self.probe_norm.detach().cpu().numpy()
obj = self.obj.detach().cpu().numpy()
@@ -1202,3 +1220,65 @@ class FancyPtycho(CDIModel):
}
return {**base_results, **results}
@classmethod
def from_results_dict(cls, results_dict):
"""Reconstructs a FancyPtycho model from a results dictionary.
Parameters
----------
results_dict : dict
The dictionary returned by save_results(), as loaded from an h5 file
or produced directly in memory.
Returns
-------
model : FancyPtycho
A fully reconstructed model with all parameters, buffers, and
training metadata restored.
"""
import numpy as np
sd = results_dict['state_dict']
# For optional Parameters (translation_offsets, weights, qe_mask, etc.),
# we pass the saved values directly so they get registered as
# Parameters/buffers before _load_results_dict overwrites them with the
# exact saved state via load_state_dict.
translation_offsets = sd.get('translation_offsets')
model = cls(
wavelength=sd['wavelength'],
detector_geometry={
'basis': sd['det_basis'],
'distance': sd.get('det_distance'),
'corner': sd.get('det_corner'),
},
obj_basis=sd['obj_basis'],
probe_guess=sd['probe'], # normalized; probe_norm restored by _load_results_dict
obj_guess=sd['obj'],
surface_normal=sd['surface_normal'],
min_translation=sd['min_translation'],
background=sd['background'], # sqrt form; restored exactly by _load_results_dict
probe_basis=sd.get('probe_basis'),
translation_offsets=translation_offsets, # overwritten by _load_results_dict
probe_fourier_shifts=sd.get('probe_fourier_shifts'), # overwritten by _load_results_dict
mask=sd.get('mask'),
weights=sd.get('weights'), # overwritten by _load_results_dict
qe_mask=sd.get('qe_mask'), # overwritten by _load_results_dict
saturation=sd.get('saturation'),
translation_scale=float(sd['translation_scale']),
oversampling=int(sd['oversampling']),
fourier_probe=bool(sd['fourier_probe']),
loss=results_dict.get('loss_function', 'amplitude mse'),
simulate_probe_translation=bool(sd['simulate_probe_translation']),
simulate_finite_pixels=bool(sd['simulate_finite_pixels']),
exponentiate_obj=bool(sd['exponentiate_obj']),
phase_only=bool(sd['phase_only']),
near_field=bool(sd['near_field']),
angular_spectrum_propagator=sd.get('angular_spectrum_propagator'),
inv_angular_spectrum_propagator=sd.get('inv_angular_spectrum_propagator'),
translations=sd.get('original_translations'),
)
model._load_results_dict(results_dict)
return model
+31 -2
View File
@@ -130,9 +130,9 @@ class SimplePtycho(CDIModel):
},
]
def save_results(self, dataset):
def save_results(self, dataset=None):
# This will save out everything needed to recreate the object
# in the same state, but it's not the best formatted.
# in the same state, but it's not the best formatted.
base_results = super().save_results()
# So we also save out the main results in a more useable format
@@ -150,3 +150,32 @@ class SimplePtycho(CDIModel):
}
return {**base_results, **results}
@classmethod
def from_results_dict(cls, results_dict):
"""Reconstructs a SimplePtycho model from a results dictionary.
Parameters
----------
results_dict : dict
The dictionary returned by save_results(), as loaded from an h5 file
or produced directly in memory.
Returns
-------
model : SimplePtycho
A fully reconstructed model with all parameters, buffers, and
training metadata restored.
"""
sd = results_dict['state_dict']
model = cls(
wavelength=sd['wavelength'],
probe_basis=sd['probe_basis'],
probe_guess=sd['probe'], # normalized; probe_norm restored by _load_results_dict
obj_guess=sd['obj'],
min_translation=sd['min_translation'],
)
model._load_results_dict(results_dict)
return model
+59
View File
@@ -166,3 +166,62 @@ def test_near_field_ptycho(near_field_ptycho_cxi, reconstruction_device, show_pl
# If this fails, the reconstruction has gotten worse
assert model.loss_history[-1] < 0.005
def test_fancy_ptycho_from_results_dict(lab_ptycho_cxi, tmp_path):
dataset = cdtools.datasets.Ptycho2DDataset.from_cxi(lab_ptycho_cxi)
t.manual_seed(42)
model = cdtools.models.FancyPtycho.from_dataset(
dataset,
n_modes=2,
)
# Verify original_translations is stored after from_dataset
assert hasattr(model, 'original_translations')
assert model.original_translations is not None
# Run a few epochs to get non-trivial state
for loss in model.Adam_optimize(5, dataset, batch_size=10):
pass
# Test from_results_dict with in-memory dict (no dataset argument needed)
results_dict = model.save_results()
loaded_model = cdtools.models.FancyPtycho.from_results_dict(results_dict)
# Test from_results_h5 via a temporary file
h5_path = str(tmp_path / 'fancy_ptycho_test.h5')
model.save_to_h5(h5_path)
loaded_model_h5 = cdtools.models.FancyPtycho.from_results_h5(h5_path)
# Verify training metadata is restored
assert loaded_model.epoch == model.epoch
assert loaded_model.loss_history == model.loss_history
# Verify original_translations round-trips correctly
assert t.allclose(
loaded_model.original_translations,
model.original_translations,
)
# Verify all parameters and buffers are restored exactly
original_sd = model.state_dict()
loaded_sd = loaded_model.state_dict()
loaded_h5_sd = loaded_model_h5.state_dict()
for key in original_sd:
assert t.allclose(original_sd[key].float(), loaded_sd[key].float()), \
f'from_results_dict: state_dict mismatch for key {key}'
assert t.allclose(original_sd[key].float(), loaded_h5_sd[key].float()), \
f'from_results_h5: state_dict mismatch for key {key}'
# Verify forward pass produces identical output
(indices, translations), patterns = dataset[:5]
with t.no_grad():
original_out = model(indices, translations)
loaded_out = loaded_model(indices, translations)
loaded_h5_out = loaded_model_h5(indices, translations)
assert t.allclose(original_out, loaded_out), \
'from_results_dict: forward pass output mismatch'
assert t.allclose(original_out, loaded_h5_out), \
'from_results_h5: forward pass output mismatch'
+46
View File
@@ -9,6 +9,52 @@ import cdtools
t.manual_seed(0)
def test_simple_ptycho_from_results_dict(lab_ptycho_cxi, tmp_path):
dataset = cdtools.datasets.Ptycho2DDataset.from_cxi(lab_ptycho_cxi)
t.manual_seed(42)
model = cdtools.models.SimplePtycho.from_dataset(dataset)
# Run a few epochs to get non-trivial state
for loss in model.Adam_optimize(5, dataset, batch_size=10):
pass
# Test from_results_dict with in-memory dict (no dataset argument needed)
results_dict = model.save_results()
loaded_model = cdtools.models.SimplePtycho.from_results_dict(results_dict)
# Test from_results_h5 via a temporary file
h5_path = str(tmp_path / 'simple_ptycho_test.h5')
model.save_to_h5(h5_path)
loaded_model_h5 = cdtools.models.SimplePtycho.from_results_h5(h5_path)
# Verify training metadata is restored
assert loaded_model.epoch == model.epoch
assert loaded_model.loss_history == model.loss_history
# Verify all parameters and buffers are restored exactly
original_sd = model.state_dict()
loaded_sd = loaded_model.state_dict()
loaded_h5_sd = loaded_model_h5.state_dict()
for key in original_sd:
assert t.allclose(original_sd[key].float(), loaded_sd[key].float()), \
f'from_results_dict: state_dict mismatch for key {key}'
assert t.allclose(original_sd[key].float(), loaded_h5_sd[key].float()), \
f'from_results_h5: state_dict mismatch for key {key}'
# Verify forward pass produces identical output
(indices, translations), patterns = dataset[:5]
with t.no_grad():
original_out = model(indices, translations)
loaded_out = loaded_model(indices, translations)
loaded_h5_out = loaded_model_h5(indices, translations)
assert t.allclose(original_out, loaded_out), \
'from_results_dict: forward pass output mismatch'
assert t.allclose(original_out, loaded_h5_out), \
'from_results_h5: forward pass output mismatch'
@pytest.mark.slow
def test_simple_ptycho(lab_ptycho_cxi, reconstruction_device, show_plot):
dataset = cdtools.datasets.Ptycho2DDataset.from_cxi(lab_ptycho_cxi)