added projectors, propagators

This commit is contained in:
Maddie Cain committed 2019-03-27 21:57:23 -04:00
commit f1c3df3545
10 files changed
+1266 -316

No files matched your search

+1
View File
@@ -1,3 +1,4 @@
from __future__ import division, print_function, absolute_import
from CDTools import tools
from CDTools import datasets
+216
View File
@@ -0,0 +1,216 @@
from __future__ import division, print_function, absolute_import
import numpy as np
import torch as t
from copy import copy
# The naming overlap here is definitely going to get confusing.
from CDTools.tools import data as cdtdata
from torch.utils import data as torchdata
__all__ = ['CDataset', 'Ptycho_2D_Dataset']
#
# I think that here should live a variety of datasets that all
# subclass the basic dataset class. They should be able to contain
# all the information that would be available in a CDI-type experiment.
#
# It's important for each kind of dataset to have tools to:
#
# * Pass too and from the GPU (where to store the data)
# * Be able to load itself intelligently from a cxi file
# * Be able to save itself intelligently to a cxi file
#
# And in addition, it should be relatively simple to initialize from
# data held as python structures
#
# The datasets need to be able to work just like pytorch datasets,
# in fact, they should subclass them, where they return the relevant
# information for each diffraction pattern when sliced.
#
#
# One question, though, is how to deal with information that models need
# to know as well as the datasets? The basic issue is that it would be
# nice to be able to write down a forward model and have it populate
# a dataset object with the simulated data. But you also want to be able
# to write a forward model that pulls the relevant information from a
# dataset that has been read from a real data's cxi file.
#
# One workaround would be to just have the things defined twice, but to
# make sure that every model can load/initialize itself from a dataset.
# If it also knows how to define itself from an initialization function
# or a python dataset, then it can easily also have a generic function to
# simulate the action of the model and create a dataset from that simulation.
#
# This is unrelated, but it will then be important to be able to save and load
# models easily from a predefined format. To be honest, this could just
# literally be a snippet of python code that redefines the specific model
# using only the base CDTools packages. That way a saved model could be a
# .py file that just creates the model on the spot. Not sure I love this
# though
#
# Is the trio of data, model, and reconstruction the correct thing though?
# Perhaps the reconstruction doesn't need to exist. The model can have
# a few more things involved in it, and the reconstructions could be a
# single function, for example, that only uses what's in the model.
#
# The issue with the automatic differentiation stuff is that it's history
# dependent in a sense. So you generate an optimization object and you take
# a step with it.
#
# Can all reconstruction algorithms be formulated in terms of an explicit
# forward step, backward step, and gradient with respect to the forward
# step? No, for example the position annealing step needs to do it's own
# thing. Okay, well then it could be possible to write an automatic
# differentiation sequence that can generate n steps of a given automatic
# differentiation solved. So given a model, you could just generically
# create n steps of automatic differentiation. But then also write an
# explicit ePIE step, etc. It could be interesting if the reconstruction
# algorithms defined generators which would run one step and then yield
# the loss at the end of that step. This would let you easily save out
# the loss but also would make it simple to include live plotting
#
#
# Then instead of a plan, you would just write a python script to follow
# the steps you wanted to follow.
#
#
# I think there should be a base CDataSet that has entry_info, sample_info, detector_info, and mask attributes. It can know how to load this data from a cxi file and write it out to a cxi file. It also can know how to pass this info to and from the GPU (not all of it needs to be passed in that way).
# Then, I will write a few common datasets as a demonstration. First is
# A 2D CDI dataset, second is a 2D Ptycho dataset.
#
# This loads and stores all the kinds of metadata that are common to
# All different kinds of diffraction experiments
# Other datasets can subclass this and not worry about loading and
# saving that metadata.
#
class CDataset(torchdata.Dataset):
def __init__(self, entry_info=None, sample_info=None,
wavelength=None,
detector_geometry=None, mask=None):
# Force pass-by-value-like behavior to stop strangeness
self.entry_info = copy(entry_info)
self.sample_info = copy(sample_info)
self.wavelength = wavelength
self.detector_geometry = copy(detector_geometry)
if mask is not None:
self.mask = t.tensor(mask)
else:
self.mask = None
def to(self,*args,**kwargs):
# The mask should always stay a uint8, but it should switch devices
mask_kwargs = copy(kwargs)
try:
mask_kwargs.pop('dtype')
except KeyError as r:
pass
if self.mask is not None:
self.mask = self.mask.to(*args,**mask_kwargs)
@classmethod
def from_cxi(cls, cxi_file):
entry_info = cdtdata.get_entry_info(cxi_file)
sample_info = cdtdata.get_sample_info(cxi_file)
wavelength = cdtdata.get_wavelength(cxi_file)
distance, basis, corner = cdtdata.get_detector_geometry(cxi_file)
detector_geometry = {'distance' : distance,
'basis' : basis,
'corner' : corner}
mask = cdtdata.get_mask(cxi_file)
return cls(entry_info = entry_info,
sample_info = sample_info,
wavelength=wavelength,
detector_geometry=detector_geometry,
mask=mask)
def to_cxi(self, cxi_file):
if self.entry_info is not None:
cdtdata.add_entry_info(cxi_file, self.entry_info)
if self.sample_info is not None:
cdtdata.add_sample_info(cxi_file, self.sample_info)
if self.wavelength is not None:
cdtdata.add_source(cxi_file, self.wavelength)
if self.detector_geometry is not None:
if 'corner' in self.detector_geometry:
corner = self.detector_geometry['corner']
else:
corner = None
cdtdata.add_detector(cxi_file,
self.detector_geometry['distance'],
self.detector_geometry['basis'],
corner = corner)
if self.mask is not None:
cdtdata.add_mask(cxi_file, self.mask)
#
# This is the standard dataset for a 2D ptychography experiment,
# which saves and loads files compatible with most reconstruction
# programs (only tested against SHARP)
#
class Ptycho_2D_Dataset(CDataset):
def __init__(self, translations, patterns, axes=None, *args, **kwargs):
super(Ptycho_2D_Dataset,self).__init__(*args, **kwargs)
self.axes = copy(axes)
self.translations = t.tensor(translations)
self.patterns = t.tensor(patterns)
def __len__(self):
return self.patterns.shape[0]
def __getitem__(self,index):
return index, self.translations[index], self.patterns[index]
def to(self, *args, **kwargs):
super(Ptycho_2D_Dataset,self).to(*args,**kwargs)
self.translations = self.translations.to(*args, **kwargs)
self.patterns = self.patterns.to(*args, **kwargs)
# It sucks that I can't reuse the base factory method here,
# perhaps there is a way but I couldn't figure it out.
@classmethod
def from_cxi(cls, cxi_file):
entry_info = cdtdata.get_entry_info(cxi_file)
sample_info = cdtdata.get_sample_info(cxi_file)
wavelength = cdtdata.get_wavelength(cxi_file)
distance, basis, corner = cdtdata.get_detector_geometry(cxi_file)
detector_geometry = {'distance' : distance,
'basis' : basis,
'corner' : corner}
mask = cdtdata.get_mask(cxi_file)
patterns, axes = cdtdata.get_data(cxi_file)
translations = cdtdata.get_ptycho_translations(cxi_file)
return cls(translations, patterns, axes=axes,
entry_info = entry_info,
sample_info = sample_info,
wavelength=wavelength,
detector_geometry=detector_geometry,
mask=mask)
def to_cxi(self, cxi_file):
super(Ptycho_2D_Dataset,self).to_cxi(cxi_file)
cdtdata.add_data(cxi_file, self.patterns, axes=self.axes)
cdtdata.add_ptycho_translations(cxi_file, self.translations)
+1
View File
@@ -0,0 +1 @@
from __future__ import division, print_function, absolute_import
+96
View File
@@ -0,0 +1,96 @@
from __future__ import division, print_function, absolute_import
import h5py
import numpy as np
import os
from PyQt5 import QtWidgets
from matplotlib import pyplot as plt
import signal
import sys
import argparse
#
# The plotting was broken by the move to CDTools, at some point this should
# be fixed
#
def view_cxi(filename):
"""Opens a popup window displaying the contents of ``filename``.
relevant attributes."""
signal.signal(signal.SIGINT, signal.SIG_DFL)
class Viewer(QtWidgets.QMainWindow):
def __init__(self, datafile):
self.datafile = datafile
QtWidgets.QMainWindow.__init__(self)
self.tree = QtWidgets.QTreeWidget(self)
self.tree.setColumnWidth(0, 200)
self.setCentralWidget(self.tree)
self.buildTree()
self.tree.itemClicked.connect(self.handleClick)
def handleClick(self,item,column):
if(item.text(column) == 'Click to print to console'):
data = self.data_full[str(item.text(2))]
print(np.asarray(data))
if(item.text(column) == 'Click to display'):
data = self.datasets[str(item.text(2))]
plt.plot(data)
plt.title(item.text(0).capitalize())
plt.show()
def closeWindow(self):
pass
def buildTree(self):
self.datasets = {}
self.data_full = {}
self.tree.setColumnCount(2)
self.f = h5py.File(self.datafile, 'r')
item = QtWidgets.QTreeWidgetItem(['/'])
self.tree.addTopLevelItem(item)
self.buildBranch(self.f,item)
def buildBranch(self,group,item):
for g in group.keys():
lst = [g]
self.data_full[group[g].name] = group[g]
if(isinstance(group[g],h5py.Group)):
child = QtWidgets.QTreeWidgetItem(lst)
self.buildBranch(group[g],child)
item.addChild(child)
else:
if len(group[g].shape)>2:
lst.append('Click to print to console')
lst.append(group[g].name)
self.datasets[group[g].name] = group[g]
item.addChild(QtWidgets.QTreeWidgetItem(lst))
if len(group[g].shape)==2 or len(group[g].shape)==1:
lst.append('Click to display')
lst.append(group[g].name)
self.datasets[group[g].name] = group[g]
item.addChild(QtWidgets.QTreeWidgetItem(lst))
else:
lst.append('Click to print to console')
lst.append(group[g].name)
self.datasets[group[g].name] = group[g]
item.addChild(QtWidgets.QTreeWidgetItem(lst))
filename = os.path.expanduser(filename)
app = QtWidgets.QApplication(sys.argv)
viewer = Viewer(filename)
viewer.setFixedSize(500, 500)
viewer.show()
app.exec_()
def make_argparser():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('file', help='The cxi file to view')
return parser
if __name__ == '__main__':
args = make_argparser().parse_args()
view_cxi(args.file)
+3
View File
@@ -3,8 +3,11 @@ from __future__ import division, print_function, absolute_import
from CDTools.tools import cmath
from CDTools.tools import losses
from CDTools.tools import data
<<<<<<< HEAD
from CDTools.tools import image_processing
from CDTools.tools import initializers
from CDTools.tools import losses
from CDTools.tools import projectors
from CDTools.tools import propagators
=======
>>>>>>> bad75422717fd0eecfe1a136fddc2b96828a3321
+267 -27
View File
@@ -2,6 +2,11 @@ from __future__ import division, print_function, absolute_import
import h5py
import numpy as np
import numbers
import datetime
import dateutil.parser
import torch as t
from contextlib import contextmanager
__all__ = ['get_entry_info',
'get_sample_info',
@@ -9,7 +14,15 @@ __all__ = ['get_entry_info',
'get_detector_geometry',
'get_mask',
'get_data',
'get_ptycho_translations']
'get_ptycho_translations',
'create_cxi',
'add_entry_info',
'add_sample_info',
'add_source',
'add_detector',
'add_mask',
'add_data',
'add_ptycho_translations']
#
#
@@ -56,13 +69,19 @@ __all__ = ['get_entry_info',
#
#
# Functions to inspect the basic attributes of a cxi file represented as an
# h5 file object
#
def get_entry_info(cxi_file):
"""Returns a dictionary with the basic metadata from the cxi file's entry_1 attribute
String type metadata is read out as a string, and datetime metadata
is converted to python datetime objects if the string is properly
formatted.
Args:
cxi_file (h5py.File) : a file object to be read
@@ -75,11 +94,17 @@ def get_entry_info(cxi_file):
metadata_attrs = ['title',
'experiment_identifier',
'experiment_description',
'program_name',
'start_time',
'end_time']
'program_name']
metadata = {attr: str(e1[attr][()].decode()) for attr in metadata_attrs
if attr in e1}
datetime_attrs = ['start_time',
'end_time']
for attr in datetime_attrs:
if attr in e1:
try:
metadata[attr] = dateutil.parser.parse(str(e1[attr][()].decode()))
except ValueError:
metadata[attr] = str(e1[attr][()].decode())
return metadata
@@ -118,6 +143,10 @@ def get_sample_info(cxi_file):
# TODO: I should add the sample geometry as a valid metadata that can
# be copied over
# Check if the metadata is empty
if metadata == {}:
metadata = None
return metadata
@@ -290,12 +319,10 @@ def get_data(cxi_file):
def get_ptycho_translations(cxi_file):
"""Gets an array of x,y,z translations, if such an array has been defined in the file
It applies two operations to the translations. First, it negates them,
because the CXI file format is designed to specify translations of the
samples and the CDTools code specifies translations of the optics.
Second, it transposes the array so that the first axis is translation
ID and the second axis is the (x,y,z) components of the translation
It negates the translations, because the CXI file format is designed
to specify translations of the samples and the CDTools code specifies
translations of the optics.
Args:
cxi_file (h5py.File) : a file object to be read
@@ -313,32 +340,245 @@ def get_ptycho_translations(cxi_file):
else:
raise KeyError('Translations are not defined within cxi file')
translations = -np.array(cxi_file[pull_from]).astype(np.float32).transpose()
translations = -np.array(cxi_file[pull_from]).astype(np.float32)
return translations
#
# It might be useful to make some helper functions to help write cxi files
#
#
# A function to place the skeleton of a cxi file down
#
#
# A function to define the source attributes
#
def create_cxi(filename):
"""Creates a new cxi file with a single entry group
#
# A function to define the detector geometry
#
Args:
filename (str) : The path at which to create the file
"""
file_obj = h5py.File(filename,'w')
file_obj.create_dataset('cxi_version', data=160)
file_obj.create_dataset('number_of_entries',data=1)
e1f = file_obj.create_group('entry_1')
return file_obj
#
# A function to save out a mask, converting it to the correct format
#
#
# Perhaps a function to store the data and link it correctly? But this might
# have to change too much situation to situation
#
def add_entry_info(cxi_file, metadata):
"""Adds a dictionary of entry metadata to the entry_1 group of a cxi file object
Args:
cxi_file (h5py.File) : The file to add the info to
metadata (dict) : A dictionary containing all the metadata to be stored
"""
# Just the string and datetime types should be relevant but all are
# included in case the cxi spec becomes more permissive
for key, value in metadata.items():
if isinstance(value,(str,bytes)):
cxi_file['entry_1'][key] = np.string_(value)
elif isinstance(value, datetime.datetime):
cxi_file['entry_1'][key] = np.string_(value.isoformat())
elif isinstance(value, numbers.Number):
si[key] = value
elif isinstance(value, (np.ndarray,list,tuple)):
s1.create_dataset(key, data=np.asarray(value))
elif isinstance(value, t.Tensor):
asnumpy = value.detach().cpu().numpy()
cxi_file['entry_1'].create_dataset(key, data=asnumpy)
def add_sample_info(cxi_file, metadata):
"""Adds a dictionary of entry metadata to the entry_1/sample_1 group of a cxi file object
This function will create the sample_1 attribute if it doesn't already exist
Args:
cxi_file (h5py.File) : The file to add the info to
metadata (dict) : A dictionary containing all the metadata to be stored
"""
if 'entry_1/sample_1' not in cxi_file:
cxi_file['entry_1'].create_group('sample_1')
s1 = cxi_file['entry_1/sample_1']
for key, value in metadata.items():
if isinstance(value,(str,bytes)):
s1[key] = np.string_(value)
elif isinstance(value, datetime.datetime):
s1[key] = np.string_(value.isoformat())
elif isinstance(value, numbers.Number):
s1[key] = value
elif isinstance(value, (np.ndarray,list,tuple)):
s1.create_dataset(key, data=np.asarray(value))
elif isinstance(value, t.Tensor):
asnumpy = value.detach().cpu().numpy()
s1.create_dataset(key, data=asnumpy)
def add_source(cxi_file, wavelength):
"""Adds the entry_1/source_1 group to a cxi file object
It stores the energy and wavelength attributes in the source_1 group,
given a wavelength to define them from.
Args:
cxi_file (h5py.File) : The file to add the source to
wavelength (float) : The wavelength of light
"""
if 'entry_1/instrument_1' not in cxi_file:
cxi_file['entry_1'].create_group('instrument_1')
i1 = cxi_file['entry_1/instrument_1']
if 'source_1' not in i1:
i1.create_group('source_1')
s1 = i1['source_1']
s1['wavelength'] = np.float32(wavelength)
s1['energy'] = np.float32(1.9864459e-25 / wavelength)
def add_detector(cxi_file, distance, basis, corner=None):
"""Adds the entry_1/instrument_1/detector_1 group to a cxi file object
It will define all the relevant parameters - distance, pixel size,
detector basis, and corner position (if relevant) based on the provided
information
Args:
cxi_file (h5py.File) : The file to add the detector to
distance (float) : The sample to detector distance
basis (array_like) : The detector basis
corner (array_like) : Optional, the corner position of the detector
"""
if 'entry_1/instrument_1' not in cxi_file:
cxi_file['entry_1'].create_group('instrument_1')
i1 = cxi_file['entry_1/instrument_1']
if 'detector_1' not in i1:
i1.create_group('detector_1')
d1 = i1['detector_1']
d1['distance'] = np.float32(distance)
d1['x_pixel_size'] = np.linalg.norm(basis[:,1])
d1['y_pixel_size'] = np.linalg.norm(basis[:,0])
if isinstance(basis, t.Tensor):
basis = basis.detach().cpu().numpy()
d1.create_dataset('basis_vectors', data=basis)
if corner is not None:
if isinstance(corner, t.Tensor):
corner = corner.detach().cpu().numpy()
d1.create_dataset('corner_position',data=corner)
def add_mask(cxi_file, mask):
"""Adds the specified mask to the cxi file
It places the mask into the mask dataset under
entry_1/instrument_1/detector_1. The internal mask is defined
simply as a 1 for an "on" pixel and a 0 for an "off" pixel, and
the saved mask is exactly the opposite. This is simpler than the
most general mask allowed by the cxi file format but it captures the
distinction between pixels to be used and pixels not to be used.
Args:
cxi_file (h5py.File) : The file to add the mask to
mask (array_like) : The mask to save out to the file
"""
if 'entry_1/instrument_1' not in cxi_file:
cxi_file['entry_1'].create_group('instrument_1')
i1 = cxi_file['entry_1/instrument_1']
if 'detector_1' not in i1:
i1.create_group('detector_1')
d1 = i1['detector_1']
if isinstance(mask, t.Tensor):
mask = mask.detach().cpu().numpy()
mask_to_save = np.zeros(mask.shape).astype(np.uint32)
mask_to_save[mask == 0] = 1
d1.create_dataset('mask',data=mask_to_save)
def add_data(cxi_file, data, axes=None):
"""Adds the specified data to the cxi file
It will add the data unchanged to the file, placing it in two spots:
1) The entry_1/instrument_1/detector_1/data path
2) A softlink at entry_1/data_1/data
Args:
cxi_file (h5py.File) : The file to add the data to
data (array_like) : The data to be saved
axes (list) : Optional, a list of axis names to be saved in the axes attribute
"""
if 'entry_1/data_1' not in cxi_file:
cxi_file['entry_1'].create_group('data_1')
data1 = cxi_file['entry_1/data_1']
if 'entry_1/instrument_1' not in cxi_file:
cxi_file['entry_1'].create_group('instrument_1')
i1 = cxi_file['entry_1/instrument_1']
if 'detector_1' not in i1:
i1.create_group('detector_1')
det1 = i1['detector_1']
if isinstance(data, t.Tensor):
data = data.detach().cpu().numpy()
det1.create_dataset('data', data=data)
data1['data'] = h5py.SoftLink('/entry_1/instrument_1/detector_1/data')
if axes is not None:
if isinstance(axes, list):
axes_str = ':'.join(axes)
else:
axes_str = str(axes)
det1['data'].attrs['axes'] = np.string_(axes_str)
def add_ptycho_translations(cxi_file, translations):
"""Adds the specified translations to the cxi file
It will add the translations to the file, negating them to conform to
the standard in cxi files that the translations refer to the object's
translation.
It will generally store them in 3 places:
1) The entry_1/sample_1/geometry_1/translation path
2) A softlink at entry_1/data_1/translation
3) A softlink at entry_1/instrument_1/detector_1/translation
Args:
cxi_file (h5py.File) : The file to add the translations to
translations (array_like) : The translations to be saved
"""
if 'entry_1/sample_1' not in cxi_file:
cxi_file['entry_1'].create_group('sample_1')
s1 = cxi_file['entry_1/sample_1']
if 'geometry_1' not in s1:
s1.create_group('geometry_1')
g1 = s1['geometry_1']
if 'entry_1/data_1' not in cxi_file:
cxi_file['entry_1'].create_group('data_1')
data1 = cxi_file['entry_1/data_1']
if 'entry_1/instrument_1' not in cxi_file:
cxi_file['entry_1'].create_group('instrument_1')
i1 = cxi_file['entry_1/instrument_1']
if 'detector_1' not in i1:
i1.create_group('detector_1')
det1 = i1['detector_1']
if isinstance(translations, t.Tensor):
translations = translations.detach().cpu().numpy()
# accounting for the different definition between cxi files and
# CDTools
translations = -translations
g1.create_dataset('translation', data=translations)
data1['translation'] = h5py.SoftLink('/entry_1/sample_1/geometry_1/translation')
det1['translation'] = h5py.SoftLink('/entry_1/sample_1/geometry_1/translation')
+1
View File
@@ -16,6 +16,7 @@ setuptools.setup(
"numpy",
"scipy",
"matplotlib",
"dateutil",
#"pytorch",
"h5py"],
packages=setuptools.find_packages(),
+283
View File
@@ -0,0 +1,283 @@
from __future__ import division, print_function, absolute_import
import numpy as np
import h5py
import pytest
import datetime
#
#
# The following few fixtures define some standard data files
# for use to test the data loading capabilities, whether in the
# datasets directly or in the data tools file
#
#
@pytest.fixture(scope='module')
def ptycho_cxi_1():
"""Creates an example file for CXI ptychography. This file is defined
to have everything done as correctly as possible with lots of attributes
defined. It will return both a dictionary describing what is expected
to be loaded and a file with the data stored in it.
"""
expected = {}
f = h5py.File('ptycho_cxi_1',driver='core',backing_store=False)
# Start by defining the basic structure
f.create_dataset('cxi_version', data=150)
f.create_dataset('number_of_entries',data=1)
# Then define a bunch of metadata for entry_1
e1f = f.create_group('entry_1')
expected['entry metadata'] = {}
e1e = expected['entry metadata']
e1e['start_time'] = datetime.datetime.now()
e1f['start_time'] = np.string_(e1e['start_time'].isoformat())
e1e['end_time'] = datetime.datetime.now()
e1f['end_time'] = np.string_(e1e['end_time'].isoformat())
e1e['experiment_identifier'] = 'Fake Experiment 1'
e1f['experiment_identifier'] = np.string_(e1e['experiment_identifier'])
e1e['experiment_description'] = 'A fully defined ptychography experiment to test the data loading'
e1f['experiment_description'] = np.string_(e1e['experiment_description'])
e1e['program_name'] = 'CDTools'
e1f['program_name'] = np.string_(e1e['program_name'])
e1e['title'] = 'The one experiment we did'
e1f['title'] = np.string_(e1e['title'])
# Set up the sample info
s1f = e1f.create_group('sample_1')
expected['sample info'] = {}
s1e = expected['sample info']
s1e['name'] = 'Fake Sample'
s1f['name'] = np.string_(s1e['name'])
s1e['description'] = 'A sample that isn\'t real'
s1f['description'] = np.string_(s1e['description'])
s1e['unit_cell_group'] = 'P1'
s1f['unit_cell_group'] = np.string_(s1e['unit_cell_group'])
s1e['concentration'] = np.float32(np.random.rand())
s1f['concentration'] = s1e['concentration']
s1e['mass'] = np.float32(np.random.rand())
s1f['mass'] = s1e['mass']
s1e['temperature'] = np.float32(np.random.rand()*100)
s1f['temperature'] = s1e['temperature']
s1e['thickness'] = np.float32(np.random.rand()*1e-7)
s1f['thickness'] = s1e['thickness']
s1e['unit_cell_volume'] = np.float32(np.random.rand() * 1e-27)
s1f['unit_cell_volume'] = s1e['unit_cell_volume']
s1e['unit_cell'] = np.array([1,1,1,90,90,90]).astype(np.float32)
s1f.create_dataset('unit_cell',data = s1e['unit_cell'])
i1f = e1f.create_group('instrument_1')
source1f = i1f.create_group('source_1')
energy = np.float32(1.3618e-16) #Joules, = 850 eV
source1f['energy'] = energy
expected['wavelength'] = np.float32(1.9864459e-25) / energy
source1f['wavelength'] = expected['wavelength']
d1f = i1f.create_group('detector_1')
expected['detector'] = {}
d1e = expected['detector']
d1e['distance'] = np.float32(0.3)
d1f['distance'] = d1e['distance']
d1e['basis'] = np.array([[0,-30e-6,0],
[-20e-6,0,0]]).astype(np.float32).transpose()
d1f.create_dataset('basis_vectors',data=d1e['basis'])
d1f['x_pixel_size'] = np.float32(20e-6)
d1f['y_pixel_size'] = np.float32(30e-6)
d1e['corner'] = np.array((2550e-6,3825e-6,0.3)).astype(np.float32)
d1f.create_dataset('corner_position', data=d1e['corner'])
# Remember the format for the CXI file differs from the format used
# internally
mask = np.zeros((100,256,256)).astype(np.uint32)
expected['mask'] = np.ones((100,256,256)).astype(np.uint8)
d1f.create_dataset('mask',data=mask)
data1f = e1f.create_group('data_1')
data = np.random.rand(100,256,256).astype(np.float32)
expected['data'] = data
d1f.create_dataset('data',data=data)
data1f['data'] = h5py.SoftLink('/entry_1/instrument_1/detector_1/data')
d1f['data'].attrs['axes'] = np.string_('translation:y:x')
expected['axes'] = ['translation','y','x']
g1f = s1f.create_group('geometry_1')
translations = np.arange(300).reshape((100,3)).astype(np.float32)
g1f.create_dataset('translation',data=translations)
data1f['translation'] = h5py.SoftLink('/entry_1/sample_1/geometry_1/translation')
d1f['translation'] = h5py.SoftLink('/entry_1/sample_1/geometry_1/translation')
expected['translations'] = -translations
yield f, expected
f.close()
@pytest.fixture(scope='module')
def ptycho_cxi_2():
"""Creates an example file for CXI ptychography. This file is defined
to have a subset of things missing. In particular, it:
* Defines the wavelength but not the energy
* Defines the corner position but not the sample-detector distance
* Defines pixel sizes but no basis vectors
* Doesn't define a mask
* Only defines data in the relevant places, not under data_1
* Doesn't explicitly define axes for the data arrays
* Is missing many allowed metadata attributes
"""
expected = {}
f = h5py.File('ptycho_cxi_2',driver='core',backing_store=False)
# Start by defining the basic structure
f.create_dataset('cxi_version', data=150)
f.create_dataset('number_of_entries',data=1)
# Then define a bunch of metadata for entry_1
e1f = f.create_group('entry_1')
expected['entry metadata'] = {}
e1e = expected['entry metadata']
e1e['title'] = 'The one experiment we did'
e1f['title'] = np.string_(e1e['title'])
# Set up the sample info
s1f = e1f.create_group('sample_1')
expected['sample info'] = {}
s1e = expected['sample info']
s1e['temperature'] = np.float32(np.random.rand()*100)
s1f['temperature'] = s1e['temperature']
i1f = e1f.create_group('instrument_1')
source1f = i1f.create_group('source_1')
energy = np.float32(1.3618e-16) #Joules, = 850 eV
expected['wavelength'] = np.float32(1.9864459e-25) / energy
source1f['wavelength'] = expected['wavelength']
d1f = i1f.create_group('detector_1')
expected['detector'] = {}
d1e = expected['detector']
d1e['distance'] = np.float32(0.3)
d1e['basis'] = np.array([[0,-30e-6,0],
[-20e-6,0,0]]).astype(np.float32).transpose()
d1f['x_pixel_size'] = np.float32(20e-6)
d1f['y_pixel_size'] = np.float32(30e-6)
d1e['corner'] = np.array((2550e-6,3825e-6,0.3)).astype(np.float32)
d1f.create_dataset('corner_position', data=d1e['corner'])
# Remember the format for the CXI file differs from the format used
# internally
expected['mask'] = None
data1f = e1f.create_group('data_1')
data = np.random.rand(100,256,256).astype(np.float32)
expected['data'] = data
d1f.create_dataset('data',data=data)
expected['axes'] = None
g1f = s1f.create_group('geometry_1')
translations = np.arange(300).reshape((100,3)).astype(np.float32)
g1f.create_dataset('translation',data=translations)
expected['translations'] = -translations
yield f, expected
f.close()
@pytest.fixture(scope='module')
def ptycho_cxi_3():
"""Creates an example file for CXI ptychography. This file is defined
to have a different subset of information missing. In particular, it:
* Has no sample_1 group
* Defines the energy but not the wavelength of light
* Has the data only defined under the data_1 group, not in the relevant places
* Defines the detector basis but no pixel sizes
* Defines a mask as all pixels flagged as "above the background"
* Defines the sample to detector distance but no corner location
* Is missing some of the allowed metadata
"""
expected = {}
f = h5py.File('ptycho_cxi_3',driver='core',backing_store=False)
# Start by defining the basic structure
f.create_dataset('cxi_version', data=150)
f.create_dataset('number_of_entries',data=1)
# Then define a bunch of metadata for entry_1
e1f = f.create_group('entry_1')
expected['entry metadata'] = {}
e1e = expected['entry metadata']
e1e['start_time'] = datetime.datetime.now()
e1f['start_time'] = np.string_(e1e['start_time'].isoformat())
e1e['end_time'] = datetime.datetime.now()
e1f['end_time'] = np.string_(e1e['end_time'].isoformat())
# Set up the sample info
expected['sample info'] = None
i1f = e1f.create_group('instrument_1')
source1f = i1f.create_group('source_1')
energy = np.float32(1.3618e-16) #Joules, = 850 eV
source1f['energy'] = energy
expected['wavelength'] = np.float32(1.9864459e-25) / energy
d1f = i1f.create_group('detector_1')
expected['detector'] = {}
d1e = expected['detector']
d1e['distance'] = np.float32(0.3)
d1f['distance'] = d1e['distance']
d1e['basis'] = np.array([[0,-30e-6,0],
[-20e-6,0,0]]).astype(np.float32).transpose()
d1f.create_dataset('basis_vectors',data=d1e['basis'])
d1e['corner'] = None
# Remember the format for the CXI file differs from the format used
# internally
mask = np.ones((100,256,256)).astype(np.uint32) * 0x00001000
expected['mask'] = np.ones((100,256,256)).astype(np.uint8)
d1f.create_dataset('mask',data=mask)
data1f = e1f.create_group('data_1')
data = np.random.rand(100,256,256).astype(np.float32)
expected['data'] = data
data1f.create_dataset('data',data=data)
data1f['data'].attrs['axes'] = np.string_('translation:y:x')
expected['axes'] = ['translation','y','x']
translations = np.arange(300).reshape((100,3)).astype(np.float32)
data1f.create_dataset('translation',data=translations)
expected['translations'] = -translations
yield f, expected
f.close()
# As specific issues start to crop up with loading CXI files from different
# beamlines, put a fixture here that replicates the issue so that we can
# ensure compatibility with many beamlines
#
@pytest.fixture(scope='module')
def test_ptycho_cxis(ptycho_cxi_1, ptycho_cxi_2, ptycho_cxi_3):
"""Loads a list of tuples of ptychography CXI files and dictionaries,
describing the expected output from various functions on being called
on the cxi files.
"""
return [ptycho_cxi_1, ptycho_cxi_2, ptycho_cxi_3]
+243
View File
@@ -0,0 +1,243 @@
from __future__ import division, print_function, absolute_import
from CDTools.datasets import *
from CDTools.tools import data as cdtdata
import numpy as np
import torch as t
import h5py
import pytest
import datetime
#
# We start by testing the CDataset base class
#
def test_CDataset_init():
entry_info = {'start_time': datetime.datetime.now(),
'title' : 'A simple test'}
sample_info = {'name': 'A test sample',
'mass' : 3.4,
'unit_cell' : np.array([1,1,1,87,84.5,90])}
wavelength = 1e-9
detector_geometry = {'distance': 0.7,
'basis': np.array([[0,-30e-6,0],
[-20e-6,0,0]]).transpose(),
'corner': np.array((2550e-6,3825e-6,0.3))}
mask = np.ones((256,256))
dataset = CDataset(entry_info, sample_info,
wavelength, detector_geometry, mask)
assert t.all(t.eq(dataset.mask,t.tensor(mask)))
assert dataset.entry_info == entry_info
assert dataset.sample_info == sample_info
assert dataset.wavelength == wavelength
assert dataset.detector_geometry == detector_geometry
def test_CDataset_from_cxi(test_ptycho_cxis):
for cxi, expected in test_ptycho_cxis:
dataset = CDataset.from_cxi(cxi)
# The entry metadata loaded
for key in expected['entry metadata']:
assert dataset.entry_info[key] == expected['entry metadata'][key]
# Don't test for fidelity since this is tested in the data, just test
# that it is loaded
if expected['sample info'] is None:
assert dataset.sample_info is None
else:
assert dataset.sample_info is not None
assert np.isclose(dataset.wavelength,expected['wavelength'])
# Just check one of the loaded attributes
assert np.isclose(dataset.detector_geometry['distance'],
expected['detector']['distance'])
# Check that the other ones are loaded but not for fidelity
assert 'basis' in dataset.detector_geometry
if expected['detector']['corner'] is not None:
assert 'corner' in dataset.detector_geometry
if expected['mask'] is not None:
assert t.all(t.eq(t.tensor(expected['mask']),dataset.mask))
def test_CDataset_to_cxi(test_ptycho_cxis, tmp_path):
for cxi, expected in test_ptycho_cxis:
dataset = CDataset.from_cxi(cxi)
with cdtdata.create_cxi(tmp_path / 'test_CDataset_to_cxi.cxi') as f:
dataset.to_cxi(f)
# Now we have to check that all the stuff was written
with h5py.File(tmp_path / 'test_CDataset_to_cxi.cxi', 'r') as f:
read_dataset = CDataset.from_cxi(f)
assert dataset.entry_info == read_dataset.entry_info
if dataset.sample_info is None:
assert read_dataset.sample_info is None
else:
assert read_dataset.sample_info is not None
assert np.isclose(dataset.wavelength, read_dataset.wavelength)
# Just check one of the loaded attributes
assert np.isclose(dataset.detector_geometry['distance'],
read_dataset.detector_geometry['distance'])
# Check that the other ones are loaded but not for fidelity
assert 'basis' in read_dataset.detector_geometry
if dataset.detector_geometry['corner'] is not None:
assert 'corner' in read_dataset.detector_geometry
if dataset.mask is not None:
assert t.all(t.eq(dataset.mask,read_dataset.mask))
def test_CDataset_to(ptycho_cxi_1):
dataset = CDataset.from_cxi(ptycho_cxi_1[0])
dataset.to(dtype=t.float32)
assert dataset.mask.dtype == t.uint8
# If cuda is available, check that moving the mask to CUDA works.
if t.cuda.is_available():
dataset.to(device='cuda:0')
assert dataset.mask.device == t.device('cuda:0')
#
# And we then test the derived Ptychography class
#
#
def test_Ptycho_2D_Dataset_init():
entry_info = {'start_time': datetime.datetime.now(),
'title' : 'A simple test'}
sample_info = {'name': 'A test sample',
'mass' : 3.4,
'unit_cell' : np.array([1,1,1,87,84.5,90])}
wavelength = 1e-9
detector_geometry = {'distance': 0.7,
'basis': np.array([[0,-30e-6,0],
[-20e-6,0,0]]).transpose(),
'corner': np.array((2550e-6,3825e-6,0.3))}
mask = np.ones((256,256))
patterns = np.random.rand(20,256,256)
translations = np.random.rand(20,3)
dataset = Ptycho_2D_Dataset(translations, patterns,
entry_info=entry_info,
sample_info=sample_info,
wavelength=wavelength,
detector_geometry=detector_geometry,
mask=mask)
assert t.all(t.eq(dataset.mask,t.tensor(mask)))
assert dataset.entry_info == entry_info
assert dataset.sample_info == sample_info
assert dataset.wavelength == wavelength
assert dataset.detector_geometry == detector_geometry
assert t.allclose(dataset.patterns, t.tensor(patterns))
assert t.allclose(dataset.translations, t.tensor(translations))
def test_Ptycho_2D_Dataset_from_cxi(test_ptycho_cxis):
for cxi, expected in test_ptycho_cxis:
dataset = Ptycho_2D_Dataset.from_cxi(cxi)
# The entry metadata loaded
for key in expected['entry metadata']:
assert dataset.entry_info[key] == expected['entry metadata'][key]
# Don't test for fidelity since this is tested in the data, just test
# that it is loaded
if expected['sample info'] is None:
assert dataset.sample_info is None
else:
assert dataset.sample_info is not None
assert np.isclose(dataset.wavelength,expected['wavelength'])
# Just check one of the loaded attributes
assert np.isclose(dataset.detector_geometry['distance'],
expected['detector']['distance'])
# Check that the other ones are loaded but not for fidelity
assert 'basis' in dataset.detector_geometry
if expected['detector']['corner'] is not None:
assert 'corner' in dataset.detector_geometry
if expected['mask'] is not None:
assert t.all(t.eq(t.tensor(expected['mask']),dataset.mask))
assert t.allclose(t.tensor(expected['data']),dataset.patterns)
assert t.allclose(t.tensor(expected['translations']),dataset.translations)
def test_Ptycho_2D_Dataset_to_cxi(test_ptycho_cxis, tmp_path):
for cxi, expected in test_ptycho_cxis:
dataset = Ptycho_2D_Dataset.from_cxi(cxi)
with cdtdata.create_cxi(tmp_path / 'test_Ptycho_2D_Dataset_to_cxi.cxi') as f:
dataset.to_cxi(f)
# Now we have to check that all the stuff was written
with h5py.File(tmp_path / 'test_Ptycho_2D_Dataset_to_cxi.cxi', 'r') as f:
read_dataset = Ptycho_2D_Dataset.from_cxi(f)
assert dataset.entry_info == read_dataset.entry_info
if dataset.sample_info is None:
assert read_dataset.sample_info is None
else:
assert read_dataset.sample_info is not None
assert np.isclose(dataset.wavelength, read_dataset.wavelength)
# Just check one of the loaded attributes
assert np.isclose(dataset.detector_geometry['distance'],
read_dataset.detector_geometry['distance'])
# Check that the other ones are loaded but not for fidelity
assert 'basis' in read_dataset.detector_geometry
if dataset.detector_geometry['corner'] is not None:
assert 'corner' in read_dataset.detector_geometry
if dataset.mask is not None:
assert t.all(t.eq(dataset.mask,read_dataset.mask))
assert t.allclose(dataset.patterns, read_dataset.patterns)
assert t.allclose(dataset.translations, read_dataset.translations)
def test_Ptycho_2D_Dataset_to(ptycho_cxi_1):
dataset = Ptycho_2D_Dataset.from_cxi(ptycho_cxi_1[0])
dataset.to(dtype=t.float64)
assert dataset.mask.dtype == t.uint8
assert dataset.patterns.dtype == t.float64
assert dataset.translations.dtype == t.float64
# If cuda is available, check that moving the mask to CUDA works.
if t.cuda.is_available():
dataset.to(device='cuda:0')
assert dataset.mask.device == t.device('cuda:0')
assert dataset.patterns.device == t.device('cuda:0')
assert dataset.translations.device == t.device('cuda:0')
def test_Ptycho_2D_Dataset_ops(ptycho_cxi_1):
cxi, expected = ptycho_cxi_1
dataset = Ptycho_2D_Dataset.from_cxi(cxi)
assert len(dataset) == expected['data'].shape[0]
idx, translation, pattern = dataset[3]
assert idx == 3
assert t.allclose(translation, t.tensor(expected['translations'][3,:]))
assert t.allclose(pattern, t.tensor(expected['data'][3,:,:]))
+155 -289
View File
@@ -7,300 +7,13 @@ import h5py
import pytest
import os
import datetime
import numbers
from pathlib import Path
#
# First, we write a few fixtures to generate specific data files we
# want with deliberate pathologies.
#
# * Energy defined but no wavelength
# * Wavelength defined but no energy
# * Distance and pixel pitch defined but no corner position or basis
# * Corner position and basis defined but no distance or pixel pitch
# * No mask defined
# * Mask defined
#
# Each file will get a fixture that loads the cxi file but also loads
# a dictionary with the relevant information for the cxi file
#
#
# This just grabs the directory whose name matches the file we're running
@pytest.fixture(scope='module')
def datadir(request):
filename = request.module.__file__
test_dir, _ = os.path.splitext(filename)
return Path(test_dir)
@pytest.fixture(scope='module')
def ptycho_cxi_1():
"""Creates an example file for CXI ptychography. This file is defined
to have everything done as correctly as possible with lots of attributes
defined. It will return both a dictionary describing what is expected
to be loaded and a file with the data stored in it.
"""
expected = {}
f = h5py.File('ptycho_cxi_1',driver='core',backing_store=False)
# Start by defining the basic structure
f.create_dataset('cxi_version', data=150)
f.create_dataset('number_of_entries',data=1)
# Then define a bunch of metadata for entry_1
e1f = f.create_group('entry_1')
expected['entry metadata'] = {}
e1e = expected['entry metadata']
e1e['start_time'] = datetime.datetime.now().isoformat()
e1f['start_time'] = np.string_(e1e['start_time'])
e1e['end_time'] = datetime.datetime.now().isoformat()
e1f['end_time'] = np.string_(e1e['end_time'])
e1e['experiment_identifier'] = 'Fake Experiment 1'
e1f['experiment_identifier'] = np.string_(e1e['experiment_identifier'])
e1e['experiment_description'] = 'A fully defined ptychography experiment to test the data loading'
e1f['experiment_description'] = np.string_(e1e['experiment_description'])
e1e['program_name'] = 'CDTools'
e1f['program_name'] = np.string_(e1e['program_name'])
e1e['title'] = 'The one experiment we did'
e1f['title'] = np.string_(e1e['title'])
# Set up the sample info
s1f = e1f.create_group('sample_1')
expected['sample info'] = {}
s1e = expected['sample info']
s1e['name'] = 'Fake Sample'
s1f['name'] = np.string_(s1e['name'])
s1e['description'] = 'A sample that isn\'t real'
s1f['description'] = np.string_(s1e['description'])
s1e['unit_cell_group'] = 'P1'
s1f['unit_cell_group'] = np.string_(s1e['unit_cell_group'])
s1e['concentration'] = np.float32(np.random.rand())
s1f['concentration'] = s1e['concentration']
s1e['mass'] = np.float32(np.random.rand())
s1f['mass'] = s1e['mass']
s1e['temperature'] = np.float32(np.random.rand()*100)
s1f['temperature'] = s1e['temperature']
s1e['thickness'] = np.float32(np.random.rand()*1e-7)
s1f['thickness'] = s1e['thickness']
s1e['unit_cell_volume'] = np.float32(np.random.rand() * 1e-27)
s1f['unit_cell_volume'] = s1e['unit_cell_volume']
s1e['unit_cell'] = np.array([1,1,1,90,90,90]).astype(np.float32)
s1f.create_dataset('unit_cell',data = s1e['unit_cell'])
i1f = e1f.create_group('instrument_1')
source1f = i1f.create_group('source_1')
energy = np.float32(1.3618e-16) #Joules, = 850 eV
source1f['energy'] = energy
expected['wavelength'] = np.float32(1.9864459e-25) / energy
source1f['wavelength'] = expected['wavelength']
d1f = i1f.create_group('detector_1')
expected['detector'] = {}
d1e = expected['detector']
d1e['distance'] = np.float32(0.3)
d1f['distance'] = d1e['distance']
d1e['basis'] = np.array([[0,-30e-6,0],
[-20e-6,0,0]]).astype(np.float32).transpose()
d1f.create_dataset('basis_vectors',data=d1e['basis'])
d1f['x_pixel_size'] = np.float32(20e-6)
d1f['y_pixel_size'] = np.float32(30e-6)
d1e['corner'] = np.array((2550e-6,3825e-6,0.3)).astype(np.float32)
d1f.create_dataset('corner_position', data=d1e['corner'])
# Remember the format for the CXI file differs from the format used
# internally
mask = np.zeros((100,256,256)).astype(np.uint32)
expected['mask'] = np.ones((100,256,256)).astype(np.uint8)
d1f.create_dataset('mask',data=mask)
data1f = e1f.create_group('data_1')
data = np.random.rand(100,256,256).astype(np.float32)
expected['data'] = data
d1f.create_dataset('data',data=data)
data1f['data'] = h5py.SoftLink('/entry_1/instrument_1/detector_1/data')
d1f['data'].attrs['axes'] = np.string_('translation:y:x')
expected['axes'] = ['translation','y','x']
g1f = s1f.create_group('geometry_1')
translations = np.arange(300).reshape((100,3)).astype(np.float32)
g1f.create_dataset('translation',data=translations)
data1f['translation'] = h5py.SoftLink('/entry_1/sample_1/geometry_1/translation')
d1f['translation'] = h5py.SoftLink('/entry_1/sample_1/geometry_1/translation')
expected['translations'] = -translations.transpose()
return f, expected
@pytest.fixture(scope='module')
def ptycho_cxi_2():
"""Creates an example file for CXI ptychography. This file is defined
to have a subset of things missing. In particular, it:
* Defines the wavelength but not the energy
* Defines the corner position but not the sample-detector distance
* Defines pixel sizes but no basis vectors
* Doesn't define a mask
* Only defines data in the relevant places, not under data_1
* Doesn't explicitly define axes for the data arrays
* Is missing many allowed metadata attributes
"""
expected = {}
f = h5py.File('ptycho_cxi_2',driver='core',backing_store=False)
# Start by defining the basic structure
f.create_dataset('cxi_version', data=150)
f.create_dataset('number_of_entries',data=1)
# Then define a bunch of metadata for entry_1
e1f = f.create_group('entry_1')
expected['entry metadata'] = {}
e1e = expected['entry metadata']
e1e['title'] = 'The one experiment we did'
e1f['title'] = np.string_(e1e['title'])
# Set up the sample info
s1f = e1f.create_group('sample_1')
expected['sample info'] = {}
s1e = expected['sample info']
s1e['temperature'] = np.float32(np.random.rand()*100)
s1f['temperature'] = s1e['temperature']
i1f = e1f.create_group('instrument_1')
source1f = i1f.create_group('source_1')
energy = np.float32(1.3618e-16) #Joules, = 850 eV
expected['wavelength'] = np.float32(1.9864459e-25) / energy
source1f['wavelength'] = expected['wavelength']
d1f = i1f.create_group('detector_1')
expected['detector'] = {}
d1e = expected['detector']
d1e['distance'] = np.float32(0.3)
d1e['basis'] = np.array([[0,-30e-6,0],
[-20e-6,0,0]]).astype(np.float32).transpose()
d1f['x_pixel_size'] = np.float32(20e-6)
d1f['y_pixel_size'] = np.float32(30e-6)
d1e['corner'] = np.array((2550e-6,3825e-6,0.3)).astype(np.float32)
d1f.create_dataset('corner_position', data=d1e['corner'])
# Remember the format for the CXI file differs from the format used
# internally
expected['mask'] = None
data1f = e1f.create_group('data_1')
data = np.random.rand(100,256,256).astype(np.float32)
expected['data'] = data
d1f.create_dataset('data',data=data)
expected['axes'] = None
g1f = s1f.create_group('geometry_1')
translations = np.arange(300).reshape((100,3)).astype(np.float32)
g1f.create_dataset('translation',data=translations)
expected['translations'] = -translations.transpose()
return f, expected
@pytest.fixture(scope='module')
def ptycho_cxi_3():
"""Creates an example file for CXI ptychography. This file is defined
to have a different subset of information missing. In particular, it:
* Has no sample_1 group
* Defines the energy but not the wavelength of light
* Has the data only defined under the data_1 group, not in the relevant places
* Defines the detector basis but no pixel sizes
* Defines a mask as all pixels flagged as "above the background"
* Defines the sample to detector distance but no corner location
* Is missing some of the allowed metadata
"""
expected = {}
f = h5py.File('ptycho_cxi_3',driver='core',backing_store=False)
# Start by defining the basic structure
f.create_dataset('cxi_version', data=150)
f.create_dataset('number_of_entries',data=1)
# Then define a bunch of metadata for entry_1
e1f = f.create_group('entry_1')
expected['entry metadata'] = {}
e1e = expected['entry metadata']
e1e['start_time'] = datetime.datetime.now().isoformat()
e1f['start_time'] = np.string_(e1e['start_time'])
e1e['end_time'] = datetime.datetime.now().isoformat()
e1f['end_time'] = np.string_(e1e['end_time'])
# Set up the sample info
expected['sample info'] = None
i1f = e1f.create_group('instrument_1')
source1f = i1f.create_group('source_1')
energy = np.float32(1.3618e-16) #Joules, = 850 eV
source1f['energy'] = energy
expected['wavelength'] = np.float32(1.9864459e-25) / energy
d1f = i1f.create_group('detector_1')
expected['detector'] = {}
d1e = expected['detector']
d1e['distance'] = np.float32(0.3)
d1f['distance'] = d1e['distance']
d1e['basis'] = np.array([[0,-30e-6,0],
[-20e-6,0,0]]).astype(np.float32).transpose()
d1f.create_dataset('basis_vectors',data=d1e['basis'])
d1e['corner'] = None
# Remember the format for the CXI file differs from the format used
# internally
mask = np.ones((100,256,256)).astype(np.uint32) * 0x00001000
expected['mask'] = np.ones((100,256,256)).astype(np.uint8)
d1f.create_dataset('mask',data=mask)
data1f = e1f.create_group('data_1')
data = np.random.rand(100,256,256).astype(np.float32)
expected['data'] = data
data1f.create_dataset('data',data=data)
data1f['data'].attrs['axes'] = np.string_('translation:y:x')
expected['axes'] = ['translation','y','x']
translations = np.arange(300).reshape((100,3)).astype(np.float32)
data1f.create_dataset('translation',data=translations)
expected['translations'] = -translations.transpose()
return f, expected
#
# As specific issues start to crop up with loading CXI files from different
# beamlines, put a fixture here that replicates the issue so that we can
# ensure compatibility with many beamlines
#
@pytest.fixture(scope='module')
def test_ptycho_cxis(ptycho_cxi_1, ptycho_cxi_2, ptycho_cxi_3):
"""Loads a list of tuples of ptychography CXI files and dictionaries,
describing the expected output from various functions on being called
on the cxi files.
"""
return [ptycho_cxi_1, ptycho_cxi_2, ptycho_cxi_3]
#
# Now we have a bunch of tests of the data loading capabilities
# We start with a bunch of tests of the data loading capabilities
#
@@ -373,3 +86,156 @@ def test_get_ptycho_translations(test_ptycho_cxis):
#
def test_create_cxi(tmp_path):
data.create_cxi(tmp_path / 'test_create.cxi')
with h5py.File(tmp_path / 'test_create.cxi') as f:
assert f['cxi_version'][()] == 160
assert 'entry_1' in f
def test_add_entry_info(tmp_path):
entry_info = {'experiment_identifier':'test of cxi file writing tools',
'title': 'my cool experiment',
'start_time': datetime.datetime.now(),
'end_time': datetime.datetime.now()}
with data.create_cxi(tmp_path / 'test_add_entry_info.cxi') as f:
data.add_entry_info(f, entry_info)
with h5py.File(tmp_path / 'test_add_entry_info.cxi') as f:
read_entry_info = data.get_entry_info(f)
for key in entry_info:
if isinstance(entry_info[key], np.ndarray):
assert np.allclose(entry_info[key], read_entry_info[key])
else:
assert entry_info[key] == read_entry_info[key]
def test_add_sample_info(tmp_path):
sample_info = {'name':'A nice fake sample',
'concentration': 10,
'mass': 5.3,
'temperature': 76,
'description': 'A very nice sample',
'unit_cell': np.array([1,1,1,90.,90.,90.])}
with data.create_cxi(tmp_path / 'test_add_sample_info.cxi') as f:
data.add_sample_info(f, sample_info)
with h5py.File(tmp_path / 'test_add_sample_info.cxi') as f:
read_sample_info = data.get_sample_info(f)
for key in sample_info:
if isinstance(sample_info[key], np.ndarray):
assert np.allclose(sample_info[key], read_sample_info[key])
elif isinstance(sample_info[key], numbers.Number):
assert np.isclose(sample_info[key], read_sample_info[key])
else:
assert sample_info[key] == read_sample_info[key]
def test_add_source(tmp_path):
wavelength = 1e-9
energy = 1.9864459e-25 / wavelength
with data.create_cxi(tmp_path / 'test_add_source.cxi') as f:
data.add_source(f, wavelength)
with h5py.File(tmp_path / 'test_add_source.cxi') as f:
# Check this directly since we want to make sure it saved
# the wavelength and energy
read_wavelength = f['entry_1/instrument_1/source_1/wavelength'][()]
read_energy = f['entry_1/instrument_1/source_1/energy'][()]
assert np.isclose( wavelength, read_wavelength)
assert np.isclose( energy, read_energy)
def test_add_detector(tmp_path):
distance = 0.34
basis = np.array([[0,-30e-6,0],
[-20e-6,0,0]]).astype(np.float32).transpose()
corner = np.array((2550e-6,3825e-6,0.3)).astype(np.float32)
with data.create_cxi(tmp_path / 'test_add_detector.cxi') as f:
data.add_detector(f, distance, basis, corner=corner)
with h5py.File(tmp_path / 'test_add_detector.cxi') as f:
# Check this directly since we want to make sure it saved
# the pixel sizes
d1 = f['entry_1/instrument_1/detector_1']
read_basis = np.array(d1['basis_vectors'])
read_x_pix = np.float32(d1['x_pixel_size'])
read_y_pix = np.float32(d1['y_pixel_size'])
read_distance = np.float32(d1['distance'])
read_corner = np.array(d1['corner_position'])
assert np.isclose(distance, read_distance)
assert np.allclose(basis, read_basis)
assert np.isclose(np.linalg.norm(basis[:,1]), read_x_pix)
assert np.isclose(np.linalg.norm(basis[:,0]), read_y_pix)
assert np.allclose(corner,read_corner)
def test_add_mask(tmp_path):
mask = (np.random.rand(350,600) > 0.1).astype(np.uint8)
with data.create_cxi(tmp_path / 'test_add_mask.cxi') as f:
data.add_mask(f, mask)
with h5py.File(tmp_path / 'test_add_mask.cxi') as f:
read_mask = data.get_mask(f)
assert np.all(mask == read_mask)
def test_add_data(tmp_path):
# First test from numpy, with axes
fake_data = np.random.rand(100,256,256)
axes = ['translation','y','x']
with data.create_cxi(tmp_path / 'test_add_data.cxi') as f:
data.add_data(f, fake_data, axes)
with h5py.File(tmp_path / 'test_add_data.cxi') as f:
# Check this directly since we want to make sure it saved
# it in all the places it should have
read_data_1 = np.array(f['entry_1/data_1/data'])
read_data_2 = np.array(f['entry_1/instrument_1/detector_1/data'])
read_axes = str(f['entry_1/instrument_1/detector_1/data'].attrs['axes'].decode())
assert np.allclose(fake_data, read_data_1)
assert np.allclose(fake_data, read_data_2)
assert 'translation:y:x' == read_axes
# Then test from torch, without axes
fake_data = t.from_numpy(fake_data)
with data.create_cxi(tmp_path / 'test_add_data_torch.cxi') as f:
data.add_data(f, fake_data)
with h5py.File(tmp_path / 'test_add_data_torch.cxi') as f:
read_data, axes = data.get_data(f)
assert np.allclose(fake_data.numpy(),read_data)
def test_add_ptycho_translations(tmp_path):
translations = np.random.rand(3,100)
with data.create_cxi(tmp_path / 'test_add_ptycho_translations.cxi') as f:
data.add_ptycho_translations(f, translations)
with h5py.File(tmp_path / 'test_add_ptycho_translations.cxi') as f:
# Check this directly since we want to make sure it saved
# it in all the places it should have
read_translations_1 = np.array(f['entry_1/data_1/translation'])
read_translations_2 = np.array(f['entry_1/instrument_1/detector_1/translation'])
read_translations_3 = np.array(f['entry_1/sample_1/geometry_1/translation'])
assert np.allclose(-translations, read_translations_1)
assert np.allclose(-translations, read_translations_2)
assert np.allclose(-translations, read_translations_3)