diff --git a/CDTools/__init__.py b/CDTools/__init__.py index fbe00ae..095a51f 100644 --- a/CDTools/__init__.py +++ b/CDTools/__init__.py @@ -1,3 +1,4 @@ from __future__ import division, print_function, absolute_import from CDTools import tools +from CDTools import datasets diff --git a/CDTools/datasets.py b/CDTools/datasets.py new file mode 100644 index 0000000..d4d364b --- /dev/null +++ b/CDTools/datasets.py @@ -0,0 +1,216 @@ +from __future__ import division, print_function, absolute_import +import numpy as np +import torch as t +from copy import copy + +# The naming overlap here is definitely going to get confusing. +from CDTools.tools import data as cdtdata +from torch.utils import data as torchdata + +__all__ = ['CDataset', 'Ptycho_2D_Dataset'] + +# +# I think that here should live a variety of datasets that all +# subclass the basic dataset class. They should be able to contain +# all the information that would be available in a CDI-type experiment. +# +# It's important for each kind of dataset to have tools to: +# +# * Pass too and from the GPU (where to store the data) +# * Be able to load itself intelligently from a cxi file +# * Be able to save itself intelligently to a cxi file +# +# And in addition, it should be relatively simple to initialize from +# data held as python structures +# +# The datasets need to be able to work just like pytorch datasets, +# in fact, they should subclass them, where they return the relevant +# information for each diffraction pattern when sliced. +# + + +# +# One question, though, is how to deal with information that models need +# to know as well as the datasets? The basic issue is that it would be +# nice to be able to write down a forward model and have it populate +# a dataset object with the simulated data. But you also want to be able +# to write a forward model that pulls the relevant information from a +# dataset that has been read from a real data's cxi file. +# +# One workaround would be to just have the things defined twice, but to +# make sure that every model can load/initialize itself from a dataset. +# If it also knows how to define itself from an initialization function +# or a python dataset, then it can easily also have a generic function to +# simulate the action of the model and create a dataset from that simulation. +# +# This is unrelated, but it will then be important to be able to save and load +# models easily from a predefined format. To be honest, this could just +# literally be a snippet of python code that redefines the specific model +# using only the base CDTools packages. That way a saved model could be a +# .py file that just creates the model on the spot. Not sure I love this +# though +# +# Is the trio of data, model, and reconstruction the correct thing though? +# Perhaps the reconstruction doesn't need to exist. The model can have +# a few more things involved in it, and the reconstructions could be a +# single function, for example, that only uses what's in the model. +# +# The issue with the automatic differentiation stuff is that it's history +# dependent in a sense. So you generate an optimization object and you take +# a step with it. +# +# Can all reconstruction algorithms be formulated in terms of an explicit +# forward step, backward step, and gradient with respect to the forward +# step? No, for example the position annealing step needs to do it's own +# thing. Okay, well then it could be possible to write an automatic +# differentiation sequence that can generate n steps of a given automatic +# differentiation solved. So given a model, you could just generically +# create n steps of automatic differentiation. But then also write an +# explicit ePIE step, etc. It could be interesting if the reconstruction +# algorithms defined generators which would run one step and then yield +# the loss at the end of that step. This would let you easily save out +# the loss but also would make it simple to include live plotting +# +# +# Then instead of a plan, you would just write a python script to follow +# the steps you wanted to follow. +# +# + + +# I think there should be a base CDataSet that has entry_info, sample_info, detector_info, and mask attributes. It can know how to load this data from a cxi file and write it out to a cxi file. It also can know how to pass this info to and from the GPU (not all of it needs to be passed in that way). + +# Then, I will write a few common datasets as a demonstration. First is +# A 2D CDI dataset, second is a 2D Ptycho dataset. + + +# +# This loads and stores all the kinds of metadata that are common to +# All different kinds of diffraction experiments +# Other datasets can subclass this and not worry about loading and +# saving that metadata. +# + +class CDataset(torchdata.Dataset): + + def __init__(self, entry_info=None, sample_info=None, + wavelength=None, + detector_geometry=None, mask=None): + + # Force pass-by-value-like behavior to stop strangeness + self.entry_info = copy(entry_info) + self.sample_info = copy(sample_info) + self.wavelength = wavelength + self.detector_geometry = copy(detector_geometry) + if mask is not None: + self.mask = t.tensor(mask) + else: + self.mask = None + + + def to(self,*args,**kwargs): + # The mask should always stay a uint8, but it should switch devices + mask_kwargs = copy(kwargs) + try: + mask_kwargs.pop('dtype') + except KeyError as r: + pass + + if self.mask is not None: + self.mask = self.mask.to(*args,**mask_kwargs) + + + @classmethod + def from_cxi(cls, cxi_file): + entry_info = cdtdata.get_entry_info(cxi_file) + sample_info = cdtdata.get_sample_info(cxi_file) + wavelength = cdtdata.get_wavelength(cxi_file) + distance, basis, corner = cdtdata.get_detector_geometry(cxi_file) + detector_geometry = {'distance' : distance, + 'basis' : basis, + 'corner' : corner} + mask = cdtdata.get_mask(cxi_file) + return cls(entry_info = entry_info, + sample_info = sample_info, + wavelength=wavelength, + detector_geometry=detector_geometry, + mask=mask) + + + def to_cxi(self, cxi_file): + if self.entry_info is not None: + cdtdata.add_entry_info(cxi_file, self.entry_info) + if self.sample_info is not None: + cdtdata.add_sample_info(cxi_file, self.sample_info) + if self.wavelength is not None: + cdtdata.add_source(cxi_file, self.wavelength) + if self.detector_geometry is not None: + if 'corner' in self.detector_geometry: + corner = self.detector_geometry['corner'] + else: + corner = None + cdtdata.add_detector(cxi_file, + self.detector_geometry['distance'], + self.detector_geometry['basis'], + corner = corner) + if self.mask is not None: + cdtdata.add_mask(cxi_file, self.mask) + + + +# +# This is the standard dataset for a 2D ptychography experiment, +# which saves and loads files compatible with most reconstruction +# programs (only tested against SHARP) +# + +class Ptycho_2D_Dataset(CDataset): + + def __init__(self, translations, patterns, axes=None, *args, **kwargs): + + super(Ptycho_2D_Dataset,self).__init__(*args, **kwargs) + self.axes = copy(axes) + self.translations = t.tensor(translations) + self.patterns = t.tensor(patterns) + + + def __len__(self): + return self.patterns.shape[0] + + def __getitem__(self,index): + return index, self.translations[index], self.patterns[index] + + + def to(self, *args, **kwargs): + super(Ptycho_2D_Dataset,self).to(*args,**kwargs) + self.translations = self.translations.to(*args, **kwargs) + self.patterns = self.patterns.to(*args, **kwargs) + + + # It sucks that I can't reuse the base factory method here, + # perhaps there is a way but I couldn't figure it out. + @classmethod + def from_cxi(cls, cxi_file): + entry_info = cdtdata.get_entry_info(cxi_file) + sample_info = cdtdata.get_sample_info(cxi_file) + wavelength = cdtdata.get_wavelength(cxi_file) + distance, basis, corner = cdtdata.get_detector_geometry(cxi_file) + detector_geometry = {'distance' : distance, + 'basis' : basis, + 'corner' : corner} + mask = cdtdata.get_mask(cxi_file) + patterns, axes = cdtdata.get_data(cxi_file) + + translations = cdtdata.get_ptycho_translations(cxi_file) + return cls(translations, patterns, axes=axes, + entry_info = entry_info, + sample_info = sample_info, + wavelength=wavelength, + detector_geometry=detector_geometry, + mask=mask) + + + def to_cxi(self, cxi_file): + super(Ptycho_2D_Dataset,self).to_cxi(cxi_file) + cdtdata.add_data(cxi_file, self.patterns, axes=self.axes) + cdtdata.add_ptycho_translations(cxi_file, self.translations) diff --git a/CDTools/scripts/__init__.py b/CDTools/scripts/__init__.py new file mode 100644 index 0000000..d675d93 --- /dev/null +++ b/CDTools/scripts/__init__.py @@ -0,0 +1 @@ +from __future__ import division, print_function, absolute_import diff --git a/CDTools/scripts/view_cxi.py b/CDTools/scripts/view_cxi.py new file mode 100644 index 0000000..5f7a9c4 --- /dev/null +++ b/CDTools/scripts/view_cxi.py @@ -0,0 +1,96 @@ +from __future__ import division, print_function, absolute_import +import h5py +import numpy as np +import os +from PyQt5 import QtWidgets +from matplotlib import pyplot as plt +import signal +import sys +import argparse + +# +# The plotting was broken by the move to CDTools, at some point this should +# be fixed +# + +def view_cxi(filename): + """Opens a popup window displaying the contents of ``filename``. + relevant attributes.""" + signal.signal(signal.SIGINT, signal.SIG_DFL) + + class Viewer(QtWidgets.QMainWindow): + def __init__(self, datafile): + self.datafile = datafile + QtWidgets.QMainWindow.__init__(self) + self.tree = QtWidgets.QTreeWidget(self) + self.tree.setColumnWidth(0, 200) + self.setCentralWidget(self.tree) + self.buildTree() + self.tree.itemClicked.connect(self.handleClick) + + def handleClick(self,item,column): + if(item.text(column) == 'Click to print to console'): + data = self.data_full[str(item.text(2))] + print(np.asarray(data)) + if(item.text(column) == 'Click to display'): + data = self.datasets[str(item.text(2))] + plt.plot(data) + plt.title(item.text(0).capitalize()) + plt.show() + + def closeWindow(self): + pass + + def buildTree(self): + self.datasets = {} + self.data_full = {} + self.tree.setColumnCount(2) + self.f = h5py.File(self.datafile, 'r') + item = QtWidgets.QTreeWidgetItem(['/']) + self.tree.addTopLevelItem(item) + self.buildBranch(self.f,item) + + def buildBranch(self,group,item): + for g in group.keys(): + lst = [g] + self.data_full[group[g].name] = group[g] + if(isinstance(group[g],h5py.Group)): + child = QtWidgets.QTreeWidgetItem(lst) + self.buildBranch(group[g],child) + item.addChild(child) + else: + if len(group[g].shape)>2: + lst.append('Click to print to console') + lst.append(group[g].name) + self.datasets[group[g].name] = group[g] + item.addChild(QtWidgets.QTreeWidgetItem(lst)) + if len(group[g].shape)==2 or len(group[g].shape)==1: + lst.append('Click to display') + lst.append(group[g].name) + self.datasets[group[g].name] = group[g] + item.addChild(QtWidgets.QTreeWidgetItem(lst)) + else: + lst.append('Click to print to console') + lst.append(group[g].name) + self.datasets[group[g].name] = group[g] + item.addChild(QtWidgets.QTreeWidgetItem(lst)) + + filename = os.path.expanduser(filename) + app = QtWidgets.QApplication(sys.argv) + viewer = Viewer(filename) + viewer.setFixedSize(500, 500) + viewer.show() + app.exec_() + + +def make_argparser(): + parser = argparse.ArgumentParser(description=__doc__) + + parser.add_argument('file', help='The cxi file to view') + return parser + + +if __name__ == '__main__': + args = make_argparser().parse_args() + + view_cxi(args.file) diff --git a/CDTools/tools/__init__.py b/CDTools/tools/__init__.py index 6ea3699..a8eacbe 100644 --- a/CDTools/tools/__init__.py +++ b/CDTools/tools/__init__.py @@ -3,8 +3,11 @@ from __future__ import division, print_function, absolute_import from CDTools.tools import cmath from CDTools.tools import losses from CDTools.tools import data +<<<<<<< HEAD from CDTools.tools import image_processing from CDTools.tools import initializers from CDTools.tools import losses from CDTools.tools import projectors from CDTools.tools import propagators +======= +>>>>>>> bad75422717fd0eecfe1a136fddc2b96828a3321 diff --git a/CDTools/tools/data.py b/CDTools/tools/data.py index 4ef1796..7646391 100644 --- a/CDTools/tools/data.py +++ b/CDTools/tools/data.py @@ -2,6 +2,11 @@ from __future__ import division, print_function, absolute_import import h5py import numpy as np +import numbers +import datetime +import dateutil.parser +import torch as t +from contextlib import contextmanager __all__ = ['get_entry_info', 'get_sample_info', @@ -9,7 +14,15 @@ __all__ = ['get_entry_info', 'get_detector_geometry', 'get_mask', 'get_data', - 'get_ptycho_translations'] + 'get_ptycho_translations', + 'create_cxi', + 'add_entry_info', + 'add_sample_info', + 'add_source', + 'add_detector', + 'add_mask', + 'add_data', + 'add_ptycho_translations'] # # @@ -56,13 +69,19 @@ __all__ = ['get_entry_info', # + # # Functions to inspect the basic attributes of a cxi file represented as an # h5 file object # + def get_entry_info(cxi_file): """Returns a dictionary with the basic metadata from the cxi file's entry_1 attribute + + String type metadata is read out as a string, and datetime metadata + is converted to python datetime objects if the string is properly + formatted. Args: cxi_file (h5py.File) : a file object to be read @@ -75,11 +94,17 @@ def get_entry_info(cxi_file): metadata_attrs = ['title', 'experiment_identifier', 'experiment_description', - 'program_name', - 'start_time', - 'end_time'] + 'program_name'] metadata = {attr: str(e1[attr][()].decode()) for attr in metadata_attrs if attr in e1} + datetime_attrs = ['start_time', + 'end_time'] + for attr in datetime_attrs: + if attr in e1: + try: + metadata[attr] = dateutil.parser.parse(str(e1[attr][()].decode())) + except ValueError: + metadata[attr] = str(e1[attr][()].decode()) return metadata @@ -118,6 +143,10 @@ def get_sample_info(cxi_file): # TODO: I should add the sample geometry as a valid metadata that can # be copied over + # Check if the metadata is empty + if metadata == {}: + metadata = None + return metadata @@ -290,12 +319,10 @@ def get_data(cxi_file): def get_ptycho_translations(cxi_file): """Gets an array of x,y,z translations, if such an array has been defined in the file - It applies two operations to the translations. First, it negates them, - because the CXI file format is designed to specify translations of the - samples and the CDTools code specifies translations of the optics. - Second, it transposes the array so that the first axis is translation - ID and the second axis is the (x,y,z) components of the translation - + It negates the translations, because the CXI file format is designed + to specify translations of the samples and the CDTools code specifies + translations of the optics. + Args: cxi_file (h5py.File) : a file object to be read @@ -313,32 +340,245 @@ def get_ptycho_translations(cxi_file): else: raise KeyError('Translations are not defined within cxi file') - translations = -np.array(cxi_file[pull_from]).astype(np.float32).transpose() + translations = -np.array(cxi_file[pull_from]).astype(np.float32) return translations + # # It might be useful to make some helper functions to help write cxi files # -# -# A function to place the skeleton of a cxi file down -# -# -# A function to define the source attributes -# +def create_cxi(filename): + """Creates a new cxi file with a single entry group -# -# A function to define the detector geometry -# + Args: + filename (str) : The path at which to create the file + """ + file_obj = h5py.File(filename,'w') + file_obj.create_dataset('cxi_version', data=160) + file_obj.create_dataset('number_of_entries',data=1) + e1f = file_obj.create_group('entry_1') + return file_obj -# -# A function to save out a mask, converting it to the correct format -# -# -# Perhaps a function to store the data and link it correctly? But this might -# have to change too much situation to situation -# +def add_entry_info(cxi_file, metadata): + """Adds a dictionary of entry metadata to the entry_1 group of a cxi file object + + Args: + cxi_file (h5py.File) : The file to add the info to + metadata (dict) : A dictionary containing all the metadata to be stored + """ + # Just the string and datetime types should be relevant but all are + # included in case the cxi spec becomes more permissive + for key, value in metadata.items(): + if isinstance(value,(str,bytes)): + cxi_file['entry_1'][key] = np.string_(value) + elif isinstance(value, datetime.datetime): + cxi_file['entry_1'][key] = np.string_(value.isoformat()) + elif isinstance(value, numbers.Number): + si[key] = value + elif isinstance(value, (np.ndarray,list,tuple)): + s1.create_dataset(key, data=np.asarray(value)) + elif isinstance(value, t.Tensor): + asnumpy = value.detach().cpu().numpy() + cxi_file['entry_1'].create_dataset(key, data=asnumpy) + +def add_sample_info(cxi_file, metadata): + """Adds a dictionary of entry metadata to the entry_1/sample_1 group of a cxi file object + + This function will create the sample_1 attribute if it doesn't already exist + + Args: + cxi_file (h5py.File) : The file to add the info to + metadata (dict) : A dictionary containing all the metadata to be stored + """ + if 'entry_1/sample_1' not in cxi_file: + cxi_file['entry_1'].create_group('sample_1') + s1 = cxi_file['entry_1/sample_1'] + + for key, value in metadata.items(): + if isinstance(value,(str,bytes)): + s1[key] = np.string_(value) + elif isinstance(value, datetime.datetime): + s1[key] = np.string_(value.isoformat()) + elif isinstance(value, numbers.Number): + s1[key] = value + elif isinstance(value, (np.ndarray,list,tuple)): + s1.create_dataset(key, data=np.asarray(value)) + elif isinstance(value, t.Tensor): + asnumpy = value.detach().cpu().numpy() + s1.create_dataset(key, data=asnumpy) + + +def add_source(cxi_file, wavelength): + """Adds the entry_1/source_1 group to a cxi file object + + It stores the energy and wavelength attributes in the source_1 group, + given a wavelength to define them from. + + Args: + cxi_file (h5py.File) : The file to add the source to + wavelength (float) : The wavelength of light + """ + if 'entry_1/instrument_1' not in cxi_file: + cxi_file['entry_1'].create_group('instrument_1') + i1 = cxi_file['entry_1/instrument_1'] + if 'source_1' not in i1: + i1.create_group('source_1') + s1 = i1['source_1'] + s1['wavelength'] = np.float32(wavelength) + s1['energy'] = np.float32(1.9864459e-25 / wavelength) + + + +def add_detector(cxi_file, distance, basis, corner=None): + """Adds the entry_1/instrument_1/detector_1 group to a cxi file object + + It will define all the relevant parameters - distance, pixel size, + detector basis, and corner position (if relevant) based on the provided + information + + Args: + cxi_file (h5py.File) : The file to add the detector to + distance (float) : The sample to detector distance + basis (array_like) : The detector basis + corner (array_like) : Optional, the corner position of the detector + + """ + if 'entry_1/instrument_1' not in cxi_file: + cxi_file['entry_1'].create_group('instrument_1') + i1 = cxi_file['entry_1/instrument_1'] + if 'detector_1' not in i1: + i1.create_group('detector_1') + d1 = i1['detector_1'] + + d1['distance'] = np.float32(distance) + d1['x_pixel_size'] = np.linalg.norm(basis[:,1]) + d1['y_pixel_size'] = np.linalg.norm(basis[:,0]) + if isinstance(basis, t.Tensor): + basis = basis.detach().cpu().numpy() + d1.create_dataset('basis_vectors', data=basis) + if corner is not None: + if isinstance(corner, t.Tensor): + corner = corner.detach().cpu().numpy() + d1.create_dataset('corner_position',data=corner) + + +def add_mask(cxi_file, mask): + """Adds the specified mask to the cxi file + + It places the mask into the mask dataset under + entry_1/instrument_1/detector_1. The internal mask is defined + simply as a 1 for an "on" pixel and a 0 for an "off" pixel, and + the saved mask is exactly the opposite. This is simpler than the + most general mask allowed by the cxi file format but it captures the + distinction between pixels to be used and pixels not to be used. + + Args: + cxi_file (h5py.File) : The file to add the mask to + mask (array_like) : The mask to save out to the file + """ + + if 'entry_1/instrument_1' not in cxi_file: + cxi_file['entry_1'].create_group('instrument_1') + i1 = cxi_file['entry_1/instrument_1'] + if 'detector_1' not in i1: + i1.create_group('detector_1') + d1 = i1['detector_1'] + if isinstance(mask, t.Tensor): + mask = mask.detach().cpu().numpy() + + mask_to_save = np.zeros(mask.shape).astype(np.uint32) + mask_to_save[mask == 0] = 1 + d1.create_dataset('mask',data=mask_to_save) + + +def add_data(cxi_file, data, axes=None): + """Adds the specified data to the cxi file + + It will add the data unchanged to the file, placing it in two spots: + + 1) The entry_1/instrument_1/detector_1/data path + 2) A softlink at entry_1/data_1/data + + Args: + cxi_file (h5py.File) : The file to add the data to + data (array_like) : The data to be saved + axes (list) : Optional, a list of axis names to be saved in the axes attribute + """ + if 'entry_1/data_1' not in cxi_file: + cxi_file['entry_1'].create_group('data_1') + data1 = cxi_file['entry_1/data_1'] + + if 'entry_1/instrument_1' not in cxi_file: + cxi_file['entry_1'].create_group('instrument_1') + i1 = cxi_file['entry_1/instrument_1'] + if 'detector_1' not in i1: + i1.create_group('detector_1') + det1 = i1['detector_1'] + + if isinstance(data, t.Tensor): + data = data.detach().cpu().numpy() + + det1.create_dataset('data', data=data) + data1['data'] = h5py.SoftLink('/entry_1/instrument_1/detector_1/data') + + if axes is not None: + if isinstance(axes, list): + axes_str = ':'.join(axes) + else: + axes_str = str(axes) + det1['data'].attrs['axes'] = np.string_(axes_str) + + +def add_ptycho_translations(cxi_file, translations): + """Adds the specified translations to the cxi file + + It will add the translations to the file, negating them to conform to + the standard in cxi files that the translations refer to the object's + translation. + + It will generally store them in 3 places: + + 1) The entry_1/sample_1/geometry_1/translation path + 2) A softlink at entry_1/data_1/translation + 3) A softlink at entry_1/instrument_1/detector_1/translation + + Args: + cxi_file (h5py.File) : The file to add the translations to + translations (array_like) : The translations to be saved + """ + + if 'entry_1/sample_1' not in cxi_file: + cxi_file['entry_1'].create_group('sample_1') + s1 = cxi_file['entry_1/sample_1'] + + if 'geometry_1' not in s1: + s1.create_group('geometry_1') + g1 = s1['geometry_1'] + + if 'entry_1/data_1' not in cxi_file: + cxi_file['entry_1'].create_group('data_1') + data1 = cxi_file['entry_1/data_1'] + + if 'entry_1/instrument_1' not in cxi_file: + cxi_file['entry_1'].create_group('instrument_1') + i1 = cxi_file['entry_1/instrument_1'] + if 'detector_1' not in i1: + i1.create_group('detector_1') + det1 = i1['detector_1'] + + + if isinstance(translations, t.Tensor): + translations = translations.detach().cpu().numpy() + + # accounting for the different definition between cxi files and + # CDTools + translations = -translations + + g1.create_dataset('translation', data=translations) + data1['translation'] = h5py.SoftLink('/entry_1/sample_1/geometry_1/translation') + det1['translation'] = h5py.SoftLink('/entry_1/sample_1/geometry_1/translation') diff --git a/setup.py b/setup.py index c543375..de2f1ef 100644 --- a/setup.py +++ b/setup.py @@ -16,6 +16,7 @@ setuptools.setup( "numpy", "scipy", "matplotlib", + "dateutil", #"pytorch", "h5py"], packages=setuptools.find_packages(), diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..a062cf6 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,283 @@ +from __future__ import division, print_function, absolute_import +import numpy as np +import h5py +import pytest +import datetime + + +# +# +# The following few fixtures define some standard data files +# for use to test the data loading capabilities, whether in the +# datasets directly or in the data tools file +# +# + +@pytest.fixture(scope='module') +def ptycho_cxi_1(): + """Creates an example file for CXI ptychography. This file is defined + to have everything done as correctly as possible with lots of attributes + defined. It will return both a dictionary describing what is expected + to be loaded and a file with the data stored in it. + """ + + expected = {} + f = h5py.File('ptycho_cxi_1',driver='core',backing_store=False) + + # Start by defining the basic structure + f.create_dataset('cxi_version', data=150) + f.create_dataset('number_of_entries',data=1) + + # Then define a bunch of metadata for entry_1 + e1f = f.create_group('entry_1') + expected['entry metadata'] = {} + e1e = expected['entry metadata'] + e1e['start_time'] = datetime.datetime.now() + e1f['start_time'] = np.string_(e1e['start_time'].isoformat()) + e1e['end_time'] = datetime.datetime.now() + e1f['end_time'] = np.string_(e1e['end_time'].isoformat()) + e1e['experiment_identifier'] = 'Fake Experiment 1' + e1f['experiment_identifier'] = np.string_(e1e['experiment_identifier']) + e1e['experiment_description'] = 'A fully defined ptychography experiment to test the data loading' + e1f['experiment_description'] = np.string_(e1e['experiment_description']) + e1e['program_name'] = 'CDTools' + e1f['program_name'] = np.string_(e1e['program_name']) + e1e['title'] = 'The one experiment we did' + e1f['title'] = np.string_(e1e['title']) + + # Set up the sample info + s1f = e1f.create_group('sample_1') + expected['sample info'] = {} + s1e = expected['sample info'] + s1e['name'] = 'Fake Sample' + s1f['name'] = np.string_(s1e['name']) + s1e['description'] = 'A sample that isn\'t real' + s1f['description'] = np.string_(s1e['description']) + s1e['unit_cell_group'] = 'P1' + s1f['unit_cell_group'] = np.string_(s1e['unit_cell_group']) + s1e['concentration'] = np.float32(np.random.rand()) + s1f['concentration'] = s1e['concentration'] + s1e['mass'] = np.float32(np.random.rand()) + s1f['mass'] = s1e['mass'] + s1e['temperature'] = np.float32(np.random.rand()*100) + s1f['temperature'] = s1e['temperature'] + s1e['thickness'] = np.float32(np.random.rand()*1e-7) + s1f['thickness'] = s1e['thickness'] + s1e['unit_cell_volume'] = np.float32(np.random.rand() * 1e-27) + s1f['unit_cell_volume'] = s1e['unit_cell_volume'] + s1e['unit_cell'] = np.array([1,1,1,90,90,90]).astype(np.float32) + s1f.create_dataset('unit_cell',data = s1e['unit_cell']) + + i1f = e1f.create_group('instrument_1') + source1f = i1f.create_group('source_1') + + energy = np.float32(1.3618e-16) #Joules, = 850 eV + source1f['energy'] = energy + expected['wavelength'] = np.float32(1.9864459e-25) / energy + source1f['wavelength'] = expected['wavelength'] + + d1f = i1f.create_group('detector_1') + expected['detector'] = {} + d1e = expected['detector'] + d1e['distance'] = np.float32(0.3) + d1f['distance'] = d1e['distance'] + d1e['basis'] = np.array([[0,-30e-6,0], + [-20e-6,0,0]]).astype(np.float32).transpose() + d1f.create_dataset('basis_vectors',data=d1e['basis']) + d1f['x_pixel_size'] = np.float32(20e-6) + d1f['y_pixel_size'] = np.float32(30e-6) + d1e['corner'] = np.array((2550e-6,3825e-6,0.3)).astype(np.float32) + d1f.create_dataset('corner_position', data=d1e['corner']) + + # Remember the format for the CXI file differs from the format used + # internally + mask = np.zeros((100,256,256)).astype(np.uint32) + expected['mask'] = np.ones((100,256,256)).astype(np.uint8) + d1f.create_dataset('mask',data=mask) + + data1f = e1f.create_group('data_1') + + data = np.random.rand(100,256,256).astype(np.float32) + expected['data'] = data + d1f.create_dataset('data',data=data) + data1f['data'] = h5py.SoftLink('/entry_1/instrument_1/detector_1/data') + + d1f['data'].attrs['axes'] = np.string_('translation:y:x') + expected['axes'] = ['translation','y','x'] + + g1f = s1f.create_group('geometry_1') + translations = np.arange(300).reshape((100,3)).astype(np.float32) + g1f.create_dataset('translation',data=translations) + data1f['translation'] = h5py.SoftLink('/entry_1/sample_1/geometry_1/translation') + d1f['translation'] = h5py.SoftLink('/entry_1/sample_1/geometry_1/translation') + expected['translations'] = -translations + + yield f, expected + + f.close() + + +@pytest.fixture(scope='module') +def ptycho_cxi_2(): + """Creates an example file for CXI ptychography. This file is defined + to have a subset of things missing. In particular, it: + + * Defines the wavelength but not the energy + * Defines the corner position but not the sample-detector distance + * Defines pixel sizes but no basis vectors + * Doesn't define a mask + * Only defines data in the relevant places, not under data_1 + * Doesn't explicitly define axes for the data arrays + * Is missing many allowed metadata attributes + + """ + + expected = {} + f = h5py.File('ptycho_cxi_2',driver='core',backing_store=False) + + # Start by defining the basic structure + f.create_dataset('cxi_version', data=150) + f.create_dataset('number_of_entries',data=1) + + # Then define a bunch of metadata for entry_1 + e1f = f.create_group('entry_1') + expected['entry metadata'] = {} + e1e = expected['entry metadata'] + e1e['title'] = 'The one experiment we did' + e1f['title'] = np.string_(e1e['title']) + + # Set up the sample info + s1f = e1f.create_group('sample_1') + expected['sample info'] = {} + s1e = expected['sample info'] + s1e['temperature'] = np.float32(np.random.rand()*100) + s1f['temperature'] = s1e['temperature'] + + i1f = e1f.create_group('instrument_1') + source1f = i1f.create_group('source_1') + + energy = np.float32(1.3618e-16) #Joules, = 850 eV + expected['wavelength'] = np.float32(1.9864459e-25) / energy + source1f['wavelength'] = expected['wavelength'] + + d1f = i1f.create_group('detector_1') + expected['detector'] = {} + d1e = expected['detector'] + d1e['distance'] = np.float32(0.3) + d1e['basis'] = np.array([[0,-30e-6,0], + [-20e-6,0,0]]).astype(np.float32).transpose() + d1f['x_pixel_size'] = np.float32(20e-6) + d1f['y_pixel_size'] = np.float32(30e-6) + d1e['corner'] = np.array((2550e-6,3825e-6,0.3)).astype(np.float32) + d1f.create_dataset('corner_position', data=d1e['corner']) + + # Remember the format for the CXI file differs from the format used + # internally + expected['mask'] = None + + data1f = e1f.create_group('data_1') + + data = np.random.rand(100,256,256).astype(np.float32) + expected['data'] = data + d1f.create_dataset('data',data=data) + + expected['axes'] = None + + g1f = s1f.create_group('geometry_1') + translations = np.arange(300).reshape((100,3)).astype(np.float32) + g1f.create_dataset('translation',data=translations) + expected['translations'] = -translations + + yield f, expected + + f.close() + + +@pytest.fixture(scope='module') +def ptycho_cxi_3(): + """Creates an example file for CXI ptychography. This file is defined + to have a different subset of information missing. In particular, it: + + * Has no sample_1 group + * Defines the energy but not the wavelength of light + * Has the data only defined under the data_1 group, not in the relevant places + * Defines the detector basis but no pixel sizes + * Defines a mask as all pixels flagged as "above the background" + * Defines the sample to detector distance but no corner location + * Is missing some of the allowed metadata + """ + + expected = {} + f = h5py.File('ptycho_cxi_3',driver='core',backing_store=False) + + # Start by defining the basic structure + f.create_dataset('cxi_version', data=150) + f.create_dataset('number_of_entries',data=1) + + # Then define a bunch of metadata for entry_1 + e1f = f.create_group('entry_1') + expected['entry metadata'] = {} + e1e = expected['entry metadata'] + e1e['start_time'] = datetime.datetime.now() + e1f['start_time'] = np.string_(e1e['start_time'].isoformat()) + e1e['end_time'] = datetime.datetime.now() + e1f['end_time'] = np.string_(e1e['end_time'].isoformat()) + + # Set up the sample info + expected['sample info'] = None + + i1f = e1f.create_group('instrument_1') + source1f = i1f.create_group('source_1') + + energy = np.float32(1.3618e-16) #Joules, = 850 eV + source1f['energy'] = energy + expected['wavelength'] = np.float32(1.9864459e-25) / energy + + d1f = i1f.create_group('detector_1') + expected['detector'] = {} + d1e = expected['detector'] + d1e['distance'] = np.float32(0.3) + d1f['distance'] = d1e['distance'] + d1e['basis'] = np.array([[0,-30e-6,0], + [-20e-6,0,0]]).astype(np.float32).transpose() + d1f.create_dataset('basis_vectors',data=d1e['basis']) + d1e['corner'] = None + + # Remember the format for the CXI file differs from the format used + # internally + mask = np.ones((100,256,256)).astype(np.uint32) * 0x00001000 + expected['mask'] = np.ones((100,256,256)).astype(np.uint8) + d1f.create_dataset('mask',data=mask) + + data1f = e1f.create_group('data_1') + + data = np.random.rand(100,256,256).astype(np.float32) + expected['data'] = data + data1f.create_dataset('data',data=data) + + data1f['data'].attrs['axes'] = np.string_('translation:y:x') + expected['axes'] = ['translation','y','x'] + + translations = np.arange(300).reshape((100,3)).astype(np.float32) + data1f.create_dataset('translation',data=translations) + expected['translations'] = -translations + + yield f, expected + + f.close() + + +# As specific issues start to crop up with loading CXI files from different +# beamlines, put a fixture here that replicates the issue so that we can +# ensure compatibility with many beamlines +# + + +@pytest.fixture(scope='module') +def test_ptycho_cxis(ptycho_cxi_1, ptycho_cxi_2, ptycho_cxi_3): + """Loads a list of tuples of ptychography CXI files and dictionaries, + describing the expected output from various functions on being called + on the cxi files. + """ + return [ptycho_cxi_1, ptycho_cxi_2, ptycho_cxi_3] + diff --git a/tests/test_datasets.py b/tests/test_datasets.py new file mode 100644 index 0000000..02dc473 --- /dev/null +++ b/tests/test_datasets.py @@ -0,0 +1,243 @@ +from __future__ import division, print_function, absolute_import + +from CDTools.datasets import * +from CDTools.tools import data as cdtdata +import numpy as np +import torch as t +import h5py +import pytest +import datetime + + +# +# We start by testing the CDataset base class +# + +def test_CDataset_init(): + entry_info = {'start_time': datetime.datetime.now(), + 'title' : 'A simple test'} + sample_info = {'name': 'A test sample', + 'mass' : 3.4, + 'unit_cell' : np.array([1,1,1,87,84.5,90])} + wavelength = 1e-9 + detector_geometry = {'distance': 0.7, + 'basis': np.array([[0,-30e-6,0], + [-20e-6,0,0]]).transpose(), + 'corner': np.array((2550e-6,3825e-6,0.3))} + mask = np.ones((256,256)) + dataset = CDataset(entry_info, sample_info, + wavelength, detector_geometry, mask) + assert t.all(t.eq(dataset.mask,t.tensor(mask))) + assert dataset.entry_info == entry_info + assert dataset.sample_info == sample_info + assert dataset.wavelength == wavelength + assert dataset.detector_geometry == detector_geometry + + +def test_CDataset_from_cxi(test_ptycho_cxis): + for cxi, expected in test_ptycho_cxis: + dataset = CDataset.from_cxi(cxi) + + # The entry metadata loaded + for key in expected['entry metadata']: + assert dataset.entry_info[key] == expected['entry metadata'][key] + + # Don't test for fidelity since this is tested in the data, just test + # that it is loaded + if expected['sample info'] is None: + assert dataset.sample_info is None + else: + assert dataset.sample_info is not None + + assert np.isclose(dataset.wavelength,expected['wavelength']) + + # Just check one of the loaded attributes + assert np.isclose(dataset.detector_geometry['distance'], + expected['detector']['distance']) + # Check that the other ones are loaded but not for fidelity + assert 'basis' in dataset.detector_geometry + if expected['detector']['corner'] is not None: + assert 'corner' in dataset.detector_geometry + + if expected['mask'] is not None: + assert t.all(t.eq(t.tensor(expected['mask']),dataset.mask)) + + +def test_CDataset_to_cxi(test_ptycho_cxis, tmp_path): + for cxi, expected in test_ptycho_cxis: + dataset = CDataset.from_cxi(cxi) + with cdtdata.create_cxi(tmp_path / 'test_CDataset_to_cxi.cxi') as f: + dataset.to_cxi(f) + + # Now we have to check that all the stuff was written + with h5py.File(tmp_path / 'test_CDataset_to_cxi.cxi', 'r') as f: + read_dataset = CDataset.from_cxi(f) + + assert dataset.entry_info == read_dataset.entry_info + + if dataset.sample_info is None: + assert read_dataset.sample_info is None + else: + assert read_dataset.sample_info is not None + + assert np.isclose(dataset.wavelength, read_dataset.wavelength) + + + # Just check one of the loaded attributes + assert np.isclose(dataset.detector_geometry['distance'], + read_dataset.detector_geometry['distance']) + # Check that the other ones are loaded but not for fidelity + assert 'basis' in read_dataset.detector_geometry + if dataset.detector_geometry['corner'] is not None: + assert 'corner' in read_dataset.detector_geometry + + + if dataset.mask is not None: + assert t.all(t.eq(dataset.mask,read_dataset.mask)) + + + +def test_CDataset_to(ptycho_cxi_1): + dataset = CDataset.from_cxi(ptycho_cxi_1[0]) + + dataset.to(dtype=t.float32) + assert dataset.mask.dtype == t.uint8 + # If cuda is available, check that moving the mask to CUDA works. + if t.cuda.is_available(): + dataset.to(device='cuda:0') + assert dataset.mask.device == t.device('cuda:0') + + + +# +# And we then test the derived Ptychography class +# +# + + +def test_Ptycho_2D_Dataset_init(): + entry_info = {'start_time': datetime.datetime.now(), + 'title' : 'A simple test'} + sample_info = {'name': 'A test sample', + 'mass' : 3.4, + 'unit_cell' : np.array([1,1,1,87,84.5,90])} + wavelength = 1e-9 + detector_geometry = {'distance': 0.7, + 'basis': np.array([[0,-30e-6,0], + [-20e-6,0,0]]).transpose(), + 'corner': np.array((2550e-6,3825e-6,0.3))} + mask = np.ones((256,256)) + patterns = np.random.rand(20,256,256) + translations = np.random.rand(20,3) + + dataset = Ptycho_2D_Dataset(translations, patterns, + entry_info=entry_info, + sample_info=sample_info, + wavelength=wavelength, + detector_geometry=detector_geometry, + mask=mask) + + assert t.all(t.eq(dataset.mask,t.tensor(mask))) + assert dataset.entry_info == entry_info + assert dataset.sample_info == sample_info + assert dataset.wavelength == wavelength + assert dataset.detector_geometry == detector_geometry + assert t.allclose(dataset.patterns, t.tensor(patterns)) + assert t.allclose(dataset.translations, t.tensor(translations)) + + + +def test_Ptycho_2D_Dataset_from_cxi(test_ptycho_cxis): + for cxi, expected in test_ptycho_cxis: + dataset = Ptycho_2D_Dataset.from_cxi(cxi) + + # The entry metadata loaded + for key in expected['entry metadata']: + assert dataset.entry_info[key] == expected['entry metadata'][key] + + # Don't test for fidelity since this is tested in the data, just test + # that it is loaded + if expected['sample info'] is None: + assert dataset.sample_info is None + else: + assert dataset.sample_info is not None + + assert np.isclose(dataset.wavelength,expected['wavelength']) + + # Just check one of the loaded attributes + assert np.isclose(dataset.detector_geometry['distance'], + expected['detector']['distance']) + # Check that the other ones are loaded but not for fidelity + assert 'basis' in dataset.detector_geometry + if expected['detector']['corner'] is not None: + assert 'corner' in dataset.detector_geometry + + if expected['mask'] is not None: + assert t.all(t.eq(t.tensor(expected['mask']),dataset.mask)) + + assert t.allclose(t.tensor(expected['data']),dataset.patterns) + assert t.allclose(t.tensor(expected['translations']),dataset.translations) + + + + +def test_Ptycho_2D_Dataset_to_cxi(test_ptycho_cxis, tmp_path): + for cxi, expected in test_ptycho_cxis: + dataset = Ptycho_2D_Dataset.from_cxi(cxi) + with cdtdata.create_cxi(tmp_path / 'test_Ptycho_2D_Dataset_to_cxi.cxi') as f: + dataset.to_cxi(f) + + # Now we have to check that all the stuff was written + with h5py.File(tmp_path / 'test_Ptycho_2D_Dataset_to_cxi.cxi', 'r') as f: + read_dataset = Ptycho_2D_Dataset.from_cxi(f) + + assert dataset.entry_info == read_dataset.entry_info + + if dataset.sample_info is None: + assert read_dataset.sample_info is None + else: + assert read_dataset.sample_info is not None + + assert np.isclose(dataset.wavelength, read_dataset.wavelength) + + + # Just check one of the loaded attributes + assert np.isclose(dataset.detector_geometry['distance'], + read_dataset.detector_geometry['distance']) + # Check that the other ones are loaded but not for fidelity + assert 'basis' in read_dataset.detector_geometry + if dataset.detector_geometry['corner'] is not None: + assert 'corner' in read_dataset.detector_geometry + + + if dataset.mask is not None: + assert t.all(t.eq(dataset.mask,read_dataset.mask)) + + assert t.allclose(dataset.patterns, read_dataset.patterns) + assert t.allclose(dataset.translations, read_dataset.translations) + + +def test_Ptycho_2D_Dataset_to(ptycho_cxi_1): + dataset = Ptycho_2D_Dataset.from_cxi(ptycho_cxi_1[0]) + + dataset.to(dtype=t.float64) + assert dataset.mask.dtype == t.uint8 + assert dataset.patterns.dtype == t.float64 + assert dataset.translations.dtype == t.float64 + # If cuda is available, check that moving the mask to CUDA works. + if t.cuda.is_available(): + dataset.to(device='cuda:0') + assert dataset.mask.device == t.device('cuda:0') + assert dataset.patterns.device == t.device('cuda:0') + assert dataset.translations.device == t.device('cuda:0') + + +def test_Ptycho_2D_Dataset_ops(ptycho_cxi_1): + cxi, expected = ptycho_cxi_1 + dataset = Ptycho_2D_Dataset.from_cxi(cxi) + + assert len(dataset) == expected['data'].shape[0] + idx, translation, pattern = dataset[3] + assert idx == 3 + assert t.allclose(translation, t.tensor(expected['translations'][3,:])) + assert t.allclose(pattern, t.tensor(expected['data'][3,:,:])) diff --git a/tests/tools/test_data.py b/tests/tools/test_data.py index ed8b508..573a8e5 100644 --- a/tests/tools/test_data.py +++ b/tests/tools/test_data.py @@ -7,300 +7,13 @@ import h5py import pytest import os import datetime +import numbers from pathlib import Path -# -# First, we write a few fixtures to generate specific data files we -# want with deliberate pathologies. -# -# * Energy defined but no wavelength -# * Wavelength defined but no energy -# * Distance and pixel pitch defined but no corner position or basis -# * Corner position and basis defined but no distance or pixel pitch -# * No mask defined -# * Mask defined -# -# Each file will get a fixture that loads the cxi file but also loads -# a dictionary with the relevant information for the cxi file -# -# - - -# This just grabs the directory whose name matches the file we're running -@pytest.fixture(scope='module') -def datadir(request): - filename = request.module.__file__ - test_dir, _ = os.path.splitext(filename) - return Path(test_dir) - - -@pytest.fixture(scope='module') -def ptycho_cxi_1(): - """Creates an example file for CXI ptychography. This file is defined - to have everything done as correctly as possible with lots of attributes - defined. It will return both a dictionary describing what is expected - to be loaded and a file with the data stored in it. - """ - - expected = {} - f = h5py.File('ptycho_cxi_1',driver='core',backing_store=False) - - # Start by defining the basic structure - f.create_dataset('cxi_version', data=150) - f.create_dataset('number_of_entries',data=1) - - # Then define a bunch of metadata for entry_1 - e1f = f.create_group('entry_1') - expected['entry metadata'] = {} - e1e = expected['entry metadata'] - e1e['start_time'] = datetime.datetime.now().isoformat() - e1f['start_time'] = np.string_(e1e['start_time']) - e1e['end_time'] = datetime.datetime.now().isoformat() - e1f['end_time'] = np.string_(e1e['end_time']) - e1e['experiment_identifier'] = 'Fake Experiment 1' - e1f['experiment_identifier'] = np.string_(e1e['experiment_identifier']) - e1e['experiment_description'] = 'A fully defined ptychography experiment to test the data loading' - e1f['experiment_description'] = np.string_(e1e['experiment_description']) - e1e['program_name'] = 'CDTools' - e1f['program_name'] = np.string_(e1e['program_name']) - e1e['title'] = 'The one experiment we did' - e1f['title'] = np.string_(e1e['title']) - - # Set up the sample info - s1f = e1f.create_group('sample_1') - expected['sample info'] = {} - s1e = expected['sample info'] - s1e['name'] = 'Fake Sample' - s1f['name'] = np.string_(s1e['name']) - s1e['description'] = 'A sample that isn\'t real' - s1f['description'] = np.string_(s1e['description']) - s1e['unit_cell_group'] = 'P1' - s1f['unit_cell_group'] = np.string_(s1e['unit_cell_group']) - s1e['concentration'] = np.float32(np.random.rand()) - s1f['concentration'] = s1e['concentration'] - s1e['mass'] = np.float32(np.random.rand()) - s1f['mass'] = s1e['mass'] - s1e['temperature'] = np.float32(np.random.rand()*100) - s1f['temperature'] = s1e['temperature'] - s1e['thickness'] = np.float32(np.random.rand()*1e-7) - s1f['thickness'] = s1e['thickness'] - s1e['unit_cell_volume'] = np.float32(np.random.rand() * 1e-27) - s1f['unit_cell_volume'] = s1e['unit_cell_volume'] - s1e['unit_cell'] = np.array([1,1,1,90,90,90]).astype(np.float32) - s1f.create_dataset('unit_cell',data = s1e['unit_cell']) - - i1f = e1f.create_group('instrument_1') - source1f = i1f.create_group('source_1') - - energy = np.float32(1.3618e-16) #Joules, = 850 eV - source1f['energy'] = energy - expected['wavelength'] = np.float32(1.9864459e-25) / energy - source1f['wavelength'] = expected['wavelength'] - - d1f = i1f.create_group('detector_1') - expected['detector'] = {} - d1e = expected['detector'] - d1e['distance'] = np.float32(0.3) - d1f['distance'] = d1e['distance'] - d1e['basis'] = np.array([[0,-30e-6,0], - [-20e-6,0,0]]).astype(np.float32).transpose() - d1f.create_dataset('basis_vectors',data=d1e['basis']) - d1f['x_pixel_size'] = np.float32(20e-6) - d1f['y_pixel_size'] = np.float32(30e-6) - d1e['corner'] = np.array((2550e-6,3825e-6,0.3)).astype(np.float32) - d1f.create_dataset('corner_position', data=d1e['corner']) - - # Remember the format for the CXI file differs from the format used - # internally - mask = np.zeros((100,256,256)).astype(np.uint32) - expected['mask'] = np.ones((100,256,256)).astype(np.uint8) - d1f.create_dataset('mask',data=mask) - - data1f = e1f.create_group('data_1') - - data = np.random.rand(100,256,256).astype(np.float32) - expected['data'] = data - d1f.create_dataset('data',data=data) - data1f['data'] = h5py.SoftLink('/entry_1/instrument_1/detector_1/data') - - d1f['data'].attrs['axes'] = np.string_('translation:y:x') - expected['axes'] = ['translation','y','x'] - - g1f = s1f.create_group('geometry_1') - translations = np.arange(300).reshape((100,3)).astype(np.float32) - g1f.create_dataset('translation',data=translations) - data1f['translation'] = h5py.SoftLink('/entry_1/sample_1/geometry_1/translation') - d1f['translation'] = h5py.SoftLink('/entry_1/sample_1/geometry_1/translation') - expected['translations'] = -translations.transpose() - - return f, expected - - -@pytest.fixture(scope='module') -def ptycho_cxi_2(): - """Creates an example file for CXI ptychography. This file is defined - to have a subset of things missing. In particular, it: - - * Defines the wavelength but not the energy - * Defines the corner position but not the sample-detector distance - * Defines pixel sizes but no basis vectors - * Doesn't define a mask - * Only defines data in the relevant places, not under data_1 - * Doesn't explicitly define axes for the data arrays - * Is missing many allowed metadata attributes - - """ - - expected = {} - f = h5py.File('ptycho_cxi_2',driver='core',backing_store=False) - - # Start by defining the basic structure - f.create_dataset('cxi_version', data=150) - f.create_dataset('number_of_entries',data=1) - - # Then define a bunch of metadata for entry_1 - e1f = f.create_group('entry_1') - expected['entry metadata'] = {} - e1e = expected['entry metadata'] - e1e['title'] = 'The one experiment we did' - e1f['title'] = np.string_(e1e['title']) - - # Set up the sample info - s1f = e1f.create_group('sample_1') - expected['sample info'] = {} - s1e = expected['sample info'] - s1e['temperature'] = np.float32(np.random.rand()*100) - s1f['temperature'] = s1e['temperature'] - - i1f = e1f.create_group('instrument_1') - source1f = i1f.create_group('source_1') - - energy = np.float32(1.3618e-16) #Joules, = 850 eV - expected['wavelength'] = np.float32(1.9864459e-25) / energy - source1f['wavelength'] = expected['wavelength'] - - d1f = i1f.create_group('detector_1') - expected['detector'] = {} - d1e = expected['detector'] - d1e['distance'] = np.float32(0.3) - d1e['basis'] = np.array([[0,-30e-6,0], - [-20e-6,0,0]]).astype(np.float32).transpose() - d1f['x_pixel_size'] = np.float32(20e-6) - d1f['y_pixel_size'] = np.float32(30e-6) - d1e['corner'] = np.array((2550e-6,3825e-6,0.3)).astype(np.float32) - d1f.create_dataset('corner_position', data=d1e['corner']) - - # Remember the format for the CXI file differs from the format used - # internally - expected['mask'] = None - - data1f = e1f.create_group('data_1') - - data = np.random.rand(100,256,256).astype(np.float32) - expected['data'] = data - d1f.create_dataset('data',data=data) - - expected['axes'] = None - - g1f = s1f.create_group('geometry_1') - translations = np.arange(300).reshape((100,3)).astype(np.float32) - g1f.create_dataset('translation',data=translations) - expected['translations'] = -translations.transpose() - - return f, expected - - -@pytest.fixture(scope='module') -def ptycho_cxi_3(): - """Creates an example file for CXI ptychography. This file is defined - to have a different subset of information missing. In particular, it: - - * Has no sample_1 group - * Defines the energy but not the wavelength of light - * Has the data only defined under the data_1 group, not in the relevant places - * Defines the detector basis but no pixel sizes - * Defines a mask as all pixels flagged as "above the background" - * Defines the sample to detector distance but no corner location - * Is missing some of the allowed metadata - """ - - expected = {} - f = h5py.File('ptycho_cxi_3',driver='core',backing_store=False) - - # Start by defining the basic structure - f.create_dataset('cxi_version', data=150) - f.create_dataset('number_of_entries',data=1) - - # Then define a bunch of metadata for entry_1 - e1f = f.create_group('entry_1') - expected['entry metadata'] = {} - e1e = expected['entry metadata'] - e1e['start_time'] = datetime.datetime.now().isoformat() - e1f['start_time'] = np.string_(e1e['start_time']) - e1e['end_time'] = datetime.datetime.now().isoformat() - e1f['end_time'] = np.string_(e1e['end_time']) - - # Set up the sample info - expected['sample info'] = None - - i1f = e1f.create_group('instrument_1') - source1f = i1f.create_group('source_1') - - energy = np.float32(1.3618e-16) #Joules, = 850 eV - source1f['energy'] = energy - expected['wavelength'] = np.float32(1.9864459e-25) / energy - - d1f = i1f.create_group('detector_1') - expected['detector'] = {} - d1e = expected['detector'] - d1e['distance'] = np.float32(0.3) - d1f['distance'] = d1e['distance'] - d1e['basis'] = np.array([[0,-30e-6,0], - [-20e-6,0,0]]).astype(np.float32).transpose() - d1f.create_dataset('basis_vectors',data=d1e['basis']) - d1e['corner'] = None - - # Remember the format for the CXI file differs from the format used - # internally - mask = np.ones((100,256,256)).astype(np.uint32) * 0x00001000 - expected['mask'] = np.ones((100,256,256)).astype(np.uint8) - d1f.create_dataset('mask',data=mask) - - data1f = e1f.create_group('data_1') - - data = np.random.rand(100,256,256).astype(np.float32) - expected['data'] = data - data1f.create_dataset('data',data=data) - - data1f['data'].attrs['axes'] = np.string_('translation:y:x') - expected['axes'] = ['translation','y','x'] - - translations = np.arange(300).reshape((100,3)).astype(np.float32) - data1f.create_dataset('translation',data=translations) - expected['translations'] = -translations.transpose() - - return f, expected # -# As specific issues start to crop up with loading CXI files from different -# beamlines, put a fixture here that replicates the issue so that we can -# ensure compatibility with many beamlines -# - - -@pytest.fixture(scope='module') -def test_ptycho_cxis(ptycho_cxi_1, ptycho_cxi_2, ptycho_cxi_3): - """Loads a list of tuples of ptychography CXI files and dictionaries, - describing the expected output from various functions on being called - on the cxi files. - """ - return [ptycho_cxi_1, ptycho_cxi_2, ptycho_cxi_3] - - - -# -# Now we have a bunch of tests of the data loading capabilities +# We start with a bunch of tests of the data loading capabilities # @@ -373,3 +86,156 @@ def test_get_ptycho_translations(test_ptycho_cxis): # +def test_create_cxi(tmp_path): + data.create_cxi(tmp_path / 'test_create.cxi') + with h5py.File(tmp_path / 'test_create.cxi') as f: + assert f['cxi_version'][()] == 160 + assert 'entry_1' in f + + +def test_add_entry_info(tmp_path): + entry_info = {'experiment_identifier':'test of cxi file writing tools', + 'title': 'my cool experiment', + 'start_time': datetime.datetime.now(), + 'end_time': datetime.datetime.now()} + + with data.create_cxi(tmp_path / 'test_add_entry_info.cxi') as f: + data.add_entry_info(f, entry_info) + + with h5py.File(tmp_path / 'test_add_entry_info.cxi') as f: + read_entry_info = data.get_entry_info(f) + + for key in entry_info: + if isinstance(entry_info[key], np.ndarray): + assert np.allclose(entry_info[key], read_entry_info[key]) + else: + assert entry_info[key] == read_entry_info[key] + + +def test_add_sample_info(tmp_path): + sample_info = {'name':'A nice fake sample', + 'concentration': 10, + 'mass': 5.3, + 'temperature': 76, + 'description': 'A very nice sample', + 'unit_cell': np.array([1,1,1,90.,90.,90.])} + + with data.create_cxi(tmp_path / 'test_add_sample_info.cxi') as f: + data.add_sample_info(f, sample_info) + + with h5py.File(tmp_path / 'test_add_sample_info.cxi') as f: + read_sample_info = data.get_sample_info(f) + + for key in sample_info: + if isinstance(sample_info[key], np.ndarray): + assert np.allclose(sample_info[key], read_sample_info[key]) + elif isinstance(sample_info[key], numbers.Number): + assert np.isclose(sample_info[key], read_sample_info[key]) + else: + assert sample_info[key] == read_sample_info[key] + + +def test_add_source(tmp_path): + wavelength = 1e-9 + energy = 1.9864459e-25 / wavelength + + with data.create_cxi(tmp_path / 'test_add_source.cxi') as f: + data.add_source(f, wavelength) + + with h5py.File(tmp_path / 'test_add_source.cxi') as f: + # Check this directly since we want to make sure it saved + # the wavelength and energy + read_wavelength = f['entry_1/instrument_1/source_1/wavelength'][()] + read_energy = f['entry_1/instrument_1/source_1/energy'][()] + + assert np.isclose( wavelength, read_wavelength) + assert np.isclose( energy, read_energy) + + +def test_add_detector(tmp_path): + distance = 0.34 + basis = np.array([[0,-30e-6,0], + [-20e-6,0,0]]).astype(np.float32).transpose() + corner = np.array((2550e-6,3825e-6,0.3)).astype(np.float32) + + with data.create_cxi(tmp_path / 'test_add_detector.cxi') as f: + data.add_detector(f, distance, basis, corner=corner) + + with h5py.File(tmp_path / 'test_add_detector.cxi') as f: + # Check this directly since we want to make sure it saved + # the pixel sizes + d1 = f['entry_1/instrument_1/detector_1'] + read_basis = np.array(d1['basis_vectors']) + read_x_pix = np.float32(d1['x_pixel_size']) + read_y_pix = np.float32(d1['y_pixel_size']) + read_distance = np.float32(d1['distance']) + read_corner = np.array(d1['corner_position']) + + assert np.isclose(distance, read_distance) + assert np.allclose(basis, read_basis) + assert np.isclose(np.linalg.norm(basis[:,1]), read_x_pix) + assert np.isclose(np.linalg.norm(basis[:,0]), read_y_pix) + assert np.allclose(corner,read_corner) + + +def test_add_mask(tmp_path): + mask = (np.random.rand(350,600) > 0.1).astype(np.uint8) + + with data.create_cxi(tmp_path / 'test_add_mask.cxi') as f: + data.add_mask(f, mask) + + with h5py.File(tmp_path / 'test_add_mask.cxi') as f: + read_mask = data.get_mask(f) + + assert np.all(mask == read_mask) + + +def test_add_data(tmp_path): + # First test from numpy, with axes + fake_data = np.random.rand(100,256,256) + axes = ['translation','y','x'] + + with data.create_cxi(tmp_path / 'test_add_data.cxi') as f: + data.add_data(f, fake_data, axes) + + with h5py.File(tmp_path / 'test_add_data.cxi') as f: + # Check this directly since we want to make sure it saved + # it in all the places it should have + read_data_1 = np.array(f['entry_1/data_1/data']) + read_data_2 = np.array(f['entry_1/instrument_1/detector_1/data']) + read_axes = str(f['entry_1/instrument_1/detector_1/data'].attrs['axes'].decode()) + + + assert np.allclose(fake_data, read_data_1) + assert np.allclose(fake_data, read_data_2) + assert 'translation:y:x' == read_axes + + # Then test from torch, without axes + fake_data = t.from_numpy(fake_data) + + with data.create_cxi(tmp_path / 'test_add_data_torch.cxi') as f: + data.add_data(f, fake_data) + + with h5py.File(tmp_path / 'test_add_data_torch.cxi') as f: + read_data, axes = data.get_data(f) + + assert np.allclose(fake_data.numpy(),read_data) + + +def test_add_ptycho_translations(tmp_path): + + translations = np.random.rand(3,100) + + with data.create_cxi(tmp_path / 'test_add_ptycho_translations.cxi') as f: + data.add_ptycho_translations(f, translations) + + with h5py.File(tmp_path / 'test_add_ptycho_translations.cxi') as f: + # Check this directly since we want to make sure it saved + # it in all the places it should have + read_translations_1 = np.array(f['entry_1/data_1/translation']) + read_translations_2 = np.array(f['entry_1/instrument_1/detector_1/translation']) + read_translations_3 = np.array(f['entry_1/sample_1/geometry_1/translation']) + + assert np.allclose(-translations, read_translations_1) + assert np.allclose(-translations, read_translations_2) + assert np.allclose(-translations, read_translations_3)