From 25224d4d44773d65dba01889cd424725e277e647 Mon Sep 17 00:00:00 2001 From: allevitan Date: Sun, 13 Oct 2024 13:08:17 -0400 Subject: [PATCH 1/3] deal with the string_ related issues --- src/cdtools/tools/data/data.py | 10 +++++----- tests/conftest.py | 28 ++++++++++++++-------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/cdtools/tools/data/data.py b/src/cdtools/tools/data/data.py index 19e7467..ed13542 100644 --- a/src/cdtools/tools/data/data.py +++ b/src/cdtools/tools/data/data.py @@ -485,9 +485,9 @@ def add_entry_info(cxi_file, metadata): # included in case the cxi spec becomes more permissive for key, value in metadata.items(): if isinstance(value,(str,bytes)): - cxi_file['entry_1'][key] = np.string_(value) + cxi_file['entry_1'][key] = np.bytes_(value) elif isinstance(value, datetime.datetime): - cxi_file['entry_1'][key] = np.string_(value.isoformat()) + cxi_file['entry_1'][key] = np.bytes_(value.isoformat()) elif isinstance(value, numbers.Number): cxi_file['entry_1'][key] = value elif isinstance(value, (np.ndarray,list,tuple)): @@ -525,9 +525,9 @@ def add_sample_info(cxi_file, metadata): if key == 'orientation': continue # this is a special case if isinstance(value,(str,bytes)): - s1[key] = np.string_(value) + s1[key] = np.bytes_(value) elif isinstance(value, datetime.datetime): - s1[key] = np.string_(value.isoformat()) + s1[key] = np.bytes_(value.isoformat()) elif isinstance(value, numbers.Number): s1[key] = value elif isinstance(value, (np.ndarray,list,tuple)): @@ -701,7 +701,7 @@ def add_data(cxi_file, data, axes=None, compression='gzip', axes_str = ':'.join(axes) else: axes_str = str(axes) - det1['data'].attrs['axes'] = np.string_(axes_str) + det1['data'].attrs['axes'] = np.bytes_(axes_str) def add_shot_to_shot_info(cxi_file, data, field_name): diff --git a/tests/conftest.py b/tests/conftest.py index 9eceb0c..ca3a2c7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -77,28 +77,28 @@ def ptycho_cxi_1(): expected['entry metadata'] = {} e1e = expected['entry metadata'] e1e['start_time'] = datetime.datetime.now() - e1f['start_time'] = np.string_(e1e['start_time'].isoformat()) + e1f['start_time'] = np.bytes_(e1e['start_time'].isoformat()) e1e['end_time'] = datetime.datetime.now() - e1f['end_time'] = np.string_(e1e['end_time'].isoformat()) + e1f['end_time'] = np.bytes_(e1e['end_time'].isoformat()) e1e['experiment_identifier'] = 'Fake Experiment 1' - e1f['experiment_identifier'] = np.string_(e1e['experiment_identifier']) + e1f['experiment_identifier'] = np.bytes_(e1e['experiment_identifier']) e1e['experiment_description'] = 'A fully defined ptychography experiment to test the data loading' - e1f['experiment_description'] = np.string_(e1e['experiment_description']) + e1f['experiment_description'] = np.bytes_(e1e['experiment_description']) e1e['program_name'] = 'cdtools' - e1f['program_name'] = np.string_(e1e['program_name']) + e1f['program_name'] = np.bytes_(e1e['program_name']) e1e['title'] = 'The one experiment we did' - e1f['title'] = np.string_(e1e['title']) + e1f['title'] = np.bytes_(e1e['title']) # Set up the sample info s1f = e1f.create_group('sample_1') expected['sample info'] = {} s1e = expected['sample info'] s1e['name'] = 'Fake Sample' - s1f['name'] = np.string_(s1e['name']) + s1f['name'] = np.bytes_(s1e['name']) s1e['description'] = 'A sample that isn\'t real' - s1f['description'] = np.string_(s1e['description']) + s1f['description'] = np.bytes_(s1e['description']) s1e['unit_cell_group'] = 'P1' - s1f['unit_cell_group'] = np.string_(s1e['unit_cell_group']) + s1f['unit_cell_group'] = np.bytes_(s1e['unit_cell_group']) s1e['concentration'] = np.float32(np.random.rand()) s1f['concentration'] = s1e['concentration'] s1e['mass'] = np.float32(np.random.rand()) @@ -151,7 +151,7 @@ def ptycho_cxi_1(): d1f.create_dataset('data',data=data) data1f['data'] = h5py.SoftLink('/entry_1/instrument_1/detector_1/data') - d1f['data'].attrs['axes'] = np.string_('translation:y:x') + d1f['data'].attrs['axes'] = np.bytes_('translation:y:x') expected['axes'] = ['translation','y','x'] g1f = s1f.create_group('geometry_1') @@ -196,7 +196,7 @@ def ptycho_cxi_2(): expected['entry metadata'] = {} e1e = expected['entry metadata'] e1e['title'] = 'The one experiment we did' - e1f['title'] = np.string_(e1e['title']) + e1f['title'] = np.bytes_(e1e['title']) # Set up the sample info s1f = e1f.create_group('sample_1') @@ -277,9 +277,9 @@ def ptycho_cxi_3(): expected['entry metadata'] = {} e1e = expected['entry metadata'] e1e['start_time'] = datetime.datetime.now() - e1f['start_time'] = np.string_(e1e['start_time'].isoformat()) + e1f['start_time'] = np.bytes_(e1e['start_time'].isoformat()) e1e['end_time'] = datetime.datetime.now() - e1f['end_time'] = np.string_(e1e['end_time'].isoformat()) + e1f['end_time'] = np.bytes_(e1e['end_time'].isoformat()) # Set up the sample info expected['sample info'] = None @@ -314,7 +314,7 @@ def ptycho_cxi_3(): expected['data'] = data data1f.create_dataset('data',data=data) - data1f['data'].attrs['axes'] = np.string_('translation:y:x') + data1f['data'].attrs['axes'] = np.bytes_('translation:y:x') expected['axes'] = ['translation','y','x'] translations = np.arange(300).reshape((100,3)).astype(np.float32) From e9e2fd4beef36acc7b01beba2f091314a5d2d000 Mon Sep 17 00:00:00 2001 From: allevitan Date: Sun, 13 Oct 2024 13:14:19 -0400 Subject: [PATCH 2/3] Get all tests passing --- src/cdtools/tools/initializers/initializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cdtools/tools/initializers/initializers.py b/src/cdtools/tools/initializers/initializers.py index c0385ee..7e727af 100644 --- a/src/cdtools/tools/initializers/initializers.py +++ b/src/cdtools/tools/initializers/initializers.py @@ -57,7 +57,7 @@ def exit_wave_geometry(det_basis, det_shape, wavelength, distance, oversampling= # This method should work for a general parallelogram-shaped detector det_shape = det_basis * det_shape.to(t.float32) - pinv_basis = t.tensor(np.linalg.pinv(det_shape).transpose()).to(t.float32) + pinv_basis = t.linalg.pinv(det_shape).transpose(0,1) real_space_basis = pinv_basis * wavelength * distance return real_space_basis From 40651eaf54562d0a1ccbde78d0184664281115cb Mon Sep 17 00:00:00 2001 From: allevitan Date: Mon, 14 Oct 2024 08:30:15 -0400 Subject: [PATCH 3/3] Remove the various deprication warnings related to converting h5py datasets to arrays with np.array() --- src/cdtools/tools/analysis/analysis.py | 3 +-- src/cdtools/tools/data/data.py | 23 +++++++++++---------- tests/tools/test_data.py | 28 ++++++++++++++------------ tests/tools/test_measurements.py | 2 +- 4 files changed, 29 insertions(+), 27 deletions(-) diff --git a/src/cdtools/tools/analysis/analysis.py b/src/cdtools/tools/analysis/analysis.py index d6f2f78..0fd913e 100644 --- a/src/cdtools/tools/analysis/analysis.py +++ b/src/cdtools/tools/analysis/analysis.py @@ -352,8 +352,7 @@ def synthesize_reconstructions(probes, objects, use_probe=False, obj_slice=None, else: shift = ip.find_shift(synth_obj[obj_slice],obj[obj_slice], resolution=50) - - obj = ip.sinc_subpixel_shift(obj,np.array(shift)) + obj = ip.sinc_subpixel_shift(obj, shift) if len(probe.shape) == 3: probe = t.stack([ip.sinc_subpixel_shift(p,tuple(shift)) diff --git a/src/cdtools/tools/data/data.py b/src/cdtools/tools/data/data.py index ed13542..556aa38 100644 --- a/src/cdtools/tools/data/data.py +++ b/src/cdtools/tools/data/data.py @@ -134,18 +134,18 @@ def get_sample_info(cxi_file): metadata[attr] = np.float32(s1[attr][()]) if 'unit_cell' in s1: - metadata['unit_cell'] = np.array(s1['unit_cell']).astype(np.float32) + metadata['unit_cell'] = s1['unit_cell'][()].astype(np.float32) if 'geometry_1/orientation' in s1: - orient = np.array(s1['geometry_1/orientation']).astype(np.float32) + orient = s1['geometry_1/orientation'][()].astype(np.float32) xvec = orient[:3] / np.linalg.norm(orient[:3]) yvec = orient[3:] / np.linalg.norm(orient[3:]) metadata['orientation'] = np.array([xvec,yvec, np.cross(xvec,yvec)]) if 'geometry_1/surface_normal' in s1: - snorm = np.array(s1['geometry_1/surface_normal']).astype(np.float32) + snorm = s1['geometry_1/surface_normal'][()].astype(np.float32) xvec = np.cross(np.array([0.,1.,0.]), snorm) xvec /= np.linalg.norm(xvec) yvec = np.cross(snorm, xvec) @@ -218,7 +218,7 @@ def get_detector_geometry(cxi_file): d1 = i1['detector_1'] if 'detector_1/basis_vectors' in i1: - basis_vectors = np.array(d1['basis_vectors']) + basis_vectors = d1['basis_vectors'][()] if basis_vectors.shape == (2,3): basis_vectors = basis_vectors.T else: @@ -244,11 +244,11 @@ def get_detector_geometry(cxi_file): [-x_pixel_size,0,0]]).transpose() try: - distance = np.float32(d1['distance']) + distance = np.float32(d1['distance'][()]) except: distance = None try: - corner_position = np.array(d1['corner_position']) + corner_position = d1['corner_position'][()] except: corner_position = None @@ -292,7 +292,7 @@ def get_mask(cxi_file): i1 = cxi_file['entry_1/instrument_1'] if 'detector_1/mask' in i1: - mask = np.array(i1['detector_1/mask']).astype(np.uint32) + mask = i1['detector_1/mask'][()].astype(np.uint32) mask_on = np.equal(mask,np.uint32(0)) mask_has_signal = np.equal(mask,np.uint32(0x00001000)) return np.logical_or(mask_on,mask_has_signal).astype(bool) @@ -324,7 +324,7 @@ def get_dark(cxi_file): i1 = cxi_file['entry_1/instrument_1'] if 'detector_1/data_dark' in i1: - darks = np.array(i1['detector_1/data_dark']) + darks = i1['detector_1/data_dark'][()] dims = tuple(range(len(darks.shape) - 2)) darks = np.nanmean(darks,axis=dims) else: @@ -426,7 +426,7 @@ def get_shot_to_shot_info(cxi_file, field_name): else: raise KeyError('Data is not defined within cxi file') - return np.array(cxi_file[pull_from]).astype(np.float32) + return cxi_file[pull_from][()].astype(np.float32) def get_ptycho_translations(cxi_file): @@ -851,11 +851,12 @@ def h5_to_nested_dict(h5_file): for key in h5_file.keys(): value = h5_file[key] if isinstance(value, h5py.Dataset): - arr = np.array(value) + arr = value[()] if arr.dtype == object: d[key] = arr.ravel()[0].decode('utf-8') elif arr.ndim == 0: - d[key] = arr.ravel()[0] + # TODO is this needed with arr = value[()]? + d[key] = arr.ravel()[0] else: d[key] = arr diff --git a/tests/tools/test_data.py b/tests/tools/test_data.py index 03998dd..bf048e5 100644 --- a/tests/tools/test_data.py +++ b/tests/tools/test_data.py @@ -182,11 +182,11 @@ def test_add_detector(tmp_path): # Check this directly since we want to make sure it saved # the pixel sizes d1 = f['entry_1/instrument_1/detector_1'] - read_basis = np.array(d1['basis_vectors']) - read_x_pix = np.float32(d1['x_pixel_size']) - read_y_pix = np.float32(d1['y_pixel_size']) - read_distance = np.float32(d1['distance']) - read_corner = np.array(d1['corner_position']) + read_basis = d1['basis_vectors'][()] + read_x_pix = d1['x_pixel_size'][()] + read_y_pix = d1['y_pixel_size'][()] + read_distance = d1['distance'][()] + read_corner = d1['corner_position'][()] assert np.isclose(distance, read_distance) assert np.allclose(basis, read_basis) @@ -232,8 +232,8 @@ def test_add_data(tmp_path): with h5py.File(tmp_path / 'test_add_data.cxi','r') as f: # Check this directly since we want to make sure it saved # it in all the places it should have - read_data_1 = np.array(f['entry_1/data_1/data']) - read_data_2 = np.array(f['entry_1/instrument_1/detector_1/data']) + read_data_1 = f['entry_1/data_1/data'][()] + read_data_2 = f['entry_1/instrument_1/detector_1/data'][()] read_axes = str(f['entry_1/instrument_1/detector_1/data'].attrs['axes'].decode()) assert np.allclose(fake_data, read_data_1) @@ -261,9 +261,10 @@ def test_add_shot_to_shot_info(tmp_path): with h5py.File(tmp_path / 'test_add_shot_to_shot_info.cxi') as f: # Check this directly since we want to make sure it saved # it in all the places it should have - read_analyzer_1 = np.array(f['entry_1/data_1/analyzer_angle']) - read_analyzer_2 = np.array(f['entry_1/instrument_1/detector_1/analyzer_angle']) - read_analyzer_3 = np.array(f['entry_1/sample_1/geometry_1/analyzer_angle']) + read_analyzer_1 = f['entry_1/data_1/analyzer_angle'][()] + read_analyzer_2 = \ + f['entry_1/instrument_1/detector_1/analyzer_angle'][()] + read_analyzer_3 = f['entry_1/sample_1/geometry_1/analyzer_angle'][()] assert np.allclose(analyzer, read_analyzer_1) assert np.allclose(analyzer, read_analyzer_2) @@ -280,9 +281,10 @@ def test_add_ptycho_translations(tmp_path): with h5py.File(tmp_path / 'test_add_ptycho_translations.cxi','r') as f: # Check this directly since we want to make sure it saved # it in all the places it should have - read_translations_1 = np.array(f['entry_1/data_1/translation']) - read_translations_2 = np.array(f['entry_1/instrument_1/detector_1/translation']) - read_translations_3 = np.array(f['entry_1/sample_1/geometry_1/translation']) + read_translations_1 = f['entry_1/data_1/translation'][()] + read_translations_2 = \ + f['entry_1/instrument_1/detector_1/translation'][()] + read_translations_3 = f['entry_1/sample_1/geometry_1/translation'][()] assert np.allclose(-translations, read_translations_1) assert np.allclose(-translations, read_translations_2) diff --git a/tests/tools/test_measurements.py b/tests/tools/test_measurements.py index 78eceb0..bfa06de 100644 --- a/tests/tools/test_measurements.py +++ b/tests/tools/test_measurements.py @@ -6,7 +6,7 @@ import numpy as np def test_intensity(): wavefields = t.rand((5,10,10)) + 1j * t.rand((5,10,10)) epsilon=1e-6 - np_result = np.abs(t.as_tensor(wavefields))**2 + epsilon + np_result = np.abs(wavefields.numpy())**2 + epsilon assert t.allclose(measurements.intensity(wavefields,epsilon=epsilon), t.as_tensor(np_result))