diff --git a/CDTools/datasets/__init__.py b/CDTools/datasets/__init__.py index 30b0322..504647a 100644 --- a/CDTools/datasets/__init__.py +++ b/CDTools/datasets/__init__.py @@ -32,8 +32,11 @@ import numpy as np import torch as t from copy import copy import h5py -import pathlib - +try: + import pathlib +except ImportError: + import pathlib2 as pathlib + from CDTools.tools import data as cdtdata from CDTools.tools import plotting from torch.utils import data as torchdata @@ -98,7 +101,10 @@ class CDataset(torchdata.Dataset): self.wavelength = wavelength self.detector_geometry = copy(detector_geometry) if mask is not None: - self.mask = t.tensor(mask) + if isinstance(mask, t.Tensor): + self.mask = mask.detach().to(dtype=t.bool) + else: + self.mask = t.BoolTensor(mask) else: self.mask = None if background is not None: diff --git a/CDTools/datasets/ptycho_2d_dataset.py b/CDTools/datasets/ptycho_2d_dataset.py index fbd5f9a..57ebe63 100644 --- a/CDTools/datasets/ptycho_2d_dataset.py +++ b/CDTools/datasets/ptycho_2d_dataset.py @@ -3,7 +3,10 @@ import numpy as np import torch as t from copy import copy import h5py -import pathlib +try: + import pathlib +except ImportError: + import pathlib2 as pathlib from CDTools.datasets import CDataset from CDTools.tools import data as cdtdata diff --git a/CDTools/tools/losses.py b/CDTools/tools/losses.py index f49ac8f..7b92030 100644 --- a/CDTools/tools/losses.py +++ b/CDTools/tools/losses.py @@ -21,7 +21,11 @@ def amplitude_mse(intensities, sim_intensities, mask=None): This function calculates the mean squared error between their associated amplitudes. Because this is not well defined for negative numbers, make sure that all the intensities are >0 before using this - loss. + loss. Note that this is actually a sum-squared error, because this + formulation makes it vastly simpler to compare error calculations + between reconstructions with different minibatch size. I hope to + find a better way to do this that is more honest with this + cost function, though. It can accept intensity and simulated intensity tensors of any shape as long as their shapes match, and the provided mask array can be diff --git a/conda_requirements.txt b/conda_requirements.txt new file mode 100644 index 0000000..48fe15a --- /dev/null +++ b/conda_requirements.txt @@ -0,0 +1,8 @@ +numpy>=1.0 +scipy>=1.0 +matplotlib>=2.0 +python-dateutil +pytorch>=1.2.0 +h5py>=2.1 +pytest +sphinx diff --git a/docs/source/installation.rst b/docs/source/installation.rst index be1ab4f..7147748 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -13,6 +13,18 @@ It is recommended that you clone the repository, rather than just downloading th Step 2: Install Dependencies ---------------------------- +The dependencies for CDTools can be installed, if you are managing your environment with anaconda, by running + +.. code:: bash + + $ conda install --file conda_requirements.txt + +There are two optional dependencies which are not installed via this procedure - the dependency sphinx-argparse for building the docs, and the pathlib2 module that provides python 2 compatibility. These can either be installed manually via conda-forge, or otherwise they will be installed automatically by pip during the final installation step if needed. + +If you manage your environment with pip, all required packges should be installed automatically. The only thing to be aware of is that pytorch must be compiled with MKL support, and CUDA support if you would like to use the GPU. For this reason, using anaconda python is strongly recommended. + +For convenience, the full set of dependencies are noted below: + CDTools depends on the following packages: * `numpy `_ @@ -27,6 +39,7 @@ And has optional dependencies on * `pytest `_ * `sphinx `_ * `sphinx-argparse `_ + * `pathlib2 `_ All of these can be installed via pip or conda. Finally, CDTools is written to be python 2.7+ compatible, but is only actively tested on python 3. @@ -42,7 +55,9 @@ To install in CDTools in developer mode (recommended, to allow any updates to be .. code:: bash - $ pip install -e . + $ pip install -e .[tests,docs] + +If you don't need to run the tests, or don't need to build the docs, you can omit the relevant option or options. If you prefer to use a tool other than pip, CDTools can be installed via any other package management tool that works with a setup.py file. diff --git a/setup.py b/setup.py index 4d70f23..f44fb81 100644 --- a/setup.py +++ b/setup.py @@ -13,12 +13,18 @@ setuptools.setup( long_description_content_type="text/markdown", url="https://github.mit.edu/scattering/CDTools.git", install_requires=[ - "numpy", - "scipy", - "matplotlib", + "numpy>=1.0", + "scipy>=1.0", + "matplotlib>=2.0", "python-dateutil", - "torch", - "h5py"], + "torch>=1.2.0", #1.2.0 introduced boolean tensors in a breaking way, we use the boolean tensors here for masking + "h5py>=2.1", + "pathlib2 ; python_version<'3.4'"], + extras_require={ + 'tests': ["pytest"], + 'docs': ["sphinx","sphinx-argparse"], + ":python_version<'3.4'": ["pathlib2"], + }, packages=setuptools.find_packages(), classifiers=[ "Programming Language :: Python :: 3", diff --git a/tests/conftest.py b/tests/conftest.py index f1abff3..0894750 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -103,8 +103,8 @@ def ptycho_cxi_1(): # Remember the format for the CXI file differs from the format used # internally - mask = np.zeros((256,256)).astype(np.uint32) - expected['mask'] = np.ones((256,256)).astype(np.uint8) + mask = np.zeros((256,256)).astype(np.int32) + expected['mask'] = np.ones((256,256)).astype(np.bool) d1f.create_dataset('mask',data=mask) # Create an initial background @@ -272,7 +272,7 @@ def ptycho_cxi_3(): # Remember the format for the CXI file differs from the format used # internally mask = np.ones((256,256)).astype(np.uint32) * 0x00001000 - expected['mask'] = np.ones((256,256)).astype(np.uint8) + expected['mask'] = np.ones((256,256)).astype(np.bool) d1f.create_dataset('mask',data=mask) expected['dark'] = None diff --git a/tests/test_datasets.py b/tests/test_datasets.py index 933bd70..c83e204 100644 --- a/tests/test_datasets.py +++ b/tests/test_datasets.py @@ -27,8 +27,8 @@ def test_CDataset_init(): mask = np.ones((256,256)) dataset = CDataset(entry_info, sample_info, wavelength, detector_geometry, mask) - - assert t.all(t.eq(dataset.mask,t.tensor(mask))) + + assert t.all(t.eq(dataset.mask,t.tensor(mask.astype(np.bool)))) assert dataset.entry_info == entry_info assert dataset.sample_info == sample_info assert dataset.wavelength == wavelength @@ -110,7 +110,7 @@ def test_CDataset_to(ptycho_cxi_1): dataset = CDataset.from_cxi(ptycho_cxi_1[0]) dataset.to(dtype=t.float32) - assert dataset.mask.dtype == t.uint8 + assert dataset.mask.dtype == t.bool # If cuda is available, check that moving the mask to CUDA works. if t.cuda.is_available(): dataset.to(device='cuda:0') @@ -147,7 +147,7 @@ def test_Ptycho2DDataset_init(): detector_geometry=detector_geometry, mask=mask) - assert t.all(t.eq(dataset.mask,t.tensor(mask))) + assert t.all(t.eq(dataset.mask,t.BoolTensor(mask))) assert dataset.entry_info == entry_info assert dataset.sample_info == sample_info assert dataset.wavelength == wavelength @@ -241,7 +241,7 @@ def test_Ptycho2DDataset_to(ptycho_cxi_1): dataset = Ptycho2DDataset.from_cxi(ptycho_cxi_1[0]) dataset.to(dtype=t.float64) - assert dataset.mask.dtype == t.uint8 + assert dataset.mask.dtype == t.bool assert dataset.patterns.dtype == t.float64 assert dataset.translations.dtype == t.float64 # If cuda is available, check that moving the mask to CUDA works. diff --git a/tests/tools/test_data.py b/tests/tools/test_data.py index 5045365..0732bbd 100644 --- a/tests/tools/test_data.py +++ b/tests/tools/test_data.py @@ -8,7 +8,10 @@ import pytest import os import datetime import numbers -from pathlib import Path +try: + import pathlib +except ImportError: + import pathlib2 as pathlib diff --git a/tests/tools/test_losses.py b/tests/tools/test_losses.py index d6f9b71..de80a6e 100644 --- a/tests/tools/test_losses.py +++ b/tests/tools/test_losses.py @@ -16,17 +16,17 @@ def test_amplitude_mse(): # And add some noise to it sim = data + 0.1 * np.random.rand(10,100,100) # and define a simple mask that needs to be broadcast - mask = (np.random.rand(100,100) > 0.1).astype(np.uint8) + mask = (np.random.rand(100,100) > 0.1).astype(np.bool) # First, test without a mask np_result = np.sum((np.sqrt(data) - np.sqrt(sim))**2) - np_result /= data.size + #np_result /= data.size torch_result = losses.amplitude_mse(t.from_numpy(data),t.from_numpy(sim)) assert np.isclose(np_result, np.take(torch_result.numpy(),0)) # Then, test with a mask np_result = np.sum(mask * (np.sqrt(data) - np.sqrt(sim))**2) - np_result /= np.count_nonzero(mask * np.ones_like(data)) + #np_result /= np.count_nonzero(mask * np.ones_like(data)) torch_result = losses.amplitude_mse(t.from_numpy(data),t.from_numpy(sim), mask = t.from_numpy(mask)) assert np.isclose(np_result, np.take(torch_result.numpy(),0)) @@ -38,7 +38,7 @@ def test_intensity_mse(): # And add some noise to it sim = data + 0.1 * np.random.rand(10,100,100) # and define a simple mask that needs to be broadcast - mask = (np.random.rand(100,100) > 0.1).astype(np.uint8) + mask = (np.random.rand(100,100) > 0.1).astype(np.bool) # First, test without a mask @@ -61,7 +61,7 @@ def test_poisson_ml(): # And add some noise to it sim = data + 0.1 * np.random.rand(10,100,100) # and define a simple mask that needs to be broadcast - mask = (np.random.rand(100,100) > 0.1).astype(np.uint8) + mask = (np.random.rand(100,100) > 0.1).astype(np.bool) # First, test without a mask