Update installation instructions for simplicity and also fix the tests to reflect new usage

CDTools/datasets/__init__.py

@@ -32,8 +32,11 @@ import numpy as np
 import torch as t
 from copy import copy
 import h5py
-import pathlib
-
+try:
+    import pathlib
+except ImportError:
+    import pathlib2 as pathlib
+    
 from CDTools.tools import data as cdtdata
 from CDTools.tools import plotting
 from torch.utils import data as torchdata
@@ -98,7 +101,10 @@ class CDataset(torchdata.Dataset):
         self.wavelength = wavelength
         self.detector_geometry = copy(detector_geometry)
         if mask is not None:
-            self.mask = t.tensor(mask)
+            if isinstance(mask, t.Tensor):
+                self.mask = mask.detach().to(dtype=t.bool)
+            else:
+                self.mask = t.BoolTensor(mask)
         else:
             self.mask = None
         if background is not None:

CDTools/datasets/ptycho_2d_dataset.py

@@ -3,7 +3,10 @@ import numpy as np
 import torch as t
 from copy import copy
 import h5py
-import pathlib
+try:
+    import pathlib
+except ImportError:
+    import pathlib2 as pathlib
 
 from CDTools.datasets import CDataset
 from CDTools.tools import data as cdtdata

CDTools/tools/losses.py

@@ -21,7 +21,11 @@ def amplitude_mse(intensities, sim_intensities, mask=None):
     This function calculates the mean squared error between their
     associated amplitudes. Because this is not well defined for negative
     numbers, make sure that all the intensities are >0 before using this
-    loss.
+    loss. Note that this is actually a sum-squared error, because this
+    formulation makes it vastly simpler to compare error calculations
+    between reconstructions with different minibatch size. I hope to
+    find a better way to do this that is more honest with this
+    cost function, though.
 
     It can accept intensity and simulated intensity tensors of any shape
     as long as their shapes match, and the provided mask array can be

conda_requirements.txt

@@ -0,0 +1,8 @@
+numpy>=1.0
+scipy>=1.0
+matplotlib>=2.0
+python-dateutil
+pytorch>=1.2.0
+h5py>=2.1
+pytest
+sphinx

docs/source/installation.rst

@@ -13,6 +13,18 @@ It is recommended that you clone the repository, rather than just downloading th
 Step 2: Install Dependencies
 ----------------------------
 
+The dependencies for CDTools can be installed, if you are managing your environment with anaconda, by running
+
+.. code:: bash
+	  
+   $ conda install --file conda_requirements.txt
+
+There are two optional dependencies which are not installed via this procedure - the dependency sphinx-argparse for building the docs, and the pathlib2 module that provides python 2 compatibility. These can either be installed manually via conda-forge, or otherwise they will be installed automatically by pip during the final installation step if needed.
+
+If you manage your environment with pip, all required packges should be installed automatically. The only thing to be aware of is that pytorch must be compiled with MKL support, and CUDA support if you would like to use the GPU. For this reason, using anaconda python is strongly recommended.
+
+For convenience, the full set of dependencies are noted below:
+   
 CDTools depends on the following packages:
 
    * `numpy <http://www.numpy.org>`_
@@ -27,6 +39,7 @@ And has optional dependencies on
    * `pytest <https://docs.pytest.org/>`_
    * `sphinx <https://www.sphinx-doc.org/>`_
    * `sphinx-argparse <https://sphinx-argparse.readthedocs.io>`_
+   * `pathlib2 <https://pypi.org/project/pathlib2/>`_
      
 All of these can be installed via pip or conda. Finally, CDTools is written to be python 2.7+ compatible, but is only actively tested on python 3.
 
@@ -42,7 +55,9 @@ To install in CDTools in developer mode (recommended, to allow any updates to be
 
 .. code:: bash
 	  
-   $ pip install -e .
+   $ pip install -e .[tests,docs]
+
+If you don't need to run the tests, or don't need to build the docs, you can omit the relevant option or options.
 
 If you prefer to use a tool other than pip, CDTools can be installed via any other package management tool that works with a setup.py file.
 

setup.py

@@ -13,12 +13,18 @@ setuptools.setup(
     long_description_content_type="text/markdown",
     url="https://github.mit.edu/scattering/CDTools.git",
     install_requires=[
-        "numpy",
-        "scipy",
-        "matplotlib",
+        "numpy>=1.0",
+        "scipy>=1.0",
+        "matplotlib>=2.0",
         "python-dateutil",
-        "torch",
-        "h5py"],
+        "torch>=1.2.0", #1.2.0 introduced boolean tensors in a breaking way, we use the boolean tensors here for masking
+        "h5py>=2.1",
+        "pathlib2 ; python_version<'3.4'"],
+    extras_require={
+        'tests': ["pytest"],
+        'docs': ["sphinx","sphinx-argparse"],
+        ":python_version<'3.4'": ["pathlib2"],
+    },
     packages=setuptools.find_packages(),
     classifiers=[
         "Programming Language :: Python :: 3",

tests/conftest.py

@@ -103,8 +103,8 @@ def ptycho_cxi_1():
 
     # Remember the format for the CXI file differs from the format used
     # internally
-    mask = np.zeros((256,256)).astype(np.uint32)
-    expected['mask'] = np.ones((256,256)).astype(np.uint8)
+    mask = np.zeros((256,256)).astype(np.int32)
+    expected['mask'] = np.ones((256,256)).astype(np.bool)
     d1f.create_dataset('mask',data=mask)
 
     # Create an initial background
@@ -272,7 +272,7 @@ def ptycho_cxi_3():
     # Remember the format for the CXI file differs from the format used
     # internally
     mask = np.ones((256,256)).astype(np.uint32) * 0x00001000
-    expected['mask'] = np.ones((256,256)).astype(np.uint8)
+    expected['mask'] = np.ones((256,256)).astype(np.bool)
     d1f.create_dataset('mask',data=mask)
     expected['dark'] = None
     

tests/test_datasets.py

@@ -27,8 +27,8 @@ def test_CDataset_init():
     mask = np.ones((256,256))
     dataset = CDataset(entry_info, sample_info,
                        wavelength, detector_geometry, mask)
-    
-    assert t.all(t.eq(dataset.mask,t.tensor(mask)))
+
+    assert t.all(t.eq(dataset.mask,t.tensor(mask.astype(np.bool))))
     assert dataset.entry_info == entry_info
     assert dataset.sample_info == sample_info
     assert dataset.wavelength == wavelength
@@ -110,7 +110,7 @@ def test_CDataset_to(ptycho_cxi_1):
     dataset = CDataset.from_cxi(ptycho_cxi_1[0])
 
     dataset.to(dtype=t.float32)
-    assert dataset.mask.dtype == t.uint8
+    assert dataset.mask.dtype == t.bool
     # If cuda is available, check that moving the mask to CUDA works.
     if t.cuda.is_available():
         dataset.to(device='cuda:0')
@@ -147,7 +147,7 @@ def test_Ptycho2DDataset_init():
                                 detector_geometry=detector_geometry,
                                 mask=mask)
 
-    assert t.all(t.eq(dataset.mask,t.tensor(mask)))
+    assert t.all(t.eq(dataset.mask,t.BoolTensor(mask)))
     assert dataset.entry_info == entry_info
     assert dataset.sample_info == sample_info
     assert dataset.wavelength == wavelength
@@ -241,7 +241,7 @@ def test_Ptycho2DDataset_to(ptycho_cxi_1):
     dataset = Ptycho2DDataset.from_cxi(ptycho_cxi_1[0])
     
     dataset.to(dtype=t.float64)
-    assert dataset.mask.dtype == t.uint8
+    assert dataset.mask.dtype == t.bool
     assert dataset.patterns.dtype == t.float64
     assert dataset.translations.dtype == t.float64
     # If cuda is available, check that moving the mask to CUDA works.

tests/tools/test_data.py

@@ -8,7 +8,10 @@ import pytest
 import os
 import datetime
 import numbers
-from pathlib import Path
+try:
+    import pathlib
+except ImportError:
+    import pathlib2 as pathlib
 
 
 

tests/tools/test_losses.py

@@ -16,17 +16,17 @@ def test_amplitude_mse():
     # And add some noise to it
     sim = data + 0.1 * np.random.rand(10,100,100)
     # and define a simple mask that needs to be broadcast
-    mask = (np.random.rand(100,100) > 0.1).astype(np.uint8)
+    mask = (np.random.rand(100,100) > 0.1).astype(np.bool)
 
     # First, test without a mask
     np_result = np.sum((np.sqrt(data) - np.sqrt(sim))**2)
-    np_result /=  data.size
+    #np_result /=  data.size
     torch_result = losses.amplitude_mse(t.from_numpy(data),t.from_numpy(sim))
     assert np.isclose(np_result, np.take(torch_result.numpy(),0))
 
     # Then, test with a mask
     np_result = np.sum(mask * (np.sqrt(data) - np.sqrt(sim))**2)
-    np_result /=  np.count_nonzero(mask * np.ones_like(data))
+    #np_result /=  np.count_nonzero(mask * np.ones_like(data))
     torch_result = losses.amplitude_mse(t.from_numpy(data),t.from_numpy(sim),
                          mask = t.from_numpy(mask))
     assert np.isclose(np_result, np.take(torch_result.numpy(),0))
@@ -38,7 +38,7 @@ def test_intensity_mse():
     # And add some noise to it
     sim = data + 0.1 * np.random.rand(10,100,100)
     # and define a simple mask that needs to be broadcast
-    mask = (np.random.rand(100,100) > 0.1).astype(np.uint8)
+    mask = (np.random.rand(100,100) > 0.1).astype(np.bool)
 
 
     # First, test without a mask
@@ -61,7 +61,7 @@ def test_poisson_ml():
     # And add some noise to it
     sim = data + 0.1 * np.random.rand(10,100,100)
     # and define a simple mask that needs to be broadcast
-    mask = (np.random.rand(100,100) > 0.1).astype(np.uint8)
+    mask = (np.random.rand(100,100) > 0.1).astype(np.bool)
 
 
     # First, test without a mask