From 9159afa987ed6b38a6517e71b4cd38c970beba53 Mon Sep 17 00:00:00 2001 From: gnzng Date: Tue, 25 Feb 2025 14:27:15 -0800 Subject: [PATCH 1/9] add github workflow using pip installation for python 3.8 - 3.12 --- .github/workflows/main.yml | 32 ++++++++++++++++++++++++++++++++ constraints.txt | 11 +++++++++++ requirements.txt | 2 +- 3 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/main.yml create mode 100644 constraints.txt diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..6502b1d --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,32 @@ +name: main + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + continue-on-error: true + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + pip install --upgrade pip + pip install -r requirements.txt + pip install -e . --no-deps + + - name: Run tests + run: pytest diff --git a/constraints.txt b/constraints.txt new file mode 100644 index 0000000..6afb860 --- /dev/null +++ b/constraints.txt @@ -0,0 +1,11 @@ +numpy>=1.0 +scipy>=1.0 +matplotlib>=2.0 # 2.0 introduces better colormaps which are used by default +pytorch>=1.9.0 #1.9.0 implements support for autograd on indexed complex tensors +h5py>=2.1 +python-dateutil +pytest +pooch +sphinx>=4.3.0 # Fixes a bug with bulleted lists +sphinx-argparse +sphinx_rtd_theme>=0.5.1 # Fixes a bug with bulleted lists diff --git a/requirements.txt b/requirements.txt index 6afb860..1c584c8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ numpy>=1.0 scipy>=1.0 matplotlib>=2.0 # 2.0 introduces better colormaps which are used by default -pytorch>=1.9.0 #1.9.0 implements support for autograd on indexed complex tensors +torch>=1.9.0 #1.9.0 implements support for autograd on indexed complex tensors h5py>=2.1 python-dateutil pytest From 5f30990e55499a3dc3b227f08d6eb2d4b0f1f163 Mon Sep 17 00:00:00 2001 From: Damian Guenzing <65827185+gnzng@users.noreply.github.com> Date: Tue, 25 Feb 2025 17:07:30 -0800 Subject: [PATCH 2/9] Delete constraints.txt constraints not needed anymore, relict from rebasing --- constraints.txt | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 constraints.txt diff --git a/constraints.txt b/constraints.txt deleted file mode 100644 index 6afb860..0000000 --- a/constraints.txt +++ /dev/null @@ -1,11 +0,0 @@ -numpy>=1.0 -scipy>=1.0 -matplotlib>=2.0 # 2.0 introduces better colormaps which are used by default -pytorch>=1.9.0 #1.9.0 implements support for autograd on indexed complex tensors -h5py>=2.1 -python-dateutil -pytest -pooch -sphinx>=4.3.0 # Fixes a bug with bulleted lists -sphinx-argparse -sphinx_rtd_theme>=0.5.1 # Fixes a bug with bulleted lists From a7904491bea254f3628a6027e0a9fb7dc52b702b Mon Sep 17 00:00:00 2001 From: Dayne Yoshiki Sasaki <37006268+yoshikisd@users.noreply.github.com> Date: Tue, 25 Feb 2025 21:52:04 -0800 Subject: [PATCH 3/9] Added a function to allow for deletion of translation positions with logical indexing (#16) * Added a function to allow for deletion of translation positions via logical indexing In ptycho_2d_dataset.py - Created remove_translations_mask. This handles logical-indexing-based deletion of translation points - Refactored crop_translations to be dependent on remove_translations_mask * Add validation and tests for remove_translations_mask method in Ptycho2DDataset * Switch mask to mask_remove --------- Co-authored-by: gnzng Co-authored-by: Clemens Schmid --- src/cdtools/datasets/ptycho_2d_dataset.py | 43 ++++++++++++++++++----- tests/test_datasets.py | 28 +++++++++++++-- 2 files changed, 61 insertions(+), 10 deletions(-) diff --git a/src/cdtools/datasets/ptycho_2d_dataset.py b/src/cdtools/datasets/ptycho_2d_dataset.py index bdf4e60..8b6f6d8 100644 --- a/src/cdtools/datasets/ptycho_2d_dataset.py +++ b/src/cdtools/datasets/ptycho_2d_dataset.py @@ -400,9 +400,40 @@ class Ptycho2DDataset(CDataset): divisor_override=1)[0,0] + def remove_translations_mask(self, mask_remove): + """Removes one or more translation positions, and their associated + properties, from the dataset using logical indexing. + + This takes a 1D mask (boolean torch tensor) with the length + self.translations.shape[0] (i.e., the number of individual + translated points). Patterns, translations, and intensities + associated with indices that are "True" will be removed. + + Parameters: + ---------- + mask_remove : 1D torch.tensor(dtype=torch.bool) + The boolean mask indicating which elements are to be removed from + the dataset. True indicates that the corresponding element will be + removed. + """ + + # Check that the mask is the right size + if mask_remove.shape != t.Size([self.translations.shape[0]]): + raise ValueError( + 'The mask must have the same length as the number of translations in the dataset.' + ) + + # Update patterns, translations, and intensities + self.patterns = self.patterns[~mask_remove] + self.translations = self.translations[~mask_remove] + + if hasattr(self, 'intensities') and self.intensities is not None: + self.intensities = self.intensities[~mask_remove] + + def crop_translations(self, roi): """Shrinks the range of translation positions that are analyzed - + This deletes all diffraction patterns associated with x- and y-translations that lie outside of a specified rectangular region of interest. In essence, this operation crops the "relative @@ -420,7 +451,7 @@ class Ptycho2DDataset(CDataset): do not matter as long as roi[:2] and roi[2:] correspond with the x and y coordinates, respectively. """ - + # Pull out the bounds of the ROI, ensuring that left < right and # top < bottom x_left, x_right = sorted(roi[:2]) @@ -441,9 +472,5 @@ class Ptycho2DDataset(CDataset): '(i.e., patterns and translations will be empty).' ' Please redefine the bounds of the roi.') - # Update patterns and translations - self.patterns = self.patterns[inside_roi] - self.translations = self.translations[inside_roi] - - if hasattr(self, 'intensities') and self.intensities is not None: - self.intensities = self.intensities[inside_roi] \ No newline at end of file + # Remove translations outside the ROI + self.remove_translations_mask(~inside_roi) diff --git a/tests/test_datasets.py b/tests/test_datasets.py index 9e72fb7..6897991 100644 --- a/tests/test_datasets.py +++ b/tests/test_datasets.py @@ -1,4 +1,4 @@ -from cdtools.datasets import * +from cdtools.datasets import CDataset, Ptycho2DDataset from cdtools.tools import data as cdtdata import numpy as np import torch as t @@ -340,7 +340,31 @@ def test_Ptycho2DDataset_downsample(test_ptycho_cxis): if dataset.background is not None: assert np.allclose(np.array(dataset.background.shape) // factor, np.array(copied_dataset.background.shape)) - + + +def test_Ptycho2DDataset_remove_translations_mask(ptycho_cxi_1): + # Grab dataset + cxi, expected = ptycho_cxi_1 + dataset = Ptycho2DDataset.from_cxi(cxi) + copied_dataset = deepcopy(dataset) + + # Test 1: Complain when the the mask is not the same shape as the pattern + # length + with pytest.raises(ValueError) as excinfo: + copied_dataset.remove_translations_mask(mask_remove=t.zeros(10)) + assert ('The mask must have the same length') in str(excinfo.value) + + # Test 2: Remove the mask from the dataset + mask_success = t.zeros(len(copied_dataset.patterns)) + mask_success[1] = 1 + mask_success[10] = 1 + mask_success[-1] = 1 + mask_success = mask_success.bool() + copied_dataset.remove_translations_mask(mask_remove=mask_success) + + # test if the mask is removed and patterns length is correct + assert len(copied_dataset.patterns) == len(mask_success) - 3 + def test_Ptycho2DDataset_crop_translations(ptycho_cxi_1): # Grab dataset From b39cd2b7055139158bc09b81987b0da9f55ae491 Mon Sep 17 00:00:00 2001 From: Damian Guenzing <65827185+gnzng@users.noreply.github.com> Date: Wed, 26 Feb 2025 15:24:29 +0000 Subject: [PATCH 4/9] updated installation.rst for pip installation --- docs/source/installation.rst | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/source/installation.rst b/docs/source/installation.rst index a92a322..1b3f2ba 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -13,20 +13,22 @@ The repository remains under active development as of late 2024. Step 2: Install Dependencies ---------------------------- -CDTools requires python 3.7 or greater. +CDTools is regularly tested with Python versions 3.8 to 3.12, so it is recommended to use one of these versions. In general, CDTools requires Python 3.7 or higher. The major dependency for CDTools is pytorch (version 1.9.0 or greater). Because the details of the installation can vary depending on platform, GPU availability, etc, it is recommended that you follow the install instructions on `the pytorch site`_ to install pytorch before installing the remaining dependencies. .. _`the pytorch site`: https://pytorch.org/get-started/locally/ -If you manage your environment with conda, the remaining dependencies can be installed by running the following command in the top level directory of the package: +pytorch stopped supporting installation using conda for installation, so it is recommended continue the installation using pip. .. code:: bash - $ conda install --file requirements.txt -c conda-forge + $ pip install -r requirements.txt + +This will install all required dependencies and verify that they meet the pytorch version requirements. Additionally, several optional dependencies used for testing and documentation will also be installed. The full set of dependencies and minimum requirements are listed below is listed below. + +CDTools is reguarly tested with the latest versions of the packages shown below. -This will install all required dependencies *except for pytorch*, as well as several optional dependencies which are used for the tests and documentation. The full set of dependencies are noted below. - Required dependencies: * `numpy `_ >= 1.0 From a097dd50d22c20098915074f375f0bae820a7dd8 Mon Sep 17 00:00:00 2001 From: gnzng Date: Sun, 2 Mar 2025 13:05:39 -0800 Subject: [PATCH 5/9] Update .gitignore to include .DS_Store files --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index dcb9f36..b126f78 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ docs/build build/* dist */example_data/* -*.h5 \ No newline at end of file +*.h5 +.DS_Store \ No newline at end of file From e1b76f09591d368916cf2f00e53797f7a0c1e6f7 Mon Sep 17 00:00:00 2001 From: gnzng Date: Sun, 2 Mar 2025 13:07:30 -0800 Subject: [PATCH 6/9] DOCS: Force-Push due to rebasing unnecessary large commits to just add an extra doc publishing action --- .github/workflows/main.yml | 58 ++++++++++++++++++++++++++++++++++++ .gitignore | 1 + docs/source/installation.rst | 2 +- 3 files changed, 60 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6502b1d..55adfa6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -30,3 +30,61 @@ jobs: - name: Run tests run: pytest + + build-docs: + if: github.event_name == 'push' && github.ref == 'refs/heads/master' + runs-on: ubuntu-latest + permissions: + contents: read + pages: write + id-token: write + + concurrency: + group: "pages" + cancel-in-progress: false + + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.9' + + - name: Install dependencies + run: | + pip install --upgrade pip + pip install -r requirements.txt + pip install sphinx sphinx_rtd_theme sphinx-argparse + pip install -e . --no-deps + + - name: Build docs + working-directory: docs + run: | + make html + + - name: List files in docs/build/html + run: | + ls -la docs/build/html + + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + with: + path: 'docs/build/html' + name: github-pages + + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + timeout: 600000 + error_count: 10 + reporting_interval: 5000 + artifact_name: github-pages + preview: false \ No newline at end of file diff --git a/.gitignore b/.gitignore index b126f78..ea4c46c 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ *.egg-info .pytest_cache docs/build +docs/_build build/* dist */example_data/* diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 1b3f2ba..d998f93 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -8,7 +8,7 @@ The source code for CDTools is hosted on `Github`_. At the moment, the repositor .. _`Github`: https://github.com/cdtools-developers/cdtools -The repository remains under active development as of late 2024. +The repository remains under active development as of early 2025. Step 2: Install Dependencies ---------------------------- From efadaf807e769a39bb264bb306cec672e5180635 Mon Sep 17 00:00:00 2001 From: gnzng Date: Tue, 4 Mar 2025 14:20:09 -0800 Subject: [PATCH 7/9] Rename probe_size parameter to probe_shape in FancyPtycho and MultislicePtycho classes to match the gaussian_probe function --- src/cdtools/models/fancy_ptycho.py | 5 ++--- src/cdtools/models/multislice_ptycho.py | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/cdtools/models/fancy_ptycho.py b/src/cdtools/models/fancy_ptycho.py index 46e7db6..1be40c6 100644 --- a/src/cdtools/models/fancy_ptycho.py +++ b/src/cdtools/models/fancy_ptycho.py @@ -195,7 +195,7 @@ class FancyPtycho(CDIModel): @classmethod def from_dataset(cls, dataset, - probe_size=None, + probe_shape=None, randomize_ang=0, n_modes=1, n_obj_modes=1, @@ -277,7 +277,7 @@ class FancyPtycho(CDIModel): ) # Finally, initialize the probe and object using this information - if probe_size is None: + if probe_shape is None: probe = tools.initializers.SHARP_style_probe( dataset, propagation_distance=propagation_distance, @@ -288,7 +288,6 @@ class FancyPtycho(CDIModel): dataset, obj_basis, probe_shape, - probe_size, propagation_distance=propagation_distance, ) diff --git a/src/cdtools/models/multislice_ptycho.py b/src/cdtools/models/multislice_ptycho.py index 23df19a..afcd83e 100644 --- a/src/cdtools/models/multislice_ptycho.py +++ b/src/cdtools/models/multislice_ptycho.py @@ -181,7 +181,7 @@ class MultislicePtycho(CDIModel): dataset, dz, nz, - probe_size=None, + probe_shape=None, randomize_ang=0, n_modes=1, n_obj_modes=1, @@ -262,7 +262,7 @@ class MultislicePtycho(CDIModel): ) # Finally, initialize the probe and object using this information - if probe_size is None: + if probe_shape is None: probe = tools.initializers.SHARP_style_probe( dataset, propagation_distance=propagation_distance, @@ -273,7 +273,6 @@ class MultislicePtycho(CDIModel): dataset, obj_basis, probe_shape, - probe_size, propagation_distance=propagation_distance, ) From f8bebd0436aeb468a2168fc6af9a6e079f52fa59 Mon Sep 17 00:00:00 2001 From: gnzng Date: Tue, 4 Mar 2025 14:21:51 -0800 Subject: [PATCH 8/9] typo, factor -> factors --- src/cdtools/tools/initializers/initializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cdtools/tools/initializers/initializers.py b/src/cdtools/tools/initializers/initializers.py index 7e727af..bdb91b9 100644 --- a/src/cdtools/tools/initializers/initializers.py +++ b/src/cdtools/tools/initializers/initializers.py @@ -234,7 +234,7 @@ def gaussian_probe(dataset, basis, shape, sigma, propagation_distance=0, polariz polarizer = dataset.polarizer.tolist() analyzer = dataset.analyzer.tolist() factors = [(math.cos(math.radians(polarizer[idx] - analyzer[idx])))**2 for idx in range(len(dataset)) if (abs(polarizer[idx] - analyzer[idx]) > 5)] - avg_intensities = [t.sum(dataset[idx][1]) / factor[idx] for idx in range(len(dataset))] + avg_intensities = [t.sum(dataset[idx][1]) / factors[idx] for idx in range(len(dataset))] avg_intensity = t.mean(t.tensor(avg_intensities)) probe_intensity = t.sum(t.abs(probe)**2) From 48ca3a675af65083d70cb564f2d383eb7ff21271 Mon Sep 17 00:00:00 2001 From: gnzng Date: Fri, 7 Mar 2025 08:48:33 -0800 Subject: [PATCH 9/9] remove environment from github actions --- .github/workflows/main.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 55adfa6..5cc6d9b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -43,10 +43,6 @@ jobs: group: "pages" cancel-in-progress: false - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - steps: - name: Checkout uses: actions/checkout@v4