Merge code

.gitea/workflows/action.yaml

@@ -1,24 +1,17 @@
 name: Build and Publish
+
 on:
   push:
     tags:
       - '*'
     branches:
       - master
-      - deploy
-      - 'feature/**'
-      - gitea-pages
   pull_request:
     branches:
       - master
-      - deploy
   pull_request_target:
     types: [closed]
 
-env:
-  DEPLOY_BRANCH: gitea-pages
-  PREVIEWS_DIR: previews
-
 jobs:
   build:
     runs-on: ubuntu-latest
@@ -35,24 +28,8 @@ jobs:
 
       - name: Build the common wheel
         run: |
-          source venv/bin/activate
-          cd common/
-          python -m build
-          mv dist/* ../dist
-
-      - name: Build the GUI wheel
-        run: |
-          source venv/bin/activate
-          cd gui/
-          python -m build
-          mv dist/* ../dist
-
-      - name: Build the DAQ wheel
-        run: |
-          source venv/bin/activate
-          cd daq/
-          python -m build
-          mv dist/* ../dist
+          source venv/bin/activate          
+          python -m build          
 
       - name: Upload Package to Gitea PyPI
         if: github.ref == 'refs/heads/main' && contains(github.event.head_commit.message, 'build and publish')
@@ -63,146 +40,3 @@ jobs:
           source venv/bin/activate
           twine upload --repository-url https://gitea.psi.ch/api/packages/mx/pypi \
           dist/*
-  
-  docs:
-    name: Build and Deploy Docs
-    runs-on: ubuntu-latest
-    needs: build
-    steps:
-      - name: Checkout Repository
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0  # needed for pushing branches/tags
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.12'
-
-      - name: Build Sphinx HTML
-        working-directory: daq/docs
-        run: |
-          # ensure on master
-          git fetch origin master:master || true
-          git checkout master || true
-        
-          # create venv one level up and activate it
-          python3 -m venv ../venv
-          source ../venv/bin/activate
-        
-          # ensure pip and Sphinx are installed in this venv
-          pip install --upgrade pip
-          pip install "Sphinx==8.2.3"
-          pip install myst-parser
-          pip install sphinx_immaterial
-          pip install linkify-it-py
-          
-          if [ -f requirements.txt ]; then
-            pip install -r requirements.txt
-          elif [ -f ../docs/requirements.txt ]; then
-            pip install -r ../docs/requirements.txt
-          else
-          # fallback to known packages if requirements file is not present
-            pip install "Sphinx==8.2.3" myst-parser sphinx_immaterial linkify-it-py
-          fi
-        
-        
-          # install project editable without deps (avoid private packages)
-          pip install -e ../ --no-deps
-          
-          # prefer the sphinx-apidoc CLI if available; otherwise try module fallback
-          if command -v sphinx-apidoc > /dev/null 2>&1; then
-            sphinx-apidoc -o modules/ ../src --separate --module-first --force --remove-old
-          else
-            # module fallback: some Sphinx installs expose the module at sphinx.ext.apidoc
-            python -m sphinx.ext.apidoc -o modules/ ../src --separate --module-first --force --remove-old
-          fi
-          
-        
-          # Build HTML (source is '.' because working-directory is docs)
-          # Run sphinx-build and capture exit code and full output to help debugging if it fails.
-          sphinx-build -b html . _build/html || {
-            echo "=== Sphinx build failed with exit $?: showing build output and tree ==="
-            echo "Contents of docs/ after build attempt:"
-            ls -la
-            echo "Contents of daq/docs/_build (if any):"
-            ls -la _build || true
-            # If sphinx produced a build log file, print it
-            if [ -f sphinx-build.log ]; then
-              echo "---- sphinx-build.log ----"
-              cat sphinx-build.log
-            fi
-            # Exit with non-zero to fail the job (avoid deploying empty site)
-            exit 1
-          }
-
-          # Sanity check: ensure HTML output exists before continuing to deploy
-          if [ ! -d _build/html ] || [ -z "$(ls -A _build/html)" ]; then
-            echo "ERROR: daq/docs/_build/html does not exist or is empty after successful sphinx-build."
-            ls -la _build || true
-            exit 1
-          fi
-
-      - name: Deploy to gitea-pages branch (Gitea)
-        env:
-          GIT_USER: "Martin Appleby (Gitea)"
-          GIT_EMAIL: "martin.appleby@psi.ch"
-          DEPLOY_TOKEN: ${{ secrets.DOCS_DEPLOY_TOKEN }}
-        run: |
-          set -e
-          git config --global user.name "$GIT_USER"
-          git config --global user.email "$GIT_EMAIL"
-          
-          # Deploy strategy: create/update branch gitea-pages and replace its contents
-          # with the generated daq/docs/_build/html. This keeps only the documentation
-          # on the gitea-pages branch.
-          # Confirm build output exists (relative to repository root)
-          BUILD_DIR="daq/docs/_build/html"
-          echo "Checking build output at: $BUILD_DIR"
-          if [ ! -d "$BUILD_DIR" ] || [ -z "$(ls -A "$BUILD_DIR")" ]; then
-            echo "ERROR: $BUILD_DIR does not exist or is empty"
-            echo "Listing docs/:"
-            ls -la docs || true
-            echo "Listing daq/docs/_build:"
-            ls -la daq/docs/_build || true
-            exit 1
-          fi
-          # Work in a temporary clone to avoid touching the runner workspace
-          TMPDIR=$(mktemp -d)
-          REPO_URL="https://$DEPLOY_TOKEN@gitea.psi.ch/${{ github.repository }}"
-          git clone "$REPO_URL" "$TMPDIR"
-          cd "$TMPDIR"
-
-          # Create or switch to orphan branch gitea-pages
-          if git rev-parse --verify --quiet gitea-pages >/dev/null; then
-            git checkout gitea-pages
-            # Make sure working tree matches branch (hard reset)
-            git fetch --prune origin gitea-pages || true
-            git reset --hard origin/gitea-pages || git reset --hard
-          else
-            git checkout --orphan gitea-pages
-          fi
-          # Remove all files tracked in the branch (leave .git)
-          git rm -rf . || true
-          # Also remove untracked files to ensure clean branch state
-          git clean -fdx || true
-
-          # Copy built HTML into the clone root
-          echo "Copying built HTML into branch root..."
-          (cd "${GITHUB_WORKSPACE}/${BUILD_DIR}" && tar -cf - .) | tar -xf - -C "$TMPDIR"
-
-          # Ensure index.html exists in destination as sanity check
-          if [ ! -f "$TMPDIR/index.html" ] && [ ! -f "$TMPDIR/index.htm" ]; then
-            # If site root index is not present, try to check subfolder
-            echo "Warning: no index.html found at branch root after copy. Listing files:"
-            ls -la "$TMPDIR" || true
-          fi
-         
-          git add --all
-          if git diff --quiet --cached; then
-            echo "No changes to deploy"
-            exit 0
-          fi
-          COMMIT_DATE="$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
-          git commit -m "docs: (${COMMIT_DATE}) | auto-update documentation from ${GITHUB_SHA}"
-          git push --force-with-lease origin gitea-pages

CHANGELOG.md

@@ -1,5 +1,6 @@
 # Changelog
-## 0.2.70
+## 0.3.0
+- Merge three repos into one, remove Ultralytics as dependency
 
 ### 0.2.69
 - check beamline state before running raster, rotation and automation

common/pyproject.toml

@@ -1,25 +0,0 @@
-[project]
-name = "aaredaqlib"
-version = "0.2.69"
-description = "Libraries shared between AareDAQ and AareGUI"
-readme = "README.md"
-requires-python = ">=3.11"
-dependencies = [
-    "pydantic==2.11.4",
-    "numpy==2.2.5",
-    "jfjoch_client==1.0.0rc122",
-]
-
-[lint]
-ignore = ["F401", "F541", "W503", "W504"]
-
-[tool.uv.sources]
-aaredb = { index = "psi"}
-
-[[tool.uv.index]]
-name = "psi"
-url = "https://gitea.psi.ch/api/packages/mx/pypi/simple"
-
-[build-system]
-requires = ["setuptools>=75.6.0"]
-build-backend = "setuptools.build_meta"

daq/docs/conf.py

@@ -1,149 +0,0 @@
-# Configuration file for the Sphinx documentation builder.
-#
-# For the full list of built-in configuration values, see the documentation:
-# https://www.sphinx-doc.org/en/master/usage/configuration.html
-
-# -- Project information -----------------------------------------------------
-# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
-import os
-import sys
-
-# Add project root to Python path so autodoc can import modules
-sys.path.insert(0, os.path.abspath('..'))
-sys.path.insert(0, os.path.abspath('../src'))  # <-- ensure 'src' is on path
-sys.path.insert(0, os.path.abspath('../src/aaredaq'))
-sys.path.insert(0, os.path.abspath('../src/mxlibs3'))
-sys.path.insert(0, os.path.abspath('../../common'))
-sys.path.insert(0, os.path.abspath('../../common/src'))
-sys.path.insert(0, os.path.abspath('../../common/src/aaredaqlib'))
-sys.path.insert(0, os.path.abspath('../../gui'))
-sys.path.insert(0, os.path.abspath('../../gui/src'))
-sys.path.insert(0, os.path.abspath('../../gui/src/aaregui'))
-
-project = 'AareDAQ'
-copyright = '2025, Paul Scherrer Institute'
-author = 'M. Vears Appleby'
-release = '0.2.70'
-master_doc = 'index'
-root_doc = 'index'
-# -- General configuration ---------------------------------------------------
-# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
-
-autodoc_mock_imports = [
-    'cv2',
-    'redis',
-    'jfjoch_client',
-    'epics',
-    'sseclient',
-    'jwt',
-    'aaredb',
-    'aareDBclient',
-    'numpy',
-    'requests',
-    'PyYAML',
-    'loguru',
-    'utility_tools',
-    'extract_results',
-    'testxds',
-    'db_slurm_handler',
-    'websocket',
-    'dateutil',
-    'PySide6',
-    'scipy',
-    'redis_lock',
-    'fastapi',
-    'uvicorn',
-    'pydantic'
-]
-
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
-
-extensions = [
-    'myst_parser',
-    'sphinx_immaterial',
-    'sphinx.ext.autodoc',      # Add this
-    'sphinx.ext.viewcode',     # Optional: source code links
-    'sphinx.ext.napoleon'      # Optional: Google/NumPy docstring support
-]
-
-
-templates_path = ['_templates']
-exclude_patterns = []
-myst_enable_extensions = ['linkify', 'smartquotes']
-
-
-myst_heading_anchor = 3
-
-# -- Options for HTML output -------------------------------------------------
-# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
-
-html_theme = 'sphinx_immaterial'
-# html_static_path = ['../broker/redoc-static.html']
-#html_theme_options = {
-#    'sidebar_width': '300px',
-#    'page_width': 'auto'
-#}
-
-# html_theme_options = {
-#     'repo_url': 'https://gitea.psi.ch/mx/aaredaq',
-#     'repo_name': 'AareDaq',
-#     'nav_title': 'PSI AareDaq',
-#     # 'html_minify': True,
-#     # 'css_minify': True,
-#     'globaltoc_depth': 2,
-#     # 'repo_type': 'gitlab',
-#     'color_primary': 'indigo',
-#     'color_accent': 'lime',
-#     # 'logo_icon': '&#xe30d'
-# }
-
-# html_logo = 'jfjoch.png'
-# html_favicon = 'jfjoch.png'
-html_theme_options = {
-    "icon": {
-        "repo": "fontawesome/brands/github", # or gitlab/gitea icon
-    },
-    "site_url": "https://gitea.psi.ch/mx/aaredaq",
-    "repo_url": "https://gitea.psi.ch/mx/aaredaq",
-    "repo_name": "AareDAQ",
-    "palette": [
-        {
-            "media": "(prefers-color-scheme: light)",
-            "scheme": "default",
-            "primary": "indigo",
-            "accent": "lime",
-            "toggle": {
-                "icon": "material/lightbulb-outline",
-                "name": "Switch to dark mode",
-            },
-        },
-        {
-            "media": "(prefers-color-scheme: dark)",
-            "scheme": "slate",
-            "primary": "indigo",
-            "accent": "lime",
-            "toggle": {
-                "icon": "material/lightbulb",
-                "name": "Switch to light mode",
-            },
-        },
-    ],
-    "features": [
-        "navigation.expand",
-        "navigation.sections",
-        "navigation.top",
-        "search.share",
-        "toc.follow",
-    ],
-    "globaltoc_collapse": False,
-}
-html_theme_options["google_fonts"] = []
-html_sidebars = {
-    "**": ["logo-text.html", "globaltoc.html", "searchbox.html"]
-}
-
-
-html_show_sourcelink = False
-
-# language = "Python"
-

daq/docs/index.rst

@@ -1,30 +0,0 @@
-.. AareProc documentation master file, created by
-   sphinx-quickstart on Sun Dec 21 14:22:05 2025.
-   You can adapt this file completely to your liking, but it should at least
-   contain the root `toctree` directive.
-
-Welcome to AareDAQ's documentation!
-====================================
-
-.. toctree::
-   :maxdepth: 2
-   :caption: Contents:
-   :glob:
-
-   README
-   CHANGELOG
-
-.. toctree::
-   :maxdepth: 4
-   :caption: OpenAPI Python client
-
-   modules/src.aaredaq
-   modules/src.mxlibs3
-
-Indices and tables
-==================
-
-* :ref:`genindex`
-* :ref:`modindex`
-* :ref:`search`
-

daq/docs/requirements.txt

@@ -1,37 +0,0 @@
-alabaster==1.0.0
-babel==2.16.0
-beautifulsoup4==4.12.3
-certifi==2024.8.30
-charset-normalizer==3.4.0
-css-html-js-minify==2.5.5
-docutils==0.21.2
-idna==3.10
-imagesize==1.4.1
-Jinja2==3.1.4
-linkify-it-py==2.0.3
-lxml==5.3.0
-markdown-it-py==3.0.0
-MarkupSafe==3.0.2
-mdit-py-plugins==0.4.2
-mdurl==0.1.2
-myst-parser==4.0.0
-packaging==24.2
-Pygments==2.18.0
-python-slugify==8.0.4
-PyYAML==6.0.2
-requests==2.32.3
-snowballstemmer==2.2.0
-soupsieve==2.6
-Sphinx==8.1.3
-sphinx-material==0.0.36
-sphinxcontrib-applehelp==2.0.0
-sphinxcontrib-devhelp==2.0.0
-sphinxcontrib-htmlhelp==2.1.0
-sphinxcontrib-jsmath==1.0.1
-sphinxcontrib-qthelp==2.0.0
-sphinxcontrib-serializinghtml==2.0.0
-text-unidecode==1.3
-uc-micro-py==1.0.3
-Unidecode==1.3.8
-urllib3==2.2.3
-sphinx_immaterial==0.13.8

gui/pyproject.toml

@@ -1,28 +0,0 @@
-[project]
-name = "aaregui"
-version = "0.2.69"
-description = "Beamline control GUI"
-readme = "README.md"
-requires-python = ">=3.11"
-dependencies = [
-    "pyjwt==2.10.1",
-    "pyzmq==26.4.0",
-    "opencv-python-headless==4.11.0.86",
-    "PySide6==6.9.0",
-    "aaredaqlib==0.2.69"
-]
-
-[lint]
-ignore = ["F401", "F541", "W503", "W504"]
-
-[build-system]
-requires = ["setuptools>=75.6.0"]
-build-backend = "setuptools.build_meta"
-
-[tool.uv.sources]
-aaredb = { index = "psi"}
-
-[[tool.uv.index]]
-name = "psi"
-url = "https://gitea.psi.ch/api/packages/mx/pypi/simple"
-

pyproject.toml

@@ -1,15 +1,21 @@
 [project]
 name = "aaredaq"
-version = "0.2.69"
-description = "AareDAQ data acquisition server"
+version = "0.3.0"
+description = "AareDAQ (with GUI)"
 readme = "README.md"
 requires-python = ">=3.11"
 dependencies = [
+    "pydantic==2.11.4",
+    "numpy==2.2.5",
+    "jfjoch_client==1.0.0rc122",
+    "pyJWT==2.10.1",
+    "pyzmq==26.4.0",
+    "opencv-python-headless==4.11.0.86",
+    "PySide6==6.9.0",
     "requests==2.32.4",
     "pyepics==3.5.8",
     "redis==6.2.0",
     "python-redis-lock==4.0.0",
-    "PyJWT==2.10.1",
     "fastapi==0.115.13",
     "uvicorn==0.34.2",
     "aaredb==0.1.1a33",
@@ -17,7 +23,6 @@ dependencies = [
     "python_multipart==0.0.20",
     "websocket-client==1.8.0",
     "sseclient-py==1.8.0",
-    "aaredaqlib==0.2.69"
 ]
 
 [lint]

scripts/knife_edge_scan.ipynb

@@ -16,7 +16,7 @@
    "cell_type": "code",
    "source": [
     "from aaredaq.devices import BeamlineDevices\n",
-    "from aaredaqlib.beamline import MXBeamline"
+    "from aaredaqlib import MXBeamline"
    ],
    "id": "d2efa08de2dbd46b",
    "outputs": [],

src/aaredaq/daq.py

@@ -1,5 +1,4 @@
 import copy
-import json
 import secrets
 import time
 from datetime import datetime
@@ -22,7 +21,7 @@ from aaredaqlib.beamline import MXBeamline
 from aaredaqlib.coordinate import Coordinate, SmargonCoordinate
 from aaredaqlib.diffraction_geometry import DiffractionGeometry
 from aaredaqlib.find_xtal import raster_centre_of_mass, create_quality_filtered_array, identify_crystal_raster, \
-    get_result_list_from_com, get_best_b_factor, get_best_res, get_xtal_size, has_sufficient_low_res_spots
+    get_result_list_from_com, get_best_b_factor, get_best_res, get_xtal_size
 from aaredaqlib.logger_config import setup_logger
 from aaredaqlib.models import (
     SampleShortInfo,

src/aaredaq/server.py

@@ -1,7 +1,7 @@
 import asyncio
 import io
 import os, time
-from typing import Tuple, AsyncGenerator
+from typing import AsyncGenerator
 import json
 import cv2
 import urllib3
@@ -20,7 +20,6 @@ from fastapi import HTTPException
 from fastapi import status as api_status
 from fastapi.security import OAuth2PasswordBearer, OAuth2PasswordRequestForm
 from starlette.responses import StreamingResponse
-from urllib3.exceptions import InsecureRequestWarning
 
 from aaredaq import auth
 from aaredaqlib.beamline import mx_beamline