Introduce DataCollectionParameters model with validation and update dependencies

Added a new `DataCollectionParameters` model in `aaredaqlib` with extensive field validation and default handling for increased robustness. Updated `aaredaqlib`, `aaredb`, and related dependencies to version `0.2.4` for consistency across modules. Enhanced logging for better visibility during validation.

VERSION

@@ -1 +1 @@
-0.2.3
+0.2.4

common/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "aaredaqlib"
-version = "0.2.3"
+version = "0.2.4"
 description = "Libraries shared between AareDAQ and AareGUI"
 readme = "README.md"
 requires-python = ">=3.11"
@@ -8,7 +8,6 @@ dependencies = [
     "pydantic==2.11.4",
     "numpy==2.2.5",
     "jfjoch_client==1.0.0rc61",
-    "aareDB==0.1.1a22"
 ]
 
 [lint]

common/src/aaredaqlib/models.py

@@ -1,13 +1,20 @@
+import logging
+from pathlib import Path
+import re
 from enum import Enum
 from typing import Annotated, Literal, Tuple, List, Optional
 
-from aareDBclient.models.data_collection_parameters import DataCollectionParameters
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, field_validator
 
 from aaredaqlib.coordinate import Coordinate
 from aaredaqlib.diffraction_geometry import DiffractionGeometry
 from aaredaqlib.sample_geometry import SampleGeometryModel
 
+
+logging.basicConfig(level=logging.DEBUG)
+logger = logging.getLogger(__name__)
+
+
 class StagePositionEnum(Enum):
     MEASURE = 0
     PARK = 1
@@ -44,6 +51,339 @@ class PuckLoadedInfo(BaseModel):
     puck_name: str
     location: DewarAddress
 
+class DataCollectionParameters(BaseModel):
+    directory: Optional[str] = None
+    oscillation: Optional[float] = None  # Only accept positive float
+    exposure: Optional[float] = None  # Only accept positive floats between 0 and 1
+    totalrange: Optional[int] = None  # Only accept positive integers between 0 and 360
+    transmission: Optional[
+        int
+    ] = None  # Only accept positive integers between 0 and 100
+    targetresolution: Optional[float] = None  # Only accept positive float
+    aperture: Optional[str] = None  # Optional string field
+    datacollectiontype: Optional[
+        str
+    ] = None  # Only accept "standard", other types might be added later
+    processingpipeline: Optional[
+        str
+    ] = ""  # Only accept "gopy", "autoproc", "xia2dials"
+    spacegroupnumber: Optional[
+        int
+    ] = None  # Only accept positive integers between 1 and 230
+    cellparameters: Optional[
+        str
+    ] = None  # Must be a set of six positive floats or integers
+    rescutkey: Optional[str] = None  # Only accept "is" or "cchalf"
+    rescutvalue: Optional[
+        float
+    ] = None  # Must be a positive float if rescutkey is provided
+    userresolution: Optional[float] = None
+    pdbid: Optional[
+        str
+    ] = ""  # Accepts either the format of the protein data bank code or {provided}
+    autoprocfull: Optional[bool] = None
+    procfull: Optional[bool] = None
+    adpenabled: Optional[bool] = None
+    noano: Optional[bool] = None
+    ffcscampaign: Optional[bool] = None
+    trustedhigh: Optional[float] = None  # Should be a float between 0 and 2.0
+    autoprocextraparams: Optional[str] = None  # Optional string field
+    chiphiangles: Optional[float] = None  # Optional float field between 0 and 30
+    dose: Optional[float] = None  # Optional float field
+    cloud: bool = True
+    pdbmodel: Optional[str] = None
+
+    def to_dict(self):
+        """Convert the model instance to a dictionary."""
+        return self.dict(
+            exclude_unset=True
+        )  # Use this built-in method for serialization
+
+    class Config:
+        from_attributes = True
+
+    @field_validator("directory", mode="after")
+    @classmethod
+    def directory_characters(cls, v):
+        logger.debug(f"Validating 'directory' field with initial value: {repr(v)}")
+
+        # Default directory value if empty
+        if not v:  # Handles None or empty cases
+            default_value = "{sgPuck}/{sgPosition}"
+            logger.warning(
+                f"'directory' field is empty or None. Assigning default value: "
+                f"{default_value}"
+            )
+            return default_value
+
+        # Strip trailing slashes and store original value for comparison
+        v = str(v).strip("/")  # Ensure it's a string and no trailing slashes
+        original_value = v
+
+        # Replace spaces with underscores
+        v = v.replace(" ", "_")
+        logger.debug(f"Corrected 'directory', spaces replaced: {repr(v)}")
+
+        # Validate directory pattern with macros and allowed characters
+        valid_macros = [
+            "{date}",
+            "{prefix}",
+            "{sgPuck}",
+            "{sgPosition}",
+            "{beamline}",
+            "{sgPrefix}",
+            "{sgPriority}",
+            "{protein}",
+            "{method}",
+        ]
+        valid_macro_pattern = re.compile(
+            "|".join(re.escape(macro) for macro in valid_macros)
+        )
+
+        # Check if the value contains valid macros
+        allowed_chars_pattern = "[a-z0-9_.+-/]"
+        v_without_macros = valid_macro_pattern.sub("macro", v)
+
+        allowed_path_pattern = re.compile(
+            f"^(({allowed_chars_pattern}+|macro)*/*)*$", re.IGNORECASE
+        )
+        if not allowed_path_pattern.match(v_without_macros):
+            raise ValueError(
+                f"'{v}' is not valid. Value must be a valid path or macro."
+            )
+
+        # Log and return corrected value
+        if v != original_value:
+            logger.info(f"Directory was corrected from '{original_value}' to '{v}'")
+        return v
+
+    @field_validator("aperture", mode="before")
+    @classmethod
+    def aperture_selection(cls, v):
+        if v is not None:
+            try:
+                v = int(float(v))
+                if v not in {1, 2, 3}:
+                    raise ValueError(f" '{v}' is not valid. Value must be 1, 2, or 3.")
+            except (ValueError, TypeError) as e:
+                raise ValueError(
+                    f" '{v}' is not valid. Value must be 1, 2, or 3."
+                ) from e
+        return v
+
+    @field_validator("oscillation", mode="before")
+    @classmethod
+    def positive_float_validator(cls, v):
+        if v is None:
+            return None
+        try:
+            v = float(v)
+            if v <= 0:
+                raise ValueError(f"'{v}' is not valid. Value must be a positive float.")
+        except (ValueError, TypeError) as e:
+            raise ValueError(
+                f"'{v}' is not valid. Value must be a positive float."
+            ) from e
+        return v
+
+    @field_validator("exposure", mode="before")
+    @classmethod
+    def exposure_in_range(cls, v):
+        if v is not None:
+            try:
+                v = float(v)
+                if not (0 <= v <= 1):
+                    raise ValueError(
+                        f" '{v}' is not valid. Value must be a float between 0 and 1."
+                    )
+            except (ValueError, TypeError) as e:
+                raise ValueError(
+                    f" '{v}' is not valid. Value must be a float between 0 and 1."
+                ) from e
+        return v
+
+    @field_validator("totalrange", mode="before")
+    @classmethod
+    def totalrange_in_range(cls, v):
+        if v is not None:
+            try:
+                v = int(v)
+                if not (0 <= v <= 360):
+                    raise ValueError(
+                        f" '{v}' is not valid."
+                        f"Value must be an integer between 0 and 360."
+                    )
+            except (ValueError, TypeError) as e:
+                raise ValueError(
+                    f" '{v}' is not valid."
+                    f"Value must be an integer between 0 and 360."
+                ) from e
+        return v
+
+    @field_validator("transmission", mode="before")
+    @classmethod
+    def transmission_fraction(cls, v):
+        if v is not None:
+            try:
+                v = int(v)
+                if not (0 <= v <= 100):
+                    raise ValueError(
+                        f" '{v}' is not valid."
+                        f"Value must be an integer between 0 and 100."
+                    )
+            except (ValueError, TypeError) as e:
+                raise ValueError(
+                    f" '{v}' is not valid."
+                    f"Value must be an integer between 0 and 100."
+                ) from e
+        return v
+
+    @field_validator("datacollectiontype", mode="before")
+    @classmethod
+    def datacollectiontype_allowed(cls, v):
+        allowed = {"standard"}  # Other types of data collection might be added later
+        if v and v.lower() not in allowed:
+            raise ValueError(f" '{v}' is not valid." f"Value must be one of {allowed}.")
+        return v
+
+    @field_validator("processingpipeline", mode="before")
+    @classmethod
+    def processingpipeline_allowed(cls, v):
+        allowed = {"aareproc", "autoproc"}
+        if v and v.lower() not in allowed:
+            raise ValueError(f" '{v}' is not valid." f"Value must be one of {allowed}.")
+        return v
+
+    @field_validator("spacegroupnumber", mode="before")
+    @classmethod
+    def spacegroupnumber_allowed(cls, v):
+        if v is not None:
+            try:
+                v = int(v)
+                if not (1 <= v <= 230):
+                    raise ValueError(
+                        f" '{v}' is not valid."
+                        f"Value must be an integer between 1 and 230."
+                    )
+            except (ValueError, TypeError) as e:
+                raise ValueError(
+                    f" '{v}' is not valid."
+                    f"Value must be an integer between 1 and 230."
+                ) from e
+        return v
+
+    @field_validator("cellparameters", mode="before")
+    @classmethod
+    def cellparameters_format(cls, v):
+        if v:
+            # Replace commas with spaces, then split on whitespace
+            tokens = v.replace(",", " ").split()
+            try:
+                values = [float(i) for i in tokens]
+            except ValueError:
+                raise ValueError(
+                    f" '{v}' is not valid."
+                    " Value must be a set of six positive floats"
+                    " or integers (separated by space or comma)."
+                )
+            if len(values) != 6 or any(val <= 0 for val in values):
+                raise ValueError(
+                    f" '{v}' is not valid."
+                    " Value must be a set of six positive floats"
+                    " or integers (separated by space or comma)."
+                )
+        return v
+
+    # @field_validator("rescutkey", "rescutvalue", mode="before")
+    # @classmethod
+    # def rescutkey_value_pair(cls, values):
+    #    rescutkey = values.get("rescutkey")
+    #    rescutvalue = values.get("rescutvalue")
+    #    if rescutkey and rescutvalue:
+    #        if rescutkey not in {"is", "cchalf"}:
+    #            raise ValueError("Rescutkey must be either 'is' or 'cchalf'")
+    #        if not isinstance(rescutvalue, float) or rescutvalue <= 0:
+    #            raise ValueError(
+    #                "Rescutvalue must be a positive float if rescutkey is provided"
+    #            )
+    #    return values
+
+    @field_validator("trustedhigh", mode="before")
+    @classmethod
+    def trustedhigh_allowed(cls, v):
+        if v is not None:
+            try:
+                v = float(v)
+                if not (0 <= v <= 2.0):
+                    raise ValueError(
+                        f" '{v}' is not valid."
+                        f"Value must be a float between 0 and 2.0."
+                    )
+            except (ValueError, TypeError) as e:
+                raise ValueError(
+                    f" '{v}' is not valid." f"Value must be a float between 0 and 2.0."
+                ) from e
+        return v
+
+    @field_validator("chiphiangles", mode="before")
+    @classmethod
+    def chiphiangles_allowed(cls, v):
+        if v is not None:
+            try:
+                v = float(v)
+                if not (0 <= v <= 30):
+                    raise ValueError(
+                        f" '{v}' is not valid."
+                        f"Value must be a float between 0 and 30."
+                    )
+            except (ValueError, TypeError) as e:
+                raise ValueError(
+                    f" '{v}' is not valid. Value must be a float between 0 and 30."
+                ) from e
+        return v
+
+    @field_validator("dose", mode="before")
+    @classmethod
+    def dose_positive(cls, v):
+        if v is not None:
+            try:
+                v = float(v)
+                if v <= 0:
+                    raise ValueError(
+                        f" '{v}' is not valid. Value must be a positive float."
+                    )
+            except (ValueError, TypeError) as e:
+                raise ValueError(
+                    f" '{v}' is not valid. Value must be a positive float."
+                ) from e
+        return v
+
+    @field_validator("pdbmodel", mode="after")
+    @classmethod
+    def validate_filepath(cls, v):
+        if v is None:
+            return v
+        v_str = str(v)  # Ensure v is a string for further checks
+        if any(c in v_str for c in '<>:"|?*'):
+            raise ValueError("File path contains invalid characters.")
+        path = Path(v_str)
+        if not path.parts:
+            raise ValueError("Not a valid path.")
+        return v_str  # Return as string for JSON serialization
+
+    @field_validator("cloud", mode="before")
+    @classmethod
+    def coerce_cloud_default(cls, v):
+        if v in ("", None):
+            return True
+        if isinstance(v, bool):
+            return v
+        v_str = str(v).strip().lower()
+        if v_str in {"true", "yes", "1"}:
+            return True
+        if v_str in {"false", "no", "0"}:
+            return False
+        raise ValueError("cloud must be blank for default, or True/False")
 
 # From database to TELL after loading
 class SampleShortInfo(BaseModel):

daq/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "aaredaq"
-version = "0.2.3"
+version = "0.2.4"
 description = "AareDAQ data acquisition server"
 readme = "README.md"
 requires-python = ">=3.11"
@@ -13,12 +13,12 @@ dependencies = [
     "fastapi==0.115.13",
     "uvicorn==0.34.2",
     "ultralytics==8.3.133",
-    "aaredb==0.1.1a12",
+    "aaredb==0.1.1a24",
     "opencv-python-headless==4.11.0.86",
     "python_multipart==0.0.20",
     "websocket-client==1.8.0",
     "sseclient-py==1.8.0",
-    "aaredaqlib==0.2.3",
+    "aaredaqlib==0.2.4",
 ]
 
 [lint]

gui/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "aaregui"
-version = "0.2.3"
+version = "0.2.4"
 description = "Beamline control GUI"
 readme = "README.md"
 requires-python = ">=3.11"
@@ -9,7 +9,7 @@ dependencies = [
     "pyzmq==26.4.0",
     "opencv-python-headless==4.11.0.86",
     "PySide6==6.9.0",
-    "aaredaqlib==0.2.3"
+    "aaredaqlib==0.2.4"
 ]
 
 [lint]