fix: towards a first working version

backend/bec_atlas/datasources/datasource_manager.py

@@ -17,11 +17,16 @@ class DatasourceManager:
             datasource.connect()
 
     def load_datasources(self):
-        logger.info(f"Loading datasources with config: {self.config}")
         for datasource_name, datasource_config in self.config.items():
             if datasource_name == "redis":
+                logger.info(
+                    f"Loading Redis datasource. Host: {datasource_config.get('host')}, Port: {datasource_config.get('port')}, Username: {datasource_config.get('username')}"
+                )
                 self.datasources[datasource_name] = RedisDatasource(datasource_config)
             if datasource_name == "mongodb":
+                logger.info(
+                    f"Loading MongoDB datasource. Host: {datasource_config.get('host')}, Port: {datasource_config.get('port')}, Username: {datasource_config.get('username')}"
+                )
                 self.datasources[datasource_name] = MongoDBDatasource(datasource_config)
 
     def shutdown(self):

backend/bec_atlas/datasources/mongodb/mongodb.py

@@ -1,5 +1,6 @@
 import json
 import os
+from typing import Literal
 
 import pymongo
 from bec_lib.logger import bec_logger
@@ -21,10 +22,21 @@ class MongoDBDatasource:
         """
         Connect to the MongoDB database.
         """
-        host = self.config.get("host", "localhost")
+        host = self.config.get("host")
-        port = self.config.get("port", 27017)
+        port = self.config.get("port")
+        username = self.config.get("username")
+        password = self.config.get("password")
+        if username and password:
+            self.client = pymongo.MongoClient(
+                f"mongodb://{username}:{password}@{host}:{port}/?authSource=bec_atlas"
+            )
+        else:
+            self.client = pymongo.MongoClient(f"mongodb://{host}:{port}/")
+
+        # Check if the connection is successful
+        self.client.list_databases()
+
         logger.info(f"Connecting to MongoDB at {host}:{port}")
-        self.client = pymongo.MongoClient(f"mongodb://{host}:{port}/")
         self.db = self.client["bec_atlas"]
         if include_setup:
             self.db["users"].create_index([("email", 1)], unique=True)
@@ -55,7 +67,7 @@ class MongoDBDatasource:
                 {
                     "email": "jane.doe@bec_atlas.ch",
                     "password": "atlas",
-                    "groups": ["demo_user"],
+                    "groups": ["demo"],
                     "first_name": "Jane",
                     "last_name": "Doe",
                     "owner_groups": ["admin"],
@@ -136,30 +148,91 @@ class MongoDBDatasource:
         out = self.db[collection].find(query_filter)
         return [dtype(**x) for x in out]
 
-    def add_user_filter(self, user: User, query_filter: dict) -> dict:
+    def aggregate(
+        self, collection: str, pipeline: list[dict], dtype: BaseModel, user: User | None = None
+    ) -> list[BaseModel]:
+        """
+        Aggregate documents in the collection.
+
+        Args:
+            collection (str): The collection name
+            pipeline (list[dict]): The aggregation pipeline
+            dtype (BaseModel): The data type to return
+            user (User): The user making the request
+
+        Returns:
+            list[BaseModel]: The data type with the document data
+        """
+        if user is not None:
+            # Add the user filter to the lookup pipeline
+
+            for pipe in pipeline:
+                if "$lookup" not in pipe:
+                    continue
+                if "pipeline" not in pipe["$lookup"]:
+                    continue
+                lookup = pipe["$lookup"]
+                lookup_pipeline = lookup["pipeline"]
+                access_filter = {"$match": self._read_only_user_filter(user)}
+                lookup_pipeline.insert(0, access_filter)
+            # pipeline = self.add_user_filter(user, pipeline)
+        out = self.db[collection].aggregate(pipeline)
+        return [dtype(**x) for x in out]
+
+    def add_user_filter(
+        self, user: User, query_filter: dict, operation: Literal["r", "w"] = "r"
+    ) -> dict:
         """
         Add the user filter to the query filter.
 
         Args:
             user (User): The user making the request
             query_filter (dict): The query filter
+            operation (Literal["r", "w"]): The operation to perform
+
+        Returns:
+            dict: The updated query filter
+        """
+        if operation == "r":
+            user_filter = self._read_only_user_filter(user)
+        else:
+            user_filter = self._write_user_filter(user)
+        if user_filter:
+            query_filter = {"$and": [query_filter, user_filter]}
+        return query_filter
+
+    def _read_only_user_filter(self, user: User) -> dict:
+        """
+        Add the user filter to the query filter.
+
+        Args:
+            user (User): The user making the request
 
         Returns:
             dict: The updated query filter
         """
         if "admin" not in user.groups:
-            query_filter = {
+            return {
-                "$and": [
+                "$or": [
-                    query_filter,
+                    {"owner_groups": {"$in": user.groups}},
-                    {
+                    {"access_groups": {"$in": user.groups}},
-                        "$or": [
-                            {"owner_groups": {"$in": user.groups}},
-                            {"access_groups": {"$in": user.groups}},
-                        ]
-                    },
                 ]
             }
-        return query_filter
+        return {}
+
+    def _write_user_filter(self, user: User) -> dict:
+        """
+        Add the user filter to the query filter.
+
+        Args:
+            user (User): The user making the request
+
+        Returns:
+            dict: The updated query filter
+        """
+        if "admin" not in user.groups:
+            return {"$or": [{"owner_groups": {"$in": user.groups}}]}
+        return {}
 
     def shutdown(self):
         """

backend/bec_atlas/datasources/redis_datasource.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 from typing import TYPE_CHECKING
 
 from bec_lib.redis_connector import RedisConnector
+from redis.asyncio import Redis as AsyncRedis
 from redis.exceptions import AuthenticationError
 
 if TYPE_CHECKING:
@@ -13,14 +14,26 @@ class RedisDatasource:
     def __init__(self, config: dict):
         self.config = config
         self.connector = RedisConnector(f"{config.get('host')}:{config.get('port')}")
+        username = config.get("username")
+        password = config.get("password")
 
         try:
-            self.connector._redis_conn.auth(config.get("password", "ingestor"), username="ingestor")
+            self.connector._redis_conn.auth(password, username=username)
             self.reconfigured_acls = False
         except AuthenticationError:
             self.setup_acls()
-            self.connector._redis_conn.auth(config.get("password", "ingestor"), username="ingestor")
+            self.connector._redis_conn.auth(password, username=username)
             self.reconfigured_acls = True
+
+        self.connector._redis_conn.connection_pool.connection_kwargs["username"] = username
+        self.connector._redis_conn.connection_pool.connection_kwargs["password"] = password
+
+        self.async_connector = AsyncRedis(
+            host=config.get("host"),
+            port=config.get("port"),
+            username="ingestor",
+            password=config.get("password"),
+        )
         print("Connected to Redis")
 
     def setup_acls(self):
@@ -32,7 +45,7 @@ class RedisDatasource:
         self.connector._redis_conn.acl_setuser(
             "ingestor",
             enabled=True,
-            passwords=f'+{self.config.get("password", "ingestor")}',
+            passwords=f'+{self.config.get("password")}',
             categories=["+@all"],
             keys=["*"],
             channels=["*"],
@@ -71,6 +84,8 @@ class RedisDatasource:
             channels=[
                 f"internal/deployment/{deployment.id}/*/state",
                 f"internal/deployment/{deployment.id}/*",
+                f"internal/deployment/{deployment.id}/request",
+                f"internal/deployment/{deployment.id}/request_response/*",
             ],
             commands=[f"+keys|internal/deployment/{deployment.id}/*/state"],
             reset_channels=True,

backend/bec_atlas/ingestor/data_ingestor.py

@@ -135,6 +135,9 @@ class DataIngestor:
 
         """
         while not self.shutdown_event.is_set():
+            if not self.available_deployments:
+                self.shutdown_event.wait(1)
+                continue
             streams = {
                 f"internal/deployment/{deployment['id']}/ingest": ">"
                 for deployment in self.available_deployments

backend/bec_atlas/main.py

@@ -1,17 +1,15 @@
-import socketio
 import uvicorn
 from fastapi import FastAPI
 
 from bec_atlas.datasources.datasource_manager import DatasourceManager
 from bec_atlas.router.deployments_router import DeploymentsRouter
 from bec_atlas.router.realm_router import RealmRouter
-from bec_atlas.router.redis_router import RedisWebsocket
+from bec_atlas.router.redis_router import RedisRouter, RedisWebsocket
 from bec_atlas.router.scan_router import ScanRouter
 from bec_atlas.router.user_router import UserRouter
 
 CONFIG = {
     "redis": {"host": "localhost", "port": 6380},
-    "scylla": {"hosts": ["localhost"]},
     "mongodb": {"host": "localhost", "port": 27017},
 }
 
@@ -40,6 +38,7 @@ class AtlasApp:
         self.datasources.shutdown()
 
     def add_routers(self):
+        # pylint: disable=attribute-defined-outside-init
         if not self.datasources.datasources:
             raise ValueError("Datasources not loaded")
         self.scan_router = ScanRouter(prefix=self.prefix, datasources=self.datasources)
@@ -50,12 +49,13 @@ class AtlasApp:
         self.app.include_router(self.deployment_router.router, tags=["Deployment"])
         self.realm_router = RealmRouter(prefix=self.prefix, datasources=self.datasources)
         self.app.include_router(self.realm_router.router, tags=["Realm"])
+        self.redis_router = RedisRouter(prefix=self.prefix, datasources=self.datasources)
+        self.app.include_router(self.redis_router.router, tags=["Redis"])
 
-        if "redis" in self.datasources.datasources:
+        self.redis_websocket = RedisWebsocket(
-            self.redis_websocket = RedisWebsocket(
+            prefix=self.prefix, datasources=self.datasources, app=self
-                prefix=self.prefix, datasources=self.datasources, app=self
+        )
-            )
+        self.app.mount("/", self.redis_websocket.app)
-            self.app.mount("/", self.redis_websocket.app)
 
     def run(self, port=8000):
         config = uvicorn.Config(self.app, host="localhost", port=port)
@@ -66,12 +66,18 @@ class AtlasApp:
 
 def main():
     import argparse
+    import logging
+
+    from bec_atlas.utils.env_loader import load_env
+
+    config = load_env()
+    logging.basicConfig(level=logging.INFO)
 
     parser = argparse.ArgumentParser(description="Run the BEC Atlas API")
     parser.add_argument("--port", type=int, default=8000, help="Port to run the API on")
 
     args = parser.parse_args()
-    horizon_app = AtlasApp()
+    horizon_app = AtlasApp(config=config)
     horizon_app.run(port=args.port)
 
 

backend/bec_atlas/model/model.py

@@ -1,5 +1,7 @@
+from __future__ import annotations
+
 import uuid
-from typing import Literal
+from typing import Any, Literal
 
 from bec_lib import messages
 from bson import ObjectId
@@ -23,6 +25,11 @@ class AccessProfile(BaseModel):
     access_groups: list[str] = []
 
 
+class AccessProfilePartial(AccessProfile):
+    owner_groups: list[str] | None = None
+    access_groups: list[str] | None = None
+
+
 class ScanStatus(MongoBaseModel, AccessProfile, messages.ScanStatusMessage): ...
 
 
@@ -44,10 +51,25 @@ class UserInfo(BaseModel):
 
 
 class Deployments(MongoBaseModel, AccessProfile):
-    realm_id: str
+    realm_id: str | ObjectId
     name: str
     deployment_key: str = Field(default_factory=lambda: str(uuid.uuid4()))
-    active_session_id: str | None = None
+    active_session_id: str | ObjectId | None = None
+    config_templates: list[str | ObjectId] = []
+
+
+class DeploymentsPartial(MongoBaseModel, AccessProfilePartial):
+    realm_id: str | ObjectId | None = None
+    name: str | None = None
+    deployment_key: str | None = None
+    active_session_id: str | ObjectId | None = None
+    config_templates: list[str | ObjectId] | None = None
+
+
+class Realm(MongoBaseModel, AccessProfile):
+    realm_id: str
+    deployments: list[Deployments | DeploymentsPartial] = []
+    name: str
 
 
 class Experiments(AccessProfile):
@@ -85,12 +107,6 @@ class Session(MongoBaseModel, AccessProfile):
     name: str
 
 
-class Realm(MongoBaseModel, AccessProfile):
-    realm_id: str
-    deployments: list[Deployments] = []
-    name: str
-
-
 class Datasets(AccessProfile):
     realm_id: str
     dataset_id: str
@@ -126,18 +142,33 @@ class DeviceConfig(AccessProfile):
     software_trigger: bool
 
 
-class SignalData(AccessProfile):
+class SignalData(AccessProfile, MongoBaseModel):
-    scan_id: str
+    """
-    device_id: str
+    Signal data for a device. This is the ophyd signal data,
-    device_name: str
+    aggregated for a single scan. Upon completion of a scan,
+    the data is aggregated and stored in this format. If possible,
+    the data ingestor will calculate the average, standard deviation,
+    min, and max values for the signal.
+    """
+
+    scan_id: str | ObjectId | None = None
+    device_id: str | ObjectId
     signal_name: str
-    data: float | int | str | bool | bytes | dict | list | None
+    data: list[Any]
-    timestamp: float
+    timestamps: list[float]
-    kind: Literal["hinted", "omitted", "normal", "config"]
+    kind: Literal["hinted", "normal", "config", "omitted"]
+    average: float | None = None
+    std_dev: float | None = None
+    min: float | None = None
+    max: float | None = None
 
 
-class DeviceData(AccessProfile):
+class DeviceData(AccessProfile, MongoBaseModel):
-    scan_id: str | None
+    scan_id: str | ObjectId | None = None
-    device_name: str
+    name: str
-    device_config_id: str
+    device_config_id: str | ObjectId
     signals: list[SignalData]
+
+
+if __name__ == "__main__":
+    out = DeploymentsPartial(realm_id="123")

backend/bec_atlas/router/realm_router.py

@@ -1,7 +1,8 @@
-from fastapi import APIRouter
+from fastapi import APIRouter, Depends
 
+from bec_atlas.authentication import get_current_user
 from bec_atlas.datasources.mongodb.mongodb import MongoDBDatasource
-from bec_atlas.model.model import Realm
+from bec_atlas.model.model import Realm, UserInfo
 from bec_atlas.router.base_router import BaseRouter
 
 
@@ -14,27 +15,48 @@ class RealmRouter(BaseRouter):
             "/realms",
             self.realms,
             methods=["GET"],
-            description="Get all deployments for the realm",
+            description="Get all realms",
             response_model=list[Realm],
+            response_model_exclude_none=True,
         )
         self.router.add_api_route(
             "/realms/{realm_id}",
             self.realm_with_id,
             methods=["GET"],
-            description="Get a single deployment by id for a realm",
+            description="Get a single realm by id",
             response_model=Realm,
+            response_model_exclude_none=True,
         )
 
-    async def realms(self) -> list[Realm]:
+    async def realms(
+        self, include_deployments: bool = False, current_user: UserInfo = Depends(get_current_user)
+    ) -> list[Realm]:
         """
         Get all realms.
 
+        Args:
+            include_deployments (bool): Include deployments in the response
+
         Returns:
             list[Realm]: List of realms
         """
-        return self.db.find("realms", {}, Realm)
+        if include_deployments:
+            include = [
+                {
+                    "$lookup": {
+                        "from": "deployments",
+                        "let": {"realm_id": "$_id"},
+                        "pipeline": [{"$match": {"$expr": {"$eq": ["$realm_id", "$$realm_id"]}}}],
+                        "as": "deployments",
+                    }
+                }
+            ]
+            return self.db.aggregate("realms", include, Realm, user=current_user)
+        return self.db.find("realms", {}, Realm, user=current_user)
 
-    async def realm_with_id(self, realm_id: str):
+    async def realm_with_id(
+        self, realm_id: str, current_user: UserInfo = Depends(get_current_user)
+    ):
         """
         Get realm with id.
 
@@ -44,4 +66,4 @@ class RealmRouter(BaseRouter):
         Returns:
             Realm: The realm with the id
         """
-        return self.db.find_one("realms", {"_id": realm_id}, Realm)
+        return self.db.find_one("realms", {"_id": realm_id}, Realm, user=current_user)

backend/bec_atlas/router/redis_router.py

@@ -3,13 +3,14 @@ import functools
 import inspect
 import json
 import traceback
+import uuid
 from typing import TYPE_CHECKING, Any
 
 import socketio
 from bec_lib.endpoints import EndpointInfo, MessageEndpoints, MessageOp
 from bec_lib.logger import bec_logger
-from bec_lib.serialization import json_ext
+from bec_lib.serialization import MsgpackSerialization, json_ext
-from fastapi import APIRouter
+from fastapi import APIRouter, Query, Response
 
 from bec_atlas.router.base_router import BaseRouter
 
@@ -67,6 +68,40 @@ class RedisAtlasEndpoints:
         """
         return f"socketio/rooms/{deployment}/{endpoint}"
 
+    @staticmethod
+    def redis_request(deployment: str):
+        """
+        Endpoint for the redis request for a deployment and endpoint.
+
+        Args:
+            deployment (str): The deployment name
+
+        Returns:
+            str: The endpoint for the redis request
+        """
+        return f"internal/deployment/{deployment}/request"
+
+    @staticmethod
+    def redis_request_response(deployment: str, request_id: str):
+        """
+        Endpoint for the redis request response for a deployment and endpoint.
+
+        Args:
+            deployment (str): The deployment name
+            request_id (str): The request id
+
+        Returns:
+            str: The endpoint for the redis request response
+        """
+        return f"internal/deployment/{deployment}/request_response/{request_id}"
+
+
+class MsgResponse(Response):
+    media_type = "application/json"
+
+    def render(self, content: Any) -> bytes:
+        return content.encode()
+
 
 class RedisRouter(BaseRouter):
     """
@@ -76,14 +111,30 @@ class RedisRouter(BaseRouter):
 
     def __init__(self, prefix="/api/v1", datasources=None):
         super().__init__(prefix, datasources)
-        self.redis = self.datasources.datasources["redis"].connector
+        self.redis = self.datasources.datasources["redis"].async_connector
+
         self.router = APIRouter(prefix=prefix)
-        self.router.add_api_route("/redis", self.redis_get, methods=["GET"])
+        self.router.add_api_route(
+            "/redis/{deployment}", self.redis_get, methods=["GET"], response_class=MsgResponse
+        )
         self.router.add_api_route("/redis", self.redis_post, methods=["POST"])
         self.router.add_api_route("/redis", self.redis_delete, methods=["DELETE"])
 
-    async def redis_get(self, key: str):
+    async def redis_get(self, deployment: str, key: str = Query(...)):
-        return self.redis.get(key)
+        request_id = uuid.uuid4().hex
+        response_endpoint = RedisAtlasEndpoints.redis_request_response(deployment, request_id)
+        request_endpoint = RedisAtlasEndpoints.redis_request(deployment)
+        pubsub = self.redis.pubsub()
+        pubsub.ignore_subscribe_messages = True
+        await pubsub.subscribe(response_endpoint)
+        data = {"action": "get", "key": key, "response_endpoint": response_endpoint}
+        await self.redis.publish(request_endpoint, json.dumps(data))
+        response = await pubsub.get_message(timeout=10)
+        print(response)
+        response = await pubsub.get_message(timeout=10)
+        out = MsgpackSerialization.loads(response["data"])
+
+        return json_ext.dumps({"data": out.content, "metadata": out.metadata})
 
     async def redis_post(self, key: str, value: str):
         return self.redis.set(key, value)
@@ -129,9 +180,9 @@ class BECAsyncRedisManager(socketio.AsyncRedisManager):
 
     def start_update_loop(self):
         self.started_update_loop = True
-        # loop = asyncio.get_event_loop()
+        loop = asyncio.get_event_loop()
-        # task = loop.create_task(self._backend_heartbeat())
+        task = loop.create_task(self._backend_heartbeat())
-        # return task
+        return task
 
     async def disconnect(self, sid, namespace, **kwargs):
         if kwargs.get("ignore_queue"):
@@ -205,6 +256,8 @@ class RedisWebsocket:
         redis_port = datasources.datasources["redis"].config["port"]
         redis_password = datasources.datasources["redis"].config.get("password", "ingestor")
         self.socket = socketio.AsyncServer(
+            transports=["websocket"],
+            ping_timeout=60,
             cors_allowed_origins="*",
             async_mode="asgi",
             client_manager=BECAsyncRedisManager(
@@ -239,7 +292,10 @@ class RedisWebsocket:
         """
         if not http_query:
             raise ValueError("Query parameters not found")
-        query = json.loads(http_query)
+        if isinstance(http_query, str):
+            query = json.loads(http_query)
+        else:
+            query = http_query
 
         if "user" not in query:
             raise ValueError("User not found in query parameters")
@@ -256,12 +312,12 @@ class RedisWebsocket:
         return user, deployment
 
     @safe_socket
-    async def connect_client(self, sid, environ=None):
+    async def connect_client(self, sid, environ=None, auth=None, **kwargs):
         if sid in self.users:
             logger.info("User already connected")
             return
 
-        http_query = environ.get("HTTP_QUERY")
+        http_query = environ.get("HTTP_QUERY") or auth
 
         user, deployment = self._validate_new_user(http_query)
 
@@ -283,9 +339,9 @@ class RedisWebsocket:
 
         if user in info:
             self.users[sid] = {"user": user, "subscriptions": [], "deployment": deployment}
-            for endpoint in set(info[user]):
+            for endpoint, endpoint_request in info[user]:
                 print(f"Registering {endpoint}")
-                await self._update_user_subscriptions(sid, endpoint)
+                await self._update_user_subscriptions(sid, endpoint, endpoint_request)
         else:
             self.users[sid] = {"user": user, "subscriptions": [], "deployment": deployment}
 
@@ -321,13 +377,16 @@ class RedisWebsocket:
 
         # check if the endpoint receives arguments
         if len(inspect.signature(endpoint).parameters) > 0:
-            endpoint: MessageEndpoints = endpoint(data.get("args"))
+            args = data.get("args", [])
+            if not isinstance(args, list):
+                args = [args]
+            endpoint: MessageEndpoints = endpoint(*args)
         else:
             endpoint: MessageEndpoints = endpoint()
 
-        await self._update_user_subscriptions(sid, endpoint.endpoint)
+        await self._update_user_subscriptions(sid, endpoint.endpoint, msg)
 
-    async def _update_user_subscriptions(self, sid: str, endpoint: str):
+    async def _update_user_subscriptions(self, sid: str, endpoint: str, endpoint_request: str):
         deployment = self.users[sid]["deployment"]
 
         endpoint_info = EndpointInfo(
@@ -335,20 +394,31 @@ class RedisWebsocket:
         )
 
         room = RedisAtlasEndpoints.socketio_endpoint_room(deployment, endpoint)
-        self.redis.register(endpoint_info, cb=self.on_redis_message, parent=self, room=room)
+        self.redis.register(
+            endpoint_info,
+            cb=self.on_redis_message,
+            parent=self,
+            room=room,
+            endpoint_request=endpoint_request,
+        )
         if endpoint not in self.users[sid]["subscriptions"]:
             await self.socket.enter_room(sid, room)
-            self.users[sid]["subscriptions"].append(endpoint)
+            self.users[sid]["subscriptions"].append((endpoint, endpoint_request))
             await self.socket.manager.update_websocket_states()
 
     @staticmethod
-    def on_redis_message(message, parent, room):
+    def on_redis_message(message, parent, room, endpoint_request):
         async def emit_message(message):
             if "pubsub_data" in message:
                 msg = message["pubsub_data"]
             else:
                 msg = message["data"]
-            outgoing = {"data": msg.content, "metadata": msg.metadata}
+            outgoing = {
+                "data": msg.content,
+                "metadata": msg.metadata,
+                "endpoint": room.split("/", 3)[-1],
+                "endpoint_request": endpoint_request,
+            }
             outgoing = json_ext.dumps(outgoing)
             await parent.socket.emit("message", data=outgoing, room=room)
 

backend/bec_atlas/utils/demo_database_setup.py

@@ -16,19 +16,32 @@ class DemoSetupLoader:
         self.load_deployments()
 
     def load_realm(self):
-        realm = Realm(realm_id="demo_beamline_1", name="Demo Beamline 1", owner_groups=["admin"])
+        realm = Realm(
+            realm_id="demo_beamline_1",
+            name="Demo Beamline 1",
+            owner_groups=["admin"],
+            access_groups=["auth_user"],
+        )
         realm._id = realm.realm_id
         if self.db["realms"].find_one({"realm_id": realm.realm_id}) is None:
             self.db["realms"].insert_one(realm.__dict__)
 
-        realm = Realm(realm_id="demo_beamline_2", name="Demo Beamline 2", owner_groups=["admin"])
+        realm = Realm(
+            realm_id="demo_beamline_2",
+            name="Demo Beamline 2",
+            owner_groups=["admin"],
+            access_groups=["auth_user"],
+        )
         realm._id = realm.realm_id
         if self.db["realms"].find_one({"realm_id": realm.realm_id}) is None:
             self.db["realms"].insert_one(realm.__dict__)
 
     def load_deployments(self):
         deployment = Deployments(
-            realm_id="demo_beamline_1", name="Demo Deployment 1", owner_groups=["admin", "demo"]
+            realm_id="demo_beamline_1",
+            name="Demo Deployment 1",
+            owner_groups=["admin", "demo"],
+            access_groups=["demo"],
         )
         if self.db["deployments"].find_one({"name": deployment.name}) is None:
             self.db["deployments"].insert_one(deployment.__dict__)
@@ -36,7 +49,10 @@ class DemoSetupLoader:
         if self.db["sessions"].find_one({"name": "_default_"}) is None:
             deployment = self.db["deployments"].find_one({"name": deployment["name"]})
             default_session = Session(
-                owner_groups=["admin", "demo"], deployment_id=deployment["_id"], name="_default_"
+                owner_groups=["admin", "demo"],
+                access_groups=["demo"],
+                deployment_id=deployment["_id"],
+                name="_default_",
             )
             self.db["sessions"].insert_one(default_session.model_dump(exclude_none=True))
 

backend/bec_atlas/utils/env_loader.py

@@ -0,0 +1,25 @@
+import os
+
+import yaml
+
+
+def load_env() -> dict:
+    """
+    Load the environment variables from the .env file.
+    """
+    env_file = "./.env.yaml"
+
+    if not os.path.exists(env_file):
+        env_file = os.path.join(os.path.dirname(__file__), ".env.yaml")
+
+    if not os.path.exists(env_file):
+        # check if there is an env file in the parent directory
+        current_dir = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
+        env_file = os.path.join(current_dir, ".env.yaml")
+
+    if not os.path.exists(env_file):
+        raise FileNotFoundError(f"Could not find .env file in {os.getcwd()} or {current_dir}")
+
+    with open(env_file, "r", encoding="utf-8") as file:
+        yaml_config = yaml.safe_load(file)
+    return yaml_config

backend/tests/conftest.py

@@ -132,7 +132,12 @@ def backend(redis_container, mongo_container):
     redis_host, redis_port = redis_container
     mongo_host, mongo_port = mongo_container
     config = {
-        "redis": {"host": redis_host, "port": redis_port},
+        "redis": {
+            "host": redis_host,
+            "port": redis_port,
+            "username": "ingestor",
+            "password": "ingestor",
+        },
         "mongodb": {"host": mongo_host, "port": mongo_port},
     }
 

backend/utils/sls_deployments.yaml

@@ -0,0 +1,67 @@
+
+ADDAMS:
+    x04sa-bec-001.psi.ch:
+        name: production
+        description: Primary deployment for ADDAMS
+cSAXS:
+    x12sa-bec-001.psi.ch:
+        name: production
+        description: Primary deployment for cSAXS
+    x12sa-bec-002.psi.ch:
+        name: test
+        description: Test environment for cSAXS
+Debye:
+    x01da-bec-001.psi.ch:
+        name: production
+        description: Primary deployment for Debye
+MicroXAS:
+    x05la-bec-001.psi.ch:
+        name: production
+        description: Primary deployment for MicroXAS
+    x05la-bec-002.psi.ch:
+        name: test
+        description: Test environment for MicroXAS
+Phoenix:
+    x07mb-bec-001.psi.ch:
+        name: production
+        description: Primary deployment for Phoenix
+PolLux:
+    x07da-bec-001.psi.ch:
+        name: production
+        description: Primary deployment for PolLux
+PXI:
+    x06sa-bec-001.psi.ch:
+        name: production
+        description: Primary deployment for PXI
+PXII:
+    x10sa-bec-001.psi.ch:
+        name: production
+        description: Primary deployment for PXII
+PXIII:
+    x06da-bec-001.psi.ch:
+        name: production
+        description: Primary deployment for PXIII
+SIM:
+    x11ma-bec-001.psi.ch:
+        name: production
+        description: Primary deployment for SIM
+SuperXAS:
+    x10da-bec-001.psi.ch: 
+        name: production
+        description: Primary deployment for SuperXAS
+I-TOMCAT:
+    x02da-bec-001.psi.ch:
+        name: production
+        description: Primary deployment for I-TOMCAT
+    x02da-bec-002.psi.ch:
+        name: test
+        description: Test environment for I-TOMCAT
+X-Treme:
+    x07ma-bec-001.psi.ch:
+        name: production
+        description: Primary deployment for X-Treme
+
+
+
+
+

frontend/bec_atlas/.editorconfig

@@ -0,0 +1,17 @@
+# Editor configuration, see https://editorconfig.org
+root = true
+
+[*]
+charset = utf-8
+indent_style = space
+indent_size = 2
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[*.ts]
+quote_type = single
+ij_typescript_use_double_quotes = false
+
+[*.md]
+max_line_length = off
+trim_trailing_whitespace = false

frontend/bec_atlas/.gitignore

@@ -0,0 +1,42 @@
+# See https://docs.github.com/get-started/getting-started-with-git/ignoring-files for more about ignoring files.
+
+# Compiled output
+/dist
+/tmp
+/out-tsc
+/bazel-out
+
+# Node
+/node_modules
+npm-debug.log
+yarn-error.log
+
+# IDEs and editors
+.idea/
+.project
+.classpath
+.c9/
+*.launch
+.settings/
+*.sublime-workspace
+
+# Visual Studio Code
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+.history/*
+
+# Miscellaneous
+/.angular/cache
+.sass-cache/
+/connect.lock
+/coverage
+/libpeerconnection.log
+testem.log
+/typings
+
+# System files
+.DS_Store
+Thumbs.db

frontend/bec_atlas/README.md

@@ -0,0 +1,59 @@
+# BecAtlas
+
+This project was generated using [Angular CLI](https://github.com/angular/angular-cli) version 19.0.5.
+
+## Development server
+
+To start a local development server, run:
+
+```bash
+ng serve
+```
+
+Once the server is running, open your browser and navigate to `http://localhost:4200/`. The application will automatically reload whenever you modify any of the source files.
+
+## Code scaffolding
+
+Angular CLI includes powerful code scaffolding tools. To generate a new component, run:
+
+```bash
+ng generate component component-name
+```
+
+For a complete list of available schematics (such as `components`, `directives`, or `pipes`), run:
+
+```bash
+ng generate --help
+```
+
+## Building
+
+To build the project run:
+
+```bash
+ng build
+```
+
+This will compile your project and store the build artifacts in the `dist/` directory. By default, the production build optimizes your application for performance and speed.
+
+## Running unit tests
+
+To execute unit tests with the [Karma](https://karma-runner.github.io) test runner, use the following command:
+
+```bash
+ng test
+```
+
+## Running end-to-end tests
+
+For end-to-end (e2e) testing, run:
+
+```bash
+ng e2e
+```
+
+Angular CLI does not come with an end-to-end testing framework by default. You can choose one that suits your needs.
+
+## Additional Resources
+
+For more information on using the Angular CLI, including detailed command references, visit the [Angular CLI Overview and Command Reference](https://angular.dev/tools/cli) page.

frontend/bec_atlas/angular.json

@@ -0,0 +1,104 @@
+{
+  "$schema": "./node_modules/@angular/cli/lib/config/schema.json",
+  "version": 1,
+  "newProjectRoot": "projects",
+  "projects": {
+    "bec_atlas": {
+      "projectType": "application",
+      "schematics": {
+        "@schematics/angular:component": {
+          "style": "scss"
+        }
+      },
+      "root": "",
+      "sourceRoot": "src",
+      "prefix": "app",
+      "architect": {
+        "build": {
+          "builder": "@angular-devkit/build-angular:application",
+          "options": {
+            "outputPath": "dist/bec_atlas",
+            "index": "src/index.html",
+            "browser": "src/main.ts",
+            "polyfills": [
+              "zone.js"
+            ],
+            "tsConfig": "tsconfig.app.json",
+            "inlineStyleLanguage": "scss",
+            "assets": [
+              {
+                "glob": "**/*",
+                "input": "public"
+              }
+            ],
+            "styles": [
+              "@angular/material/prebuilt-themes/cyan-orange.css",
+              "src/styles.scss"
+            ],
+            "scripts": []
+          },
+          "configurations": {
+            "production": {
+              "budgets": [
+                {
+                  "type": "initial",
+                  "maximumWarning": "500kB",
+                  "maximumError": "1MB"
+                },
+                {
+                  "type": "anyComponentStyle",
+                  "maximumWarning": "4kB",
+                  "maximumError": "8kB"
+                }
+              ],
+              "outputHashing": "all"
+            },
+            "development": {
+              "optimization": false,
+              "extractLicenses": false,
+              "sourceMap": true
+            }
+          },
+          "defaultConfiguration": "production"
+        },
+        "serve": {
+          "builder": "@angular-devkit/build-angular:dev-server",
+          "configurations": {
+            "production": {
+              "buildTarget": "bec_atlas:build:production"
+            },
+            "development": {
+              "buildTarget": "bec_atlas:build:development"
+            }
+          },
+          "defaultConfiguration": "development"
+        },
+        "extract-i18n": {
+          "builder": "@angular-devkit/build-angular:extract-i18n"
+        },
+        "test": {
+          "builder": "@angular-devkit/build-angular:karma",
+          "options": {
+            "polyfills": [
+              "zone.js",
+              "zone.js/testing"
+            ],
+            "tsConfig": "tsconfig.spec.json",
+            "inlineStyleLanguage": "scss",
+            "assets": [
+              {
+                "glob": "**/*",
+                "input": "public"
+              }
+            ],
+            "styles": [
+              "@angular/material/prebuilt-themes/cyan-orange.css",
+              "src/styles.scss"
+            ],
+            "scripts": []
+          }
+        }
+      }
+    }
+  }
+}

frontend/bec_atlas/package.json

@@ -0,0 +1,44 @@
+{
+  "name": "bec-atlas",
+  "version": "0.0.0",
+  "scripts": {
+    "ng": "ng",
+    "start": "ng serve",
+    "build": "ng build",
+    "watch": "ng build --watch --configuration development",
+    "test": "ng test"
+  },
+  "private": true,
+  "dependencies": {
+    "@angular/animations": "^19.0.0",
+    "@angular/cdk": "^19.0.4",
+    "@angular/common": "^19.0.0",
+    "@angular/compiler": "^19.0.0",
+    "@angular/core": "^19.0.0",
+    "@angular/forms": "^19.0.0",
+    "@angular/material": "^19.0.4",
+    "@angular/platform-browser": "^19.0.0",
+    "@angular/platform-browser-dynamic": "^19.0.0",
+    "@angular/router": "^19.0.0",
+    "angular-gridster2": "^19.0.0",
+    "gridstack": "^11.1.2",
+    "gridstack-angular": "^0.6.0-dev",
+    "rxjs": "~7.8.0",
+    "socket.io-client": "^4.8.1",
+    "tslib": "^2.3.0",
+    "zone.js": "~0.15.0"
+  },
+  "devDependencies": {
+    "@angular-devkit/build-angular": "^19.0.5",
+    "@angular/cli": "^19.0.5",
+    "@angular/compiler-cli": "^19.0.0",
+    "@types/jasmine": "~5.1.0",
+    "jasmine-core": "~5.4.0",
+    "karma": "~6.4.0",
+    "karma-chrome-launcher": "~3.2.0",
+    "karma-coverage": "~2.2.0",
+    "karma-jasmine": "~5.1.0",
+    "karma-jasmine-html-reporter": "~2.1.0",
+    "typescript": "~5.6.2"
+  }
+}

frontend/bec_atlas/src/app/app.component.html

@@ -0,0 +1,6 @@
+
+
+<!-- <app-dashboard></app-dashboard> -->
+<!-- <app-gridstack-test></app-gridstack-test> -->
+ <app-device-box [device]="'samx'" [signal_name]="'samx'"></app-device-box>
+<router-outlet />

frontend/bec_atlas/src/app/app.component.spec.ts

@@ -0,0 +1,29 @@
+import { TestBed } from '@angular/core/testing';
+import { AppComponent } from './app.component';
+
+describe('AppComponent', () => {
+  beforeEach(async () => {
+    await TestBed.configureTestingModule({
+      imports: [AppComponent],
+    }).compileComponents();
+  });
+
+  it('should create the app', () => {
+    const fixture = TestBed.createComponent(AppComponent);
+    const app = fixture.componentInstance;
+    expect(app).toBeTruthy();
+  });
+
+  it(`should have the 'bec_atlas' title`, () => {
+    const fixture = TestBed.createComponent(AppComponent);
+    const app = fixture.componentInstance;
+    expect(app.title).toEqual('bec_atlas');
+  });
+
+  it('should render title', () => {
+    const fixture = TestBed.createComponent(AppComponent);
+    fixture.detectChanges();
+    const compiled = fixture.nativeElement as HTMLElement;
+    expect(compiled.querySelector('h1')?.textContent).toContain('Hello, bec_atlas');
+  });
+});

frontend/bec_atlas/src/app/app.component.ts

@@ -0,0 +1,25 @@
+import { Component } from '@angular/core';
+import { RouterOutlet } from '@angular/router';
+import { DashboardComponent } from './dashboard/dashboard.component';
+import { GridStackTestComponent } from './gridstack-test/gridstack-test.component';
+import { CommonModule } from '@angular/common';
+import { RedisConnectorService } from './core/redis-connector.service';
+import { DeviceBoxComponent } from './device-box/device-box.component';
+
+@Component({
+  selector: 'app-root',
+  imports: [
+    RouterOutlet,
+    DashboardComponent,
+    CommonModule,
+    GridStackTestComponent,
+    DeviceBoxComponent,
+  ],
+  templateUrl: './app.component.html',
+  styleUrl: './app.component.scss',
+})
+export class AppComponent {
+  title = 'bec_atlas';
+
+  constructor(private redisConnector: RedisConnectorService) {}
+}

frontend/bec_atlas/src/app/app.config.ts

@@ -0,0 +1,17 @@
+import {
+  ApplicationConfig,
+  provideEnvironmentInitializer,
+  provideZoneChangeDetection,
+} from '@angular/core';
+import { provideRouter } from '@angular/router';
+
+import { routes } from './app.routes';
+import { GridstackComponent } from 'gridstack/dist/angular';
+import { provideAnimationsAsync } from '@angular/platform-browser/animations/async';
+
+export const appConfig: ApplicationConfig = {
+  providers: [
+    provideZoneChangeDetection({ eventCoalescing: true }),
+    provideRouter(routes), provideAnimationsAsync(),
+  ],
+};

frontend/bec_atlas/src/app/app.routes.ts

@@ -0,0 +1,3 @@
+import { Routes } from '@angular/router';
+
+export const routes: Routes = [];

frontend/bec_atlas/src/app/core/redis-connector.service.spec.ts

@@ -0,0 +1,16 @@
+import { TestBed } from '@angular/core/testing';
+
+import { RedisConnectorService } from './redis-connector.service';
+
+describe('RedisConnectorService', () => {
+  let service: RedisConnectorService;
+
+  beforeEach(() => {
+    TestBed.configureTestingModule({});
+    service = TestBed.inject(RedisConnectorService);
+  });
+
+  it('should be created', () => {
+    expect(service).toBeTruthy();
+  });
+});

frontend/bec_atlas/src/app/core/redis-connector.service.ts

@@ -0,0 +1,142 @@
+import { Injectable, signal, WritableSignal } from '@angular/core';
+import { io, Socket } from 'socket.io-client';
+import { Observable } from 'rxjs';
+import { MessageEndpoints, EndpointInfo } from './redis_endpoints';
+
+@Injectable({
+  providedIn: 'root',
+})
+export class RedisConnectorService {
+  private socket!: Socket;
+  private signals: Map<string, WritableSignal<any>> = new Map();
+  private signalReferenceCount: Map<string, number> = new Map();
+
+  constructor() {
+    this.connect();
+  }
+
+  /**
+   * Connect to the WebSocket server using socket.io
+   */
+  private connect(): void {
+    this.socket = io('http://localhost', {
+      transports: ['websocket'], // Use WebSocket only
+      autoConnect: true, // Automatically connect
+      reconnection: true, // Enable automatic reconnection
+      timeout: 5000, // Connection timeout in milliseconds
+      auth: {
+        user: 'john_doe',
+        token: '1234',
+        deployment: '67599761f44165e0ad56ce0f',
+      },
+    });
+
+    this.socket.onAny((event, ...args) => {
+      console.log('Received event:', event, 'with data:', args);
+    });
+
+    this.socket.on('connect', () => {
+      console.log('Connected to WebSocket server');
+      // this.register(MessageEndpoints.device_readback('samx'));
+    });
+
+    this.socket.on('message', (data: any) => {
+      console.log('Received message:', data);
+      const dataObj = JSON.parse(data);
+      const signal = this.signals.get(dataObj.endpoint_request);
+      if (signal) {
+        signal.set(dataObj);
+      }
+    });
+
+    this.socket.on('disconnect', (reason: string) => {
+      console.log('Disconnected from WebSocket server:', reason);
+    });
+
+    this.socket.on('connect_error', (error: Error) => {
+      console.error('Connection error:', error);
+    });
+
+    this.socket.on('reconnect_attempt', (attempt: number) => {
+      console.log('Reconnection attempt:', attempt);
+    });
+
+    this.socket.on('error', (error: Error) => {
+      console.error('Socket error:', error);
+    });
+
+    this.socket.on('ping', () => {
+      console.log('Ping received');
+    });
+  }
+
+  /**
+   * Emit an event to the WebSocket server
+   * @param event Event name
+   * @param data Data to send
+   */
+  public emit(event: string, data: any): void {
+    this.socket.emit(event, data);
+  }
+
+  /**
+   * Register an endpoint to listen for events
+   * @param endpoint Endpoint to listen for
+   * @returns Signal for the endpoint
+   */
+  public register(endpoint: EndpointInfo): WritableSignal<any> {
+    // Convert endpoint to string for use as a key
+    const endpoint_str = JSON.stringify(endpoint);
+
+    let endpoint_signal: WritableSignal<any>;
+
+    if (this.signals.has(endpoint_str)) {
+      // If the signal already exists, return it
+      endpoint_signal = this.signals.get(endpoint_str) as WritableSignal<any>;
+    } else {
+      // Otherwise, create a new signal
+      endpoint_signal = signal(null);
+      this.signals.set(endpoint_str, endpoint_signal);
+    }
+
+    const signalReferenceCount =
+      this.signalReferenceCount.get(endpoint_str) || 0;
+
+    if (signalReferenceCount === 0) {
+      // If no references to the signal, register the endpoint
+      this.emit('register', endpoint_str);
+    }
+
+    this.signals.set(endpoint_str, endpoint_signal);
+    this.signalReferenceCount.set(endpoint_str, signalReferenceCount + 1);
+    return endpoint_signal;
+  }
+
+  /**
+   * Listen for an event from the WebSocket server
+   * @param event Event name
+   * @returns Observable for the event data
+   */
+  public on<T>(event: string): Observable<T> {
+    return new Observable<T>((observer) => {
+      this.socket.on(event, (data: T) => {
+        observer.next(data);
+      });
+
+      // Cleanup when unsubscribed
+      return () => {
+        this.socket.off(event);
+      };
+    });
+  }
+
+  /**
+   * Disconnect from the WebSocket server
+   */
+  public disconnect(): void {
+    if (this.socket) {
+      this.socket.disconnect();
+      console.log('Disconnected from WebSocket server');
+    }
+  }
+}

frontend/bec_atlas/src/app/core/redis_endpoints.ts

@@ -0,0 +1,31 @@
+export interface EndpointInfo {
+  endpoint: string;
+  args: Array<string>;
+}
+
+export class MessageEndpoints {
+  /**
+   *
+   * @param device Device name
+   * @returns Endpoint for device readback
+   */
+  static device_readback(device: string): EndpointInfo {
+    const out: EndpointInfo = {
+      endpoint: 'device_readback',
+      args: [device],
+    };
+    return out;
+  }
+
+  /**
+   *
+   * @returns Endpoint for scan segment
+   */
+  static scan_segment(): EndpointInfo {
+    const out: EndpointInfo = {
+      endpoint: 'scan_segment',
+      args: [],
+    };
+    return out;
+  }
+}

frontend/bec_atlas/src/app/dashboard/dashboard.component.html

@@ -0,0 +1,19 @@
+<div>
+  <div class="gridster-container-toolbar">
+    <gridster [options]="toolbarOptions">
+      <gridster-item [item]="item" *ngFor="let item of dashboard; let i = index" class="gridster-item">
+        {{ item }}
+        
+      </gridster-item>
+    </gridster>
+  </div>
+  
+  <div class="gridster-container">
+    <gridster [options]="optionsEdit">
+      <gridster-item [item]="item" *ngFor="let item of dashboard; let i = index" class="gridster-item">
+        {{ item }}
+        
+      </gridster-item>
+    </gridster>
+  </div>
+</div>

frontend/bec_atlas/src/app/dashboard/dashboard.component.scss

@@ -0,0 +1,61 @@
+.gridster-container {
+  height: calc(100vh - 120px);
+  background-color: rgb(37, 94, 75);
+}
+.gridster-container-toolbar {
+  height: 50px;
+  background-color: aqua;
+}
+
+.gridster-container-toolbar .gridster-item {
+  background-color: aqua;
+}
+
+::ng-deep .gridster {
+  background-color: #848484;
+  padding: 2px;
+}
+
+::ng-deep  .gridster-edit{
+  background-color:#ca000093;
+  /* margin-top: 80px; */
+}
+
+::ng-deep  .gridster{
+  background-color:#ca000093;
+  /* margin-top: 80px; */
+}
+::ng-deep  .gridster-edit{
+  background-color:#ca000093;
+  /* margin-top: 80px; */
+}
+
+.gridster-item{
+  padding: 10px 15px;
+  background-color: #848484;
+  color: white;
+  border-radius: 2px; 
+  overflow: hidden;
+}
+
+// .gridster-container{
+//   /* top: 50px; */
+//   padding-bottom: 100px;
+//   padding-right: 20px;
+//   padding-left: 20px;
+
+//   height: calc(100vh - 120px);
+//   width: 100wh;
+//   overflow: hidden;
+// }
+
+.dashboard-item{
+  height:inherit;
+  overflow: inherit;
+  
+}
+
+.dashboard-edit{
+  float: right;
+  height: 36px;
+}

frontend/bec_atlas/src/app/dashboard/dashboard.component.spec.ts

@@ -0,0 +1,23 @@
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { DashboardComponent } from './dashboard.component';
+
+describe('DashboardComponent', () => {
+  let component: DashboardComponent;
+  let fixture: ComponentFixture<DashboardComponent>;
+
+  beforeEach(async () => {
+    await TestBed.configureTestingModule({
+      imports: [DashboardComponent]
+    })
+    .compileComponents();
+
+    fixture = TestBed.createComponent(DashboardComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});

frontend/bec_atlas/src/app/dashboard/dashboard.component.ts

@@ -0,0 +1,145 @@
+import { Component, OnInit } from '@angular/core';
+import { CommonModule } from '@angular/common';
+import {
+  CompactType,
+  DisplayGrid,
+  GridsterComponent,
+  GridsterConfig,
+  GridsterItem,
+  GridsterItemComponent,
+  GridType,
+} from 'angular-gridster2';
+
+@Component({
+  selector: 'app-dashboard',
+  templateUrl: './dashboard.component.html',
+  styleUrl: './dashboard.component.scss',
+  imports: [CommonModule, GridsterItemComponent, GridsterComponent],
+})
+export class DashboardComponent implements OnInit {
+  dashboard: Array<GridsterItem>;
+
+  options: GridsterConfig = {
+    gridType: GridType.Fit,
+    compactType: CompactType.None,
+    margin: 1,
+    outerMargin: true,
+    outerMarginTop: null,
+    outerMarginRight: null,
+    outerMarginBottom: null,
+    outerMarginLeft: null,
+    useTransformPositioning: true,
+    mobileBreakpoint: 640,
+    minCols: 40,
+    maxCols: 40,
+    minRows: 20,
+    maxRows: 20,
+    minColWidth: 300,
+    maxItemCols: 100,
+    minItemCols: 1,
+    maxItemRows: 100,
+    minItemRows: 1,
+    maxItemArea: 2500,
+    minItemArea: 1,
+    defaultItemCols: 1,
+    defaultItemRows: 1,
+    // fixedColWidth: 105,
+    // fixedRowHeight: 105,
+    keepFixedHeightInMobile: false,
+    keepFixedWidthInMobile: false,
+    scrollSensitivity: 50,
+    scrollSpeed: 20,
+    enableEmptyCellClick: false,
+    enableEmptyCellContextMenu: false,
+    enableEmptyCellDrop: false,
+    enableEmptyCellDrag: false,
+    enableOccupiedCellDrop: false,
+    emptyCellDragMaxCols: 50,
+    emptyCellDragMaxRows: 50,
+    ignoreMarginInRow: false,
+    draggable: {
+      enabled: true,
+    },
+    resizable: {
+      enabled: true,
+    },
+    swap: true,
+    pushItems: true,
+    disablePushOnDrag: false,
+    disablePushOnResize: false,
+    pushDirections: { north: true, east: true, south: true, west: true },
+    pushResizeItems: false,
+    displayGrid: DisplayGrid.None,
+    disableWindowResize: false,
+    disableWarnings: false,
+    scrollToNewItems: false,
+  };
+
+  optionsEdit: GridsterConfig;
+  toolbarOptions: GridsterConfig;
+
+  constructor() {
+    this.dashboard = [];
+    this.optionsEdit = JSON.parse(JSON.stringify(this.options));
+    this.toolbarOptions = JSON.parse(JSON.stringify(this.options));
+  }
+
+  ngOnInit(): void {
+    this.optionsEdit = JSON.parse(JSON.stringify(this.options)); // seriously??? I cannot believe that's the only way to perform a deep copy of an object
+    this.optionsEdit.draggable = { enabled: true };
+    this.optionsEdit.resizable = { enabled: true };
+    this.optionsEdit.displayGrid = DisplayGrid.Always;
+    this.toolbarOptions.minCols = 40;
+    this.toolbarOptions.maxCols = 40;
+    this.toolbarOptions.minRows = 1;
+    this.toolbarOptions.maxRows = 1;
+
+    this.dashboard = [
+      { cols: 2, rows: 1, y: 0, x: 0 },
+      { cols: 2, rows: 2, y: 0, x: 2, hasContent: true },
+      { cols: 1, rows: 1, y: 0, x: 4 },
+      { cols: 1, rows: 1, y: 2, x: 5 },
+      { cols: 1, rows: 1, y: 1, x: 0 },
+      { cols: 1, rows: 1, y: 1, x: 0 },
+      {
+        cols: 2,
+        rows: 2,
+        y: 3,
+        x: 5,
+        minItemRows: 2,
+        minItemCols: 2,
+        label: 'Min rows & cols = 2',
+      },
+      {
+        cols: 2,
+        rows: 2,
+        y: 2,
+        x: 0,
+        maxItemRows: 2,
+        maxItemCols: 2,
+        label: 'Max rows & cols = 2',
+      },
+      {
+        cols: 2,
+        rows: 1,
+        y: 2,
+        x: 2,
+        dragEnabled: true,
+        resizeEnabled: true,
+        label: 'Drag&Resize Enabled',
+      },
+      {
+        cols: 1,
+        rows: 1,
+        y: 2,
+        x: 4,
+        dragEnabled: false,
+        resizeEnabled: false,
+        label: 'Drag&Resize Disabled',
+      },
+      { cols: 1, rows: 1, y: 2, x: 6 },
+    ];
+
+    console.log('DashboardComponent initialized');
+  }
+}

frontend/bec_atlas/src/app/device-box/device-box.component.html

@@ -0,0 +1,9 @@
+<div class="device-box">
+    <mat-card appearance="outlined">
+        <mat-card-title>{{ device }}</mat-card-title>
+    
+    <mat-card class="inner-card">
+        <mat-card-content class="center-content">{{ readback_signal() }}</mat-card-content>
+    </mat-card>
+    </mat-card>
+</div>

frontend/bec_atlas/src/app/device-box/device-box.component.scss

@@ -0,0 +1,44 @@
+
+.device-box {
+    max-width: 100px;
+    max-height: 100px;
+}
+.center-content {
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    height: 100%; /* Ensures the mat-card-content takes the full height */
+    text-align: center; /* Centers text within the content */
+}
+
+mat-card {
+    width: 100%; /* Ensure card takes the full width of its container */
+    height: 100%; /* Ensure card takes the full height of its container */
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    background: var(--mat-sys-secondary-container);
+    color: var(--mat-sys-on-secondary-container);
+}
+
+mat-card.inner-card {
+    flex-grow: 1;
+    width: 100%;
+    height: auto; /* Adjust height as needed */
+    border-top-left-radius: 0px; /* Removes border radius */
+    border-top-right-radius: 0px; /* Removes border radius */
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    position: relative;
+    color: var(--mat-sys-on-primary); /* Sets the text color */
+    background: var(--mat-sys-primary) /* Sets the background color */
+}
+
+mat-card-content {
+    text-align: center; /* Ensures text itself is centered */
+}
+
+mat-card-title {
+    font-size: 0.8em; /* Increases the font size of the title */
+}

frontend/bec_atlas/src/app/device-box/device-box.component.spec.ts

@@ -0,0 +1,23 @@
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { DeviceBoxComponent } from './device-box.component';
+
+describe('DeviceBoxComponent', () => {
+  let component: DeviceBoxComponent;
+  let fixture: ComponentFixture<DeviceBoxComponent>;
+
+  beforeEach(async () => {
+    await TestBed.configureTestingModule({
+      imports: [DeviceBoxComponent]
+    })
+    .compileComponents();
+
+    fixture = TestBed.createComponent(DeviceBoxComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});

frontend/bec_atlas/src/app/device-box/device-box.component.ts

@@ -0,0 +1,42 @@
+import { Component, computed, Input, Signal } from '@angular/core';
+import { RedisConnectorService } from '../core/redis-connector.service';
+import { MessageEndpoints } from '../core/redis_endpoints';
+import { MatCardModule } from '@angular/material/card';
+
+@Component({
+  selector: 'app-device-box',
+  imports: [MatCardModule],
+  templateUrl: './device-box.component.html',
+  styleUrl: './device-box.component.scss',
+})
+export class DeviceBoxComponent {
+  signal!: Signal<any>;
+  readback_signal!: Signal<number>;
+
+  @Input()
+  device!: string;
+
+  @Input()
+  signal_name!: string;
+
+  constructor(private redisConnector: RedisConnectorService) {}
+
+  ngOnInit(): void {
+    this.signal = this.redisConnector.register(
+      MessageEndpoints.device_readback(this.device)
+    );
+    this.readback_signal = computed(() => {
+      let data = this.signal();
+      if (!data) {
+        return 'N/A';
+      }
+      if (!data.data.signals[this.signal_name]) {
+        return 'N/A';
+      }
+      if (typeof data.data.signals[this.signal_name].value === 'number') {
+        return data.data.signals[this.signal_name].value.toFixed(2);
+      }
+      return data.data.signals[this.signal_name].value;
+    });
+  }
+}

frontend/bec_atlas/src/app/gridstack-test/gridstack-test.component.html

@@ -0,0 +1,22 @@
+
+<!-- <button (click)="add()">add item</button>
+    <button (click)="delete()">remove item</button>
+    <button (click)="modify()">modify item</button>
+    <button (click)="newLayout()">new layout</button> -->
+<div class="grid-stack">
+    <!-- using angular templating to create DOM, otherwise an easier way is to simply call grid.load(items)
+    NOTE: this example is NOT complete as there are many more properties than listed (minW, maxW, etc....)
+    -->
+    <div
+      *ngFor="let n of items; trackBy: identify"
+      class="grid-stack-item"
+      [attr.gs-id]="n.id"
+      [attr.gs-x]="n.x"
+      [attr.gs-y]="n.y"
+      [attr.gs-w]="n.w"
+      [attr.gs-h]="n.h"
+      #gridStackItem
+    >
+      <div class="grid-stack-item-content">item {{ n.id }}</div>
+    </div>
+  </div>

frontend/bec_atlas/src/app/gridstack-test/gridstack-test.component.scss

@@ -0,0 +1,37 @@
+
+:host {
+    display: block;
+    height: 100vh; /* Full-screen height */
+    width: 100vw; /* Full-screen width */
+    overflow: hidden;
+  }
+
+  .grid-stack {
+    display: block;
+    overflow: hidden;
+    height: 100%;
+    min-height: 100% !important;
+  }
+
+  .grid-stack-item-content {
+    display: flex;
+    justify-content: center;
+    align-items: center;
+    // background: #007bff;
+    // background-color: #18bc9c;
+    color: rgb(24, 7, 7);
+    // border: 1px solid #ddd;
+  }
+  $columns: 20;
+  @function fixed($float) {
+    @return round($float * 1000) / 1000; // total 2+3 digits being %
+  }
+  .gs-#{$columns} > .grid-stack-item {
+  
+    width: fixed(100% / $columns);
+  
+    @for $i from 1 through $columns - 1 {
+      &[gs-x='#{$i}'] { left: fixed((100% / $columns) * $i); }
+      &[gs-w='#{$i+1}'] { width: fixed((100% / $columns) * ($i+1)); }
+    }
+  }

frontend/bec_atlas/src/app/gridstack-test/gridstack-test.component.spec.ts

@@ -0,0 +1,23 @@
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { GridstackTestComponent } from './gridstack-test.component';
+
+describe('GridstackTestComponent', () => {
+  let component: GridstackTestComponent;
+  let fixture: ComponentFixture<GridstackTestComponent>;
+
+  beforeEach(async () => {
+    await TestBed.configureTestingModule({
+      imports: [GridstackTestComponent]
+    })
+    .compileComponents();
+
+    fixture = TestBed.createComponent(GridstackTestComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});

frontend/bec_atlas/src/app/gridstack-test/gridstack-test.component.ts

@@ -0,0 +1,141 @@
+/**
+ * Example using Angular ngFor to loop through items and create DOM items
+ */
+
+import { CommonModule } from '@angular/common';
+import {
+  Component,
+  AfterViewInit,
+  Input,
+  ViewChildren,
+  QueryList,
+  ElementRef,
+} from '@angular/core';
+import {
+  GridItemHTMLElement,
+  GridStack,
+  GridStackNode,
+  GridStackWidget,
+  Utils,
+  GridStackOptions,
+} from 'gridstack';
+
+// unique ids sets for each item for correct ngFor updating
+let ids = 1;
+
+@Component({
+  selector: 'app-gridstack-test',
+  imports: [CommonModule],
+  templateUrl: './gridstack-test.component.html',
+  styleUrls: ['./gridstack-test.component.scss'],
+})
+export class GridStackTestComponent implements AfterViewInit {
+  /** list of HTML items that we track to know when the DOM has been updated to make/remove GS widgets */
+  @ViewChildren('gridStackItem') gridstackItems!: QueryList<
+    ElementRef<GridItemHTMLElement>
+  >;
+
+  /** set the items to display. */
+  @Input() public set items(list: GridStackWidget[]) {
+    this._items = list || [];
+    this._items.forEach((w) => (w.id = w.id || String(ids++))); // make sure a unique id is generated for correct ngFor loop update
+  }
+  public get items(): GridStackWidget[] {
+    return this._items;
+  }
+
+  private grid!: GridStack;
+  public _items!: GridStackWidget[];
+
+  constructor() {
+    this.items = [
+      { x: 0, y: 0 },
+      { x: 1, y: 1 },
+      { x: 2, y: 2 },
+      { x: 2, y: 3 },
+    ];
+  }
+
+  // wait until after DOM is ready to init gridstack - can't be ngOnInit() as angular ngFor needs to run first!
+  public ngAfterViewInit() {
+    const N_ROWS = 30;
+    this.grid = GridStack.init({
+      margin: 0,
+      float: true,
+      animate: true,
+      minRow: 4,
+      maxRow: N_ROWS,
+      cellHeight: 100 / N_ROWS,
+      cellHeightUnit: '%',
+      column: 20,
+      alwaysShowResizeHandle: true,
+    }).on('change added', (event: Event, nodes: GridStackNode[]) =>
+      this.onChange(nodes)
+    );
+
+    // sync initial actual valued rendered (in case init() had to merge conflicts)
+    this.onChange();
+
+    this.gridstackItems.changes.subscribe(() => {
+      const layout: GridStackWidget[] = [];
+      this.gridstackItems.forEach((ref) => {
+        const n =
+          ref.nativeElement.gridstackNode ||
+          this.grid.makeWidget(ref.nativeElement).gridstackNode;
+        if (n) layout.push(n);
+      });
+      this.grid.load(layout); // efficient that does diffs only
+    });
+  }
+
+  /** Optional: called when given widgets are changed (moved/resized/added) - update our list to match.
+   * Note this is not strictly necessary as demo works without this
+   */
+  public onChange(list = this.grid.engine.nodes) {
+    setTimeout(
+      () =>
+        // prevent new 'added' items from ExpressionChangedAfterItHasBeenCheckedError. TODO: find cleaner way to sync outside Angular change detection ?
+        list.forEach((n) => {
+          const item = this._items.find((i) => i.id === n.id);
+          if (item) Utils.copyPos(item, n);
+        }),
+      0
+    );
+  }
+
+  /**
+   * CRUD operations
+   */
+  public add() {
+    // new array isn't required as Angular seem to detect changes to content
+    // this.items = [...this.items, { x:3, y:0, w:3, id:String(ids++) }];
+    this.items.push({ x: 3, y: 0, w: 3, id: String(ids++) });
+  }
+
+  public delete() {
+    this.items.pop();
+  }
+
+  public modify() {
+    // this will only update the DOM attr (from the ngFor loop in our template above)
+    // but not trigger gridstackItems.changes for GS to auto-update, so call GS update() instead
+    // this.items[0].w = 2;
+    const n = this.grid.engine.nodes[0];
+    if (n?.el) this.grid.update(n.el, { w: 3 });
+  }
+
+  public newLayout() {
+    this.items = [
+      // test updating existing and creating new one
+      { x: 0, y: 1, id: '1' },
+      { x: 1, y: 1, id: '2' },
+      // {x:2, y:1, id:3}, // delete item
+      { x: 3, y: 0, w: 3 }, // new item
+    ];
+  }
+
+  // ngFor unique node id to have correct match between our items used and GS
+  identify(index: number, w: GridStackWidget) {
+    return w.id;
+  }
+}

frontend/bec_atlas/src/index.html

@@ -0,0 +1,15 @@
+<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <title>BecAtlas</title>
+  <base href="/">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <link rel="icon" type="image/x-icon" href="favicon.ico">
+  <link href="https://fonts.googleapis.com/css2?family=Roboto:wght@300;400;500&display=swap" rel="stylesheet">
+  <link href="https://fonts.googleapis.com/icon?family=Material+Icons" rel="stylesheet">
+</head>
+<body class="mat-typography">
+  <app-root></app-root>
+</body>
+</html>

frontend/bec_atlas/src/main.ts

@@ -0,0 +1,6 @@
+import { bootstrapApplication } from '@angular/platform-browser';
+import { appConfig } from './app/app.config';
+import { AppComponent } from './app/app.component';
+
+bootstrapApplication(AppComponent, appConfig)
+  .catch((err) => console.error(err));

frontend/bec_atlas/src/styles.scss

@@ -0,0 +1,45 @@
+/* You can add global styles to this file, and also import other style files */
+@use '@angular/material' as mat;
+@import "gridstack/dist/gridstack.min.css";
+@import "gridstack/dist/gridstack-extra.min.css";
+
+// gridstack {
+//     display: grid;
+//     width: 100%; /* Ensure the grid uses the full width */
+//     height: 100%; /* Ensure grid height is sufficient */
+//   }
+  
+//   gridstack-item {
+//     display: block;
+//   }
+  
+html, body { height: 100%; }
+body { margin: 0; font-family: Roboto, "Helvetica Neue", sans-serif; }
+
+
+
+html {
+    color-scheme: light dark;
+  
+    // Light theme
+    @media (prefers-color-scheme: light) {
+      @include mat.theme((
+        color: mat.$violet-palette,
+        typography: Roboto,
+        density: 0
+      ), $overrides: (
+        primary-container: orange, // Light-specific override
+      ));
+    }
+  
+    // Dark theme
+    @media (prefers-color-scheme: dark) {
+      @include mat.theme((
+        color: mat.$violet-palette,
+        typography: Roboto,
+        density: 0
+      ), $overrides: (
+        primary-container: darkorange, // Dark-specific override
+      ));
+    }
+  }

frontend/bec_atlas/tsconfig.app.json

@@ -0,0 +1,15 @@
+/* To learn more about Typescript configuration file: https://www.typescriptlang.org/docs/handbook/tsconfig-json.html. */
+/* To learn more about Angular compiler options: https://angular.dev/reference/configs/angular-compiler-options. */
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "outDir": "./out-tsc/app",
+    "types": []
+  },
+  "files": [
+    "src/main.ts"
+  ],
+  "include": [
+    "src/**/*.d.ts"
+  ]
+}

frontend/bec_atlas/tsconfig.json

@@ -0,0 +1,27 @@
+/* To learn more about Typescript configuration file: https://www.typescriptlang.org/docs/handbook/tsconfig-json.html. */
+/* To learn more about Angular compiler options: https://angular.dev/reference/configs/angular-compiler-options. */
+{
+  "compileOnSave": false,
+  "compilerOptions": {
+    "outDir": "./dist/out-tsc",
+    "strict": true,
+    "noImplicitOverride": true,
+    "noPropertyAccessFromIndexSignature": true,
+    "noImplicitReturns": true,
+    "noFallthroughCasesInSwitch": true,
+    "skipLibCheck": true,
+    "isolatedModules": true,
+    "esModuleInterop": true,
+    "experimentalDecorators": true,
+    "moduleResolution": "bundler",
+    "importHelpers": true,
+    "target": "ES2022",
+    "module": "ES2022"
+  },
+  "angularCompilerOptions": {
+    "enableI18nLegacyMessageIdFormat": false,
+    "strictInjectionParameters": true,
+    "strictInputAccessModifiers": true,
+    "strictTemplates": true
+  }
+}

frontend/bec_atlas/tsconfig.spec.json

@@ -0,0 +1,15 @@
+/* To learn more about Typescript configuration file: https://www.typescriptlang.org/docs/handbook/tsconfig-json.html. */
+/* To learn more about Angular compiler options: https://angular.dev/reference/configs/angular-compiler-options. */
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "outDir": "./out-tsc/spec",
+    "types": [
+      "jasmine"
+    ]
+  },
+  "include": [
+    "src/**/*.spec.ts",
+    "src/**/*.d.ts"
+  ]
+}