From 51c4e2f971d41131d1b6dbc01fb0dd6fb3eb028a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mose=20M=C3=BCller?= Date: Thu, 2 Nov 2023 18:21:43 +0100 Subject: [PATCH 1/5] adds StateManager --- src/pydase/data_service/state_manager.py | 146 +++++++++++++++++++++++ src/pydase/utils/helpers.py | 84 +++++++++++++ 2 files changed, 230 insertions(+) create mode 100644 src/pydase/data_service/state_manager.py diff --git a/src/pydase/data_service/state_manager.py b/src/pydase/data_service/state_manager.py new file mode 100644 index 0000000..b6c3bc5 --- /dev/null +++ b/src/pydase/data_service/state_manager.py @@ -0,0 +1,146 @@ +import json +import logging +import os +from typing import TYPE_CHECKING, Any, cast + +import pydase.units as u +from pydase.utils.helpers import ( + generate_paths_from_DataService_dict, + get_nested_value_from_DataService_by_path_and_key, + set_nested_value_in_dict, +) + +if TYPE_CHECKING: + from pydase import DataService + +logger = logging.getLogger(__name__) + + +class StateManager: + """ + Manages the state of a DataService instance, serving as both a cache and a + persistence layer. It is designed to provide quick access to the latest known state + for newly connecting web clients without the need for expensive property accesses + that may involve complex calculations or I/O operations. + + The StateManager listens for state change notifications from the DataService's + callback manager and updates its cache accordingly. This cache does not always + reflect the most current complex property states but rather retains the value from + the last known state, optimizing for performance and reducing the load on the + system. + + While the StateManager ensures that the cached state is as up-to-date as possible, + it does not autonomously update complex properties of the DataService. Such + properties must be updated programmatically, for instance, by invoking specific + tasks or methods that trigger the necessary operations to refresh their state. + + The cached state maintained by the StateManager is particularly useful for web + clients that connect to the system and need immediate access to the current state of + the DataService. By avoiding direct and potentially costly property accesses, the + StateManager provides a snapshot of the DataService's state that is sufficiently + accurate for initial rendering and interaction. + + Attributes: + cache (dict[str, Any]): + A dictionary cache of the DataService's state. + filename (str): + The file name used for storing the DataService's state. + service (DataService): + The DataService instance whose state is being managed. + + Note: + The StateManager's cache updates are triggered by notifications and do not + include autonomous updates of complex DataService properties, which must be + managed programmatically. The cache serves the purpose of providing immediate + state information to web clients, reflecting the state after the last property + update. + """ + + def __init__(self, service: "DataService"): + self.cache: dict[str, Any] = {} # Initialize an empty cache + self.filename = service._filename + self.service = service + self.service._callback_manager.add_notification_callback(self.update_cache) + + def update_cache(self, parent_path: str, name: str, value: Any) -> None: + # Remove the part before the first "." in the parent_path + parent_path = ".".join(parent_path.split(".")[1:]) + + # Construct the full path + full_path = f"{parent_path}.{name}" if parent_path else name + + set_nested_value_in_dict(self.cache, full_path, value) + + def save_state(self) -> None: + """ + Serialize the DataService instance and write it to a JSON file. + + Args: + filename (str): The name of the file to write to. + """ + if self.filename is not None: + with open(self.filename, "w") as f: + json.dump(self.cache, f, indent=4) + else: + logger.error( + f"Class {self.__class__.__name__} was not initialised with a filename. " + 'Skipping "write_to_file"...' + ) + + def load_state(self) -> None: + # Traverse the serialized representation and set the attributes of the class + if self.cache == {}: + self.cache = self.service.serialize() + + json_dict = self._load_state_from_file() + if json_dict == {}: + logger.debug("Could not load the service state.") + return + + serialized_class = self.cache + for path in generate_paths_from_DataService_dict(json_dict): + value = get_nested_value_from_DataService_by_path_and_key( + json_dict, path=path + ) + value_type = get_nested_value_from_DataService_by_path_and_key( + json_dict, path=path, key="type" + ) + class_value_type = get_nested_value_from_DataService_by_path_and_key( + serialized_class, path=path, key="type" + ) + if class_value_type == value_type: + class_attr_is_read_only = ( + get_nested_value_from_DataService_by_path_and_key( + serialized_class, path=path, key="readonly" + ) + ) + if class_attr_is_read_only: + logger.debug( + f'Attribute "{path}" is read-only. Ignoring value from JSON ' + "file..." + ) + continue + # Split the path into parts + parts = path.split(".") + attr_name = parts[-1] + + # Convert dictionary into Quantity + if class_value_type == "Quantity": + value = u.convert_to_quantity(value) + + self.service.update_DataService_attribute(parts[:-1], attr_name, value) + else: + logger.info( + f'Attribute type of "{path}" changed from "{value_type}" to ' + f'"{class_value_type}". Ignoring value from JSON file...' + ) + + def _load_state_from_file(self) -> dict[str, Any]: + if self.filename is not None: + # Check if the file specified by the filename exists + if os.path.exists(self.filename): + with open(self.filename, "r") as f: + # Load JSON data from file and update class attributes with these + # values + return cast(dict[str, Any], json.load(f)) + return {} diff --git a/src/pydase/utils/helpers.py b/src/pydase/utils/helpers.py index 8d2d053..14c327f 100644 --- a/src/pydase/utils/helpers.py +++ b/src/pydase/utils/helpers.py @@ -272,6 +272,90 @@ def get_nested_value_from_DataService_by_path_and_key( return current_data.get(key, None) +def set_nested_value_in_dict(data_dict: dict[str, Any], path: str, value: Any) -> None: + """ + Set the value associated with a specific key in a dictionary given a path. + + This function traverses the dictionary according to the path provided and + sets the value at that path. The path is a string with dots connecting + the levels and brackets indicating list indices. + + Args: + cache (dict): The cache dictionary to set the value in. + path (str): The path to where the value should be set in the dictionary. + value (Any): The value to be set at the specified path in the dictionary. + + Examples: + Let's consider the following dictionary: + + cache = { + "attr1": {"type": "int", "value": 10}, + "attr2": { + "type": "MyClass", + "value": {"attr3": {"type": "float", "value": 20.5}} + } + } + + The function can be used to set the value of 'attr1' as follows: + set_nested_value_in_cache(cache, "attr1", 15) + + It can also be used to set the value of 'attr3', which is nested within 'attr2', + as follows: + set_nested_value_in_cache(cache, "attr2.attr3", 25.0) + """ + + parts = path.split(".") + current_dict: dict[str, Any] = data_dict + index: Optional[int] = None + + for attr_name in parts: + # Check if the key contains an index part like '[]' + if "[" in attr_name and attr_name.endswith("]"): + attr_name, index_part = attr_name.split("[", 1) + index_part = index_part.rstrip("]") # remove the closing bracket + + # Convert the index part to an integer + if index_part.isdigit(): + index = int(index_part) + else: + logger.error(f"Invalid index format in key: {attr_name}") + + current_dict = cast(dict[str, Any], current_dict.get(attr_name, None)) + + if not isinstance(current_dict, dict): + # key does not exist in dictionary, e.g. when class does not have this + # attribute + return + + if index is not None: + if 0 <= index < len(current_dict["value"]): + try: + current_dict = cast(dict[str, Any], current_dict["value"][index]) + except Exception as e: + logger.error(f"Could not change {path}. Exception: {e}") + return + else: + # TODO: appending to a list will probably be done here + logger.error(f"Could not change {path}...") + return + + # When the attribute is a class instance, the attributes are nested in the + # "value" key + if ( + current_dict["type"] not in STANDARD_TYPES + and current_dict["type"] != "method" + ): + current_dict = cast(dict[str, Any], current_dict.get("value", None)) # type: ignore + + index = None + + # setting the new value + try: + current_dict["value"] = value + except Exception as e: + logger.error(e) + + def convert_arguments_to_hinted_types( args: dict[str, Any], type_hints: dict[str, Any] ) -> dict[str, Any] | str: From 6ab11394fa302fdc9d6f8dfd60c02d6061f419a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mose=20M=C3=BCller?= Date: Thu, 2 Nov 2023 18:22:32 +0100 Subject: [PATCH 2/5] using StateManger in DataService --- .../data_service/abstract_data_service.py | 11 +++++--- src/pydase/data_service/data_service.py | 27 +++++-------------- src/pydase/server/server.py | 5 ++-- src/pydase/server/web_server.py | 2 +- 4 files changed, 18 insertions(+), 27 deletions(-) diff --git a/src/pydase/data_service/abstract_data_service.py b/src/pydase/data_service/abstract_data_service.py index 88683ef..4a5d59b 100644 --- a/src/pydase/data_service/abstract_data_service.py +++ b/src/pydase/data_service/abstract_data_service.py @@ -1,16 +1,19 @@ from __future__ import annotations from abc import ABC -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Optional if TYPE_CHECKING: - from .callback_manager import CallbackManager - from .data_service import DataService - from .task_manager import TaskManager + from pydase.data_service.callback_manager import CallbackManager + from pydase.data_service.data_service import DataService + from pydase.data_service.state_manager import StateManager + from pydase.data_service.task_manager import TaskManager class AbstractDataService(ABC): __root__: DataService _task_manager: TaskManager _callback_manager: CallbackManager + _state_manager: StateManager _autostart_tasks: dict[str, tuple[Any]] + _filename: Optional[str] diff --git a/src/pydase/data_service/data_service.py b/src/pydase/data_service/data_service.py index eee31ec..0514908 100644 --- a/src/pydase/data_service/data_service.py +++ b/src/pydase/data_service/data_service.py @@ -9,6 +9,7 @@ import rpyc import pydase.units as u from pydase.data_service.abstract_data_service import AbstractDataService from pydase.data_service.callback_manager import CallbackManager +from pydase.data_service.state_manager import StateManager from pydase.data_service.task_manager import TaskManager from pydase.utils.helpers import ( convert_arguments_to_hinted_types, @@ -41,8 +42,10 @@ def process_callable_attribute(attr: Any, args: dict[str, Any]) -> Any: class DataService(rpyc.Service, AbstractDataService): def __init__(self, filename: Optional[str] = None) -> None: + self._filename: Optional[str] = filename self._callback_manager: CallbackManager = CallbackManager(self) self._task_manager = TaskManager(self) + self._state_manager = StateManager(self) if not hasattr(self, "_autostart_tasks"): self._autostart_tasks = {} @@ -51,12 +54,11 @@ class DataService(rpyc.Service, AbstractDataService): """Keep track of the root object. This helps to filter the emission of notifications.""" - self._filename: Optional[str] = filename - self._callback_manager.register_callbacks() self.__check_instance_classes() self._initialised = True - self._load_values_from_json() + + self._state_manager.load_state() def __setattr__(self, __name: str, __value: Any) -> None: # converting attributes that are not properties @@ -129,15 +131,6 @@ class DataService(rpyc.Service, AbstractDataService): # allow all other attributes setattr(self, name, value) - def _load_values_from_json(self) -> None: - if self._filename is not None: - # Check if the file specified by the filename exists - if os.path.exists(self._filename): - with open(self._filename, "r") as f: - # Load JSON data from file and update class attributes with these - # values - self.load_DataService_from_JSON(cast(dict[str, Any], json.load(f))) - def write_to_file(self) -> None: """ Serialize the DataService instance and write it to a JSON file. @@ -145,14 +138,8 @@ class DataService(rpyc.Service, AbstractDataService): Args: filename (str): The name of the file to write to. """ - if self._filename is not None: - with open(self._filename, "w") as f: - json.dump(self.serialize(), f, indent=4) - else: - logger.error( - f"Class {self.__class__.__name__} was not initialised with a filename. " - 'Skipping "write_to_file"...' - ) + if self._state_manager is not None: + self._state_manager.save_state() def load_DataService_from_JSON(self, json_dict: dict[str, Any]) -> None: # Traverse the serialized representation and set the attributes of the class diff --git a/src/pydase/server/server.py b/src/pydase/server/server.py index 6dc7dd0..5402e68 100644 --- a/src/pydase/server/server.py +++ b/src/pydase/server/server.py @@ -17,6 +17,7 @@ from uvicorn.server import HANDLED_SIGNALS import pydase.units as u from pydase import DataService +from pydase.data_service.state_manager import StateManager from pydase.version import __version__ from .web_server import WebAPI @@ -323,8 +324,8 @@ class Server: logger.info("Shutting down") logger.info(f"Saving data to {self._service._filename}.") - if self._service._filename is not None: - self._service.write_to_file() + if self._service._state_manager is not None: + self._service._state_manager.save_state() await self.__cancel_servers() await self.__cancel_tasks() diff --git a/src/pydase/server/web_server.py b/src/pydase/server/web_server.py index 1ac6152..647d2c1 100644 --- a/src/pydase/server/web_server.py +++ b/src/pydase/server/web_server.py @@ -114,7 +114,7 @@ class WebAPI: @app.get("/service-properties") def service_properties() -> dict[str, Any]: - return self.service.serialize() + return self.service._state_manager.cache # exposing custom.css file provided by user if self.css is not None: From a837e1bce8a6d5c68542ff13987c48638a091a64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mose=20M=C3=BCller?= Date: Thu, 2 Nov 2023 18:25:55 +0100 Subject: [PATCH 3/5] removing unused imports --- src/pydase/data_service/data_service.py | 4 +--- src/pydase/server/server.py | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/pydase/data_service/data_service.py b/src/pydase/data_service/data_service.py index 0514908..a4dcea4 100644 --- a/src/pydase/data_service/data_service.py +++ b/src/pydase/data_service/data_service.py @@ -1,8 +1,6 @@ -import json import logging -import os from enum import Enum -from typing import Any, Optional, cast, get_type_hints +from typing import Any, Optional, get_type_hints import rpyc diff --git a/src/pydase/server/server.py b/src/pydase/server/server.py index 5402e68..e2ce365 100644 --- a/src/pydase/server/server.py +++ b/src/pydase/server/server.py @@ -17,7 +17,6 @@ from uvicorn.server import HANDLED_SIGNALS import pydase.units as u from pydase import DataService -from pydase.data_service.state_manager import StateManager from pydase.version import __version__ from .web_server import WebAPI From 621bed94afd8e1bc92c35f791b560ab89d6da818 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mose=20M=C3=BCller?= Date: Fri, 3 Nov 2023 08:50:03 +0100 Subject: [PATCH 4/5] removing unnecessary console log commands --- frontend/src/components/DataServiceComponent.tsx | 2 -- 1 file changed, 2 deletions(-) diff --git a/frontend/src/components/DataServiceComponent.tsx b/frontend/src/components/DataServiceComponent.tsx index ae4d38e..2146c16 100644 --- a/frontend/src/components/DataServiceComponent.tsx +++ b/frontend/src/components/DataServiceComponent.tsx @@ -28,9 +28,7 @@ export const DataServiceComponent = React.memo( if (name) { fullAccessPath = parentPath.concat('.' + name); } - console.log(fullAccessPath); const id = getIdFromFullAccessPath(fullAccessPath); - console.log(id); return (
From 55ce32e105a74e0c56d1b139725fd066f3ba1892 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mose=20M=C3=BCller?= Date: Fri, 3 Nov 2023 08:50:33 +0100 Subject: [PATCH 5/5] fix: only update connection toast to reconnecting when still disconnected --- frontend/src/App.tsx | 7 +++++-- src/pydase/frontend/asset-manifest.json | 6 +++--- src/pydase/frontend/index.html | 2 +- .../static/js/{main.964bc334.js => main.16698eff.js} | 6 +++--- ...64bc334.js.LICENSE.txt => main.16698eff.js.LICENSE.txt} | 0 src/pydase/frontend/static/js/main.16698eff.js.map | 1 + src/pydase/frontend/static/js/main.964bc334.js.map | 1 - 7 files changed, 13 insertions(+), 10 deletions(-) rename src/pydase/frontend/static/js/{main.964bc334.js => main.16698eff.js} (97%) rename src/pydase/frontend/static/js/{main.964bc334.js.LICENSE.txt => main.16698eff.js.LICENSE.txt} (100%) create mode 100644 src/pydase/frontend/static/js/main.16698eff.js.map delete mode 100644 src/pydase/frontend/static/js/main.964bc334.js.map diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index b52e4b9..9a04631 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -109,7 +109,6 @@ const reducer = (state: State, action: Action): State => { throw new Error(); } }; - const App = () => { const [state, dispatch] = useReducer(reducer, null); const stateRef = useRef(state); // Declare a reference to hold the current state @@ -150,7 +149,11 @@ const App = () => { socket.on('disconnect', () => { setConnectionStatus('disconnected'); setTimeout(() => { - setConnectionStatus('reconnecting'); + // Only set "reconnecting" is the state is still "disconnected" + // E.g. when the client has already reconnected + setConnectionStatus((currentState) => + currentState === 'disconnected' ? 'reconnecting' : currentState + ); }, 2000); }); diff --git a/src/pydase/frontend/asset-manifest.json b/src/pydase/frontend/asset-manifest.json index ff1c535..30118a5 100644 --- a/src/pydase/frontend/asset-manifest.json +++ b/src/pydase/frontend/asset-manifest.json @@ -1,13 +1,13 @@ { "files": { "main.css": "/static/css/main.c444b055.css", - "main.js": "/static/js/main.964bc334.js", + "main.js": "/static/js/main.16698eff.js", "index.html": "/index.html", "main.c444b055.css.map": "/static/css/main.c444b055.css.map", - "main.964bc334.js.map": "/static/js/main.964bc334.js.map" + "main.16698eff.js.map": "/static/js/main.16698eff.js.map" }, "entrypoints": [ "static/css/main.c444b055.css", - "static/js/main.964bc334.js" + "static/js/main.16698eff.js" ] } \ No newline at end of file diff --git a/src/pydase/frontend/index.html b/src/pydase/frontend/index.html index a64c694..a65539d 100644 --- a/src/pydase/frontend/index.html +++ b/src/pydase/frontend/index.html @@ -1 +1 @@ -pydase App
\ No newline at end of file +pydase App
\ No newline at end of file diff --git a/src/pydase/frontend/static/js/main.964bc334.js b/src/pydase/frontend/static/js/main.16698eff.js similarity index 97% rename from src/pydase/frontend/static/js/main.964bc334.js rename to src/pydase/frontend/static/js/main.16698eff.js index f48699c..6069635 100644 --- a/src/pydase/frontend/static/js/main.964bc334.js +++ b/src/pydase/frontend/static/js/main.16698eff.js @@ -1,3 +1,3 @@ -/*! For license information please see main.964bc334.js.LICENSE.txt */ -!function(){var e={694:function(e,t){var n;!function(){"use strict";var r={}.hasOwnProperty;function a(){for(var e=[],t=0;t