usov_i-org/bec
1
0
Fork 0
mirror of https://github.com/ivan-usov-org/bec.git synced 2025-04-21 18:20:01 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix: bugfix within scibec metadata handler to accomodate changes of metadata

Browse Source
This commit is contained in:
appel_c 2024-07-03 11:01:16 +02:00
parent 390fde56ec
commit eef2764f44
6 changed files with 26 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bec_lib/bec_lib/messages.py
View File

@ -573,6 +573,7 @@ class FileContentMessage(BECMessage):
Args: Args:
file_path (str): Path to the file. file_path (str): Path to the file.
data (str): Content of the file. data (str): Content of the file.
scan_info (dict): Scan information.
metadata (dict, optional): Status metadata. Defaults to None. metadata (dict, optional): Status metadata. Defaults to None.
""" """
@ -580,6 +581,7 @@ class FileContentMessage(BECMessage):
msg_type: ClassVar[str] = "file_content_message" msg_type: ClassVar[str] = "file_content_message"
file_path: str file_path: str
data: dict data: dict
scan_info: dict
class VariableMessage(BECMessage): class VariableMessage(BECMessage):

4
bec_lib/tests/test_bec_messages.py
View File

@ -324,7 +324,9 @@ def test_wrong_DAPRequestMessage():
def test_FileContentMessage(): def test_FileContentMessage():
msg = messages.FileContentMessage(file_path="/path/to/file", data={}, metadata={"RID": "1234"}) msg = messages.FileContentMessage(
file_path="/path/to/file", data={}, scan_info={}, metadata={"RID": "1234"}
)
res = MsgpackSerialization.dumps(msg) res = MsgpackSerialization.dumps(msg)
res_loaded = MsgpackSerialization.loads(res) res_loaded = MsgpackSerialization.loads(res)
assert res_loaded == msg assert res_loaded == msg

2
bec_server/bec_server/file_writer/file_writer.py
View File

@ -265,7 +265,7 @@ class HDF5FileWriter:
file_data = {} file_data = {}
for key, val in device_storage.items(): for key, val in device_storage.items():
file_data[key] = val if not isinstance(val, list) else merge_dicts(val) file_data[key] = val if not isinstance(val, list) else merge_dicts(val)
msg_data = {"file_path": file_path, "data": file_data} msg_data = {"file_path": file_path, "data": file_data, "scan_info": info_storage}
msg = messages.FileContentMessage(**msg_data) msg = messages.FileContentMessage(**msg_data)
self.file_writer_manager.connector.set_and_publish(MessageEndpoints.file_content(), msg) self.file_writer_manager.connector.set_and_publish(MessageEndpoints.file_content(), msg)

6
bec_server/bec_server/scihub/scibec/config_handler.py
View File

@ -68,7 +68,11 @@ class ConfigHandler:
config = msg.content["config"] config = msg.content["config"]
scibec = self.scibec_connector.scibec scibec = self.scibec_connector.scibec
logger.debug(self.scibec_connector.scibec_info) logger.debug(self.scibec_connector.scibec_info)
experiment = self.scibec_connector.scibec_info.get("beamline", {}).get("activeExperiment") beamline = self.scibec_connector.scibec_info.get("beamline", {})
if beamline:
experiments = beamline.experiments
else:
experiments = None
msg.metadata["updated_config"] = False msg.metadata["updated_config"] = False
for name, device in config.items(): for name, device in config.items():

9
bec_server/bec_server/scihub/scibec/scibec_metadata_handler.py
View File

@ -168,22 +168,23 @@ class SciBecMetadataHandler:
return self.serialize_special_data(data.tolist()) return self.serialize_special_data(data.tolist())
return json_ext.dumps(data) return json_ext.dumps(data)
def update_scan_data(self, file_path: str, data: dict): def update_scan_data(self, file_path: str, data: dict, scan_info: dict):
""" """
Update the scan data in SciBec Update the scan data in SciBec
Args: Args:
file_path(str): The path to the original NeXuS file file_path(str): The path to the original NeXuS file
data(dict): The scan data data(dict): The scan data
scan_info(dict): The scan information
""" """
scibec = self.scibec_connector.scibec scibec = self.scibec_connector.scibec
if not scibec: if not scibec:
return return
scan_filter = py_scibec.bec.ScanFilterWhere(where={"scanId": data["metadata"]["scan_id"]}) scan_filter = py_scibec.bec.ScanFilterWhere(where={"scanId": scan_info["bec"]["scan_id"]})
scan = scibec.scan.scan_controller_find(scan_filter) scan = scibec.scan.scan_controller_find(scan_filter)
if not scan: if not scan:
logger.warning( logger.warning(
f"Could not find scan with scan_id {data['metadata']['scan_id']}. Cannot write scan" f"Could not find scan with scan_id {scan_info['bec']['scan_id']}. Cannot write scan"
" data to SciBec." " data to SciBec."
) )
return return
@ -211,7 +212,7 @@ class SciBecMetadataHandler:
) )
scibec.scan_data.scan_data_controller_create_many(new_scan_data) scibec.scan_data.scan_data_controller_create_many(new_scan_data)
logger.info( logger.info(
f"Wrote scan data to SciBec for scan_id {data['metadata']['scan_id']} in {time.time() - start} seconds." f"Wrote scan data to SciBec for scan_id {scan_info['bec']['scan_id']} in {time.time() - start} seconds."
) )
def _write_scan_data_chunks( def _write_scan_data_chunks(

18
bec_server/tests/tests_scihub/test_scibec_metadata_handler.py
View File

@ -79,7 +79,7 @@ def test_update_scan_status_patch(md_handler, active_experiment, scan_document):
def test_handle_file_content(md_handler): def test_handle_file_content(md_handler):
# pylint: disable=protected-access # pylint: disable=protected-access
msg = messages.FileContentMessage(file_path="my_file.h5", data={"data": {}}) msg = messages.FileContentMessage(file_path="my_file.h5", data={"data": {}}, scan_info={})
msg_raw = MessageObject(value=msg, topic="file_content") msg_raw = MessageObject(value=msg, topic="file_content")
with mock.patch.object(md_handler, "update_scan_data") as mock_update_scan_data: with mock.patch.object(md_handler, "update_scan_data") as mock_update_scan_data:
md_handler._handle_file_content(msg_raw, parent=md_handler) md_handler._handle_file_content(msg_raw, parent=md_handler)
@ -88,7 +88,7 @@ def test_handle_file_content(md_handler):
def test_handle_file_content_ignores_errors(md_handler): def test_handle_file_content_ignores_errors(md_handler):
# pylint: disable=protected-access # pylint: disable=protected-access
msg = messages.FileContentMessage(file_path="my_file.h5", data={"data": {}}) msg = messages.FileContentMessage(file_path="my_file.h5", data={"data": {}}, scan_info={})
msg_raw = MessageObject(value=msg, topic="file_content") msg_raw = MessageObject(value=msg, topic="file_content")
with mock.patch("bec_server.scihub.scibec.scibec_metadata_handler.logger") as mock_logger: with mock.patch("bec_server.scihub.scibec.scibec_metadata_handler.logger") as mock_logger:
with mock.patch.object(md_handler, "update_scan_data") as mock_update_scan_data: with mock.patch.object(md_handler, "update_scan_data") as mock_update_scan_data:
@ -103,7 +103,7 @@ def test_handle_file_content_ignores_errors(md_handler):
def test_update_scan_data_return_without_scibec(md_handler): def test_update_scan_data_return_without_scibec(md_handler):
# pylint: disable=protected-access # pylint: disable=protected-access
md_handler.scibec_connector.scibec = None md_handler.scibec_connector.scibec = None
md_handler.update_scan_data(file_path="my_file.h5", data={"data": {}}) md_handler.update_scan_data(file_path="my_file.h5", data={"data": {}}, scan_info={})
def test_update_scan_data_without_scan(md_handler): def test_update_scan_data_without_scan(md_handler):
@ -112,7 +112,7 @@ def test_update_scan_data_without_scan(md_handler):
md_handler.scibec_connector.scibec = scibec md_handler.scibec_connector.scibec = scibec
scibec.scan.scan_controller_find = mock.MagicMock(return_value=[]) scibec.scan.scan_controller_find = mock.MagicMock(return_value=[])
md_handler.update_scan_data( md_handler.update_scan_data(
file_path="my_file.h5", data={"data": {}, "metadata": {"scan_id": "scan_id"}} file_path="my_file.h5", data={"data": {}}, scan_info={"bec": {"scan_id": "scan_id"}}
) )
@ -122,7 +122,7 @@ def test_update_scan_data(md_handler, scan_document):
md_handler.scibec_connector.scibec = scibec md_handler.scibec_connector.scibec = scibec
scibec.scan.scan_controller_find = mock.MagicMock(return_value=[scan_document]) scibec.scan.scan_controller_find = mock.MagicMock(return_value=[scan_document])
md_handler.update_scan_data( md_handler.update_scan_data(
file_path="my_file.h5", data={"data": {}, "metadata": {"scan_id": "scan_id"}} file_path="my_file.h5", data={"data": {}}, scan_info={"bec": {"scan_id": "scan_id"}}
) )
scibec.scan_data.scan_data_controller_create_many.assert_called_once_with( scibec.scan_data.scan_data_controller_create_many.assert_called_once_with(
NewScanData( NewScanData(
@ -132,7 +132,8 @@ def test_update_scan_data(md_handler, scan_document):
"owner": ["owner"], "owner": ["owner"],
"scanId": "dummy_id", "scanId": "dummy_id",
"filePath": "my_file.h5", "filePath": "my_file.h5",
"data": {"data": {}, "metadata": {"scan_id": "scan_id"}}, "data": {"data": {}},
"scaninfo": {"bec": {"scan_id": "scan_id"}},
} }
) )
) )
@ -145,8 +146,9 @@ def test_update_scan_data_exceeding_limit(md_handler, scan_document):
md_handler.scibec_connector.scibec = scibec md_handler.scibec_connector.scibec = scibec
scibec.scan.scan_controller_find = mock.MagicMock(return_value=[scan_document]) scibec.scan.scan_controller_find = mock.MagicMock(return_value=[scan_document])
data_block = {f"key_{i}": {"signal": list(range(100))} for i in range(10)} data_block = {f"key_{i}": {"signal": list(range(100))} for i in range(10)}
data_block.update({"metadata": {"scan_id": "scan_id"}}) md_handler.update_scan_data(
md_handler.update_scan_data(file_path="my_file.h5", data=data_block) file_path="my_file.h5", data=data_block, scan_info={"bec": {"scan_id": "scan_id"}}
)
num_calls = scibec.scan_data.scan_data_controller_create_many.call_count num_calls = scibec.scan_data.scan_data_controller_create_many.call_count
assert num_calls == 5 assert num_calls == 5