diff --git a/README.md b/README.md index b481fd4..3d9bbac 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ ## Automatic Processing tool aka Apocalypse (apo) -Apocalypse (apo) is a library that provides the SwissFEL user platform to effortlessly run (and re run) their scripts on the the SwissFEL infrastructure -- compute cluster "Eris" (login node: sf-eris) with [sf-daq](https://github.com/paulscherrerinstitute/sf_daq_broker) output files. +Apocalypse (apo) is a library that provides the SwissFEL user platform to effortlessly run (and re run) their scripts on the SwissFEL infrastructure -- compute cluster "Eris" (login node: sf-eris) with [sf-daq](https://github.com/paulscherrerinstitute/sf_daq_broker) output files. More information about [Eris](https://hpc-sysadmins.gitpages.psi.ch/swissfel/compute.html) @@ -16,16 +16,21 @@ ssh sf-cn-# --input .h5 ``` - `sf-cn-#` -- corresponds to the node allocated for interactive use and displayed after `salloc` -- `` -- is top level script that should run exactly as stated. That means any additional actions (e.g. setting enviroment, adding pmodules, additional parameter handling) should be included in it. See [examples](https://gitea.psi.ch/woznic_n/dummy_apocalypse/src/branch/main/examples) +- `` -- is top level script that should run exactly as stated. That means any additional actions (e.g. setting environment, adding pmodules, additional parameter handling) should be included in it. See [examples](https://gitea.psi.ch/woznic_n/dummy_apocalypse/src/branch/main/examples) - `.h5` is the path to file that you would like to process on `/sf//data//raw/` -Once your tests are succesfull remember to cancel your allocation by: +Once your tests are successful remember to cancel your allocation by: ```bash scancel ``` - `` is slurm's job ID, displayed when allocating resources, you can also check it by `squeue -u $USER` #### Setup +Run apo from Eris: +```bash +ssh sf-eris +``` + If you have `/sf/daq/bin` on your path, there is no need to do anything, otherwise: ```bash export PATH=$PATH:/sf/daq/bin @@ -92,4 +97,4 @@ buffer - file type correspondence: - data3buffer = BSDATA.h5 - imagebuffer = CAMERAS.h5 - detector_buffer = JF.h5 -- apocalypse = all files when "re-run" \ No newline at end of file +- apocalypse = all files when "re-run" diff --git a/apocalypse/send_done.py b/apocalypse/send_done.py index e349849..c0524b4 100755 --- a/apocalypse/send_done.py +++ b/apocalypse/send_done.py @@ -112,7 +112,9 @@ def setup_msg(args, msg, tstamp): def parse_apo_meta(pth): - loader = LOADER.get(pth.suffix, lambda x: x) + loader = LOADER.get(pth.suffix, lambda x: { + "table": x + }) with open(pth, "r") as f: data = f.read() try: diff --git a/apocalypse/wait_for_file.py b/apocalypse/wait_for_file.py index d6b0f07..758dfe8 100644 --- a/apocalypse/wait_for_file.py +++ b/apocalypse/wait_for_file.py @@ -49,8 +49,8 @@ def on_status(chn, _method_frame, header_frame, body, args): local_args.header = header local_args.request = request local_args.correlation_id = header_frame.correlation_id - if request.get("metadata", {}).get("general/instrument") == local_args.endstation: - print(header) + # if request.get("metadata", {}).get("general/instrument") == local_args.endstation: + # print(header) # print(body) if ( header.get("action") is None or request.get("writer_type") is None diff --git a/examples/simple1/pixi.toml b/examples/simple1/pixi.toml index 533b9e4..091b935 100644 --- a/examples/simple1/pixi.toml +++ b/examples/simple1/pixi.toml @@ -11,4 +11,4 @@ version = "0.1.0" python = "==3.12" sfdata = "*" logzero = "*" - +bitshuffle = {version="==0.5.2",build="py312h5fdea32_5"} diff --git a/examples/simple1/run_simple1.sh b/examples/simple1/run_simple1.sh index e037412..3290dbd 100755 --- a/examples/simple1/run_simple1.sh +++ b/examples/simple1/run_simple1.sh @@ -1,5 +1,5 @@ #! /bin/bash SCRIPT_PTH=./examples/simple1/ - -pixi run --manifest-path ${SCRIPT_PTH}pixi.toml python ${SCRIPT_PTH}simple1.py -f $2 +unset PIXI_IN_SHELL +pixi run --manifest-path $SCRIPT_PTH python ${SCRIPT_PTH}simple1.py -f $2 diff --git a/examples/simple2/pixi.toml b/examples/simple2/pixi.toml index a1500b8..1f0a0e0 100644 --- a/examples/simple2/pixi.toml +++ b/examples/simple2/pixi.toml @@ -12,3 +12,4 @@ python="==3.12" sfdata="*" numpy="*" logzero = "*" +bitshuffle = {version="==0.5.2",build="py312h5fdea32_5"} diff --git a/examples/simple2/run_simple2.sh b/examples/simple2/run_simple2.sh index 7dafab9..f652712 100755 --- a/examples/simple2/run_simple2.sh +++ b/examples/simple2/run_simple2.sh @@ -1,4 +1,5 @@ #! /bin/bash -SCRIPT_PTH=./examples/ -pixi run --manifest-path ${SCRIPT_PTH}pixi.toml python ${SCRIPT_PTH}simple2.py -f $2 -b bg.CAMERAS.h5 +SCRIPT_PTH=./examples/simple2/ +unset PIXI_IN_SHELL +pixi run --manifest-path ${SCRIPT_PTH} python ${SCRIPT_PTH}simple2.py -f $2 -b bg.CAMERAS.h5 diff --git a/tests/apo/test_send_msg.py b/tests/apo/test_send_msg.py index 846f16e..5495594 100644 --- a/tests/apo/test_send_msg.py +++ b/tests/apo/test_send_msg.py @@ -268,7 +268,9 @@ def test_send_msg_body_req_succ_double_meta(mock_apo_input): }, ".json"), ("key: value\nnum: 2", { "key": "value", "num": 2 - }, ".yaml"), ("plain meta txt", "plain meta txt", ".txt")] + }, ".yaml"), ("plain meta txt", { + "table": "plain meta txt" + }, ".txt")] ) def test_parse_apo_meta(content, expected, suffix): string_io = io.StringIO(content)