sf-mx/SwissMX
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
master
SwissMX/Log.md

18 KiB
Raw Permalink Blame History

16.9.22 remote Deltatau test:

(s.a. /home/zamofing_t/Documents/prj/SwissFEL/PBTools/pbtools/gather/PBGatherPlot.py

PPMAC=SAR-CPPM-EXPMX1
rsync -va ~/Documents/prj/SwissFEL/PBTools/pbtools/gather/gather_server root@$PPMAC:/tmp/

ssh root@$PPMAC
LD_LIBRARY_PATH=/opt/ppmac/libppmac/ /tmp/gather_server
ssh -L 10001:localhost:22 root@$PPMAC 'uname -a'
ssh -L 10002:localhost:2332 root@$PPMAC 'uname -a'
Deltatau host in config:
localhost:10001:10002 (instead SAR-CPPM-EXPMX1)

ssh gac-cristall@saresc-cons-03
pw: ValToira_2021

4.10.22 final test:

zamofing_t@ganymede:~$ ssh gac-cristall@saresc-cons-03
cd /sf/cristallina/applications/mx/zamofing_t/ESB_MX/python/SwissMX/
# conda env list
conda activate crmx38
python swissmx.py 

ssh gac-cristall@saresc-cons-03 ls /sf/cristallina/applications/mx/zamofing_t/ESB_MX/python/SwissMX

git@git.psi.ch:epics_ioc_modules/ESB_MX.git

git remote add sf-cristallina gac-cristall@saresc-cons-03:/sf/cristallina/applications/mx/zamofing_t/ESB_MX/python/SwissMX

git fetch sf-cristallina 

# do local changes and commit stuff
git push sf-cristallina

22.6.23 debug segmentation fault

THE CHRISTALLINA CONTROL ROOM RUNS NORMALLY ON saresc-cons-05
[saresc-cons-05 ~]$ 
ulimit -a
ulimit -c unlimited
python -X faulthandler -X tracemalloc -X dev swissmx.py 2>&1 | tee /tmp/swissmx000.log
reset;tail -c+0 -F /tmp/swissmx000.log 

python -X faulthandler -X tracemalloc -X importtime -X dev swissmx.py  --sim 0xff
python -X faulthandler -X tracemalloc -X dev swissmx.py  --sim 0xff

zamofing_t@ganymede:~/Documents/prj/SwissFEL/epics_ioc_modules/ESB_MX/python/SwissMX$
rsync -vai swissmx.py saresc-cons-03:/sf/cristallina/applications/mx/zamofing_t/ESB_MX/python/SwissMX/swissmx_segFault.py

read: https://docs.python.org/3/library/faulthandler.html

coredump erzeugen
cat /proc/sys/kernel/core_pattern
sudo echo "/tmp/core" > /proc/sys/kernel/core_pattern

ulimit -c unlimited
python -c "import ctypes; ctypes.string_at(0)"
python -X faulthandler -c "import ctypes; ctypes.string_at(0)"

Rene did:
[root@saresc-cons-05 ~]# sysctl -w kernel.core_pattern="/tmp/%e_core_dump.%p"
cat /proc/sys/kernel/core_pattern
/tmp/%e_core_dump.%p

ll /tmp/python_core_dump.*

gdb python /tmp/core 
bt  (for facktrace)

trying python c code extention:
/home/zamofing_t/Documents/prj/scratch/python/sample_c_extension

Thread 0x00007fd2e16e8700 (most recent call first):
  File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 620 in get_with_metadata
  ...
  File "/gfa/.mounts/sf_cristallina/applications/mx/zamofing_t/ESB_MX/python/SwissMX/epics_widgets/MotorTweak.py", line 216 in update_label
  ...
  File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 48 in wrapped

29.6.23 Coredump 18h40

grep -c '5000/5000' *
swissmx000.log:57
swissmx001.log:36
swissmx002.log:35

swissmx002.log 35*5000 frames
35*5000/100/60 -> every 29.16min of acquisition a crash

rsync -vai gac-cristall@saresc-cons-05:/tmp/swissmx* ~/Documents/prj/SwissFEL/epics_ioc_modules/ESB_MX/python/SwissMX/log

ll /tmp/swissmx*
ll /tmp/python_core_dump.*

Thread 0x00007fb9b13f7700 (most recent call first):
  File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 48 in wrapped
  File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 620 in get_with_metadata
  File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 48 in wrapped
  File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 620 in get_with_metadata
  File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 48 in wrapped
  File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/ca.py", line 1122 in element_count
  File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 489 in get
  File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 48 in wrapped
  File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 620 in get_with_metadata
  File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 48 in wrapped
  File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 981 in nelm
  File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/ca.py", line 579 in wrapper
  File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/ca.py", line 1122 in element_count
  File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 48 in wrapped
  File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 48 in wrapped
  File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/ca.py", line 549 in wrapper
  File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/ca.py", line 871 in current_context
  File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 48 in wrapped

conda activate crmx38
ll /tmp/python_core_dump.*
gdb python /tmp/python_core_dump.21072 
Core was generated by `python -X faulthandler -X tracemalloc -X dev swissmx.py'.
Program terminated with signal 11, Segmentation fault.

bt
0  write_thread_id.isra.3 (is_current=0, fd=<optimized out>) at /opt/conda/conda-bld/python-split_1648465063888/work/Python/traceback.c:849
#1  _Py_DumpTracebackThreads () at /opt/conda/conda-bld/python-split_1648465063888/work/Python/traceback.c:914
#2  0x000055c2b7ef6635 in faulthandler_dump_traceback.isra.2 (fd=fd@entry=2, all_threads=1) at /opt/conda/conda-bld/python-split_1648465063888/work/Modules/faulthandler.c:242
#3  0x000055c2b7ef67a3 in faulthandler_fatal_error (signum=11) at /opt/conda/conda-bld/python-split_1648465063888/work/Modules/faulthandler.c:348
#4  <signal handler called>
#5  0x00007fb9c7ed4ef4 in QWidgetTextControl::document() const () from /sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/PyQt5/../../../libQt5Widgets.so.5
#6  0x00007fb9c7e676e1 in ?? () from /sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/PyQt5/../../../libQt5Widgets.so.5
#7  0x00007fb9c7e69230 in QLabel::paintEvent(QPaintEvent*) () from /sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/PyQt5/../../../libQt5Widgets.so.5
#8  0x00007fb9d7fa16d3 in sipQLabel::paintEvent(QPaintEvent*) () from /sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/PyQt5/QtWidgets.abi3.so
#9  0x00007fb9c7dcc580 in QWidget::event(QEvent*) () from /sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/PyQt5/../../../libQt5Widgets.so.5
#10 0x00007fb9c7e36203 in QFrame::event(QEvent*) () from /sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/PyQt5/../../../libQt5Widgets.so.5
#11 0x00007fb9d7fa2d03 in sipQLabel::event(QEvent*) () from /sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/PyQt5/QtWidgets.abi3.so
#12 0x00007fb9c7da20f1 in QApplicationPrivate::notify_helper(QObject*, QEvent*) () from /sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/PyQt5/../../../libQt5Widgets.so.5
#13 0x00007fb9d8097afe in sipQApplication::notify(QObject*, QEvent*) () from /sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/PyQt5/QtWidgets.abi3.so
#14 0x00007fb9e6cadd62 in QCoreApplication::notifyInternal2(QObject*, QEvent*) () from /sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/PyQt5/../../../libQt5Core.so.5
#15 0x00007fb9c7dc6de6 in QWidgetPrivate::sendPaintEvent(QRegion const&) () from /sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/PyQt5/../../../libQt5Widgets.so.5
#16 0x00007fb9c7dc774e in QWidgetPrivate::drawWidget(QPaintDevice*, QRegion const&, QPoint const&, QFlags<QWidgetPrivate::DrawWidgetFlag>, QPainter*, QWidgetRepaintManager*) ()
   from /sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/PyQt5/../../../libQt5Widgets.so.5
#17 0x00007fb9c7dc8147 in QWidgetPrivate::paintSiblingsRecursive(QPaintDevice*, QList<QObject*> const&, int, QRegion const&, QPoint const&, QFlags<QWidgetPrivate::DrawWidgetFlag>, QPainter*, QWidgetRe

Localize monitors and callbacks:

grep -Rn 'add_callback' *.py

swissmx.py:698:        self.sigNewCamImg.connect(self.cb_update_img)
swissmx.py:745:  def cb_update_img(self):
epics_widgets/MotorTweak.py:88:    m.set_callback('RBV', self.emit_signals, {'source_field': 'RBV'})
epics_widgets/SmaractMotorTweak.py:100:        self._pv_readback.add_callback(self.update_label)
swissmx.py:717:         cam.run(self.cb_new_frame_pv)
swissmx.py:743:    self.sigNewCamImg.emit()
swissmx.py:745:  def cb_update_img(self):
swissmx.py:698:        self.sigNewCamImg.connect(self.cb_update_img)
camera.py:159:      self._pv['pic'] = epics.PV(self._prefix + "FPICTURE", auto_monitor=True, callback=cb)

-> try to turn of the monitors during ascuisition:
zamofing_t@ganymede:~/Documents/prj/SwissFEL/epics_ioc_modules/ESB_MX$
grep -r set_callback *

https://pyepics.github.io/pyepics/pv.html#automatic-monitoring-of-a-pv

Try with:
pv.clear_auto_monitor() -> pv.reconnect()

pv.remove_callback(index=None) -> pv.add_callback(callback=None[, index=None [, with_ctrlvars=True[, **kw]]) 
pv.set_callback

FixTargetFrame ->  paint -> _log.debug()

Turn off jungfrau.
no motion -> wait 1h
constant up-dow motion code -> wait 1h

23.9.24 spitting/moving repositories

https://jira.psi.ch/browse/SFELPHOTON-1337: SwissMX split/ cleanup/move repositories

SCRATCH

Deploy stuff (22.8.22) quick and dirty (21.9.22 rewworked)

DST=/sf/cristallina/applications/mx/zamofing_t/

ssh saresc-cons-03 mkdir $DST
# add '--delete'  if needed
rsync -vai ~/Documents/prj/SwissFEL/epics_ioc_modules/ESB_MX saresc-cons-03:$DST -n
rsync -vai ~/Documents/prj/SwissFEL/PBTools saresc-cons-03:$DST -n
ssh saresc-cons-03 saresc-cons-03 chmod -R go+w $DST/ESB_MX/python/SwissMX

cd /sf/cristallina/applications/mx/zamofing_t/ESB_MX/python/SwissMX/
/opt/gfa/python-3.8/latest/bin/pip install qtawesome --user
/opt/gfa/python-3.8/latest/bin/pip install opencv-python
/opt/gfa/python-3.8/latest/bin/python swissmx.py 

rsync -vai gac-cristall@saresc-cons-03:~/.config/PSI/SwissMX.conf /home/zamofing_t/Documents/prj/SwissFEL/epics_ioc_modules/ESB_MX/python/SwissMX/

cd $DST
/opt/gfa/python-3.8/latest/bin/python -m pdb swissmx.py 
rsync -vai saresc-cons-03:/tmp/image*.png ~/Documents/prj/SwissFEL/epics_ioc_modules/ESB_MX/python/SwissMX/scratch/
/home/zamofing_t/Documents/prj/SwissFEL/epics_ioc_modules/ESB_MX/python/SwissMX/scratch/autofocus2

pyqtgraph examples

ipython3
import pyqtgraph.examples
pyqtgraph.examples.run()

EPICS simulator

This provides very dummy images and motor(to be done) records

cd /home/zamofing_t/Documents/prj/SwissFEL/epics_ioc_modules/ESB_MX/EpicsSim/iocBoot/iocSwissMxSim
./st.cmd

simulate camera (with EPICS simulator)

cd /home/zamofing_t/Documents/prj/SwissFEL/epics_ioc_modules/ESB_MX/python/SwissMX
EPICS_CA_ADDR_LIST=localhost
./simCam.py

test camera display

cd /home/zamofing_t/Documents/prj/SwissFEL/epics_ioc_modules/ESB_MX/python/SwissMX
EPICS_CA_ADDR_LIST=localhost
./camera.py -u -b SwissMxSim

test at ESC

https://git.psi.ch/SwissMX/swissmx_cristallina/-/wikis/Instructions%20on%20how%20to%20use%20software

P=ESB_MX/python/SwissMX/
ssh saresc-cons-02 mkdir -p /tmp/zamofing_t/$P
rsync -vai ~/Documents/prj/SwissFEL/epics_ioc_modules/$P saresc-cons-02:/tmp/zamofing_t/$P
cd /tmp/zamofing_t/ESB_MX/python/SwissMX
/opt/gfa/python-3.7/2018.12/bin/python camera.py -u 1 -p SARES30-CAMS156-SMX-OAV

/opt/gfa/python-3.7/2018.12/bin/python swissmx.py


caqtdm -macro 'NAME=SARES30-CAMS156-SMX-OAV,CAMNAME=SARES30-CAMS156-SMX-OAV' /sf/controls/config/qt/Camera/CameraExpert_RF.ui

try revive Zac code

cd /tmp/; git clone https://github.com/malcolmreynolds/transformations.git
cd /tmp/transformations.git
-> modify __init__.py -> .transformation (add dot)
setup.py install --user


cd /tmp/; git clone https://github.com/spyder-ide/qtawesome.git
cd /tmp/qtawesome

#To have epics channels we must be connected to the ESC network
EPICS_CA_ADDR_LIST='129.129.244.255 sf-saresc-cagw.psi.ch:5062 sf-saresc-cagw.psi.ch:5066'
cd /home/zamofing_t/Documents/prj/SwissFEL/epics_ioc_modules/ESB_MX/python/oldRepos/app/src
python swissmx.py

pyqtgraph examples

import pyqtgraph.examples
pyqtgraph.examples.run()

-0.952 pp_comm.py_: @property def fast_gather(self): tries to connect to port and to query_types(), if fails starts the fast gather process.

triggerSync.c: root@:/opt/ppmac# LD_LIBRARY_PATH=/opt/ppmac/libppmac/ /tmp/triggerSync

usage: /tmp/triggerSync pt2ptTime timeOfs mode

pt2ptTime: time in ms (float value) from point to point == frequency if FEL timeOfs: time offset in ms (float value) for motion relative to the FEL timing

The program changes the speed of motion pshm->Coord[1].DesTimeBase= (default serverPeriod= 0.2?)

SIMFLAG0 (pshm->Coord[1].Q[10]) -> start trigger SIMFLAG1 (pshm->Coord[1].Q[11]) -> FEL pulse trigger //Power PMAC Software Reference Manual.pdf Gate3[i].Chan[j].Status -> page 919 UserFlag #define FLAG0 (gate3_1->Chan[0].Status&0x800) -> start trigger #define FLAG1 (gate3_1->Chan[1].Status&0x800) -> FEL pulse trigger

mode: bit0:1: sync mode bit1:2: simulate start trigger bit2:4: simulate frame trigger bit3:8: verbose

simulate start trigger: set pshm->Coord[1].Q[10]=1 to simulate a Jungfrau aquire start

simulate frame trigger is output to pshm->Coord[1].Q[11] 1: synchronize real frame and start triggers 3: synchronize real frame and simulated start triggers 6: simulated frame and start triggers (no sync) 7: synchronize simulated frame and start triggers

in simulate mode: set pshm->Coord[1].Q[10]=1 to simulate a Jungfrau aquire start set pshm->Coord[1].Q[10]=2 to stop simulate trigger generation Coord[1].Q[11] is the simulated frame trigger

in synchronize mode Coord[1].Q[0]=-2 : trigsync_func start, Wait for 'arm' trigger Coord[1].Q[0]=-1 : got 'arm' trigger, wait frame trigger Coord[1].Q[0]= 0 : got frame trigger 0 Coord[1].Q[0] is incremented at each trigger sync task ends when Gather.Enable==0

// /tmp/triggerSync 40 11 trigger all 40 ms, simulated start, use real frame triggers, verbose // /tmp/triggerSync 40 14 trigger all 40 ms, simulated start and frame triggers, no sync, verbose // /tmp/triggerSync 40 15 trigger all 40 ms, simulated start and frame triggers, with sync, verbose // /tmp/triggerSync 40 7 trigger all 40 ms, simulated start and frame triggers, with sync, minimal verbose

78x78 points =6084pts a 10ms =60 sec. -> 70 sec. -> we are at 100 Hz !!!

remote ssh tunnel + start gather_server

PPMAC=SAR-CPPM-EXPMX1
rsync -va ~/Documents/prj/SwissFEL/PBTools/pbtools/gather/gather_server root@$PPMAC:/tmp/
lsof -i -n | grep '127.0.0.1:1000'
ssh -L 10001:localhost:22 root@$PPMAC 'uname -a'
ssh -L 10002:localhost:2332 root@$PPMAC 'uname -a'
ssh root@$PPMAC
LD_LIBRARY_PATH=/opt/ppmac/libppmac/ /tmp/gather_server

cleanup /tmp/

PPMAC=SAR-CPPM-EXPMX1
ssh root@$PPMAC rm /tmp/gather_server /tmp/triggerSync
ssh root@$PPMAC ls -l /tmp

start debug tools

PPMAC=SAR-CPPM-EXPMX1
PBInspect --host=$PPMAC&
gpasciiCommander --host $PPMAC -i
ssh root@$PPMAC
ssh root@$PPMAC rm /tmp/gather_server /tmp/triggerSync
ssh root@$PPMAC ls -l /tmp

restart IOC

ssh saresc-cons-03
PPMAC=SAR-CPPM-EXPMX1
telnet $PPMAC 50001
Ctrl-X
dbgf SAR-CPPM-EXPMX1:MOD_VER
caget SAR-CPPM-EXPMX1:MOD_VER

checking versions

git: 7a968aac967
asyn             427.0.2
motorBase        alpha_220518
asynMotor        alpha_220518
powerPmac        alpha_220518
PB_COMMON        2.0.1
gpasciiCommander 0.9.0
ESB_MX           0.0.2

zamofing_t@ganymede:~/Documents/prj/SwissFEL/epics_ioc_modules/ESB_MX$ git loggraph -10

  • 7a968aa 2022-09-20 (HEAD -> master, tag: latest, tag: 0.0.2, psigithub/master) change speeds and directions [Thierry Zamofing]
  • 00588f8 2022-09-16 minor changes [Thierry Zamofing]
  • d52a6ce 2022-08-30 minor changes [Thierry Zamofing]
  • f47e111 2022-05-20 (tag: 0.0.1) add DET_Z motor [Thierry Zamofing]
  • 8e5b15e 2022-05-20 wip [Thierry Zamofing]
  • 399282c 2019-03-20 enhance triggering [Thierry Zamofing]
  • eda8caf 2019-03-19 wip [Thierry Zamofing]
  • 0c45705 2019-03-08 optimize [Thierry Zamofing]
  • c962ebd 2019-03-06 documentation [Thierry Zamofing] git reset --hard 399282c rmake -e LIBVERSION=42.42.42 uninstall install -> restart IOC ssh root@$PPMAC rm /tmp/triggerSync cd python git dt latest -- shapepath.py ./shapepath.py --host=localhost:10001:10002 removing test verion ssh sf-lc7 ls -l /ioc/modules/ESB_MX/ ssh sf-lc7 rm -rf /ioc/modules/ESB_MX/42.42.42 IOC locations
~/Documents/prj/SwissFEL/epics_ioc_boot_sf/ESC_all/ESB_MX_PowerBrick
They are just using the new alphy driver. That should have no impact on the motion config.

test shapepath

cd ~/Documents/prj/SwissFEL/epics_ioc_modules/ESB_MX
./shapepath.py --host=localhost:10001:10002
Reference in New Issue View Git Blame Copy Permalink