put try/catch for all monitor callbacks (suggestion Xiaoqiang)
This commit is contained in:
131
Readme.md
131
Readme.md
@@ -169,6 +169,137 @@ cd /sf/cristallina/applications/mx/zamofing_t/ESB_MX
|
||||
git checkout master
|
||||
cd /sf/cristallina/applications/mx/zamofing_t/ESB_MX/python/SwissMX
|
||||
git checkout master
|
||||
```
|
||||
|
||||
22.6.23 debug segmentation fault
|
||||
--------------------------------
|
||||
```
|
||||
THE CHRISTALLINA CONTROL ROOM RUNS NORMALLY ON saresc-cons-05
|
||||
|
||||
[saresc-cons-05 ~]$
|
||||
ulimit -a
|
||||
ulimit -c unlimited
|
||||
python -X faulthandler -X tracemalloc -X dev swissmx.py 2>&1 | tee /tmp/swissmx000.log
|
||||
reset;tail -c+0 -F /tmp/swissmx000.log
|
||||
|
||||
python -X faulthandler -X tracemalloc -X importtime -X dev swissmx.py --sim 0xff
|
||||
python -X faulthandler -X tracemalloc -X dev swissmx.py --sim 0xff
|
||||
|
||||
zamofing_t@ganymede:~/Documents/prj/SwissFEL/epics_ioc_modules/ESB_MX/python/SwissMX$
|
||||
rsync -vai swissmx.py saresc-cons-03:/sf/cristallina/applications/mx/zamofing_t/ESB_MX/python/SwissMX/swissmx_segFault.py
|
||||
|
||||
read: https://docs.python.org/3/library/faulthandler.html
|
||||
|
||||
coredump erzeugen
|
||||
cat /proc/sys/kernel/core_pattern
|
||||
sudo echo "/tmp/core" > /proc/sys/kernel/core_pattern
|
||||
|
||||
ulimit -c unlimited
|
||||
python -c "import ctypes; ctypes.string_at(0)"
|
||||
python -X faulthandler -c "import ctypes; ctypes.string_at(0)"
|
||||
|
||||
Rene did:
|
||||
[root@saresc-cons-05 ~]# sysctl -w kernel.core_pattern="/tmp/%e_core_dump.%p"
|
||||
cat /proc/sys/kernel/core_pattern
|
||||
/tmp/%e_core_dump.%p
|
||||
|
||||
ll /tmp/python_core_dump.*
|
||||
|
||||
|
||||
gdb python /tmp/core
|
||||
bt (for facktrace)
|
||||
|
||||
trying python c code extention:
|
||||
/home/zamofing_t/Documents/prj/scratch/python/sample_c_extension
|
||||
|
||||
|
||||
Thread 0x00007fd2e16e8700 (most recent call first):
|
||||
File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 620 in get_with_metadata
|
||||
...
|
||||
File "/gfa/.mounts/sf_cristallina/applications/mx/zamofing_t/ESB_MX/python/SwissMX/epics_widgets/MotorTweak.py", line 216 in update_label
|
||||
...
|
||||
File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 48 in wrapped
|
||||
```
|
||||
|
||||
Coredump 29.6.23 18h40
|
||||
----------------------
|
||||
```
|
||||
grep -c '5000/5000' *
|
||||
swissmx000.log:57
|
||||
swissmx001.log:36
|
||||
swissmx002.log:35
|
||||
|
||||
swissmx002.log 35*5000 frames
|
||||
35*5000/100/60 -> every 29.16min of acquisition a crash
|
||||
|
||||
|
||||
rsync -vai gac-cristall@saresc-cons-05:/tmp/swissmx* ~/Documents/prj/SwissFEL/epics_ioc_modules/ESB_MX/python/SwissMX/log
|
||||
|
||||
ll /tmp/swissmx*
|
||||
ll /tmp/python_core_dump.*
|
||||
|
||||
Thread 0x00007fb9b13f7700 (most recent call first):
|
||||
File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 48 in wrapped
|
||||
File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 620 in get_with_metadata
|
||||
File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 48 in wrapped
|
||||
File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 620 in get_with_metadata
|
||||
File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 48 in wrapped
|
||||
File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/ca.py", line 1122 in element_count
|
||||
File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 489 in get
|
||||
File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 48 in wrapped
|
||||
File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 620 in get_with_metadata
|
||||
File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 48 in wrapped
|
||||
File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 981 in nelm
|
||||
File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/ca.py", line 579 in wrapper
|
||||
File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/ca.py", line 1122 in element_count
|
||||
File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 48 in wrapped
|
||||
File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 48 in wrapped
|
||||
File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/ca.py", line 549 in wrapper
|
||||
File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/ca.py", line 871 in current_context
|
||||
File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 48 in wrapped
|
||||
|
||||
conda activate crmx38
|
||||
ll /tmp/python_core_dump.*
|
||||
gdb python /tmp/python_core_dump.21072
|
||||
Core was generated by `python -X faulthandler -X tracemalloc -X dev swissmx.py'.
|
||||
Program terminated with signal 11, Segmentation fault.
|
||||
|
||||
bt
|
||||
0 write_thread_id.isra.3 (is_current=0, fd=<optimized out>) at /opt/conda/conda-bld/python-split_1648465063888/work/Python/traceback.c:849
|
||||
#1 _Py_DumpTracebackThreads () at /opt/conda/conda-bld/python-split_1648465063888/work/Python/traceback.c:914
|
||||
#2 0x000055c2b7ef6635 in faulthandler_dump_traceback.isra.2 (fd=fd@entry=2, all_threads=1) at /opt/conda/conda-bld/python-split_1648465063888/work/Modules/faulthandler.c:242
|
||||
#3 0x000055c2b7ef67a3 in faulthandler_fatal_error (signum=11) at /opt/conda/conda-bld/python-split_1648465063888/work/Modules/faulthandler.c:348
|
||||
#4 <signal handler called>
|
||||
#5 0x00007fb9c7ed4ef4 in QWidgetTextControl::document() const () from /sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/PyQt5/../../../libQt5Widgets.so.5
|
||||
#6 0x00007fb9c7e676e1 in ?? () from /sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/PyQt5/../../../libQt5Widgets.so.5
|
||||
#7 0x00007fb9c7e69230 in QLabel::paintEvent(QPaintEvent*) () from /sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/PyQt5/../../../libQt5Widgets.so.5
|
||||
#8 0x00007fb9d7fa16d3 in sipQLabel::paintEvent(QPaintEvent*) () from /sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/PyQt5/QtWidgets.abi3.so
|
||||
#9 0x00007fb9c7dcc580 in QWidget::event(QEvent*) () from /sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/PyQt5/../../../libQt5Widgets.so.5
|
||||
#10 0x00007fb9c7e36203 in QFrame::event(QEvent*) () from /sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/PyQt5/../../../libQt5Widgets.so.5
|
||||
#11 0x00007fb9d7fa2d03 in sipQLabel::event(QEvent*) () from /sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/PyQt5/QtWidgets.abi3.so
|
||||
#12 0x00007fb9c7da20f1 in QApplicationPrivate::notify_helper(QObject*, QEvent*) () from /sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/PyQt5/../../../libQt5Widgets.so.5
|
||||
#13 0x00007fb9d8097afe in sipQApplication::notify(QObject*, QEvent*) () from /sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/PyQt5/QtWidgets.abi3.so
|
||||
#14 0x00007fb9e6cadd62 in QCoreApplication::notifyInternal2(QObject*, QEvent*) () from /sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/PyQt5/../../../libQt5Core.so.5
|
||||
#15 0x00007fb9c7dc6de6 in QWidgetPrivate::sendPaintEvent(QRegion const&) () from /sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/PyQt5/../../../libQt5Widgets.so.5
|
||||
#16 0x00007fb9c7dc774e in QWidgetPrivate::drawWidget(QPaintDevice*, QRegion const&, QPoint const&, QFlags<QWidgetPrivate::DrawWidgetFlag>, QPainter*, QWidgetRepaintManager*) ()
|
||||
from /sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/PyQt5/../../../libQt5Widgets.so.5
|
||||
#17 0x00007fb9c7dc8147 in QWidgetPrivate::paintSiblingsRecursive(QPaintDevice*, QList<QObject*> const&, int, QRegion const&, QPoint const&, QFlags<QWidgetPrivate::DrawWidgetFlag>, QPainter*, QWidgetRe
|
||||
|
||||
|
||||
```
|
||||
|
||||
Localize mionitors and callbacks:
|
||||
```
|
||||
|
||||
grep -n 'def cb_update_img(self):' *.py
|
||||
|
||||
|
||||
swissmx.py:698: self.sigNewCamImg.connect(self.cb_update_img)
|
||||
swissmx.py:745: def cb_update_img(self):
|
||||
|
||||
|
||||
|
||||
|
||||
```
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user