diff --git a/Readme.md b/Readme.md index 3fc06cd..d8b0e3a 100644 --- a/Readme.md +++ b/Readme.md @@ -169,6 +169,137 @@ cd /sf/cristallina/applications/mx/zamofing_t/ESB_MX git checkout master cd /sf/cristallina/applications/mx/zamofing_t/ESB_MX/python/SwissMX git checkout master +``` + +22.6.23 debug segmentation fault +-------------------------------- +``` +THE CHRISTALLINA CONTROL ROOM RUNS NORMALLY ON saresc-cons-05 + +[saresc-cons-05 ~]$ +ulimit -a +ulimit -c unlimited +python -X faulthandler -X tracemalloc -X dev swissmx.py 2>&1 | tee /tmp/swissmx000.log +reset;tail -c+0 -F /tmp/swissmx000.log + +python -X faulthandler -X tracemalloc -X importtime -X dev swissmx.py --sim 0xff +python -X faulthandler -X tracemalloc -X dev swissmx.py --sim 0xff + +zamofing_t@ganymede:~/Documents/prj/SwissFEL/epics_ioc_modules/ESB_MX/python/SwissMX$ +rsync -vai swissmx.py saresc-cons-03:/sf/cristallina/applications/mx/zamofing_t/ESB_MX/python/SwissMX/swissmx_segFault.py + +read: https://docs.python.org/3/library/faulthandler.html + +coredump erzeugen +cat /proc/sys/kernel/core_pattern +sudo echo "/tmp/core" > /proc/sys/kernel/core_pattern + +ulimit -c unlimited +python -c "import ctypes; ctypes.string_at(0)" +python -X faulthandler -c "import ctypes; ctypes.string_at(0)" + +Rene did: +[root@saresc-cons-05 ~]# sysctl -w kernel.core_pattern="/tmp/%e_core_dump.%p" +cat /proc/sys/kernel/core_pattern +/tmp/%e_core_dump.%p + +ll /tmp/python_core_dump.* + + +gdb python /tmp/core +bt (for facktrace) + +trying python c code extention: +/home/zamofing_t/Documents/prj/scratch/python/sample_c_extension + + +Thread 0x00007fd2e16e8700 (most recent call first): + File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 620 in get_with_metadata + ... + File "/gfa/.mounts/sf_cristallina/applications/mx/zamofing_t/ESB_MX/python/SwissMX/epics_widgets/MotorTweak.py", line 216 in update_label + ... + File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 48 in wrapped +``` + +Coredump 29.6.23 18h40 +---------------------- +``` +grep -c '5000/5000' * +swissmx000.log:57 +swissmx001.log:36 +swissmx002.log:35 + +swissmx002.log 35*5000 frames +35*5000/100/60 -> every 29.16min of acquisition a crash + + +rsync -vai gac-cristall@saresc-cons-05:/tmp/swissmx* ~/Documents/prj/SwissFEL/epics_ioc_modules/ESB_MX/python/SwissMX/log + +ll /tmp/swissmx* +ll /tmp/python_core_dump.* + +Thread 0x00007fb9b13f7700 (most recent call first): + File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 48 in wrapped + File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 620 in get_with_metadata + File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 48 in wrapped + File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 620 in get_with_metadata + File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 48 in wrapped + File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/ca.py", line 1122 in element_count + File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 489 in get + File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 48 in wrapped + File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 620 in get_with_metadata + File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 48 in wrapped + File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 981 in nelm + File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/ca.py", line 579 in wrapper + File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/ca.py", line 1122 in element_count + File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 48 in wrapped + File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 48 in wrapped + File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/ca.py", line 549 in wrapper + File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/ca.py", line 871 in current_context + File "/sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/pyepics-3.4.3-py3.9.egg/epics/pv.py", line 48 in wrapped + +conda activate crmx38 +ll /tmp/python_core_dump.* +gdb python /tmp/python_core_dump.21072 +Core was generated by `python -X faulthandler -X tracemalloc -X dev swissmx.py'. +Program terminated with signal 11, Segmentation fault. + +bt +0 write_thread_id.isra.3 (is_current=0, fd=) at /opt/conda/conda-bld/python-split_1648465063888/work/Python/traceback.c:849 +#1 _Py_DumpTracebackThreads () at /opt/conda/conda-bld/python-split_1648465063888/work/Python/traceback.c:914 +#2 0x000055c2b7ef6635 in faulthandler_dump_traceback.isra.2 (fd=fd@entry=2, all_threads=1) at /opt/conda/conda-bld/python-split_1648465063888/work/Modules/faulthandler.c:242 +#3 0x000055c2b7ef67a3 in faulthandler_fatal_error (signum=11) at /opt/conda/conda-bld/python-split_1648465063888/work/Modules/faulthandler.c:348 +#4 +#5 0x00007fb9c7ed4ef4 in QWidgetTextControl::document() const () from /sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/PyQt5/../../../libQt5Widgets.so.5 +#6 0x00007fb9c7e676e1 in ?? () from /sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/PyQt5/../../../libQt5Widgets.so.5 +#7 0x00007fb9c7e69230 in QLabel::paintEvent(QPaintEvent*) () from /sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/PyQt5/../../../libQt5Widgets.so.5 +#8 0x00007fb9d7fa16d3 in sipQLabel::paintEvent(QPaintEvent*) () from /sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/PyQt5/QtWidgets.abi3.so +#9 0x00007fb9c7dcc580 in QWidget::event(QEvent*) () from /sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/PyQt5/../../../libQt5Widgets.so.5 +#10 0x00007fb9c7e36203 in QFrame::event(QEvent*) () from /sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/PyQt5/../../../libQt5Widgets.so.5 +#11 0x00007fb9d7fa2d03 in sipQLabel::event(QEvent*) () from /sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/PyQt5/QtWidgets.abi3.so +#12 0x00007fb9c7da20f1 in QApplicationPrivate::notify_helper(QObject*, QEvent*) () from /sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/PyQt5/../../../libQt5Widgets.so.5 +#13 0x00007fb9d8097afe in sipQApplication::notify(QObject*, QEvent*) () from /sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/PyQt5/QtWidgets.abi3.so +#14 0x00007fb9e6cadd62 in QCoreApplication::notifyInternal2(QObject*, QEvent*) () from /sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/PyQt5/../../../libQt5Core.so.5 +#15 0x00007fb9c7dc6de6 in QWidgetPrivate::sendPaintEvent(QRegion const&) () from /sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/PyQt5/../../../libQt5Widgets.so.5 +#16 0x00007fb9c7dc774e in QWidgetPrivate::drawWidget(QPaintDevice*, QRegion const&, QPoint const&, QFlags, QPainter*, QWidgetRepaintManager*) () + from /sf/cristallina/applications/conda/envs/crmx38/lib/python3.8/site-packages/PyQt5/../../../libQt5Widgets.so.5 +#17 0x00007fb9c7dc8147 in QWidgetPrivate::paintSiblingsRecursive(QPaintDevice*, QList const&, int, QRegion const&, QPoint const&, QFlags, QPainter*, QWidgetRe + + +``` + +Localize mionitors and callbacks: +``` + +grep -n 'def cb_update_img(self):' *.py + + +swissmx.py:698: self.sigNewCamImg.connect(self.cb_update_img) +swissmx.py:745: def cb_update_img(self): + + + + ``` diff --git a/app_config.py b/app_config.py index 270d514..1755fd7 100644 --- a/app_config.py +++ b/app_config.py @@ -144,7 +144,7 @@ class AppCfg(QSettings): if AppCfg.DAQ_PV_CH not in keys: dflt.append((AppCfg.DAQ_PV_CH, () )) #list of PVs if AppCfg.DFT_POS_GONIO not in keys: - dflt.append((AppCfg.DFT_POS_GONIO, {'mount':(0.,0.,0.,0.),'align':(0.,0.,0.,0.)}))#default positions + dflt.append((AppCfg.DFT_POS_GONIO, {'pos_mount':(0.,0.,0.,0.),'pos_align':(0.,0.,0.,0.)}))#default positions if AppCfg.DFT_POS_BKLGT not in keys: dflt.append((AppCfg.DFT_POS_BKLGT, {'pos_in': -30000.0, 'pos_out': 1000.0, 'pos_diode': -30000.0}))#default positions @@ -363,8 +363,8 @@ verbose bits: {'name':'set_out', 'title':'use current position as "out"', 'type':'action'}, ]}, {'name':AppCfg.DFT_POS_GONIO, 'title':'gonio reference positions', 'type':'group', 'expanded':False, 'children':[ - {'name':'pos_mount', 'title':'Mount position', 'value':dft_pos_gonio.get('mount'), 'type':'str' }, - {'name':'pos_align', 'title':'Align position', 'value':dft_pos_gonio.get('align'), 'type':'str' }, + {'name':'pos_mount', 'title':'Mount position', 'value':dft_pos_gonio.get('pos_mount'), 'type':'str' }, + {'name':'pos_align', 'title':'Align position', 'value':dft_pos_gonio.get('pos_align'), 'type':'str' }, {'name':'set_mount', 'title':'use current position as "mount"', 'type':'action'}, {'name':'set_align', 'title':'use current position as "align"', 'type':'action'}, ]}, diff --git a/epics_widgets/MotorTweak.py b/epics_widgets/MotorTweak.py index 4bbbf47..22ff727 100644 --- a/epics_widgets/MotorTweak.py +++ b/epics_widgets/MotorTweak.py @@ -95,10 +95,13 @@ class MotorTweak(QWidget, Ui_MotorTweak): def set_val(self, **kw): - v = kw['char_value'] - _log.debug('updating VAL = {}'.format(v)) - self._val=float(v) # rewrite in case of tweaking - self._drive_val.setText(v) + try: #pv-monitor-func + v = kw['char_value'] + _log.debug('updating VAL = {}'.format(v)) + self._val=float(v) # rewrite in case of tweaking + self._drive_val.setText(v) + except Exception as e: + _log.critical(f'{e}') def set_motor_validator(self, **kwargs): @@ -187,33 +190,39 @@ class MotorTweak(QWidget, Ui_MotorTweak): :return: ''' - field = kw['motor_field'] - src = kw['source_field'] - kw['alias'] = self._label - if field != src: - return - if field == 'VAL': - self.event_val.emit(self._rec_name, kw) - elif field == 'RBV': - self.event_rbv.emit(kw['alias'], kw['value'], kw) - elif field == 'LVIO': - self.event_soft_limit.emit(self._rec_name, kw) - elif field == 'HLS': - self.event_high_hard_limit.emit(self._rec_name, kw) - self.event_axis_fault.emit(self._rec_name, kw) - elif field == 'LVIO': - self.event_low_hard_limit.emit(self._rec_name, kw) - self.event_axis_fault.emit(self._rec_name, kw) - elif field == 'STAT': - self.event_axis_fault.emit(self._rec_name, kw) + try: #pv-monitor-func + field = kw['motor_field'] + src = kw['source_field'] + kw['alias'] = self._label + if field != src: + return + if field == 'VAL': + self.event_val.emit(self._rec_name, kw) + elif field == 'RBV': + self.event_rbv.emit(kw['alias'], kw['value'], kw) + elif field == 'LVIO': + self.event_soft_limit.emit(self._rec_name, kw) + elif field == 'HLS': + self.event_high_hard_limit.emit(self._rec_name, kw) + self.event_axis_fault.emit(self._rec_name, kw) + elif field == 'LVIO': + self.event_low_hard_limit.emit(self._rec_name, kw) + self.event_axis_fault.emit(self._rec_name, kw) + elif field == 'STAT': + self.event_axis_fault.emit(self._rec_name, kw) + except Exception as e: + _log.critical(f'{e}') def update_label(self, **kwargs): - m = self._motor - self.label.setText(self._templates[self._label_style].format(rbv=m.readback)) - self.jog_forward.setToolTip('jog forward at {:.3f} {}/s'.format(m.jog_speed, m.units)) - self.jog_reverse.setToolTip('jog reverse at {:.3f} {}/s'.format(m.jog_speed, m.units)) - self.tweak_forward.setToolTip('tweak forward by {:.3f} {}'.format(m.tweak_val, m.units)) - self.tweak_reverse.setToolTip('tweak reverse by {:.3f} {}'.format(m.tweak_val, m.units)) + try: #pv-monitor-func + m = self._motor + self.label.setText(self._templates[self._label_style].format(rbv=m.readback)) + self.jog_forward.setToolTip('jog forward at {:.3f} {}/s'.format(m.jog_speed, m.units)) + self.jog_reverse.setToolTip('jog reverse at {:.3f} {}/s'.format(m.jog_speed, m.units)) + self.tweak_forward.setToolTip('tweak forward by {:.3f} {}'.format(m.tweak_val, m.units)) + self.tweak_reverse.setToolTip('tweak reverse by {:.3f} {}'.format(m.tweak_val, m.units)) + except Exception as e: + _log.critical(f'{e}') def update_jog_speed(self, event): m = self._motor diff --git a/swissmx.py b/swissmx.py index a8231fa..4c45cfa 100755 --- a/swissmx.py +++ b/swissmx.py @@ -717,30 +717,33 @@ class WndSwissMx(QMainWindow, Ui_MainWindow): cam.run(self.cb_new_frame_pv) def cb_new_frame_pv(self, **kwargs): - #thrd=threading.current_thread() - #_log.debug(f'thread:{thrd.getName()}, {thrd.native_id}') - #_log.debug(f"{kwargs['timestamp']}") + try: #pv-monitor-func + #thrd=threading.current_thread() + #_log.debug(f'thread:{thrd.getName()}, {thrd.native_id}') + #_log.debug(f"{kwargs['timestamp']}") - app=QApplication.instance() - cam=app._camera - sz=cam._sz - if kwargs['count']==sz[0]*sz[1]: - pic=kwargs['value'].reshape(sz[::-1]) - else: - sz=app._camera.update_size() - pic=kwargs['value'].reshape(sz[::-1]) - if pic.dtype==np.int16: - pic.dtype=np.uint16 - camera.epics_cam.set_fiducial(pic, 255) - cam._pic=pic - cam._timestamp=kwargs['timestamp'] - try: - cam.process() - except AttributeError as e: - pass - # self._goImg.setImage(cam._pic) caused some deadlocks. - # therefore try to update the image with signals instead - self.sigNewCamImg.emit() + app=QApplication.instance() + cam=app._camera + sz=cam._sz + if kwargs['count']==sz[0]*sz[1]: + pic=kwargs['value'].reshape(sz[::-1]) + else: + sz=app._camera.update_size() + pic=kwargs['value'].reshape(sz[::-1]) + if pic.dtype==np.int16: + pic.dtype=np.uint16 + camera.epics_cam.set_fiducial(pic, 255) + cam._pic=pic + cam._timestamp=kwargs['timestamp'] + try: + cam.process() + except AttributeError as e: + pass + # self._goImg.setImage(cam._pic) caused some deadlocks. + # therefore try to update the image with signals instead + self.sigNewCamImg.emit() + except Exception as e: + _log.critical(f'{e}') def cb_update_img(self): #thrd=threading.current_thread() @@ -748,22 +751,43 @@ class WndSwissMx(QMainWindow, Ui_MainWindow): app=QApplication.instance() cam=app._camera + self._goImg.setImage(cam._pic) #vb.setRange(QRectF(-1300,-1100,1400,1200)) + #force a segmentation fault + #try: + # dbgSegFault=self._dbgSegFault + #except AttributeError as e: + # dbgSegFault=self._dbgSegFault=0 + #_log.warning(f'dbgSegFault:{dbgSegFault}') + #if dbgSegFault>100: + # _log.critical('Xforce a segmentation fault') + # import ctypes + # ctypes.string_at(0) + #self._dbgSegFault+=1 + def cb_new_frame_sim(self, **kwargs): app=QApplication.instance() sim=app._camera._sim imgSeq=sim['imgSeq'] idx=sim['imgIdx'] - sim['imgIdx']=(idx+1)%imgSeq.shape[0] + sim['imgIdx']=(idx+1) + imgIdx=idx%imgSeq.shape[0] # _log.info('simulated idx:{}'.format(idx)) - pic=imgSeq[idx] + pic=imgSeq[imgIdx] self._goImg.setImage(pic) delay=500 # ms -> 2fps QtCore.QTimer.singleShot(delay, self.cb_new_frame_sim) + #force a segmentation fault + #_log.warning(f'imgIdx:{imgIdx},idx:{idx}') + #if idx==20: + # _log.critical('force a segmentation fault') + # import ctypes + # ctypes.string_at(0) + def load_stylesheet(self): with open("swissmx.css", "r") as sheet: self.setStyleSheet(sheet.read()) @@ -969,6 +993,14 @@ class WndSwissMx(QMainWindow, Ui_MainWindow): ] qutilities.add_item_to_toolbox(toolbox,"Fast Stage",widget_list=widgets) + #pv-monitor-func + #_log.info('modify monitors') + #for w in widgets[1:-1]: #ignore last item + # for k,pv in w._motor._pvs.items(): + # pv.auto_monitor=False + # for k,pv in w._motor._pvs.items(): + # pv.auto_monitor=True + def build_group_collimator(self, toolbox): pfx=QApplication.instance()._cfg.value(AppCfg.GBL_DEV_PREFIX)[1] c=QWidget()