towards jungfrau and shapepath...

app_config.py

@@ -33,6 +33,7 @@ class AppCfg(QSettings):
   CRYOJET_NOZZLE_OUT="cryojet/nozzle_out"
   CRYOJET_NOZZLE_IN="cryojet/nozzle_in"
 
+  DELTATAU_HOST="deltatau/host"
   DELTATAU_SHOW_PLOTS="deltatau/show_plots"
   DELTATAU_OMEGACOS="deltatau/omegacos"
   DELTATAU_SORT_POINTS="deltatau/sort_points"

deltatau.py

@@ -0,0 +1,34 @@
+import sys,os
+sys.path.insert(0, os.path.expanduser('~/Documents/prj/SwissFEL/PBTools/'))
+sys.path.insert(0, os.path.expanduser('..'))
+
+import logging
+_log=logging.getLogger(__name__)
+
+from PyQt5.QtWidgets import (QApplication,)
+from app_config import AppCfg #settings, option, toggle_option
+
+
+from pbtools.misc.pp_comm import PPComm
+from pbtools.misc.gather import Gather
+import shapepath
+
+class Deltatau:
+  def __init__(self):
+    app=QApplication.instance()
+    cfg=app._cfg
+    # cfg.setValue(AppCfg.DELTATAU_HOST, 'SAR-CPPM-EXPMX1')
+    cfg.setValue(AppCfg.DELTATAU_HOST, 'localhost:10001:10002')
+    host=cfg.value(AppCfg.DELTATAU_HOST)
+
+    hpp=host.split(':')
+    param={'host':hpp[0]}
+    if len(hpp)>1:
+      param['port']=int(hpp[1])
+    if len(hpp)>2:
+      param['fast_gather_port']=int(hpp[2])
+      _log.info(' -> ssh-tunneling PPComm({host}:{port} {host}:{fast_gather_port})'.format(**param))
+    self._comm=comm=PPComm(**param)
+    self._gather=gather=Gather(comm)
+    verbose=0xff
+    self._shapepath=sp=shapepath.ShapePath(comm, gather, verbose, sync_mode=1, sync_flag=3)

detector.py

@@ -0,0 +1,306 @@
+import logging
+_log=logging.getLogger(__name__)
+
+#from datetime import datetime
+#from glob import glob
+
+from PyQt5.QtCore import QObject, pyqtSignal, pyqtSlot
+
+#import eventer
+#import qsingleton
+#from detector_integration_api import DetectorIntegrationClient
+#from app_config import settings, appsconf, simulated
+#config = appsconf['jungfrau']
+#dia_client = DetectorIntegrationClient(config['uri'])
+
+class JungfrauWaiter(QObject):
+    finished = pyqtSignal()
+
+    def __init__(self):
+        super().__init__()
+
+    @pyqtSlot()
+    def work(self):
+        pass
+
+class JungfrauMissingConfiguration(Exception):
+    pass
+
+class Jungfrau(QObject, metaclass=qsingleton.Singleton):
+    pedestal_selected = pyqtSignal(str)
+    configured = pyqtSignal()
+    armed = pyqtSignal()
+    started = pyqtSignal()
+    triggered = pyqtSignal()
+    aborted = pyqtSignal()
+
+    def __init__(self, parent=None, **kwargs):
+        super(Jungfrau, self).__init__(parent, **kwargs)
+        self._simulated = kwargs.get("simulate", simulated)
+        self._save_raw = True
+        self._runJungfrau = False
+        self._number_of_frames = 1
+        self._gain_maps = config["gain_maps"]
+        self._uid = None
+        self._serial_number = config['serialnr']
+        self.writer_config = {}
+        self.backend_config = {}
+        self.detector_config = {}
+        self.bsread_config = {}
+
+    @pyqtSlot(bool)
+    def set_run_jungfrau(self, run):
+        """running or not jungfrau, global simulated flag overrides"""
+        logger.warning(f"{'' if run else 'not '}running jungfrau detector")
+        if not self._simulated:
+            self._runJungfrau = run
+        else:
+            self._runJungfrau = False
+
+    @pyqtSlot(bool)
+    def set_save_raw(self, save_raw):
+        """enable or not writer_config"""
+        logger.warning(f"{'' if save_raw else 'not '}saving data")
+        self._save_raw = save_raw
+
+    @pyqtSlot(str)
+    def set_detector_serial(self, serial_number):
+        """set serial number for main detector"""
+        logger.info(f"jungfrau serial number: {serial_number}")
+        self._serial_number = serial_number
+
+    def is_saving_data(self):
+        """if False writer_config is not issued"""
+        return self._save_raw
+
+    def is_running_detector(self):
+        """if False nothing is actually called"""
+        return self._runJungfrau
+
+    @pyqtSlot(int)
+    def set_number_of_frames(self, frames):
+        logger.info(f"jungfrau number of frames: {frames}")
+        self._number_of_frames = frames
+
+    @pyqtSlot(int)
+    def set_data_owner_uid(self, uid):
+        logger.info(f"data owner set to UID = {uid}")
+        self._uid = uid
+
+    @pyqtSlot(str)
+    def set_gain_filename(self, gainfile:str):
+        logger.info(f"gain file: {gainfile}")
+        self._gain_maps = gainfile
+
+    def configure(self, outfile=None, outdir=None, **kwargs):
+        if not self._runJungfrau:
+            logger.warning("jungfrau not required: configure dismissed")
+            return
+
+        if self._gain_maps is None:
+            raise JungfrauMissingConfiguration("missing gain maps filename")
+        gain_filename = self._gain_maps
+
+        if self._uid is None:
+            raise JungfrauMissingConfiguration("missing data owner p-group UID")
+        uid = self._uid
+
+        n_frames = self._number_of_frames
+
+        pede_pattern = "/sf/bernina/data/p{}/res/JF_pedestals/pedestal_*.JF07T32V01.res.h5".format(
+            uid
+        )
+        try:
+            pf = sorted(glob(pede_pattern))[-1]
+            pf = pf[: pf.index(".JF07T32V01")]
+        except IndexError:
+            pf = None
+
+        pede_filename = pf
+        logger.info(f"Pedestal file {pede_filename}")
+
+        exptime = 0.000005
+        instrument = "Bernina"
+
+        self.writer_config = {
+            "output_file": outdir + "/" + outfile,
+            "user_id": uid,
+            "n_frames": n_frames,
+            "general/user": str(uid),
+            "general/process": __name__,
+            "general/created": str(datetime.now()),
+            "general/instrument": instrument,
+        }
+
+        if not self._save_raw:
+            self.writer_config["output_file"] = "/dev/null"
+
+        self.detector_config = {
+            "exptime": exptime,
+            "frames": 1,
+            "cycles": n_frames,
+            "timing": "trigger",
+            "dr": 16,
+        }
+
+        self.backend_config = {
+            "n_frames": n_frames,
+            "bit_depth": 16,
+            "run_name": outfile,
+            "beam_center_x": 2110.0,
+            "beam_center_y": 2210.0,
+            "beam_energy": 6000.0,
+            "detector_distance": 0.06,
+            "hitfinder_min_snr": 6.0,
+            "hitfinder_min_pix_count": 5,
+            "hitfinder_adc_thresh": 20.0,
+            "swissmx_trajectory_method": "grid",
+            "swissmx_trajectory_details_1": [100],
+            "swissmx_trajectory_details_2": [n_frames // 100 + 1],
+        }
+
+        backend_extras = kwargs.get("backend_extras", None)
+        if backend_extras:
+            self.backend_config.update(backend_extras)
+
+        # """
+        # "swissmx_trajectory_method": "grid",
+        # "swissmx_trajectory_details_1": [100],
+        # "swissmx_trajectory_details_2": [n_frames//100+1],
+        #
+        # "swissmx_trajectory_method": "rotation",
+        # "swissmx_trajectory_details_1": [360],
+        # "swissmx_trajectory_details_2": [0.1],
+        #
+        # "swissmx_trajectory_method": "trajectory",
+        # "swissmx_trajectory_details_1": [random()*100 for i in range(int(n_frames*0.9))],
+        # "swissmx_trajectory_details_2": [random()*100 for i in range(int(n_frames*0.9))],
+        #
+        # """
+        self.bsread_config = {
+            "output_file": outdir + "/" + outfile,
+            "user_id": uid,
+            "general/user": str(uid),
+            "general/process": __name__,
+            "general/created": str(datetime.now()),
+            "general/instrument": instrument,
+        }
+
+        if gain_filename != "" or pede_filename != "":
+            self.backend_config["gain_corrections_filename"] = gain_filename
+            self.backend_config["gain_corrections_dataset"] = "gains"
+            self.backend_config["pede_corrections_filename"] = pede_filename
+            self.backend_config["pede_corrections_dataset"] = "gains"
+            self.backend_config["pede_mask_dataset"] = "pixel_mask"
+            self.backend_config["activate_corrections_preview"] = True
+            logger.info("Corrections in online viewer activated")
+
+        configuration = {
+            "writer": self.writer_config,
+            "backend": self.backend_config,
+            "detector": self.detector_config,
+            "bsread": self.bsread_config,
+        }
+
+        logger.info("resetting & configuring detector: approx. 6 seconds")
+        dia_client.reset()
+        dia_client.set_config(configuration)
+        logger.info(dia_client.get_config())
+
+    def arm(self) -> int:
+        """arm detector and return an ID for this series"""
+        if not self._runJungfrau:
+            logger.warning("jungfrau not required: arm dismissed")
+            return
+
+        logger.info("arming detector: approx. 2 seconds")
+        dia_client.start(trigger_start=False)
+
+    def wait_finished(self):
+        """close shutter after this wait is finished regardless of exceptions"""
+        if not self._runJungfrau:
+            logger.warning("jungfrau not required: wait_finished dismissed")
+            return
+
+        try:
+            dia_client.wait_for_status(["IntegrationStatus.FINISHED"], polling_interval=0.1)
+        except:
+            logger.info("Got IntegrationStatus ERROR")
+            logger.info(dia_client.get_status())
+            logger.info(dia_client.get_status_details())
+
+        logger.info("Stopping acquisition")
+        dia_client.reset()
+
+        logger.info("Done")
+
+    def disarm(self) -> int:
+        """arm detector and return an ID for this series"""
+        if not self._runJungfrau:
+            logger.warning("jungfrau not required: disarm dismissed")
+            return
+
+        dia_client.reset()
+
+    def trigger(self):
+        if not self._runJungfrau:
+            logger.warning("jungfrau not required: trigger dismissed")
+            return
+        logger.info("Trigger in detector is called")
+        eventer.start()
+
+    def abort(self):
+        if not self._runJungfrau:
+            logger.warning("jungfrau not required: abort dismissed")
+            return
+
+        self.disarm()
+        eventer.stop()
+
+
+jungfrau_detector = Jungfrau()
+
+def main():
+    import argparse
+
+    date_string = datetime.now().strftime("%Y%m%d_%H%M")
+
+    parser = argparse.ArgumentParser(description="Create a pedestal file for Jungrau")
+    parser.add_argument("--api", default="http://sf-daq-1:10000", required=True)
+    parser.add_argument(
+        "--filename", default="run_%s.h5" % date_string, help="Output file name"
+    )
+    parser.add_argument(
+        "--pede", default="", help="File containing pedestal corrections"
+    )
+    parser.add_argument("--gain", default="", help="File containing gain corrections")
+    parser.add_argument(
+        "--directory", default="/sf/bernina/data/raw/p16582", help="Output directory"
+    )
+    parser.add_argument(
+        "--uid", default=16582, help="User ID which needs to own the file", type=int
+    )
+    parser.add_argument(
+        "--period", default=0.01, help="Period (default is 10Hz - 0.01)", type=float
+    )
+    parser.add_argument(
+        "--exptime",
+        default=0.000010,
+        help="Integration time (default 0.000010 - 10us)",
+        type=float,
+    )
+    parser.add_argument(
+        "--frames", default=10, help="Number of frames to take", type=int
+    )
+    parser.add_argument(
+        "--save", default=False, help="Save data file", action="store_true"
+    )
+    parser.add_argument(
+        "--highgain", default=False, help="Enable High Gain (HG0)", action="store_true"
+    )
+    parser.add_argument(
+        "--instrument", default="", help="Name of the instrument, e.g. Alvra"
+    )
+    # parser.add_argument("--caput", default=False, help="Use the CAPUT trick (experts only!!!)", action="store_true")
+
+    args = parser.parse_args()

pyqtUsrObj.py

@@ -149,6 +149,9 @@ class Grid(pg.ROI):
     self.addScaleHandle([0, 0], [1, 1])
     self.addScaleRotateHandle([1, 0], [0, 0])
 
+  def get_points(self):
+    return np.array(((1,2),(3,4),(5,6)),dtype=np.float)
+
   def paint(self, p, *args):
     #pg.ROI.paint(self, p, *args)
     sz=self.state['size']

swissmx.py

@@ -2194,14 +2194,6 @@ class Main(QMainWindow, Ui_MainWindow):
     params = (xp[:, 0].tolist(), xp[:, 1].tolist())
     self.daq_collect_points(points, visualizer_method=method, visualizer_params=params)
 
-  def daq_grid_collect_grid(self):
-    grid = self._grids[0]  # FIXME one grid at a time only
-    points = np.array(grid.get_grid_targets())
-    method = "grid"
-    params = grid._grid_dimensions
-    # params = ([grid._grid_dimensions[0]], [grid._grid_dimensions[1]]) # FIXME something wrong here<
-    self.daq_collect_points(points, visualizer_method=method, visualizer_params=params)
-
   def daq_grid_findxtals(self):
     feature_size = self._sb_findxtals_feature_size.value()
     image = sample_camera.get_image()
@@ -2232,27 +2224,75 @@ class Main(QMainWindow, Ui_MainWindow):
     return True
 
   def daq_collect_points(self, points, visualizer_method, visualizer_params):
+    app = QApplication.instance()
+    cfg = app._cfg
+    verbose=0xff
+    fn='/tmp/shapepath'
+    try:
+      dt=app._deltatau
+    except AttributeError:
+      import matplotlib.pyplot as plt
+      import deltatau
+      app._deltatau=dt=deltatau.Deltatau()
+    try:
+      jf=app._jungfrau
+    except AttributeError:
+      import detector
+      app._jungfrau=jf=detector.Jungfrau()
+
+      dt=app._deltatau
+
+
+    sp=dt._shapepath
+
+    sp.gen_grid_points(w=15, h=15, pitch=3, rnd=0, ofs=(0, +2000));
+    sp.sort_points(False, 15);
+    sp.meta['pt2pt_time']=10
+
+    gtMaxLn=116508
+    ovhdTime=100
+    acq_per=int(np.ceil((sp.meta['pt2pt_time']*sp.points.shape[0]+ovhdTime)/(gtMaxLn*sp.meta['srv_per'])))
+    sp.setup_gather(acq_per=acq_per)
+    sp.setup_sync(verbose=verbose&32, timeOfs=0.05)
+    sp.setup_coord_trf()  # reset to shape path system
+    # sp.meta['pt2pt_time']=10 #put between setup_sync and setup_motion to have more motion points than FEL syncs
+    sp.setup_motion(fnPrg=fn+'.prg', mode=3, scale=1., dwell=10)
+    sp.homing()  # homing if needed
+    sp.run()  # start motion program
+    sp.wait_armed()  # wait until motors are at first position
+    sp.trigger(0.5)  # send a start trigger (if needed) ater given time
+    if not dt._comm is None:
+      while True:
+        p=sp.progress()
+        if p<0: break
+        print('progress %d/%d'%(p, sp.points.shape[0]));
+        time.sleep(.1)
+      sp.gather_upload(fnRec=fn+'.npz')
+      dp=deltatau.shapepath.DebugPlot(sp);
+      dp.plot_gather(mode=11)
+
+    print('done')
+    plt.show(block=False)
+    return
+
+
+
     task = self.active_task()
     XDIR = -1
 
-    folders.make_if_needed()
+    #folders.make_if_needed()
+    #if ( cfg.option(AppCfg.ACTIVATE_PULSE_PICKER) and not jungfrau_detector.is_running_detector()):
+    #  if QMessageBox.No == QMessageBox.question(self, "X-rays but no Jungfrau",
+    #                                            "X-rays will be used bu the Jungfrau will not run.\n\n\tContinue?",):
+    #    _log.warning("user forgot to turn on the jungfrau")
+    #    return
 
-    if ( option(ACTIVATE_PULSE_PICKER) and not jungfrau_detector.is_running_detector()):
+    #if option(ACTIVATE_PULSE_PICKER) or not option(SKIP_ESCAPE_TRANSITIONS_IF_SAFE):
-      if QMessageBox.No == QMessageBox.question(self, "X-rays but no Jungfrau",
+    #  self.escape_goToDataCollection()
-                                                "X-rays will be used bu the Jungfrau will not run.\n\n\tContinue?",):
-        _log.warning("user forgot to turn on the jungfrau")
-        return
 
-    if option(ACTIVATE_PULSE_PICKER) or not option(SKIP_ESCAPE_TRANSITIONS_IF_SAFE):
-      self.escape_goToDataCollection()
-
-    ntrigger = len(points)
     points *= 1000  # delta tau uses micrometers
     points[:, 0] *= XDIR  # fast X axis is reversed
 
-    etime = settings.value("exposureTime", type=float)
-    vscale = settings.value(DELTATAU_VELOCITY_SCALE, 1.0, type=float)
-    sort_points = option(DELTATAU_SORT_POINTS)
 
     # sync_mode : default=2
     #         0 : no sync at all
@@ -2264,38 +2304,48 @@ class Main(QMainWindow, Ui_MainWindow):
     #             sync_run are the commands to run the whole program
     # sync_flag if not using jungfrau =1 otherwise =0
     # D.O.        shapepath.meta.update(sync_mode=2, sync_flag=1)
-    shapepath.meta.update(sync_mode=0, sync_flag=0)
+    sp.meta.update(sync_mode=0, sync_flag=0)
+
     maxacq_points = 116508
     samp_time = 0.0002  # s
     overhead_time = 0.1
-    acq_per = int(np.ceil((etime * ntrigger + overhead_time) / (maxacq_points * samp_time)))
+    etime=10
+    vscale=1.0
+    #etime = settings.value("exposureTime", type=float)
+    #vscale = settings.value(DELTATAU_VELOCITY_SCALE, 1.0, type=float)
+    #sort_points = option(DELTATAU_SORT_POINTS)
+
+    acq_per = int(np.ceil((etime * len(points) + overhead_time) / (maxacq_points * samp_time)))
     _log.info(f"gather acquisotion period = {acq_per}")
     _log.info(f"velocity scale {vscale}")
-    shapepath.setup_gather(acq_per=acq_per)
+    sp.setup_gather(acq_per=acq_per)
-    shapepath.setup_sync(verbose=True)
+    sp.setup_sync(verbose=True)
-    shapepath.setup_coord_trf()
+    sp.setup_coord_trf()
-    shapepath.points = np.copy(points)
+
+    assert(points.dtcfgype==np.float)
+    sp.points = points
 
     if TASK_GRID == task:
-      width, height = visualizer_params
+      # width, height = visualizer_params
-      _log.debug(f"grid: {width} x {height}")
+      # _log.debug(f"grid: {width} x {height}")
-      details_1 = [width]
+      # details_1 = [width]
-      details_2 = [height]
+      # details_2 = [height]
-      shapepath.sort_points(xy=False, grp_sz=height)
+      # sp.sort_points(xy=False, grp_sz=height)
+      pass
     elif task in (TASK_PRELOCATED, TASK_EMBL):
       if sort_points:
         shapepath.sort_points()
       self.daq_method_prelocated_remove_markers()
       details_1, details_2 = visualizer_params
 
-    shapepath.setup_motion(
+    sp.setup_motion(
       mode=3,  # 1 = bad pvt 3 = pft (pvt via inverse fourier transform)
       pt2pt_time=etime * 1000.,
-      fnPrg=folders.get_prefixed_file("_program.prg"),
+      #fnPrg=folders.get_prefixed_file("_program.prg"),
       scale=vscale,  # velocity at target position scaling: 1=optimal speed, 0=zero speed
       dwell=10,  # milli-seconds wait
     )
-    shapepath.run()
+    sp.run()
 
     self.qprogress = QProgressDialog(self)
     self.qprogress.setRange(0, 0)
@@ -2735,7 +2785,14 @@ class Main(QMainWindow, Ui_MainWindow):
       )
       self._inspect = self._grid_inspect_area
       self._inspect.setPlainText("")
-      self.daq_grid_collect_grid()
+      for fixTrg in self._goTracked['objLst']:
+        points=fixTrg.get_points()
+        method="grid"
+        params=None #grid._grid_dimensions
+        # params = ([grid._grid_dimensions[0]], [grid._grid_dimensions[1]]) # FIXME something wrong here<
+        self.daq_collect_points(points, visualizer_method=method, visualizer_params=params)
+
+      #self.daq_grid_collect_grid()
     elif task == TASK_PRELOCATED:
       self._inspect = self._preloc_inspect_area
       self._inspect.setPlainText("")