ctb: add patternstart command, xilinx: fix frequency (#1307)

* add patternstart command for CTB, block end of execution udp packets if pattern was started by patternstart command

* update docs

* Dhanya's comments

* more Dhanya comments

* refactored

* fixed tests for startpatttern, also clkfrequency not properly used in server

* xilinx: fixed setfrequency, tick clock (with sync clock), clkfrequency set from getfrequency to get the exact value

* xilinx freq in kHz, updated default values and prints

---------

Co-authored-by: Martin Mueller <martin.mueller@psi.ch>
Co-authored-by: Dhanya Thattil <dhanya.thattil@psi.ch>

docs/src/pattern.rst

@@ -28,7 +28,7 @@ This instructs the firmware to execute the commands from address 0 to 4 (includi
 .. code-block::
 
    start [Ctb, Xilinx_Ctb]
-   patternstart [Mythen3, Xilinx_Ctb]
+   patternstart [Mythen3, Ctb, Xilinx_Ctb]
 
 The maximal number of patword addresses is 8192. However, it is possible to extend the length of the pattern sequence using loops and wait commands. Loops can be configured with the following commands:
 
@@ -70,11 +70,11 @@ The mappings of bit positions in the pattern word to signals/pads of the FPGA ar
 
 .. table:: 
 
-   +----+---+------+----+----------+-------------------+----------------+
-   | 63 | 62| 61-57| 56 |  55-48   |  47-32            |  31-0          |
-   +----+---+------+----+----------+-------------------+----------------+
-   |  A |  D|  --- |  T | EXTIO    | DO, stream source | DIO            |
-   +----+---+------+----+----------+-------------------+----------------+
+   +----+---+------+----+----------+----------+----------------+
+   | 63 | 62| 61-57| 56 |  55-48   |  47-32   |  31-0          |
+   +----+---+------+----+----------+----------+----------------+
+   |  A |  D|  --- |  T | EXTIO    | DO       | DIO            |
+   +----+---+------+----+----------+----------+----------------+
 
 DIO: Driving the 32 FPGA pins corresponding to the lowest 32 bits of the patioctrl command. If bits in patioctrl are 0, the same bit positions in DIO will switch to input pins and connect to dbit sampling. Additionally, some of these 32 bits have an automatic override by detector-specific statemachines which is active whenever one of these statemachines is running (currently bits 7,8,11,14 and 20).
 
@@ -120,3 +120,11 @@ DIO: Driving the 32 FPGA pins corresponding to the lowest 32 bits of the patioct
    +---------+-----+-------+-------+----+-------+---------+--------+
 
 For Mythen3 the pattern word only connects to output pins of the FPGA when the pattern is running. Afterwards the signals will switch back to other logic in the FPGA. Both CTB's hold the last executed pattern word until a new pattern is started.
+
+**Relation of received data to pattern execution**
+
+In the default configuration the Ctb will send out udp packets to the sls_receiver for every end of a pattern execution. This behavior can be changed using STREAMING_CTRL_REG, where one can configure a bit position in the 64-bit pattern word to trigger udp packets. This allows to send more than one packet per pattern or also no packets at all.
+
+The "patternstart" command on the ctb exectues the pattern. As long as streaming_ctrl_reg is disabeld, every pattern execution using this command will not send UDP packets.
+
+For Mythen3 the sending of udp packets is not connected to pattern execution.

slsDetectorServers/ctbDetectorServer/RegisterDefs.h

@@ -384,6 +384,13 @@
 #define PLL_CNTRL_ADDR_OFST     (16)
 #define PLL_CNTRL_ADDR_MSK      (0x0000003F << PLL_CNTRL_ADDR_OFST)
 
+/* Streaming Control RW regiser */
+#define STREAMING_CTRL_REG         (0x3D << MEM_MAP_SHIFT)
+#define STREAMING_CTRL_ENA_OFST    (15)
+#define STREAMING_CTRL_ENA_MSK     (0x00000001 << STREAMING_CTRL_ENA_OFST)
+#define STREAMING_CTRL_SELECT_OFST (0)
+#define STREAMING_CTRL_SELECT_MSK  (0x0000003F << STREAMING_CTRL_SELECT_OFST)
+
 /* Pattern Control RW register */
 #define PATTERN_CNTRL_REG (0x88 << MEM_MAP_SHIFT)
 

slsDetectorServers/slsDetectorServer/include/XILINX_PLL.h

@@ -4,7 +4,7 @@
 #include <stdbool.h>
 #include <stdint.h>
 
-void XILINX_PLL_setFrequency(uint32_t clk_index, uint32_t freq);
+int XILINX_PLL_setFrequency(uint32_t clk_index, uint32_t freq);
 uint32_t XILINX_PLL_getFrequency(uint32_t clkIDX);
 bool XILINX_PLL_isLocked();
 void XILINX_PLL_reset();

slsDetectorServers/slsDetectorServer/include/loadPattern.h

@@ -58,7 +58,8 @@ uint64_t getPatternMask();
 void setPatternBitMask(uint64_t mask);
 uint64_t getPatternBitMask();
 
-#if defined(MYTHEN3D) || defined(XILINX_CHIPTESTBOARDD)
+#if defined(MYTHEN3D) || defined(XILINX_CHIPTESTBOARDD) ||                     \
+    defined(CHIPTESTBOARDD)
 void startPattern();
 #endif
 char *getPatternFileName();

slsDetectorServers/slsDetectorServer/src/XILINX_PLL.c

@@ -72,14 +72,14 @@
 // clang-format on
 
 // freq in kHz !!
-void XILINX_PLL_setFrequency(uint32_t clk_index, uint32_t freq) {
+int XILINX_PLL_setFrequency(uint32_t clk_index, uint32_t freq) {
     if (clk_index >= XILINX_PLL_NUM_CLKS) {
         LOG(logERROR, ("XILINX_PLL: Invalid clock index %d\n", clk_index));
-        return;
+        return 1;
     }
     if (freq < XILINX_PLL_MIN_FREQ || freq > XILINX_PLL_MAX_FREQ) {
         LOG(logERROR, ("XILINX_PLL: Frequency %d kHz is out of range\n", freq));
-        return;
+        return 1;
     }
 
     // calculate base clock frequency
@@ -103,7 +103,7 @@ void XILINX_PLL_setFrequency(uint32_t clk_index, uint32_t freq) {
     if (clk_div < 1 || clk_div > XILINX_PLL_MAX_CLK_DIV) {
         LOG(logERROR,
             ("XILINX_PLL: Invalid clock divider, need to change base clock\n"));
-        return;
+        return 1;
     }
 
     uint32_t clk_div_frac = 0;
@@ -140,18 +140,19 @@ void XILINX_PLL_setFrequency(uint32_t clk_index, uint32_t freq) {
 
     // wait for firmware to measure the actual frequency
     usleep(2 * 1000 * 1000);
+    return 0;
 }
 
 uint32_t XILINX_PLL_getFrequency(uint32_t clk_index) {
     if (clk_index >= XILINX_PLL_NUM_CLKS) {
         LOG(logERROR, ("XILINX_PLL: Invalid clock index %d\n", clk_index));
-        return 0;
+        return -1;
     }
     if (clk_index > XILINX_PLL_MAX_NUM_CLKS_FOR_GET) {
         LOG(logERROR,
             ("XILINX_PLL: get frequency not implemented for this clock %d\n",
              clk_index));
-        return 0;
+        return -1;
     }
 
     uint32_t base_addr = XILINX_PLL_MEASURE_BASE_ADDR0;

slsDetectorServers/slsDetectorServer/src/loadPattern.c

@@ -623,28 +623,43 @@ uint64_t getPatternBitMask() {
     return getU64BitReg(PATTERN_SET_LSB_REG, PATTERN_SET_MSB_REG);
 }
 
-#ifdef MYTHEN3D
 void startPattern() {
     LOG(logINFOBLUE, ("Starting Pattern\n"));
+#ifdef MYTHEN3D
     bus_w(CONTROL_REG, bus_r(CONTROL_REG) | CONTROL_STRT_PATTERN_MSK);
     usleep(1);
     while (bus_r(PAT_STATUS_REG) & PAT_STATUS_RUN_BUSY_MSK) {
         usleep(1);
     }
-    LOG(logINFOBLUE, ("Pattern done\n"));
-}
-#endif
-#ifdef XILINX_CHIPTESTBOARDD
-void startPattern() {
-    LOG(logINFOBLUE, ("Starting Pattern\n"));
+#elif CHIPTESTBOARDD
+    // we only want to run the pattern here. No acquisition, no UDP packets
+
+    // disable 10G UDP temporarily
+    // except if the pattern explicitly contains udp trigger points
+    uint32_t conf_reg_tmp = bus_r(CONFIG_REG);
+    if ((bus_r(STREAMING_CTRL_REG) & STREAMING_CTRL_ENA_MSK) == 0) {
+        bus_w(CONFIG_REG, conf_reg_tmp & ~CONFIG_GB10_SND_UDP_MSK);
+    }
+
+    // run the pattern, wait till done
+    bus_w(CONTROL_REG, bus_r(CONTROL_REG) | CONTROL_STRT_ACQSTN_MSK);
+    bus_w(CONTROL_REG, bus_r(CONTROL_REG) & ~CONTROL_STRT_ACQSTN_MSK);
+    usleep(1);
+    while (bus_r(STATUS_REG) & STATUS_RN_BSY_MSK) {
+        usleep(1);
+    }
+
+    // go back to original config
+    bus_w(CONFIG_REG, conf_reg_tmp);
+#elif XILINX_CHIPTESTBOARDD
     bus_w(FLOW_CONTROL_REG, bus_r(FLOW_CONTROL_REG) | START_F_MSK);
     usleep(1);
     while (bus_r(FLOW_CONTROL_REG) & RSM_BUSY_MSK) {
         usleep(1);
     }
+#endif
     LOG(logINFOBLUE, ("Pattern done\n"));
 }
-#endif
 
 char *getPatternFileName() { return clientPatternfile; }
 

slsDetectorServers/slsDetectorServer/src/slsDetectorServer_funcs.c

@@ -5839,15 +5839,24 @@ int set_clock_frequency(int file_des) {
                 LOG(logINFO, ("Same %s: %d %s\n", modeName, val,
                               myDetectorType == GOTTHARD2 ? "Hz" : "MHz"));
             } else {
-                setFrequency(c, val);
-                int retval = getFrequency(c);
-                LOG(logDEBUG1, ("retval %s: %d %s\n", modeName, retval,
-                                myDetectorType == GOTTHARD2 ? "Hz" : "MHz"));
-#if !defined(                                                                  \
-    XILINX_CHIPTESTBOARDD) // XCTB will give the actual frequency, which is not
-                           // 100% identical to the set frequency
-                validate(&ret, mess, val, retval, modeName, DEC);
+                int ret = setFrequency(c, val);
+                if (ret == FAIL) {
+                    sprintf(mess, "Could not set %s to %d %s\n", modeName, val,
+                            myDetectorType == XILINX_CHIPTESTBOARD ? "kHz"
+                                                                   : "MHz");
+                    LOG(logERROR, (mess));
+                } else {
+                    int retval = getFrequency(c);
+                    LOG(logDEBUG1,
+                        ("retval %s: %d %s\n", modeName, retval,
+                         myDetectorType == XILINX_CHIPTESTBOARD ? "kHz"
+                                                                : "MHz"));
+#if !defined(XILINX_CHIPTESTBOARDD)
+                    // XCTB will give the actual frequency, which is not
+                    // 100% identical to the set frequency
+                    validate(&ret, mess, val, retval, modeName, DEC);
 #endif
+                }
             }
         }
     }
@@ -5902,8 +5911,11 @@ int get_clock_frequency(int file_des) {
         LOG(logDEBUG1,
             ("retval %s clock (%d) frequency: %d %s\n", clock_names[c], (int)c,
              retval,
-             myDetectorType == GOTTHARD2 || myDetectorType == MYTHEN3 ? "Hz"
-                                                                      : "MHz"));
+             myDetectorType == XILINX_CHIPTESTBOARD
+                 ? "kHz"
+                 : (myDetectorType == GOTTHARD2 || myDetectorType == MYTHEN3
+                        ? "Hz"
+                        : "MHz")));
     }
 #endif
     return Server_SendResult(file_des, INT32, &retval, sizeof(retval));
@@ -7468,7 +7480,8 @@ int start_pattern(int file_des) {
     memset(mess, 0, sizeof(mess));
 
     LOG(logDEBUG1, ("Starting Pattern\n"));
-#if !defined(MYTHEN3D) && !defined(XILINX_CHIPTESTBOARDD)
+#if !defined(MYTHEN3D) && !defined(XILINX_CHIPTESTBOARDD) &&                   \
+    !defined(CHIPTESTBOARDD)
     functionNotImplemented();
 #else
     // only set

slsDetectorServers/xilinx_ctbDetectorServer/slsDetectorFunctionList.c

@@ -40,8 +40,7 @@ char initErrorMessage[MAX_STR_LENGTH];
 
 int detPos[2] = {0, 0};
 
-uint32_t clkFrequency[NUM_CLOCKS] = {20, 100, 20, 100};
-
+uint32_t clkFrequency[NUM_CLOCKS] = {};
 int chipConfigured = 0;
 int analogEnable = 0;
 int digitalEnable = 0;
@@ -376,6 +375,10 @@ void setupDetector() {
     LOG(logINFO, ("Setting up Server for 1 Xilinx Chip Test Board\n"));
 
     // default variables
+    clkFrequency[RUN_CLK] = DEFAULT_RUN_CLK;
+    clkFrequency[ADC_CLK] = DEFAULT_ADC_CLK;
+    clkFrequency[SYNC_CLK] = DEFAULT_SYNC_CLK;
+    clkFrequency[DBIT_CLK] = DEFAULT_DBIT_CLK;
     chipConfigured = 0;
     analogEnable = 0;
     digitalEnable = 0;
@@ -1061,12 +1064,12 @@ int setPeriod(int64_t val) {
         return FAIL;
     }
     LOG(logINFO, ("Setting period %lld ns\n", (long long int)val));
-    val *= (1E-3 * RUN_CLK);
+    val *= (1E-3 * clkFrequency[RUN_CLK]);
     setU64BitReg(val, PERIOD_IN_REG_1, PERIOD_IN_REG_2);
 
     // validate for tolerance
     int64_t retval = getPeriod();
-    val /= (1E-3 * RUN_CLK);
+    val /= (1E-3 * clkFrequency[RUN_CLK]);
     if (val != retval) {
         return FAIL;
     }
@@ -1074,7 +1077,8 @@ int setPeriod(int64_t val) {
 }
 
 int64_t getPeriod() {
-    return getU64BitReg(PERIOD_IN_REG_1, PERIOD_IN_REG_2) / (1E-3 * RUN_CLK);
+    return getU64BitReg(PERIOD_IN_REG_1, PERIOD_IN_REG_2) /
+           (1E-3 * clkFrequency[RUN_CLK]);
 }
 
 int setDelayAfterTrigger(int64_t val) {
@@ -1083,12 +1087,12 @@ int setDelayAfterTrigger(int64_t val) {
         return FAIL;
     }
     LOG(logINFO, ("Setting delay after trigger %ld ns\n", val));
-    val *= (1E-3 * RUN_CLK);
+    val *= (1E-3 * clkFrequency[RUN_CLK]);
     setU64BitReg(val, DELAY_IN_REG_1, DELAY_IN_REG_2);
 
     // validate for tolerance
     int64_t retval = getDelayAfterTrigger();
-    val /= (1E-3 * RUN_CLK);
+    val /= (1E-3 * clkFrequency[RUN_CLK]);
     if (val != retval) {
         return FAIL;
     }
@@ -1096,7 +1100,8 @@ int setDelayAfterTrigger(int64_t val) {
 }
 
 int64_t getDelayAfterTrigger() {
-    return getU64BitReg(DELAY_IN_REG_1, DELAY_IN_REG_2) / (1E-3 * RUN_CLK);
+    return getU64BitReg(DELAY_IN_REG_1, DELAY_IN_REG_2) /
+           (1E-3 * clkFrequency[RUN_CLK]);
 }
 
 int64_t getNumFramesLeft() {
@@ -1108,11 +1113,13 @@ int64_t getNumTriggersLeft() {
 }
 
 int64_t getDelayAfterTriggerLeft() {
-    return getU64BitReg(DELAY_OUT_REG_1, DELAY_OUT_REG_2) / (1E-3 * RUN_CLK);
+    return getU64BitReg(DELAY_OUT_REG_1, DELAY_OUT_REG_2) /
+           (1E-3 * clkFrequency[RUN_CLK]);
 }
 
 int64_t getPeriodLeft() {
-    return getU64BitReg(PERIOD_OUT_REG_1, PERIOD_OUT_REG_2) / (1E-3 * RUN_CLK);
+    return getU64BitReg(PERIOD_OUT_REG_1, PERIOD_OUT_REG_2) /
+           (1E-3 * clkFrequency[RUN_CLK]);
 }
 
 int64_t getFramesFromStart() {
@@ -1122,12 +1129,12 @@ int64_t getFramesFromStart() {
 
 int64_t getActualTime() {
     return getU64BitReg(TIME_FROM_START_OUT_REG_1, TIME_FROM_START_OUT_REG_2) /
-           (1E-3 * TICK_CLK);
+           (1E-3 * clkFrequency[SYNC_CLK]);
 }
 
 int64_t getMeasurementTime() {
     return getU64BitReg(FRAME_TIME_OUT_REG_1, FRAME_TIME_OUT_REG_2) /
-           (1E-3 * TICK_CLK);
+           (1E-3 * clkFrequency[SYNC_CLK]);
 }
 
 /* parameters - dac, adc, hv */
@@ -1792,10 +1799,16 @@ int setFrequency(enum CLKINDEX ind, int val) {
     }
 
     char *clock_names[] = {CLK_NAMES};
-    LOG(logINFO, ("\tSetting %s clock (%d) frequency to %d MHz\n",
+    LOG(logINFO, ("\tSetting %s clock (%d) frequency to %d kHz\n",
                   clock_names[ind], ind, val));
 
-    XILINX_PLL_setFrequency(ind, val);
+    if (XILINX_PLL_setFrequency(ind, val) == FAIL) {
+        LOG(logERROR, ("\tCould not set %s clock (%d) frequency to %d kHz\n",
+                       clock_names[ind], ind, val));
+        return FAIL;
+    }
+    clkFrequency[ind] = val;
+    // TODO later: connect setPhase as phase gets reset on freq change
     return OK;
 }
 
@@ -1804,5 +1817,8 @@ int getFrequency(enum CLKINDEX ind) {
         LOG(logERROR, ("Unknown clock index %d to get frequency\n", ind));
         return -1;
     }
-    return XILINX_PLL_getFrequency(ind);
+#ifndef VIRTUAL
+    clkFrequency[ind] = XILINX_PLL_getFrequency(ind);
+#endif
+    return clkFrequency[ind];
 }

slsDetectorServers/xilinx_ctbDetectorServer/slsDetectorServer_defs.h

@@ -71,8 +71,6 @@
 #define POWER_RGLTR_MAX (2661)
 #define VIO_MIN_MV      (1200) // for fpga to function
 
-#define TICK_CLK (20) // MHz (trig_timeFromStart, frametime, timeFromStart)
-
 /* Defines in the Firmware */
 #define WAIT_TIME_PATTERN_READ  (10)
 #define WAIT_TIME_OUT_0US_TIMES (35000) // 2s
@@ -157,3 +155,8 @@ typedef struct udp_header_struct {
 
 enum CLKINDEX { RUN_CLK, ADC_CLK, SYNC_CLK, DBIT_CLK, NUM_CLOCKS };
 #define CLK_NAMES "run", "adc", "sync", "dbit"
+
+#define DEFAULT_RUN_CLK  (20000)  // 20 MHz
+#define DEFAULT_ADC_CLK  (100000) // 100 MHz
+#define DEFAULT_SYNC_CLK (20000)  // 20 MHz
+#define DEFAULT_DBIT_CLK (100000) // 100 MHz

slsDetectorSoftware/generator/commands.yaml

@@ -1791,7 +1791,7 @@ defaultpattern:
 
 patternstart:
   inherit_actions: EXECUTE_SET_COMMAND
-  help: "\n\t[Mythen3][Xilinx Ctb] Starts Pattern"
+  help: "\n\t[Mythen3][Ctb][Xilinx Ctb] Starts Pattern"
   actions:
     PUT:
       function: startPattern

slsDetectorSoftware/generator/extended_commands.yaml

@@ -6517,7 +6517,7 @@ patternstart:
         store_result_in_t: false
   command_name: patternstart
   function_alias: patternstart
-  help: "\n\t[Mythen3][Xilinx Ctb] Starts Pattern"
+  help: "\n\t[Mythen3][Ctb][Xilinx Ctb] Starts Pattern"
   infer_action: true
   template: true
 patwait:

slsDetectorSoftware/include/sls/Detector.h

@@ -1943,7 +1943,7 @@ class Detector {
      * selected bits */
     void setPatternBitMask(uint64_t mask, Positions pos = {});
 
-    /** [Mythen3][Xilinx CTB] */
+    /** [CTB][Mythen3][Xilinx CTB] */
     void startPattern(Positions pos = {});
     ///@}
 

slsDetectorSoftware/tests/Caller/test-Caller-pattern.cpp

@@ -436,7 +436,8 @@ TEST_CASE("patternstart", "[.cmdcall]") {
     Caller caller(&det);
     REQUIRE_THROWS(caller.call("patternstart", {}, -1, GET));
     auto det_type = det.getDetectorType().squash();
-    if (det_type == defs::MYTHEN3) {
+    if (det_type == defs::MYTHEN3 || det_type == defs::CHIPTESTBOARD ||
+        det_type == defs::XILINX_CHIPTESTBOARD) {
         REQUIRE_NOTHROW(caller.call("patternstart", {}, -1, PUT));
     } else {
         REQUIRE_THROWS(caller.call("patternstart", {}, -1, PUT));

slsSupportLib/include/sls/versionAPI.h

@@ -3,10 +3,10 @@
 /** API versions */
 #define APILIB       "0.0.0 0x250909"
 #define APIRECEIVER  "0.0.0 0x250822"
-#define APICTB       "0.0.0 0x250909"
+#define APICTB       "0.0.0 0x250922"
 #define APIGOTTHARD2 "0.0.0 0x250909"
 #define APIMOENCH    "0.0.0 0x250909"
 #define APIEIGER     "0.0.0 0x250909"
-#define APIXILINXCTB "0.0.0 0x250916"
+#define APIXILINXCTB "0.0.0 0x250923"
 #define APIJUNGFRAU  "0.0.0 0x250909"
-#define APIMYTHEN3   "0.0.0 0x250909"
+#define APIMYTHEN3   "0.0.0 0x250922"