diff --git a/python/slsdet/detector.py b/python/slsdet/detector.py index 581de9337..c00cda762 100755 --- a/python/slsdet/detector.py +++ b/python/slsdet/detector.py @@ -2652,7 +2652,12 @@ class Detector(CppDetectorApi): @property @element def tengiga(self): - """[Eiger][Ctb][Mythen3] 10GbE Enable.""" + """[Eiger][Ctb][Xilinx Ctb][Mythen3] 10GbE Enable. + + Note + ----- + [Xilinx Ctb] Default is enabled. If disabled, the detector server reads out only the digital and transceiver data and sends them via the 1GbE interface. Analog readout is not implemented for 1GbE. + """ return self.getTenGiga() @tengiga.setter diff --git a/slsDetectorServers/slsDetectorServer/include/slsDetectorServer_funcs.h b/slsDetectorServers/slsDetectorServer/include/slsDetectorServer_funcs.h index b61751541..d09b38cd6 100644 --- a/slsDetectorServers/slsDetectorServer/include/slsDetectorServer_funcs.h +++ b/slsDetectorServers/slsDetectorServer/include/slsDetectorServer_funcs.h @@ -52,7 +52,8 @@ int set_settings(int); int get_threshold_energy(int); int acquire(int blocking, int file_des); void *start_state_machine(void *arg); -#if defined(CHIPTESTBOARDD) && !defined(VIRTUAL) +#if (defined(CHIPTESTBOARDD) || defined(XILINX_CHIPTESTBOARDD)) && \ + !defined(VIRTUAL) void *start_reading_and_sending_udp_frames(void *arg); #endif int start_acquisition(int); diff --git a/slsDetectorServers/slsDetectorServer/src/slsDetectorServer_funcs.c b/slsDetectorServers/slsDetectorServer/src/slsDetectorServer_funcs.c index 6075ae064..929601b53 100644 --- a/slsDetectorServers/slsDetectorServer/src/slsDetectorServer_funcs.c +++ b/slsDetectorServers/slsDetectorServer/src/slsDetectorServer_funcs.c @@ -1900,6 +1900,17 @@ int acquire(int blocking, int file_des) { LOG(logERROR, (mess)); } else #endif +#if defined(XILINX_CHIPTESTBOARDD) + // adcs do not exist right now + if (!enableTenGigabitEthernet(GET_FLAG) && + (getReadoutMode() == ANALOG_ONLY || + getReadoutMode() == ANALOG_AND_DIGITAL)) { + ret = FAIL; + strcpy(mess, "Could not start acquisition. Analog readout is not " + "implemented.\n"); + LOG(logERROR, (mess)); + } else +#endif #ifdef EIGERD // check for hardware mac and hardware ip if (udpDetails[0].srcmac != getDetectorMAC()) { @@ -2022,7 +2033,7 @@ void *start_state_machine(void *arg) { ret = startStateMachine(); LOG(logDEBUG2, ("Starting Acquisition ret: %d\n", ret)); if (ret == FAIL) { -#if defined(CHIPTESTBOARDD) || defined(VIRTUAL) +#if defined(CHIPTESTBOARDD) || defined(XILINX_CHIPTESTBOARDD) || defined(VIRTUAL) sprintf(mess, "Could not start acquisition. Could not create udp " "socket in server. Check udp_dstip & udp_dstport.\n"); #else @@ -2036,9 +2047,9 @@ void *start_state_machine(void *arg) { } break; } -// only 1g real ctb needs to read from fifo +// only 1g real ctb and xilinx ctb needs to read from fifo // to avoid blocking, in another thread -#if defined(CHIPTESTBOARDD) && !defined(VIRTUAL) +#if (defined(CHIPTESTBOARDD) || defined(XILINX_CHIPTESTBOARDD)) && !defined(VIRTUAL) if (!enableTenGigabitEthernet(-1)) { ret = validateUDPSocket(); if (ret == FAIL) { @@ -2094,7 +2105,7 @@ void *start_state_machine(void *arg) { return NULL; } -#if defined(CHIPTESTBOARDD) && !defined(VIRTUAL) +#if (defined(CHIPTESTBOARDD) || defined(XILINX_CHIPTESTBOARDD)) && !defined(VIRTUAL) void *start_reading_and_sending_udp_frames(void *arg) { readandSendUDPFrames(); return NULL; @@ -3077,8 +3088,7 @@ int enable_ten_giga(int file_des) { return printSocketReadError(); LOG(logDEBUG, ("Setting 10GbE: %d\n", arg)); -#if defined(JUNGFRAUD) || defined(MOENCHD) || defined(GOTTHARD2D) || \ - defined(XILINX_CHIPTESTBOARDD) +#if defined(JUNGFRAUD) || defined(MOENCHD) || defined(GOTTHARD2D) functionNotImplemented(); #else // set & get @@ -7223,7 +7233,8 @@ int get_receiver_parameters(int file_des) { return printSocketReadError(); // 10 gbe -#if defined(EIGERD) || defined(CHIPTESTBOARDD) || defined(MYTHEN3D) +#if defined(EIGERD) || defined(CHIPTESTBOARDD) || defined(MYTHEN3D) || \ + defined(XILINX_CHIPTESTBOARDD) i32 = enableTenGigabitEthernet(GET_FLAG); #else i32 = 0; diff --git a/slsDetectorServers/xilinx_ctbDetectorServer/CMakeLists.txt b/slsDetectorServers/xilinx_ctbDetectorServer/CMakeLists.txt index 2fee93ca8..0aa8c2831 100644 --- a/slsDetectorServers/xilinx_ctbDetectorServer/CMakeLists.txt +++ b/slsDetectorServers/xilinx_ctbDetectorServer/CMakeLists.txt @@ -12,7 +12,8 @@ add_executable(xilinx_ctbDetectorServer_virtual ../slsDetectorServer/src/sharedMemory.c ../slsDetectorServer/src/loadPattern.c ../slsDetectorServer/src/programViaArm.c - ../slsDetectorServer/src/communication_funcs_UDP.c + ../slsDetectorServer/src/communication_funcs_UDP.c + ../slsDetectorServer/src/UDPPacketHeaderGenerator.c ../../slsSupportLib/src/md5.c ../slsDetectorServer/src/LTC2620_Driver.c ) diff --git a/slsDetectorServers/xilinx_ctbDetectorServer/Makefile b/slsDetectorServers/xilinx_ctbDetectorServer/Makefile index cea87a249..4e6f3b66e 100755 --- a/slsDetectorServers/xilinx_ctbDetectorServer/Makefile +++ b/slsDetectorServers/xilinx_ctbDetectorServer/Makefile @@ -24,7 +24,7 @@ DESTDIR ?= bin INSTMODE = 0777 SRCS = slsDetectorFunctionList.c -SRCS += $(main_src)slsDetectorServer.c $(main_src)slsDetectorServer_funcs.c $(main_src)communication_funcs.c $(main_src)arm64.c $(main_src)XILINX_PLL.c $(main_src)XILINX_FMC.c $(main_src)common.c $(main_src)/sharedMemory.c $(main_src)/loadPattern.c $(md5_dir)md5.c $(main_src)programViaArm.c $(main_src)LTC2620_Driver.c +SRCS += $(main_src)slsDetectorServer.c $(main_src)slsDetectorServer_funcs.c $(main_src)communication_funcs.c $(main_src)communication_funcs_UDP.c $(main_src)UDPPacketHeaderGenerator.c $(main_src)arm64.c $(main_src)XILINX_PLL.c $(main_src)XILINX_FMC.c $(main_src)common.c $(main_src)/sharedMemory.c $(main_src)/loadPattern.c $(md5_dir)md5.c $(main_src)programViaArm.c $(main_src)LTC2620_Driver.c OBJS = $(SRCS:.c=.o) diff --git a/slsDetectorServers/xilinx_ctbDetectorServer/slsDetectorFunctionList.c b/slsDetectorServers/xilinx_ctbDetectorServer/slsDetectorFunctionList.c index 8cf755dda..3842d8577 100644 --- a/slsDetectorServers/xilinx_ctbDetectorServer/slsDetectorFunctionList.c +++ b/slsDetectorServers/xilinx_ctbDetectorServer/slsDetectorFunctionList.c @@ -9,11 +9,12 @@ #include "sls/versionAPI.h" #include "LTC2620_Driver.h" +#include "UDPPacketHeaderGenerator.h" #include "XILINX_FMC.h" #include "XILINX_PLL.h" +#include "communication_funcs_UDP.h" #include "loadPattern.h" #ifdef VIRTUAL -#include "communication_funcs_UDP.h" #include //ceil #include #endif @@ -49,6 +50,23 @@ int powerValues[NPWR - 1] = {}; // powerIndex (A->IO) // software limit that depends on the current chip on the ctb int vLimit = 0; +// 1g readout (server reads the fifos and sends the udp packets) +// analog is not read out in 1g, its variables are only required by the udp +// packet header generator +int analogDataBytes = 0; +int digitalDataBytes = 0; +int transceiverDataBytes = 0; +char *analogData = NULL; +char *digitalData = NULL; +char *transceiverData = NULL; +char *digitalDataPtr = NULL; +char *transceiverDataPtr = NULL; +char udpPacketData[UDP_PACKET_DATA_BYTES + sizeof(sls_detector_header)]; +// configuration of the acquisition being read out (0 samples if not enabled) +int numDigitalSamples1g = 0; +int numTransceiverSamples1g = 0; +uint32_t transceiverMask1g = 0; + #ifdef VIRTUAL pthread_t pthread_virtual_tid; #endif @@ -387,16 +405,31 @@ void setupDetector() { for (int i = 0; i != NPWR - 1; ++i) powerValues[i] = -1; vLimit = DEFAULT_VLIMIT; + analogDataBytes = 0; + digitalDataBytes = 0; + transceiverDataBytes = 0; + free(digitalData); + digitalData = NULL; + free(transceiverData); + transceiverData = NULL; + digitalDataPtr = NULL; + transceiverDataPtr = NULL; + numDigitalSamples1g = 0; + numTransceiverSamples1g = 0; + transceiverMask1g = 0; #ifdef VIRTUAL if (isControlServer) { sharedMemory_setStatus(IDLE); - setupUDPCommParameters(); initializePatternWord(); } else { sharedMemory_setStop(0); } #endif + // udp socket in the server (virtual server and 1g readout) + if (isControlServer) { + setupUDPCommParameters(); + } // initialization only at start up (restart fpga) initError = waitTransceiverReset(initErrorMessage); if (initError == FAIL) { @@ -437,6 +470,7 @@ void setupDetector() { initializePatternAddresses(); LOG(logINFOBLUE, ("Setting Default readout\n")); + enableTenGigabitEthernet(DEFAULT_TEN_GIGA); setNumAnalogSamples(DEFAULT_NUM_ASAMPLES); setNumDigitalSamples(DEFAULT_NUM_DSAMPLES); setADCEnableMask_10G(BIT32_MSK); @@ -1495,13 +1529,13 @@ int configureMAC() { "\tDest Port : %hu\n", src_ip, src_mac, srcport, dst_ip, dst_mac, dstport)); -#ifdef VIRTUAL + // udp socket in the server (virtual server and 1g readout) if (setUDPDestinationDetails(0, 0, dst_ip, dstport) == FAIL) { LOG(logERROR, ("could not set udp destination IP and port\n")); return FAIL; } -#endif + // udp header in the firmware (10g readout) // build udp header udp_header header; udp_header *udp = &header; @@ -1545,6 +1579,21 @@ int configureMAC() { return OK; } +int enableTenGigabitEthernet(int val) { + uint32_t addr = ENABLE_1G; + + // set + if (val != -1) { + LOG(logINFO, ("Setting 10Gbe: %d\n", (val > 0) ? 1 : 0)); + if (val > 0) { + bus_w(addr, bus_r(addr) & ~ENABLE_1G_MSK); + } else { + bus_w(addr, bus_r(addr) | ENABLE_1G_MSK); + } + } + return ((bus_r(addr) & ENABLE_1G_MSK) ? 0 : 1); +} + /* aquisition */ int startStateMachine() { @@ -1570,6 +1619,21 @@ int startStateMachine() { return OK; #endif + // 1 giga udp + if (!enableTenGigabitEthernet(-1)) { // read enable status from FW reg, not SW + if (updateDatabytesandAllocateRAM() == FAIL) { + return FAIL; + } + // create udp socket + if (createUDPSocket(0) != OK) { + return FAIL; + } + // update header with modId, detType and version. Reset offset and fnum + createUDPPacketHeader(udpPacketData, 0); + // no data from a previous acquisition + cleanFifos(); + } + LOG(logINFOBLUE, ("Starting readout\n")); bus_w(FLOW_CONTROL_REG, bus_r(FLOW_CONTROL_REG) | START_F_MSK); return OK; @@ -1597,7 +1661,8 @@ void *start_timer(void *arg) { int numFrames = (getNumFrames() * getNumTriggers()); int imageSize = calculateDataBytes(); - int maxDataSize = MAX_DATA_SIZE_IN_PACKET; + int maxDataSize = enableTenGigabitEthernet(-1) ? MAX_DATA_SIZE_IN_PACKET + : UDP_PACKET_DATA_BYTES; int packetSize = sizeof(sls_detector_header) + maxDataSize; int packetsPerFrame = ceil((double)imageSize / (double)maxDataSize); @@ -1748,6 +1813,10 @@ enum runStatus getRunStatus() { LOG(logINFO, ("Status Register: %08x\n", retval)); if (retval == 0x0) { + // 1g might still be reading from fifo + if (!enableTenGigabitEthernet(-1) && checkDataInFifo()) { + return TRANSMITTING; + } return IDLE; } @@ -1786,37 +1855,238 @@ void waitForAcquisitionEnd() { LOG(logINFOGREEN, ("Blocking Acquisition done\n")); } +int validateUDPSocket() { + if (getUdPSocketDescriptor(0, 0) <= 0) { + return FAIL; + } + return OK; +} + +int updateDatabytesandAllocateRAM() { + // configuration of the acquisition to be read out + // (analog is not read out in 1g) + numDigitalSamples1g = digitalEnable ? getNumDigitalSamples() : 0; + numTransceiverSamples1g = + transceiverEnable ? getNumTransceiverSamples() : 0; + transceiverMask1g = getTransceiverEnableMask(); + int ntchans = __builtin_popcount(transceiverMask1g); + + int nDigitalBytes = sizeof(uint64_t) * numDigitalSamples1g; + int nTransceiverBytes = + ntchans * (NBITS_PER_TRANSCEIVER / 8) * numTransceiverSamples1g; + LOG(logINFO, ("\t1G Databytes [Digital:%d, Transceiver:%d (%d channels)]\n", + nDigitalBytes, nTransceiverBytes, ntchans)); + + if (nDigitalBytes == 0 && nTransceiverBytes == 0) { + LOG(logERROR, ("Can not allocate RAM for 0 bytes.\n")); + return FAIL; + } + + // update only if change in databytes + if (nDigitalBytes != digitalDataBytes) { + free(digitalData); + digitalData = NULL; + digitalDataBytes = 0; + if (nDigitalBytes) { + digitalData = malloc(nDigitalBytes); + if (digitalData == NULL) { + LOG(logERROR, ("Can not allocate digital data RAM.\n")); + return FAIL; + } + digitalDataBytes = nDigitalBytes; + } + } + if (nTransceiverBytes != transceiverDataBytes) { + free(transceiverData); + transceiverData = NULL; + transceiverDataBytes = 0; + if (nTransceiverBytes) { + transceiverData = malloc(nTransceiverBytes); + if (transceiverData == NULL) { + LOG(logERROR, ("Can not allocate transceiver data RAM.\n")); + return FAIL; + } + transceiverDataBytes = nTransceiverBytes; + } + } + return OK; +} + +void readandSendUDPFrames() { + LOG(logDEBUG1, ("Reading from 1G UDP\n")); + int dataBytes = digitalDataBytes + transceiverDataBytes; + + // read every frame + while (readFrameFromFifo() == OK) { + int bytesToSend = 0, n = 0; + while ((bytesToSend = fillUDPPacket(udpPacketData))) { + n += sendUDPPacket(0, 0, udpPacketData, bytesToSend); + } + } + cleanFifos(); + closeUDPSocket(0); +} + +uint32_t checkDataInFifo() { + uint32_t dataPresent = 0; + // from the register, as it is also used by the stop server + uint32_t reg = bus_r(FIFO_TO_GB_CONTROL_REG); + if (reg & RO_MODE_D_MSK) { + uint32_t fifoEmpty = + (bus_r(D_FIFO_EMPTY_STATUS_REG) & D_FIFO_EMPTY_STATUS_MSK); + LOG(logDEBUG1, ("Digital Fifo Empty: 0x%x\n", fifoEmpty)); + dataPresent = (fifoEmpty ? 0 : 1); + } + if (!dataPresent && (reg & RO_MODE_X_MSK)) { + uint32_t mask = + ((reg & ENABLED_CHANNELS_X_MSK) >> ENABLED_CHANNELS_X_OFST); + uint32_t fifoEmpty = + ((bus_r(X_FIFO_EMPTY_STATUS_REG) & X_FIFO_EMPTY_STATUS_MSK) >> + X_FIFO_EMPTY_STATUS_OFST); + LOG(logDEBUG1, ("Transceiver Fifo Empty: 0x%x (enabled: 0x%x)\n", + fifoEmpty, mask)); + // data in any of the enabled channels + dataPresent = (~fifoEmpty & mask); + } + LOG(logDEBUG1, ("Data in Fifo :0x%x\n", dataPresent)); + return dataPresent; +} + +// only called for starting of a new frame +int checkFifoForEndOfAcquisition() { + LOG(logDEBUG1, ("Check for end of acq\n")); + uint32_t dataPresent = checkDataInFifo(); + + while (!dataPresent) { + if (!runBusy()) { + // wait to be sure there is no data in fifo + usleep(WAIT_TIME_US_FR_ACQDONE_REG); + + // still no data + if (!checkDataInFifo()) { + LOG(logINFO, ("Acquisition Finished (State: 0x%08x), " + "no frame found .\n", + bus_r(STATUS_REG))); + return FAIL; + } + // got data, exit + break; + } + // check if data in fifo again + dataPresent = checkDataInFifo(); + } + LOG(logDEBUG1, ("Got data :0x%x\n", dataPresent)); + return OK; +} + +int waitForDataInFifo(uint32_t addr, uint32_t emptyMask) { + // as long as any of the fifos is empty + while (bus_r(addr) & emptyMask) { + if (!runBusy()) { + // wait to be sure there is no data in fifo + usleep(WAIT_TIME_US_FR_ACQDONE_REG); + if (bus_r(addr) & emptyMask) { + return FAIL; + } + } + } + return OK; +} + +int readSample(int ns) { + + // digital: 64 bit + if (ns < numDigitalSamples1g) { + if (waitForDataInFifo(D_FIFO_EMPTY_STATUS_REG, + D_FIFO_EMPTY_STATUS_MSK) == FAIL) { + return FAIL; + } + bus_r_block(D_FIFO_READ, digitalDataPtr, sizeof(uint64_t), true); + digitalDataPtr += sizeof(uint64_t); + } + + // transceivers: 64 bit for every enabled channel + if (ns < numTransceiverSamples1g) { + if (waitForDataInFifo(X_FIFO_EMPTY_STATUS_REG, + ((transceiverMask1g << X_FIFO_EMPTY_STATUS_OFST) & + X_FIFO_EMPTY_STATUS_MSK)) == FAIL) { + return FAIL; + } + const uint32_t addr[NCHAN_TRANSCEIVER] = {X0_FIFO_READ, X1_FIFO_READ, + X2_FIFO_READ, X3_FIFO_READ}; + for (int ich = 0; ich != NCHAN_TRANSCEIVER; ++ich) { + if ((1 << ich) & transceiverMask1g) { + bus_r_block(addr[ich], transceiverDataPtr, + NBITS_PER_TRANSCEIVER / 8, true); + transceiverDataPtr += (NBITS_PER_TRANSCEIVER / 8); + } + } + } + return OK; +} + +int readFrameFromFifo() { + digitalDataPtr = digitalData; + if (digitalDataBytes) { + memset(digitalData, 0, digitalDataBytes); + } + transceiverDataPtr = transceiverData; + if (transceiverDataBytes) { + memset(transceiverData, 0, transceiverDataBytes); + } + + // no more frames + if (checkFifoForEndOfAcquisition() == FAIL) { + return FAIL; + } + + // read Sample + int maxSamples = numDigitalSamples1g; + if (numTransceiverSamples1g > maxSamples) + maxSamples = numTransceiverSamples1g; + for (int ns = 0; ns != maxSamples; ++ns) { + if (readSample(ns) == FAIL) { + LOG(logWARNING, ("Acquisition finished with an incomplete frame " + "(%d of %d samples). Discarding it.\n", + ns, maxSamples)); + return FAIL; + } + } + + return OK; +} + int calculateDataBytes() { - int analogDataBytes = 0; - int digitalDataBytes = 0; - int transceiverDataBytes = 0; + int nAnalogBytes = 0; + int nDigitalBytes = 0; + int nTransceiverBytes = 0; int nachans = 0, ndchans = 0, ntchans = 0; if (analogEnable) { nachans = __builtin_popcount(getADCEnableMask_10G()); - analogDataBytes = nachans * (DYNAMIC_RANGE / 8) * getNumAnalogSamples(); + nAnalogBytes = nachans * (DYNAMIC_RANGE / 8) * getNumAnalogSamples(); LOG(logINFO, ("\t#Analog Channels:%d, Databytes:%d\n", nachans, - analogDataBytes)); + nAnalogBytes)); } if (digitalEnable) { ndchans = 64; - digitalDataBytes = (sizeof(uint64_t) * getNumDigitalSamples()); + nDigitalBytes = (sizeof(uint64_t) * getNumDigitalSamples()); LOG(logINFO, ("\t#Digital Channels:%d, Databytes:%d\n", ndchans, - digitalDataBytes)); + nDigitalBytes)); } if (transceiverEnable) { ntchans = __builtin_popcount(getTransceiverEnableMask()); - transceiverDataBytes = + nTransceiverBytes = ntchans * (NBITS_PER_TRANSCEIVER / 8) * getNumTransceiverSamples(); LOG(logINFO, ("\t#Transceiver Channels:%d, Databytes:%d\n", ntchans, - transceiverDataBytes)); + nTransceiverBytes)); } // total int nchans = nachans + ndchans + ntchans; - int dataBytes = analogDataBytes + digitalDataBytes + transceiverDataBytes; + int dataBytes = nAnalogBytes + nDigitalBytes + nTransceiverBytes; LOG(logINFO, ("\t#Total Channels:%d, Total Databytes:%d\n", nchans, dataBytes)); diff --git a/slsDetectorServers/xilinx_ctbDetectorServer/slsDetectorFunctionList.h b/slsDetectorServers/xilinx_ctbDetectorServer/slsDetectorFunctionList.h index 14de21741..d2254a939 100644 --- a/slsDetectorServers/xilinx_ctbDetectorServer/slsDetectorFunctionList.h +++ b/slsDetectorServers/xilinx_ctbDetectorServer/slsDetectorFunctionList.h @@ -155,6 +155,8 @@ int configureMAC(); int setDetectorPosition(int pos[]); int *getDetectorPosition(); +int enableTenGigabitEthernet(int val); + // very detector specific // chip test board specific - configure frequency, phase, pll, @@ -173,6 +175,19 @@ enum runStatus getRunStatus(); void waitForAcquisitionEnd(); u_int32_t runBusy(); +// 1g readout (digital and transceiver only) +int validateUDPSocket(); +/** for the acquisition about to start */ +int updateDatabytesandAllocateRAM(); +void readandSendUDPFrames(); +uint32_t checkDataInFifo(); +int checkFifoForEndOfAcquisition(); +/** waits until none of the fifos in emptyMask are empty + * @returns FAIL if the acquisition is done and they are still empty */ +int waitForDataInFifo(uint32_t addr, uint32_t emptyMask); +int readSample(int ns); +int readFrameFromFifo(); + // common int calculateDataBytes(); int getTotalNumberOfChannels(); diff --git a/slsDetectorServers/xilinx_ctbDetectorServer/slsDetectorServer_defs.h b/slsDetectorServers/xilinx_ctbDetectorServer/slsDetectorServer_defs.h index 393bb6874..d03694288 100644 --- a/slsDetectorServers/xilinx_ctbDetectorServer/slsDetectorServer_defs.h +++ b/slsDetectorServers/xilinx_ctbDetectorServer/slsDetectorServer_defs.h @@ -51,6 +51,7 @@ #define DEFAULT_PERIOD (300 * 1000) // 300us #define DEFAULT_READOUT_MODE (TRANSCEIVER_ONLY) #define DEFAULT_READOUT_MODE_STR "transceiver_only" +#define DEFAULT_TEN_GIGA (1) #define DEFAULT_TRANSCEIVER_MASK (0x3) // TODO: check #define DEFAULT_NUM_ASAMPLES (1) #define DEFAULT_NUM_DSAMPLES (1) @@ -76,6 +77,8 @@ /* Defines in the Firmware */ #define WAIT_TIME_PATTERN_READ (10) #define WAIT_TIME_OUT_0US_TIMES (35000) // 2s +// wait time in us after acquisition done to ensure there is no data in fifo +#define WAIT_TIME_US_FR_ACQDONE_REG (100) #define BIT32_MSK (0xFFFFFFFF) #define BIT16_MASK (0xFFFF) diff --git a/slsDetectorSoftware/generator/commands.yaml b/slsDetectorSoftware/generator/commands.yaml index d9dd5deba..dd0e344c7 100644 --- a/slsDetectorSoftware/generator/commands.yaml +++ b/slsDetectorSoftware/generator/commands.yaml @@ -894,7 +894,7 @@ udp_dstmac2: cast_input: [ false ] tengiga: - help: "[0, 1]\n\t[Eiger][Ctb][Mythen3] 10GbE Enable." + help: "[0, 1]\n\t[Eiger][Ctb][Xilinx Ctb][Mythen3] 10GbE Enable.\n\t[Xilinx Ctb] Default is enabled. If disabled, the detector server reads out only the digital and transceiver data and sends them via the 1GbE interface. Analog readout is not implemented for 1GbE." inherit_actions: INTEGER_COMMAND_VEC_ID actions: GET: diff --git a/slsDetectorSoftware/include/sls/Detector.h b/slsDetectorSoftware/include/sls/Detector.h index 7ddd5680c..928f34a8e 100644 --- a/slsDetectorSoftware/include/sls/Detector.h +++ b/slsDetectorSoftware/include/sls/Detector.h @@ -839,10 +839,13 @@ class Detector { Result printRxConfiguration(Positions pos = {}) const; - /** [Eiger][CTB][Mythen3] */ + /** [Eiger][CTB][Xilinx CTB][Mythen3] */ Result getTenGiga(Positions pos = {}) const; - /** [Eiger][CTB][Mythen3] */ + /** [Eiger][CTB][Xilinx CTB][Mythen3] \n[Xilinx CTB] Default is enabled. If + * disabled, the detector server reads out only the digital and transceiver + * data and sends them via the 1GbE interface. Analog readout is not + * implemented for 1GbE. */ void setTenGiga(bool value, Positions pos = {}); /** [Eiger][Jungfrau][Moench] */ diff --git a/slsDetectorSoftware/tests/Caller/test-Caller-acquire.cpp b/slsDetectorSoftware/tests/Caller/test-Caller-acquire.cpp index 8084efd84..cfae83230 100644 --- a/slsDetectorSoftware/tests/Caller/test-Caller-acquire.cpp +++ b/slsDetectorSoftware/tests/Caller/test-Caller-acquire.cpp @@ -248,6 +248,42 @@ void test_ctb_binary_file_size(Detector &det) { REQUIRE_NOTHROW( acquire_and_check_file_size(det, std::make_optional(ctb_state))); } + + // xilinx ctb 1g readout (altera ctb is in 1g by default) + if (detType == defs::XILINX_CHIPTESTBOARD) { + { + ctb_state = acq::default_ctb_state(detType); + ctb_state.readout_mode = defs::DIGITAL_AND_TRANSCEIVER; + ctb_state.ten_giga = false; + acq::set_ctb_state(det, ctb_state); + REQUIRE_NOTHROW(acquire_and_check_file_size( + det, std::make_optional(ctb_state))); + } + { + ctb_state = acq::default_ctb_state(detType); + ctb_state.readout_mode = defs::DIGITAL_ONLY; + ctb_state.ten_giga = false; + acq::set_ctb_state(det, ctb_state); + REQUIRE_NOTHROW(acquire_and_check_file_size( + det, std::make_optional(ctb_state))); + } + { + ctb_state = acq::default_ctb_state(detType); + ctb_state.readout_mode = defs::TRANSCEIVER_ONLY; + ctb_state.ten_giga = false; + acq::set_ctb_state(det, ctb_state); + REQUIRE_NOTHROW(acquire_and_check_file_size( + det, std::make_optional(ctb_state))); + } + // analog readout is not implemented for 1g + { + ctb_state = acq::default_ctb_state(detType); + ctb_state.readout_mode = defs::ANALOG_AND_DIGITAL; + ctb_state.ten_giga = false; + acq::set_ctb_state(det, ctb_state); + REQUIRE_THROWS(det.startDetector()); + } + } } // disable for jungfrau as it requires higher maximum receive buffer size diff --git a/slsDetectorSoftware/tests/Caller/test-Caller-rx-running.cpp b/slsDetectorSoftware/tests/Caller/test-Caller-rx-running.cpp index 68fade955..aa955df0b 100644 --- a/slsDetectorSoftware/tests/Caller/test-Caller-rx-running.cpp +++ b/slsDetectorSoftware/tests/Caller/test-Caller-rx-running.cpp @@ -184,7 +184,8 @@ TEST_CASE("tengiga - cant put if receiver is not idle", auto det_type = det.getDetectorType().squash(); if (det_type == defs::EIGER || det_type == defs::MYTHEN3 || - det_type == defs::CHIPTESTBOARD) { + det_type == defs::CHIPTESTBOARD || + det_type == defs::XILINX_CHIPTESTBOARD) { auto prev_tengiga = det.getTenGiga(); // start receiver diff --git a/slsDetectorSoftware/tests/Caller/test-Caller.cpp b/slsDetectorSoftware/tests/Caller/test-Caller.cpp index 4c677c0a3..e5c9b2c91 100644 --- a/slsDetectorSoftware/tests/Caller/test-Caller.cpp +++ b/slsDetectorSoftware/tests/Caller/test-Caller.cpp @@ -2963,7 +2963,7 @@ TEST_CASE("tengiga", "[.detectorintegration]") { auto det_type = det.getDetectorType().squash(); if (det_type == defs::EIGER || det_type == defs::CHIPTESTBOARD || - det_type == defs::MYTHEN3) { + det_type == defs::XILINX_CHIPTESTBOARD || det_type == defs::MYTHEN3) { auto tengiga = det.getTenGiga(); det.setTenGiga(false); diff --git a/slsDetectorSoftware/tests/acquire/CTBState.h b/slsDetectorSoftware/tests/acquire/CTBState.h index da5eca98e..350b41d0f 100644 --- a/slsDetectorSoftware/tests/acquire/CTBState.h +++ b/slsDetectorSoftware/tests/acquire/CTBState.h @@ -49,8 +49,7 @@ inline CTBState get_ctb_state(const Detector &det) { "Inconsistent number of digital samples"), det.getNumberOfTransceiverSamples().tsquash( "Inconsistent number of transceiver samples"), - isAltera ? det.getTenGiga().tsquash("Inconsisten ten giga enable") - : true, + det.getTenGiga().tsquash("Inconsisten ten giga enable"), isAltera ? det.getADCEnableMask().tsquash("Inconsistent adc enable mask") : 0, @@ -67,8 +66,8 @@ inline void set_ctb_state(Detector &det, const CTBState &s) { auto isAltera = (det.getDetectorType().squash(defs::GENERIC) == defs::CHIPTESTBOARD); det.setReadoutMode(s.readout_mode); + det.setTenGiga(s.ten_giga); if (isAltera) { - det.setTenGiga(s.ten_giga); det.setADCEnableMask(s.adc_enable_1g); } det.setNumberOfAnalogSamples(s.num_adc_samples); diff --git a/slsDetectorSoftware/tests/checks/MasterFileChecks.h b/slsDetectorSoftware/tests/checks/MasterFileChecks.h index d5770913e..6d3823c71 100644 --- a/slsDetectorSoftware/tests/checks/MasterFileChecks.h +++ b/slsDetectorSoftware/tests/checks/MasterFileChecks.h @@ -416,18 +416,14 @@ void check_ctb_metadata(CheckerT &checker, const acq::ExpectedState &expected) { check_exptime(checker, st.exptime); check_period(checker, st.period); - // 10g for xilinx, 1g/10g for altera ctb - bool ten_giga = true; - if (expected.common_state.det_type == defs::CHIPTESTBOARD) { - check_ten_giga(checker, st.ctb_acq_state.ten_giga); - if (!st.ctb_acq_state.ten_giga) { - ten_giga = false; - } - } - if (ten_giga) - check_adc_mask(checker, st.ctb_acq_state.adc_enable_10g); - else + check_ten_giga(checker, st.ctb_acq_state.ten_giga); + + // separate 1g adc mask only for altera ctb + if (expected.common_state.det_type == defs::CHIPTESTBOARD && + !st.ctb_acq_state.ten_giga) check_adc_mask(checker, st.ctb_acq_state.adc_enable_1g); + else + check_adc_mask(checker, st.ctb_acq_state.adc_enable_10g); check_readout_mode(checker, st.ctb_acq_state.readout_mode); check_analog_samples(checker, st.ctb_acq_state.num_adc_samples); diff --git a/slsReceiverSoftware/src/ClientInterface.cpp b/slsReceiverSoftware/src/ClientInterface.cpp index 9cc70d535..560101840 100644 --- a/slsReceiverSoftware/src/ClientInterface.cpp +++ b/slsReceiverSoftware/src/ClientInterface.cpp @@ -402,7 +402,7 @@ int ClientInterface::setup_receiver(Interface &socket) { } impl()->setTimingMode(arg.timMode); if (detType == EIGER || detType == CHIPTESTBOARD || - detType == MYTHEN3) { + detType == XILINX_CHIPTESTBOARD || detType == MYTHEN3) { impl()->setTenGigaEnable(arg.tenGiga); } if (detType == CHIPTESTBOARD || detType == XILINX_CHIPTESTBOARD) { @@ -898,7 +898,8 @@ int ClientInterface::get_overwrite(Interface &socket) { int ClientInterface::enable_tengiga(Interface &socket) { auto val = socket.Receive(); - if (detType != EIGER && detType != CHIPTESTBOARD && detType != MYTHEN3) + if (detType != EIGER && detType != CHIPTESTBOARD && + detType != XILINX_CHIPTESTBOARD && detType != MYTHEN3) functionNotImplemented(); if (val >= 0) { diff --git a/slsReceiverSoftware/src/GeneralData.h b/slsReceiverSoftware/src/GeneralData.h index 4b61fdf56..f30453981 100644 --- a/slsReceiverSoftware/src/GeneralData.h +++ b/slsReceiverSoftware/src/GeneralData.h @@ -726,8 +726,6 @@ class XilinxChipTestBoardData : public GeneralData { framesPerFile = XILINX_CTB_MAX_FRAMES_PER_FILE; fifoDepth = 2500; standardheader = true; - dataSize = 8144; - packetSize = headerSizeinPacket + dataSize; tengigaEnable = true; ctbDbitReorder = true; UpdateImageSize(); @@ -785,6 +783,11 @@ class XilinxChipTestBoardData : public GeneralData { UpdateImageSize(); }; + void SetTenGigaEnable(bool tg) { + tengigaEnable = tg; + UpdateImageSize(); + }; + private: void UpdateImageSize() { // used in calculations so cant remove now - TODO: remove later @@ -794,6 +797,7 @@ class XilinxChipTestBoardData : public GeneralData { CtbImageInputs inputs{}; inputs.mode = readoutType; inputs.nAnalogSamples = nAnalogSamples; + // only one adc mask (no analog readout in 1g) inputs.adcMask = adcEnableMaskTenGiga; inputs.nTransceiverSamples = nTransceiverSamples; inputs.transceiverMask = transceiverMask; @@ -816,6 +820,8 @@ class XilinxChipTestBoardData : public GeneralData { LOG(logDEBUG1) << "Actual image size: " << actualImageSize; // calculate network parameters + dataSize = tengigaEnable ? 8144 : UDP_PACKET_DATA_BYTES; + packetSize = headerSizeinPacket + dataSize; packetsPerFrame = ceil((double)imageSize / (double)dataSize); LOG(logDEBUG1) << "Total Number of Channels:" << nPixelsX << " Databytes: " << imageSize; diff --git a/slsReceiverSoftware/src/Implementation.cpp b/slsReceiverSoftware/src/Implementation.cpp index 60c704578..fced3544a 100644 --- a/slsReceiverSoftware/src/Implementation.cpp +++ b/slsReceiverSoftware/src/Implementation.cpp @@ -966,9 +966,11 @@ void Implementation::StartMasterWriter() { masterAttributes.quad = quadEnable; masterAttributes.readNRows = readNRows; masterAttributes.ratecorr = rateCorrections; - masterAttributes.adcMask = generalData->tengigaEnable - ? generalData->adcEnableMaskTenGiga - : generalData->adcEnableMaskOneGiga; + masterAttributes.adcMask = + (generalData->tengigaEnable || + generalData->detType == XILINX_CHIPTESTBOARD) + ? generalData->adcEnableMaskTenGiga + : generalData->adcEnableMaskOneGiga; masterAttributes.analog = (generalData->readoutType == ANALOG_ONLY || generalData->readoutType == ANALOG_AND_DIGITAL) diff --git a/slsReceiverSoftware/src/MasterAttributes.cpp b/slsReceiverSoftware/src/MasterAttributes.cpp index 8ef96cd46..a005a671f 100644 --- a/slsReceiverSoftware/src/MasterAttributes.cpp +++ b/slsReceiverSoftware/src/MasterAttributes.cpp @@ -225,6 +225,7 @@ void MasterAttributes::WriteCtbHDF5Attributes(H5::Group *group) { void MasterAttributes::GetXilinxCtbBinaryAttributes(writer *w) { WriteBinaryExposureTme(w); WriteBinaryAcquisitionPeriod(w); + WriteBinaryTenGiga(w); WriteBinaryAdcMask(w); WriteBinaryAnalogFlag(w); WriteBinaryAnalogSamples(w); @@ -242,6 +243,7 @@ void MasterAttributes::GetXilinxCtbBinaryAttributes(writer *w) { void MasterAttributes::WriteXilinxCtbHDF5Attributes(H5::Group *group) { WriteHDF5ExposureTime(group); WriteHDF5AcquisitionPeriod(group); + WriteHDF5TenGiga(group); WriteHDF5AdcMask(group); WriteHDF5AnalogFlag(group); WriteHDF5AnalogSamples(group);