From b75b9fa3b73b70a34309815df2f7a69474b244cc Mon Sep 17 00:00:00 2001 From: Dhanya Thattil Date: Thu, 26 Aug 2021 16:29:36 +0200 Subject: [PATCH] eiger round robin --- slsDetectorServers/eigerDetectorServer/Beb.c | 65 +++++++++++++++++-- slsDetectorServers/eigerDetectorServer/Beb.h | 6 +- .../eigerDetectorServer/FebRegisterDefs.h | 1 + .../slsDetectorFunctionList.c | 63 +++++++----------- .../slsDetectorFunctionList.c | 11 ++-- .../include/slsDetectorFunctionList.h | 7 +- .../src/slsDetectorServer_funcs.c | 33 ++++++---- slsDetectorSoftware/include/sls/Detector.h | 4 +- slsDetectorSoftware/src/CmdProxy.h | 2 +- 9 files changed, 123 insertions(+), 69 deletions(-) diff --git a/slsDetectorServers/eigerDetectorServer/Beb.c b/slsDetectorServers/eigerDetectorServer/Beb.c index 19c1513c5..1c31e4c06 100644 --- a/slsDetectorServers/eigerDetectorServer/Beb.c +++ b/slsDetectorServers/eigerDetectorServer/Beb.c @@ -45,6 +45,7 @@ int Beb_deactivated_transmission_delay_left = 0; int Beb_deactivated_transmission_delay_right = 0; int Beb_deactivated_left_datastream = 1; int Beb_deactivated_right_datastream = 1; +int Beb_deactivated_num_destinations = 1; void BebInfo_BebInfo(struct BebInfo *bebInfo, unsigned int beb_num) { bebInfo->beb_number = beb_num; @@ -1199,9 +1200,7 @@ int Beb_StopAcquisition() { return 1; } -int Beb_RequestNImages(unsigned int beb_number, int ten_gig, - unsigned int dst_number, unsigned int nimages, - int test_just_send_out_packets_no_wait) { +int Beb_RequestNImages(int ten_gig, unsigned int nimages, int test_just_send_out_packets_no_wait) { if (!Beb_activated) return 1; @@ -1225,10 +1224,10 @@ int Beb_RequestNImages(unsigned int beb_number, int ten_gig, unsigned int packet_size = ten_gig ? 0x200 : 0x80; // 4k or 1k packets LOG(logDEBUG1, ("----Beb_RequestNImages Start----\n")); - LOG(logINFO, ("beb_number:%d, ten_gig:%d,dst_number:%d, npackets:%d, " + LOG(logINFO, ("ten_gig:%d, npackets:%d, " "Beb_bit_mode:%d, header_size:%d, nimages:%d, " "test_just_send_out_packets_no_wait:%d\n", - beb_number, ten_gig, dst_number, npackets, Beb_bit_mode, + ten_gig, npackets, Beb_bit_mode, header_size, nimages, test_just_send_out_packets_no_wait)); u_int32_t right_port_value = 0x2000; @@ -1610,6 +1609,62 @@ int Beb_GetNextFrameNumber(uint64_t *retval, int tengigaEnable) { void Beb_SetPartialReadout(int value) { Beb_partialReadout = value; } +int Beb_GetNumberofDestinations(int *retval) { + if (!Beb_activated) { + *retval = Beb_deactivated_num_destinations; + return OK; + } + u_int32_t offset[2] = {LEFT_OFFSET + NUM_UDP_DEST_OFFSET, + RIGHT_OFFSET + NUM_UDP_DEST_OFFSET}; + u_int32_t *csp0base = 0; + int fd = Beb_open(&csp0base, XPAR_CMD_GENERATOR); + if (fd <= 0) { + LOG(logERROR, ("Could not read register to get number of udp " + "destinations. FAIL\n")); + return FAIL; + } else { + int retval1[2] = {0, 0}; + retval1[0] = Beb_Read32(csp0base, offset[0]); + retval1[1] = Beb_Read32(csp0base, offset[1]); + Beb_close(fd, csp0base); + if (retval1[0] != retval1[1]) { + LOG(logERROR, ("Inconsistent values on left (%d) and right (%d) " + "fpga for number of destinations. FAIL\n", + retval1[0], retval1[1])); + return FAIL; + } + *retval = retval1[0]; + return OK; + } +} + +int Beb_SetNumberofDestinations(int value) { + LOG(logINFO, ("Setting number of destinations to %d\n", value)); + if (value < 0 || value >= MAX_UDP_DESTINATION) { + LOG(logERROR, ("Invalid number of destinations %d\n", value)); + return FAIL; + } + if (!Beb_activated) { + Beb_deactivated_num_destinations = value; + return FAIL; + } + u_int32_t offset[2] = {LEFT_OFFSET + NUM_UDP_DEST_OFFSET, + RIGHT_OFFSET + NUM_UDP_DEST_OFFSET}; + u_int32_t *csp0base = 0; + int fd = Beb_open(&csp0base, XPAR_CMD_GENERATOR); + if (fd <= 0) { + LOG(logERROR, ("Could not read register to set number of udp " + "destinations. FAIL\n")); + return FAIL; + } else { + Beb_Write32(csp0base, offset[0], value); + Beb_Write32(csp0base, offset[1], value); + Beb_deactivated_num_destinations = value; + Beb_close(fd, csp0base); + return OK; + } +} + uint16_t Beb_swap_uint16(uint16_t val) { return (val << 8) | (val >> 8); } int Beb_open(u_int32_t **csp0base, u_int32_t offset) { diff --git a/slsDetectorServers/eigerDetectorServer/Beb.h b/slsDetectorServers/eigerDetectorServer/Beb.h index b29e252ea..e4981df75 100644 --- a/slsDetectorServers/eigerDetectorServer/Beb.h +++ b/slsDetectorServers/eigerDetectorServer/Beb.h @@ -92,8 +92,7 @@ int Beb_SetUpTransferParameters(short the_bit_mode); * ten_gig, unsigned int dst_number, unsigned int nimages, int * test_just_send_out_packets_no_wait=0); //all images go to the same * destination!*/ -int Beb_RequestNImages(unsigned int beb_number, int ten_gig, - unsigned int dst_number, unsigned int nimages, +int Beb_RequestNImages(int ten_gig, unsigned int nimages, int test_just_send_out_packets_no_wait); int Beb_Test(unsigned int beb_number); @@ -110,6 +109,9 @@ int Beb_GetNextFrameNumber(uint64_t *retval, int tengigaEnable); void Beb_SetPartialReadout(int value); +int Beb_GetNumberofDestinations(int *retval); +int Beb_SetNumberofDestinations(int value); + uint16_t Beb_swap_uint16(uint16_t val); int Beb_open(u_int32_t **csp0base, u_int32_t offset); u_int32_t Beb_Read32(u_int32_t *baseaddr, u_int32_t offset); diff --git a/slsDetectorServers/eigerDetectorServer/FebRegisterDefs.h b/slsDetectorServers/eigerDetectorServer/FebRegisterDefs.h index da4728be6..d1349735e 100644 --- a/slsDetectorServers/eigerDetectorServer/FebRegisterDefs.h +++ b/slsDetectorServers/eigerDetectorServer/FebRegisterDefs.h @@ -208,6 +208,7 @@ #define STOP_ACQ_BIT 0x40000000 #define TWO_REQUESTS_OFFSET 0x1c #define TWO_REQUESTS_BIT 0x80000000 +#define NUM_UDP_DEST_OFFSET 0x20 // version #define FIRMWARE_VERSION_OFFSET 0x4 diff --git a/slsDetectorServers/eigerDetectorServer/slsDetectorFunctionList.c b/slsDetectorServers/eigerDetectorServer/slsDetectorFunctionList.c index 2495243af..f5c66aafb 100644 --- a/slsDetectorServers/eigerDetectorServer/slsDetectorFunctionList.c +++ b/slsDetectorServers/eigerDetectorServer/slsDetectorFunctionList.c @@ -41,11 +41,7 @@ int *detectorChans = NULL; int *detectorDacs = NULL; int send_to_ten_gig = 0; -int ndsts_in_use = 32; unsigned int nimages_per_request = 1; -int on_dst = 0; -int dst_requested[32] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; int top = 0; int master = 0; @@ -837,9 +833,6 @@ int setDynamicRange(int dr) { #ifndef VIRTUAL sharedMemory_lockLocalLink(); if (Feb_Control_SetDynamicRange(dr)) { - on_dst = 0; - for (int i = 0; i < 32; ++i) - dst_requested[i] = 0; // clear dst requested if (!Beb_SetUpTransferParameters(dr)) { LOG(logERROR, ("Could not set bit mode in the back end\n")); sharedMemory_unlockLocalLink(); @@ -917,10 +910,6 @@ void setNumFrames(int64_t val) { sharedMemory_lockLocalLink(); if (Feb_Control_SetNExposures((unsigned int)val * eiger_ntriggers)) { eiger_nexposures = val; - on_dst = 0; - for (int i = 0; i < 32; ++i) - dst_requested[i] = 0; // clear dst requested - ndsts_in_use = 1; nimages_per_request = eiger_nexposures * eiger_ntriggers; } sharedMemory_unlockLocalLink(); @@ -940,9 +929,6 @@ void setNumTriggers(int64_t val) { sharedMemory_lockLocalLink(); if (Feb_Control_SetNExposures((unsigned int)val * eiger_nexposures)) { eiger_ntriggers = val; - on_dst = 0; - for (int i = 0; i < 32; ++i) - dst_requested[i] = 0; // clear dst requested nimages_per_request = eiger_nexposures * eiger_ntriggers; } sharedMemory_unlockLocalLink(); @@ -1521,6 +1507,24 @@ enum timingMode getTiming() { /* configure mac */ +int getNumberofDestinations(int *retval) { +#ifdef VIRTUAL + *retval = numUdpDestinations; + return OK; +#else + return Beb_GetNumberofDestinations(retval); +#endif +} + +int setNumberofDestinations(int value) { +#ifdef VIRTUAL + // already set in funcs.c + return OK; +#else + return Beb_SetNumberofDestinations(value); +#endif +} + int configureMAC() { LOG(logINFOBLUE, ("Configuring MAC\n")); @@ -1595,12 +1599,6 @@ int configureMAC() { } else { return FAIL; } - - on_dst = 0; - - for (int i = 0; i < 32; ++i) - dst_requested[i] = 0; // clear dst requested - nimages_per_request = eiger_nexposures * eiger_ntriggers; #endif } return OK; @@ -2594,30 +2592,13 @@ int softwareTrigger(int block) { } int startReadOut() { - LOG(logINFO, ("Requesting images...\n")); -#ifdef VIRTUAL - return OK; -#else - // RequestImages(); - int ret_val = 0; - dst_requested[0] = 1; - while (dst_requested[on_dst]) { - // waits on data - int beb_num = detid; - if ((ret_val = (!Beb_RequestNImages(beb_num, send_to_ten_gig, on_dst, - nimages_per_request, 0)))) - break; - - dst_requested[on_dst++] = 0; - on_dst %= ndsts_in_use; - } - - if (ret_val) +#ifndef VIRTUAL + if (!Beb_RequestNImages(send_to_ten_gig, nimages_per_request, 0)) { return FAIL; - else - return OK; + } #endif + return OK; } enum runStatus getRunStatus() { diff --git a/slsDetectorServers/jungfrauDetectorServer/slsDetectorFunctionList.c b/slsDetectorServers/jungfrauDetectorServer/slsDetectorFunctionList.c index 77585cc12..8043eac8f 100644 --- a/slsDetectorServers/jungfrauDetectorServer/slsDetectorFunctionList.c +++ b/slsDetectorServers/jungfrauDetectorServer/slsDetectorFunctionList.c @@ -1331,14 +1331,18 @@ int getNumberofUDPInterfaces() { return ((bus_r(CONFIG_REG) & CONFIG_OPRTN_MDE_2_X_10GbE_MSK) ? 2 : 1); } -int getNumberofDestinations() { - return (((bus_r(CONTROL_REG) & CONTROL_RX_ADDTNL_ENDPTS_NUM_MSK) >> CONTROL_RX_ADDTNL_ENDPTS_NUM_OFST) + 1); +int getNumberofDestinations(int *retval) { + retval = (((bus_r(CONTROL_REG) & CONTROL_RX_ADDTNL_ENDPTS_NUM_MSK) >> + CONTROL_RX_ADDTNL_ENDPTS_NUM_OFST) + + 1); + return OK; } -void setNumberofDestinations(int value) { +int setNumberofDestinations(int value) { LOG(logINFO, ("Setting number of entries to %d\n", value)); --value; bus_w(CONTROL_REG, bus_r(CONTROL_REG) | ((value << CONTROL_RX_ADDTNL_ENDPTS_NUM_OFST) & CONTROL_RX_ADDTNL_ENDPTS_NUM_MSK)); + return OK; } int getFirstUDPDestination() { @@ -1544,7 +1548,6 @@ int configureMAC() { } } } - setNumberofDestinations(numUdpDestinations); setNumberofUDPInterfaces(numInterfaces); selectPrimaryInterface(selInterface); diff --git a/slsDetectorServers/slsDetectorServer/include/slsDetectorFunctionList.h b/slsDetectorServers/slsDetectorServer/include/slsDetectorFunctionList.h index 7079bfac1..f9c14f38a 100644 --- a/slsDetectorServers/slsDetectorServer/include/slsDetectorFunctionList.h +++ b/slsDetectorServers/slsDetectorServer/include/slsDetectorFunctionList.h @@ -385,9 +385,12 @@ void calcChecksum(mac_conf *mac, int sourceip, int destip); void setNumberofUDPInterfaces(int val); int getNumberofUDPInterfaces(); #endif + +#if defined(JUNGFRAUD) || defined(EIGERD) +int getNumberofDestinations(int *retval); +int setNumberofDestinations(int value); +#endif #ifdef JUNGFRAUD -int getNumberofDestinations(); -void setNumberofDestinations(int value); int getFirstUDPDestination(); void setFirstUDPDestination(int value); void selectPrimaryInterface(int val); diff --git a/slsDetectorServers/slsDetectorServer/src/slsDetectorServer_funcs.c b/slsDetectorServers/slsDetectorServer/src/slsDetectorServer_funcs.c index a504b3658..0981918ef 100644 --- a/slsDetectorServers/slsDetectorServer/src/slsDetectorServer_funcs.c +++ b/slsDetectorServers/slsDetectorServer/src/slsDetectorServer_funcs.c @@ -9156,15 +9156,14 @@ int get_num_dest_list(int file_des) { functionNotImplemented(); #else retval = numUdpDestinations; -#ifdef JUNGFRAUD - if (getNumberofDestinations() != retval) { + int retval1 = 0; + if (getNumberofDestinations(&retval1) == FAIL || retval1 != retval) { ret = FAIL; sprintf( mess, - "Could not get number of udp destinations. (server reads %d, fpga reads %d).\n", getNumberofDestinations(), retval); + "Could not get number of udp destinations. (server reads %d, fpga reads %d).\n", retval1, retval); LOG(logERROR, (mess)); } -#endif #endif LOG(logDEBUG1, ("numUdpDestinations retval: 0x%x\n", retval)); return Server_SendResult(file_des, INT32, &retval, sizeof(retval)); @@ -9190,8 +9189,15 @@ int set_num_dest_list(int file_des) { LOG(logERROR, (mess)); } else { if (check_detector_idle("set number of udp destinations") == OK) { - numUdpDestinations = arg; - configure_mac(); + if (setNumberofDestinations(arg) == FAIL) { + ret = FAIL; + strcpy(mess, + "Could not set number of udp destinations.\n"); + LOG(logERROR, (mess)); + } else { + numUdpDestinations = arg; + configure_mac(); + } } } #endif @@ -9202,11 +9208,10 @@ int get_udp_first_dest(int file_des) { ret = OK; memset(mess, 0, sizeof(mess)); int retval = -1; -#if !defined(JUNGFRAUD) && !defined(EIGERD) +#ifndef JUNGFRAUD functionNotImplemented(); #else retval = firstUDPDestination; -#ifdef JUNGFRAUD if (getFirstUDPDestination() != retval) { ret = FAIL; sprintf(mess, @@ -9215,7 +9220,6 @@ int get_udp_first_dest(int file_des) { getFirstUDPDestination(), retval); LOG(logERROR, (mess)); } -#endif #endif LOG(logDEBUG1, ("first udp destination retval: 0x%x\n", retval)); return Server_SendResult(file_des, INT32, &retval, sizeof(retval)); @@ -9230,7 +9234,7 @@ int set_udp_first_dest(int file_des) { return printSocketReadError(); LOG(logDEBUG1, ("Setting first udp destination to %d\n", arg)); -#if !defined(JUNGFRAUD) && !defined(EIGERD) +#ifndef JUNGFRAUD functionNotImplemented(); #else if (arg < 0 || arg >= MAX_UDP_DESTINATION) { @@ -9240,8 +9244,13 @@ int set_udp_first_dest(int file_des) { LOG(logERROR, (mess)); } else { if (check_detector_idle("set first udp destination") == OK) { - firstUDPDestination = arg; - configure_mac(); + setFirstUDPDestination(arg); + int retval = getFirstUDPDestination(); + validate(&ret, mess, arg, retval, "set udp first destination", DEC); + if (ret == OK) { + firstUDPDestination = arg; + configure_mac(); + } } } #endif diff --git a/slsDetectorSoftware/include/sls/Detector.h b/slsDetectorSoftware/include/sls/Detector.h index 0e53c892b..664e3f0c6 100644 --- a/slsDetectorSoftware/include/sls/Detector.h +++ b/slsDetectorSoftware/include/sls/Detector.h @@ -677,10 +677,10 @@ class Detector { /**[Jungfrau][Eiger] Options 1-32 */ void setNumberofUDPDestinations(const int value, Positions pos = {}); - /** [Jungfrau][Eiger] */ + /** [Jungfrau] */ Result getFirstUDPDestination(Positions pos = {}) const; - /**[Jungfrau][Eiger] Options 0-31 */ + /**[Jungfrau] Options 0-31 */ void setFirstUDPDestination(const int value, Positions pos = {}); Result getDestinationUDPIP(Positions pos = {}) const; diff --git a/slsDetectorSoftware/src/CmdProxy.h b/slsDetectorSoftware/src/CmdProxy.h index 5d2708678..3e5f4ddde 100644 --- a/slsDetectorSoftware/src/CmdProxy.h +++ b/slsDetectorSoftware/src/CmdProxy.h @@ -1543,7 +1543,7 @@ class CmdProxy { INTEGER_COMMAND_VEC_ID( udp_firstdst, getFirstUDPDestination, setFirstUDPDestination, StringTo, - "[0 - 31]\n\t[Jungfrau][Eiger] One can set which is the first " + "[0 - 31]\n\t[Jungfrau] One can set which is the first " "destination that the detector will stream images " "out from in a round robin fashion. The entry must not have been " "empty. Default: 0");