rsrv: add rsrvCS_shutdown to avoid race in destroyAllChannels()

A race can occur between rsrv and cac by way of asLib
whereby casAccessRightsCB() is called after
a channel_in_use has been removed from
chanList/chanPendingUpdateARList.
casAccessRightsCB() would then attempt to remove
the node again, causing corruption.

Create a new rsrvCS_shutdown state when the
channel is not in either list.

Attempt to resolve lp:1571224

src/rsrv/camessage.c

@@ -1072,7 +1072,7 @@ unsigned    cid
  * casAccessRightsCB()
  *
  * If access right state changes then inform the client.
- *
+ * asLock is held
  */
 static void casAccessRightsCB(ASCLIENTPVT ascpvt, asClientStatus type)
 {
@@ -1521,6 +1521,9 @@ static void sendAllUpdateAS ( struct client *client )
         else if ( pciu->state == rsrvCS_inServiceUpdatePendAR ) {
              access_rights_reply ( pciu );
         }
+        else if ( pciu->state == rsrvCS_shutdown ) {
+            /* no-op */
+        }
         else {
             errlogPrintf (
             "%s at %d: corrupt channel state detected durring AR update\n",
@@ -2003,10 +2006,15 @@ static int clear_channel_reply ( caHdrLargeArray *mp,
      if ( pciu->state == rsrvCS_inService || 
             pciu->state == rsrvCS_pendConnectResp  ) {
         ellDelete ( &client->chanList, &pciu->node );
+        pciu->state = rsrvCS_shutdown;
      }
      else if ( pciu->state == rsrvCS_inServiceUpdatePendAR ||
             pciu->state == rsrvCS_pendConnectRespUpdatePendAR ) {
         ellDelete ( &client->chanPendingUpdateARList, &pciu->node );
+        pciu->state = rsrvCS_shutdown;
+     }
+     else if ( pciu->state == rsrvCS_shutdown ) {
+         /* no-op */
      }
      else {
         epicsMutexUnlock( client->chanListLock );

src/rsrv/caservertask.c

@@ -622,6 +622,7 @@ static void destroyAllChannels (
 
         epicsMutexMustLock ( client->chanListLock );
         pciu = (struct channel_in_use *) ellGet ( pList );
+        if(pciu) pciu->state = rsrvCS_shutdown;
         epicsMutexUnlock ( client->chanListLock );
 
         if ( ! pciu ) {

src/rsrv/server.h

@@ -98,12 +98,25 @@ typedef struct client {
   char                  disconnect; /* disconnect detected */
 } client;
 
+/* Channel state shows which struct client list a
+ * channel_in_us::node is in.
+ *
+ * client::chanList
+ *   rsrvCS_pendConnectResp, rsrvCS_inService
+ * client::chanPendingUpdateARList
+ *   rsrvCS_pendConnectRespUpdatePendAR, rsrvCS_inServiceUpdatePendAR
+ * Not in any list
+ *   rsrvCS_shutdown
+ *
+ * rsrvCS_invalid is not used
+ */
 enum rsrvChanState { 
     rsrvCS_invalid,
     rsrvCS_pendConnectResp,
     rsrvCS_inService,
     rsrvCS_pendConnectRespUpdatePendAR,
-    rsrvCS_inServiceUpdatePendAR 
+    rsrvCS_inServiceUpdatePendAR,
+    rsrvCS_shutdown
 };
 
 /*