From 8f704ebab9425f626ca92aef1587c0211d9268f5 Mon Sep 17 00:00:00 2001 From: Mark Koennecke Date: Wed, 27 Aug 2014 16:24:39 +0200 Subject: [PATCH] Improvements for BOA - Allowed write buffers to become very large in asynnet. Thus in order to allow for the transfer of large images. tested ub to 2kx2k, 16MB - Handle lack of space in the write buffer more gracefully: just skip the image --- asynnet.c | 18 ++++++++++++++++-- asynnet.h | 8 ++++++++ conman.c | 25 +++++++++++++++++++++++++ rwpuffer.c | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- rwpuffer.h | 15 +++++++++++++++ 5 files changed, 116 insertions(+), 4 deletions(-) diff --git a/asynnet.c b/asynnet.c index b9e688d5..d10c8c5b 100644 --- a/asynnet.c +++ b/asynnet.c @@ -45,7 +45,8 @@ #define MAXCONNECTIONS 1024 #define RBUFFERSIZE 262144 /* 256kb */ #define WBUFFERSIZE 20*262144 -/* #define WBUFFERSIZE 100*262144 /* +/* #define WBUFFERSIZE 100*262144 */ +#define MAXWBUFFERSIZE 128*1000*1024 /*--------------------------------------------------------------------------*/ typedef struct { int socket; @@ -204,7 +205,7 @@ int ANETregisterSocket(int socket) flags =1; setsockopt(socket,IPPROTO_TCP,TCP_NODELAY,(char *) &flags, sizeof(int)); socke.readBuffer = MakeRWPuffer(RBUFFERSIZE); - socke.writeBuffer = MakeRWPuffer(WBUFFERSIZE); + socke.writeBuffer = MakeBigRWPuffer(WBUFFERSIZE, MAXWBUFFERSIZE); if (socke.readBuffer == NULL || socke.writeBuffer == NULL) { return ANETMEM; } @@ -498,7 +499,20 @@ int ANETinfo(int handle, char *hostname, int hostnameLen) } return 1; } +/*---------------------------------------------------------------------------*/ +int ANETcanWrite(int handle, void *buffer, int count) +{ + pSocketDescriptor con = NULL; + int status; + con = findSocketDescriptor(handle); + if (con == NULL) { + return ANETDISCONNECTED; + } else { + ANETprocess(); + return CanStoreRWBuffer(con->writeBuffer, buffer, count); + } +} /*---------------------------------------------------------------------------*/ int ANETwrite(int handle, void *buffer, int count) { diff --git a/asynnet.h b/asynnet.h index 9912b70f..ae2f114d 100644 --- a/asynnet.h +++ b/asynnet.h @@ -145,6 +145,14 @@ int ANETinfo(int handle, char *hostname, int hostNameLen); * \return 1 on success, 0 on failure */ int ANETwrite(int handle, void *buffer, int count); +/** + * \brief Test if the buffer can be written to the network + * \param handle The handle for the connection + * \param buffer A pointer to the data to write + * \param count The number of bytes to write. + * \return 1 when possible, 0 when buffer overrun + */ +int ANETcanWrite(int handle, void *buffer, int count); /** * \brief copy at max bufferLength bytes into buffer. The data is not deleted from * the read buffer yet. diff --git a/conman.c b/conman.c index a053e2c4..70c1b25c 100644 --- a/conman.c +++ b/conman.c @@ -1178,6 +1178,8 @@ int SCWriteZipped(SConnection * self, char *pName, void *pData, SCWrite(self, "ERROR: no data to write in SCWriteZiped", eError); return 0; } + + pBuf = malloc(iDataLen*sizeof(char)); memset(pBuf,0,iDataLen*sizeof(char)); @@ -1204,6 +1206,19 @@ int SCWriteZipped(SConnection * self, char *pName, void *pData, } compressedLength = compStream.total_out; + /* + If data is large, test if we can do it + */ + if(compressedLength > 2*1000*1024) { + if(!ANETcanWrite(self->sockHandle,pData,compressedLength)){ + SCWrite(self,"WARNING: skipping excessive data in SCWriteZipped",eLogError); + deflateEnd(&compStream); + free(pBuf); + return 0; + } + } + + /* write header line */ memset(outBuf, 0, 65536); @@ -1270,6 +1285,16 @@ int SCWriteBinary(SConnection * self, char *pName, void *pData, return 0; } + /* + If data is large, test if we can do it + */ + if(iDataLen > 2*1000*1024) { + if(!ANETcanWrite(self->sockHandle,pData,iDataLen)){ + SCWrite(self,"WARNING: skipping excessive data in SCWriteBinary",eLogError); + return 0; + } + } + /* write header line */ memset(outBuf, 0, 65536); diff --git a/rwpuffer.c b/rwpuffer.c index a678e700..500a8595 100644 --- a/rwpuffer.c +++ b/rwpuffer.c @@ -4,6 +4,11 @@ * copyright: see file COPYRIGHT * * Mark Koennecke, January 2009 + * + * added resizing option and MakeBigRWPuffer in order to support transfer + * of large amounts of image data on few connections + * + * Mark Koennecke, August 2014 */ #include #include @@ -15,6 +20,7 @@ typedef struct __RWBuffer { int length; int startPtr; int endPtr; + int maxSize; } RWBuffer; /*----------------------------------------------------------------------*/ prwBuffer MakeRWPuffer(int size) @@ -32,9 +38,18 @@ prwBuffer MakeRWPuffer(int size) self->length = size; self->startPtr = 0; self->endPtr = 0; + self->maxSize = size; return self; } - +/*------------------------------------------------------------------------*/ +prwBuffer MakeBigRWPuffer(int size, int maxSize) +{ + prwBuffer result = MakeRWPuffer(size); + if(result != NULL){ + result->maxSize = maxSize; + } + return result; +} /*------------------------------------------------------------------------*/ void KillRWBuffer(prwBuffer self) { @@ -46,16 +61,51 @@ void KillRWBuffer(prwBuffer self) } free(self); } +/*------------------------------------------------------------------------*/ +int CanStoreRWBuffer(prwBuffer self, void *data, int count) +{ + int length; + char *ptr; + length = self->endPtr - self->startPtr; + if (count + length >= self->length ) { + if(self->length < self->maxSize){ + ptr = calloc(self->maxSize,sizeof(char)); + if(ptr == NULL) { + return 0; + } + memcpy(ptr,self->data, length*sizeof(char)); + free(self->data); + self->data = ptr; + self->length = self->maxSize; + } else { + return 0; + } + } + return 1; +} /*------------------------------------------------------------------------*/ int StoreRWBuffer(prwBuffer self, void *data, int count) { int length; + char *ptr; length = self->endPtr - self->startPtr; if (count + length >= self->length ) { + if(self->length < self->maxSize){ + ptr = calloc(self->maxSize,sizeof(char)); + if(ptr == NULL) { printf("HELP: RWBuffer overrun!!!!\n"); - return 0; + return 0; + } + memcpy(ptr,self->data, length*sizeof(char)); + free(self->data); + self->data = ptr; + self->length = self->maxSize; + } else { + printf("HELP: RWBuffer overrun!!!!\n"); + return 0; + } } if (count + self->endPtr > self->length) { memmove(self->data, self->data + self->startPtr, length); diff --git a/rwpuffer.h b/rwpuffer.h index e69795b4..59ec439e 100644 --- a/rwpuffer.h +++ b/rwpuffer.h @@ -16,6 +16,13 @@ typedef struct __RWBuffer *prwBuffer; * \return NULL on success, else a pointer to t a new rwPuffer */ prwBuffer MakeRWPuffer(int size); +/** + * \brief create a RW buffer which can grow. + * \param size The size of the buffer. + * \param maxSize The maximum size of the buffer. + * \return NULL on success, else a pointer to t a new rwPuffer + */ +prwBuffer MakeBigRWPuffer(int size, int maxSize); /** * \brief delete a rw buffer. * \param self The rwPuffer to delete. @@ -29,6 +36,14 @@ void KillRWBuffer(prwBuffer self); * \return 1 on success, 0 on failure */ int StoreRWBuffer(prwBuffer self, void *data, int count); +/** + * \brief Test if the data can be stored in the rwBuffer + * \param self The rw buffer to store the data in + * \param data pointer to the data to store + * \param count The number of bytes to store + * \return 1 when OK, 0 when buffer full + */ +int CanStoreRWBuffer(prwBuffer self, void *data, int count); /** * \brief Get a pointer to the current buffer data * \param self the buffer to get the data from