From abb5bfb6e4b45a1bcb4bd11634c8b06c1a822177 Mon Sep 17 00:00:00 2001
From: Dhanya Maliakal <dhanya.maliakal@psi.ch>
Date: Tue, 3 Jan 2017 08:57:59 +0100
Subject: [PATCH] creating new datasets every 10,000 images in same file

---
 .../include/UDPStandardImplementation.h       |  1 +
 .../src/UDPStandardImplementation.cpp         | 37 ++++++++++++++++---
 2 files changed, 32 insertions(+), 6 deletions(-)

diff --git a/slsReceiverSoftware/include/UDPStandardImplementation.h b/slsReceiverSoftware/include/UDPStandardImplementation.h
index d85dc668e..b413842d3 100644
--- a/slsReceiverSoftware/include/UDPStandardImplementation.h
+++ b/slsReceiverSoftware/include/UDPStandardImplementation.h
@@ -646,6 +646,7 @@ private:
 	DataSet *hdf5_datasetId[MAX_NUMBER_OF_WRITER_THREADS];
 	H5File *hdf5_fileId[MAX_NUMBER_OF_WRITER_THREADS];
 	DataType hdf5_datatype;
+	const static int MAX_IMAGES_IN_DATASET = 10000;
 #endif
 	//***acquisition indices/count parameters***
 	/** Frame Number of First Frame of an entire Acquisition (including all scans) */
diff --git a/slsReceiverSoftware/src/UDPStandardImplementation.cpp b/slsReceiverSoftware/src/UDPStandardImplementation.cpp
index 5911e7f0e..2b78db696 100644
--- a/slsReceiverSoftware/src/UDPStandardImplementation.cpp
+++ b/slsReceiverSoftware/src/UDPStandardImplementation.cpp
@@ -1856,19 +1856,21 @@ int UDPStandardImplementation::createNewFile(int ithread){
 
 				//data
 				//create property list for a dataset and set up fill values
-				int fillvalue = -1;
+
+				/*int fillvalue = -1; //Aldo suggested its time consuming
 				DSetCreatPropList plist;
-				plist.setFillValue(hdf5_datatype, &fillvalue);
+				plist.setFillValue(hdf5_datatype, &fillvalue);*/
 				//create dataspace for the dataset in the file
-				hsize_t srcdims[3] = {NY,NX,numberOfFrames};
+				int numimagesindataset = ((numberOfFrames < MAX_IMAGES_IN_DATASET)? numberOfFrames:MAX_IMAGES_IN_DATASET);
+				hsize_t srcdims[3] = {NY,NX,numimagesindataset};
 				if(dynamicRange == 4)
 					srcdims[1] = NX/2;
 				hdf5_dataspaceId[ithread] = new DataSpace (3,srcdims);
 				char dsetname[100];
-				sprintf(dsetname, "/entry/data/data_%06lld", (long long int)currentFrameNumber[ithread]+1);
+				sprintf(dsetname, "/entry/data/data_%06lld", 0);//(long long int)currentFrameNumber[ithread]+1)
 				//Create dataset and write it into the file
 				hdf5_datasetId[ithread] = new DataSet (hdf5_fileId[ithread]->createDataSet(
-						dsetname, hdf5_datatype, *hdf5_dataspaceId[ithread], plist));
+						dsetname, hdf5_datatype, *hdf5_dataspaceId[ithread]));/*, plist));*/
 			}
 			catch(Exception error){
 				cprintf(RED,"Error in creating HDF5 handles in thread %d\n",ithread);
@@ -3138,9 +3140,32 @@ void UDPStandardImplementation::handleCompleteFramesOnly(int ithread, char* wbuf
 		else if (fileFormatType == HDF5){
 			pthread_mutex_lock(&writeMutex);
 
+			if(tempframenumber && (tempframenumber%MAX_IMAGES_IN_DATASET) == 0){
+				try{
+					Exception::dontPrint(); //to handle errors
+					if(hdf5_datasetId[ithread])		{delete hdf5_datasetId[ithread];	hdf5_datasetId[ithread] = 0;}
+					char dsetname[100];
+					sprintf(dsetname, "/entry/data/data_%06lld", (long long int)currentFrameNumber[ithread]+1);
+					//create new dataspace if fewer than max images/dataset
+					int numimagesindataset = (((numberOfFrames-tempframenumber) < MAX_IMAGES_IN_DATASET)? (numberOfFrames-tempframenumber):MAX_IMAGES_IN_DATASET);
+					if(numimagesindataset<MAX_IMAGES_IN_DATASET){
+						hsize_t srcdims[3] = {NY,NX,numimagesindataset};
+						if(dynamicRange == 4)
+							srcdims[1] = NX/2;
+						hdf5_dataspaceId[ithread] = new DataSpace (3,srcdims);
+					}
+					hdf5_datasetId[ithread] = new DataSet (hdf5_fileId[ithread]->createDataSet(
+											dsetname, hdf5_datatype, *hdf5_dataspaceId[ithread]));
+				}
+				catch(Exception error){
+					cprintf(RED,"Error in closing HDF5 dataset to create a new one in thread %d\n",ithread);
+					error.printError();
+				}
+			}
+
 			//wite to file
 			hsize_t count[3] = {NY,NX,1};
-			hsize_t start[3] = {0, 0, tempframenumber};
+			hsize_t start[3] = {0, 0, tempframenumber%MAX_IMAGES_IN_DATASET};
 			hsize_t dims2[2]={NY,NX};
 			if(dynamicRange == 4){
 				dims2[1] = NX/2;