detectors/slsDetectorPackage
25
0
Fork 0
mirror of https://github.com/slsdetectorgroup/slsDetectorPackage.git synced 2025-04-24 23:30:03 +02:00
Code Issues Actions Packages Releases Activity

in between

Browse Source
This commit is contained in:
Dhanya Maliakal 2017-01-17 10:56:01 +01:00
parent adb5ab5ed1
commit f6927ad1bf
3 changed files with 261 additions and 193 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
slsReceiverSoftware/include/UDPStandardImplementation.h
View File

@ -645,9 +645,8 @@ private:
DataSpace *hdf5_dataspaceId[MAX_NUMBER_OF_WRITER_THREADS]; DataSpace *hdf5_dataspaceId[MAX_NUMBER_OF_WRITER_THREADS];
DataSet *hdf5_datasetId[MAX_NUMBER_OF_WRITER_THREADS]; DataSet *hdf5_datasetId[MAX_NUMBER_OF_WRITER_THREADS];
H5File *hdf5_fileId[MAX_NUMBER_OF_WRITER_THREADS]; H5File *hdf5_fileId[MAX_NUMBER_OF_WRITER_THREADS];
H5File* hdf5_masterFileId;
DataType hdf5_datatype; DataType hdf5_datatype;
const static int MAX_IMAGES_IN_DATASET = 10000;
const static int MAX_CHUNKED_IMAGES = 1000;
#endif #endif
//***acquisition indices/count parameters*** //***acquisition indices/count parameters***
/** Frame Number of First Frame of an entire Acquisition (including all scans) */ /** Frame Number of First Frame of an entire Acquisition (including all scans) */

138
slsReceiverSoftware/include/receiver_defs.h
View File

@ -2,7 +2,6 @@
#define RECEIVER_DEFS_H #define RECEIVER_DEFS_H
#include "sls_receiver_defs.h" #include "sls_receiver_defs.h"
#include <stdint.h> #include <stdint.h>
@ -19,6 +18,7 @@ typedef struct {
unsigned char portIndex[1]; unsigned char portIndex[1];
unsigned char dynamicRange[1]; unsigned char dynamicRange[1];
} eiger_packet_header_t; } eiger_packet_header_t;
/** /**
* structure of an eiger packet footer * structure of an eiger packet footer
* framenum 48 bit frame number (already written by firmware) * framenum 48 bit frame number (already written by firmware)
@ -44,158 +44,126 @@ typedef struct {
} jfrau_packet_header_t; } jfrau_packet_header_t;
#define WRITER_VERSION 1.0 #define HDF5_WRITER_VERSION 1.0
#define GOODBYE -200 #define GOODBYE -200
#define DO_NOTHING 0
#define CREATE_FILES 1
#define DO_EVERYTHING 2
#define DO_NOTHING 0
#define CREATE_FILES 1
#define DO_EVERYTHING 2
#define BUF_SIZE (16*1024*1024) //16mb
#define SAMPLE_TIME_IN_NS 100000000//100ms
#define MAX_JOBS_PER_THREAD 1000
#define HEADER_SIZE_NUM_TOT_PACKETS 4 #define HEADER_SIZE_NUM_TOT_PACKETS 4
#define HEADER_SIZE_NUM_FRAMES 2 #define SAMPLE_TIME_IN_NS 100000000//100ms
#define HEADER_SIZE_NUM_PACKETS 1 #define MAX_JOBS_PER_THREAD 1000
#define ALL_MASK_32 0xFFFFFFFF #define ALL_MASK_32 0xFFFFFFFF
//binary file size and file header
#define FILE_BUF_SIZE (16*1024*1024) //16mb
#define FILE_FRAME_HEADER_LENGTH 16 #define FILE_FRAME_HEADER_LENGTH 16
#define FILE_HEADER_BUNCHID_OFFSET 8 #define FILE_HEADER_BUNCHID_OFFSET 8
//HDF5
#define MAX_CHUNKED_IMAGES 1
#define MAX_IMAGES_IN_DATASET 1000
//all max frames defined in sls_receiver_defs.h. 20000 gotthard, 100000 for short gotthard, 1000 for moench, eiger 20000 //all max frames per file defined in sls_receiver_defs.h
#define GOTTHARD_FIFO_SIZE 25000 //cannot be less than max jobs per thread = 1000 #define GOTTHARD_FIFO_SIZE 25000 //cannot be less than max jobs per thread = 1000
/*#define GOTTHARD_ALIGNED_FRAME_SIZE 4096*/ #define GOTTHARD_PIXELS_IN_ROW 1280
#define GOTTHARD_PIXELS_IN_COL 1
#define GOTTHARD_PACKETS_PER_FRAME 2 #define GOTTHARD_PACKETS_PER_FRAME 2
#define GOTTHARD_ONE_PACKET_SIZE 1286 #define GOTTHARD_ONE_PACKET_SIZE 1286
#define GOTTHARD_ONE_DATA_SIZE 1280 #define GOTTHARD_ONE_DATA_SIZE 1280
#define GOTTHARD_BUFFER_SIZE (GOTTHARD_ONE_PACKET_SIZE*GOTTHARD_PACKETS_PER_FRAME) //1286*2 #define GOTTHARD_BUFFER_SIZE (GOTTHARD_ONE_PACKET_SIZE*GOTTHARD_PACKETS_PER_FRAME) //1286*2
#define GOTTHARD_DATA_BYTES (GOTTHARD_ONE_DATA_SIZE*GOTTHARD_PACKETS_PER_FRAME) //1280*2 #define GOTTHARD_DATA_BYTES (GOTTHARD_ONE_DATA_SIZE*GOTTHARD_PACKETS_PER_FRAME) //1280*2
#define GOTTHARD_FRAME_INDEX_MASK 0xFFFFFFFE #define GOTTHARD_FRAME_INDEX_MASK 0xFFFFFFFE
#define GOTTHARD_FRAME_INDEX_OFFSET 1 #define GOTTHARD_FRAME_INDEX_OFFSET 1
#define GOTTHARD_PACKET_INDEX_MASK 0x1 #define GOTTHARD_PACKET_INDEX_MASK 0x1
//short
#define GOTTHARD_PIXELS_IN_ROW 1280 #define GOTTHARD_SHORT_PIXELS_IN_ROW 256
#define GOTTHARD_PIXELS_IN_COL 1 #define GOTTHARD_SHORT_PIXELS_IN_COL 1
#define GOTTHARD_SHORT_PACKETS_PER_FRAME 1 #define GOTTHARD_SHORT_PACKETS_PER_FRAME 1
#define GOTTHARD_SHORT_ONE_PACKET_SIZE 518 #define GOTTHARD_SHORT_ONE_PACKET_SIZE 518
#define GOTTHARD_SHORT_BUFFER_SIZE 518 #define GOTTHARD_SHORT_ONE_DATA_SIZE 512
#define GOTTHARD_SHORT_DATABYTES 512 #define GOTTHARD_SHORT_BUFFER_SIZE (GOTTHARD_SHORT_ONE_PACKET_SIZE*GOTTHARD_SHORT_PACKETS_PER_FRAME)//518*1
#define GOTTHARD_SHORT_DATABYTES (GOTTHARD_SHORT_ONE_DATA_SIZE*GOTTHARD_SHORT_PACKETS_PER_FRAME) //512*1
#define GOTTHARD_SHORT_FRAME_INDEX_MASK 0xFFFFFFFF #define GOTTHARD_SHORT_FRAME_INDEX_MASK 0xFFFFFFFF
#define GOTTHARD_SHORT_FRAME_INDEX_OFFSET 0 #define GOTTHARD_SHORT_FRAME_INDEX_OFFSET 0
#define GOTTHARD_SHORT_PACKET_INDEX_MASK 0 #define GOTTHARD_SHORT_PACKET_INDEX_MASK 0
#define GOTTHARD_SHORT_PIXELS_IN_ROW 256
#define GOTTHARD_SHORT_PIXELS_IN_COL 1
#define PROPIX_PIXELS_IN_ROW 22
#define PROPIX_PIXELS_IN_COL 22
#define PROPIX_DATABYTES_PER_PIXEL 2
#define PROPIX_FIFO_SIZE 25000 //cannot be less than max jobs per thread = 1000 #define PROPIX_FIFO_SIZE 25000 //cannot be less than max jobs per thread = 1000
#define PROPIX_PIXELS_IN_ROW 22
#define PROPIX_PIXELS_IN_COL 22
#define PROPIX_PACKETS_PER_FRAME 2 #define PROPIX_PACKETS_PER_FRAME 2
#define PROPIX_ONE_PACKET_SIZE 1286 #define PROPIX_ONE_PACKET_SIZE 1286
#define PROPIX_ONE_DATA_SIZE 1280
#define PROPIX_BUFFER_SIZE (PROPIX_ONE_PACKET_SIZE*PROPIX_PACKETS_PER_FRAME) //1286*2 #define PROPIX_BUFFER_SIZE (PROPIX_ONE_PACKET_SIZE*PROPIX_PACKETS_PER_FRAME) //1286*2
//#define PROPIX_DATA_BYTES (1280*PROPIX_PACKETS_PER_FRAME) //1280*2 #define PROPIX_DATABYTES_PER_PIXEL 2
//#define PROPIX_DATA_BYTES (1280*PROPIX_PACKETS_PER_FRAME) //1280*2
#define PROPIX_DATA_BYTES (PROPIX_PIXELS_IN_ROW * PROPIX_PIXELS_IN_COL * PROPIX_DATABYTES_PER_PIXEL) //22 * 22 * 2 #define PROPIX_DATA_BYTES (PROPIX_PIXELS_IN_ROW * PROPIX_PIXELS_IN_COL * PROPIX_DATABYTES_PER_PIXEL) //22 * 22 * 2
#define PROPIX_FRAME_INDEX_MASK 0xFFFFFFFE #define PROPIX_FRAME_INDEX_MASK 0xFFFFFFFE
#define PROPIX_FRAME_INDEX_OFFSET 1 #define PROPIX_FRAME_INDEX_OFFSET 1
#define PROPIX_PACKET_INDEX_MASK 0x1 #define PROPIX_PACKET_INDEX_MASK 0x1
#define MOENCH_FIFO_SIZE 2500 //cannot be less than max jobs per thread = 1000 #define MOENCH_FIFO_SIZE 2500 //cannot be less than max jobs per thread = 1000
/*#define MOENCH_ALIGNED_FRAME_SIZE 65536*/ #define MOENCH_BYTES_IN_ONE_ROW (MOENCH_PIXELS_IN_ONE_ROW*2)
#define MOENCH_BYTES_PER_ADC (40*2)
#define MOENCH_PIXELS_IN_ONE_ROW 160
#define MOENCH_PACKETS_PER_FRAME 40 #define MOENCH_PACKETS_PER_FRAME 40
#define MOENCH_ONE_PACKET_SIZE 1286 #define MOENCH_ONE_PACKET_SIZE 1286
#define MOENCH_ONE_DATA_SIZE 1280 #define MOENCH_ONE_DATA_SIZE 1280
#define MOENCH_BUFFER_SIZE (MOENCH_ONE_PACKET_SIZE*MOENCH_PACKETS_PER_FRAME) //1286*40 #define MOENCH_BUFFER_SIZE (MOENCH_ONE_PACKET_SIZE*MOENCH_PACKETS_PER_FRAME) //1286*40
#define MOENCH_DATA_BYTES (MOENCH_ONE_DATA_SIZE*MOENCH_PACKETS_PER_FRAME) //1280*40 #define MOENCH_DATA_BYTES (MOENCH_ONE_DATA_SIZE*MOENCH_PACKETS_PER_FRAME) //1280*40
#define MOENCH_FRAME_INDEX_MASK 0xFFFFFF00 #define MOENCH_FRAME_INDEX_MASK 0xFFFFFF00
#define MOENCH_FRAME_INDEX_OFFSET 8 #define MOENCH_FRAME_INDEX_OFFSET 8
#define MOENCH_PACKET_INDEX_MASK 0xFF #define MOENCH_PACKET_INDEX_MASK 0xFF
#define MOENCH_BYTES_PER_ADC (40*2)
#define MOENCH_PIXELS_IN_ONE_ROW 160
#define MOENCH_BYTES_IN_ONE_ROW (MOENCH_PIXELS_IN_ONE_ROW*2)
#define JFRAU_FIFO_SIZE 2500 //cannot be less than max jobs per thread = 1000 #define JFRAU_FIFO_SIZE 2500 //cannot be less than max jobs per thread = 1000
#define JFRAU_PACKETS_PER_FRAME 128
#define JFRAU_HEADER_LENGTH 22
#define JFRAU_ONE_DATA_SIZE 8192
#define JFRAU_ONE_PACKET_SIZE (JFRAU_HEADER_LENGTH+JFRAU_ONE_DATA_SIZE) //8214
#define JFRAU_DATA_BYTES (JFRAU_ONE_DATA_SIZE*JFRAU_PACKETS_PER_FRAME) //8192*128
#define JFRAU_BUFFER_SIZE (JFRAU_ONE_PACKET_SIZE*JFRAU_PACKETS_PER_FRAME) //8214*128
#define JFRAU_FRAME_INDEX_MASK 0xffffff //mask after using struct (48 bit)
#define JFRAU_FRAME_INDEX_OFFSET 0x0 //Not Applicable, use struct
#define JFRAU_PACKET_INDEX_MASK 0x0//Not Applicable, use struct
#define JFRAU_PIXELS_IN_ONE_ROW (256*4) #define JFRAU_PIXELS_IN_ONE_ROW (256*4)
#define JFRAU_PIXELS_IN_ONE_COL (256) #define JFRAU_PIXELS_IN_ONE_COL (256)
#define JFRAU_BYTES_IN_ONE_ROW (JFRAU_PIXELS_IN_ONE_ROW*2) #define JFRAU_BYTES_IN_ONE_ROW (JFRAU_PIXELS_IN_ONE_ROW*2)
#define JFRAU_PACKETS_PER_FRAME 128
#define JFRAU_HEADER_LENGTH 22
#define JFRAU_ONE_DATA_SIZE 8192
#define JFRAU_ONE_PACKET_SIZE (JFRAU_HEADER_LENGTH+JFRAU_ONE_DATA_SIZE) //8214
#define JFRAU_BUFFER_SIZE (JFRAU_ONE_PACKET_SIZE*JFRAU_PACKETS_PER_FRAME) //8214*128
#define JFRAU_DATA_BYTES (JFRAU_ONE_DATA_SIZE*JFRAU_PACKETS_PER_FRAME) //8192*128
#define JFRAU_FRAME_INDEX_MASK 0xffffff //mask after using struct (48 bit)
#define JFRAU_FRAME_INDEX_OFFSET 0x0 //Not Applicable, use struct
#define JFRAU_PACKET_INDEX_MASK 0x0 //Not Applicable, use struct
#define JCTB_FIFO_SIZE 2500 //cannot be less than max jobs per thread = 1000 #define JCTB_FIFO_SIZE 2500 //cannot be less than max jobs per thread = 1000
/*#define MOENCH_ALIGNED_FRAME_SIZE 65536*/
#define JCTB_PACKETS_PER_FRAME 1
#define JCTB_ONE_PACKET_SIZE 8224
#define JCTB_BUFFER_SIZE (JCTB_ONE_PACKET_SIZE*40)
#define JCTB_DATA_BYTES (8192*JCTB_PACKETS_PER_FRAME)
#define JCTB_FRAME_INDEX_MASK 0xFFFFFFFF
#define JCTB_FRAME_INDEX_OFFSET 6+8
#define JCTB_PACKET_INDEX_MASK 0xFFFFFFFF
#define JCTB_BYTES_PER_ADC (2)
#define JCTB_PIXELS_IN_ONE_ROW 32 #define JCTB_PIXELS_IN_ONE_ROW 32
#define JCTB_BYTES_IN_ONE_ROW (JCTB_PIXELS_IN_ONE_ROW*2) #define JCTB_BYTES_IN_ONE_ROW (JCTB_PIXELS_IN_ONE_ROW*2)
#define JCTB_BYTES_PER_ADC (2)
#define JCTB_PACKETS_PER_FRAME 1
#define JCTB_ONE_PACKET_SIZE 8224
#define JCTB_BUFFER_SIZE (JCTB_ONE_PACKET_SIZE*40)
#define JCTB_DATA_BYTES (8192*JCTB_PACKETS_PER_FRAME)
#define JCTB_FRAME_INDEX_MASK 0xFFFFFFFF
#define JCTB_FRAME_INDEX_OFFSET 6+8
#define JCTB_PACKET_INDEX_MASK 0xFFFFFFFF
#define EIGER_MAX_PORTS 2
#define EIGER_HEADER_PACKET_LENGTH 48
#define EIGER_FIFO_SIZE 100 #define EIGER_FIFO_SIZE 100
/*#define EIGER_ALIGNED_FRAME_SIZE 65536*/ #define EIGER_PIXELS_IN_ONE_ROW (256*2)
#define EIGER_PIXELS_IN_ONE_COL (256)
#define EIGER_MAX_PORTS 2
#define EIGER_HEADER_PACKET_LENGTH 48
#define EIGER_ONE_GIGA_CONSTANT 16 #define EIGER_ONE_GIGA_CONSTANT 16
#define EIGER_TEN_GIGA_CONSTANT 4 #define EIGER_TEN_GIGA_CONSTANT 4
//#define EIGER_PACKETS_PER_FRAME_COSTANT (16*EIGER_MAX_PORTS)//*bit mode 4*16=64, 8*16=128, 16*16=256, 32*16=512
#define EIGER_ONE_GIGA_ONE_PACKET_SIZE 1040 #define EIGER_ONE_GIGA_ONE_PACKET_SIZE 1040
#define EIGER_ONE_GIGA_ONE_DATA_SIZE 1024 #define EIGER_ONE_GIGA_ONE_DATA_SIZE 1024
#define EIGER_TEN_GIGA_ONE_PACKET_SIZE 4112 #define EIGER_TEN_GIGA_ONE_PACKET_SIZE 4112
#define EIGER_TEN_GIGA_ONE_DATA_SIZE 4096 #define EIGER_TEN_GIGA_ONE_DATA_SIZE 4096
#define EIGER_DATA_PACKET_HEADER_SIZE 8 #define EIGER_DATA_PACKET_HEADER_SIZE 8
//#define EIGER_BUFFER_SIZE_CONSTANT (EIGER_ONE_PACKET_SIZE*EIGER_PACKETS_PER_FRAME_COSTANT)//1040*16*2//*bit mode
//#define EIGER_DATA_BYTES_CONSTANT (EIGER_ONE_DATA_SIZE*EIGER_PACKETS_PER_FRAME_COSTANT) //1024*16*2//*bit mode
#define EIGER_FRAME_INDEX_MASK 0xFFFFFFFF //32 bit for now #define EIGER_FRAME_INDEX_MASK 0xFFFFFFFF //32 bit for now
#define EIGER_FRAME_INDEX_OFFSET 0 #define EIGER_FRAME_INDEX_OFFSET 0
#define EIGER_PACKET_INDEX_MASK 0x0 #define EIGER_PACKET_INDEX_MASK 0x0
//for each thread
#define EIGER_PIXELS_IN_ONE_ROW (256*2)
#define EIGER_PIXELS_IN_ONE_COL (256)
#endif #endif

313
slsReceiverSoftware/src/UDPStandardImplementation.cpp
View File

@ -160,6 +160,7 @@ void UDPStandardImplementation::initializeMembers(){
#endif #endif
} }
#ifdef HDF5C #ifdef HDF5C
hdf5_masterFileId = 0;
hdf5_datatype = PredType::STD_U16LE; hdf5_datatype = PredType::STD_U16LE;
#endif #endif
maxFramesPerFile = 0; maxFramesPerFile = 0;
@ -1020,6 +1021,18 @@ int UDPStandardImplementation::startReceiver(char *c){
} }
pthread_mutex_lock(&writeMutex); pthread_mutex_lock(&writeMutex);
#ifdef HDF5C
try{
Exception::dontPrint(); //to handle errors
if(hdf5_masterFileId) {delete hdf5_masterFileId; hdf5_masterFileId = 0;}
}
catch(Exception error){
cprintf(RED,"Error in closing master HDF5 handle\n");
error.printError();
pthread_mutex_unlock(&writeMutex);
return FAIL;
}
#endif
packetsCaught = 0; packetsCaught = 0;
totalPacketsCaught = 0; totalPacketsCaught = 0;
pthread_mutex_unlock(&writeMutex); pthread_mutex_unlock(&writeMutex);
@ -1036,13 +1049,14 @@ int UDPStandardImplementation::startReceiver(char *c){
FILE_LOG(logINFO) << "Data Compression has been " << stringEnable(dataCompressionEnable); FILE_LOG(logINFO) << "Data Compression has been " << stringEnable(dataCompressionEnable);
} }
FILE_LOG(logINFO) << "Number of Jobs Per Buffer: " << numberofJobsPerBuffer; FILE_LOG(logINFO) << "Number of Jobs Per Buffer: " << numberofJobsPerBuffer;
if(fileFormatType == BINARY) if(fileFormatType == BINARY){
FILE_LOG(logINFO) << "Max Frames Per File:" << maxFramesPerFile; FILE_LOG(logINFO) << "Max Frames Per File:" << maxFramesPerFile;
if(frameToGuiFrequency) }
if(frameToGuiFrequency){
FILE_LOG(logINFO) << "Frequency of frames sent to gui: " << frameToGuiFrequency; FILE_LOG(logINFO) << "Frequency of frames sent to gui: " << frameToGuiFrequency;
else }else{
FILE_LOG(logINFO) << "Frequency of frames sent to gui: Random"; FILE_LOG(logINFO) << "Frequency of frames sent to gui: Random";
}
//create UDP sockets //create UDP sockets
@ -1254,6 +1268,7 @@ void UDPStandardImplementation::closeFile(int ithread){
if(hdf5_dataspaceId[ithread]) {delete hdf5_dataspaceId[ithread]; hdf5_dataspaceId[ithread] = 0;} if(hdf5_dataspaceId[ithread]) {delete hdf5_dataspaceId[ithread]; hdf5_dataspaceId[ithread] = 0;}
if(hdf5_datasetId[ithread]) {delete hdf5_datasetId[ithread]; hdf5_datasetId[ithread] = 0;} if(hdf5_datasetId[ithread]) {delete hdf5_datasetId[ithread]; hdf5_datasetId[ithread] = 0;}
if(hdf5_fileId[ithread]) {delete hdf5_fileId[ithread]; hdf5_fileId[ithread] = 0;} if(hdf5_fileId[ithread]) {delete hdf5_fileId[ithread]; hdf5_fileId[ithread] = 0;}
if(hdf5_masterFileId) {delete hdf5_masterFileId; hdf5_masterFileId = 0;}
} }
catch(Exception error){ catch(Exception error){
cprintf(RED,"Error in closing HDF5 handles\n"); cprintf(RED,"Error in closing HDF5 handles\n");
@ -1639,12 +1654,9 @@ int UDPStandardImplementation::createNewFile(int ithread){
//create file name //create file name
if(!frameIndexEnable) if(!frameIndexEnable)
sprintf(completeFileName[ithread], "%s/%s_%lld", filePath,fileNamePerThread[ithread],(long long int)fileIndex); sprintf(completeFileName[ithread], "%s/%s_%lld", filePath,fileNamePerThread[ithread],(long long int)fileIndex);
else{ else
if(fileFormatType == BINARY) sprintf(completeFileName[ithread], "%s/%s_f%012lld_%lld", filePath,fileNamePerThread[ithread],(long long int)lastFrameNumberInFile[ithread]+1,(long long int)fileIndex);
sprintf(completeFileName[ithread], "%s/%s_f%012lld_%lld", filePath,fileNamePerThread[ithread],(long long int)lastFrameNumberInFile[ithread]+1,(long long int)fileIndex);
else
sprintf(completeFileName[ithread], "%s/%s_%lld", filePath,fileNamePerThread[ithread],(long long int)fileIndex);
}
//file type //file type
switch(fileFormatType){ switch(fileFormatType){
#ifdef HDF5C #ifdef HDF5C
@ -1693,7 +1705,7 @@ int UDPStandardImplementation::createNewFile(int ithread){
return FAIL; return FAIL;
} }
//setting file buffer size to 16mb //setting file buffer size to 16mb
setvbuf(sfilefd[ithread],NULL,_IOFBF,BUF_SIZE); setvbuf(sfilefd[ithread],NULL,_IOFBF,FILE_BUF_SIZE);
//Print packet loss and filenames //Print packet loss and filenames
@ -1748,6 +1760,7 @@ int UDPStandardImplementation::createNewFile(int ithread){
if(hdf5_dataspaceId[ithread]) {delete hdf5_dataspaceId[ithread]; hdf5_dataspaceId[ithread] = 0;} if(hdf5_dataspaceId[ithread]) {delete hdf5_dataspaceId[ithread]; hdf5_dataspaceId[ithread] = 0;}
if(hdf5_datasetId[ithread]) {delete hdf5_datasetId[ithread]; hdf5_datasetId[ithread] = 0;} if(hdf5_datasetId[ithread]) {delete hdf5_datasetId[ithread]; hdf5_datasetId[ithread] = 0;}
if(hdf5_fileId[ithread]) {delete hdf5_fileId[ithread]; hdf5_fileId[ithread] = 0; } if(hdf5_fileId[ithread]) {delete hdf5_fileId[ithread]; hdf5_fileId[ithread] = 0; }
if(hdf5_masterFileId) {delete hdf5_masterFileId; hdf5_masterFileId = 0;}
} }
catch(AttributeIException error){ catch(AttributeIException error){
cprintf(RED,"Error in creating attributes in thread %d\n",ithread); cprintf(RED,"Error in creating attributes in thread %d\n",ithread);
@ -1762,116 +1775,175 @@ int UDPStandardImplementation::createNewFile(int ithread){
return FAIL; return FAIL;
}//end of closing file }//end of closing file
char masterFileName[1000]="";
sprintf(masterFileName, "%s/%s_master_%lld.h5", filePath,fileName,(long long int)fileIndex);
//creating file //creating file
try{ try{
Exception::dontPrint(); //to handle errors Exception::dontPrint(); //to handle errors
//creating master file with metadata
try{
if(!detID && !ithread){
//creating file
FileAccPropList flist;
flist.setFcloseDegree(H5F_CLOSE_STRONG);
if(!overwriteEnable)
hdf5_masterFileId = new H5File( masterFileName, H5F_ACC_EXCL, NULL, flist );
else
hdf5_masterFileId = new H5File( masterFileName, H5F_ACC_TRUNC, NULL, flist );
//create attributes
DataSpace dataspace = DataSpace (H5S_SCALAR);
Attribute attribute;
double dValue=0;
//version
dValue=HDF5_WRITER_VERSION;
attribute = hdf5_masterFileId->createAttribute("version",PredType::NATIVE_DOUBLE, dataspace);
attribute.write(PredType::NATIVE_DOUBLE, &dValue);
//Create a group in the file
Group group1( hdf5_masterFileId->createGroup( "entry" ) );
Group group2( group1.createGroup("data") );
Group group3( group1.createGroup("instrument") );
Group group4( group3.createGroup("beam") );
Group group5( group3.createGroup("detector") );
Group group6( group1.createGroup("sample") );
int iValue=0;
StrType strdatatype(PredType::C_S1,256);
DataSet dataset;
//Dynamic Range
dataset = group5.createDataSet ( "dynamic range", PredType::NATIVE_INT, dataspace );
dataset.write ( &dynamicRange, PredType::NATIVE_INT);
attribute = dataset.createAttribute("unit",strdatatype, dataspace);
attribute.write(strdatatype, string("bits"));
//Ten Giga
iValue = tengigaEnable;
dataset = group5.createDataSet ( "ten giga enable", PredType::NATIVE_INT, dataspace );
dataset.write ( &iValue, PredType::NATIVE_INT);
//Image Size
dataset = group5.createDataSet ( "image size", PredType::NATIVE_INT, dataspace );
dataset.write ( &bufferSize, PredType::NATIVE_INT);
attribute = dataset.createAttribute("unit",strdatatype, dataspace);
attribute.write(strdatatype, string("bytes"));
//x
dataset = group5.createDataSet ( "number of pixels in x axis", PredType::NATIVE_INT, dataspace );
dataset.write ( &NX, PredType::NATIVE_INT);
//y
dataset = group5.createDataSet ( "number of pixels in y axis", PredType::NATIVE_INT, dataspace );
dataset.write ( &NY, PredType::NATIVE_INT);
//Total Frames
dataset = group5.createDataSet ( "total frames", PredType::STD_U64LE, dataspace );
dataset.write ( &numberOfFrames, PredType::STD_U64LE);
//Exptime
dataset = group5.createDataSet ( "exposure time", PredType::STD_U64LE, dataspace );
dataset.write ( &acquisitionTime, PredType::STD_U64LE);
attribute = dataset.createAttribute("unit",strdatatype, dataspace);
attribute.write(strdatatype, string("ns"));
//Period
dataset = group5.createDataSet ( "acquisition period", PredType::STD_U64LE, dataspace );
dataset.write ( &acquisitionPeriod, PredType::STD_U64LE);
attribute = dataset.createAttribute("unit",strdatatype, dataspace);
attribute.write(strdatatype, string("ns"));
//Timestamp
time_t t = time(0);
dataset = group5.createDataSet ( "timestamp", strdatatype, dataspace );
dataset.write ( string(ctime(&t)), strdatatype );
group1.close();
group2.close();
group3.close();
group4.close();
group5.close();
group6.close();
}
}catch(Exception error){
cprintf(RED,"Error in creating HDF5 master file in thread %d\n",ithread);
error.printError();
pthread_mutex_unlock(&writeMutex);
return FAIL;
}
//creating file //creating file
FileAccPropList flist; FileAccPropList flist;
flist.setFcloseDegree(H5F_CLOSE_STRONG); flist.setFcloseDegree(H5F_CLOSE_STRONG);
try{
if(!overwriteEnable) if(!overwriteEnable)
hdf5_fileId[ithread] = new H5File( completeFileName[ithread], H5F_ACC_EXCL, NULL,flist ); hdf5_fileId[ithread] = new H5File( completeFileName[ithread], H5F_ACC_EXCL, NULL,flist );
else else
hdf5_fileId[ithread] = new H5File( completeFileName[ithread], H5F_ACC_TRUNC, NULL, flist ); hdf5_fileId[ithread] = new H5File( completeFileName[ithread], H5F_ACC_TRUNC, NULL, flist );
}catch(Exception error){
//create attributes cprintf(RED,"Error in creating HDF5 file %s in thread %d\n",completeFileName[ithread], ithread);
DataSpace dataspace = DataSpace (H5S_SCALAR); error.printError();
Attribute attribute; hdf5_masterFileId->close();
double dValue=0; pthread_mutex_unlock(&writeMutex);
return FAIL;
//version
dValue=WRITER_VERSION;
attribute = hdf5_fileId[ithread]->createAttribute("version",PredType::NATIVE_DOUBLE, dataspace);
attribute.write(PredType::NATIVE_DOUBLE, &dValue);
//Create a group in the file
Group group1( hdf5_fileId[ithread]->createGroup( "entry" ));
Group group2(group1.createGroup("data"));
Group group3(group1.createGroup("instrument"));
Group group4(group3.createGroup("detector"));
Group group5(group4.createGroup("detector specific"));
int iValue=0;
StrType strdatatype(PredType::C_S1,256);
DataSet dataset;
if(myDetectorType == EIGER){
//top
iValue = (flippedData[0]?0:1);
dataset = group5.createDataSet ( "top", PredType::NATIVE_INT, dataspace );
dataset.write ( &iValue, PredType::NATIVE_INT);
//left
iValue = (ithread?0:1);
dataset = group5.createDataSet ( "left", PredType::NATIVE_INT, dataspace );
dataset.write ( &iValue, PredType::NATIVE_INT);
//active
iValue = activated;
dataset = group5.createDataSet ( "active", PredType::NATIVE_INT, dataspace );
dataset.write ( &iValue, PredType::NATIVE_INT);
} }
//Dynamic Range //create attributes
dataset = group4.createDataSet ( "dynamic range", PredType::NATIVE_INT, dataspace );
dataset.write ( &dynamicRange, PredType::NATIVE_INT); try{
attribute = dataset.createAttribute("unit",strdatatype, dataspace); DataSpace dataspace = DataSpace (H5S_SCALAR);
attribute.write(strdatatype, string("bits")); Attribute attribute;
//Ten Giga double dValue=0;
iValue = tengigaEnable;
dataset = group4.createDataSet ( "ten giga enable", PredType::NATIVE_INT, dataspace ); //version
dataset.write ( &iValue, PredType::NATIVE_INT); dValue=HDF5_WRITER_VERSION;
//Image Size attribute = hdf5_fileId[ithread]->createAttribute("version",PredType::NATIVE_DOUBLE, dataspace);
dataset = group4.createDataSet ( "image size", PredType::NATIVE_INT, dataspace ); attribute.write(PredType::NATIVE_DOUBLE, &dValue);
dataset.write ( &bufferSize, PredType::NATIVE_INT);
attribute = dataset.createAttribute("unit",strdatatype, dataspace);
attribute.write(strdatatype, string("bytes"));
//x
dataset = group4.createDataSet ( "number of pixels in x axis", PredType::NATIVE_INT, dataspace );
dataset.write ( &NX, PredType::NATIVE_INT);
//y
dataset = group4.createDataSet ( "number of pixels in y axis", PredType::NATIVE_INT, dataspace );
dataset.write ( &NY, PredType::NATIVE_INT);
//Total Frames
dataset = group4.createDataSet ( "total frames", PredType::STD_U64LE, dataspace );
dataset.write ( &numberOfFrames, PredType::STD_U64LE);
//Exptime
dataset = group4.createDataSet ( "exposure time", PredType::STD_U64LE, dataspace );
dataset.write ( &acquisitionTime, PredType::STD_U64LE);
attribute = dataset.createAttribute("unit",strdatatype, dataspace);
attribute.write(strdatatype, string("ns"));
//Period
dataset = group4.createDataSet ( "acquisition period", PredType::STD_U64LE, dataspace );
dataset.write ( &acquisitionPeriod, PredType::STD_U64LE);
attribute = dataset.createAttribute("unit",strdatatype, dataspace);
attribute.write(strdatatype, string("ns"));
//Timestamp
time_t t = time(0);
dataset = group4.createDataSet ( "timestamp", strdatatype, dataspace );
dataset.write ( string(ctime(&t)), strdatatype );
//group4.link(H5G_LINK_HARD,"/entry/data","/entry/instrument/detector/data"); int iValue=0;
group1.close(); StrType strdatatype(PredType::C_S1,256);
group2.close(); DataSet dataset;
group3.close(); if(myDetectorType == EIGER){
group4.close(); //top
group5.close(); iValue = (flippedData[0]?0:1);
attribute = hdf5_fileId[ithread]->createAttribute("top",PredType::NATIVE_INT, dataspace);
attribute.write(PredType::NATIVE_INT, &iValue);
/*dataset = group5.createDataSet ( "top", PredType::NATIVE_INT, dataspace );dataset.write ( &iValue, PredType::NATIVE_INT);*/
//left
iValue = (ithread?0:1);
attribute = hdf5_fileId[ithread]->createAttribute("left",PredType::NATIVE_INT, dataspace);
attribute.write(PredType::NATIVE_INT, &iValue);
/*dataset = group5.createDataSet ( "left", PredType::NATIVE_INT, dataspace );dataset.write ( &iValue, PredType::NATIVE_INT);*/
//active
iValue = activated;
attribute = hdf5_fileId[ithread]->createAttribute("active",PredType::NATIVE_INT, dataspace);
attribute.write(PredType::NATIVE_INT, &iValue);
/*dataset = group5.createDataSet ( "active", PredType::NATIVE_INT, dataspace );dataset.write ( &iValue, PredType::NATIVE_INT);*/
}
}catch(Exception error){
cprintf(RED,"Error in creating attribute to file in thread %d\n", ithread);
error.printError();
hdf5_masterFileId->close();
pthread_mutex_unlock(&writeMutex);
return FAIL;
}
//data //data
//create dataspace for the dataset in the file //create dataspace for the dataset in the file
int numimagesindataset = ((numberOfFrames < MAX_IMAGES_IN_DATASET)? numberOfFrames:MAX_IMAGES_IN_DATASET);
hsize_t srcdims[3] = {NY,NX,numimagesindataset};
if(dynamicRange == 4)
srcdims[1] = NX/2;
hdf5_dataspaceId[ithread] = new DataSpace (3,srcdims);
char dsetname[100]; char dsetname[100];
sprintf(dsetname, "/entry/data/data_%06lld", 0);//(long long int)currentFrameNumber[ithread]+1) try{
int numimagesindataset = ((numberOfFrames < MAX_IMAGES_IN_DATASET)? numberOfFrames:MAX_IMAGES_IN_DATASET);
hsize_t srcdims[3] = {numimagesindataset,NY,NX};
if(dynamicRange == 4)
srcdims[2] = NX/2;
hdf5_dataspaceId[ithread] = new DataSpace (3,srcdims);
sprintf(dsetname, "/data_%012lld", (long long int)currentFrameNumber[ithread]+1);
//create chunked dataset if greater than max_chunked_images //create chunked dataset if greater than max_chunked_images
if(numimagesindataset > MAX_CHUNKED_IMAGES){ if(numimagesindataset > MAX_CHUNKED_IMAGES){
//create property list for a dataset //create property list for a dataset
DSetCreatPropList plist; DSetCreatPropList plist;
/*//set up fill values //set up fill values
int fillvalue = -1; //Aldo suggested its time consuming int fillvalue = -1; /*Aldo suggested its time consuming*/
plist.setFillValue(hdf5_datatype, &fillvalue);*/ plist.setFillValue(hdf5_datatype, &fillvalue);
hsize_t chunk_dims[3] ={NY, srcdims[1],MAX_CHUNKED_IMAGES}; hsize_t chunk_dims[3] ={MAX_CHUNKED_IMAGES, NY, srcdims[2]};
plist.setChunk(3, chunk_dims); plist.setChunk(3, chunk_dims);
//Create dataset and write it into the file //Create dataset and write it into the file
hdf5_datasetId[ithread] = new DataSet (hdf5_fileId[ithread]->createDataSet( hdf5_datasetId[ithread] = new DataSet (hdf5_fileId[ithread]->createDataSet(
@ -1879,14 +1951,39 @@ int UDPStandardImplementation::createNewFile(int ithread){
}else }else
hdf5_datasetId[ithread] = new DataSet (hdf5_fileId[ithread]->createDataSet( hdf5_datasetId[ithread] = new DataSet (hdf5_fileId[ithread]->createDataSet(
dsetname, hdf5_datatype, *hdf5_dataspaceId[ithread])); dsetname, hdf5_datatype, *hdf5_dataspaceId[ithread]));
}catch(Exception error){
cprintf(RED,"Error in creating dataset in thread %d\n",ithread);
error.printError();
hdf5_masterFileId->close();
pthread_mutex_unlock(&writeMutex);
return FAIL;
}
/*//link ... master file should create link to the virtual file..
try{
char linkPath[1000]="";
sprintf(linkPath,"/entry/data/%s",dsetname);
//herr_t H5Lcreate_external( const char *target_file_name, const char *target_obj_name, hid_t link_loc_id, const char *link_name, hid_t lcpl_id, hid_t lapl_id )
H5Lcreate_external(masterFileName, dsetname, "/entry/data",dsetname,H5P_DEFAULT,H5P_DEFAULT
//hdf5_fileId[ithread]->link(H5G_LINK_HARD,dsetname,linkPath);
}catch(Exception error){
cprintf(RED,"Error in creating link in thread %d\n", ithread);
error.printError();
pthread_mutex_unlock(&writeMutex);
return FAIL;
}
*/
} }
catch(Exception error){ catch(Exception error){
cprintf(RED,"Error in creating HDF5 handles in thread %d\n",ithread); cprintf(RED,"Error in creating HDF5 handles in thread %d\n",ithread);
error.printError(); error.printError();
hdf5_masterFileId->close();
pthread_mutex_unlock(&writeMutex); pthread_mutex_unlock(&writeMutex);
return FAIL; return FAIL;
}//end of creating file }//end of creating file
hdf5_masterFileId->close();
pthread_mutex_unlock(&writeMutex); pthread_mutex_unlock(&writeMutex);
if(!totalWritingPacketCount[ithread]) if(!totalWritingPacketCount[ithread])
@ -3153,22 +3250,22 @@ void UDPStandardImplementation::handleCompleteFramesOnly(int ithread, char* wbuf
Exception::dontPrint(); //to handle errors Exception::dontPrint(); //to handle errors
if(hdf5_datasetId[ithread]) {delete hdf5_datasetId[ithread]; hdf5_datasetId[ithread] = 0;} if(hdf5_datasetId[ithread]) {delete hdf5_datasetId[ithread]; hdf5_datasetId[ithread] = 0;}
char dsetname[100]; char dsetname[100];
sprintf(dsetname, "/entry/data/data_%06lld", (long long int)currentFrameNumber[ithread]+1); sprintf(dsetname, "/entry/data/data_%012lld", (long long int)currentFrameNumber[ithread]+1);
//create new dataspace if fewer than max images/dataset //create new dataspace if fewer than max images/dataset
int numimagesindataset = (((numberOfFrames-tempframenumber) < MAX_IMAGES_IN_DATASET)? (numberOfFrames-tempframenumber):MAX_IMAGES_IN_DATASET); int numimagesindataset = (((numberOfFrames-tempframenumber) < MAX_IMAGES_IN_DATASET)? (numberOfFrames-tempframenumber):MAX_IMAGES_IN_DATASET);
if(numimagesindataset<MAX_IMAGES_IN_DATASET){ if(numimagesindataset<MAX_IMAGES_IN_DATASET){
hsize_t srcdims[3] = {NY,NX,numimagesindataset}; hsize_t srcdims[3] = {numimagesindataset,NY,NX};
if(dynamicRange == 4) if(dynamicRange == 4)
srcdims[1] = NX/2; srcdims[2] = NX/2;
hdf5_dataspaceId[ithread] = new DataSpace (3,srcdims); hdf5_dataspaceId[ithread] = new DataSpace (3,srcdims);
} }
//create chunked dataset if greater than max_chunked_images //create chunked dataset if greater than max_chunked_images
if(numimagesindataset > MAX_CHUNKED_IMAGES){ if(numimagesindataset > MAX_CHUNKED_IMAGES){
DSetCreatPropList plist; DSetCreatPropList plist;
hsize_t chunk_dims[3] ={NY, NX, MAX_CHUNKED_IMAGES}; hsize_t chunk_dims[3] ={MAX_CHUNKED_IMAGES, NY, NX};
if(dynamicRange == 4) if(dynamicRange == 4)
chunk_dims[1] = NX/2; chunk_dims[2] = NX/2;
plist.setChunk(3, chunk_dims); plist.setChunk(3, chunk_dims);
hdf5_datasetId[ithread] = new DataSet (hdf5_fileId[ithread]->createDataSet( hdf5_datasetId[ithread] = new DataSet (hdf5_fileId[ithread]->createDataSet(
@ -3178,6 +3275,10 @@ void UDPStandardImplementation::handleCompleteFramesOnly(int ithread, char* wbuf
hdf5_datasetId[ithread] = new DataSet (hdf5_fileId[ithread]->createDataSet( hdf5_datasetId[ithread] = new DataSet (hdf5_fileId[ithread]->createDataSet(
dsetname, hdf5_datatype, *hdf5_dataspaceId[ithread])); dsetname, hdf5_datatype, *hdf5_dataspaceId[ithread]));
//link
char linkPath[1000]="";
sprintf(linkPath,"/entry/data/%s",dsetname);
hdf5_fileId[ithread]->link(H5G_LINK_HARD,dsetname,linkPath);
} }
catch(Exception error){ catch(Exception error){
cprintf(RED,"Error in closing HDF5 dataset to create a new one in thread %d\n",ithread); cprintf(RED,"Error in closing HDF5 dataset to create a new one in thread %d\n",ithread);
@ -3186,8 +3287,8 @@ void UDPStandardImplementation::handleCompleteFramesOnly(int ithread, char* wbuf
} }
//wite to file //wite to file
hsize_t count[3] = {NY,NX,1}; hsize_t count[3] = {1, NY,NX};
hsize_t start[3] = {0, 0, tempframenumber%MAX_IMAGES_IN_DATASET}; hsize_t start[3] = {tempframenumber%MAX_IMAGES_IN_DATASET, 0 , 0};
hsize_t dims2[2]={NY,NX}; hsize_t dims2[2]={NY,NX};
if(dynamicRange == 4){ if(dynamicRange == 4){
dims2[1] = NX/2; dims2[1] = NX/2;