detectors/slsDetectorPackage
25
0
Fork 0
mirror of https://github.com/slsdetectorgroup/slsDetectorPackage.git synced 2025-06-19 00:07:13 +02:00
Code Issues Actions Packages Releases Activity

changed to have only one virtual file with bunch id, subfnum and data datasets all linked in master file

Browse Source
This commit is contained in:
Dhanya Maliakal
2017-03-03 08:39:53 +01:00
parent 537260879a
commit d8b0a13ee2
7 changed files with 268 additions and 241 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
slsReceiverSoftware/src/DataProcessor.cpp
View File

@ -249,6 +249,12 @@ void DataProcessor::CloseFiles() {
file->CloseAllFiles();
}
void DataProcessor::EndofAcquisition(uint64_t numf) {
if (*fileWriteEnable && file->GetFileType() == HDF5 && numf) {
file->EndofAcquisition(numf);
}
}
void DataProcessor::ThreadExecution() {
char* buffer=0;
@ -307,7 +313,7 @@ void DataProcessor::ProcessAnImage(char* buf) {
RecordFirstIndices(fnum);
}
if (fileWriteEnable && *callbackAction == DO_EVERYTHING)
if (*fileWriteEnable && *callbackAction == DO_EVERYTHING)
file->WriteToFile(buf, generalData->fifoBufferSize + FILE_FRAME_HEADER_SIZE, fnum-firstMeasurementIndex);
}

122
slsReceiverSoftware/src/HDF5File.cpp
View File

@ -26,6 +26,9 @@ HDF5File::HDF5File(int ind, int* nd, char* fname, char* fpath, uint64_t* findex,
dataspace(0),
dataset(0),
datatype(PredType::STD_U16LE),
nPixelsX(nx),
nPixelsY(ny),
numFramesInFile(0),
dataspace_para(0),
@ -35,11 +38,7 @@ HDF5File::HDF5File(int ind, int* nd, char* fname, char* fpath, uint64_t* findex,
para2("bunch_id"),
dataset_para2(0),
datatype_para2(PredType::STD_U64LE),
nPixelsX(nx),
nPixelsY(ny),
numFramesInFile(0)
datatype_para2(PredType::STD_U64LE)
{
#ifdef VERBOSE
PrintMembers();
@ -65,29 +64,27 @@ void HDF5File::PrintMembers() {
}
}
void HDF5File::SetNumberofPixels(uint32_t nx, uint32_t ny) {
nPixelsX = nx;
nPixelsY = ny;
}
slsReceiverDefs::fileFormat HDF5File::GetFileType() {
return HDF5;
}
void HDF5File::UpdateDataType() {
switch(*dynamicRange){
case 16:
datatype = PredType::STD_U16LE;
break;
case 32:
datatype = PredType::STD_U32LE;
break;
default:
datatype = PredType::STD_U8LE;
break;
case 16: datatype = PredType::STD_U16LE; break;
case 32: datatype = PredType::STD_U32LE; break;
default: datatype = PredType::STD_U8LE; break;
}
}
int HDF5File::CreateFile(uint64_t fnum) {
numFramesInFile = 0;
currentFileName = HDF5FileStatic::CreateFileName(filePath, fileNamePrefix, *fileIndex,
@ -110,23 +107,17 @@ int HDF5File::CreateFile(uint64_t fnum) {
}
pthread_mutex_unlock(&Mutex);
printf("%d HDF5 File: %s\n", index, currentFileName.c_str());
//virtual file
if (master && (*detIndex==0))
return CreateVirtualFile(fnum);
return OK;
}
void HDF5File::CloseCurrentFile() {
pthread_mutex_lock(&Mutex);
HDF5FileStatic::CloseDataFile(index, filefd, dataspace, dataset, dataset_para1, dataset_para2);
if (master && (*detIndex==0)) {
HDF5FileStatic::CloseVirtualDataFile(virtualfd);
}
pthread_mutex_unlock(&Mutex);
}
void HDF5File::CloseAllFiles() {
pthread_mutex_lock(&Mutex);
HDF5FileStatic::CloseDataFile(index, filefd, dataspace, dataset, dataset_para1, dataset_para2);
@ -139,7 +130,7 @@ void HDF5File::CloseAllFiles() {
int HDF5File::WriteToFile(char* buffer, int buffersize, uint64_t fnum) {
if (numFramesInFile >= maxFramesPerFile) {/**max multiply by 100?????????????*/
if (numFramesInFile >= maxFramesPerFile) {
CloseCurrentFile();
CreateFile(fnum);
}
@ -168,6 +159,7 @@ int HDF5File::WriteToFile(char* buffer, int buffersize, uint64_t fnum) {
int HDF5File::CreateMasterFile(bool en, uint32_t size,
uint32_t nx, uint32_t ny, uint64_t at, uint64_t ap) {
if (master && (*detIndex==0)) {
virtualfd = 0;
masterFileName = HDF5FileStatic::CreateMasterFileName(filePath, fileNamePrefix, *fileIndex);
printf("Master File: %s\n", masterFileName.c_str());
pthread_mutex_lock(&Mutex);
@ -180,77 +172,29 @@ int HDF5File::CreateMasterFile(bool en, uint32_t size,
}
int HDF5File::CreateVirtualFile(uint64_t fnum) {
void HDF5File::EndofAcquisition(uint64_t numf) {
//not created before
if (!virtualfd)
CreateVirtualFile(numf);
}
int HDF5File::CreateVirtualFile(uint64_t numf) {
if (master && (*detIndex==0)) {
//file name
string virtualFileName = HDF5FileStatic::CreateVirtualFileName(filePath, fileNamePrefix, *fileIndex, *frameIndexEnable, fnum);
printf("Virtual File: %s\n", virtualFileName.c_str());
//source file names
int numReadouts = numDetX * numDetY;
string fileNames[numReadouts];
for (int i = 0; i < numReadouts; ++i) {
fileNames[i] = HDF5FileStatic::CreateFileName(filePath, fileNamePrefix, *fileIndex,
*frameIndexEnable, fnum, *detIndex, *numUnitsPerDetector, i);
#ifdef VERBOSE
printf("%d: File Name: %s\n", i, fileNames[i].c_str());
#endif
}
//datatype
hid_t cdatatype;
switch(*dynamicRange){
case 16:
cdatatype = H5T_STD_U16LE;
break;
case 32:
cdatatype = H5T_STD_U32LE;
break;
default:
cdatatype = H5T_STD_U8LE;
break;
}
//source dataset name
ostringstream osfn;
osfn << "/data";
if (*frameIndexEnable) osfn << "_f" << setfill('0') << setw(12) << fnum;
string srcDatasetName = osfn.str();
//virtual dataset name
osfn.str(""); osfn.clear();
osfn << "/virtual_data";
if (*frameIndexEnable) osfn << "_f" << setfill('0') << setw(12) << fnum;
string virtualDatasetName = osfn.str();
//parameter 1 name
osfn.str(""); osfn.clear();
osfn << "/virtual_" << para1;
if (*frameIndexEnable) osfn << "_f" << setfill('0') << setw(12) << fnum;
string vpara1DatasetName = osfn.str();
//parameter 2 name
osfn.str(""); osfn.clear();
osfn << "/virtual_" << para2;
if (*frameIndexEnable) osfn << "_f" << setfill('0') << setw(12) << fnum;
string vpara2DatasetName = osfn.str();
uint64_t framestosave = ((*numImages - fnum) > maxFramesPerFile) ? maxFramesPerFile : (*numImages-fnum);
//create virtual file
pthread_mutex_lock(&Mutex);
int ret = HDF5FileStatic::CreateVirtualDataFile(virtualfd, virtualFileName, virtualDatasetName, srcDatasetName,
para1, vpara1DatasetName, H5T_STD_U32LE,
para2, vpara2DatasetName, H5T_STD_U64LE,
numReadouts, fileNames, *overWriteEnable, cdatatype,
framestosave, nPixelsY, ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX),
framestosave, numDetY * nPixelsY, numDetX * ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX), HDF5_WRITER_VERSION);
if (ret == OK)
ret = HDF5FileStatic::LinkVirtualInMaster(masterFileName, virtualFileName, virtualDatasetName, vpara1DatasetName, vpara2DatasetName);
int ret = HDF5FileStatic::CreateVirtualDataFile(
virtualfd, masterFileName,
filePath, fileNamePrefix, *fileIndex, *frameIndexEnable,
*detIndex, *numUnitsPerDetector,
maxFramesPerFile, numf,
"data", para1, para2,
datatype, datatype_para1, datatype_para2,
numDetY, numDetX, nPixelsY, ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX),
HDF5_WRITER_VERSION);
pthread_mutex_unlock(&Mutex);
return ret;
}
return OK;
}

18
slsReceiverSoftware/src/UDPStandardImplementation.cpp
View File

@ -536,6 +536,17 @@ void UDPStandardImplementation::stopReceiver(){
while(DataProcessor::GetRunningMask()){
usleep(5000);
}
//create virtual file
if (fileWriteEnable && fileFormatType == HDF5) {
uint64_t maxFramescaught = 0;
for (vector<DataProcessor*>::const_iterator it = dataProcessor.begin(); it != dataProcessor.end(); ++it) {
maxFramescaught = max(maxFramescaught, (*it)->GetNumFramesCaught());
}
if (maxFramescaught)
dataProcessor[0]->EndofAcquisition(maxFramescaught);
}
while(DataStreamer::GetRunningMask()){
usleep(5000);
}
@ -637,8 +648,13 @@ void UDPStandardImplementation::shutDownUDPSockets() {
void UDPStandardImplementation::closeFiles() {
for (vector<DataProcessor*>::const_iterator it = dataProcessor.begin(); it != dataProcessor.end(); ++it)
uint64_t maxFramescaught = 0;
for (vector<DataProcessor*>::const_iterator it = dataProcessor.begin(); it != dataProcessor.end(); ++it) {
(*it)->CloseFiles();
maxFramescaught = max(maxFramescaught, (*it)->GetNumFramesCaught());
}
if (maxFramescaught)
dataProcessor[0]->EndofAcquisition(maxFramescaught);
}