1077 lines
41 KiB
C++
1077 lines
41 KiB
C++
/* Description
|
|
*
|
|
* This application merges and slices one, two or more H5Part files
|
|
* into a single one. The range of each file which should be merged/
|
|
* sliced can be specified on the command line where each input file
|
|
* is handled similar to a python array. Negative indices are counted
|
|
* from the end of the file i.e. -1 signifies the last step.
|
|
*
|
|
* Copyright by Christof Kraus, 2007-2008, all rights reserved.
|
|
*
|
|
* \author Christof Kraus
|
|
*
|
|
* \date 2007 dec 27
|
|
*
|
|
* \warning none
|
|
*
|
|
* \attention none required
|
|
*
|
|
* \bug this code is under permanent development.
|
|
*
|
|
* \todo ??
|
|
*/
|
|
#include <iostream>
|
|
#include <fstream>
|
|
#include <string>
|
|
#include <sstream>
|
|
#include <map>
|
|
#include "H5Part.h"
|
|
#include "optparse.h"
|
|
|
|
#ifdef USE_BOOST
|
|
#include <boost/any.hpp>
|
|
#endif
|
|
|
|
#define MAX_LEN 128
|
|
|
|
using namespace std;
|
|
struct strCmp {
|
|
bool operator()( const char* s1, const char* s2 ) const {
|
|
return strcmp( s1, s2 ) < 0;
|
|
}
|
|
};
|
|
|
|
struct StringCmp {
|
|
bool operator()( const string s1, const string s2 ) const {
|
|
return s1 < s2;
|
|
}
|
|
};
|
|
|
|
#ifdef USE_BOOST
|
|
using boost::any_cast;
|
|
typedef map<string, boost::any, StringCmp> StringAnyValueMap;
|
|
#endif
|
|
|
|
typedef map<string, int, StringCmp> StringIntMap;
|
|
typedef map<string, h5part_int64_t, StringCmp> StringInt64Map;
|
|
|
|
h5part_int64_t copyFileAttribute(H5PartFile* In, H5PartFile* Out, string AttrName, h5part_int64_t Type, h5part_int64_t Num);
|
|
h5part_int64_t copyFileToStepAttribute(H5PartFile* In, H5PartFile* Out, string AttrName, h5part_int64_t Type, h5part_int64_t Num);
|
|
|
|
#ifdef USE_BOOST
|
|
h5part_int64_t copyFileAttribute(H5PartFile* In, H5PartFile* Out, StringAnyValueMap &FileAttributes, string AttrName, h5part_int64_t Type, h5part_int64_t Num);
|
|
h5part_int64_t copyFileToStepAttribute(H5PartFile* In, H5PartFile* Out, StringAnyValueMap &FileAttributes, string AttrName, h5part_int64_t Type, h5part_int64_t Num);
|
|
#endif
|
|
|
|
h5part_int64_t copyStepAttribute(H5PartFile* In, H5PartFile* Out, string AttrName, h5part_int64_t Type, h5part_int64_t Num);
|
|
h5part_int64_t copyDataset(H5PartFile* In, H5PartFile* Out, string SetName, h5part_int64_t Type, h5part_float64_t* FloatArray, h5part_int64_t* IntArray);
|
|
|
|
int main(int argc, char **argv){
|
|
|
|
using optparse::STORE;
|
|
using optparse::STORE_TRUE;
|
|
using optparse::STORE_FALSE;
|
|
using optparse::STRING;
|
|
using optparse::BOOL;
|
|
using optparse::INT;
|
|
using optparse::DOUBLE;
|
|
|
|
H5PartFile *H5OutFile, *H5InFile;
|
|
ifstream testopen;
|
|
|
|
int NumInput = 0;
|
|
int effNumInput = 0;
|
|
string *InputFilenames = NULL;
|
|
string Input;
|
|
string Range;
|
|
string from_str;
|
|
int *from = NULL;
|
|
string to_str;
|
|
int *to = NULL;
|
|
|
|
|
|
string OutputFilename;
|
|
bool isOutputInput = false;
|
|
int OutputIsInput = 0;
|
|
|
|
char AttrName[MAX_LEN];
|
|
int NumFileAttr;
|
|
StringIntMap FileAttribName;
|
|
StringInt64Map FileAttribType;
|
|
StringInt64Map FileAttribNumType;
|
|
|
|
#ifdef USE_BOOST
|
|
StringAnyValueMap FileAttributes;
|
|
#endif
|
|
|
|
char SetName[MAX_LEN];
|
|
int NumDataSets;
|
|
StringIntMap DataSetName;
|
|
StringInt64Map DataSetType;
|
|
|
|
char StepAttrName[MAX_LEN];
|
|
int NumStepAttr;
|
|
StringIntMap StepAttribName;
|
|
StringInt64Map StepAttribType;
|
|
StringInt64Map StepAttribNumType;
|
|
|
|
StringIntMap::iterator itName;
|
|
StringInt64Map::iterator itType;
|
|
StringInt64Map::iterator itNumType;
|
|
|
|
h5part_int64_t NumParticles = 0;
|
|
h5part_int64_t maxNumParticles = 0;
|
|
|
|
h5part_int64_t NumSteps;
|
|
h5part_int64_t effStep = 0;
|
|
bool firstStepInput = true;
|
|
|
|
h5part_float64_t* FloatValues = NULL;
|
|
h5part_int64_t* IntegerValues = NULL;
|
|
|
|
h5part_int64_t Type;
|
|
h5part_int64_t SetType;
|
|
h5part_int64_t NumType;
|
|
h5part_int64_t SetNum;
|
|
|
|
int verbosity = 0;
|
|
h5part_int64_t rc;
|
|
|
|
string usage("H5merge\n\nmerges two or more H5Part files, but can also be used to slice a H5Part file\n\nUsage: H5merge [[options]] input1.h5[from1:to1] [input2.h5[from2:to2] ... inputN.h5[fromN:toN]] output.h5,\n where fromX and toX are the step numbers which determine the range to be merged.\n Negative values are counted from the end of the file\n\noptions:");
|
|
optparse::OptionParser parser(usage);
|
|
|
|
parser.add_option("-h", "--help", "help","Show this message",STORE_TRUE, BOOL, "0");
|
|
parser.add_option("-v", "--verbose", "verbosity", "increase verbosity", STORE, INT, "0");
|
|
parser.add_option("-r", "--replace", "replace", "If output file name is the same as input file replace the input file at the end", STORE_TRUE, BOOL, "0");
|
|
parser.add_option("-f", "--intersect-file-attributes", "intersect-file-attributes", "merge only the intersection of file attributes from all input files", STORE_TRUE, BOOL, "0");
|
|
parser.add_option("-s", "--intersect-step-attributes", "intersect-step-attributes", "merge only the intersection of step attributes from all steps in all input files", STORE_TRUE, BOOL, "0");
|
|
parser.add_option("-d", "--intersect-datasets", "intersect-datasets", "merge only the intersection of datasets of all input files", STORE_TRUE, BOOL, "0");
|
|
try {
|
|
parser.parse_args(argc, argv);
|
|
if (parser.get<bool>("help")) {
|
|
parser.help(cerr);
|
|
exit(0);
|
|
}
|
|
}
|
|
catch(optparse::OptionError e){
|
|
cerr << e.what() << endl << endl;
|
|
parser.help(cerr);
|
|
exit(1);
|
|
}
|
|
|
|
if (parser.arguments.size() < 2){
|
|
cerr << "No output file found." << endl << endl;
|
|
parser.help(cerr);
|
|
exit(1);
|
|
}
|
|
|
|
NumInput = parser.arguments.size(); //argc - 2;
|
|
InputFilenames = new string[NumInput];
|
|
from = new int[NumInput];
|
|
to = new int[NumInput];
|
|
|
|
verbosity = parser.get<int>("verbosity");
|
|
H5PartSetVerbosityLevel(verbosity);
|
|
|
|
/**************************************************************\
|
|
* Parse input file names and ranges to be merged
|
|
\**************************************************************/
|
|
|
|
for (int i = 0; i < NumInput - 1; ++i){
|
|
Input = string(parser.arguments[i]);
|
|
|
|
if (Input.find("[") != string::npos && Input.find("]") != string::npos){
|
|
Range = Input;
|
|
Range.erase(0,Range.find_first_of("[")+1);
|
|
Range.erase(Range.find_first_of("]"));
|
|
if (Range.find(":") != string::npos){
|
|
from_str = Range;
|
|
from_str.erase(from_str.find_first_of(":"));
|
|
if (from_str.length() == 0)
|
|
from[effNumInput] = 0;
|
|
else
|
|
from[effNumInput] = atoi(from_str.c_str());
|
|
|
|
to_str = Range;
|
|
to_str.erase(0,to_str.find_first_of(":")+1);
|
|
if (to_str.length() == 0)
|
|
to[effNumInput] = -1;
|
|
else
|
|
to[effNumInput] = atoi(to_str.c_str());
|
|
}else{
|
|
if (Range.length() > 0){
|
|
from[effNumInput] = atoi(Range.c_str());
|
|
to[effNumInput] = from[effNumInput];
|
|
}else{
|
|
from[effNumInput] = -1;
|
|
to[effNumInput] = 0;
|
|
}
|
|
}
|
|
Input.erase(Input.find_first_of("["));
|
|
|
|
}else{
|
|
if (Input.find("[") != string::npos){
|
|
Input.erase(Input.find_first_of("["));
|
|
cerr << "error when assigning a range for file " << Input << endl;
|
|
exit(1);
|
|
}
|
|
if (Input.find("]") != string::npos){
|
|
Input.erase(Input.find_first_of("]"));
|
|
cerr << "error when assigning a range for file " << Input << endl;
|
|
exit(1);
|
|
}
|
|
from[effNumInput] = 0;
|
|
to[effNumInput] = -1;
|
|
}
|
|
testopen.open(Input.c_str(),ios::in);
|
|
if (testopen.good())
|
|
{
|
|
InputFilenames[effNumInput] = Input;
|
|
effNumInput++;
|
|
}
|
|
else
|
|
{
|
|
cerr << "Can't open " << Input << endl;
|
|
exit(1);
|
|
}
|
|
testopen.close();
|
|
}
|
|
|
|
OutputFilename = string(parser.arguments[NumInput - 1]);
|
|
|
|
/**************************************************************\
|
|
* check whether output file name is also an input filename;
|
|
* if true then append '_tmp' to the base name;
|
|
* rename it in the end
|
|
\**************************************************************/
|
|
for (int i = 0; i < effNumInput; ++i){
|
|
if (InputFilenames[i] == OutputFilename){
|
|
if (!parser.get<bool>("replace"))
|
|
{
|
|
cout << "Do you really want to overwrite the input file " << InputFilenames[i] << "? [y/N] ";
|
|
char answer = '0';
|
|
cin.get(answer);
|
|
|
|
if (answer != 'y' && answer != 'Y')
|
|
{
|
|
cout << "Aborting..." << endl;
|
|
exit(1);
|
|
}
|
|
}
|
|
ostringstream tmp;
|
|
isOutputInput = true;
|
|
OutputIsInput = i;
|
|
OutputFilename.erase(OutputFilename.find_last_of("."));
|
|
tmp << OutputFilename << "_tmp.h5";
|
|
OutputFilename = tmp.str();
|
|
break;
|
|
}
|
|
}
|
|
|
|
/**************************************************************\
|
|
* check whether output file allready exists
|
|
\**************************************************************/
|
|
if (!isOutputInput)
|
|
{
|
|
testopen.open(OutputFilename.c_str(), ios::in);
|
|
if (testopen.good())
|
|
{
|
|
testopen.close();
|
|
if (!parser.get<bool>("replace"))
|
|
{
|
|
cout << "Do you really want to overwrite the file " << OutputFilename << "? [y/N] ";
|
|
char answer = '0';
|
|
cin.get(answer);
|
|
|
|
if (answer != 'y' && answer != 'Y')
|
|
{
|
|
cout << "Aborting..." << endl;
|
|
exit(1);
|
|
}
|
|
}
|
|
}
|
|
else if (parser.get<bool>("replace"))
|
|
{
|
|
cerr << "WARNING: option 'replace' chosen but output filename is not the same" << endl
|
|
<< " as any of the input filenames" << endl;
|
|
}
|
|
}
|
|
|
|
for (int i = 0; i < effNumInput; ++i)
|
|
{
|
|
H5InFile = H5PartOpenFile(InputFilenames[i].c_str(), H5PART_READ);
|
|
|
|
/**************************************************************\
|
|
* fix range if values are negative
|
|
\**************************************************************/
|
|
int n_steps = H5PartGetNumSteps(H5InFile);
|
|
|
|
if (from[i] >= n_steps)
|
|
from[i] = n_steps - 1;
|
|
else
|
|
while (from[i] < 0)
|
|
from[i] += n_steps;
|
|
|
|
if (to[i] >= n_steps)
|
|
to[i] = n_steps - 1;
|
|
else
|
|
while (to[i] < 0)
|
|
to[i] += n_steps;
|
|
if (to[i] < from[i])
|
|
{
|
|
int tmp;
|
|
tmp = to[i];
|
|
to[i] = from[i];
|
|
from[i] = tmp;
|
|
}
|
|
|
|
NumSteps = to[i] - from[i] + 1;
|
|
|
|
// cout << InputFilenames[i] << "[" << from[i] << ":" << to[i] << "]" << endl;
|
|
|
|
for (int k = from[i]; k <= to[i]; ++k)
|
|
{
|
|
H5PartSetStep(H5InFile,k);
|
|
// cout << "Step set to " << k << endl;
|
|
|
|
/**************************************************************\
|
|
* determine maximum number of particles in all input files
|
|
* and all steps
|
|
\**************************************************************/
|
|
h5part_int64_t localNumParticles = H5PartGetNumParticles(H5InFile);
|
|
if (localNumParticles > maxNumParticles)
|
|
maxNumParticles = localNumParticles;
|
|
}
|
|
|
|
|
|
/**************************************************************\
|
|
* get the *intersection* of names of all file attributes
|
|
* --------------- " ------------------- step attributes
|
|
* --------------- " ------------------- datasets
|
|
* from all files which are to be merged
|
|
\**************************************************************/
|
|
if (parser.get<bool>("intersect-file-attributes") ||
|
|
parser.get<bool>("intersect-datasets") ||
|
|
parser.get<bool>("intersect-step-attributes") )
|
|
{
|
|
StringIntMap localStepAttribName;
|
|
StringInt64Map localStepAttribType;
|
|
StringInt64Map localStepAttribNumType;
|
|
StringIntMap localDataSetName;
|
|
StringInt64Map localDataSetType;
|
|
StringIntMap::iterator itlocalName;
|
|
StringInt64Map::iterator itlocalType;
|
|
StringInt64Map::iterator itlocalNumType;
|
|
|
|
|
|
if (parser.get<bool>("intersect-file-attributes"))
|
|
{
|
|
NumFileAttr = H5PartGetNumFileAttribs(H5InFile);
|
|
// cout << NumFileAttr << " file attributes" << endl;
|
|
for (int l = 0; l < NumFileAttr; ++l)
|
|
{
|
|
|
|
H5PartGetFileAttribInfo(H5InFile, l, AttrName, MAX_LEN, &Type, &NumType);
|
|
// cout << "Found file attribute '" << AttrName << "'" << endl;
|
|
itName = FileAttribName.find(string(AttrName));
|
|
if (itName == FileAttribName.end())
|
|
{
|
|
FileAttribName.insert(make_pair(string(AttrName), 1));
|
|
FileAttribType.insert(make_pair(string(AttrName), Type));
|
|
FileAttribNumType.insert(make_pair(string(AttrName), NumType));
|
|
// cout << "Added file attribute " << AttrName << endl;
|
|
}
|
|
else
|
|
{
|
|
if (FileAttribType[string(AttrName)] == Type)
|
|
{
|
|
if (FileAttribNumType[string(AttrName)] == NumType)
|
|
{
|
|
FileAttribName[string(AttrName)] += 1;
|
|
}
|
|
else
|
|
{
|
|
cerr << "WARNING: found same file attribute (" << AttrName << ") " << endl;
|
|
cerr << " with same type but different length! Will be skipped!" << endl;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
cerr << "WARNING: found same file attribute (" << AttrName << ") " << endl;
|
|
cerr << " with different type! Will be skipped!" << endl;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
for (int k = from[i]; k <= to[i]; ++k)
|
|
{
|
|
H5PartSetStep(H5InFile,k);
|
|
|
|
if (parser.get<bool>("intersect-datasets"))
|
|
{
|
|
NumDataSets = H5PartGetNumDatasets(H5InFile);
|
|
for (h5part_int64_t m = 0; m < NumDataSets; ++m)
|
|
{
|
|
H5PartGetDatasetInfo(H5InFile, m, SetName, MAX_LEN, &SetType, &SetNum);
|
|
// cout << "Found dataset '" << SetName << "' " << endl;
|
|
itName = localDataSetName.find(string(SetName));
|
|
if (itName == localDataSetName.end())
|
|
{
|
|
localDataSetName.insert(make_pair(string(SetName), 1));
|
|
localDataSetType.insert(make_pair(string(SetName), SetType));
|
|
// cout << "Added dataset " << SetName << endl;
|
|
}
|
|
else
|
|
{
|
|
if (localDataSetType[string(SetName)] == SetType)
|
|
{
|
|
localDataSetName[string(SetName)] += 1;
|
|
}
|
|
else
|
|
{
|
|
cerr << "WARNING: found same dataset (" << SetName << ")" << endl;
|
|
cerr << " with different type! Will be skipped!" << endl;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (parser.get<bool>("intersect-step-attributes"))
|
|
{
|
|
NumStepAttr = H5PartGetNumStepAttribs(H5InFile);
|
|
for (int m = 0; m < NumStepAttr; ++m)
|
|
{
|
|
H5PartGetStepAttribInfo(H5InFile, m, StepAttrName, MAX_LEN, &Type, &NumType);
|
|
// cout << "Found Step Attribute '" << StepAttrName << "' " << endl;
|
|
itlocalName = localStepAttribName.find(string(StepAttrName));
|
|
if (itlocalName == localStepAttribName.end())
|
|
{
|
|
localStepAttribName.insert(make_pair(string(StepAttrName), 1));
|
|
localStepAttribType.insert(make_pair(string(StepAttrName), Type));
|
|
localStepAttribNumType.insert(make_pair(string(StepAttrName), NumType));
|
|
}
|
|
else
|
|
{
|
|
if (localStepAttribType[string(StepAttrName)] == Type)
|
|
{
|
|
if (localStepAttribNumType[string(StepAttrName)] == NumType)
|
|
{
|
|
localStepAttribName[string(StepAttrName)] += 1;
|
|
}
|
|
else
|
|
{
|
|
cerr << "WARNING: found same step attribute (" << StepAttrName << ") " << endl;
|
|
cerr << " with same type but different length! Will be skipped!" << endl;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
cerr << "WARNING: found same step attribute (" << StepAttrName << ")" << endl;
|
|
cerr << " with different type! Will be skipped!" << endl;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (parser.get<bool>("intersect-datasets"))
|
|
{
|
|
|
|
// exclude those datasets which do not occur in all steps in file i
|
|
for (itlocalName = localDataSetName.begin(), itlocalType = localDataSetType.begin(); itlocalName != localDataSetName.end(); itlocalName++, itlocalType++)
|
|
{
|
|
if (itlocalName->second != NumSteps)
|
|
{
|
|
cerr << "WARNING: dataset '" << itlocalName->first << "' in file " << InputFilenames[i] << " only found in " << itlocalName->second << " of " << NumSteps << " steps;" << endl;
|
|
cerr << " Skipping this dataset;" << endl;
|
|
localDataSetName.erase(itlocalName);
|
|
}
|
|
else
|
|
{
|
|
itName = DataSetName.find(itlocalName->first);
|
|
if (itName == DataSetName.end())
|
|
{
|
|
DataSetName.insert(make_pair(itlocalName->first, 1));
|
|
DataSetType.insert(make_pair(itlocalName->first, itlocalType->second));
|
|
}
|
|
else
|
|
{
|
|
if (DataSetType[itlocalName->first] == itlocalType->second)
|
|
{
|
|
DataSetName[itlocalName->first] += 1;
|
|
}
|
|
else
|
|
{
|
|
cerr << "WARNING: found same dataset (" << itlocalName->first << ")" << endl;
|
|
cerr << " with different type! Will be skipped!" << endl;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (parser.get<bool>("intersect-step-attributes"))
|
|
{
|
|
// exclude those step attributes which do not occur in all steps in file i
|
|
itlocalType = localStepAttribType.begin();
|
|
itlocalNumType = localStepAttribNumType.begin();
|
|
for (itlocalName = localStepAttribName.begin(); itlocalName != localStepAttribName.end(); itlocalName++, itlocalType++, itlocalNumType++)
|
|
{
|
|
if (itlocalName->second != NumSteps){
|
|
cerr << "WARNING: step attribute '" << itlocalName->first << "' in file " << InputFilenames[i] << " only found in " << itlocalName->second << " of " << NumSteps << " steps;" << endl;
|
|
cerr << " Skipping this step attribute;" << endl;
|
|
// localStepAttribName.erase(itlocalName);
|
|
// localStepAttribType.erase(itlocalType);
|
|
// localStepAttribNumType.erase(itlocalNumType);
|
|
}
|
|
else
|
|
{
|
|
itName = StepAttribName.find(itlocalName->first);
|
|
if (itName == StepAttribName.end())
|
|
{
|
|
StepAttribName.insert( make_pair(itlocalName->first, 1));
|
|
StepAttribType.insert( make_pair(itlocalName->first, itlocalType->second));
|
|
StepAttribNumType.insert( make_pair(itlocalName->first, itlocalNumType->second));
|
|
}
|
|
else
|
|
{
|
|
if (StepAttribType[itlocalName->first] == itlocalType->second)
|
|
{
|
|
if (StepAttribNumType[itlocalName->first] == itlocalNumType->second)
|
|
{
|
|
StepAttribName[itlocalName->first] += 1;
|
|
}
|
|
else
|
|
{
|
|
cerr << "WARNING: found same step attribute (" << itlocalName->first << ") " << endl;
|
|
cerr << " with same type but different length! Will be skipped!" << endl;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
cerr << "WARNING: found same step attribute (" << itlocalName->first << ")" << endl;
|
|
cerr << " with different type! Will be skipped!" << endl;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
H5PartCloseFile(H5InFile);
|
|
}
|
|
}
|
|
|
|
|
|
if (parser.get<bool>("intersect-file-attributes"))
|
|
{
|
|
//erase all those file attributes which do not occur in all files
|
|
itType = FileAttribType.begin();
|
|
itNumType = FileAttribNumType.begin();
|
|
for (itName = FileAttribName.begin(); itName != FileAttribName.end(); itName++, itType++, itNumType++)
|
|
{
|
|
if (itName->second != effNumInput)
|
|
{
|
|
cerr << "WARNING: file attribute '" << itName->first << "' only found in " << itName->second << " of " << effNumInput << " input files;" << endl;
|
|
cerr << " skipping this attribute!" << endl;
|
|
FileAttribName.erase(itName);
|
|
FileAttribType.erase(itType);
|
|
FileAttribNumType.erase(itNumType);
|
|
}
|
|
}
|
|
|
|
if (verbosity > 0)
|
|
{
|
|
cout << "found the following file attributes (intersection of all file attributes):" << endl;
|
|
for (itName = FileAttribName.begin(); itName != FileAttribName.end(); ++itName)
|
|
{
|
|
cout << "file attribute '" << itName->first << "'" << endl;
|
|
}
|
|
cout << endl;
|
|
}
|
|
}
|
|
|
|
if (parser.get<bool>("intersect-datasets"))
|
|
{
|
|
//erase all those datasets which do not occur in all steps of all files
|
|
for (itName = DataSetName.begin(), itType = DataSetType.begin(); itName != DataSetName.end(); itName++, itType++)
|
|
{
|
|
if (itName->second != effNumInput)
|
|
{
|
|
cerr << "WARNING: dataset '" << itName->first << "' only found in " << itName->second << " of " << effNumInput << " input files;" << endl;
|
|
cerr << " skipping this data set!" << endl;
|
|
DataSetName.erase(itName);
|
|
DataSetType.erase(itType);
|
|
}
|
|
}
|
|
|
|
if (verbosity > 0)
|
|
{
|
|
cout << "found the following data sets (intersection of all datasets):" << endl;
|
|
for (itName = DataSetName.begin(); itName != DataSetName.end(); itName++)
|
|
cout << "dataset '" << itName->first << "'" << endl;
|
|
cout << endl;
|
|
}
|
|
}
|
|
|
|
if (parser.get<bool>("intersect-step-attributes"))
|
|
{
|
|
//erase all those step attributes which do not occur in all steps of all fiels
|
|
itType = StepAttribType.begin();
|
|
itNumType = StepAttribNumType.begin();
|
|
for (itName = StepAttribName.begin(); itName != StepAttribName.end(); itName++, itType++, itNumType++)
|
|
{
|
|
if (itName->second != effNumInput)
|
|
{
|
|
cerr << "WARNING: step attribute '" << itName->first << "' only found in " << itName->second << " of " << effNumInput << " input files;" << endl;
|
|
cerr << " skipping this attribute!" << endl;
|
|
StepAttribName.erase(itName);
|
|
StepAttribType.erase(itType);
|
|
StepAttribNumType.erase(itNumType);
|
|
}
|
|
}
|
|
|
|
if (verbosity > 0)
|
|
{
|
|
cout << "found the following step attributes (intersection of all step attributes):" << endl;
|
|
for (itName = StepAttribName.begin(); itName != StepAttribName.end(); ++itName)
|
|
{
|
|
cout << "step attribute '" << itName->first << "'" << endl;
|
|
}
|
|
cout << endl;
|
|
}
|
|
}
|
|
|
|
|
|
H5OutFile = H5PartOpenFile(OutputFilename.c_str(), H5PART_WRITE);
|
|
if (H5OutFile == NULL)
|
|
{
|
|
cerr << "ABORT: could not open '" << OutputFilename << "' for writing!" << endl;
|
|
exit(1);
|
|
}
|
|
|
|
FloatValues = new h5part_float64_t[maxNumParticles];
|
|
IntegerValues = new h5part_int64_t[maxNumParticles];
|
|
|
|
// cout << "ready to write data " << maxNumParticles << endl;
|
|
for (int i = 0; i < effNumInput; ++i)
|
|
{
|
|
H5InFile = H5PartOpenFile(InputFilenames[i].c_str(), H5PART_READ);
|
|
|
|
H5PartSetStep(H5InFile,from[i]);
|
|
H5PartSetStep(H5OutFile,effStep);
|
|
|
|
if (parser.get<bool>("intersect-file-attributes"))
|
|
{
|
|
itType = FileAttribType.begin();
|
|
itNumType = FileAttribNumType.begin();
|
|
for (itName = FileAttribName.begin(); itName != FileAttribName.end(); itName++, itType++, itNumType++)
|
|
{
|
|
// cout << "writing file attribute " << itName->first << endl;
|
|
if (i == 0)
|
|
{
|
|
#ifdef USE_BOOST
|
|
rc = copyFileAttribute(H5InFile, H5OutFile, FileAttributes, itName->first, itType->second, itNumType->second);
|
|
#else
|
|
rc = copyFileAttribute(H5InFile, H5OutFile, itName->first, itType->second, itNumType->second);
|
|
#endif
|
|
}
|
|
else
|
|
{
|
|
#ifdef USE_BOOST
|
|
rc = copyFileToStepAttribute(H5InFile, H5OutFile, FileAttributes, itName->first, itType->second, itNumType->second);
|
|
#else
|
|
rc = copyFileToStepAttribute(H5InFile, H5OutFile, itName->first, itType->second, itNumType->second);
|
|
#endif
|
|
}
|
|
if (rc != H5PART_SUCCESS)
|
|
cerr << "WARNING: could not write file attribute '" << AttrName << "'" << endl;
|
|
}
|
|
// cout << "wrote file attributes" << endl;
|
|
}
|
|
else
|
|
{
|
|
h5part_int64_t NumFileAttr = H5PartGetNumFileAttribs(H5InFile);
|
|
// cout << NumFileAttr << " file attributes" << endl;
|
|
for (h5part_int64_t l = 0; l < NumFileAttr; ++l)
|
|
{
|
|
|
|
H5PartGetFileAttribInfo(H5InFile, l, AttrName, MAX_LEN, &Type, &NumType);
|
|
// cout << "Found file attribute '" << AttrName << "'" << endl;
|
|
if (i == 0)
|
|
{
|
|
#ifdef USE_BOOST
|
|
rc = copyFileAttribute(H5InFile, H5OutFile, FileAttributes, string(AttrName), Type, NumType);
|
|
#else
|
|
rc = copyFileAttribute(H5InFile, H5OutFile, string(AttrName), Type, NumType);
|
|
#endif
|
|
}
|
|
else
|
|
{
|
|
#ifdef USE_BOOST
|
|
rc = copyFileToStepAttribute(H5InFile, H5OutFile, FileAttributes, string(AttrName), Type, NumType);
|
|
#else
|
|
rc = copyFileToStepAttribute(H5InFile, H5OutFile, string(AttrName), Type, NumType);
|
|
#endif
|
|
}
|
|
if (rc != H5PART_SUCCESS)
|
|
cerr << "WARNING: could not write file attribute '" << AttrName << "'" << endl;
|
|
}
|
|
}
|
|
|
|
firstStepInput = true;
|
|
for (int l = from[i]; l <= to[i]; ++l, ++effStep)
|
|
{
|
|
if(!firstStepInput)
|
|
{
|
|
rc = H5PartSetStep(H5InFile, l);
|
|
if (rc != H5PART_SUCCESS)
|
|
{
|
|
cerr << "ABORT: could not change to step #" << l << " in input file '" << InputFilenames[i] << "'" << endl;
|
|
rc = H5PartCloseFile(H5InFile);
|
|
rc = H5PartCloseFile(H5OutFile);
|
|
exit(1);
|
|
}
|
|
rc = H5PartSetStep(H5OutFile, effStep);
|
|
if (rc != H5PART_SUCCESS)
|
|
{
|
|
cerr << "ABORT: could not change to step #" << effStep << " in output file '" << OutputFilename << "'" << endl;
|
|
rc = H5PartCloseFile(H5InFile);
|
|
rc = H5PartCloseFile(H5OutFile);
|
|
exit(1);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
firstStepInput = false;
|
|
}
|
|
NumParticles = H5PartGetNumParticles(H5InFile);
|
|
rc = H5PartSetNumParticles(H5OutFile, NumParticles);
|
|
if (rc != H5PART_SUCCESS)
|
|
{
|
|
cerr << "ABORT: could not set the number of particles in output file '" << OutputFilename << "'" << endl;
|
|
rc = H5PartCloseFile(H5InFile);
|
|
rc = H5PartCloseFile(H5OutFile);
|
|
exit(1);
|
|
}
|
|
if (parser.get<bool>("intersect-step-attributes"))
|
|
{
|
|
itType = StepAttribType.begin();
|
|
itNumType = StepAttribNumType.begin();
|
|
for (itName = StepAttribName.begin(); itName != StepAttribName.end(); ++itName, ++itType, ++itNumType)
|
|
{
|
|
rc = copyStepAttribute(H5InFile, H5OutFile, itName->first, itType->second, itNumType->second);
|
|
if (rc != H5PART_SUCCESS)
|
|
cerr << "WARNING: could not write step attribute '" << itName->first << "'" << endl;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
h5part_int64_t NumStepAttr = H5PartGetNumStepAttribs(H5InFile);
|
|
for (h5part_int64_t m = 0; m < NumStepAttr; ++m)
|
|
{
|
|
H5PartGetStepAttribInfo(H5InFile, m, StepAttrName, MAX_LEN, &Type, &NumType);
|
|
rc = copyStepAttribute(H5InFile, H5OutFile, string(StepAttrName), Type, NumType);
|
|
if (rc != H5PART_SUCCESS)
|
|
cerr << "WARNING: could not write step attribute '" << StepAttrName << "'" << endl;
|
|
}
|
|
}
|
|
|
|
if (parser.get<bool>("intersect-datasets"))
|
|
{
|
|
|
|
for (itType = DataSetType.begin(); itType != DataSetType.end(); ++itType)
|
|
{
|
|
rc = copyDataset(H5InFile, H5OutFile, itType->first, itType->second, FloatValues, IntegerValues);
|
|
if (rc != H5PART_SUCCESS)
|
|
cerr << "WARNING: could not write dataset '" << itType->first << "'" << endl;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
h5part_int64_t NumDataSets = H5PartGetNumDatasets(H5InFile);
|
|
for (h5part_int64_t m = 0; m < NumDataSets; ++m)
|
|
{
|
|
H5PartGetDatasetInfo(H5InFile, m, SetName, MAX_LEN, &SetType, &SetNum);
|
|
rc = copyDataset(H5InFile, H5OutFile, string(SetName), SetType, FloatValues, IntegerValues);
|
|
if (rc != H5PART_SUCCESS)
|
|
cerr << "WARNING: could not write dataset '" << SetName << "'" << endl;
|
|
}
|
|
}
|
|
|
|
}
|
|
rc = H5PartCloseFile(H5InFile);
|
|
}
|
|
rc = H5PartCloseFile(H5OutFile);
|
|
|
|
if (isOutputInput)
|
|
{
|
|
rename(OutputFilename.c_str(), InputFilenames[OutputIsInput].c_str());
|
|
}
|
|
|
|
delete[] FloatValues;
|
|
delete[] IntegerValues;
|
|
delete[] InputFilenames;
|
|
delete[] from;
|
|
delete[] to;
|
|
}
|
|
|
|
h5part_int64_t copyFileAttribute(H5PartFile* In, H5PartFile* Out, string AttrName, h5part_int64_t Type, h5part_int64_t Num)
|
|
{
|
|
h5part_int64_t rc;
|
|
if (Type == H5T_NATIVE_DOUBLE)
|
|
{
|
|
void *Attrib = (double*)malloc(sizeof(double)*Num);
|
|
rc = H5PartReadFileAttrib(In, AttrName.c_str(), (double*)Attrib);
|
|
if (rc == H5PART_SUCCESS)
|
|
rc = H5PartWriteFileAttrib(Out, AttrName.c_str(), Type, (double*)Attrib, Num);
|
|
free(Attrib);
|
|
return rc;
|
|
}
|
|
else if (Type == H5T_NATIVE_CHAR)
|
|
{
|
|
void *Attrib = (char*)malloc(sizeof(char)*Num);
|
|
rc = H5PartReadFileAttrib(In, AttrName.c_str(), (char*)Attrib);
|
|
if (rc == H5PART_SUCCESS)
|
|
rc = H5PartWriteFileAttrib(Out, AttrName.c_str(), Type, (char*)Attrib, Num);
|
|
free(Attrib);
|
|
return rc;
|
|
}
|
|
else if (Type == H5T_NATIVE_INT64)
|
|
{
|
|
void *Attrib = (h5part_int64_t*)malloc(sizeof(h5part_int64_t)*Num);
|
|
rc = H5PartReadFileAttrib(In, AttrName.c_str(), (h5part_int64_t*)Attrib);
|
|
if (rc == H5PART_SUCCESS)
|
|
rc = H5PartWriteFileAttrib(Out, AttrName.c_str(), Type, (h5part_int64_t*)Attrib, Num);
|
|
free(Attrib);
|
|
return rc;
|
|
}
|
|
else
|
|
{
|
|
cerr << "WARNING: unknown type for file attribue " << AttrName << "! Skipping it. " << endl;
|
|
return H5PART_ERR_INVAL;
|
|
}
|
|
}
|
|
|
|
|
|
h5part_int64_t copyFileToStepAttribute(H5PartFile* In, H5PartFile* Out, string AttrName, h5part_int64_t Type, h5part_int64_t Num)
|
|
{
|
|
h5part_int64_t rc;
|
|
if (Type == H5T_NATIVE_DOUBLE)
|
|
{
|
|
void *Attrib = (double*)malloc(sizeof(double)*Num);
|
|
rc = H5PartReadFileAttrib(In, AttrName.c_str(), (double*)Attrib);
|
|
if (rc == H5PART_SUCCESS)
|
|
rc = H5PartWriteStepAttrib(Out, AttrName.c_str(), Type, (double*)Attrib, Num);
|
|
free(Attrib);
|
|
return rc;
|
|
}
|
|
else if (Type == H5T_NATIVE_CHAR)
|
|
{
|
|
void *Attrib = (char*)malloc(sizeof(char)*Num);
|
|
rc = H5PartReadFileAttrib(In, AttrName.c_str(), (char*)Attrib);
|
|
if (rc == H5PART_SUCCESS)
|
|
rc = H5PartWriteStepAttrib(Out, AttrName.c_str(), Type, (char*)Attrib, Num);
|
|
free(Attrib);
|
|
return rc;
|
|
}
|
|
else if (Type == H5T_NATIVE_INT64)
|
|
{
|
|
void *Attrib = (h5part_int64_t*)malloc(sizeof(h5part_int64_t)*Num);
|
|
rc = H5PartReadFileAttrib(In, AttrName.c_str(), (h5part_int64_t*)Attrib);
|
|
if (rc == H5PART_SUCCESS)
|
|
rc = H5PartWriteStepAttrib(Out, AttrName.c_str(), Type, (h5part_int64_t*)Attrib, Num);
|
|
free(Attrib);
|
|
return rc;
|
|
}
|
|
else
|
|
{
|
|
cerr << "WARNING: unknown type for file attribue " << AttrName << "! Skipping it. " << endl;
|
|
return H5PART_ERR_INVAL;
|
|
}
|
|
}
|
|
|
|
h5part_int64_t copyStepAttribute(H5PartFile* In, H5PartFile* Out, string AttrName, h5part_int64_t Type, h5part_int64_t Num)
|
|
{
|
|
h5part_int64_t rc;
|
|
if (Type == H5T_NATIVE_DOUBLE)
|
|
{
|
|
void* Attrib = (double*)malloc(sizeof(double)*Num);;
|
|
rc = H5PartReadStepAttrib(In, AttrName.c_str(), (double*)Attrib);
|
|
if (rc == H5PART_SUCCESS)
|
|
rc = H5PartWriteStepAttrib(Out, AttrName.c_str(), Type, (double*)Attrib, Num);
|
|
free(Attrib);
|
|
return rc;
|
|
}
|
|
else if (Type == H5T_NATIVE_CHAR)
|
|
{
|
|
void* Attrib = (char*)malloc(sizeof(char)*Num);;
|
|
rc = H5PartReadStepAttrib(In, AttrName.c_str(), (char*)Attrib);
|
|
if (rc == H5PART_SUCCESS)
|
|
rc = H5PartWriteStepAttrib(Out, AttrName.c_str(), Type, (char*)Attrib, Num);
|
|
free(Attrib);
|
|
return rc;
|
|
}
|
|
else if (Type == H5T_NATIVE_INT64)
|
|
{
|
|
void* Attrib = (h5part_int64_t*)malloc(sizeof(h5part_int64_t)*Num);
|
|
rc = H5PartReadStepAttrib(In, AttrName.c_str(), (h5part_int64_t*)Attrib);
|
|
if (rc == H5PART_SUCCESS)
|
|
rc = H5PartWriteStepAttrib(Out, AttrName.c_str(), Type, (h5part_int64_t*)Attrib, Num);
|
|
free(Attrib);
|
|
return rc;
|
|
}
|
|
else
|
|
{
|
|
cerr << "WARNING: unknown type for step attribute " << AttrName << "! Skipping it." << endl;
|
|
return H5PART_ERR_INVAL;
|
|
}
|
|
}
|
|
|
|
|
|
h5part_int64_t copyDataset(H5PartFile* In, H5PartFile* Out, string SetName, h5part_int64_t Type, h5part_float64_t* FloatArray, h5part_int64_t* IntegerArray)
|
|
{
|
|
h5part_int64_t rc;
|
|
if (Type == H5T_NATIVE_DOUBLE)
|
|
{
|
|
rc = H5PartReadDataFloat64(In, SetName.c_str(), FloatArray);
|
|
if (rc == H5PART_SUCCESS)
|
|
rc = H5PartWriteDataFloat64(Out, SetName.c_str(), FloatArray);
|
|
return rc;
|
|
}
|
|
else if (Type == H5T_NATIVE_INT64)
|
|
{
|
|
rc = H5PartReadDataInt64(In, SetName.c_str(), IntegerArray);
|
|
if (rc == H5PART_SUCCESS)
|
|
rc = H5PartWriteDataInt64(Out, SetName.c_str(), IntegerArray);
|
|
return rc;
|
|
}
|
|
else
|
|
{
|
|
cerr << "WARNING: unknown type for dataset " << SetName << "! Skipping it." << endl;
|
|
return H5PART_ERR_INVAL;
|
|
}
|
|
}
|
|
|
|
#ifdef USE_BOOST
|
|
h5part_int64_t copyFileAttribute(H5PartFile* In, H5PartFile* Out, StringAnyValueMap &FileAttributes, string AttrName, h5part_int64_t Type, h5part_int64_t Num)
|
|
{
|
|
h5part_int64_t rc;
|
|
if (Type == H5T_NATIVE_DOUBLE)
|
|
{
|
|
void *Attrib = (double*)malloc(sizeof(double)*Num);
|
|
rc = H5PartReadFileAttrib(In, AttrName.c_str(), (double*)Attrib);
|
|
if (rc == H5PART_SUCCESS)
|
|
rc = H5PartWriteFileAttrib(Out, AttrName.c_str(), Type, (double*)Attrib, Num);
|
|
if (rc == H5PART_SUCCESS)
|
|
FileAttributes.insert(make_pair(AttrName,boost::any(*(double*)Attrib)));
|
|
free(Attrib);
|
|
return rc;
|
|
}
|
|
else if (Type == H5T_NATIVE_CHAR)
|
|
{
|
|
void *Attrib = (char*)malloc(sizeof(char)*Num);
|
|
rc = H5PartReadFileAttrib(In, AttrName.c_str(), (char*)Attrib);
|
|
if (rc == H5PART_SUCCESS)
|
|
rc = H5PartWriteFileAttrib(Out, AttrName.c_str(), Type, (char*)Attrib, Num);
|
|
if (rc == H5PART_SUCCESS)
|
|
FileAttributes.insert(make_pair(AttrName,boost::any(string((char*)Attrib))));
|
|
free(Attrib);
|
|
return rc;
|
|
}
|
|
else if (Type == H5T_NATIVE_INT64)
|
|
{
|
|
void *Attrib = (h5part_int64_t*)malloc(sizeof(h5part_int64_t)*Num);
|
|
rc = H5PartReadFileAttrib(In, AttrName.c_str(), (h5part_int64_t*)Attrib);
|
|
if (rc == H5PART_SUCCESS)
|
|
rc = H5PartWriteFileAttrib(Out, AttrName.c_str(), Type, (h5part_int64_t*)Attrib, Num);
|
|
if (rc == H5PART_SUCCESS)
|
|
FileAttributes.insert(make_pair(AttrName,boost::any(*(h5part_int64_t*)Attrib)));
|
|
free(Attrib);
|
|
return rc;
|
|
}
|
|
else
|
|
{
|
|
cerr << "WARNING: unknown type for file attribue " << AttrName << "! Skipping it. " << endl;
|
|
return H5PART_ERR_INVAL;
|
|
}
|
|
}
|
|
|
|
|
|
h5part_int64_t copyFileToStepAttribute(H5PartFile* In, H5PartFile* Out, StringAnyValueMap &FileAttributes, string AttrName, h5part_int64_t Type, h5part_int64_t Num)
|
|
{
|
|
h5part_int64_t rc;
|
|
if (Type == H5T_NATIVE_DOUBLE)
|
|
{
|
|
void *Attrib = (double*)malloc(sizeof(double)*Num);
|
|
rc = H5PartReadFileAttrib(In, AttrName.c_str(), (double*)Attrib);
|
|
if (rc == H5PART_SUCCESS)
|
|
{
|
|
if (FileAttributes.find(AttrName) == FileAttributes.end())
|
|
{
|
|
rc = H5PartWriteStepAttrib(Out, AttrName.c_str(), Type, (double*)Attrib, Num);
|
|
if (rc == H5PART_SUCCESS)
|
|
FileAttributes.insert(make_pair(AttrName,boost::any(*(double*)Attrib)));
|
|
}
|
|
else if(any_cast<double>(FileAttributes[AttrName]) != *(double*)Attrib)
|
|
{
|
|
rc = H5PartWriteStepAttrib(Out, (AttrName + " changed to").c_str(), Type, (double*)Attrib, Num);
|
|
if (rc == H5PART_SUCCESS)
|
|
FileAttributes[AttrName] = boost::any(*(double*)Attrib);
|
|
}
|
|
}
|
|
free(Attrib);
|
|
return rc;
|
|
}
|
|
else if (Type == H5T_NATIVE_CHAR)
|
|
{
|
|
void *Attrib = (char*)malloc(sizeof(char)*Num);
|
|
rc = H5PartReadFileAttrib(In, AttrName.c_str(), (char*)Attrib);
|
|
if (rc == H5PART_SUCCESS)
|
|
{
|
|
if (FileAttributes.find(AttrName) == FileAttributes.end())
|
|
{
|
|
rc = H5PartWriteStepAttrib(Out, AttrName.c_str(), Type, (char*)Attrib, Num);
|
|
if (rc == H5PART_SUCCESS)
|
|
FileAttributes.insert(make_pair(AttrName,boost::any(string((char*)Attrib))));
|
|
}
|
|
else if(strcmp((any_cast<string>(FileAttributes[AttrName])).c_str(),(char*)Attrib) != 0)
|
|
{
|
|
rc = H5PartWriteStepAttrib(Out, (AttrName + " changed to").c_str(), Type, (char*)Attrib, Num);
|
|
if (rc == H5PART_SUCCESS)
|
|
FileAttributes[AttrName] = boost::any(string((char*)Attrib));
|
|
}
|
|
}
|
|
free(Attrib);
|
|
return rc;
|
|
}
|
|
else if (Type == H5T_NATIVE_INT64)
|
|
{
|
|
void *Attrib = (h5part_int64_t*)malloc(sizeof(h5part_int64_t)*Num);
|
|
rc = H5PartReadFileAttrib(In, AttrName.c_str(), (h5part_int64_t*)Attrib);
|
|
if (rc == H5PART_SUCCESS)
|
|
{
|
|
if (FileAttributes.find(AttrName) == FileAttributes.end())
|
|
{
|
|
rc = H5PartWriteStepAttrib(Out, AttrName.c_str(), Type, (h5part_int64_t*)Attrib, Num);
|
|
FileAttributes.insert(make_pair(AttrName,boost::any((h5part_int64_t*)Attrib)));
|
|
}
|
|
else if(any_cast<h5part_int64_t>(FileAttributes[AttrName]) != *(h5part_int64_t*)Attrib)
|
|
{
|
|
rc = H5PartWriteStepAttrib(Out, (AttrName + " changed to").c_str(), Type, (h5part_int64_t*)Attrib, Num);
|
|
if (rc == H5PART_SUCCESS)
|
|
FileAttributes[AttrName] = boost::any((h5part_int64_t*)Attrib);
|
|
}
|
|
}
|
|
free(Attrib);
|
|
return rc;
|
|
}
|
|
else
|
|
{
|
|
cerr << "WARNING: unknown type for file attribue " << AttrName << "! Skipping it. " << endl;
|
|
return H5PART_ERR_INVAL;
|
|
}
|
|
}
|
|
#endif
|