doc/2006-Berkeley-Visit

doc/H5xxx
	- moved to repo "H5Memo"
This commit is contained in:
2006-09-26 04:41:46 +00:00
parent 70d1a26d4c
commit 56f13c3b68
11 changed files with 0 additions and 2874 deletions
-10
View File
@@ -40,13 +40,6 @@ Hdf5fed/makefile.am -text
/config.sub -text
/configure.ac -text
/depcomp -text
doc/2006-Berkeley-Visit/BethelSAP_1page-4.doc -text
doc/2006-Berkeley-Visit/H5Part.ppt -text
doc/2006-Berkeley-Visit/H5xxx.ppt -text
doc/2006-Berkeley-Visit/H5xxxVisIt.fig -text
doc/2006-Berkeley-Visit/SAP-Combined-v6.doc -text
doc/2006-Berkeley-Visit/aqpsi2006mar22.pdf -text
doc/2006-Berkeley-Visit/hdf.ps -text
doc/Building.html -text
doc/BuildingHDF5.html -text
doc/Downloading.html -text
@@ -64,9 +57,6 @@ doc/H5PartVisIt/visit_query1.png -text
doc/H5PartVisIt/visit_query1.s.png -text
doc/H5tools/H5tools.htm -text
doc/H5tools/H5tools_files/snapshot1.jpg -text
doc/H5xxx/H5xxx.tex -text
doc/H5xxx/JAC2003.cls -text
doc/H5xxx/makefile -text
doc/InternalLayout.html -text
doc/ReferencePages/H5PartF_8c.html -text
doc/ReferencePages/H5Part_8c.html -text
Binary file not shown.
Binary file not shown.
Binary file not shown.
-294
View File
@@ -1,294 +0,0 @@
#FIG 3.2
Landscape
Center
Inches
Letter
100.00
Single
-2
1200 2
0 32 #c6b797
0 33 #eff8ff
0 34 #dccba6
0 35 #404040
0 36 #808080
0 37 #c0c0c0
0 38 #e0e0e0
0 39 #8e8f8e
0 40 #aaaaaa
0 41 #555555
0 42 #bebebe
0 43 #515151
0 44 #e7e3e7
0 45 #000049
0 46 #797979
0 47 #303430
0 48 #414141
0 49 #c7b696
0 50 #8e8e8e
0 51 #414541
0 52 #8e8e8e
0 53 #000079
0 54 #ff8200
0 55 #007d00
0 56 #0000be
# Desktop computer
6 375 2400 2025 4050
6 1575 3825 2025 4050
2 3 0 1 -1 7 2 0 20 0.000 0 0 -1 0 0 8
1615 3916 1719 3874 1908 3874 1950 3853 1972 3853 1846 3937
1615 3937 1615 3916
2 3 0 1 -1 7 2 0 20 0.000 0 0 -1 0 0 6
1888 3853 1846 3874 1908 3874 1950 3853 1908 3853 1888 3853
2 3 0 1 -1 7 2 0 20 0.000 0 0 -1 0 0 5
1846 3874 1888 3853 1846 3853 1803 3874 1846 3874
2 3 0 1 -1 7 2 0 20 0.000 0 0 -1 0 0 5
1803 3874 1740 3874 1803 3853 1824 3853 1803 3874
2 3 0 1 -1 7 2 0 20 0.000 0 0 -1 0 0 6
1846 3958 1866 3916 1972 3853 1992 3895 1846 3979 1846 3958
2 3 0 1 -1 7 2 0 20 0.000 0 0 -1 0 0 5
1615 3937 1866 3937 1846 3958 1615 3958 1615 3937
2 3 0 1 -1 7 2 0 20 0.000 0 0 -1 0 0 9
1615 3958 1594 4000 1719 4000 1846 4000 1866 4000 1992 3916
1992 3895 1846 3958 1615 3958
-6
6 600 2400 1950 3675
2 3 0 1 -1 7 2 0 15 0.000 0 0 -1 0 0 6
669 2530 648 3307 1594 3307 1615 2468 1594 2468 669 2530
2 3 0 1 -1 7 0 0 10 0.000 0 0 -1 0 0 5
1615 2468 1594 3307 1719 3307 1719 2509 1615 2468
2 3 0 1 -1 7 0 0 15 0.000 0 0 -1 0 0 5
1762 2677 1908 2719 1908 3139 1762 3203 1762 2677
2 3 0 1 -1 7 2 0 15 0.000 0 0 -1 0 0 6
732 3349 732 3412 1594 3454 1594 3371 753 3349 732 3349
2 3 0 1 -1 7 1 0 15 0.000 0 0 -1 0 0 5
1594 3371 1846 3280 1846 3348 1594 3454 1594 3371
2 3 0 1 -1 7 2 0 15 0.000 0 0 -1 0 0 5
732 3433 732 3559 1594 3601 1594 3454 732 3433
2 3 0 1 -1 0 2 0 20 0.000 0 0 -1 0 0 5
1594 3454 732 3412 753 3433 1572 3454 1594 3454
2 3 0 1 -1 7 2 0 8 0.000 0 0 -1 0 0 5
1594 3454 1594 3601 1635 3580 1635 3433 1594 3454
2 3 0 1 -1 7 2 0 8 0.000 0 0 -1 0 0 5
1719 3538 1846 3475 1846 3349 1719 3391 1719 3538
2 3 0 1 -1 7 2 0 10 0.000 0 0 -1 0 0 5
1635 3496 1719 3454 1719 3391 1635 3433 1635 3496
2 3 0 1 -1 7 2 0 10 0.000 0 0 -1 0 0 5
1635 3580 1719 3538 1719 3475 1635 3517 1635 3580
2 1 0 2 -1 7 2 0 8 0.000 0 0 -1 0 0 2
1635 3496 1719 3454
2 1 0 2 -1 7 2 0 8 0.000 0 0 -1 0 0 2
1635 3517 1719 3475
2 3 0 1 -1 7 3 0 10 0.000 0 0 -1 0 0 7
732 3559 732 3601 816 3601 963 3559 963 3538 816 3559
732 3559
2 3 0 1 -1 7 3 0 10 0.000 1 0 -1 0 0 5
1490 3596 1490 3641 1594 3643 1594 3601 1490 3596
2 3 0 1 -1 7 3 0 10 0.000 1 0 -1 0 0 5
1594 3643 1635 3601 1635 3580 1594 3601 1594 3643
2 3 0 1 -1 7 3 0 10 0.000 0 0 -1 0 0 5
1846 3475 1846 3496 1803 3517 1803 3496 1846 3475
2 3 0 1 -1 7 3 0 10 0.000 0 0 -1 0 0 4
1803 3517 1740 3517 1803 3496 1803 3517
2 3 0 1 -1 7 4 0 15 0.000 1 0 -1 0 0 6
879 3307 732 3349 1594 3371 1846 3279 1762 3265 879 3307
2 3 0 1 -1 7 0 0 10 0.000 1 0 -1 0 0 5
1719 2509 1719 3307 1762 3265 1762 2552 1719 2509
3 1 0 1 -1 7 1 0 20 0.000 0 0 0 11
774 2656 816 2614 1488 2572 1510 2593 1530 2636 1530 3139
1510 3181 1467 3203 837 3203 795 3181 774 3160
1.000 1.000 1.000 1.000 1.000 1.000 1.000 1.000
1.000 1.000 1.000
-6
6 375 3675 1650 4050
2 3 0 1 -1 7 0 0 20 0.000 0 0 -1 0 0 8
1207 3935 1205 3897 1229 3888 1265 3890 1264 3922 1244 3937
1212 3937 1207 3935
2 3 0 1 -1 7 0 0 20 0.000 0 0 -1 0 0 8
1264 3939 1261 3900 1285 3892 1320 3893 1319 3925 1300 3940
1268 3940 1264 3939
2 3 0 1 -1 7 0 0 20 0.000 0 0 -1 0 0 8
1318 3950 1315 3912 1339 3904 1374 3905 1374 3937 1355 3953
1322 3953 1318 3950
2 3 0 1 -1 7 0 0 20 0.000 0 0 -1 0 0 8
1151 3935 1150 3897 1173 3888 1209 3890 1207 3922 1188 3937
1157 3937 1151 3935
2 3 0 1 -1 7 1 0 20 0.000 0 0 -1 0 0 8
1169 3900 1166 3861 1190 3853 1226 3854 1224 3886 1205 3901
1173 3901 1169 3900
2 3 0 1 -1 7 1 0 20 0.000 0 0 -1 0 0 8
1225 3904 1223 3865 1247 3857 1282 3858 1281 3891 1262 3906
1230 3906 1225 3904
2 3 0 1 -1 7 1 0 20 0.000 0 0 -1 0 0 8
1281 3908 1279 3869 1303 3860 1338 3862 1337 3893 1318 3909
1286 3909 1281 3908
2 3 0 1 -1 7 1 0 20 0.000 0 0 -1 0 0 8
1344 3909 1342 3870 1366 3862 1401 3863 1399 3895 1381 3911
1349 3911 1344 3909
2 3 0 1 -1 7 0 0 20 0.000 0 0 -1 0 0 9
1214 3854 1214 3835 1233 3822 1272 3825 1272 3831 1260 3839
1258 3856 1211 3854 1214 3854
2 3 0 1 -1 7 0 0 20 0.000 0 0 -1 0 0 9
1258 3855 1258 3836 1277 3823 1315 3826 1315 3831 1304 3839
1302 3857 1255 3855 1258 3855
2 3 0 1 -1 7 0 0 20 0.000 0 0 -1 0 0 9
1300 3862 1300 3843 1319 3830 1359 3833 1359 3838 1346 3846
1344 3863 1297 3862 1300 3862
2 3 0 1 -1 7 0 0 20 0.000 0 0 -1 0 0 9
1344 3863 1344 3845 1363 3831 1402 3835 1402 3839 1390 3848
1389 3865 1341 3863 1344 3863
2 3 0 1 -1 7 0 0 20 0.000 0 0 -1 0 0 9
1387 3869 1387 3850 1406 3836 1444 3839 1444 3845 1436 3871
1417 3882 1383 3869 1387 3869
2 3 0 1 -1 7 0 0 20 0.000 0 0 -1 0 0 9
708 3798 708 3779 727 3766 766 3769 766 3774 755 3783
752 3800 705 3798 708 3798
2 3 0 1 -1 7 0 0 20 0.000 0 0 -1 0 0 9
747 3803 747 3784 766 3771 805 3775 805 3780 793 3788
791 3805 744 3803 747 3803
2 3 0 1 -1 7 0 0 20 0.000 0 0 -1 0 0 9
794 3805 794 3787 813 3773 852 3777 852 3781 840 3790
839 3807 791 3805 794 3805
2 3 0 1 -1 7 0 0 20 0.000 0 0 -1 0 0 9
832 3810 832 3792 850 3778 889 3781 889 3787 878 3795
876 3812 828 3810 832 3810
2 3 0 1 -1 7 0 0 20 0.000 0 0 -1 0 0 9
876 3812 876 3793 894 3780 934 3784 934 3788 921 3797
919 3814 872 3812 876 3812
2 3 0 1 -1 7 0 0 20 0.000 0 0 -1 0 0 9
919 3816 919 3797 938 3784 977 3787 977 3792 965 3800
964 3817 916 3816 919 3816
2 3 0 1 -1 7 0 0 20 0.000 0 0 -1 0 0 9
962 3816 962 3797 981 3784 1019 3787 1019 3792 1008 3800
1006 3817 959 3816 962 3816
2 3 0 1 -1 7 0 0 20 0.000 0 0 -1 0 0 9
1001 3819 1001 3800 1019 3787 1058 3790 1058 3795 1047 3803
1045 3820 997 3819 1001 3819
2 3 0 1 -1 7 0 0 20 0.000 0 0 -1 0 0 9
1049 3819 1049 3800 1067 3787 1106 3790 1106 3795 1094 3803
1092 3820 1045 3819 1049 3819
2 3 0 1 -1 7 0 0 20 0.000 0 0 -1 0 0 9
661 3822 661 3803 679 3790 718 3793 718 3799 706 3807
704 3824 658 3822 661 3822
2 3 0 1 -1 7 0 0 20 0.000 0 0 -1 0 0 9
704 3831 704 3812 724 3799 762 3802 762 3807 750 3816
748 3832 702 3831 704 3831
2 3 0 1 -1 7 0 0 20 0.000 0 0 -1 0 0 9
748 3834 748 3816 768 3802 806 3805 806 3810 794 3819
793 3835 746 3834 748 3834
2 3 0 1 -1 7 0 0 20 0.000 0 0 -1 0 0 9
789 3838 789 3819 808 3805 847 3808 847 3814 835 3822
833 3839 786 3838 789 3838
2 3 0 1 -1 7 0 0 20 0.000 0 0 -1 0 0 9
832 3840 832 3822 850 3808 889 3812 889 3817 878 3825
876 3842 828 3840 832 3840
2 3 0 1 -1 7 0 0 20 0.000 0 0 -1 0 0 9
874 3842 874 3824 893 3810 932 3814 932 3819 919 3827
918 3844 871 3842 874 3842
2 3 0 1 -1 7 0 0 20 0.000 0 0 -1 0 0 9
916 3846 916 3827 934 3814 974 3817 974 3822 962 3831
960 3847 913 3846 916 3846
2 3 0 1 -1 7 0 0 20 0.000 0 0 -1 0 0 9
961 3846 961 3828 980 3814 1018 3817 1018 3823 1007 3831
1005 3847 957 3846 961 3846
2 3 0 1 -1 7 0 0 20 0.000 0 0 -1 0 0 9
1005 3849 1005 3831 1023 3817 1062 3821 1062 3826 1050 3834
1049 3851 1002 3849 1005 3849
2 3 0 1 -1 7 0 0 20 0.000 0 0 -1 0 0 9
1049 3851 1049 3832 1067 3819 1107 3823 1107 3828 1095 3836
1093 3853 1045 3851 1049 3851
2 3 0 1 -1 7 1 0 20 0.000 0 0 -1 0 0 8
656 3866 655 3827 678 3819 714 3820 713 3853 693 3868
662 3868 656 3866
2 3 0 1 -1 7 1 0 20 0.000 0 0 -1 0 0 8
700 3869 698 3831 722 3822 757 3824 755 3856 737 3871
704 3871 700 3869
2 3 0 1 -1 7 1 0 20 0.000 0 0 -1 0 0 8
744 3876 741 3838 766 3829 801 3831 799 3862 781 3878
748 3878 744 3876
2 3 0 1 -1 7 1 0 20 0.000 0 0 -1 0 0 8
792 3882 790 3842 814 3834 849 3835 848 3868 829 3883
797 3883 792 3882
2 3 0 1 -1 7 1 0 20 0.000 0 0 -1 0 0 8
832 3884 830 3846 854 3837 889 3839 887 3870 869 3886
836 3886 832 3884
2 3 0 1 -1 7 1 0 20 0.000 0 0 -1 0 0 8
880 3890 879 3851 902 3842 938 3844 936 3876 918 3892
886 3892 880 3890
2 3 0 1 -1 7 1 0 20 0.000 0 0 -1 0 0 8
932 3893 930 3854 954 3846 989 3847 988 3879 969 3895
936 3895 932 3893
2 3 0 1 -1 7 1 0 20 0.000 0 0 -1 0 0 8
982 3895 981 3856 1004 3847 1040 3849 1038 3882 1019 3897
988 3897 982 3895
2 3 0 1 -1 7 1 0 20 0.000 0 0 -1 0 0 8
1025 3898 1023 3860 1047 3851 1082 3853 1080 3885 1062 3900
1029 3900 1025 3898
2 3 0 1 -1 7 1 0 20 0.000 0 0 -1 0 0 8
618 3861 616 3822 640 3814 675 3816 674 3847 655 3862
623 3862 618 3861
2 3 0 1 -1 7 1 0 20 0.000 0 0 -1 0 0 8
572 3888 571 3849 594 3840 630 3842 629 3875 609 3890
578 3890 572 3888
2 3 0 1 -1 7 0 0 20 0.000 0 0 -1 0 0 9
609 3888 609 3869 629 3856 667 3860 667 3864 655 3873
654 3890 607 3888 609 3888
2 3 0 1 -1 7 0 0 20 0.000 0 0 -1 0 0 7
654 3893 984 3934 988 3898 664 3866 652 3882 652 3888
654 3893
2 3 0 1 -1 7 0 0 20 0.000 0 0 -1 0 0 9
986 3931 986 3913 1004 3900 1043 3902 1043 3908 1032 3916
1029 3933 982 3931 986 3931
2 3 0 1 -1 7 5 0 15 0.000 0 0 -1 0 0 9
429 3871 683 3743 1551 3816 1576 3831 1573 3840 1426 3990
427 3883 427 3869 429 3871
2 1 0 1 -1 7 0 0 -1 0.000 0 0 -1 0 0 3
423 3910 1434 4014 1575 3875
2 3 0 0 -1 7 5 0 10 0.000 0 0 -1 0 0 8
429 3885 429 3928 1430 4050 1580 3895 1576 3831 1423 3990
425 3882 429 3885
-6
-6
6 4050 3000 7500 3750
2 1 0 1 0 0 0 0 -1 0.000 0 0 -1 0 0 4
7500 3000 5430 3000 6120 3690 4050 3690
-6
# TB Data
6 8025 6000 9000 7425
5 1 0 1 -1 -1 0 0 -1 0.000 0 1 0 0 8490.000 5700.000 8040 6300 8490 6450 8940 6300
5 1 0 1 -1 -1 0 0 -1 0.000 0 1 0 0 8490.000 6600.000 8040 7200 8490 7350 8940 7200
1 2 0 1 -1 -1 0 0 -1 0.000 1 0.0000 8490 6150 450 150 8040 6000 8940 6300
2 1 0 1 -1 -1 0 0 -1 0.000 0 0 0 0 0 2
8940 6225 8940 7200
2 1 0 1 -1 -1 0 0 -1 0.000 0 0 0 0 0 2
8040 6225 8040 7200
4 1 -1 0 0 2 12 0.0000 6 135 270 8490 6675 TB\001
4 1 -1 0 0 2 12 0.0000 6 135 405 8490 7275 Data\001
-6
2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 1 1 2
0 0 1.00 60.00 120.00
0 0 1.00 60.00 120.00
2025 3075 2475 3075
2 4 0 1 0 7 50 0 -1 0.000 0 0 7 0 0 5
3750 4650 3750 2100 2550 2100 2550 4650 3750 4650
2 4 0 1 0 7 50 0 -1 0.000 0 0 7 0 0 5
9075 4650 9075 2100 7875 2100 7875 4650 9075 4650
2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 0 1 2
0 0 3.00 60.00 120.00
8400 4800 8400 6000
2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 0 1 2
0 0 3.00 60.00 120.00
8250 4800 8250 6000
2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 0 1 2
0 0 3.00 60.00 120.00
8100 4800 8100 6000
2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 0 1 2
0 0 3.00 60.00 120.00
8850 4800 8850 6000
2 4 0 1 0 7 50 -1 -1 0.000 0 0 7 0 0 5
12000 4500 12000 3975 8175 3975 8175 4500 12000 4500
4 0 0 50 0 2 15 0.0000 4 30 360 8475 5550 ......\001
4 0 0 50 0 0 20 0.0000 4 195 570 2625 3375 VisIt\001
4 0 0 50 0 0 20 0.0000 4 195 570 7950 3375 VisIt\001
4 0 0 50 -1 2 16 0.0000 4 225 1410 9375 4275 H5xxx plugin\001
4 0 0 50 0 0 20 0.0000 4 255 1650 6450 5550 n-way parallel\001
Binary file not shown.
Binary file not shown.
File diff suppressed because one or more lines are too long
-345
View File
@@ -1,345 +0,0 @@
\documentclass[acus]{JAC2003}
%%
%% Use \documentclass[boxit]{JAC2003}
%% to draw a frame with the correct margins on the output.
%%
%% Use \documentclass[acus]{JAC2003}
%% for US letter paper layout
%%
\usepackage{graphicx}
%%
%% VARIABLE HEIGHT FOR THE TITLE BOX (default 35mm)
%%
\setlength{\titleblockheight}{40mm}
\begin{document}
\title{H5Pie: A Portable High Performance Parallel Data Interface for Electromagnetics Simulations Including Particles\thanks{
This work (LBNL-57607) was supported by the Director, Office of Science, Office of Advanced Scientific Computing Research, of the U.S. Department of Energy under Contract No. DE-AC03-76SF00098 and the DOE SciDAC project on ``Advanced Computing for 21st Century Accelerator Science and Technology.''.
}}
\author{A. Adelmann, A. Gsell, B. Oswald, PSI, Villigen, Switzerland \\
R.D. Ryne, LBNL/AFR, Berkeley, California, USA\\
W. Bethel, J.M. Shalf, C. Siegerist, LBNL/NERSC, Berkeley, California, USA}
\maketitle
\begin{abstract}
The very largest parallel particle simulations, for problems
involving six dimensional phase space,
generate vast quantities of data. It is desirable to
store such enormous datasets efficiently and also to share data
effortlessly between data analysis tools
such as VisIt or ParaView
among other groups who are working on particle-based accelerator
simulations. We define a very simple file schema built on top of HDF5~\cite{hdf5hp}
(Hierarchical Data Format version 5) as well as an API that simplifies
the reading/writing of the data to the HDF5 file format. HDF5 offers a
self-describing machine-independent binary file format that supports
scalable parallel I/O performance for MPI codes on computer systems
ranging from laptops to supercomputers.
H5Pie is actually a composition of three parts. On a very abstract view we have,
H5Part which handles the macro particles of a simulation. H5Mesh defines the (spatial) geometry of the
computational domain and H5Fields represents scalar and vector filds. (H5Pie = H5{\bf P} art + H5F{\bf i}eld+ H5M{\bf e}sh)
The sample H5Pie
API will be available for C, C++, and Fortran codes.
The common file format will enable groups that
use completely different simulation implementations to transparently share
datasets.
\end{abstract}
\section{Motivation}
The motivation for this undertaking is to create a file format that is
suitable for large-scale parallel simulation codes.
A suitable data format must have the following properties:
it must be a machine-independent binary representation that is
self-describing, easily extensible, language independent, efficient
(both for serial and parallel), and produces files that are seamlessly sharable
between different programs. In the following sections we describe
the motivation for these features and how they can be accomplished
using the proposed implementation.
\subsection{Machine Independence}
Processor architectures use different binary representations
for data. While the IEEE 754 standard has decreased the number of
differing floating point number represenations, byte-order still remains
a source of incompatibility. While it is trivial to byte-swap a file from
a programming standpoint, it creates a number of long-term file management
difficulties for groups that are sharing or maintaining a repository of simulation
data. Given files that store data structures with differently sized elements,
one must know the storage format precisely in order to apply byte-swapping
properly.
HDF5 does not suffer from these problems because the file format is completely
self-describing and the internal binary structures are all machine independent.
The HDF5 library is able to convert data that is stored in any native machine
represenation in the data file into a native binary representation in memory
as part of the reading process with little performance penalty.
\subsection{Language Independence}
The three most common languages used for implementing applications
in the arena of computational sciences are
Fortran, C and C++. The file format and associated API must hide
differences in the binary file-storage conventions of these languages
as well as offering native API bindings for each of these
programming languages.
For instance, Fortran unformatted binary files contain integer fields
at the beginning and the end of each record that describe the length
of the record. The size of these integer tags is usually 32-bits, but some
fortran compiler implementations will use larger tags in order to represent
larger record sizes. C and C++ binary files have no such convention for
record-oriented storage. The language-dependent differences in
binary storage layout conventions
can cause difficulties for scientists who wish to share data files between
Fortran and C/C++ implementations of a code, or with visualization tools
that are primarily written in C/C++. The API bindings and underlying file format
provided by the H5Part API and the underlying HDF5 file format are able to
hide these differences in order to provide symmetric access via all languages.
\subsection{Self-Describing}
The data is accessed by names, for example, one might ask for `the
column of data called $p_x$' -- affording a layer of file-layout independence.
In other words, self-describing data is
not accessed by a position in a file but by name of the datasets. Various attributes
of the data that may be necessary to using it are available. For
example, one can ask ``what are the units of column $p_x$?'.
There are a number of examples of self-describing file formats. Examples include HDF earlier HDF implementations leading up to HDF version 5 and the Unidata NetCDF format. Another
very popular approach is the Self Describing Data Sets (SDDS) \cite{SDDS} although it
is only serial in nature. HDF5 is a complete rewrite of the HDF file format that supports
parallel I/O and offers a much leaner, more flexible interface.
Because it is self-describing, the entire contents of an HDF5 file can be
browsed and even converted to ASCII text, including XML syntax text files,
using the built-in 'h5ls' and 'h5dump'
tools without specific knowledge of the internal file format.
\begin{figure}[htb]
\begin{minipage}[b]{0.5\textwidth}
%\includegraphics[scale=0.095]{./partview1.eps}
%\includegraphics[scale=0.097]{./density70.eps}
\end{minipage}
\caption{color: A common self-describing file format allows different codes to share a common set of visualization and data analysis tools. PartView and AVS/Express, pictured above, are able to read and display contents of an H5Part/HDF5 file written on any machine in any language, regardless of how many processors are used.}
\label{fig:vis}
\end{figure}
The primary advantage of accessing data and its attributes is that one
can then construct more flexible data manipulation tools that are capable
of surviving the natural evolution of file formats. Data formats can be extended
to include additional information without breaking older file readers.
Self-describing data contains all the information
that analysis tools need to manipulate various types of data
correctly. Two examples of such tools using the proposed file format
are shown in Figure ~\ref{fig:vis} As a result, data exchange between different simulations tools is
much simpler, robust and better defined by using self-describing data sets.
\subsection{High Performance}
The HDF5 file format allows data elements to be written to disk in the
native binary representation. The file format stores a description of
the native data representation of the machine that wrote the data so
that it can be automatically translated to the native binary
representation of the machine that reading the data (eg. if the byte
order differs). In contrast to XDR, where the data always gets
translated to/from an intermediate machine-neutral format, the HDF5
data conversion only occurs if the stored data represenation is
different from the native binary representation of the machine that is
reading the file, so there is no performance penalty if the machines
have the compatible binary data formats. [ada: need work] In general, HDF5 offers
performance that is very close to what can be achieved by writing an
ad-hoc machine-dependent binary for F77 unformatted data file.
\subsection{Parallel I/O}
HDF5 also supports parallel I/O capabilities for MPI programs.
The naive approach to writing data from a parallel program is to
write one file per processor. While this is simple to implement and
very efficient on most cluster filesystems, it leads to file management
headaches when it comes the time to analyze the data. One must either
recombine these separate files into a single file or create ponderous
user-interfaces that allow a data analysis application to read from a
directory full of files instead of just one file.
Parallel I/O methods, allow you to write data into a single file from all
of the tasks of a parallel program. The performance is typically lower
than that of writing one-file-per-processor, but it makes data management
much simpler after the program has finished. No additional recombining
steps are required to make the file accessible by vis-tools or for
restarting a simulation using a different number of processors.
Parallel HDF5 uses MPI-I/O for its low-level implementation. The
mechanics of using MPI-I/O are all hidden from the user by our
H5Part file API (the code looks nearly identical to reading/writing the data
from a serial program). While the performance is not as good as
writing one-file-per-processor, we demonstrate that writing files
with Parallel HDF5 is consistently faster than writing the data
in raw/native binary using the MPI-I/O library. This efficiency is
made possible through sophisticated HDF5 tuning directives that
control data alignment and caching within the HDF5 layer.
Therefore, we argue that it would be difficult to match
HDF5 performance even using a home-grown binary file format.
\section{H5Mesh File Organization and API}
\section{H5Field File Organization and API}
\section{H5Part File Organization and API}
The proposed file storage format uses HDF5 for the low-level file
storage and a simple API to provide a high-level interface to that
file format. A programmer can either use the H5Part API to access
the data files or write directly to the file format using some simple
conventions for organizing and naming the objects stored in the
file.
The HDF5 format, its benefits, and its file organization is decribed
at \cite{hdf5hp}. The file format was also adopted by the DOE
ASCI-VIEWS effort, so the library has been tuned and adapted to read
and write data efficiently on large-scale parallel computing
systems. We adopted HDF5 for our file storage needs because it offers
all that is needed as stipulated in the motivation section.
We describe now the H5Part conventions for storing
objects in the HDF5 file as well as some examples of the API.
\subsection{H5Part File Organization}
In order to store Particle Data in the HDF5 file format, we have
formalized the hierarchical arrangement of the datasets and naming
conventions for the groups and associated datasets. The sample H5Part API formally
encodes these conventions in order to provide a simple and uniform
way to access these files from C, C++, and Fortran codes. The API makes
it easier to write very portable data adaptors for visualization tools in order to
expand the number of tools available to access the data. Even so,
anyone can use the HDF5 $h5ls$ utility to examine the organization
of the H5Part files and even write their own HDF5-based interface for reading and writing the
file format. The standards offered by the sample API are completely independent of the
standard for organizing data within the file.
The file format supports the storage of multiple timesteps of
datasets that contain multiple fields. The fields correspond to different properties of
the particles at that particular time step -- for instance,
the 3-dimensional cartesian position of the particles
$(X,Y,Z)$ as well as the 3-dimensional phase of each
particle $(PX,PY,PZ)$. These two degrees of freedom
are organized such that the timesteps are groups (time groups) that are added sequentially
to the root group (``/''). The fields are datasets that are nested within the
time groups. The convention for naming the time group is $Particles<integer>$ where $<integer>$ is a monotonically increasing counter for the number of timesteps stored in the file.
The fields contained within a given time group are simply named for the property of the particle they represent. For instance, the phase of the particle stored in a simulation variable called 'px' is simply named $``px''$. The field names are user-defined and
can be understood automatically by the visualization
tools that read the file. The only other convention is that each time group must contain the same set of fields -- the contents of the fields will change, but the set of names for these fields must remain the same for all timesteps.
The fields can be either integer or real data types. Initially, the file format supports double precision float and 64-bit integers in order to simplify the requirements for file readers, but HDF5 is capable of automatically down-converting to 32-bit data types upon request. The API will be extended accordingly to support these conversions.
Finally, the file, the individual timesteps, and the individual data arrays can contain {\em attributes} that provide additional information about the data. For instance, the datasets can be annotated with attributes containing {\em units} for a given data field, simulation parameters, or code revision information. The {\em attributes} are key-value pairs where the $key$ is a string that is associated with the file, group, or dataset, and the $value$ is either a string, a real value, or an integer associated with that key.
%Visualization and data analysis tools can take advantage of this additional information if they are programmed to recognize it, but less-sophisticated tools can safely ignore it without compromising their ability to read the file format.
% ada: to philosophic for a 3 page paper, although it is very important and of course right!
%The ability to ignore such information is arguably one of the most powerful capabilities of the HDF5 file format. Whereas changes to headers or data layout in conventional, non-self-describing file formats can render an older reader interface useless, one can continuously add new descriptive information to the HDF5 file without having any detrimental effect on readers that were designed to work with older versions of the file format. This provides enormous benefits for the long term maintenance and evolution of the file format.
\section{GENERAL FORM IN PSEUDOCODE}
In Figure \ref{fig:usage} we show the very simple API for writing
data. The API for reading is almost symmetric. It is also worth to
note that there are minimal differences whether one read/write serial
or in parallel. The API consists of a small number of C, C++ and
Fortran functions and will be described elsewhere.
In the parallel case the original domain decomposition can be used or
the data can be decomposed according to the new number of processor
nodes available.
The resulting HDF5 file will contains a simple directory structure that can be navigated using the generic 'h5ls' utility;
\begin{figure}[h!] \label{fig:sbendVect}
\newsavebox{\gogo}
\setbox\gogo=\hbox{%
\begin{minipage}{0.4\textwidth}
\small
\begin{tabbing}
111111\=aaaa\=aaaa\=aaaa\=\kill
\texttt{if(not parallel);}\\
\quad\texttt{filehandle=OpenFile(filename,mode)} \\
\texttt{else}\\
\quad\texttt{filehandle=OpenFile(filename,mode,mpicomm)} \\
\texttt{SetNumberOfParticles(filehandle);}\\
\texttt{loop(step=1,NSteps);}\\
\quad\texttt{SetStep(filehandle,step);}\\
\quad\texttt{WriteData(filehandle,fieldname1,data1);}\\
\centerline{\texttt{{\bf write more data}}}\\
\quad\texttt{WriteData(filehandle,fieldname<n>,data<n>);}\\
\texttt{CloseFile(filehandle);}
\end{tabbing}
\end{minipage}
}
\fbox{\usebox{\gogo}}
\caption{Usage of H5Part in pseudo-code}
\label{fig:usage}
\end{figure}
% \begin{figure}[h!] \label{fig:sbendVect}
% \newsavebox{\gugu}
% \setbox\gugu=\hbox{%
% \begin{minipage}{0.4\textwidth}
% \small
% \begin{tabbing}
% 111111\=aaaa\=aaaa\=aaaa\=\kill
%\texttt{/Particles1/fieldname1 } \\
%\texttt{/Particles1/fieldname2} \\
%\texttt{ . . . } \\
%\texttt{/Particles1/fieldname<n> } \\
%\texttt{/Particles2/fieldname1} \\
%\texttt{ . . .}\\
%\texttt{ /Particles<nsteps>/fieldname<n>}
%\end{tabbing}
%\end{minipage}
%}
%\fbox{\usebox{\gugu}}
%\end{figure}
\vspace{-5mm}
\section{PERFORMANCE}
% [ada] We also have data for 4 procs
% [ada] name for one file per proc
Preliminary performance estimations, looking at global (GD) and local data (LD)
rates, suggests that our HDF5 writing has a very good performance even
with respect to raw mpi, as shown in Table \ref{tab:perf}.
\begin{table}[h!]
\begin{flushleft} \footnotesize
\begin{tabular}{|l|l|l|}
\hline
\bf Mode & \bf GD [MB/s] & \bf LD [MB/s]\\
\hline
mpi-io (one file) & 241 & 3.7 \\
\hline
one file per proc & 1288 & 20 \\
\hline
H5Part/pHDFf5 (one file) & 773 & 12 \\
\hline
\end{tabular}
\end{flushleft}
\caption{\label{tab:perf}{64 IBM SP-3 nodes writing $51e6$ particles (6D). }}
%, each having 6 double-precision floating point fields for 64 timesteps over 3 trials.}}
\end{table}
\vspace{-4mm}
\section{Conclusions and Future Work}
The file format will be extended in the near future to integrate fast bitmap indexing
technology~\cite{DEX} in order to provide accelerated queries of data
stored in the file. With fastbit technolgy, a user can efficiently extract subsets of data
using compound query expressions such
as {\it $(velocity > 1e6)$ AND $(0.4 < phase < 1.0)$}.
%Such a query-driven approach offers advantages over scalable technologies aimed at visualizing ever-larger datasets.
We are also constantly tuning the performance of the parallel data
file format implementation. We will also be porting the H5Part reader
to a wider variety of visualization tools.
%in order to expand the arena
%of tools available for analyzing the stored particle data.
\begin{thebibliography}{9} % Use for 1-9 references
\bibitem{FPAT082}
A. Adelmann, R.D Ryne, C. Siegerist, J. Shalf, "From Visualization to Data Mining With Large Datasets," PAC, 2005.
\bibitem{hdf5hp}
HDF5 Home Page, http://hdf.ncsa.uiuc.edu/HDF5.
\bibitem{SDDS}
Definitions and libraries for SDDS implementation may be found at the link http://www.aps.anl.gov/asd/oag/oagPackages.shtml.
\bibitem{DEX}
K. Stockinger, J. Shalf, W. Bethel, K. Wu. "DEX: Increasing the Capability of Scientific Data Analysis Pipelines by Using Efficient Bitmap Indices to Accelerate Scientific Visualization." Scientific and Statistical Database Management Conference (SDDBM), 2005.
\end{thebibliography}
\end{document}
-217
View File
@@ -1,217 +0,0 @@
%%
%% This file has been developed as a common template for papers
%% destined for electronic production using Adobe Acrobat
%% software for Accelerator Conferences
%%
%% See the JACo Website for more information
%%
%% http://www.cern.ch/accelconf/
%%
% Special thanks to John Jowett and Michel Goossens from CERN and
% Martin Comyn at TRIUMF for their significant contributions to
% this class file over the period 1996 to 2000.
% John Poole
% March 2000
% JAC2001.cls is a modified version of JAC2000.cls to produce indented
% first paragraphs after section, subsection and subsubsection headings.
%
% Martin Comyn April 2001
\def\fileversion{1.2}
\def\filedate{2003/03/10}
\def\docdate {2003/03/10}
\newlength{\dsep}
\newlength{\dwid}
\newlength{\dht}
\NeedsTeXFormat{LaTeX2e}
\ProvidesClass{JAC2003}[\filedate\space Version \fileversion]
\DeclareOption{acus}
{\setlength{\dwid}{6mm}%
\setlength{\dht}{-18mm}
\setlength{\paperheight}{11in}
\setlength{\paperwidth}{8.5in}
\typeout{Setup for US LETTER PAPER}}
\DeclareOption{aca4}
{\setlength{\dwid}{0mm}%
\setlength{\dht}{0mm}}
\DeclareOption{boxit}
%%
%% This option draws a box on the output text with the correct margins
%%
{\special{!userdict begin /bop-hook{
gsave
newpath
58 55 moveto
58 737 lineto
539 737 lineto
539 55 lineto
closepath
stroke
grestore
}def end}}
\DeclareOption*{\PassOptionsToClass{\CurrentOption}{article}}
\ExecuteOptions{aca4}
\ProcessOptions
\LoadClass[10pt]{article}
%%
%% Using Times fonts produces a Uniform, compact document
%% suitable for the Acrobat Software and printable anywhere.
%%
%% In a 'modern' TeX installation
%% e.g TeXLive Version 3 or later (more information on the JACo Website)
%% one can use the following command
%%
\renewcommand{\rmdefault}{ptm}
%%
%% Older installations may require
%% \RequirePackage{times}
%%
\typeout{------------------------------------------------------------------------}
\typeout{LaTeX2e Class file for Accelerator Conference publication for LaTeX2e users}
\typeout{ }
\typeout{Use the boxit option to draw a box on page showing the correct margins}
\typeout{ }
\typeout{Itemize, Enumerate and Description environments are compact versions}
\typeout{------------------------------------------------------------------------}
\typeout{ }
%
% DIMENSION OF TEXT:
\setlength{\textheight}{241mm}
\setlength{\textwidth}{170mm}
\setlength{\topmargin}{23mm}
\addtolength{\topmargin}{\dht}
\setlength{\oddsidemargin}{20mm}
\setlength{\evensidemargin}{20mm}
\setlength{\columnsep}{5mm}
\hoffset -25.4mm
\voffset -25.4mm
\columnseprule 0pt % Width of rule between columns.
% Use Michel Goossens' dense lists
% Prevent infinite loops
\let\Otemize =\itemize
\let\Onumerate =\enumerate
\let\Oescription =\description
% Zero the vertical spacing parameters
\def\Nospacing{\itemsep=0pt\topsep=0pt\partopsep=0pt\parskip=0pt\parsep=0pt}
% Redefine the environments in terms of the original values
\newenvironment{Itemize}{\Otemize\Nospacing}{\endlist}
\newenvironment{Enumerate}{\Onumerate\Nospacing}{\endlist}
\newenvironment{Description}{\Oescription\Nospacing}{\endlist}
\renewcommand{\topfraction}{.95}
\renewcommand{\bottomfraction}{.95}
\renewcommand{\textfraction}{0.1}
\renewcommand{\floatpagefraction}{0.8}
\def\@oddhead{}\def\@evenhead{}
\def\@oddfoot{}
\def\@evenfoot{\@oddfoot}
%-----------------------------------------------------------------------
% We acknowledge the borrowing of the clever part of
% Steven Gildea's definition of \section
% from geophysi.sty.
%
% \section is tricky because we want to both support \section* and have
% the section title uppercased.
\def\section{\secdef\@sectionb\@sections}
\def\@sectionb[#1]#2{\@sections{#2}}
\def\@sections#1{\@startsection {section}{1}{\z@}
{2.8ex plus 0.8ex minus .1ex}{1.5ex plus .2ex}
{\large\bf\boldmath\centering}{\uppercase{#1}}}
% Should there be any problem with this, we can revert to the following
% but then the onus is on the user to type the argument of \section in
% uppercase in the input file.
%
%\def\section{\@startsection {section}{1}{\z@}{2.8ex plus 0.8ex minus.1ex}
% {1.5ex plus .2ex}{\large\bf\boldmath\centering}}
\def\subsection{\@startsection{subsection}{2}{\z@}{2.6ex plus .8ex minus
.17ex}{1.2ex plus .17ex}{\large\it}}
\def\subsubsection{\@startsection{subsubsection}{3}{\parindent}
{2.5ex plus .7ex minus .17ex}{-1em}{\normalsize\bf}}
\def\paragraph{\@startsection
{paragraph}{4}{\z@}{2.5ex plus .7ex minus .17ex}{-1em}{\normalsize\it}}
\def\subparagraph{\@startsection
{subparagraph}{4}{\parindent}{2.25ex plus .7ex minus
.17ex}{-1em}{\normalsize\bf}}
\setcounter{secnumdepth}{0}
% This definition of \maketitle taken from article.sty, and has been
% somewhat modified.
\def\maketitle{\par
\begingroup
\def\thefootnote{\fnsymbol{footnote}}
\def\@makefnmark{\hbox
to 5pt{$^{\@thefnmark}$\hss}}
\twocolumn[\@maketitle]
\@thanks
\endgroup
\setcounter{footnote}{0}
\let\maketitle\relax
\let\@maketitle\relax
\gdef\@thanks{}\gdef\@author{}\gdef\@title{}\let\thanks\relax}
\newlength{\titleblockheight} % so user can change it if need be
\setlength{\titleblockheight}{3.5cm}
\def\@maketitle{\vbox to \titleblockheight {\hsize\textwidth
\linewidth\hsize \vskip 3pt \centering
{\Large\bf \@title \par}
\vskip 2em % Vertical space after title.
{\large\begin{tabular}[t]{@{}c@{}}\@author \end{tabular}\par}
\vfil}}
% The \copyrightspace command is used to produce a blank space in the first
% column where a copyright notice may go. It works by producing a
% blank footnote of the appropriate size. Note that it should appear
% after any \footnote commands that produce footnotes for the first
% column. Adjusted to leave just 1cm.
\def\copyrightspace{\footnotetext[0]{\mbox{}\vrule height 1cm width 0pt}}
\def\abstract{\subsection*{Abstract}}
\def\endabstract{\par}
% Redefine to use smaller fonts
\def\thebibliography#1{\setlength{\itemsep}{0pt}\setlength{\parsep}{0pt}
\section*{REFERENCES\@mkboth
{REFERENCES}{REFERENCES}}\small\list
{[\arabic{enumi}]}{\settowidth\labelwidth{[#1]}\leftmargin\labelwidth
\advance\leftmargin\labelsep
\usecounter{enumi}}
\def\newblock{\hskip .11em plus .33em minus .07em}
\sloppy\clubpenalty4000\widowpenalty4000
\sfcode`\.=1000\relax}
\let\endthebibliography=\endlist
\twocolumn
\sloppy
\parindent 1em
\leftmargini 2em
\leftmargin\leftmargini
\leftmarginv .5em
\leftmarginvi .5em
\flushbottom
%-----------------------------------------------------------------------
-14
View File
@@ -1,14 +0,0 @@
#################################################
##BEGIN OF makefile "makefile" - using the gnu-compiler g++
##This file is used by the Make command.
###################################################
FILE=H5xxx
all: $(FILE).tex
latex $(FILE).tex
#bibtex $(FILE)
dvips -j0 $(FILE).dvi -o $(FILE).ps
# dvips $(FILE).dvi -o $(FILE).ps -x 1200 -O 10mm,15mm
clean:
rm *~ *.bbl *.aux *.blg *.log *.dvi *.ps