mirror of
https://github.com/paulscherrerinstitute/sf_daq_buffer.git
synced 2026-11-10 23:23:53 +01:00
Merge branch 'master' into eiger
This commit is contained in:
Hax Damiani Leonardo committed
commit
3f7fef4f61
11 files changed
+198
-82
No files matched your search
@@ -17,6 +17,7 @@ Documentation of individual components:
|
||||
- [sf-stream](sf-stream) (Live streaming of detector data)
|
||||
- [sf-writer](sf-writer) (Read from buffer and write H5)
|
||||
- [sf-utils](sf-utils) (Small utilities for debugging and testing)
|
||||
- [jf-live-writer](jf-live-writer) (Live writer to high performance detectors)
|
||||
|
||||
## Design goals
|
||||
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 39 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 104 KiB |
@@ -1 +1 @@
|
||||
<mxfile host="Electron" modified="2020-07-27T14:56:31.435Z" agent="5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) draw.io/12.9.9 Chrome/80.0.3987.163 Electron/8.2.1 Safari/537.36" etag="ylPe3tuhqdRVXZDWMhbB" version="12.9.9" type="device" pages="6"><diagram name="overview" id="muXDs5UsbG9aPfefTxRP">7V1rV6pMFP41fdQFw/1jF63O21WtzulLCwWVJDDAtH79Oygoly0OOKgVnrVOOiDis599nT3jEXf6Njt31PHw2tZ08wgx2uyIOztCCDECwn/8kc/FCIuHFiMDx9CCsdVA2/jSg0EmGJ0Ymu4uxoIhz7ZNzxjHB3u2Zek9LzamOo49dWMf0rfN+KeO1YGeGmj3VDM9+mRo3jAYFRlmdeBCNwbD4KOV8MCbGp4cDLhDVbOnkSGuccSdOrbtLZ69zU5100cvxOW0d25Z15/Psjq0/z51bv/w40FtcbFmnrcsv4KjW17hS58xlnTDjT6cm8bJ5eB61LK+WsFbmA/VnAR4Bd/V+wwBdOyJpen+RZgj7mQ6NDy9PVZ7/tEp5gweG3pvJn7F4qeu59gj/dQ2bWf+bu5k/m95JBSBf27fMM3Imc35wx+3LS/gEScGryPnMfMHHldNY2DhMVPvY1ROCEEKwPzQHU+fRSgSgHau22+653ziU4KjohSA9Bkn9nTFJiQEY8MIkeSQSGrA4MHy0isp4SeBoGCh/WuNT82PM6YnXJ+r7bv/XnuuRV1oUUFYtqWn5BgMUkd4CinkEFLGbTA0X6R7fnb/yHTOu19P3M3730GzxkqbQRxgFMdbfumlYVO74WUZGIzwXWycbjKf5huLgUmDJQhlYSUUxiob+63BqqE4WDUW1YUUXCC1SkNLTIF1bE7VTxePORPLMqyBj5Fj93TXxY4xCSRGxMuyqYEuAjob2sMeRl/H4yc+vgb2hMfBgTdD0/yPAe1B3GJEbDDPgKqfzRRi6ypLocDWm1dhJdWoBPmybENx05APFHK6R8CQACRK4zLajASOisb+096ku45aES53Fyy76i4H1N5oMOfe7cQzDZ/I83FNdUa3+DKG5/OCqeOoMzaI5qNsnKpori5B6MDMSe2pnmH75Ff815rhYLgXA5bt+JgmwxBN1eV+DwplxJ6sd/vkyrDOyq03/UldqEHWjAeMmVQWAbiKAHslwGpkBwzoPo5OX72ncfPspq3VrOfGVfudJNjcj++XeCLfD4ElLs+kHyul3T8pXpnwU/EdOweDTYFxtwh8fEDG6tTCAQdiuviTRPXNtxRW1x3PFYwpd+hHBV3iGo4Q5xhQzIUAqpQWcbEEgUYuK7MOksMPudjf6HIFXdZ4yOXKqMuJYg5dQGsE/52CLpavKLBfCuw07Prz+DS+vG49fHwIjQfxscM2hu81RGARdW2gt4OXPqr2wLZUs7EaTfiX1TlXtj0OJP6qe95nIEh14tmJ0mCkFlvAJbn2xOnpWSfKixP9r5IpKkc3MaU+oifBqAdvvbMNfIdLEfMcikmY45NBj6c6A90L3pYQ3vI+ttBpGZCnaHoBxjHBiu8TOzxQW+jVMT6BQ+PZ6iB+NvD/dn11ccJr4XtbXG5xkCDQKSmAQUCFvz9/pOYCOEjrl1X+0oIggY1TQgFUnoV0XqAQBU1PblXpQ5sqzourDsS3P025AYXLpFFQuTVpLh4u8lI6XmSRCEDFSmVhpRSGKht6KgHjrsEgiBfygKF8EywuP487z6gtvXH62cizJl/K3T1tLPitsQinEYW00iwVKwYNogDNs3LiTrgn7UOqdV5fZFEavSAi+/Lj4soi1bxMYpFPbSTqU1yaABIg/yUp6MufIKas5F+a/OW9yx8qLdCIQd3+C4ZXV99+UxhKhySJIjbgJFgZIAmNGNS5um6eW+ZIe7D19qR1ef91d0niPkMT0Tf12bHfoYXh0S0teHrWM1XXNXpQOTUKNWmXz7ouoRAhVqxLEs8qnMDLnCwxiri4neDaQl2RFExbJMiMyAtscDS8E8TXUeyRkwRZMcPGnDZabs2Q8Zapr5jokZAT1Fnk6KnEN3UdQc6+Dr0EGiQmQQ9KRcxvRUyOoUPMZYJROjEfJg/Pt19NSx1coP/Uu4bDORckPRK/vYVSUeIS2mkLJSi00iIhz3b8hmTEmOrnNmW5w5x/LIcOEtDiCPKBRtEApEPxokHJzaDc5plasL5CRXMagvjXNPRRv4+t55P32r8ayLV0DeH8rtn+tozOpAP5DFJcSpzE1bnoIy00cDKJr4vRh1SSCMvxWGtiKCQCGZjc03tgUt6VBV4oRVL705i0r1loDNOd9Pu+j6hUZ6U6SJIOWXVypMea6ql+OEBQRkvVvFIKVRb6QsJwhcu94isz0nizMhsXE7894GDNChFP9AQwBsOka6bKmiPbaYgLIhfe0M9MTLLJks8dgKGmotSV6KMsKUFzMjQykYnru5afkIZE1Zqhz4QwCUFCHTBoG3mScDzi9jyROq9nmjVpMu8PVuelfd9oTR+BsHuNGdwSH3K7F1ashQQCZLixErMb4NLR15nu4e9nO4vvigluHr4iID7DWKZlnkkhYpVgORmU0SbZhr1W1GVJkLiE0Zc6l3B+n7YEFUhq4j6tIfr/DsmnheWVeCQiMlJdUGSOxYIRkSzKKSGC6ZBU5xVOlBhZxhcQFArq2dGG4hf391E463RupNqrI4sW1OvamI11x3jT597vp2ooJa+1bBbZqJMUwkywURVYSH1x1hTwSNPAKFZyA+WGgBa60uQGK17amFZy2yg3fpf6BsqteONjWYEg3O7CA+0u0K4FMlOXKSRXIFYEC7m3y4DJo4XVtDGtaCGTHcTBAuf7+XSeu6wH1RWBXT1SEmX5eJasyIAqgBehLmyowb4SdkzYCCUWukhsXWGiTQZJ+fLA0je5rshS5FFWaEiwknZXAqUf628p0A1GVeDqCorkbTRWK4IdJQSLFfdTluBFQkfNcmxdpAAPHGSlyzYn/pZm32HWJ2eAlUkP8nyGYetIVPAnSYv/JUiKdSb64KDoq87J4SX8/ylUJODsh8BIhSUJy/bWCSHaUp2ng/qgTFo55QuWsG1aKS1RIphHqUS8jYg5wpURNEQM7oJY1hwMtvMfRq/qBsskQ3J/xZ12g4GE/tETp3SExiUaB/Y/3V2WDqvjsYk1aL7K6Efo8Ybp1HL4sVOlBkNRgvLAIaythrbiARtwxLLinRzhTrXQIptsB7LQAiU9bMGFFlw4WbfmOvQWWoCgFt+WM9MgUFnlDpcWaOgomHYSTAJ854nwTPEXzjL8ssGqvwex6dICOBEu1svqbcxTPKhMbbYjPhBTyya31hPZgsZWSQQGQnLJ7xpri5mhfkZOG/snuDnuOPygFbMXlyxqy19GLn97fyE0/rU6Y0GbSlJrVAUZP4/5SXuLkt6vKPNTF6IXZ8AFv8JxRsmZgph0aCkHtuMNZdBv3KuyyIYimTz7vhvKoN+4UeXhyH/vG8ogaFOGalPDIrvJ0GFIwkNAu8nsdEfD4puh77ToJgJdfSwSecCX0kjpW/d/GueD1j/XfprcXw3+2S2mW0XEFMpuAd0OJiKmtPPOioql73ACbmpVvMkn0yhsrcUK1ERdWgQMQoMKQ5OJNJ2aZGlQXNvNF+ul3eqf38/My0vvePD1j2SCpYoFN/GocCyIEstVlfTC+dJiQ5AOVWq4TzrsMjUAxQ92TVHaazLcN+P3ZAd0SJLKDuJ9lFB3f0nJAri6s4qAf1oErCR/y7RoRVjYVfxb08Vnq3bRv3maGbzs9V47fbMi5o8jZrIjoigxRS77OvSICf9IDAEz91JfkZKzoECpmoUCEjY5XUqtrYkgU/McQ7UGax3/YXRLROPQqe7CDRTZbCm6O1QNaE5b5pPUk0xQigRZxQbRqe4Y4+ebamPmR3GH1fmyreD2Iil4ZQzUo5QvAUASlACstm85sPifA1wzpfifkj4n6pGQUYZqRnxZqwcPdqY90ZMLZNNrftamtJmU4p4+E/mtoULAOkuWYcoyNWAiTLA1fB5s1qlVfhopaWx23KGRo8Pwx1ThivySZCavvmVRFqQDQQt1RYfS6LD3fo2yfoPS7b9MHcyTgwvJdl2SzU2RwynJwg0+OQxGVfrKjicOpPSV7CLNXfoqvdSlCRMejfvse+NS5c9Prg30MK1qsJuJmCAYacK0LyJKiX5xvmi/uJgoNvLJNcaUVkqkbjj4HFoLJUDeE5QtI7kNcc5SnBxrzAj0m8Dl5TogVFXHwfqOA4AYUQKsp95BNqODt1v1ou9T/LvMbcDb3b4VHck4t5knNYn0xvSdHL60oU8Pse1Epprj0KfKHn/iFLzbfE3p+3Op0LZIO3apv7FeVKZNFb+XS61aevcp/r27VAozuBtcqmqp5qdruJVTzUuW3TlV/NKxfeGt0loM3vDa1nT/jP8B</diagram><diagram name="sf_buffer" id="McOcRQAG4h0-hEUw4Ais">7VxrU+I8FP41fMTpveWjgKD74uJlGVa/OGmbQrU0tU3l8uvfpBfoJWCVoqjr7AztSZqePOeSc07SbYid2aLvA296iUzoNATOXDTEbkMQBE4WyA+lLGMKT0gxZeLbZkLbEG7tFUyIXEINbRMGMS0hYYQcbHt5ooFcFxo4RwO+j+ZB7iUWcvJv9cAElgi3BnDK1LFt4mlCVThu03AO7ck0eXUrbZiBtHNCCKbARPMMSTxriB0fIRxfzRYd6FD0UlwumvLD8NfloHnvCHfnw8GNN9eb8WC9tzyynoIPXfzuoaf3amD5rnW7uuv/etT/DICP06FfgBMmeCVzxcsUQB+FrgnpIFxDbM+nNoa3HjBo65zoDLine truncated
|
||||
<mxfile host="Electron" modified="2021-04-15T13:20:22.394Z" agent="5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) draw.io/12.9.9 Chrome/80.0.3987.163 Electron/8.2.1 Safari/537.36" etag="nM78PTzm0mEIcblxpjCY" version="12.9.9" type="device" pages="8"><diagram name="overview" id="muXDs5UsbG9aPfefTxRP">7V3ZdqJKFP2aPOqCYn7MoEn6ZlSTdOclCwWVSMAARs3X30JBGY5YYqHGQK/V0QJR99lnrFPlCXf+Mbl01GH/1tZ08wQx2uSEuzhBCDECwn/8kel8hMVD85GeY2jB2HKgaXzrwSATjI4MTXfnY8GQZ9umZwzjgx3bsvSOFxtTHcceu7E36dpm/F2Hak9PDTQ7qpkefTE0rx+MigyzPHGlG71+8NZKeOJDDS8OBty+qtnjyBBXO+HOHdv25o8+Jue66aMX4nLeubSs2+mrrPbtvy+t+z/8sFeZ36y+yUsWX8HRLS/3rS8YS7rjBl/OXe3sunc7aFjfjeAlzJdqjgK8gu/qTUMAHXtkabp/E+aEOxv3DU9vDtWOf3aMOYPH+t6HiZ+x+KHrOfZAP7dN25m9mjub/VucCUXgX9s1TDNyZX12+OO25QU84sTgeeQ6ZnbgcdU0ehYeM/UuRuWMEKQAzC/d8fRJhCIBaJe6/aF7zhRfEpwVpQCkaZzY4yWbkBCM9SNEkkMiqQGDe4tbL6WEHwSCgoX2rzE8N78umI5we6k2H/5777gWdaFFBWHZlp6SYzBIHeExpJB9SBm3wdB8kx75yeMz07psf79wd59/e/UKK60HsYdRHG75pReGTW2Ht2VgMMJXsXG6yXyabywGJg2WIBSFlZAbq2zstwarguJgVVhUFVJwgdQqDC0xBdapOVanLh5zRpZlWD0fI8fu6K6LHWMSSIyIl2VTA10EdDa0hx2Mvo7Hz3x8DewJT4MTH4am+W8D2oO4xYjYYJ4BVT+bKcTWVZZCga02r8JSqlEJ8kXZhvymYTNQyOkeAUMCkCiMy2g9EjgqGvoPO6P2KmpFuNyes+ymvRhQO4PejHv3I880fCLPxjXVGdzj2xiezwumiqPO2CCajbJxqqKZugShAzMjtad6hu2TX/Gfa4aD4Z4PWLbjY5oMQzRVl7sdKJQRO7Le7pIrwyort9r0J3WhAlkzHjBmUlEE4EoC7JUAy5EdMKD9PDh/916G9Yu7plaxXms3zU+SYHM/vl/iiXw/BJa4uJJ+rJR2/6R4ZcJPxXfsHAw2BcbDPPDxARmqYwsHHIhp43cS1Q/fUlhtdzhTMKbYoaMKusQVHCHOMaCYCwFUKSziYgkCjY2szCpIDj/kYn+jyxV0WeMhlyujNieKG+gCWiH4nxR0sXxJgf1SYKdh15/nl+H1bePp60uoPYnPLbbW/6wgAouoaz29GTz1UbV7tqWateVowr8sr7mx7WEg8Xfd86aBINWRZydKg5FabA6X5Nojp6NnXSjPL/S/SqaoHN3ElPqKXgSjHrz0wTbwJ1yImOdQTMIcnwx6PNXp6V7wsoTwFp9jC52WAXmKphdgHBOs+DmywxOVuV6d4gs4NJwsT+JHPf9v21cXJ7wX/mzz281PEgQ6BQUwCKjwd2dHai6Ag7R+UeUvLAgS2DglFEDlWUjnBQpR0PjsXpW+tLHivLlqT/z4U5drULhMGgUVW5Pm4uEiL6XjRRaJAFSsVBRWSm6osqGnEjDuGgyCeGETMJQfgsX19LT1iprSB6dfDDxr9K08PNLGgt8ai3AaUUgrzUKxYtAgCtC8KmfuiHvRvqRK6/1NFqXBGyKyL0cXV+ap5mUSi3xqI1Gf4tIEkAD5L0hBX/4EMWUp/8LkL+9d/lBpgUYM6nbfMLy6+vGbwlA6JEkUsQEnwcoASWjEoM7Nbf3SMgfak603R43rx++HaxL3GZqIrqlPTv0OLQyPbmnBw4uOqbqu0YHKqVGoSbt8VnUJhQixYlWSeFbhBF7mZIlRxPnHCe4tVBVJwbRFgsyIvMAGZ8NPgvgqih0bkiArZlib00bLrRky3jL1FRM9EnKCOvMcPZX4pu4jyNn3oZdAg8Qk6EEpifmjiMkxdIi5SDAKJ+bT6On1/rtuqb0r9J/6UHM454qkR+K3t1AqSlxCO22hBIVWWCTk2Y7fkIwYU51uU5Y7zPnHYuggAS2OIB9oFA1AOuQvGhTcDMqtn6kF6ytUNKcmiH9NQx90u9h6vnjv3ZueXEnXEC4f6s0fy+hMOpDPIMWlxElclYseaaGBk0l8VYweUkEiLMZjrYihkAhkYHJH74BJeVsWeKEQSe1PY9K+Zq4xTHvU7fo+olSdpeogSTpk1dkgPdZUT/XDAYIyWqrmlVKootAXEoYrXO4VX5mRxpuV2biY+O0BB2tWiHiiJ4AxGCZdM1XUHNlOQ1wQufADHWdikk2WzdwBGGoqSlWJHkVJCZqToZGJjFzftRxDGhJVa4Y+E8IkBAlVwKCt5UnC8Yjb80RqvV9o1qjOfD5ZrbfmY60xfgbC7hVmcEt8yO1eWLEWEgiQ4cZKzG6AS0dfF7qHv5/tzL8rJrh5+IqA+AxjmZZ5JoWIVYLlZFBG62Qb9lpRlyVB4hJGX+pMwpv7tAWoQFIT92k10f93SD4tLK/EIxGRkaqCInMsFoyIZFFOCRFMh6Qqr3CixMgyvoGgUFDPltYXv7m/z8JFq3UnVd4dWbSgXtfaZKg7xoc+837HqqGUvNaiWWStTlIIM8FGVWAh9dVFXcAjdQOjWMoNlBsCWugKkxuseGljWsptrdz4XeobKLf8jY9FBYJwuwsPtLtAuxbITFWmkFyBWBEs5N4uAyaPFpbTxrSihUx2EAcLnO/n03nuoh5UVQR2eaQkyvLxLFmRAVUAb0Jd2FCDfSnsmLARSix0kdiqwkSbDJLy5YGlb3JVkaXIUVRoSLCSdlcCpR/rbynQNUZV4KoKiuRtNFYrgh0lBIsV91OW4EVCR81ybFWkAA8cZKXLNmf+lmY/YdZnwwArkx7k+QzDVpGo4HeS5v9LkBSrTPTgoOirysnhLfz/KVQk4OyHwEiFJQnL9lYJIdpSvUkH9UGZtGLKFyxh27RSWKJEMI9SingbEXOEKyNoiBjcBbGoORhs57+MTtkNlkmG5P6KO+0GAwl91BOndITGJRoH9j/dXZQOq8OhiTVotsroKPR4zXRqMfzYqVKDoShBeeAQ1lZDW/GADThiUfHOBuFOudAim2wHstACJT1szoUWXDhZt+I+9BZagKDm35Yz0yBQWeUOlxZo6CiYdhJMAvzkifBM8efOMvyywbK/B7Hp0gI4ES5Wi+pt3KR4UJrabEd8IKaWTW6tJ7I5ja2SCAyE5JLfFdYWM0OdRi4b+he4G3zi8I2WzJ7fMq8tfxu4/P3jlVD712gNBW0sSY1BGWQcH/OT9hYlvV9e5qduRC/OgAt+ueOMgjMFMenQUg5sxxvKoN+4V2WeDUUyefZzN5RBv3GjysOR/943lEHQpgzlpoZ5dpOhw5CEh4B2k9npjob5N0PfadFNBLr6WCTygC+lkdI3Hv/ULnuNf679Mnq86f2zG0y7jIgplN0Cuh1MRExp550lFQvf4QTc1Cp/k0+mUdhaixWoibqwCBiEBuWGJhNpOjXJwqC4tetv1luz0b18nJjX195p7/sfyQRLGQuu41HuWBAllqsq6YXzhcWGIB3K1HCfdNhlagCKH+yaorTXZLhvxu/JDuiQJJUdxPsooe7+gpIFcHVnGQEfWwSsJH/LNG9FWNhV/FvRxVerctW9e5kYvOx13ltdsyTm0REz2RGRl5gil30fesSEfySGgJl7qa9IyVlQoFTNQgEJm5wupdbWRJCpeY6hWr2Vjv8wuiWicehYd+EGimy25N0dqgI0py3ySepJJihFgqxijehUd4jx8021MfGjuMPqfNlWcHuRFLwyBupR2iwBQBKUACy3bzmw+J8DXDOl+J+SPifqkZBRhmpGfFGrBw92pj3Rkwtk0yt+1qawmZT8nj4T+a2hQsA6S5ZhijI1YCJMsDX8JtisUqvNaaSksdlxh8YGHYZHU4XL80uSmbz6kUVZkA4ELdQlHQqjw977NYr6DUq3+zZ2ME8OLiTbdUl2Y4ocTkkWbvDZwGCUpa/seOJASl/JLtKNS1+Fl7o0YcSjYZf9rF2r/OXZrYGexmUN9ugaxqVEvzift19cTBQb+eQaY0orJVIfOHgfWgslQN4TlC0Pohkd+k3g4nIdEKqy44Cs4yCTZz+iGR38BmUv+j7Fv8vcBvwG27eiIxnnNrOkJpHemL6Tw7c29PEhtp3IheU4dKiyx584Bb/AwTalJ9d37XSBFwjVb6wXUbOpq/qcf45LLVt69yn+vbtUCjO4a1yqaqnm1DXcX+9UNybL7pwqfurYvvCWaS0Gr39ra7p/xf8=</diagram><diagram name="sf_buffer" id="McOcRQAG4h0-hEUw4Ais">7Vxrd6I8EP41frSHO/ixarXd1669rMdtv/RwCUqLhEKol1//JhBUIFqqWG27PXuOMAlhMvPMZGYStia2JrNuoPvja2gBtyZw1qwmtmuCIHCygH8IZZ5QeExKKKPAsShtRbh3FoASOUqNHAuECY2SEIQucvws0YSeB0yUoelBAKdh5iU2dLNv9fURKBDuTd0tUoeOhcaUqnDcquESOKMxfXUjbZjoaWdKCMe6BadrJPGiJrYCCFFyNZm1gEukl8rlqi4/9X9d9+qPrvBw2e/d+VOjngzW+cgjyykEwEM7Dz1+VEM78Oz7xUP317Pxp6cHKB36TXcjKi86VzRPBRjAyLMAGYSric3p2EHg3tdN0jrFmMG0MZq4+I7Hl7bjui3owLine truncated
|
||||
+133
-7
@@ -1,11 +1,124 @@
|
||||
# jf-live-writer
|
||||
|
||||
The jf-live-writer is packaged as a Docker container for development and
|
||||
testing.
|
||||
This component is a PHDF5 based MPI writer for high performance detectors
|
||||
that need more than 2GB/s of write speed. It parallelizes the HDF5 writing to
|
||||
multiple processes in order to overcome the cca. 3GB/s single stream
|
||||
write limit on GPFS.
|
||||
|
||||
# Using the docker container
|
||||
It expects an input ZMQ stream that contains metadata about what and where to
|
||||
write and access to the RamBuffer of the images one wants to write.
|
||||
|
||||
This writer is stateless but tied to a specific detector. There are no
|
||||
configuration states, all the information for writing files is expected to
|
||||
come from the input metadata stream. The images data is directly taken from the
|
||||
existing buffer without additional memory copies.
|
||||
|
||||
## Overview
|
||||
|
||||
|
||||
The ZMQ store stream is a PUB/SUB stream that gets distributed to all
|
||||
ranks. This stream caries both ImageMetadata information and file writing
|
||||
metadata and is generated and sent by the writer agent.
|
||||
|
||||
For writing image data, each process decides based on its rank if the
|
||||
particular store stream message is for him and writes the requested image.
|
||||
For writing image metadata, the rank 0 is always responsible. The metadata is
|
||||
always written only by the process with **RANK = 0**.
|
||||
|
||||
When to start writing a new file or when to close an existing one is also
|
||||
decided based on the metadata in the store stream. There is no state machine
|
||||
in the writer, but which action to take is based solely on the received
|
||||
metadata. This saves us the need to have inter-rank communication and makes
|
||||
for a more simple writer.
|
||||
|
||||
### ZMQ Store Stream format
|
||||
This stream is composed by 2 parts. The first part is the already known
|
||||
**ImageMetadata** the **jf-assembler** provides, and the second part is
|
||||
provided by the **writer-agent**.
|
||||
|
||||
Each message in the stream has this format:
|
||||
|
||||
```c++
|
||||
#pragma pack(push)
|
||||
#pragma pack(1)
|
||||
struct StoreStream {
|
||||
ImageMetadata image_metadata;
|
||||
|
||||
int64_t run_id;
|
||||
uint32_t i_image;
|
||||
uint32_t n_images;
|
||||
uint32_t image_y_size;
|
||||
uint32_t image_x_size;
|
||||
uint32_t op_code;
|
||||
uint32_t bits_per_pixel;
|
||||
};
|
||||
#pragma pack(pop)
|
||||
|
||||
#pragma pack(push)
|
||||
#pragma pack(1)
|
||||
struct ImageMetadata {
|
||||
uint64_t pulse_id;
|
||||
uint64_t frame_index;
|
||||
uint32_t daq_rec;
|
||||
uint32_t is_good_image;
|
||||
};
|
||||
#pragma pack(pop)
|
||||
```
|
||||
|
||||
#### StoreStream
|
||||
|
||||
| Name | Type | Comment |
|
||||
| --- | --- | --- |
|
||||
| run_id | int64 | Run id used to construct the output file name. |
|
||||
| i_image | uint32_t | Current image index inside this run. |
|
||||
| n_images | uint32_t | Total number of images in this run. |
|
||||
| image_y_size | uint32_t | Y image size in pixels. |
|
||||
| image_x_size | uint32_t | X image size in pixels. |
|
||||
| op_code | uint32_t | State transition information for the writer. |
|
||||
| bits_per_pixel | uint32_t | How many bits does 1 pixel have. 8, 16 or 32. |
|
||||
|
||||
Some details regarding how this fields are used:
|
||||
|
||||
- **run\_id**: Currently the output file name is simply **[run\_id].h5**.
|
||||
- **i\_images**: Based on this each rank decides if the received message is for
|
||||
itself and needs to write the corresponding image data to file.
|
||||
- **op_code**: This is used to steer the file writing and to avoid the need
|
||||
for a state machine:
|
||||
- op_code = 0 (Continue - just write to the same file as you already are)
|
||||
- op_code = 1 (Start - create a new file for this run_id)
|
||||
- op_code = 2 (Stop - close the current file)
|
||||
|
||||
Since the writer is relying on the correct sequence of messages in the input
|
||||
stream instead of having an internal state machine,
|
||||
the input stream must always follow a valid pattern of messages:
|
||||
|
||||
|
||||
|
||||
The sequence must always follow:
|
||||
|
||||
- op_code = 1 (also a new run_id if you do not want to overwrite the previous file)
|
||||
- op_code = 0 (same run_id as the last message)
|
||||
- op_code = 2 (same run_id as the last message)
|
||||
- etc.
|
||||
|
||||
In case the sequence is broken (wrong send order from the writer agent or lost
|
||||
messages, etc.) the writer will ignore the received message. An operational
|
||||
state can be restored by sending a **op\_code = 2** message.
|
||||
|
||||
Images are written only when op_code = 0, meaning that the start and stop
|
||||
message are not used for writing data. This allows to send the stop message
|
||||
in case we need to reset the writer status at any point.
|
||||
|
||||
#### ImageMetadata
|
||||
This comes from jf_assembler without modifications for a particular
|
||||
image.
|
||||
|
||||
## Build
|
||||
|
||||
### Build inside docker
|
||||
The easiest way to build and test the jf-live-writer is to use the
|
||||
provided docker container. You need to start it from the project **root**:
|
||||
provided docker container. You need to start building it
|
||||
from the project **root**:
|
||||
|
||||
```bash
|
||||
docker build -f jf-live-writer/debug.Dockerfile -t jf-live-writer .
|
||||
@@ -13,9 +126,22 @@ docker build -f jf-live-writer/debug.Dockerfile -t jf-live-writer .
|
||||
(Running this command from the project root is mandatory as the entire project
|
||||
folder needs to be part of the build context.)
|
||||
|
||||
# Build on your local machine
|
||||
This will copy your current working directory to the image and build the
|
||||
jf-live-writer. Once you've dont this, you can start
|
||||
|
||||
## Building
|
||||
### Build on your machine
|
||||
|
||||
In addition to the libraries needed for sf_daq, you need **mpich** installed:
|
||||
|
||||
```bash
|
||||
yum install mpich-devel
|
||||
ln -v -s /usr/include/mpich-x86_64/* /usr/include/
|
||||
```
|
||||
|
||||
Making the soft links for mpich headers to your /usr/include is
|
||||
necessary due to HDF5.
|
||||
|
||||
#### Building with cmake
|
||||
In order to build this executable you need to specify the cmake variable
|
||||
```
|
||||
cmake3 -DBUILD_JF_LIVE_WRITER=ON
|
||||
@@ -23,7 +149,7 @@ cmake3 -DBUILD_JF_LIVE_WRITER=ON
|
||||
The project will not build if you do not have installed the PHDF5 library.
|
||||
Please follow instructions below on how to do that manually.
|
||||
|
||||
## Install PHDF5
|
||||
#### Install PHDF5
|
||||
```
|
||||
wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.12/hdf5-1.12.0/src/hdf5-1.12.0.tar.gz
|
||||
tar -xzf hdf5-1.12.0.tar.gz
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#ifndef SFWRITER_HPP
|
||||
#define SFWRITER_HPP
|
||||
#ifndef JF_LIVE_WRITER_HPP
|
||||
#define JF_LIVE_WRITER_HPP
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
@@ -32,28 +32,28 @@ class JFH5Writer {
|
||||
hid_t daq_rec_dataset_id_ = -1;
|
||||
hid_t is_good_dataset_id_ = -1;
|
||||
|
||||
hid_t get_datatype(const int bits_per_pixel);
|
||||
void open_file(const std::string& output_file, const uint32_t n_images);
|
||||
static hid_t get_datatype(int bits_per_pixel);
|
||||
void open_file(const std::string& output_file, uint32_t n_images);
|
||||
void close_file();
|
||||
|
||||
public:
|
||||
JFH5Writer(const BufferUtils::DetectorConfig config);
|
||||
explicit JFH5Writer(const BufferUtils::DetectorConfig& config);
|
||||
~JFH5Writer();
|
||||
|
||||
void open_run(const int64_t run_id,
|
||||
const uint32_t n_images,
|
||||
const uint32_t image_y_size,
|
||||
const uint32_t image_x_size,
|
||||
const uint32_t bits_per_pixel);
|
||||
void open_run(int64_t run_id,
|
||||
uint32_t n_images,
|
||||
uint32_t image_y_size,
|
||||
uint32_t image_x_size,
|
||||
uint32_t bits_per_pixel);
|
||||
void close_run();
|
||||
|
||||
void write_data(const int64_t run_id,
|
||||
const uint32_t index,
|
||||
void write_data(int64_t run_id,
|
||||
uint32_t index,
|
||||
const char* data);
|
||||
|
||||
void write_meta(const int64_t run_id,
|
||||
const uint32_t index,
|
||||
void write_meta(int64_t run_id,
|
||||
uint32_t index,
|
||||
const ImageMetadata& meta);
|
||||
};
|
||||
|
||||
#endif //SFWRITER_HPP
|
||||
#endif //JF_LIVE_WRITER_HPP
|
||||
@@ -9,25 +9,23 @@
|
||||
|
||||
class WriterStats {
|
||||
const std::string detector_name_;
|
||||
const size_t stats_modulo_;
|
||||
|
||||
uint32_t image_n_bytes_;
|
||||
uint32_t image_n_bytes_{};
|
||||
|
||||
int image_counter_;
|
||||
uint64_t total_bytes_;
|
||||
int image_counter_{};
|
||||
uint64_t total_bytes_{};
|
||||
|
||||
uint32_t total_buffer_write_us_;
|
||||
uint32_t max_buffer_write_us_;
|
||||
uint32_t total_buffer_write_us_{};
|
||||
uint32_t max_buffer_write_us_{};
|
||||
std::chrono::time_point<std::chrono::steady_clock> stats_interval_start_;
|
||||
|
||||
void reset_counters();
|
||||
void print_stats();
|
||||
|
||||
public:
|
||||
WriterStats(
|
||||
const std::string &detector_name,
|
||||
const size_t stats_modulo);
|
||||
void setup_run(const StoreStream& meta);
|
||||
explicit WriterStats(std::string detector_name);
|
||||
void start_run(const StoreStream& meta);
|
||||
void end_run();
|
||||
void start_image_write();
|
||||
void end_image_write();
|
||||
};
|
||||
|
||||
@@ -3,21 +3,18 @@
|
||||
|
||||
#include "formats.hpp"
|
||||
|
||||
const static uint8_t OP_START = 1;
|
||||
const static uint8_t OP_END = 2;
|
||||
|
||||
#pragma pack(push)
|
||||
#pragma pack(1)
|
||||
struct StoreStream {
|
||||
ImageMetadata image_metadata;
|
||||
|
||||
int64_t run_id;
|
||||
uint32_t i_image;
|
||||
uint32_t n_images;
|
||||
uint32_t image_y_size;
|
||||
uint32_t image_x_size;
|
||||
uint32_t op_code;
|
||||
uint32_t bits_per_pixel;
|
||||
|
||||
ImageMetadata image_metadata;
|
||||
};
|
||||
#pragma pack(pop)
|
||||
#endif //SF_DAQ_BUFFER_BROKER_FORMAT_HPP
|
||||
@@ -1,11 +1,7 @@
|
||||
#include "JFH5Writer.hpp"
|
||||
|
||||
#include <sstream>
|
||||
#include <cstring>
|
||||
#include <H5version.h>
|
||||
#include <iostream>
|
||||
|
||||
|
||||
#include "JFH5Writer.hpp"
|
||||
#include "live_writer_config.hpp"
|
||||
#include "buffer_config.hpp"
|
||||
#include "formats.hpp"
|
||||
@@ -19,7 +15,7 @@ using namespace std;
|
||||
using namespace buffer_config;
|
||||
using namespace live_writer_config;
|
||||
|
||||
JFH5Writer::JFH5Writer(const BufferUtils::DetectorConfig config):
|
||||
JFH5Writer::JFH5Writer(const BufferUtils::DetectorConfig& config):
|
||||
root_folder_(config.buffer_folder),
|
||||
detector_name_(config.detector_name)
|
||||
{
|
||||
@@ -140,14 +136,14 @@ void JFH5Writer::open_file(const string& output_file, const uint32_t n_images)
|
||||
}
|
||||
|
||||
hsize_t image_dataset_dims[] = {n_images, image_y_size_, image_x_size_};
|
||||
auto image_space_id = H5Screate_simple(3, image_dataset_dims, NULL);
|
||||
auto image_space_id = H5Screate_simple(3, image_dataset_dims, nullptr);
|
||||
if (image_space_id < 0) {
|
||||
throw runtime_error("Cannot create image dataset space.");
|
||||
}
|
||||
|
||||
// TODO: Enable compression.
|
||||
// bshuf_register_h5filter();
|
||||
// uint filter_prop[] = {PIXEL_N_BYTES, BSHUF_H5_COMPRESS_LZ4};
|
||||
// uint filter_prop[] = {0, BSHUF_H5_COMPRESS_LZ4};
|
||||
// if (H5Pset_filter(dcpl_id, BSHUF_H5FILTER, H5Z_FLAG_MANDATORY,
|
||||
// 2, filter_prop) < 0) {
|
||||
// throw runtime_error("Cannot set compression filter on dataset.");
|
||||
@@ -162,12 +158,12 @@ void JFH5Writer::open_file(const string& output_file, const uint32_t n_images)
|
||||
|
||||
// Create metadata datasets.
|
||||
hsize_t meta_dataset_dims[] = {n_images};
|
||||
auto meta_space_id = H5Screate_simple(1, meta_dataset_dims, NULL);
|
||||
auto meta_space_id = H5Screate_simple(1, meta_dataset_dims, nullptr);
|
||||
if (meta_space_id < 0) {
|
||||
throw runtime_error("Cannot create meta dataset space.");
|
||||
}
|
||||
|
||||
auto create_meta_dataset = [&](string name, hid_t data_type) {
|
||||
auto create_meta_dataset = [&](const string& name, hid_t data_type) {
|
||||
auto dataset_id = H5Dcreate(
|
||||
data_group_id, name.c_str(), data_type, meta_space_id,
|
||||
H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
|
||||
|
||||
@@ -1,14 +1,12 @@
|
||||
#include <iostream>
|
||||
#include <utility>
|
||||
#include "WriterStats.hpp"
|
||||
|
||||
using namespace std;
|
||||
using namespace chrono;
|
||||
|
||||
WriterStats::WriterStats(
|
||||
const string& detector_name,
|
||||
const size_t stats_modulo) :
|
||||
detector_name_(detector_name),
|
||||
stats_modulo_(stats_modulo)
|
||||
WriterStats::WriterStats(string detector_name) :
|
||||
detector_name_(std::move(detector_name))
|
||||
{
|
||||
reset_counters();
|
||||
}
|
||||
@@ -26,13 +24,6 @@ void WriterStats::start_image_write()
|
||||
stats_interval_start_ = steady_clock::now();
|
||||
}
|
||||
|
||||
void WriterStats::setup_run(const StoreStream& meta)
|
||||
{
|
||||
image_n_bytes_ = (meta.image_y_size *
|
||||
meta.image_x_size *
|
||||
meta.bits_per_pixel) / 8;
|
||||
}
|
||||
|
||||
void WriterStats::end_image_write()
|
||||
{
|
||||
image_counter_++;
|
||||
@@ -43,11 +34,19 @@ void WriterStats::end_image_write()
|
||||
|
||||
total_buffer_write_us_ += write_us_duration;
|
||||
max_buffer_write_us_ = max(max_buffer_write_us_, write_us_duration);
|
||||
}
|
||||
|
||||
if (image_counter_ == stats_modulo_) {
|
||||
print_stats();
|
||||
reset_counters();
|
||||
}
|
||||
void WriterStats::start_run(const StoreStream& meta)
|
||||
{
|
||||
image_n_bytes_ = (meta.image_y_size *
|
||||
meta.image_x_size *
|
||||
meta.bits_per_pixel) / 8;
|
||||
}
|
||||
|
||||
void WriterStats::end_run()
|
||||
{
|
||||
print_stats();
|
||||
reset_counters();
|
||||
}
|
||||
|
||||
void WriterStats::print_stats()
|
||||
|
||||
+17
-18
@@ -1,14 +1,14 @@
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
#include <zmq.h>
|
||||
#include <RamBuffer.hpp>
|
||||
#include <BufferUtils.hpp>
|
||||
#include <mpi.h>
|
||||
|
||||
#include "RamBuffer.hpp"
|
||||
#include "BufferUtils.hpp"
|
||||
#include "live_writer_config.hpp"
|
||||
#include "WriterStats.hpp"
|
||||
#include "broker_format.hpp"
|
||||
#include <mpi.h>
|
||||
#include <JFH5Writer.hpp>
|
||||
|
||||
#include "JFH5Writer.hpp"
|
||||
|
||||
using namespace std;
|
||||
using namespace buffer_config;
|
||||
@@ -27,7 +27,7 @@ int main (int argc, char *argv[])
|
||||
|
||||
auto const config = BufferUtils::read_json_config(string(argv[1]));
|
||||
|
||||
MPI_Init(NULL, NULL);
|
||||
MPI_Init(nullptr, nullptr);
|
||||
|
||||
int n_writers;
|
||||
MPI_Comm_size(MPI_COMM_WORLD, &n_writers);
|
||||
@@ -43,27 +43,21 @@ int main (int argc, char *argv[])
|
||||
RamBuffer ram_buffer(config.detector_name, config.n_modules);
|
||||
|
||||
JFH5Writer writer(config);
|
||||
WriterStats stats(config.detector_name, STATS_MODULO);
|
||||
WriterStats stats(config.detector_name);
|
||||
|
||||
StoreStream meta = {};
|
||||
while (true) {
|
||||
zmq_recv(receiver, &meta, sizeof(meta), 0);
|
||||
|
||||
if (meta.op_code == OP_START) {
|
||||
// i_image == 0 -> we have a new run.
|
||||
if (meta.i_image == 0) {
|
||||
writer.open_run(meta.run_id,
|
||||
meta.n_images,
|
||||
meta.image_y_size,
|
||||
meta.image_x_size,
|
||||
meta.bits_per_pixel);
|
||||
|
||||
stats.setup_run(meta);
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if (meta.op_code == OP_END) {
|
||||
writer.close_run();
|
||||
continue;
|
||||
stats.start_run(meta);
|
||||
}
|
||||
|
||||
// Fair distribution of images among writers.
|
||||
@@ -79,7 +73,12 @@ int main (int argc, char *argv[])
|
||||
if (i_writer == 0) {
|
||||
writer.write_meta(meta.run_id, meta.i_image, meta.image_metadata);
|
||||
}
|
||||
}
|
||||
|
||||
MPI_Finalize();
|
||||
// i_image + 1 == meta.n_images -> we received the last image.
|
||||
if (meta.i_image+1 == meta.n_images) {
|
||||
writer.close_run();
|
||||
|
||||
stats.end_run();
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user