From 456a68a32efcf112f21d10dee80f72e2445454dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 23 Jun 2022 16:25:00 +0200 Subject: [PATCH 1/3] Use span-based API where applicable --- Source/Diagnostics/WarpXOpenPMD.cpp | 67 +++++++++++------------------ 1 file changed, 26 insertions(+), 41 deletions(-) diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index 6e73ab55d5c..6ce68ad1d67 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -756,14 +756,10 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, auto const positionComponents = detail::getParticlePositionComponentLabels(); #if defined(WARPX_DIM_RZ) { - std::shared_ptr z( - new amrex::ParticleReal[numParticleOnTile], - [](amrex::ParticleReal const *p) { delete[] p; } - ); + auto z = currSpecies["position"]["z"].storeChunk({offset}, {numParticleOnTile64}); + auto zBuffer = z.currentBuffer(); for (auto i = 0; i < numParticleOnTile; i++) - z.get()[i] = aos[i].pos(1); // {0: "r", 1: "z"} - std::string const positionComponent = "z"; - currSpecies["position"]["z"].storeChunk(z, {offset}, {numParticleOnTile64}); + zBuffer[i] = aos[i].pos(1); // {0: "r", 1: "z"} } // reconstruct x and y from polar coordinates r, theta @@ -773,47 +769,39 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, WARPX_ALWAYS_ASSERT_WITH_MESSAGE(int(soa.GetRealData(PIdx::theta).size()) == numParticleOnTile, "openPMD: theta and tile size do not match"); { - std::shared_ptr< amrex::ParticleReal > x( - new amrex::ParticleReal[numParticleOnTile], - [](amrex::ParticleReal const *p){ delete[] p; } - ); - std::shared_ptr< amrex::ParticleReal > y( - new amrex::ParticleReal[numParticleOnTile], - [](amrex::ParticleReal const *p){ delete[] p; } - ); + auto x = currSpecies["position"]["x"].storeChunk({offset}, {numParticleOnTile64}); + auto y = currSpecies["position"]["y"].storeChunk({offset}, {numParticleOnTile64}); + auto xBuffer = x.currentBuffer(); + auto yBuffer = y.currentBuffer(); for (auto i=0; i curr( - new amrex::ParticleReal[numParticleOnTile], - [](amrex::ParticleReal const *p) { delete[] p; } - ); - for (auto i = 0; i < numParticleOnTile; i++) { - curr.get()[i] = aos[i].pos(currDim); - } + // std::shared_ptr curr( + // new amrex::ParticleReal[numParticleOnTile], + // [](amrex::ParticleReal const *p) { delete[] p; } + // ); std::string const positionComponent = positionComponents[currDim]; - currSpecies["position"][positionComponent].storeChunk(curr, {offset}, + auto curr = currSpecies["position"][positionComponent].storeChunk({offset}, {numParticleOnTile64}); + auto currBuffer = curr.currentBuffer(); + for (auto i = 0; i < numParticleOnTile; i++) { + currBuffer[i] = aos[i].pos(currDim); + } } #endif // save particle ID after converting it to a globally unique ID - std::shared_ptr ids( - new uint64_t[numParticleOnTile], - [](uint64_t const *p) { delete[] p; } - ); + auto const scalar = openPMD::RecordComponent::SCALAR; + auto ids = currSpecies["id"][scalar].storeChunk({offset}, {numParticleOnTile64}); + auto idsBuffer = ids.currentBuffer(); for (auto i = 0; i < numParticleOnTile; i++) { - ids.get()[i] = ablastr::particles::localIDtoGlobal(aos[i].id(), aos[i].cpu()); + idsBuffer[i] = ablastr::particles::localIDtoGlobal(aos[i].id(), aos[i].cpu()); } - auto const scalar = openPMD::RecordComponent::SCALAR; - currSpecies["id"][scalar].storeChunk(ids, {offset}, {numParticleOnTile64}); } // save "extra" particle properties in AoS and SoA @@ -932,16 +920,13 @@ WarpXOpenPMDPlot::SaveRealProperty (ParticleIter& pti, auto currRecord = currSpecies[record_name]; auto currRecordComp = currRecord[component_name]; - std::shared_ptr< amrex::ParticleReal > d( - new amrex::ParticleReal[numParticleOnTile], - [](amrex::ParticleReal const *p){ delete[] p; } - ); + auto d = currRecordComp.storeChunk( + {offset}, {numParticleOnTile64}); + auto dBuffer = d.currentBuffer(); for( auto kk=0; kk Date: Fri, 24 Jun 2022 10:15:55 +0200 Subject: [PATCH 2/3] Avoid flushing before Iteration::close() --- Source/Diagnostics/WarpXOpenPMD.cpp | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index 6ce68ad1d67..43c717af492 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -732,6 +732,7 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, } // open files from all processors, in case some will not contribute below + // @todo move to SetStep m_Series->flush(); // dump individual particles @@ -814,7 +815,15 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, offset += numParticleOnTile64; } } - m_Series->flush(); + /* + * Deliberately not flushing here. + * The ADIOS2 BP5 engine can read data directly from user-specified pointers + * if there happens no adios2::Engine::PerformPuts() (invoked by Series::flush()) + * before adios2::Engine::EndStep() (invoked by Iteration::close()). + * So, in order to avoid unneeded data copies, don't flush anymore until + * closing this iteration. + */ + // m_Series->flush(); } void @@ -1414,8 +1423,15 @@ WarpXOpenPMDPlot::WriteOpenPMDFieldsAll ( //const std::string& filename, #ifdef AMREX_USE_GPU amrex::Gpu::streamSynchronize(); #endif - // Flush data to disk after looping over all components - m_Series->flush(); + /* + * Deliberately not flushing here. + * The ADIOS2 BP5 engine can read data directly from user-specified pointers + * if there happens no adios2::Engine::PerformPuts() (invoked by Series::flush()) + * before adios2::Engine::EndStep() (invoked by Iteration::close()). + * So, in order to avoid unneeded data copies, don't flush anymore until + * closing this iteration. + */ + // m_Series->flush(); } // levels loop (i) } #endif // WARPX_USE_OPENPMD From a5de601324afc137ef5820b6094f84a8ed1fdb56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 27 Jun 2022 12:49:36 +0200 Subject: [PATCH 3/3] Revert "Avoid flushing before Iteration::close()" This reverts commit b0f93b42d4be82a2072553df3dd29a7c0df1a17b. --- Source/Diagnostics/WarpXOpenPMD.cpp | 22 +++------------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index 43c717af492..6ce68ad1d67 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -732,7 +732,6 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, } // open files from all processors, in case some will not contribute below - // @todo move to SetStep m_Series->flush(); // dump individual particles @@ -815,15 +814,7 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, offset += numParticleOnTile64; } } - /* - * Deliberately not flushing here. - * The ADIOS2 BP5 engine can read data directly from user-specified pointers - * if there happens no adios2::Engine::PerformPuts() (invoked by Series::flush()) - * before adios2::Engine::EndStep() (invoked by Iteration::close()). - * So, in order to avoid unneeded data copies, don't flush anymore until - * closing this iteration. - */ - // m_Series->flush(); + m_Series->flush(); } void @@ -1423,15 +1414,8 @@ WarpXOpenPMDPlot::WriteOpenPMDFieldsAll ( //const std::string& filename, #ifdef AMREX_USE_GPU amrex::Gpu::streamSynchronize(); #endif - /* - * Deliberately not flushing here. - * The ADIOS2 BP5 engine can read data directly from user-specified pointers - * if there happens no adios2::Engine::PerformPuts() (invoked by Series::flush()) - * before adios2::Engine::EndStep() (invoked by Iteration::close()). - * So, in order to avoid unneeded data copies, don't flush anymore until - * closing this iteration. - */ - // m_Series->flush(); + // Flush data to disk after looping over all components + m_Series->flush(); } // levels loop (i) } #endif // WARPX_USE_OPENPMD