From 0f1c3242e50cc1e679c6fb036680d724ba2d3c51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 8 Mar 2024 17:10:28 +0100 Subject: [PATCH 01/16] Use more reliable logic for specifying overridable dataset config Copied from HDF5 backend --- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 10 ++- src/IO/ADIOS/ADIOS2IOHandler.cpp | 74 +++++++++++++++----- 2 files changed, 66 insertions(+), 18 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index 6c3d499779..74008d41c5 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -305,9 +305,12 @@ class ADIOS2IOHandlerImpl adios2::Params params; }; - std::vector defaultOperators; + // read operators can (currently) not be specified per dataset, so parse + // them once and then buffer them + std::vector readOperators; json::TracingJSON m_config; + std::optional m_buffered_dataset_config; static json::TracingJSON nullvalue; template @@ -348,7 +351,10 @@ class ADIOS2IOHandlerImpl template std::vector getDatasetOperators( - Parameter const &, Writable *, std::string const &varName); + Parameter const &, + Writable *, + std::string const &varName, + std::vector default_operators = {}); std::string fileSuffix(bool verbose = true) const; diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 5d3ce17f36..94da058dbf 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -35,6 +35,7 @@ #include "openPMD/ThrowError.hpp" #include "openPMD/auxiliary/Environment.hpp" #include "openPMD/auxiliary/Filesystem.hpp" +#include "openPMD/auxiliary/JSON.hpp" #include "openPMD/auxiliary/JSONMatcher.hpp" #include "openPMD/auxiliary/JSON_internal.hpp" #include "openPMD/auxiliary/Mpi.hpp" @@ -233,6 +234,19 @@ void ADIOS2IOHandlerImpl::init( groupTableViaEnv == 0 ? UseGroupTable::No : UseGroupTable::Yes; } + { + constexpr char const *const init_json_shadow_str = R"( + { + "adios2": { + "dataset": { + "operators": null + } + } + })"; + auto init_json_shadow = nlohmann::json::parse(init_json_shadow_str); + json::merge(cfg.getShadow(), init_json_shadow); + } + if (cfg.json().contains("adios2")) { m_config = cfg["adios2"]; @@ -309,7 +323,7 @@ void ADIOS2IOHandlerImpl::init( auto operators = getOperators(); if (operators) { - defaultOperators = std::move(operators.value()); + readOperators = std::move(operators.value()); } } } @@ -409,27 +423,55 @@ ADIOS2IOHandlerImpl::getOperators() template auto ADIOS2IOHandlerImpl::getDatasetOperators( - Parameter const ¶meters, Writable *writable, std::string const &varName) + Parameter const ¶meters, + Writable *writable, + std::string const &varName, + std::vector operators) -> std::vector { - std::vector operators; - json::TracingJSON options = - parameters.template compileJSONConfig( - writable, *m_handler->jsonMatcher, "adios2"); - if (options.json().contains("adios2")) - { - json::TracingJSON datasetConfig(options["adios2"]); - auto datasetOperators = getOperators(datasetConfig); + json::TracingJSON config = [&]() { + if (!m_buffered_dataset_config.has_value()) + { + // we are only interested in these values from the global config + constexpr char const *const mask_for_global_conf = R"( + { + "dataset": { + "operators": null, + "shape": null + } + })"; + m_buffered_dataset_config = m_config.json(); + json::filterByTemplate( + *m_buffered_dataset_config, + nlohmann::json::parse(mask_for_global_conf)); + } + auto parsed_config = + parameters.template compileJSONConfig( + writable, *m_handler->jsonMatcher, "adios2"); + if (auto adios2_config_it = parsed_config.config.find("adios2"); + adios2_config_it != parsed_config.config.end()) + { + adios2_config_it.value() = json::merge( + *m_buffered_dataset_config, adios2_config_it.value()); + } + else + { + parsed_config.config["adios2"] = *m_buffered_dataset_config; + } + return parsed_config; + }(); - operators = datasetOperators ? std::move(datasetOperators.value()) - : defaultOperators; - } - else + if (config.json().contains("adios2")) { - operators = defaultOperators; + json::TracingJSON datasetConfig(config["adios2"]); + auto maybe_operators = getOperators(datasetConfig); + if (maybe_operators) + { + operators = std::move(*maybe_operators); + } } parameters.warnUnusedParameters( - options, + config, "adios2", "Warning: parts of the backend configuration for ADIOS2 dataset '" + varName + "' remain unused:\n"); From d5e9884cdcf715aae35b593332c6446c9462528c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 8 Mar 2024 18:16:07 +0100 Subject: [PATCH 02/16] Main implementation + testing --- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 16 ++- src/IO/ADIOS/ADIOS2File.cpp | 32 ++++- src/IO/ADIOS/ADIOS2IOHandler.cpp | 117 +++++++++++++++---- test/ParallelIOTest.cpp | 42 +++++++ 4 files changed, 181 insertions(+), 26 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index 74008d41c5..db656b4c0b 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -39,7 +39,6 @@ #include "openPMD/backend/Variant_internal.hpp" #include "openPMD/backend/Writable.hpp" #include "openPMD/config.hpp" -#include #if openPMD_HAVE_ADIOS2 #include @@ -349,12 +348,19 @@ class ADIOS2IOHandlerImpl // use m_config std::optional> getOperators(); + enum class Shape + { + GlobalArray, + LocalValue + }; + template - std::vector getDatasetOperators( + auto parseDatasetConfig( Parameter const &, Writable *, std::string const &varName, - std::vector default_operators = {}); + std::vector default_operators = {}) + -> std::tuple, Shape>; std::string fileSuffix(bool verbose = true) const; @@ -553,6 +559,10 @@ class ADIOS2IOHandlerImpl } // TODO leave this check to ADIOS? adios2::Dims shape = var.Shape(); + if (shape == adios2::Dims{adios2::LocalValueDim}) + { + return var; + } auto actualDim = shape.size(); { auto requiredDim = extent.size(); diff --git a/src/IO/ADIOS/ADIOS2File.cpp b/src/IO/ADIOS/ADIOS2File.cpp index 30d6f0e981..4ac9f52f60 100644 --- a/src/IO/ADIOS/ADIOS2File.cpp +++ b/src/IO/ADIOS/ADIOS2File.cpp @@ -110,7 +110,21 @@ void WriteDataset::call(ADIOS2File &ba, detail::BufferedPut &bp) std::nullopt, ba.variables()); - engine.Put(var, ptr); + if (var.Shape() == adios2::Dims{adios2::LocalValue}) + { + if (bp.param.extent != Extent{1}) + { + throw error::OperationUnsupportedInBackend( + "ADIOS2", + "Can only write a single element to LocalValue " + "variables (extent == Extent{1})."); + } + engine.Put(var, *ptr); + } + else + { + engine.Put(var, ptr); + } } else if constexpr (std::is_same_v< ptr_type, @@ -181,7 +195,21 @@ struct RunUniquePtrPut bufferedPut.name, std::nullopt, ba.variables()); - engine.Put(var, ptr); + if (var.Shape() == adios2::Dims{adios2::LocalValue}) + { + if (bufferedPut.extent != Extent{1}) + { + throw error::OperationUnsupportedInBackend( + "ADIOS2", + "Can only write a single element to LocalValue " + "variables (extent == Extent{1})."); + } + engine.Put(var, *ptr); + } + else + { + engine.Put(var, ptr); + } } static constexpr char const *errorMsg = "RunUniquePtrPut"; diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 94da058dbf..6e288c86ae 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -239,7 +239,8 @@ void ADIOS2IOHandlerImpl::init( { "adios2": { "dataset": { - "operators": null + "operators": null, + "shape": null } } })"; @@ -422,12 +423,12 @@ ADIOS2IOHandlerImpl::getOperators() } template -auto ADIOS2IOHandlerImpl::getDatasetOperators( +auto ADIOS2IOHandlerImpl::parseDatasetConfig( Parameter const ¶meters, Writable *writable, std::string const &varName, std::vector operators) - -> std::vector + -> std::tuple, Shape> { json::TracingJSON config = [&]() { if (!m_buffered_dataset_config.has_value()) @@ -461,21 +462,58 @@ auto ADIOS2IOHandlerImpl::getDatasetOperators( return parsed_config; }(); - if (config.json().contains("adios2")) - { - json::TracingJSON datasetConfig(config["adios2"]); - auto maybe_operators = getOperators(datasetConfig); - if (maybe_operators) + Shape arrayShape = Shape::GlobalArray; + [&]() { + if (!config.json().contains("adios2")) + { + return; + }; + json::TracingJSON adios2Config(config["adios2"]); + auto datasetOperators = getOperators(adios2Config); + if (datasetOperators.has_value()) { - operators = std::move(*maybe_operators); + operators = std::move(*datasetOperators); } - } + if (!adios2Config.json().contains("dataset")) + { + return; + } + auto datasetConfig = adios2Config["dataset"]; + if (!datasetConfig.json().contains("shape")) + { + return; + } + auto maybe_shape = + json::asLowerCaseStringDynamic(datasetConfig["shape"].json()); + if (!maybe_shape.has_value()) + { + throw error::BackendConfigSchema( + {"adios2", "dataset", "shape"}, + "Must be convertible to string type."); + } + auto const &shape = *maybe_shape; + if (shape == "global_array") + { + arrayShape = Shape::GlobalArray; + } + else if (shape == "local_value") + { + arrayShape = Shape::LocalValue; + } + else + { + throw error::BackendConfigSchema( + {"adios2", "dataset", "shape"}, + "Unknown value: '" + shape + "'."); + } + }(); + parameters.warnUnusedParameters( config, "adios2", "Warning: parts of the backend configuration for ADIOS2 dataset '" + varName + "' remain unused:\n"); - return operators; + return {std::move(operators), arrayShape}; } using AcceptedEndingsForEngine = std::map; @@ -887,15 +925,47 @@ void ADIOS2IOHandlerImpl::createDataset( filePos->gd = GroupOrDataset::DATASET; auto const varName = nameOfVariable(writable); - std::vector operators = - getDatasetOperators(parameters, writable, varName); + // Captured structured bindings are a C++20 extension... + std::vector operators; + Shape arrayShape; + std::tie(operators, arrayShape) = + parseDatasetConfig(parameters, writable, varName); - // cast from openPMD::Extent to adios2::Dims - adios2::Dims shape(parameters.extent.begin(), parameters.extent.end()); - if (auto jd = parameters.joinedDimension; jd.has_value()) - { - shape[jd.value()] = adios2::JoinedDim; - } + adios2::Dims shape = [&, arrayShape = arrayShape]() { + switch (arrayShape) + { + + case Shape::GlobalArray: { + // cast from openPMD::Extent to adios2::Dims + adios2::Dims res( + parameters.extent.begin(), parameters.extent.end()); + if (auto jd = parameters.joinedDimension; jd.has_value()) + { + res[jd.value()] = adios2::JoinedDim; + } + return res; + } + case Shape::LocalValue: { + int required_size = 1; +#if openPMD_HAVE_MPI + if (m_communicator.has_value()) + { + MPI_Comm_size(*m_communicator, &required_size); + } +#endif + if (parameters.extent != + Extent{Extent::value_type(required_size)}) + { + throw error::OperationUnsupportedInBackend( + "ADIOS2", + "Shape for local value array must be a 1D array " + "equivalent to the MPI size."); + } + return adios2::Dims{adios2::LocalValueDim}; + } + } + throw std::runtime_error("Unreachable!"); + }(); auto &fileData = getFileData(file, IfFileNotOpen::ThrowError); @@ -1160,8 +1230,8 @@ void ADIOS2IOHandlerImpl::openDataset( * reading, so the dataset-specific configuration should still be explored * here. */ - std::vector operators = - getDatasetOperators(parameters, writable, varName); + [[maybe_unused]] auto [operators, _] = + parseDatasetConfig(parameters, writable, varName); switchAdios2VariableType( *parameters.dtype, this, @@ -1265,6 +1335,11 @@ namespace detail varName, std::nullopt, ba.variables()); + if (variable.Shape() == adios2::Dims{adios2::LocalValueDim}) + { + params.out->backendManagedBuffer = false; + return; + } adios2::Dims offset(params.offset.begin(), params.offset.end()); adios2::Dims extent(params.extent.begin(), params.extent.end()); variable.SetSelection({std::move(offset), std::move(extent)}); diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index 84d746aeb7..0f58f61d51 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -439,6 +439,48 @@ void available_chunks_test(std::string const &file_ending) auto E_x = it0.meshes["E"]["x"]; E_x.resetDataset({Datatype::INT, {mpi_size, 4}}); E_x.storeChunk(data, {mpi_rank, 0}, {1, 4}); + + auto electrons = it0.particles["e"].particlePatches; + auto numParticles = electrons["numParticles"]; + auto numParticlesOffset = electrons["numParticlesOffset"]; + for (auto rc : {&numParticles, &numParticlesOffset}) + { + rc->resetDataset( + {Datatype::ULONG, + {Extent::value_type{mpi_size}}, + R"(adios2.dataset.shape = "local_value")"}); + } + numParticles.storeChunk( + std::make_unique(10), {size_t(mpi_rank)}, {1}); + numParticlesOffset.storeChunk( + std::make_unique(10 * ((unsigned long)mpi_rank)), + {size_t(mpi_rank)}, + {1}); + auto offset = electrons["offset"]; + for (auto const &dim : {"x", "y", "z"}) + { + auto rc = offset[dim]; + rc.resetDataset( + {Datatype::ULONG, + {Extent::value_type{mpi_size}}, + R"(adios2.dataset.shape = "local_value")"}); + rc.storeChunk( + std::make_unique((unsigned long)mpi_rank), + {size_t(mpi_rank)}, + {1}); + } + auto extent = electrons["extent"]; + for (auto const &dim : {"x", "y", "z"}) + { + auto rc = extent[dim]; + rc.resetDataset( + {Datatype::ULONG, + {Extent::value_type{mpi_size}}, + R"(adios2.dataset.shape = "local_value")"}); + rc.storeChunk( + std::make_unique(1), {size_t(mpi_rank)}, {1}); + } + it0.close(); } From b7778f56d318e0b1f4e3ad53adf0377e84bff7c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 11 Mar 2024 15:11:59 +0100 Subject: [PATCH 03/16] Fix bug in ADIOS2 that leaks dataset configs into other datasets --- src/IO/ADIOS/ADIOS2IOHandler.cpp | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 6e288c86ae..c4dfae9174 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -430,7 +430,7 @@ auto ADIOS2IOHandlerImpl::parseDatasetConfig( std::vector operators) -> std::tuple, Shape> { - json::TracingJSON config = [&]() { + json::TracingJSON config = [&]() -> json::ParsedConfig { if (!m_buffered_dataset_config.has_value()) { // we are only interested in these values from the global config @@ -446,18 +446,21 @@ auto ADIOS2IOHandlerImpl::parseDatasetConfig( *m_buffered_dataset_config, nlohmann::json::parse(mask_for_global_conf)); } + auto const &buffered_config = *m_buffered_dataset_config; auto parsed_config = parameters.template compileJSONConfig( writable, *m_handler->jsonMatcher, "adios2"); if (auto adios2_config_it = parsed_config.config.find("adios2"); adios2_config_it != parsed_config.config.end()) { - adios2_config_it.value() = json::merge( - *m_buffered_dataset_config, adios2_config_it.value()); + auto copy = buffered_config; + json::merge(copy, adios2_config_it.value()); + copy = nlohmann::json{{"adios2", std::move(copy)}}; + parsed_config.config = std::move(copy); } else { - parsed_config.config["adios2"] = *m_buffered_dataset_config; + parsed_config.config["adios2"] = buffered_config; } return parsed_config; }(); @@ -959,7 +962,11 @@ void ADIOS2IOHandlerImpl::createDataset( throw error::OperationUnsupportedInBackend( "ADIOS2", "Shape for local value array must be a 1D array " - "equivalent to the MPI size."); + "equivalent to the MPI size ('" + + varName + "' has shape " + + auxiliary::format_vec(parameters.extent) + + ", but should have shape [" + + std::to_string(required_size) + "])."); } return adios2::Dims{adios2::LocalValueDim}; } From 83145eb0f653b8e3d8f7067d6275749c734b0263 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 11 Mar 2024 15:12:17 +0100 Subject: [PATCH 04/16] Add particle patches to streaming example, using local_value --- examples/10_streaming_read.cpp | 19 ++++++++ examples/10_streaming_write.cpp | 86 ++++++++++++++++++++++++++++++--- 2 files changed, 98 insertions(+), 7 deletions(-) diff --git a/examples/10_streaming_read.cpp b/examples/10_streaming_read.cpp index e19db4d2ac..c346c4d6c2 100644 --- a/examples/10_streaming_read.cpp +++ b/examples/10_streaming_read.cpp @@ -1,3 +1,4 @@ +#include "openPMD/auxiliary/StringManip.hpp" #include #include @@ -55,6 +56,24 @@ int main() extents[i] = rc.getExtent(); } + auto e_patches = iteration.particles["e"].particlePatches; + for (auto key : + {"numParticles", "numParticlesOffset", "offset", "extent"}) + { + for (auto &rc : e_patches[key]) + { + std::cout << "Chunks for '" << rc.second.myPath().openPMDPath() + << "':"; + for (auto const &chunk : rc.second.availableChunks()) + { + std::cout << "\n\tRank " << chunk.sourceID << "\t" + << auxiliary::format_vec(chunk.offset) << "\t– " + << auxiliary::format_vec(chunk.extent); + } + std::cout << std::endl; + } + } + // The iteration can be closed in order to help free up resources. // The iteration's content will be flushed automatically. iteration.close(); diff --git a/examples/10_streaming_write.cpp b/examples/10_streaming_write.cpp index 6c83fcfbb4..94e78d7fc9 100644 --- a/examples/10_streaming_write.cpp +++ b/examples/10_streaming_write.cpp @@ -1,6 +1,5 @@ -#include "openPMD/Series.hpp" -#include "openPMD/snapshots/Snapshots.hpp" #include +#include #include #include @@ -21,6 +20,14 @@ int main() return 0; } + int mpi_rank{0}, mpi_size{1}; + +#if openPMD_HAVE_MPI + MPI_Init(nullptr, nullptr); + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); +#endif + // open file for writing // use QueueFullPolicy = Discard in order to create a situation where from // the reader's perspective steps are skipped. This tests the bug reported @@ -30,7 +37,13 @@ int main() // Iterations can be accessed independently from one another. This more // restricted mode enables performance optimizations in the backends, and // more importantly is compatible with streaming I/O. - Series series = Series("electrons.sst", Access::CREATE_LINEAR, R"( + Series series = Series( + "electrons.sst", + Access::CREATE_LINEAR, +#if openPMD_HAVE_MPI + MPI_COMM_WORLD, +#endif + R"( { "adios2": { "engine": { @@ -40,11 +53,13 @@ int main() } } } -})"); +})" + + ); Datatype datatype = determineDatatype(); constexpr unsigned long length = 10ul; - Extent global_extent = {length}; + Extent global_extent = {mpi_size * length}; Dataset dataset = Dataset(datatype, global_extent); std::shared_ptr local_data( new position_t[length], [](position_t const *ptr) { delete[] ptr; }); @@ -55,13 +70,66 @@ int main() Iteration iteration = iterations[i]; Record electronPositions = iteration.particles["e"]["position"]; - std::iota(local_data.get(), local_data.get() + length, i * length); + std::iota( + local_data.get(), + local_data.get() + length, + i * length * mpi_size + mpi_rank * length); for (auto const &dim : {"x", "y", "z"}) { RecordComponent pos = electronPositions[dim]; pos.resetDataset(dataset); - pos.storeChunk(local_data, Offset{0}, global_extent); + pos.storeChunk(local_data, Offset{length * mpi_rank}, {length}); + } + + // Use the `local_value` ADIOS2 dataset shape to send a dataset not via + // the data plane, but the control plane of ADIOS2 SST. This is + // advisable for datasets where each rank contributes only a single item + // since the control plane performs data aggregation, thus avoiding + // fully interconnected communication meshes for data that needs to be + // read by each reader. A local value dataset can only contain a single + // item per MPI rank, forming an array of length equal to the MPI size. + + auto e_patches = iteration.particles["e"].particlePatches; + auto numParticles = e_patches["numParticles"]; + auto numParticlesOffset = e_patches["numParticlesOffset"]; + for (auto rc : {&numParticles, &numParticlesOffset}) + { + rc->resetDataset( + {Datatype::ULONG, + {Extent::value_type(mpi_size)}, + R"(adios2.dataset.shape = "local_value")"}); } + numParticles.storeChunk( + std::make_unique(10), {size_t(mpi_rank)}, {1}); + numParticlesOffset.storeChunk( + std::make_unique(10 * ((unsigned long)mpi_rank)), + {size_t(mpi_rank)}, + {1}); + auto offset = e_patches["offset"]; + for (auto const &dim : {"x", "y", "z"}) + { + auto rc = offset[dim]; + rc.resetDataset( + {Datatype::ULONG, + {Extent::value_type(mpi_size)}, + R"(adios2.dataset.shape = "local_value")"}); + rc.storeChunk( + std::make_unique((unsigned long)mpi_rank), + {size_t(mpi_rank)}, + {1}); + } + auto extent = e_patches["extent"]; + for (auto const &dim : {"x", "y", "z"}) + { + auto rc = extent[dim]; + rc.resetDataset( + {Datatype::ULONG, + {Extent::value_type(mpi_size)}, + R"(adios2.dataset.shape = "local_value")"}); + rc.storeChunk( + std::make_unique(1), {size_t(mpi_rank)}, {1}); + } + iteration.close(); } @@ -73,6 +141,10 @@ int main() */ series.close(); +#if openPMD_HAVE_MPI + MPI_Finalize(); +#endif + return 0; #else std::cout << "The streaming example requires that openPMD has been built " From 1dbdee88a2fa86619a11be775498a6bc3fabf1e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 12 Mar 2024 10:57:09 +0100 Subject: [PATCH 05/16] Fixes --- src/IO/ADIOS/ADIOS2File.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/IO/ADIOS/ADIOS2File.cpp b/src/IO/ADIOS/ADIOS2File.cpp index 4ac9f52f60..a6abe579ec 100644 --- a/src/IO/ADIOS/ADIOS2File.cpp +++ b/src/IO/ADIOS/ADIOS2File.cpp @@ -110,14 +110,16 @@ void WriteDataset::call(ADIOS2File &ba, detail::BufferedPut &bp) std::nullopt, ba.variables()); - if (var.Shape() == adios2::Dims{adios2::LocalValue}) + if (var.Shape() == adios2::Dims{adios2::LocalValueDim}) { if (bp.param.extent != Extent{1}) { throw error::OperationUnsupportedInBackend( "ADIOS2", "Can only write a single element to LocalValue " - "variables (extent == Extent{1})."); + "variables (extent == Extent{1}, but extent of '" + + bp.name + " was " + + auxiliary::format_vec(bp.param.extent) + "')."); } engine.Put(var, *ptr); } @@ -195,14 +197,16 @@ struct RunUniquePtrPut bufferedPut.name, std::nullopt, ba.variables()); - if (var.Shape() == adios2::Dims{adios2::LocalValue}) + if (var.Shape() == adios2::Dims{adios2::LocalValueDim}) { if (bufferedPut.extent != Extent{1}) { throw error::OperationUnsupportedInBackend( "ADIOS2", "Can only write a single element to LocalValue " - "variables (extent == Extent{1})."); + "variables (extent == Extent{1}, but extent of '" + + bufferedPut.name + " was " + + auxiliary::format_vec(bufferedPut.extent) + "')."); } engine.Put(var, *ptr); } From 7f3b25ba447c1224e0a8d1c58874f14cae2ad640 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 12 Mar 2024 12:03:35 +0100 Subject: [PATCH 06/16] MPI include guard --- examples/10_streaming_write.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/examples/10_streaming_write.cpp b/examples/10_streaming_write.cpp index 94e78d7fc9..6b84a76ccb 100644 --- a/examples/10_streaming_write.cpp +++ b/examples/10_streaming_write.cpp @@ -6,6 +6,10 @@ #include #include // std::iota +#if openPMD_HAVE_MPI +#include +#endif + using std::cout; using namespace openPMD; From e79d430ee7f01cb3ec3b7c33425dca72a4776c9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 13 Mar 2024 14:44:10 +0100 Subject: [PATCH 07/16] CI fixes --- src/IO/ADIOS/ADIOS2IOHandler.cpp | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index c4dfae9174..cd838ff998 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -430,7 +430,7 @@ auto ADIOS2IOHandlerImpl::parseDatasetConfig( std::vector operators) -> std::tuple, Shape> { - json::TracingJSON config = [&]() -> json::ParsedConfig { + json::TracingJSON parsedConfig = [&]() -> json::ParsedConfig { if (!m_buffered_dataset_config.has_value()) { // we are only interested in these values from the global config @@ -467,11 +467,11 @@ auto ADIOS2IOHandlerImpl::parseDatasetConfig( Shape arrayShape = Shape::GlobalArray; [&]() { - if (!config.json().contains("adios2")) + if (!parsedConfig.json().contains("adios2")) { return; }; - json::TracingJSON adios2Config(config["adios2"]); + json::TracingJSON adios2Config(parsedConfig["adios2"]); auto datasetOperators = getOperators(adios2Config); if (datasetOperators.has_value()) { @@ -511,11 +511,21 @@ auto ADIOS2IOHandlerImpl::parseDatasetConfig( } }(); +#if 0 + std::cout << "Operations for '" << varName << "':"; + for(auto const & op: operators) + { + std::cout << " '" << op.op.Type() << "'"; + } + std::cout << std::endl; +#endif + parameters.warnUnusedParameters( - config, + parsedConfig, "adios2", "Warning: parts of the backend configuration for ADIOS2 dataset '" + varName + "' remain unused:\n"); + return {std::move(operators), arrayShape}; } @@ -934,7 +944,7 @@ void ADIOS2IOHandlerImpl::createDataset( std::tie(operators, arrayShape) = parseDatasetConfig(parameters, writable, varName); - adios2::Dims shape = [&, arrayShape = arrayShape]() { + adios2::Dims shape = [&]() { switch (arrayShape) { From 0f60ef0b69a4a97e56a6aecb1867aca58bf05049 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 19 Mar 2024 11:15:40 +0100 Subject: [PATCH 08/16] Add links to ADIOS2 documentation in comments Co-authored-by: Axel Huebl --- examples/10_streaming_write.cpp | 1 + src/IO/ADIOS/ADIOS2File.cpp | 2 ++ 2 files changed, 3 insertions(+) diff --git a/examples/10_streaming_write.cpp b/examples/10_streaming_write.cpp index 6b84a76ccb..de8d481517 100644 --- a/examples/10_streaming_write.cpp +++ b/examples/10_streaming_write.cpp @@ -92,6 +92,7 @@ int main() // fully interconnected communication meshes for data that needs to be // read by each reader. A local value dataset can only contain a single // item per MPI rank, forming an array of length equal to the MPI size. + // https://adios2.readthedocs.io/en/v2.9.2/components/components.html#shapes auto e_patches = iteration.particles["e"].particlePatches; auto numParticles = e_patches["numParticles"]; diff --git a/src/IO/ADIOS/ADIOS2File.cpp b/src/IO/ADIOS/ADIOS2File.cpp index a6abe579ec..0235994573 100644 --- a/src/IO/ADIOS/ADIOS2File.cpp +++ b/src/IO/ADIOS/ADIOS2File.cpp @@ -110,6 +110,7 @@ void WriteDataset::call(ADIOS2File &ba, detail::BufferedPut &bp) std::nullopt, ba.variables()); + // https://adios2.readthedocs.io/en/v2.9.2/components/components.html#shapes if (var.Shape() == adios2::Dims{adios2::LocalValueDim}) { if (bp.param.extent != Extent{1}) @@ -197,6 +198,7 @@ struct RunUniquePtrPut bufferedPut.name, std::nullopt, ba.variables()); + // https://adios2.readthedocs.io/en/v2.9.2/components/components.html#shapes if (var.Shape() == adios2::Dims{adios2::LocalValueDim}) { if (bufferedPut.extent != Extent{1}) From b664368296d56ca5a7197bbbb457f8a9e0157e6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 22 Apr 2024 16:40:52 +0200 Subject: [PATCH 09/16] Add documentation --- docs/source/details/backendconfig.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/source/details/backendconfig.rst b/docs/source/details/backendconfig.rst index 9d5e1dcf2c..fae9df8791 100644 --- a/docs/source/details/backendconfig.rst +++ b/docs/source/details/backendconfig.rst @@ -190,6 +190,17 @@ Explanation of the single keys: * ``type`` supported ADIOS operator type, e.g. zfp, sz * ``parameters`` is an associative map of string parameters for the operator (e.g. compression levels) +* ``adios2.dataset.shape`` (advanced): Specify the `dataset shape `_ for the ADIOS2 variable. + Note that variable shapes will generally imply a different way of interacting with a variable, and some variable shapes (such as *joined arrays*) may not be accessible via this parameter, but via different API calls instead. + This parameter's purpose to select different implementations for the same used API call. + Supported values by this parameter are: + + * ``"global_array"`` (default): The variable is a (n-dimensional) array with a globally defined size. Local blocks, subsets of the global region, are written by parallel writers. + * ``"local_value"``: Each parallel writer contributes one single value to the dataset, joined into a 1-dimensional array. + Since there (currently) exists no dedicated API call for this shape in the openPMD-api, this setting is only useful as an optimization since "local value" variables participate in ADIOS2 metadata aggregation. + Can only be applied if the global shape (1-dimensional array the same length as number of parallel instances) and the local blocks (a single data item) are specified correctly. + Use global or joined arrays otherwise. + * ``adios2.use_span_based_put``: The openPMD-api exposes the `span-based Put() API `_ of ADIOS2 via an overload of ``RecordComponent::storeChunk()``. This API is incompatible with compression operators as described above. The openPMD-api will automatically use a fallback implementation for the span-based Put() API if any operator is added to a dataset. From 55dd262025a5dfc1e533e03d51532ef9f4956fa4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 22 Apr 2024 16:43:27 +0200 Subject: [PATCH 10/16] Update examples/10_streaming_read.cpp Co-authored-by: Axel Huebl --- examples/10_streaming_read.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/10_streaming_read.cpp b/examples/10_streaming_read.cpp index c346c4d6c2..4ded07aa2d 100644 --- a/examples/10_streaming_read.cpp +++ b/examples/10_streaming_read.cpp @@ -1,4 +1,4 @@ -#include "openPMD/auxiliary/StringManip.hpp" +#include #include #include From 5d868ae67729b6a670f15e713c5afe5d5e10094d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 22 Apr 2024 17:10:09 +0200 Subject: [PATCH 11/16] Add review suggestions --- examples/10_streaming_read.cpp | 4 +-- include/openPMD/auxiliary/StringManip.hpp | 15 +++++++++ src/IO/ADIOS/ADIOS2File.cpp | 5 +-- src/IO/ADIOS/ADIOS2IOHandler.cpp | 2 +- test/ParallelIOTest.cpp | 40 +++++++++++++++++++++-- 5 files changed, 59 insertions(+), 7 deletions(-) diff --git a/examples/10_streaming_read.cpp b/examples/10_streaming_read.cpp index 4ded07aa2d..e794f16a3a 100644 --- a/examples/10_streaming_read.cpp +++ b/examples/10_streaming_read.cpp @@ -67,8 +67,8 @@ int main() for (auto const &chunk : rc.second.availableChunks()) { std::cout << "\n\tRank " << chunk.sourceID << "\t" - << auxiliary::format_vec(chunk.offset) << "\t– " - << auxiliary::format_vec(chunk.extent); + << auxiliary::vec_as_string(chunk.offset) << "\t– " + << auxiliary::vec_as_string(chunk.extent); } std::cout << std::endl; } diff --git a/include/openPMD/auxiliary/StringManip.hpp b/include/openPMD/auxiliary/StringManip.hpp index eb3799d3be..ede4ab7152 100644 --- a/include/openPMD/auxiliary/StringManip.hpp +++ b/include/openPMD/auxiliary/StringManip.hpp @@ -243,6 +243,14 @@ namespace auxiliary return std::forward(s); } + /** Write a string representation of a vector or another iterable + * container to a stream. + * + * @param s The stream to write to. + * @param vec The vector or other iterable container. + * @return The modified stream. Each item is + * formatted using the default definition for operator<<(). + */ template auto write_vec_to_stream(Stream &&s, Vec const &vec) -> Stream && { @@ -265,6 +273,13 @@ namespace auxiliary return std::forward(s); } + /** Create a string representation of a vector or another iterable + * container. + * + * @param vec The vector or other iterable container. + * @return A string that shows the items of the container. Each item is + * formatted using the default definition for operator<<(). + */ template auto vec_as_string(Vec const &vec) -> std::string { diff --git a/src/IO/ADIOS/ADIOS2File.cpp b/src/IO/ADIOS/ADIOS2File.cpp index 0235994573..c5d78f483a 100644 --- a/src/IO/ADIOS/ADIOS2File.cpp +++ b/src/IO/ADIOS/ADIOS2File.cpp @@ -120,7 +120,8 @@ void WriteDataset::call(ADIOS2File &ba, detail::BufferedPut &bp) "Can only write a single element to LocalValue " "variables (extent == Extent{1}, but extent of '" + bp.name + " was " + - auxiliary::format_vec(bp.param.extent) + "')."); + auxiliary::vec_as_string(bp.param.extent) + + "')."); } engine.Put(var, *ptr); } @@ -208,7 +209,7 @@ struct RunUniquePtrPut "Can only write a single element to LocalValue " "variables (extent == Extent{1}, but extent of '" + bufferedPut.name + " was " + - auxiliary::format_vec(bufferedPut.extent) + "')."); + auxiliary::vec_as_string(bufferedPut.extent) + "')."); } engine.Put(var, *ptr); } diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index cd838ff998..ffd50609a6 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -974,7 +974,7 @@ void ADIOS2IOHandlerImpl::createDataset( "Shape for local value array must be a 1D array " "equivalent to the MPI size ('" + varName + "' has shape " + - auxiliary::format_vec(parameters.extent) + + auxiliary::vec_as_string(parameters.extent) + ", but should have shape [" + std::to_string(required_size) + "])."); } diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index 0f58f61d51..cd7dba0bf8 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -9,6 +9,7 @@ #include "openPMD/auxiliary/Environment.hpp" #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/Mpi.hpp" +#include "openPMD/backend/PatchRecordComponent.hpp" #include "openPMD/openPMD.hpp" // @todo change includes #include "openPMD/auxiliary/OneDimensionalBlockSlicer.hpp" @@ -406,7 +407,7 @@ void available_chunks_test(std::string const &file_ending) MPI_Comm_size(MPI_COMM_WORLD, &r_mpi_size); unsigned mpi_rank{static_cast(r_mpi_rank)}, mpi_size{static_cast(r_mpi_size)}; - std::string name = "../samples/available_chunks." + file_ending; + std::string name = "../samples/parallel_available_chunks." + file_ending; /* * ADIOS2 assigns writerIDs to blocks in a BP file by id of the substream @@ -419,7 +420,6 @@ void available_chunks_test(std::string const &file_ending) { "engine": { - "type": "bp4", "parameters": { "NumAggregators":)END" @@ -440,6 +440,14 @@ void available_chunks_test(std::string const &file_ending) E_x.resetDataset({Datatype::INT, {mpi_size, 4}}); E_x.storeChunk(data, {mpi_rank, 0}, {1, 4}); + /* + * Verify that block decomposition also works in "local value" variable + * shape. That shape instructs the data to participate in ADIOS2 + * metadata aggregation, hence there is only one "real" written block, + * the aggregated one. We still need the original logical blocks to be + * present in reading. + */ + auto electrons = it0.particles["e"].particlePatches; auto numParticles = electrons["numParticles"]; auto numParticlesOffset = electrons["numParticlesOffset"]; @@ -512,12 +520,40 @@ void available_chunks_test(std::string const &file_ending) { REQUIRE(ranks[i] == i); } + + auto electrons = it0.particles["e"].particlePatches; + for (PatchRecordComponent *prc : + {static_cast(&electrons["numParticles"]), + static_cast( + &electrons["numParticlesOffset"]), + &electrons["offset"]["x"], + &electrons["offset"]["y"], + &electrons["extent"]["z"], + &electrons["offset"]["x"], + &electrons["extent"]["y"], + &electrons["extent"]["z"]}) + { + auto available_chunks = prc->availableChunks(); + REQUIRE(size_t(r_mpi_size) == available_chunks.size()); + for (size_t i = 0; i < available_chunks.size(); ++i) + { + auto const &chunk = available_chunks[i]; + REQUIRE(chunk.extent == Extent{1}); + REQUIRE(chunk.offset == Offset{i}); + REQUIRE(chunk.sourceID == i); + } + } } } TEST_CASE("available_chunks_test", "[parallel][adios]") { +#if HAS_ADIOS_2_9 + available_chunks_test("bp4"); + available_chunks_test("bp5"); +#else available_chunks_test("bp"); +#endif } #endif From e7060fcac348d875c964d7a5ae698ab1f3430777 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 11 Dec 2024 09:18:55 +0000 Subject: [PATCH 12/16] Fix includes --- examples/10_streaming_write.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/10_streaming_write.cpp b/examples/10_streaming_write.cpp index de8d481517..55975b5c3a 100644 --- a/examples/10_streaming_write.cpp +++ b/examples/10_streaming_write.cpp @@ -1,5 +1,4 @@ #include -#include #include #include From 75ab6075288a89e51e9a633832125d144a9abc56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 21 Feb 2025 11:59:28 +0100 Subject: [PATCH 13/16] Only set shape if it was changed --- src/IO/ADIOS/ADIOS2IOHandler.cpp | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index ffd50609a6..16f4c9be86 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -2617,7 +2617,23 @@ namespace detail } else { - var.SetShape(shape); + auto const &old_shape = var.Shape(); + bool shape_changed = old_shape.size() != shape.size(); + if (!shape_changed) + { + for (size_t i = 0; i < old_shape.size(); ++i) + { + if (old_shape[i] != shape[i]) + { + shape_changed = true; + break; + } + } + } + if (shape_changed) + { + var.SetShape(shape); + } if (count.size() > 0) { var.SetSelection({start, count}); From f642db9e51f38dc41a77897f153800bf3ee579c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 26 Mar 2025 13:30:57 +0100 Subject: [PATCH 14/16] Fix after rebase: merge() -> merge_internal() Function has been renamed on dev --- src/IO/ADIOS/ADIOS2IOHandler.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 16f4c9be86..8b8e036f5e 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -245,7 +245,11 @@ void ADIOS2IOHandlerImpl::init( } })"; auto init_json_shadow = nlohmann::json::parse(init_json_shadow_str); - json::merge(cfg.getShadow(), init_json_shadow); + std::cout << "Will merge:\n" + << init_json_shadow << "\ninto:\n" + << cfg.getShadow() << std::endl; + json::merge_internal( + cfg.getShadow(), init_json_shadow, /* do_prune = */ false); } if (cfg.json().contains("adios2")) @@ -454,7 +458,8 @@ auto ADIOS2IOHandlerImpl::parseDatasetConfig( adios2_config_it != parsed_config.config.end()) { auto copy = buffered_config; - json::merge(copy, adios2_config_it.value()); + json::merge_internal( + copy, adios2_config_it.value(), /* do_prune = */ false); copy = nlohmann::json{{"adios2", std::move(copy)}}; parsed_config.config = std::move(copy); } From f1d2c80802ae33079e389d2b525b747e0aebbea7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 26 Mar 2025 14:12:19 +0100 Subject: [PATCH 15/16] Consider default read operators in openDataset() @todo: should we really? --- src/IO/ADIOS/ADIOS2IOHandler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 8b8e036f5e..e5410a8d98 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -1253,7 +1253,7 @@ void ADIOS2IOHandlerImpl::openDataset( * here. */ [[maybe_unused]] auto [operators, _] = - parseDatasetConfig(parameters, writable, varName); + parseDatasetConfig(parameters, writable, varName, readOperators); switchAdios2VariableType( *parameters.dtype, this, From df87fa71770ca91a8d8a9ad88cf6fc1e68a510ee Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 3 Apr 2025 16:28:32 +0000 Subject: [PATCH 16/16] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/10_streaming_read.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/10_streaming_read.cpp b/examples/10_streaming_read.cpp index e794f16a3a..41a2379d44 100644 --- a/examples/10_streaming_read.cpp +++ b/examples/10_streaming_read.cpp @@ -67,7 +67,8 @@ int main() for (auto const &chunk : rc.second.availableChunks()) { std::cout << "\n\tRank " << chunk.sourceID << "\t" - << auxiliary::vec_as_string(chunk.offset) << "\t– " + << auxiliary::vec_as_string(chunk.offset) + << "\t– " << auxiliary::vec_as_string(chunk.extent); } std::cout << std::endl;