From b6a9ac6d61341ad7e74f6ede6c48abe12e00248e Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Tue, 16 Jan 2024 18:09:42 -0800 Subject: [PATCH 01/50] BasisWriter::writeBasis create/open a file and closes the file at the end. --- lib/linalg/BasisReader.cpp | 1 - lib/linalg/BasisWriter.cpp | 45 +++++++++++++++++++------------------- lib/utils/CSVDatabase.cpp | 2 ++ lib/utils/CSVDatabase.h | 4 ++-- lib/utils/Database.cpp | 23 +++++++++++++++++++ lib/utils/Database.h | 6 +++-- lib/utils/HDFDatabase.cpp | 4 ++++ lib/utils/HDFDatabase.h | 4 ++-- 8 files changed, 59 insertions(+), 30 deletions(-) diff --git a/lib/linalg/BasisReader.cpp b/lib/linalg/BasisReader.cpp index 1b5c8583c..d2f0464aa 100644 --- a/lib/linalg/BasisReader.cpp +++ b/lib/linalg/BasisReader.cpp @@ -48,7 +48,6 @@ BasisReader::BasisReader( d_database = new CSVDatabase(); } - std::cout << "Opening file: " << full_file_name << std::endl; d_database->open(full_file_name, "r"); int num_time_intervals; diff --git a/lib/linalg/BasisWriter.cpp b/lib/linalg/BasisWriter.cpp index eee5c3a09..7c4b14402 100644 --- a/lib/linalg/BasisWriter.cpp +++ b/lib/linalg/BasisWriter.cpp @@ -53,20 +53,19 @@ BasisWriter::BasisWriter( char tmp2[100]; sprintf(tmp2, "_snapshot.%06d", rank); snap_file_name = base_file_name + tmp2; + + // create and open snapshot/basis database + // TODO(kevin): can it be CSV at all? we might want to remove if statement here. + if (db_format_ == Database::HDF5) { + d_snap_database = new HDFDatabase(); + d_database = new HDFDatabase(); + } } BasisWriter::~BasisWriter() { - if (d_database) { - d_database->putInteger("num_time_intervals", d_num_intervals_written); - d_database->close(); - delete d_database; - } - if (d_snap_database) { - d_snap_database->putInteger("num_time_intervals", d_num_intervals_written); - d_snap_database->close(); - delete d_snap_database; - } + if (d_database) delete d_database; + if (d_snap_database) delete d_snap_database; } void @@ -81,13 +80,10 @@ BasisWriter::writeBasis(const std::string& kind) sprintf(tmp, "time_%06d", d_num_intervals_written); if (kind == "basis") { - - // create and open basis database - if (db_format_ == Database::HDF5) { - d_database = new HDFDatabase(); - } - std::cout << "Creating file: " << full_file_name << std::endl; - d_database->create(full_file_name); + if (fileExists(full_file_name)) + d_database->open(full_file_name, "wr"); + else + d_database->create(full_file_name); d_database->putDouble(tmp, time_interval_start_time); @@ -122,15 +118,15 @@ BasisWriter::writeBasis(const std::string& kind) ++d_num_intervals_written; + d_database->putInteger("num_time_intervals", d_num_intervals_written); + d_database->close(); } if (kind == "snapshot") { - // create and open snapshot database - if (db_format_ == Database::HDF5) { - d_snap_database = new HDFDatabase(); - } - std::cout << "Creating file: " << snap_file_name << std::endl; - d_snap_database->create(snap_file_name); + if (fileExists(snap_file_name)) + d_snap_database->open(snap_file_name, "wr"); + else + d_snap_database->create(snap_file_name); d_snap_database->putDouble(tmp, time_interval_start_time); @@ -143,6 +139,9 @@ BasisWriter::writeBasis(const std::string& kind) d_snap_database->putInteger(tmp, num_cols); sprintf(tmp, "snapshot_matrix_%06d", d_num_intervals_written); d_snap_database->putDoubleArray(tmp, &snapshots->item(0,0), num_rows*num_cols); + + d_snap_database->putInteger("num_time_intervals", d_num_intervals_written); + d_snap_database->close(); } } diff --git a/lib/utils/CSVDatabase.cpp b/lib/utils/CSVDatabase.cpp index 6ecf74cd8..99547d6a7 100644 --- a/lib/utils/CSVDatabase.cpp +++ b/lib/utils/CSVDatabase.cpp @@ -30,6 +30,7 @@ bool CSVDatabase::create( const std::string& file_name) { + Database::create(file_name); return true; } @@ -38,6 +39,7 @@ CSVDatabase::open( const std::string& file_name, const std::string& type) { + Database::open(file_name, type); return true; } diff --git a/lib/utils/CSVDatabase.h b/lib/utils/CSVDatabase.h index f4cfdb465..9eefa1f8a 100644 --- a/lib/utils/CSVDatabase.h +++ b/lib/utils/CSVDatabase.h @@ -47,7 +47,7 @@ class CSVDatabase : public Database virtual bool create( - const std::string& file_name); + const std::string& file_name) override; /** * @brief Opens an existing CSV database file with the supplied name. @@ -61,7 +61,7 @@ class CSVDatabase : public Database bool open( const std::string& file_name, - const std::string& type); + const std::string& type) override; /** * @brief Closes the currently open CSV database file. diff --git a/lib/utils/Database.cpp b/lib/utils/Database.cpp index ef46d3ad4..53e240dd9 100644 --- a/lib/utils/Database.cpp +++ b/lib/utils/Database.cpp @@ -11,9 +11,18 @@ // Description: The abstract database class defines interface for databases. #include "Database.h" +#include +#include namespace CAROM { +bool fileExists(const std::string& name) +{ + std::ifstream f(name.c_str()); + return f.good(); + // ifstream f will be closed upon the end of the function. +} + Database::Database() { } @@ -22,4 +31,18 @@ Database::~Database() { } +bool +Database::create(const std::string& file_name) +{ + std::cout << "Creating file: " << file_name << std::endl; +} + +bool +Database::open( + const std::string& file_name, + const std::string& type) +{ + std::cout << "Opening file: " << file_name << std::endl; +} + } diff --git a/lib/utils/Database.h b/lib/utils/Database.h index dafdb4056..e839da863 100644 --- a/lib/utils/Database.h +++ b/lib/utils/Database.h @@ -18,6 +18,8 @@ namespace CAROM { +bool fileExists(const std::string& name); + /** * Class Database is an abstract base class that provides basic ability to * write to and read from a file. It's capabilities are limited to what the @@ -47,7 +49,7 @@ class Database virtual bool create( - const std::string& file_name) = 0; + const std::string& file_name); /** * @brief Opens an existing database file with the supplied name. @@ -61,7 +63,7 @@ class Database bool open( const std::string& file_name, - const std::string& type) = 0; + const std::string& type); /** * @brief Closes the currently open database file. diff --git a/lib/utils/HDFDatabase.cpp b/lib/utils/HDFDatabase.cpp index 7f98f4e11..673723cca 100644 --- a/lib/utils/HDFDatabase.cpp +++ b/lib/utils/HDFDatabase.cpp @@ -44,6 +44,8 @@ bool HDFDatabase::create( const std::string& file_name) { + Database::create(file_name); + CAROM_VERIFY(!file_name.empty()); hid_t file_id = H5Fcreate(file_name.c_str(), H5F_ACC_TRUNC, @@ -63,6 +65,8 @@ HDFDatabase::open( const std::string& file_name, const std::string& type) { + Database::open(file_name, type); + CAROM_VERIFY(!file_name.empty()); CAROM_VERIFY(type == "r" || type == "wr"); hid_t file_id; diff --git a/lib/utils/HDFDatabase.h b/lib/utils/HDFDatabase.h index 35cbc0070..d91e707d6 100644 --- a/lib/utils/HDFDatabase.h +++ b/lib/utils/HDFDatabase.h @@ -46,7 +46,7 @@ class HDFDatabase : public Database virtual bool create( - const std::string& file_name); + const std::string& file_name) override; /** * @brief Opens an existing HDF5 database file with the supplied name. @@ -60,7 +60,7 @@ class HDFDatabase : public Database bool open( const std::string& file_name, - const std::string& type); + const std::string& type) override; /** * @brief Closes the currently open HDF5 database file. From e5c777918f8dd6f2cb096932cd5ebbf5c8c3e29c Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Tue, 16 Jan 2024 18:23:41 -0800 Subject: [PATCH 02/50] stylization. --- lib/utils/Database.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/utils/Database.cpp b/lib/utils/Database.cpp index 53e240dd9..49c6c9eb7 100644 --- a/lib/utils/Database.cpp +++ b/lib/utils/Database.cpp @@ -18,9 +18,9 @@ namespace CAROM { bool fileExists(const std::string& name) { - std::ifstream f(name.c_str()); - return f.good(); - // ifstream f will be closed upon the end of the function. + std::ifstream f(name.c_str()); + return f.good(); + // ifstream f will be closed upon the end of the function. } Database::Database() From fd53d75a79af9c4845594c1f897980673284b0ba Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Tue, 16 Jan 2024 18:48:53 -0800 Subject: [PATCH 03/50] HDFDatabase::putIntegerArray - overwrites if the dataset exists. --- lib/linalg/BasisWriter.cpp | 3 ++- lib/utils/HDFDatabase.cpp | 35 +++++++++++++++++++++++------------ 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/lib/linalg/BasisWriter.cpp b/lib/linalg/BasisWriter.cpp index 7c4b14402..3a054786a 100644 --- a/lib/linalg/BasisWriter.cpp +++ b/lib/linalg/BasisWriter.cpp @@ -80,7 +80,8 @@ BasisWriter::writeBasis(const std::string& kind) sprintf(tmp, "time_%06d", d_num_intervals_written); if (kind == "basis") { - if (fileExists(full_file_name)) + bool file_exists = fileExists(full_file_name); + if (file_exists) d_database->open(full_file_name, "wr"); else d_database->create(full_file_name); diff --git a/lib/utils/HDFDatabase.cpp b/lib/utils/HDFDatabase.cpp index 673723cca..a40d3b8ee 100644 --- a/lib/utils/HDFDatabase.cpp +++ b/lib/utils/HDFDatabase.cpp @@ -12,6 +12,7 @@ #include "HDFDatabase.h" #include "Utilities.h" +#include namespace CAROM { @@ -123,21 +124,31 @@ HDFDatabase::putIntegerArray( hid_t space = H5Screate_simple(1, dim, 0); CAROM_VERIFY(space >= 0); + const bool key_exists = (H5Lexists(d_group_id, key.c_str(), H5P_DEFAULT) > 0); + hid_t dataset; + if (key_exists) + { + std::cout << "HDF5Database: overwriting dataset " << key << std::endl; + dataset = H5Dopen(d_group_id, key.c_str(), H5P_DEFAULT); + } + else + { #if (H5_VERS_MAJOR > 1) || ((H5_VERS_MAJOR == 1) && (H5_VERS_MINOR > 6)) - hid_t dataset = H5Dcreate(d_group_id, - key.c_str(), - H5T_STD_I32BE, - space, - H5P_DEFAULT, - H5P_DEFAULT, - H5P_DEFAULT); + dataset = H5Dcreate(d_group_id, + key.c_str(), + H5T_STD_I32BE, + space, + H5P_DEFAULT, + H5P_DEFAULT, + H5P_DEFAULT); #else - hid_t dataset = H5Dcreate(d_group_id, - key.c_str(), - H5T_STD_I32BE, - space, - H5P_DEFAULT); + dataset = H5Dcreate(d_group_id, + key.c_str(), + H5T_STD_I32BE, + space, + H5P_DEFAULT); #endif + } CAROM_VERIFY(dataset >= 0); herr_t errf = H5Dwrite(dataset, From 302823aac51a5b6a98e29a392ab3d088a6bef68b Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Thu, 8 Feb 2024 16:36:27 -0800 Subject: [PATCH 04/50] enforce single time interval in Options. --- lib/linalg/Options.h | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/linalg/Options.h b/lib/linalg/Options.h index a7c6a1fb7..6b6b15f11 100644 --- a/lib/linalg/Options.h +++ b/lib/linalg/Options.h @@ -50,7 +50,15 @@ class Options samples_per_time_interval(samples_per_time_interval_), max_time_intervals(max_time_intervals_), update_right_SV(update_right_SV_), - write_snapshots(write_snapshots_) {}; + write_snapshots(write_snapshots_) + { + if (max_time_intervals > 1) + { + printf("time interval is obsolete and will be removed in the future. Set max_time_intervals=%d to 1 or -1!\n", + max_time_intervals); + } + CAROM_VERIFY(max_time_intervals <= 1); + }; /** * @brief Sets the maximum basis dimension of the SVD algorithm. From 14cbb89ec749490b93fdbf00435b9b80debb09e3 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Thu, 8 Feb 2024 16:47:43 -0800 Subject: [PATCH 05/50] HDFDatabase::putIntegerArray does not allow overwrite. --- lib/utils/HDFDatabase.cpp | 34 ++++++++++++---------------------- 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/lib/utils/HDFDatabase.cpp b/lib/utils/HDFDatabase.cpp index a40d3b8ee..60bd0ad69 100644 --- a/lib/utils/HDFDatabase.cpp +++ b/lib/utils/HDFDatabase.cpp @@ -124,31 +124,21 @@ HDFDatabase::putIntegerArray( hid_t space = H5Screate_simple(1, dim, 0); CAROM_VERIFY(space >= 0); - const bool key_exists = (H5Lexists(d_group_id, key.c_str(), H5P_DEFAULT) > 0); - hid_t dataset; - if (key_exists) - { - std::cout << "HDF5Database: overwriting dataset " << key << std::endl; - dataset = H5Dopen(d_group_id, key.c_str(), H5P_DEFAULT); - } - else - { #if (H5_VERS_MAJOR > 1) || ((H5_VERS_MAJOR == 1) && (H5_VERS_MINOR > 6)) - dataset = H5Dcreate(d_group_id, - key.c_str(), - H5T_STD_I32BE, - space, - H5P_DEFAULT, - H5P_DEFAULT, - H5P_DEFAULT); + hid_t dataset = H5Dcreate(d_group_id, + key.c_str(), + H5T_STD_I32BE, + space, + H5P_DEFAULT, + H5P_DEFAULT, + H5P_DEFAULT); #else - dataset = H5Dcreate(d_group_id, - key.c_str(), - H5T_STD_I32BE, - space, - H5P_DEFAULT); + hid_t dataset = H5Dcreate(d_group_id, + key.c_str(), + H5T_STD_I32BE, + space, + H5P_DEFAULT); #endif - } CAROM_VERIFY(dataset >= 0); herr_t errf = H5Dwrite(dataset, From b7993175bd647a94cad1c92a97e0c17143c17f9f Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Thu, 8 Feb 2024 16:53:11 -0800 Subject: [PATCH 06/50] BasisWriter::writeBasis always create the file, which will overwrite the exisiting file. --- lib/linalg/BasisWriter.cpp | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/lib/linalg/BasisWriter.cpp b/lib/linalg/BasisWriter.cpp index 3a054786a..cdde9b731 100644 --- a/lib/linalg/BasisWriter.cpp +++ b/lib/linalg/BasisWriter.cpp @@ -81,10 +81,7 @@ BasisWriter::writeBasis(const std::string& kind) if (kind == "basis") { bool file_exists = fileExists(full_file_name); - if (file_exists) - d_database->open(full_file_name, "wr"); - else - d_database->create(full_file_name); + d_database->create(full_file_name); d_database->putDouble(tmp, time_interval_start_time); @@ -124,10 +121,7 @@ BasisWriter::writeBasis(const std::string& kind) } if (kind == "snapshot") { - if (fileExists(snap_file_name)) - d_snap_database->open(snap_file_name, "wr"); - else - d_snap_database->create(snap_file_name); + d_snap_database->create(snap_file_name); d_snap_database->putDouble(tmp, time_interval_start_time); From 6d2067a9f06e25606176491aba09cf4e028ea8a8 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Thu, 8 Feb 2024 17:03:38 -0800 Subject: [PATCH 07/50] add a header and stylization. --- lib/linalg/Options.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/linalg/Options.h b/lib/linalg/Options.h index 6b6b15f11..5f8b5060d 100644 --- a/lib/linalg/Options.h +++ b/lib/linalg/Options.h @@ -14,6 +14,8 @@ #ifndef included_Options_h #define included_Options_h +#include "utils/Utilities.h" + namespace CAROM { /** @@ -55,7 +57,7 @@ class Options if (max_time_intervals > 1) { printf("time interval is obsolete and will be removed in the future. Set max_time_intervals=%d to 1 or -1!\n", - max_time_intervals); + max_time_intervals); } CAROM_VERIFY(max_time_intervals <= 1); }; From dc6304216a9cc94e95089e64dcc08d1a7c91780b Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Thu, 8 Feb 2024 17:13:33 -0800 Subject: [PATCH 08/50] remove increase time interval test, as time interval is fixed to 1. --- unit_tests/test_SVD.cpp | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/unit_tests/test_SVD.cpp b/unit_tests/test_SVD.cpp index 68a7c70fc..daa17f5bb 100644 --- a/unit_tests/test_SVD.cpp +++ b/unit_tests/test_SVD.cpp @@ -212,20 +212,6 @@ TEST(SVDSerialTest, Test_getBasisIntervalStartTime) EXPECT_DOUBLE_EQ(svd.getBasisIntervalStartTime(2), 2); } - -TEST(SVDSerialTest, Test_increaseTimeInterval) -{ - FakeSVD svd(CAROM::Options(5, 2, 2)); - - ASSERT_NO_THROW(svd.takeSample(NULL, 0, true)); - ASSERT_NO_THROW(svd.takeSample(NULL, 0.5, true)); - ASSERT_NO_THROW(svd.takeSample(NULL, 1, true)); - ASSERT_NO_THROW(svd.takeSample(NULL, 1.5, true)); - - /* The maximum number of time intervals is surpassed */ - EXPECT_DEATH(svd.takeSample(NULL, 2, true), ".*"); -} - int main(int argc, char* argv[]) { ::testing::InitGoogleTest(&argc, argv); From 8d84c1b08c6d517a98bd94eded5fa28c80399fe6 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Thu, 15 Feb 2024 11:57:55 -0800 Subject: [PATCH 09/50] add an error message for a guidance. --- lib/linalg/svd/SVD.h | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/linalg/svd/SVD.h b/lib/linalg/svd/SVD.h index 9ce034843..c56ef47d1 100644 --- a/lib/linalg/svd/SVD.h +++ b/lib/linalg/svd/SVD.h @@ -164,6 +164,7 @@ class SVD void increaseTimeInterval() { + CAROM_ERROR("SVD::increaseTimeInterval- Time interval is obsolete and will be removed in the future. You received this error presumably because the number of samples reached its limit.\n"); int num_time_intervals = static_cast(d_time_interval_start_times.size()); CAROM_VERIFY(d_max_time_intervals == -1 || From 85aa02120786bd3dc2340d43469c80053a9f1603 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Thu, 15 Feb 2024 12:27:15 -0800 Subject: [PATCH 10/50] remove test_SVD from ci workflow. --- .github/workflows/run_tests/action.yml | 1 - lib/linalg/Options.h | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/run_tests/action.yml b/.github/workflows/run_tests/action.yml index 34471de9b..7a7ee5b2b 100644 --- a/.github/workflows/run_tests/action.yml +++ b/.github/workflows/run_tests/action.yml @@ -4,7 +4,6 @@ runs: - name: Run unit tests run: | cd ${GITHUB_WORKSPACE}/build - ./tests/test_SVD ./tests/test_Vector ./tests/test_Matrix mpirun -n 3 --oversubscribe ./tests/test_Matrix diff --git a/lib/linalg/Options.h b/lib/linalg/Options.h index 5f8b5060d..be36bff52 100644 --- a/lib/linalg/Options.h +++ b/lib/linalg/Options.h @@ -44,7 +44,7 @@ class Options */ Options(int dim_, int samples_per_time_interval_, - int max_time_intervals_ = -1, + int max_time_intervals_ = 1, bool update_right_SV_ = false, bool write_snapshots_ = false ): dim(dim_), @@ -56,10 +56,10 @@ class Options { if (max_time_intervals > 1) { - printf("time interval is obsolete and will be removed in the future. Set max_time_intervals=%d to 1 or -1!\n", + printf("time interval is obsolete and will be removed in the future. Set max_time_intervals=%d to 1!\n", max_time_intervals); } - CAROM_VERIFY(max_time_intervals <= 1); + CAROM_VERIFY(max_time_intervals == 1); }; /** From 3c3c6670dc0ca70e7fc2e7f4f0e5a90914a73d42 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Thu, 15 Feb 2024 12:45:08 -0800 Subject: [PATCH 11/50] SVD::increaseTimeInterval - allow the initial time interval. --- lib/linalg/svd/SVD.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/linalg/svd/SVD.h b/lib/linalg/svd/SVD.h index c56ef47d1..cb9fb1fbf 100644 --- a/lib/linalg/svd/SVD.h +++ b/lib/linalg/svd/SVD.h @@ -164,11 +164,11 @@ class SVD void increaseTimeInterval() { - CAROM_ERROR("SVD::increaseTimeInterval- Time interval is obsolete and will be removed in the future. You received this error presumably because the number of samples reached its limit.\n"); int num_time_intervals = static_cast(d_time_interval_start_times.size()); - CAROM_VERIFY(d_max_time_intervals == -1 || - num_time_intervals < d_max_time_intervals); + CAROM_VERIFY(d_max_time_intervals == 1); + if (num_time_intervals > 0) + CAROM_ERROR("SVD::increaseTimeInterval- Time interval is obsolete and will be removed in the future. You received this error presumably because the number of samples reached its limit.\n"); d_time_interval_start_times.resize( static_cast(num_time_intervals) + 1); } From 8371e85c99e5065b35afd74b51292ca37e4064d6 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Thu, 15 Feb 2024 13:19:14 -0800 Subject: [PATCH 12/50] minor fix in test_IncrementalSVDBrand. --- unit_tests/test_IncrementalSVDBrand.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unit_tests/test_IncrementalSVDBrand.cpp b/unit_tests/test_IncrementalSVDBrand.cpp index 014c05021..a18d842d7 100644 --- a/unit_tests/test_IncrementalSVDBrand.cpp +++ b/unit_tests/test_IncrementalSVDBrand.cpp @@ -82,7 +82,7 @@ TEST(IncrementalSVDBrandTest, Test_IncrementalSVDBrand) bool fast_update = true; bool fast_update_brand = true; - CAROM::Options incremental_svd_options = CAROM::Options(d_num_rows, 3, -1, true) + CAROM::Options incremental_svd_options = CAROM::Options(d_num_rows, 3, 1, true) .setMaxBasisDimension(num_total_rows) .setIncrementalSVD(1e-1, 1e-1, From bf1e126afc18803f6d9b6f2d64d518e53a4df970 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Fri, 16 Feb 2024 09:32:52 -0800 Subject: [PATCH 13/50] reflecting the comments. --- lib/linalg/BasisWriter.cpp | 18 ++++++++++-------- lib/linalg/Options.h | 4 ++-- lib/linalg/svd/SVD.h | 5 ++++- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/lib/linalg/BasisWriter.cpp b/lib/linalg/BasisWriter.cpp index cdde9b731..3e3402a73 100644 --- a/lib/linalg/BasisWriter.cpp +++ b/lib/linalg/BasisWriter.cpp @@ -55,17 +55,15 @@ BasisWriter::BasisWriter( snap_file_name = base_file_name + tmp2; // create and open snapshot/basis database - // TODO(kevin): can it be CSV at all? we might want to remove if statement here. - if (db_format_ == Database::HDF5) { - d_snap_database = new HDFDatabase(); - d_database = new HDFDatabase(); - } + CAROM_VERIFY(db_format_ == Database::HDF5); + d_snap_database = new HDFDatabase(); + d_database = new HDFDatabase(); } BasisWriter::~BasisWriter() { - if (d_database) delete d_database; - if (d_snap_database) delete d_snap_database; + delete d_database; + delete d_snap_database; } void @@ -80,7 +78,6 @@ BasisWriter::writeBasis(const std::string& kind) sprintf(tmp, "time_%06d", d_num_intervals_written); if (kind == "basis") { - bool file_exists = fileExists(full_file_name); d_database->create(full_file_name); d_database->putDouble(tmp, time_interval_start_time); @@ -116,6 +113,7 @@ BasisWriter::writeBasis(const std::string& kind) ++d_num_intervals_written; + CAROM_VERIFY(d_num_intervals_written == 1); d_database->putInteger("num_time_intervals", d_num_intervals_written); d_database->close(); } @@ -135,6 +133,10 @@ BasisWriter::writeBasis(const std::string& kind) sprintf(tmp, "snapshot_matrix_%06d", d_num_intervals_written); d_snap_database->putDoubleArray(tmp, &snapshots->item(0,0), num_rows*num_cols); + // NOTE(kevin): number of interval will be removed in future. + // Until then, we keep the current practice of + // un-increased number of interval, that is 0. + CAROM_VERIFY(d_num_intervals_written == 0); d_snap_database->putInteger("num_time_intervals", d_num_intervals_written); d_snap_database->close(); } diff --git a/lib/linalg/Options.h b/lib/linalg/Options.h index be36bff52..816002d2b 100644 --- a/lib/linalg/Options.h +++ b/lib/linalg/Options.h @@ -56,8 +56,8 @@ class Options { if (max_time_intervals > 1) { - printf("time interval is obsolete and will be removed in the future. Set max_time_intervals=%d to 1!\n", - max_time_intervals); + printf("time interval is obsolete and will be removed in the future." + " Set max_time_intervals=%d to 1!\n", max_time_intervals); } CAROM_VERIFY(max_time_intervals == 1); }; diff --git a/lib/linalg/svd/SVD.h b/lib/linalg/svd/SVD.h index cb9fb1fbf..de2c6f389 100644 --- a/lib/linalg/svd/SVD.h +++ b/lib/linalg/svd/SVD.h @@ -168,7 +168,10 @@ class SVD static_cast(d_time_interval_start_times.size()); CAROM_VERIFY(d_max_time_intervals == 1); if (num_time_intervals > 0) - CAROM_ERROR("SVD::increaseTimeInterval- Time interval is obsolete and will be removed in the future. You received this error presumably because the number of samples reached its limit.\n"); + CAROM_ERROR("SVD::increaseTimeInterval- Time interval is obsolete" + " and will be removed in the future. " + "You received this error presumably " + "because the number of samples reached its limit.\n"); d_time_interval_start_times.resize( static_cast(num_time_intervals) + 1); } From cf6f27c77be0ba5adea4c24f88e96b537f9705dd Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Sat, 17 Feb 2024 12:00:34 -0800 Subject: [PATCH 14/50] removed the concept of time interval in BasisReader. time argument remains for backward compatibility. --- lib/linalg/BasisReader.cpp | 222 ++++++++++++------------------------- lib/linalg/BasisReader.h | 147 ++++++++++++------------ 2 files changed, 150 insertions(+), 219 deletions(-) diff --git a/lib/linalg/BasisReader.cpp b/lib/linalg/BasisReader.cpp index d2f0464aa..60d8c44a1 100644 --- a/lib/linalg/BasisReader.cpp +++ b/lib/linalg/BasisReader.cpp @@ -49,16 +49,6 @@ BasisReader::BasisReader( } d_database->open(full_file_name, "r"); - - int num_time_intervals; - double foo; - d_database->getDouble("num_time_intervals", foo); - num_time_intervals = static_cast(foo); - d_time_interval_start_times.resize(num_time_intervals); - for (int i = 0; i < num_time_intervals; ++i) { - sprintf(tmp, "time_%06d", i); - d_database->getDouble(tmp, d_time_interval_start_times[i]); - } } BasisReader::~BasisReader() @@ -71,24 +61,15 @@ Matrix* BasisReader::getSpatialBasis( double time) { - CAROM_ASSERT(0 < numTimeIntervals()); - CAROM_ASSERT(0 <= time); - int num_time_intervals = numTimeIntervals(); - int i; - for (i = 0; i < num_time_intervals-1; ++i) { - if (d_time_interval_start_times[i] <= time && - time < d_time_interval_start_times[i+1]) { - break; - } - } - d_last_basis_idx = i; - int num_rows = getDim("basis",time); - int num_cols = getNumSamples("basis",time); + d_last_basis_idx = 0; + int num_rows = getDim("basis"); + int num_cols = getNumSamples("basis"); - char tmp[100]; Matrix* spatial_basis_vectors = new Matrix(num_rows, num_cols, true); - sprintf(tmp, "spatial_basis_%06d", i); - d_database->getDoubleArray(tmp, + + // This 0 index is the remainder from time interval concept. + // This remains only for backward compatibility purpose. + d_database->getDoubleArray("spatial_basis_000000", &spatial_basis_vectors->item(0, 0), num_rows*num_cols); return spatial_basis_vectors; @@ -99,7 +80,7 @@ BasisReader::getSpatialBasis( double time, int n) { - return getSpatialBasis(time, 1, n); + return getSpatialBasis(1, n); } Matrix* @@ -108,19 +89,9 @@ BasisReader::getSpatialBasis( int start_col, int end_col) { - CAROM_ASSERT(0 < numTimeIntervals()); - CAROM_ASSERT(0 <= time); - int num_time_intervals = numTimeIntervals(); - int i; - for (i = 0; i < num_time_intervals-1; ++i) { - if (d_time_interval_start_times[i] <= time && - time < d_time_interval_start_times[i+1]) { - break; - } - } - d_last_basis_idx = i; - int num_rows = getDim("basis",time); - int num_cols = getNumSamples("basis",time); + d_last_basis_idx = 0; + int num_rows = getDim("basis"); + int num_cols = getNumSamples("basis"); char tmp[100]; CAROM_VERIFY(0 < start_col <= num_cols); @@ -128,7 +99,9 @@ BasisReader::getSpatialBasis( int num_cols_to_read = end_col - start_col + 1; Matrix* spatial_basis_vectors = new Matrix(num_rows, num_cols_to_read, true); - sprintf(tmp, "spatial_basis_%06d", i); + // This 0 index is the remainder from time interval concept. + // This remains only for backward compatibility purpose. + sprintf(tmp, "spatial_basis_000000"); d_database->getDoubleArray(tmp, &spatial_basis_vectors->item(0, 0), num_rows*num_cols_to_read, @@ -143,7 +116,7 @@ BasisReader::getSpatialBasis( double time, double ef) { - Vector* sv = getSingularValues(time); + Vector* sv = getSingularValues(); double total_energy = 0.0; double energy = 0.0; for (int i = 0; i < sv->dim(); i++) @@ -163,30 +136,22 @@ BasisReader::getSpatialBasis( } delete sv; - return getSpatialBasis(time, num_used_singular_values); + return getSpatialBasis(num_used_singular_values); } Matrix* BasisReader::getTemporalBasis( double time) { - CAROM_ASSERT(0 < numTimeIntervals()); - CAROM_ASSERT(0 <= time); - int num_time_intervals = numTimeIntervals(); - int i; - for (i = 0; i < num_time_intervals-1; ++i) { - if (d_time_interval_start_times[i] <= time && - time < d_time_interval_start_times[i+1]) { - break; - } - } - d_last_basis_idx = i; - int num_rows = getDim("temporal_basis",time); - int num_cols = getNumSamples("temporal_basis",time); + d_last_basis_idx = 0; + int num_rows = getDim("temporal_basis"); + int num_cols = getNumSamples("temporal_basis"); char tmp[100]; Matrix* temporal_basis_vectors = new Matrix(num_rows, num_cols, true); - sprintf(tmp, "temporal_basis_%06d", i); + // This 0 index is the remainder from time interval concept. + // This remains only for backward compatibility purpose. + sprintf(tmp, "temporal_basis_000000"); d_database->getDoubleArray(tmp, &temporal_basis_vectors->item(0, 0), num_rows*num_cols); @@ -198,7 +163,7 @@ BasisReader::getTemporalBasis( double time, int n) { - return getTemporalBasis(time, 1, n); + return getTemporalBasis(1, n); } Matrix* @@ -207,19 +172,9 @@ BasisReader::getTemporalBasis( int start_col, int end_col) { - CAROM_ASSERT(0 < numTimeIntervals()); - CAROM_ASSERT(0 <= time); - int num_time_intervals = numTimeIntervals(); - int i; - for (i = 0; i < num_time_intervals-1; ++i) { - if (d_time_interval_start_times[i] <= time && - time < d_time_interval_start_times[i+1]) { - break; - } - } - d_last_basis_idx = i; - int num_rows = getDim("temporal_basis",time); - int num_cols = getNumSamples("temporal_basis",time); + d_last_basis_idx = 0; + int num_rows = getDim("temporal_basis"); + int num_cols = getNumSamples("temporal_basis"); char tmp[100]; CAROM_VERIFY(0 < start_col <= num_cols); @@ -227,7 +182,9 @@ BasisReader::getTemporalBasis( int num_cols_to_read = end_col - start_col + 1; Matrix* temporal_basis_vectors = new Matrix(num_rows, num_cols_to_read, true); - sprintf(tmp, "temporal_basis_%06d", i); + // This 0 index is the remainder from time interval concept. + // This remains only for backward compatibility purpose. + sprintf(tmp, "temporal_basis_000000"); d_database->getDoubleArray(tmp, &temporal_basis_vectors->item(0, 0), num_rows*num_cols_to_read, @@ -242,7 +199,7 @@ BasisReader::getTemporalBasis( double time, double ef) { - Vector* sv = getSingularValues(time); + Vector* sv = getSingularValues(); double total_energy = 0.0; double energy = 0.0; for (int i = 0; i < sv->dim(); i++) @@ -262,31 +219,24 @@ BasisReader::getTemporalBasis( } delete sv; - return getTemporalBasis(time, num_used_singular_values); + return getTemporalBasis(num_used_singular_values); } Vector* -BasisReader::getSingularValues( - double time) +BasisReader::getSingularValues() { - CAROM_ASSERT(0 < numTimeIntervals()); - CAROM_ASSERT(0 <= time); - int num_time_intervals = numTimeIntervals(); - int i; - for (i = 0; i < num_time_intervals-1; ++i) { - if (d_time_interval_start_times[i] <= time && - time < d_time_interval_start_times[i+1]) { - break; - } - } - d_last_basis_idx = i; + d_last_basis_idx = 0; char tmp[100]; int size; - sprintf(tmp, "singular_value_size_%06d", i); + // This 0 index is the remainder from time interval concept. + // This remains only for backward compatibility purpose. + sprintf(tmp, "singular_value_size_000000"); d_database->getInteger(tmp, size); Vector* singular_values = new Vector(size, false); - sprintf(tmp, "singular_value_%06d", i); + // This 0 index is the remainder from time interval concept. + // This remains only for backward compatibility purpose. + sprintf(tmp, "singular_value_000000"); d_database->getDoubleArray(tmp, &singular_values->item(0), size); @@ -298,7 +248,7 @@ BasisReader::getSingularValues( double time, double ef) { - Vector* sv = getSingularValues(time); + Vector* sv = getSingularValues(); double total_energy = 0.0; double energy = 0.0; for (int i = 0; i < sv->dim(); i++) @@ -330,60 +280,46 @@ BasisReader::getSingularValues( int BasisReader::getDim( - const std::string kind, - double time) + const std::string kind) { - CAROM_ASSERT(0 < numTimeIntervals()); - CAROM_ASSERT(0 <= time); CAROM_ASSERT((kind == "basis") || (kind == "snapshot") || (kind == "temporal_basis")); - int num_time_intervals = numTimeIntervals(); - int i; - for (i = 0; i < num_time_intervals-1; ++i) { - if (d_time_interval_start_times[i] <= time && - time < d_time_interval_start_times[i+1]) { - break; - } - } - d_last_basis_idx = i; + d_last_basis_idx = 0; char tmp[100]; int num_rows; - if (kind == "basis") sprintf(tmp, "spatial_basis_num_rows_%06d", i); - else if (kind == "snapshot") sprintf(tmp, "snapshot_matrix_num_rows_%06d", i); - else if (kind == "temporal_basis") sprintf(tmp, "temporal_basis_num_rows_%06d", - i); + + // This 0 index is the remainder from time interval concept. + // This remains only for backward compatibility purpose. + if (kind == "basis") sprintf(tmp, "spatial_basis_num_rows_000000"); + else if (kind == "snapshot") sprintf(tmp, "snapshot_matrix_num_rows_000000"); + else if (kind == "temporal_basis") sprintf(tmp, + "temporal_basis_num_rows_000000"); + d_database->getInteger(tmp, num_rows); return num_rows; } int BasisReader::getNumSamples( - const std::string kind, - double time) + const std::string kind) { - CAROM_ASSERT(0 < numTimeIntervals()); - CAROM_ASSERT(0 <= time); CAROM_ASSERT((kind == "basis") || (kind == "snapshot") || (kind == "temporal_basis")); - int num_time_intervals = numTimeIntervals(); - int i; - for (i = 0; i < num_time_intervals-1; ++i) { - if (d_time_interval_start_times[i] <= time && - time < d_time_interval_start_times[i+1]) { - break; - } - } - d_last_basis_idx = i; + d_last_basis_idx = 0; char tmp[100]; int num_cols; - if (kind == "basis") sprintf(tmp, "spatial_basis_num_cols_%06d", i); - else if (kind == "snapshot") sprintf(tmp, "snapshot_matrix_num_cols_%06d", i); - else if (kind == "temporal_basis") sprintf(tmp, "temporal_basis_num_cols_%06d", - i); + + // This 0 index is the remainder from time interval concept. + // This remains only for backward compatibility purpose. + if (kind == "basis") sprintf(tmp, "spatial_basis_num_cols_000000"); + else if (kind == "snapshot") sprintf(tmp, "snapshot_matrix_num_cols_000000"); + else if (kind == "temporal_basis") sprintf(tmp, + "temporal_basis_num_cols_000000"); + d_database->getInteger(tmp, num_cols); return num_cols; } @@ -392,23 +328,15 @@ Matrix* BasisReader::getSnapshotMatrix( double time) { - CAROM_ASSERT(0 < numTimeIntervals()); - CAROM_ASSERT(0 <= time); - int num_time_intervals = numTimeIntervals(); - int i; - for (i = 0; i < num_time_intervals-1; ++i) { - if (d_time_interval_start_times[i] <= time && - time < d_time_interval_start_times[i+1]) { - break; - } - } - d_last_basis_idx = i; + d_last_basis_idx = 0; int num_rows = getDim("snapshot",time); int num_cols = getNumSamples("snapshot",time); char tmp[100]; Matrix* snapshots = new Matrix(num_rows, num_cols, false); - sprintf(tmp, "snapshot_matrix_%06d", i); + // This 0 index is the remainder from time interval concept. + // This remains only for backward compatibility purpose. + sprintf(tmp, "snapshot_matrix_000000"); d_database->getDoubleArray(tmp, &snapshots->item(0, 0), num_rows*num_cols); @@ -420,7 +348,7 @@ BasisReader::getSnapshotMatrix( double time, int n) { - return getSnapshotMatrix(time, 1, n); + return getSnapshotMatrix(1, n); } Matrix* @@ -429,19 +357,9 @@ BasisReader::getSnapshotMatrix( int start_col, int end_col) { - CAROM_ASSERT(0 < numTimeIntervals()); - CAROM_ASSERT(0 <= time); - int num_time_intervals = numTimeIntervals(); - int i; - for (i = 0; i < num_time_intervals-1; ++i) { - if (d_time_interval_start_times[i] <= time && - time < d_time_interval_start_times[i+1]) { - break; - } - } - d_last_basis_idx = i; - int num_rows = getDim("snapshot",time); - int num_cols = getNumSamples("snapshot",time); + d_last_basis_idx = 0; + int num_rows = getDim("snapshot"); + int num_cols = getNumSamples("snapshot"); CAROM_VERIFY(0 < start_col <= num_cols); CAROM_VERIFY(start_col <= end_col && end_col <= num_cols); @@ -449,7 +367,9 @@ BasisReader::getSnapshotMatrix( char tmp[100]; Matrix* snapshots = new Matrix(num_rows, num_cols_to_read, false); - sprintf(tmp, "snapshot_matrix_%06d", i); + // This 0 index is the remainder from time interval concept. + // This remains only for backward compatibility purpose. + sprintf(tmp, "snapshot_matrix_000000"); d_database->getDoubleArray(tmp, &snapshots->item(0, 0), num_rows*num_cols_to_read, diff --git a/lib/linalg/BasisReader.h b/lib/linalg/BasisReader.h index 5cbfb27ef..7a01ddb4e 100644 --- a/lib/linalg/BasisReader.h +++ b/lib/linalg/BasisReader.h @@ -58,10 +58,9 @@ class BasisReader { * @brief Returns true if the basis vectors at requested time are * different from the last requested basis vectors. * - * @pre 0 < numTimeIntervals() - * @pre 0 <= time - * * @param[in] time Time at which we are interested in the basis vectors. + * NOTE: this argument is obsolete and remains only for backward compatibility. + * Will be removed in future. * * @return True if the basis vectors at the requested time are different * from the last requested basis vectors. @@ -70,24 +69,7 @@ class BasisReader { isNewBasis( double time) { - CAROM_VERIFY(0 < numTimeIntervals()); - CAROM_VERIFY(0 <= time); - bool result = false; - if (d_last_basis_idx == -1) { - result = true; - } - else { - int num_time_intervals = numTimeIntervals(); - int i; - for (i = 0; i < num_time_intervals-1; ++i) { - if (d_time_interval_start_times[i] <= time && - time < d_time_interval_start_times[i+1]) { - break; - } - } - result = i != d_last_basis_idx; - } - return result; + return (d_last_basis_idx == -1); } /** @@ -95,10 +77,9 @@ class BasisReader { * @brief Returns the spatial basis vectors for the requested time as a * Matrix. * - * @pre 0 < numTimeIntervals() - * @pre 0 <= time - * * @param[in] time Time for which we want the basis vectors. + * NOTE: this argument is obsolete and remains only for backward compatibility. + * Will be removed in future. * * @return The spatial basis vectors for the requested time. */ @@ -111,11 +92,11 @@ class BasisReader { * @brief Returns the first n spatial basis vectors for the requested time * as a Matrix. * - * @pre 0 < numTimeIntervals() - * @pre 0 <= time * @pre 0 < n <= numColumns() * * @param[in] time Time for which we want the basis vectors. + * NOTE: this argument is obsolete and remains only for backward compatibility. + * Will be removed in future. * @param[in] n The number of spatial basis vectors desired. * * @return The spatial basis vectors for the requested time. @@ -130,12 +111,12 @@ class BasisReader { * @brief Returns spatial basis vectors from start_col to end_col for the * requested time as a Matrix. * - * @pre 0 < numTimeIntervals() - * @pre 0 <= time * @pre 0 < start_col <= numColumns() * @pre start_col <= end_col <= numColumns() * * @param[in] time Time for which we want the basis vectors. + * NOTE: this argument is obsolete and remains only for backward compatibility. + * Will be removed in future. * @param[in] start_col The starting column desired. * @param[in] end_col The starting column desired. * @@ -152,11 +133,11 @@ class BasisReader { * @brief Returns the first n spatial basis vectors for the requested time * as a Matrix that capture the given energy fraction. * - * @pre 0 < numTimeIntervals() - * @pre 0 <= time * @pre 0 <= ef <= 1.0 * * @param[in] time Time for which we want the basis vectors. + * NOTE: this argument is obsolete and remains only for backward compatibility. + * Will be removed in future. * @param[in] ef The desired energy fraction. * * @return The spatial basis vectors for the requested time. @@ -171,10 +152,9 @@ class BasisReader { * @brief Returns the temporal basis vectors for the requested time as * a Matrix. * - * @pre 0 < numTimeIntervals() - * @pre 0 <= time - * * @param[in] time Time for which we want the basis vectors. + * NOTE: this argument is obsolete and remains only for backward compatibility. + * Will be removed in future. * * @return The temporal basis vectors for the requested time. */ @@ -187,11 +167,11 @@ class BasisReader { * @brief Returns the first n temporal basis vectors for the requested time * as a Matrix. * - * @pre 0 < numTimeIntervals() - * @pre 0 <= time * @pre 0 < n <= numColumns() * * @param[in] time Time for which we want the basis vectors. + * NOTE: this argument is obsolete and remains only for backward compatibility. + * Will be removed in future. * @param[in] n The number of temporal basis vectors desired. * * @return The temporal basis vectors for the requested time. @@ -206,12 +186,12 @@ class BasisReader { * @brief Returns temporal basis vectors from start_col to end_col for the * requested time as a Matrix. * - * @pre 0 < numTimeIntervals() - * @pre 0 <= time * @pre 0 < start_col <= numColumns() * @pre start_col <= end_col <= numColumns() * * @param[in] time Time for which we want the basis vectors. + * NOTE: this argument is obsolete and remains only for backward compatibility. + * Will be removed in future. * @param[in] start_col The starting column desired. * @param[in] end_col The starting column desired. * @@ -228,11 +208,11 @@ class BasisReader { * @brief Returns the first n temporal basis vectors for the requested time * as a Matrix that capture the given energy fraction. * - * @pre 0 < numTimeIntervals() - * @pre 0 <= time * @pre 0 <= ef <= 1.0 * * @param[in] time Time for which we want the basis vectors. + * NOTE: this argument is obsolete and remains only for backward compatibility. + * Will be removed in future. * @param[in] ef The desired energy fraction. * * @return The temporal basis vectors for the requested time. @@ -246,27 +226,44 @@ class BasisReader { * * @brief Returns the singular values for the requested time. * - * @pre 0 < numTimeIntervals() - * @pre 0 <= time + * @param[in] time Time for which we want the basis vectors. + * NOTE: this argument is obsolete and remains only for backward compatibility. + * Will be removed in future. + * + * @return The temporal basis vectors for the requested time. + */ + Vector* + getSingularValues(); + + /** + * + * @brief Returns the singular values for the requested time. + * NOTE: this function is obsolete and remains only for backward compatibility. + * Will be removed in future. * * @param[in] time Time for which we want the basis vectors. + * NOTE: this argument is obsolete and remains only for backward compatibility. + * Will be removed in future. * * @return The temporal basis vectors for the requested time. */ Vector* getSingularValues( - double time); + double time) + { + return getSingularValues(); + } /** * * @brief Returns the largest singular values for the requested time * that capture the given energy fraction. * - * @pre 0 < numTimeIntervals() - * @pre 0 <= time * @pre 0 <= ef <= 1.0 * * @param[in] time Time for which we want the basis vectors. + * NOTE: this argument is obsolete and remains only for backward compatibility. + * Will be removed in future. * @param[in] ef The desired energy fraction. * * @return The temporal basis vectors for the requested time. @@ -283,9 +280,24 @@ class BasisReader { * @return The dimension of the system on this processor. */ int + getDim( + const std::string kind); + + /** + * + * @brief Returns the dimension of the system on this processor. + * NOTE: this function is obsolete and remains only for backward compatibility. + * Will be removed in future. + * + * @return The dimension of the system on this processor. + */ + int getDim( const std::string kind, - double time); + double time) + { + return getDim(kind); + } /** * @@ -294,18 +306,33 @@ class BasisReader { * @return The number of samples in file. */ int + getNumSamples( + const std::string kind); + + /** + * + * @brief Returns the number of samples (columns) in file. + * NOTE: this function is obsolete and remains only for backward compatibility. + * Will be removed in future. + * + * @return The number of samples in file. + */ + int getNumSamples( const std::string kind, - double time); + double time) + { + return getNumSamples(kind); + } /** * * @brief Returns the snapshot matrix for the requested time. * - * @pre 0 < numTimeIntervals() - * @pre 0 <= time * * @param[in] time Time for which we want the basis vectors. + * NOTE: this argument is obsolete and remains only for backward compatibility. + * Will be removed in future. * * @return The snapshot matrix for the requested time. */ @@ -317,11 +344,11 @@ class BasisReader { * * @brief Returns the first n columns of the snapshot matrix for the requested time. * - * @pre 0 < numTimeIntervals() - * @pre 0 <= time * @pre 0 < n <= numColumns() * * @param[in] time Time for which we want the basis vectors. + * NOTE: this argument is obsolete and remains only for backward compatibility. + * Will be removed in future. * @param[in] n The number of basis vectors desired. * * @return The snapshot matrix for the requested time. @@ -335,12 +362,12 @@ class BasisReader { * * @brief Returns the snapshot matrix from start_col to end_col for the requested time. * - * @pre 0 < numTimeIntervals() - * @pre 0 <= time * @pre 0 < start_col <= numColumns() * @pre start_col <= end_col <= numColumns() * * @param[in] time Time for which we want the basis vectors. + * NOTE: this argument is obsolete and remains only for backward compatibility. + * Will be removed in future. * @param[in] start_col The starting column desired. * @param[in] end_col The starting column desired. * @@ -371,22 +398,6 @@ class BasisReader { operator = ( const BasisReader& rhs); - /** - * @brief Number of time intervals. - * - * @return The number of time intervals. - */ - int - numTimeIntervals() - { - return static_cast(d_time_interval_start_times.size()); - } - - /** - * @brief The start time of each time interval. - */ - std::vector d_time_interval_start_times; - /** * @brief The database being read from. */ From 3153eaf80a6322ac9e43b29b82a9750693270a30 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Sat, 17 Feb 2024 12:08:14 -0800 Subject: [PATCH 15/50] BasisWriter: removed the concept of time interval. --- lib/linalg/BasisWriter.cpp | 42 ++++++++++++-------------------------- lib/linalg/BasisWriter.h | 6 ------ 2 files changed, 13 insertions(+), 35 deletions(-) diff --git a/lib/linalg/BasisWriter.cpp b/lib/linalg/BasisWriter.cpp index 3e3402a73..712d5f0c7 100644 --- a/lib/linalg/BasisWriter.cpp +++ b/lib/linalg/BasisWriter.cpp @@ -26,7 +26,6 @@ BasisWriter::BasisWriter( const std::string& base_file_name, Database::formats db_format) : d_basis_generator(basis_generator), - d_num_intervals_written(0), full_file_name(""), snap_file_name(""), db_format_(db_format), @@ -69,75 +68,60 @@ BasisWriter::~BasisWriter() void BasisWriter::writeBasis(const std::string& kind) { - CAROM_ASSERT(kind == "basis" || kind == "snapshot"); char tmp[100]; - double time_interval_start_time = - d_basis_generator->getBasisIntervalStartTime(d_num_intervals_written); - sprintf(tmp, "time_%06d", d_num_intervals_written); + // This 0 index is the remainder from time interval concept. + // This remains only for backward compatibility purpose. if (kind == "basis") { d_database->create(full_file_name); - d_database->putDouble(tmp, time_interval_start_time); - const Matrix* basis = d_basis_generator->getSpatialBasis(); int num_rows = basis->numRows(); - sprintf(tmp, "spatial_basis_num_rows_%06d", d_num_intervals_written); + sprintf(tmp, "spatial_basis_num_rows_000000"); d_database->putInteger(tmp, num_rows); int num_cols = basis->numColumns(); - sprintf(tmp, "spatial_basis_num_cols_%06d", d_num_intervals_written); + sprintf(tmp, "spatial_basis_num_cols_000000"); d_database->putInteger(tmp, num_cols); - sprintf(tmp, "spatial_basis_%06d", d_num_intervals_written); + sprintf(tmp, "spatial_basis_000000"); d_database->putDoubleArray(tmp, &basis->item(0, 0), num_rows*num_cols); if(d_basis_generator->updateRightSV()) { const Matrix* tbasis = d_basis_generator->getTemporalBasis(); num_rows = tbasis->numRows(); - sprintf(tmp, "temporal_basis_num_rows_%06d", d_num_intervals_written); + sprintf(tmp, "temporal_basis_num_rows_000000"); d_database->putInteger(tmp, num_rows); num_cols = tbasis->numColumns(); - sprintf(tmp, "temporal_basis_num_cols_%06d", d_num_intervals_written); + sprintf(tmp, "temporal_basis_num_cols_000000"); d_database->putInteger(tmp, num_cols); - sprintf(tmp, "temporal_basis_%06d", d_num_intervals_written); + sprintf(tmp, "temporal_basis_000000"); d_database->putDoubleArray(tmp, &tbasis->item(0, 0), num_rows*num_cols); } const Vector* sv = d_basis_generator->getSingularValues(); int sv_dim = sv->dim(); - sprintf(tmp, "singular_value_size_%06d", d_num_intervals_written); + sprintf(tmp, "singular_value_size_000000"); d_database->putInteger(tmp, sv_dim); - sprintf(tmp, "singular_value_%06d", d_num_intervals_written); + sprintf(tmp, "singular_value_000000"); d_database->putDoubleArray(tmp, &sv->item(0), sv_dim); - ++d_num_intervals_written; - - CAROM_VERIFY(d_num_intervals_written == 1); - d_database->putInteger("num_time_intervals", d_num_intervals_written); d_database->close(); } if (kind == "snapshot") { d_snap_database->create(snap_file_name); - d_snap_database->putDouble(tmp, time_interval_start_time); - const Matrix* snapshots = d_basis_generator->getSnapshotMatrix(); int num_rows = snapshots->numRows(); // d_dim - sprintf(tmp, "snapshot_matrix_num_rows_%06d", d_num_intervals_written); + sprintf(tmp, "snapshot_matrix_num_rows_000000"); d_snap_database->putInteger(tmp, num_rows); int num_cols = snapshots->numColumns(); // d_num_samples - sprintf(tmp, "snapshot_matrix_num_cols_%06d", d_num_intervals_written); + sprintf(tmp, "snapshot_matrix_num_cols_000000"); d_snap_database->putInteger(tmp, num_cols); - sprintf(tmp, "snapshot_matrix_%06d", d_num_intervals_written); + sprintf(tmp, "snapshot_matrix_000000"); d_snap_database->putDoubleArray(tmp, &snapshots->item(0,0), num_rows*num_cols); - // NOTE(kevin): number of interval will be removed in future. - // Until then, we keep the current practice of - // un-increased number of interval, that is 0. - CAROM_VERIFY(d_num_intervals_written == 0); - d_snap_database->putInteger("num_time_intervals", d_num_intervals_written); d_snap_database->close(); } diff --git a/lib/linalg/BasisWriter.h b/lib/linalg/BasisWriter.h index 8611c1ead..d495b6ef0 100644 --- a/lib/linalg/BasisWriter.h +++ b/lib/linalg/BasisWriter.h @@ -99,12 +99,6 @@ class BasisWriter { */ std::string full_file_name; std::string snap_file_name; - - /** - * @brief Number of time intervals for which basis vectors have been - * written. - */ - int d_num_intervals_written; }; } From 6ffa9082b7d74cb8a6ef51c68ba8f8ee3d6f49ac Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Sat, 17 Feb 2024 12:31:43 -0800 Subject: [PATCH 16/50] minor fix in BasisReader. --- lib/linalg/BasisReader.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/linalg/BasisReader.cpp b/lib/linalg/BasisReader.cpp index 60d8c44a1..0bf3dd074 100644 --- a/lib/linalg/BasisReader.cpp +++ b/lib/linalg/BasisReader.cpp @@ -80,7 +80,7 @@ BasisReader::getSpatialBasis( double time, int n) { - return getSpatialBasis(1, n); + return getSpatialBasis(time, 1, n); } Matrix* @@ -136,7 +136,7 @@ BasisReader::getSpatialBasis( } delete sv; - return getSpatialBasis(num_used_singular_values); + return getSpatialBasis(time, num_used_singular_values); } Matrix* @@ -163,7 +163,7 @@ BasisReader::getTemporalBasis( double time, int n) { - return getTemporalBasis(1, n); + return getTemporalBasis(time, 1, n); } Matrix* @@ -219,7 +219,7 @@ BasisReader::getTemporalBasis( } delete sv; - return getTemporalBasis(num_used_singular_values); + return getTemporalBasis(time, num_used_singular_values); } Vector* @@ -348,7 +348,7 @@ BasisReader::getSnapshotMatrix( double time, int n) { - return getSnapshotMatrix(1, n); + return getSnapshotMatrix(time, 1, n); } Matrix* From 8aa426d0ff9e8a8a4b6fc39de244a88360d71780 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Sat, 17 Feb 2024 13:48:57 -0800 Subject: [PATCH 17/50] SVD: removed the concept of time intervals. --- CMakeLists.txt | 1 - lib/linalg/BasisGenerator.cpp | 18 +- lib/linalg/BasisGenerator.h | 31 --- lib/linalg/svd/IncrementalSVD.cpp | 17 +- lib/linalg/svd/IncrementalSVD.h | 6 +- lib/linalg/svd/IncrementalSVDBrand.cpp | 22 +- lib/linalg/svd/IncrementalSVDBrand.h | 3 +- lib/linalg/svd/IncrementalSVDFastUpdate.cpp | 22 +- lib/linalg/svd/IncrementalSVDFastUpdate.h | 3 +- lib/linalg/svd/IncrementalSVDStandard.cpp | 22 +- lib/linalg/svd/IncrementalSVDStandard.h | 3 +- lib/linalg/svd/RandomizedSVD.cpp | 2 +- lib/linalg/svd/SVD.cpp | 6 +- lib/linalg/svd/SVD.h | 78 +------ lib/linalg/svd/StaticSVD.cpp | 49 ++--- lib/linalg/svd/StaticSVD.h | 9 +- unit_tests/test_IncrementalSVD.cpp | 3 +- unit_tests/test_SVD.cpp | 227 -------------------- 18 files changed, 47 insertions(+), 475 deletions(-) delete mode 100644 unit_tests/test_SVD.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 35919a836..9a2b489f5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -270,7 +270,6 @@ if(GTEST_FOUND) GNAT QDEIM S_OPT - SVD StaticSVD RandomizedSVD IncrementalSVD diff --git a/lib/linalg/BasisGenerator.cpp b/lib/linalg/BasisGenerator.cpp index 7c1925a78..36ba53aeb 100644 --- a/lib/linalg/BasisGenerator.cpp +++ b/lib/linalg/BasisGenerator.cpp @@ -135,6 +135,7 @@ BasisGenerator::takeSample( { CAROM_VERIFY(u_in != 0); CAROM_VERIFY(time >= 0); + CAROM_VERIFY(d_svd->getNumSamples() < d_svd->getMaxNumSamples()); // Check that u_in is not non-zero. Vector u_vec(u_in, getDim(), true); @@ -144,20 +145,7 @@ BasisGenerator::takeSample( return false; } - if (getNumBasisTimeIntervals() > 0 && - d_svd->isNewTimeInterval()) { - resetDt(dt); - if (d_basis_writer) { - if (d_write_snapshots) { - writeSnapshot(); - } - else { - d_basis_writer->writeBasis("basis"); - } - } - } - - return d_svd->takeSample(u_in, time, add_without_increase); + return d_svd->takeSample(u_in, add_without_increase); } void @@ -199,7 +187,7 @@ BasisGenerator::loadSamples(const std::string& base_file_name, u_in[i] = mat->item(i,j); } } - d_svd->takeSample(u_in, time, false); + d_svd->takeSample(u_in, false); delete[] u_in; } } diff --git a/lib/linalg/BasisGenerator.h b/lib/linalg/BasisGenerator.h index ebad59495..fee00f4ee 100644 --- a/lib/linalg/BasisGenerator.h +++ b/lib/linalg/BasisGenerator.h @@ -222,37 +222,6 @@ class BasisGenerator return d_svd->getSnapshotMatrix(); } - /** - * @brief Returns the number of time intervals on which different sets of - * basis vectors are defined. - * - * @return The number of time intervals on which there are basis vectors. - */ - int - getNumBasisTimeIntervals() const - { - return d_svd->getNumBasisTimeIntervals(); - } - - /** - * @brief Returns the start time for the requested time interval. - * - * @pre 0 <= which_interval - * @pre which_interval < getNumBasisTimeIntervals() - * - * @param[in] which_interval Time interval whose start time is needed. - * - * @return The start time for the requested time interval. - */ - double - getBasisIntervalStartTime( - int which_interval) const - { - CAROM_VERIFY(0 <= which_interval); - CAROM_VERIFY(which_interval < getNumBasisTimeIntervals()); - return d_svd->getBasisIntervalStartTime(which_interval); - } - /** * @brief Returns the number of samples taken. * diff --git a/lib/linalg/svd/IncrementalSVD.cpp b/lib/linalg/svd/IncrementalSVD.cpp index 807e8d473..c65f2a8b0 100644 --- a/lib/linalg/svd/IncrementalSVD.cpp +++ b/lib/linalg/svd/IncrementalSVD.cpp @@ -93,12 +93,6 @@ IncrementalSVD::IncrementalSVD( d_state_database = new HDFDatabase(); bool is_good = d_state_database->open(d_state_file_name, "r"); if (is_good) { - // Read time interval start time. - double time; - d_state_database->getDouble("time", time); - d_time_interval_start_times.resize(1); - d_time_interval_start_times[0] = time; - // Read d_U. int num_rows; d_state_database->getInteger("U_num_rows", num_rows); @@ -144,10 +138,7 @@ IncrementalSVD::~IncrementalSVD() // // If there are multiple time intervals then saving and restoring the state // does not make sense as there is not one, all encompassing, basis. - if (d_save_state && d_time_interval_start_times.size() == 1) { - // Save the time interval start time. - d_state_database->putDouble("time", d_time_interval_start_times[0]); - + if (d_save_state && (!isNewSample())) { // Save d_U. int num_rows = d_U->numRows(); d_state_database->putInteger("U_num_rows", num_rows); @@ -182,11 +173,9 @@ IncrementalSVD::~IncrementalSVD() bool IncrementalSVD::takeSample( double* u_in, - double time, bool add_without_increase) { CAROM_VERIFY(u_in != 0); - CAROM_VERIFY(time >= 0.0); // Check that u_in is not non-zero. Vector u_vec(u_in, d_dim, true); @@ -197,8 +186,8 @@ IncrementalSVD::takeSample( // If this is the first SVD then build it. Otherwise add this sample to the // system. bool result = true; - if (isNewTimeInterval()) { - buildInitialSVD(u_in, time); + if (isNewSample()) { + buildInitialSVD(u_in); } else { result = buildIncrementalSVD(u_in,add_without_increase); diff --git a/lib/linalg/svd/IncrementalSVD.h b/lib/linalg/svd/IncrementalSVD.h index 213151aaa..144781597 100644 --- a/lib/linalg/svd/IncrementalSVD.h +++ b/lib/linalg/svd/IncrementalSVD.h @@ -52,10 +52,8 @@ class IncrementalSVD : public SVD * @brief Sample new state, u_in, at the given time. * * @pre u_in != 0 - * @pre time >= 0.0 * * @param[in] u_in The state at the specified time. - * @param[in] time The simulation time for the state. * @param[in] add_without_increase If true, addLinearlyDependent is invoked. * * @return True if the sampling was successful. @@ -64,7 +62,6 @@ class IncrementalSVD : public SVD bool takeSample( double* u_in, - double time, bool add_without_increase = false); /** @@ -118,8 +115,7 @@ class IncrementalSVD : public SVD virtual void buildInitialSVD( - double* u, - double time) = 0; + double* u) = 0; /** * @brief Adds the new sampled the state vector, u, to the system. diff --git a/lib/linalg/svd/IncrementalSVDBrand.cpp b/lib/linalg/svd/IncrementalSVDBrand.cpp index 32cd41ca1..e2a994b35 100644 --- a/lib/linalg/svd/IncrementalSVDBrand.cpp +++ b/lib/linalg/svd/IncrementalSVDBrand.cpp @@ -65,7 +65,7 @@ IncrementalSVDBrand::~IncrementalSVDBrand() // // If there are multiple time intervals then saving and restoring the state // does not make sense as there is not one, all encompassing, basis. - if (d_save_state && d_time_interval_start_times.size() == 1) { + if (d_save_state && (!isNewSample())) { // Create state database file. d_state_database = new HDFDatabase(); d_state_database->create(d_state_file_name); @@ -106,27 +106,9 @@ IncrementalSVDBrand::getTemporalBasis() void IncrementalSVDBrand::buildInitialSVD( - double* u, - double time) + double* u) { CAROM_VERIFY(u != 0); - CAROM_VERIFY(time >= 0.0); - - // We have a new time interval. - - // If this is not the first time interval then delete d_basis, d_U, d_Up, - // and d_S of the just completed time interval. - int num_time_intervals = - static_cast(d_time_interval_start_times.size()); - if (num_time_intervals > 0) { - delete d_basis; - delete d_U; - delete d_Up; - delete d_S; - delete d_W; - } - increaseTimeInterval(); - d_time_interval_start_times[num_time_intervals] = time; // Build d_S for this new time interval. d_S = new Vector(1, false); diff --git a/lib/linalg/svd/IncrementalSVDBrand.h b/lib/linalg/svd/IncrementalSVDBrand.h index 153120c62..6b7e54482 100644 --- a/lib/linalg/svd/IncrementalSVDBrand.h +++ b/lib/linalg/svd/IncrementalSVDBrand.h @@ -98,8 +98,7 @@ class IncrementalSVDBrand : public IncrementalSVD virtual void buildInitialSVD( - double* u, - double time); + double* u); /** * @brief Adds the new sampled the state vector, u, to the system. diff --git a/lib/linalg/svd/IncrementalSVDFastUpdate.cpp b/lib/linalg/svd/IncrementalSVDFastUpdate.cpp index cbed4d250..9ce73649d 100644 --- a/lib/linalg/svd/IncrementalSVDFastUpdate.cpp +++ b/lib/linalg/svd/IncrementalSVDFastUpdate.cpp @@ -65,7 +65,7 @@ IncrementalSVDFastUpdate::~IncrementalSVDFastUpdate() // // If there are multiple time intervals then saving and restoring the state // does not make sense as there is not one, all encompassing, basis. - if (d_save_state && d_time_interval_start_times.size() == 1) { + if (d_save_state && (!isNewSample())) { // Create state database file. d_state_database = new HDFDatabase(); d_state_database->create(d_state_file_name); @@ -88,27 +88,9 @@ IncrementalSVDFastUpdate::~IncrementalSVDFastUpdate() void IncrementalSVDFastUpdate::buildInitialSVD( - double* u, - double time) + double* u) { CAROM_VERIFY(u != 0); - CAROM_VERIFY(time >= 0.0); - - // We have a new time interval. - - // If this is not the first time interval then delete d_basis, d_U, d_Up, - // and d_S of the just completed time interval. - int num_time_intervals = - static_cast(d_time_interval_start_times.size()); - if (num_time_intervals > 0) { - delete d_basis; - delete d_U; - delete d_Up; - delete d_S; - delete d_W; - } - increaseTimeInterval(); - d_time_interval_start_times[num_time_intervals] = time; // Build d_S for this new time interval. d_S = new Vector(1, false); diff --git a/lib/linalg/svd/IncrementalSVDFastUpdate.h b/lib/linalg/svd/IncrementalSVDFastUpdate.h index 2069e86e5..6d7e1f41c 100644 --- a/lib/linalg/svd/IncrementalSVDFastUpdate.h +++ b/lib/linalg/svd/IncrementalSVDFastUpdate.h @@ -80,8 +80,7 @@ class IncrementalSVDFastUpdate : public IncrementalSVD virtual void buildInitialSVD( - double* u, - double time); + double* u); /** * @brief Computes the current basis vectors. diff --git a/lib/linalg/svd/IncrementalSVDStandard.cpp b/lib/linalg/svd/IncrementalSVDStandard.cpp index c9c9dd491..de9cb3fe6 100644 --- a/lib/linalg/svd/IncrementalSVDStandard.cpp +++ b/lib/linalg/svd/IncrementalSVDStandard.cpp @@ -52,7 +52,7 @@ IncrementalSVDStandard::~IncrementalSVDStandard() // // If there are multiple time intervals then saving and restoring the state // does not make sense as there is not one, all encompassing, basis. - if (d_save_state && d_time_interval_start_times.size() == 1) { + if (d_save_state && (!isNewSample())) { // Create state database file. d_state_database = new HDFDatabase(); d_state_database->create(d_state_file_name); @@ -61,27 +61,9 @@ IncrementalSVDStandard::~IncrementalSVDStandard() void IncrementalSVDStandard::buildInitialSVD( - double* u, - double time) + double* u) { CAROM_VERIFY(u != 0); - CAROM_VERIFY(time >= 0.0); - - // We have a new time interval. - - // If this is not the first time interval then write the basis vectors for - // the just completed interval. Delete d_basis and d_S of the just - // completed time interval. - int num_time_intervals = - static_cast(d_time_interval_start_times.size()); - if (num_time_intervals > 0) { - delete d_basis; - delete d_U; - delete d_S; - delete d_W; - } - increaseTimeInterval(); - d_time_interval_start_times[num_time_intervals] = time; // Build d_S for this new time interval. d_S = new Vector(1, false); diff --git a/lib/linalg/svd/IncrementalSVDStandard.h b/lib/linalg/svd/IncrementalSVDStandard.h index f514bb5b8..ac10b1ee9 100644 --- a/lib/linalg/svd/IncrementalSVDStandard.h +++ b/lib/linalg/svd/IncrementalSVDStandard.h @@ -79,8 +79,7 @@ class IncrementalSVDStandard : public IncrementalSVD virtual void buildInitialSVD( - double* u, - double time); + double* u); /** * @brief Computes the current basis vectors. diff --git a/lib/linalg/svd/RandomizedSVD.cpp b/lib/linalg/svd/RandomizedSVD.cpp index 9d9271ebc..c5ea51f13 100644 --- a/lib/linalg/svd/RandomizedSVD.cpp +++ b/lib/linalg/svd/RandomizedSVD.cpp @@ -196,7 +196,7 @@ RandomizedSVD::computeSVD() d_basis_right->gather(); } - d_this_interval_basis_current = true; + d_basis_is_current = true; delete Q; release_context(&svd_input); diff --git a/lib/linalg/svd/SVD.cpp b/lib/linalg/svd/SVD.cpp index d81825f94..fdf3ef214 100644 --- a/lib/linalg/svd/SVD.cpp +++ b/lib/linalg/svd/SVD.cpp @@ -19,20 +19,16 @@ SVD::SVD( Options options) : d_dim(options.dim), d_num_samples(0), - d_samples_per_time_interval(options.samples_per_time_interval), - d_max_time_intervals(options.max_time_intervals), + d_max_num_samples(options.samples_per_time_interval), d_basis(NULL), d_basis_right(NULL), d_U(NULL), d_W(NULL), d_S(NULL), d_snapshots(NULL), - d_time_interval_start_times(0), d_debug_algorithm(options.debug_algorithm) { CAROM_VERIFY(options.dim > 0); - CAROM_VERIFY(options.max_time_intervals == -1 - || options.max_time_intervals > 0); CAROM_VERIFY(options.samples_per_time_interval > 0); } diff --git a/lib/linalg/svd/SVD.h b/lib/linalg/svd/SVD.h index de2c6f389..25a4662b0 100644 --- a/lib/linalg/svd/SVD.h +++ b/lib/linalg/svd/SVD.h @@ -47,10 +47,8 @@ class SVD * @brief Collect the new sample, u_in at supplied time. * * @pre u_in != 0 - * @pre time >= 0.0 * * @param[in] u_in The new sample. - * @param[in] time The simulation time of the new sample. * @param[in] add_without_increase If true, the addLinearlyDependent is invoked. * This only applies to incremental SVD. * @@ -60,7 +58,6 @@ class SVD bool takeSample( double* u_in, - double time, bool add_without_increase) = 0; /** @@ -110,39 +107,6 @@ class SVD const Matrix* getSnapshotMatrix() = 0; - /** - * @brief Returns the number of time intervals on which different sets - * of basis vectors are defined. - * - * @return The number of time intervals on which there are basis vectors. - */ - int - getNumBasisTimeIntervals() const - { - return static_cast(d_time_interval_start_times.size()); - } - - /** - * @brief Returns the start time for the requested time interval. - * - * @pre 0 <= which_interval - * @pre which_interval < getNumBasisTimeIntervals() - * - * @param[in] which_interval The time interval of interest. - * - * @return The start time for the requested time interval. - */ - double - getBasisIntervalStartTime( - int which_interval) const - { - CAROM_VERIFY(0 <= which_interval); - CAROM_VERIFY(which_interval < getNumBasisTimeIntervals()); - - std::size_t i = static_cast(which_interval); - return d_time_interval_start_times[i]; - } - /** * @brief Returns true if the next sample will result in a new time * interval. @@ -151,38 +115,27 @@ class SVD * interval. */ bool - isNewTimeInterval() const + isNewSample() const { - return (d_num_samples == 0) || - (d_num_samples >= d_samples_per_time_interval); + return (d_num_samples == 0); } /** - * @brief Increase the number of time intervals by one + * @brief Get the number of samples taken. * */ - void - increaseTimeInterval() + int getNumSamples() const { - int num_time_intervals = - static_cast(d_time_interval_start_times.size()); - CAROM_VERIFY(d_max_time_intervals == 1); - if (num_time_intervals > 0) - CAROM_ERROR("SVD::increaseTimeInterval- Time interval is obsolete" - " and will be removed in the future. " - "You received this error presumably " - "because the number of samples reached its limit.\n"); - d_time_interval_start_times.resize( - static_cast(num_time_intervals) + 1); + return d_num_samples; } /** - * @brief Get the number of samples taken. + * @brief Get the maximum number of samples that can be taken. * */ - int getNumSamples() const + int getMaxNumSamples() const { - return d_num_samples; + return d_max_num_samples; } protected: @@ -202,15 +155,9 @@ class SVD int d_num_rows_of_W; /** - * @brief The maximum number of samples to be collected for a time - * interval. - */ - const int d_samples_per_time_interval; - - /** - * @brief The maximum number of time intervals. + * @brief The maximum number of samples. */ - const int d_max_time_intervals; + const int d_max_num_samples; /** * @brief The globalized basis vectors for the current time interval. @@ -260,11 +207,6 @@ class SVD */ Matrix* d_snapshots; - /** - * @brief The simulation time at which each time interval starts. - */ - std::vector d_time_interval_start_times; - /** * @brief Flag to indicate if results of algorithm should be printed for * debugging purposes. diff --git a/lib/linalg/svd/StaticSVD.cpp b/lib/linalg/svd/StaticSVD.cpp index 085336f10..47e8344e1 100644 --- a/lib/linalg/svd/StaticSVD.cpp +++ b/lib/linalg/svd/StaticSVD.cpp @@ -36,7 +36,7 @@ StaticSVD::StaticSVD( Options options) : SVD(options), d_samples(new SLPK_Matrix), d_factorizer(new SVDManager), - d_this_interval_basis_current(false), + d_basis_is_current(false), d_max_basis_dimension(options.max_basis_dimension), d_singular_value_tol(options.singular_value_tol), d_preserve_snapshot(options.static_svd_preserve_snapshot) @@ -60,7 +60,7 @@ StaticSVD::StaticSVD( d_blocksize += 1; } - initialize_matrix(d_samples.get(), d_total_dim, d_samples_per_time_interval, + initialize_matrix(d_samples.get(), d_total_dim, d_max_num_samples, d_nprow, d_npcol, d_blocksize, d_blocksize); // TODO: should nb = 1? d_factorizer->A = nullptr; } @@ -100,12 +100,12 @@ void StaticSVD::delete_factorizer() bool StaticSVD::takeSample( double* u_in, - double time, bool add_without_increase) { CAROM_VERIFY(u_in != 0); - CAROM_VERIFY(time >= 0.0); CAROM_NULL_USE(add_without_increase); + CAROM_VERIFY(0 <= d_num_samples); + CAROM_VERIFY(d_num_samples < d_max_num_samples); // Check the u_in is not non-zero. Vector u_vec(u_in, d_dim, true); @@ -113,33 +113,14 @@ StaticSVD::takeSample( return false; } - if (isNewTimeInterval()) { + if (isNewSample()) { // We have a new time interval. delete_factorizer(); - int num_time_intervals = - static_cast(d_time_interval_start_times.size()); - if (num_time_intervals > 0) { - delete d_basis; - d_basis = nullptr; - delete d_basis_right; - d_basis_right = nullptr; - delete d_U; - d_U = nullptr; - delete d_S; - d_S = nullptr; - delete d_W; - d_W = nullptr; - delete d_snapshots; - d_snapshots = nullptr; - } d_num_samples = 0; - increaseTimeInterval(); - d_time_interval_start_times[static_cast(num_time_intervals)] = - time; d_basis = nullptr; d_basis_right = nullptr; // Set the N in the global matrix so BLACS won't complain. - d_samples->n = d_samples_per_time_interval; + d_samples->n = d_max_num_samples; } broadcast_sample(u_in); ++d_num_samples; @@ -153,7 +134,7 @@ StaticSVD::takeSample( // firstrow, 1, nrows, // d_num_samples, rank); // } - d_this_interval_basis_current = false; + d_basis_is_current = false; return true; } @@ -162,7 +143,7 @@ StaticSVD::getSpatialBasis() { // If this basis is for the last time interval then it may not be up to date // so recompute it. - if (!thisIntervalBasisCurrent()) { + if (!isBasisCurrent()) { delete d_basis; d_basis = nullptr; delete d_basis_right; @@ -178,7 +159,7 @@ StaticSVD::getSpatialBasis() else { CAROM_ASSERT(d_basis != 0); } - CAROM_ASSERT(thisIntervalBasisCurrent()); + CAROM_ASSERT(isBasisCurrent()); return d_basis; } @@ -187,7 +168,7 @@ StaticSVD::getTemporalBasis() { // If this basis is for the last time interval then it may not be up to date // so recompute it. - if (!thisIntervalBasisCurrent()) { + if (!isBasisCurrent()) { delete d_basis; d_basis = nullptr; delete d_basis_right; @@ -203,7 +184,7 @@ StaticSVD::getTemporalBasis() else { CAROM_ASSERT(d_basis_right != 0); } - CAROM_ASSERT(thisIntervalBasisCurrent()); + CAROM_ASSERT(isBasisCurrent()); return d_basis_right; } @@ -212,7 +193,7 @@ StaticSVD::getSingularValues() { // If these singular values are for the last time interval then they may not // be up to date so recompute them. - if (!thisIntervalBasisCurrent()) { + if (!isBasisCurrent()) { delete d_basis; d_basis = nullptr; delete d_basis_right; @@ -228,14 +209,14 @@ StaticSVD::getSingularValues() else { CAROM_ASSERT(d_S != 0); } - CAROM_ASSERT(thisIntervalBasisCurrent()); + CAROM_ASSERT(isBasisCurrent()); return d_S; } const Matrix* StaticSVD::getSnapshotMatrix() { - if ((!d_preserve_snapshot) && (thisIntervalBasisCurrent()) && (d_basis != 0)) + if ((!d_preserve_snapshot) && (isBasisCurrent()) && (d_basis != 0)) CAROM_ERROR("StaticSVD: snapshot matrix is modified after computeSVD." " To preserve the snapshots, set Options::static_svd_preserve_snapshot to be true!\n"); @@ -363,7 +344,7 @@ StaticSVD::computeSVD() } for (int i = 0; i < ncolumns; ++i) d_S->item(i) = d_factorizer->S[static_cast(i)]; - d_this_interval_basis_current = true; + d_basis_is_current = true; if (d_debug_algorithm) { if (d_rank == 0) { diff --git a/lib/linalg/svd/StaticSVD.h b/lib/linalg/svd/StaticSVD.h index bb8d29cbf..a650936f2 100644 --- a/lib/linalg/svd/StaticSVD.h +++ b/lib/linalg/svd/StaticSVD.h @@ -41,10 +41,8 @@ class StaticSVD : public SVD * @brief Collect the new sample, u_in at the supplied time. * * @pre u_in != 0 - * @pre time >= 0.0 * * @param[in] u_in The new sample. - * @param[in] time The simulation time of the new sample. * @param[in] add_without_increase If true, then addLinearlyDependent will be invoked * * @return True if the sampling was successful. @@ -53,7 +51,6 @@ class StaticSVD : public SVD bool takeSample( double* u_in, - double time, bool add_without_increase = false); /** @@ -128,9 +125,9 @@ class StaticSVD : public SVD * date. */ bool - thisIntervalBasisCurrent() + isBasisCurrent() { - return d_this_interval_basis_current; + return d_basis_is_current; } /** @@ -147,7 +144,7 @@ class StaticSVD : public SVD * @brief Flag to indicate if the basis vectors for the current time * interval are up to date. */ - bool d_this_interval_basis_current; + bool d_basis_is_current; /** * @brief The rank of the process this object belongs to. diff --git a/unit_tests/test_IncrementalSVD.cpp b/unit_tests/test_IncrementalSVD.cpp index 5aaa31279..2ca19d1c0 100644 --- a/unit_tests/test_IncrementalSVD.cpp +++ b/unit_tests/test_IncrementalSVD.cpp @@ -69,8 +69,7 @@ class FakeIncrementalSVD : public CAROM::IncrementalSVD } void buildInitialSVD - (__attribute__((unused)) double* u, - __attribute__((unused)) double time) + (__attribute__((unused)) double* u) { /* Do nothing */ } diff --git a/unit_tests/test_SVD.cpp b/unit_tests/test_SVD.cpp deleted file mode 100644 index daa17f5bb..000000000 --- a/unit_tests/test_SVD.cpp +++ /dev/null @@ -1,227 +0,0 @@ -/****************************************************************************** - * - * Copyright (c) 2013-2024, Lawrence Livermore National Security, LLC - * and other libROM project developers. See the top-level COPYRIGHT - * file for details. - * - * SPDX-License-Identifier: (Apache-2.0 OR MIT) - * - *****************************************************************************/ - -// Description: This source file is a test runner that uses the Google Test -// Framework to run unit tests on the CAROM::SVD class. - -#include - -#ifdef CAROM_HAS_GTEST -#include -#include -#include "linalg/Options.h" -#include "linalg/svd/SVD.h" - -/** - * Simple smoke test to make sure Google Test is properly linked - */ -TEST(GoogleTestFramework, GoogleTestFrameworkFound) { - SUCCEED(); -} - -/** - * Fake SVD to test parts of the SVD abstract base class. (For those - * unfamiliar, "fake" or "mock" is a term of art in unit testing - * referring to an implementation that simulates the behavior of - * real objects.) - */ - -class FakeSVD : public CAROM::SVD -{ -public: - - FakeSVD(CAROM::Options options) - : SVD(options) - { - } - - ~FakeSVD() - { - } - - /** - * Stub implementations of methods not really testable from the - * abstract base class, because there is no meaningful data we could - * return without implementing an actual singular value decomposition. - * - */ - virtual const CAROM::Matrix* getSpatialBasis() { - return NULL; - } - virtual const CAROM::Matrix* getTemporalBasis() { - return NULL; - } - virtual const CAROM::Vector* getSingularValues() { - return NULL; - } - virtual const CAROM::Matrix* getSnapshotMatrix() { - return NULL; - } - - /** - * The only testable methods from the SVD abstract base class are - * those with concrete implementations, namely: - * - * int getDim() const; - * - * int getNumBasisTimeIntervals() const; - * - * double getBasisIntervalStartTime(int) const; - * - * bool isNewTimeInterval() const; - */ - - /** - * This method needs to do a few things: - * - * 1) If there are no sample time intervals stored, create one and - * record its start time. Set the number of samples in the current - * time interval to zero. - * - * 2) If adding another sample to the current sample time interval - * would exceed the number of samples per time interval set in - * the constructor, create a new time interval and record its - * start time. Set the number of samples in the current time interval - * to zero. - * - * 3) Increment the number of samples in the current sample time - * interval. - * - * Implementing this behavior suffices for testing the abstract base class. - * - */ - bool takeSample - (__attribute__((unused)) double* u_in, - double time, - __attribute__((unused)) bool add_without_increase) - { - /** - If a new time interval is needed, add one and reset the number - of samples counter to zero. - */ - if (isNewTimeInterval()) - { - int num_time_intervals = - static_cast(d_time_interval_start_times.size()); - increaseTimeInterval(); - d_time_interval_start_times[num_time_intervals] = time; - d_num_samples = 0; - } - - /* Increment the number of samples in the current time interval */ - d_num_samples++; - - /** - This method should almost always succeed because it does not - do anything likely to fail, so it should return true. - */ - return true; - } - -}; - -TEST(SVDSerialTest, Test_getDim) -{ - FakeSVD svd(CAROM::Options(5, 2)); - EXPECT_EQ(svd.getDim(), 5); -} - -TEST(SVDSerialTest, Test_isNewTimeInterval) -{ - FakeSVD svd(CAROM::Options(5, 2)); - - /* 0 samples, so taking a sample will create a new time interval */ - EXPECT_TRUE(svd.isNewTimeInterval()); - - /* 1 sample; limit is 2, taking a sample won't create a new time interval */ - svd.takeSample(NULL, 0, true); - EXPECT_FALSE(svd.isNewTimeInterval()); - - /* 2 samples; limit is 2, taking a sample will create a new time interval */ - svd.takeSample(NULL, 0.5, true); - EXPECT_TRUE(svd.isNewTimeInterval()); - - /* 1 sample; limit is 2, taking a sample won't create a new time interval */ - svd.takeSample(NULL, 1, true); - EXPECT_FALSE(svd.isNewTimeInterval()); - - /* 2 samples; limit is 2, taking a sample will create a new time interval */ - svd.takeSample(NULL, 1.5, true); - EXPECT_TRUE(svd.isNewTimeInterval()); - - /* 1 sample; limit is 2, taking a sample won't create a new time interval */ - svd.takeSample(NULL, 2, true); - EXPECT_FALSE(svd.isNewTimeInterval()); -} - -TEST(SVDSerialTest, Test_getNumBasisTimeIntervals) -{ - FakeSVD svd(CAROM::Options(5, 2)); - - /* Number of time intervals starts at zero. */ - EXPECT_EQ(svd.getNumBasisTimeIntervals(), 0); - - /* Creates new time interval; number of intervals = 1 */ - svd.takeSample(NULL, 0, true); - EXPECT_EQ(svd.getNumBasisTimeIntervals(), 1); - svd.takeSample(NULL, 0.5, true); - EXPECT_EQ(svd.getNumBasisTimeIntervals(), 1); - - /* Creates new time interval; number of intervals = 2 */ - svd.takeSample(NULL, 1, true); - EXPECT_EQ(svd.getNumBasisTimeIntervals(), 2); - svd.takeSample(NULL, 1.5, true); - EXPECT_EQ(svd.getNumBasisTimeIntervals(), 2); - - /* Creates new time interval; number of intervals = 3 */ - svd.takeSample(NULL, 2, true); - EXPECT_EQ(svd.getNumBasisTimeIntervals(), 3); -} - -TEST(SVDSerialTest, Test_getBasisIntervalStartTime) -{ - FakeSVD svd(CAROM::Options(5, 2)); - - /* 1st time interval starts at time 0 */ - svd.takeSample(NULL, 0, true); - EXPECT_DOUBLE_EQ(svd.getBasisIntervalStartTime(0), 0); - - svd.takeSample(NULL, 0.5, true); - EXPECT_DOUBLE_EQ(svd.getBasisIntervalStartTime(0), 0); - - /* 2nd time interval starts at time 1 */ - svd.takeSample(NULL, 1, true); - EXPECT_DOUBLE_EQ(svd.getBasisIntervalStartTime(0), 0); - EXPECT_DOUBLE_EQ(svd.getBasisIntervalStartTime(1), 1); - - svd.takeSample(NULL, 1.5, true); - EXPECT_DOUBLE_EQ(svd.getBasisIntervalStartTime(0), 0); - EXPECT_DOUBLE_EQ(svd.getBasisIntervalStartTime(1), 1); - - /* 3rd time interval starts at time 2 */ - svd.takeSample(NULL, 2, true); - EXPECT_DOUBLE_EQ(svd.getBasisIntervalStartTime(0), 0); - EXPECT_DOUBLE_EQ(svd.getBasisIntervalStartTime(1), 1); - EXPECT_DOUBLE_EQ(svd.getBasisIntervalStartTime(2), 2); -} - -int main(int argc, char* argv[]) -{ - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} -#else // #ifndef CAROM_HAS_GTEST -int main() -{ - std::cout << "libROM was compiled without Google Test support, so unit " - << "tests have been disabled. To enable unit tests, compile " - << "libROM with Google Test support." << std::endl; -} -#endif // #endif CAROM_HAS_GTEST From b7cb8e2b89e5d7530cd29d3ae1e63ac6cc864b4a Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Sat, 17 Feb 2024 14:07:37 -0800 Subject: [PATCH 18/50] BasisGenerator: removed the concept of time interval. --- lib/linalg/BasisGenerator.cpp | 13 +++++------ lib/linalg/BasisReader.cpp | 13 +++++------ lib/linalg/BasisReader.h | 42 ++++++++++++++++++++++++++++++++--- lib/linalg/Options.h | 15 +++++++------ lib/linalg/svd/SVD.cpp | 4 ++-- 5 files changed, 59 insertions(+), 28 deletions(-) diff --git a/lib/linalg/BasisGenerator.cpp b/lib/linalg/BasisGenerator.cpp index 36ba53aeb..dcf59d5b0 100644 --- a/lib/linalg/BasisGenerator.cpp +++ b/lib/linalg/BasisGenerator.cpp @@ -34,7 +34,7 @@ BasisGenerator::BasisGenerator( d_write_snapshots(options.write_snapshots) { CAROM_VERIFY(options.dim > 0); - CAROM_VERIFY(options.samples_per_time_interval > 0); + CAROM_VERIFY(options.max_num_samples > 0); CAROM_VERIFY(options.singular_value_tol >= 0); CAROM_VERIFY(options.max_time_intervals == -1 || options.max_time_intervals > 0); @@ -134,14 +134,12 @@ BasisGenerator::takeSample( bool add_without_increase) { CAROM_VERIFY(u_in != 0); - CAROM_VERIFY(time >= 0); CAROM_VERIFY(d_svd->getNumSamples() < d_svd->getMaxNumSamples()); // Check that u_in is not non-zero. Vector u_vec(u_in, getDim(), true); if (u_vec.norm() == 0.0) { - printf("WARNING: BasisGenerator::takeSample skipped trivial sample at time %.4E\n", - time); + printf("WARNING: BasisGenerator::takeSample skipped trivial sample.\n"); return false; } @@ -160,16 +158,15 @@ BasisGenerator::loadSamples(const std::string& base_file_name, if (d_basis_reader) delete d_basis_reader; d_basis_reader = new BasisReader(base_file_name, db_format); - double time = 0.0; const Matrix* mat; const Vector* singular_vals; if (kind == "basis") { - mat = d_basis_reader->getSpatialBasis(time); - singular_vals = d_basis_reader->getSingularValues(time); + mat = d_basis_reader->getSpatialBasis(); + singular_vals = d_basis_reader->getSingularValues(); } else if (kind == "snapshot") { - mat = d_basis_reader->getSnapshotMatrix(time); + mat = d_basis_reader->getSnapshotMatrix(); } int num_rows = mat->numRows(); diff --git a/lib/linalg/BasisReader.cpp b/lib/linalg/BasisReader.cpp index 0bf3dd074..4cecaae45 100644 --- a/lib/linalg/BasisReader.cpp +++ b/lib/linalg/BasisReader.cpp @@ -58,8 +58,7 @@ BasisReader::~BasisReader() } Matrix* -BasisReader::getSpatialBasis( - double time) +BasisReader::getSpatialBasis() { d_last_basis_idx = 0; int num_rows = getDim("basis"); @@ -140,8 +139,7 @@ BasisReader::getSpatialBasis( } Matrix* -BasisReader::getTemporalBasis( - double time) +BasisReader::getTemporalBasis() { d_last_basis_idx = 0; int num_rows = getDim("temporal_basis"); @@ -325,12 +323,11 @@ BasisReader::getNumSamples( } Matrix* -BasisReader::getSnapshotMatrix( - double time) +BasisReader::getSnapshotMatrix() { d_last_basis_idx = 0; - int num_rows = getDim("snapshot",time); - int num_cols = getNumSamples("snapshot",time); + int num_rows = getDim("snapshot"); + int num_cols = getNumSamples("snapshot"); char tmp[100]; Matrix* snapshots = new Matrix(num_rows, num_cols, false); diff --git a/lib/linalg/BasisReader.h b/lib/linalg/BasisReader.h index 7a01ddb4e..f1c209574 100644 --- a/lib/linalg/BasisReader.h +++ b/lib/linalg/BasisReader.h @@ -72,10 +72,21 @@ class BasisReader { return (d_last_basis_idx == -1); } + /** + * + * @brief Returns the spatial basis vectors as a Matrix. + * + * @return The spatial basis vectors. + */ + Matrix* + getSpatialBasis(); + /** * * @brief Returns the spatial basis vectors for the requested time as a * Matrix. + * NOTE: this function is obsolete and remains only for backward compatibility. + * Will be removed in future. * * @param[in] time Time for which we want the basis vectors. * NOTE: this argument is obsolete and remains only for backward compatibility. @@ -85,7 +96,8 @@ class BasisReader { */ Matrix* getSpatialBasis( - double time); + double time) + { return getSpatialBasis(); } /** * @@ -152,6 +164,18 @@ class BasisReader { * @brief Returns the temporal basis vectors for the requested time as * a Matrix. * + * @return The temporal basis vectors for the requested time. + */ + Matrix* + getTemporalBasis(); + + /** + * + * @brief Returns the temporal basis vectors for the requested time as + * a Matrix. + * NOTE: this function is obsolete and remains only for backward compatibility. + * Will be removed in future. + * * @param[in] time Time for which we want the basis vectors. * NOTE: this argument is obsolete and remains only for backward compatibility. * Will be removed in future. @@ -160,7 +184,8 @@ class BasisReader { */ Matrix* getTemporalBasis( - double time); + double time) + { return getTemporalBasis(); } /** * @@ -329,6 +354,16 @@ class BasisReader { * * @brief Returns the snapshot matrix for the requested time. * + * @return The snapshot matrix for the requested time. + */ + Matrix* + getSnapshotMatrix(); + + /** + * + * @brief Returns the snapshot matrix for the requested time. + * NOTE: this function is obsolete and remains only for backward compatibility. + * Will be removed in future. * * @param[in] time Time for which we want the basis vectors. * NOTE: this argument is obsolete and remains only for backward compatibility. @@ -338,7 +373,8 @@ class BasisReader { */ Matrix* getSnapshotMatrix( - double time); + double time) + { return getSnapshotMatrix(); } /** * diff --git a/lib/linalg/Options.h b/lib/linalg/Options.h index 816002d2b..87aa8cfac 100644 --- a/lib/linalg/Options.h +++ b/lib/linalg/Options.h @@ -30,11 +30,11 @@ class Options * @brief Constructor. * * @pre dim_ > 0 - * @pre samples_per_time_interval_ > 0 + * @pre max_num_samples_ > 0 * @pre max_time_intervals == -1 || max_time_intervals > 0 * * @param[in] dim_ The dimension of the system on this processor. - * @param[in] samples_per_time_interval_ The maximum number of samples in + * @param[in] max_num_samples_ The maximum number of samples in * each time interval. * @param[in] max_time_intervals_ The maximum number of time intervals. * @param[in] update_right_SV_ Whether to update the right SV or not. @@ -43,13 +43,13 @@ class Options * */ Options(int dim_, - int samples_per_time_interval_, + int max_num_samples_, int max_time_intervals_ = 1, bool update_right_SV_ = false, bool write_snapshots_ = false ): dim(dim_), - max_basis_dimension(samples_per_time_interval_), - samples_per_time_interval(samples_per_time_interval_), + max_basis_dimension(max_num_samples_), + max_num_samples(max_num_samples_), max_time_intervals(max_time_intervals_), update_right_SV(update_right_SV_), write_snapshots(write_snapshots_) @@ -226,12 +226,13 @@ class Options int dim = -1; /** - * @brief The number of samples per time interval. + * @brief The maximum number of samples. */ - int samples_per_time_interval = -1; + int max_num_samples = -1; /** * @brief The maximum number of time intervals. + * NOTE: this variable is obsolte and will be removed in future. */ int max_time_intervals = -1; diff --git a/lib/linalg/svd/SVD.cpp b/lib/linalg/svd/SVD.cpp index fdf3ef214..fe6547625 100644 --- a/lib/linalg/svd/SVD.cpp +++ b/lib/linalg/svd/SVD.cpp @@ -19,7 +19,7 @@ SVD::SVD( Options options) : d_dim(options.dim), d_num_samples(0), - d_max_num_samples(options.samples_per_time_interval), + d_max_num_samples(options.max_num_samples), d_basis(NULL), d_basis_right(NULL), d_U(NULL), @@ -29,7 +29,7 @@ SVD::SVD( d_debug_algorithm(options.debug_algorithm) { CAROM_VERIFY(options.dim > 0); - CAROM_VERIFY(options.samples_per_time_interval > 0); + CAROM_VERIFY(options.max_num_samples > 0); } SVD::~SVD() From 3d43de04644614cad30a09cc833badf2ea925851 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Sat, 17 Feb 2024 14:24:42 -0800 Subject: [PATCH 19/50] add test_SVD.cpp for resolving conflict. --- unit_tests/test_SVD.cpp | 241 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 241 insertions(+) create mode 100644 unit_tests/test_SVD.cpp diff --git a/unit_tests/test_SVD.cpp b/unit_tests/test_SVD.cpp new file mode 100644 index 000000000..68a7c70fc --- /dev/null +++ b/unit_tests/test_SVD.cpp @@ -0,0 +1,241 @@ +/****************************************************************************** + * + * Copyright (c) 2013-2024, Lawrence Livermore National Security, LLC + * and other libROM project developers. See the top-level COPYRIGHT + * file for details. + * + * SPDX-License-Identifier: (Apache-2.0 OR MIT) + * + *****************************************************************************/ + +// Description: This source file is a test runner that uses the Google Test +// Framework to run unit tests on the CAROM::SVD class. + +#include + +#ifdef CAROM_HAS_GTEST +#include +#include +#include "linalg/Options.h" +#include "linalg/svd/SVD.h" + +/** + * Simple smoke test to make sure Google Test is properly linked + */ +TEST(GoogleTestFramework, GoogleTestFrameworkFound) { + SUCCEED(); +} + +/** + * Fake SVD to test parts of the SVD abstract base class. (For those + * unfamiliar, "fake" or "mock" is a term of art in unit testing + * referring to an implementation that simulates the behavior of + * real objects.) + */ + +class FakeSVD : public CAROM::SVD +{ +public: + + FakeSVD(CAROM::Options options) + : SVD(options) + { + } + + ~FakeSVD() + { + } + + /** + * Stub implementations of methods not really testable from the + * abstract base class, because there is no meaningful data we could + * return without implementing an actual singular value decomposition. + * + */ + virtual const CAROM::Matrix* getSpatialBasis() { + return NULL; + } + virtual const CAROM::Matrix* getTemporalBasis() { + return NULL; + } + virtual const CAROM::Vector* getSingularValues() { + return NULL; + } + virtual const CAROM::Matrix* getSnapshotMatrix() { + return NULL; + } + + /** + * The only testable methods from the SVD abstract base class are + * those with concrete implementations, namely: + * + * int getDim() const; + * + * int getNumBasisTimeIntervals() const; + * + * double getBasisIntervalStartTime(int) const; + * + * bool isNewTimeInterval() const; + */ + + /** + * This method needs to do a few things: + * + * 1) If there are no sample time intervals stored, create one and + * record its start time. Set the number of samples in the current + * time interval to zero. + * + * 2) If adding another sample to the current sample time interval + * would exceed the number of samples per time interval set in + * the constructor, create a new time interval and record its + * start time. Set the number of samples in the current time interval + * to zero. + * + * 3) Increment the number of samples in the current sample time + * interval. + * + * Implementing this behavior suffices for testing the abstract base class. + * + */ + bool takeSample + (__attribute__((unused)) double* u_in, + double time, + __attribute__((unused)) bool add_without_increase) + { + /** + If a new time interval is needed, add one and reset the number + of samples counter to zero. + */ + if (isNewTimeInterval()) + { + int num_time_intervals = + static_cast(d_time_interval_start_times.size()); + increaseTimeInterval(); + d_time_interval_start_times[num_time_intervals] = time; + d_num_samples = 0; + } + + /* Increment the number of samples in the current time interval */ + d_num_samples++; + + /** + This method should almost always succeed because it does not + do anything likely to fail, so it should return true. + */ + return true; + } + +}; + +TEST(SVDSerialTest, Test_getDim) +{ + FakeSVD svd(CAROM::Options(5, 2)); + EXPECT_EQ(svd.getDim(), 5); +} + +TEST(SVDSerialTest, Test_isNewTimeInterval) +{ + FakeSVD svd(CAROM::Options(5, 2)); + + /* 0 samples, so taking a sample will create a new time interval */ + EXPECT_TRUE(svd.isNewTimeInterval()); + + /* 1 sample; limit is 2, taking a sample won't create a new time interval */ + svd.takeSample(NULL, 0, true); + EXPECT_FALSE(svd.isNewTimeInterval()); + + /* 2 samples; limit is 2, taking a sample will create a new time interval */ + svd.takeSample(NULL, 0.5, true); + EXPECT_TRUE(svd.isNewTimeInterval()); + + /* 1 sample; limit is 2, taking a sample won't create a new time interval */ + svd.takeSample(NULL, 1, true); + EXPECT_FALSE(svd.isNewTimeInterval()); + + /* 2 samples; limit is 2, taking a sample will create a new time interval */ + svd.takeSample(NULL, 1.5, true); + EXPECT_TRUE(svd.isNewTimeInterval()); + + /* 1 sample; limit is 2, taking a sample won't create a new time interval */ + svd.takeSample(NULL, 2, true); + EXPECT_FALSE(svd.isNewTimeInterval()); +} + +TEST(SVDSerialTest, Test_getNumBasisTimeIntervals) +{ + FakeSVD svd(CAROM::Options(5, 2)); + + /* Number of time intervals starts at zero. */ + EXPECT_EQ(svd.getNumBasisTimeIntervals(), 0); + + /* Creates new time interval; number of intervals = 1 */ + svd.takeSample(NULL, 0, true); + EXPECT_EQ(svd.getNumBasisTimeIntervals(), 1); + svd.takeSample(NULL, 0.5, true); + EXPECT_EQ(svd.getNumBasisTimeIntervals(), 1); + + /* Creates new time interval; number of intervals = 2 */ + svd.takeSample(NULL, 1, true); + EXPECT_EQ(svd.getNumBasisTimeIntervals(), 2); + svd.takeSample(NULL, 1.5, true); + EXPECT_EQ(svd.getNumBasisTimeIntervals(), 2); + + /* Creates new time interval; number of intervals = 3 */ + svd.takeSample(NULL, 2, true); + EXPECT_EQ(svd.getNumBasisTimeIntervals(), 3); +} + +TEST(SVDSerialTest, Test_getBasisIntervalStartTime) +{ + FakeSVD svd(CAROM::Options(5, 2)); + + /* 1st time interval starts at time 0 */ + svd.takeSample(NULL, 0, true); + EXPECT_DOUBLE_EQ(svd.getBasisIntervalStartTime(0), 0); + + svd.takeSample(NULL, 0.5, true); + EXPECT_DOUBLE_EQ(svd.getBasisIntervalStartTime(0), 0); + + /* 2nd time interval starts at time 1 */ + svd.takeSample(NULL, 1, true); + EXPECT_DOUBLE_EQ(svd.getBasisIntervalStartTime(0), 0); + EXPECT_DOUBLE_EQ(svd.getBasisIntervalStartTime(1), 1); + + svd.takeSample(NULL, 1.5, true); + EXPECT_DOUBLE_EQ(svd.getBasisIntervalStartTime(0), 0); + EXPECT_DOUBLE_EQ(svd.getBasisIntervalStartTime(1), 1); + + /* 3rd time interval starts at time 2 */ + svd.takeSample(NULL, 2, true); + EXPECT_DOUBLE_EQ(svd.getBasisIntervalStartTime(0), 0); + EXPECT_DOUBLE_EQ(svd.getBasisIntervalStartTime(1), 1); + EXPECT_DOUBLE_EQ(svd.getBasisIntervalStartTime(2), 2); +} + + +TEST(SVDSerialTest, Test_increaseTimeInterval) +{ + FakeSVD svd(CAROM::Options(5, 2, 2)); + + ASSERT_NO_THROW(svd.takeSample(NULL, 0, true)); + ASSERT_NO_THROW(svd.takeSample(NULL, 0.5, true)); + ASSERT_NO_THROW(svd.takeSample(NULL, 1, true)); + ASSERT_NO_THROW(svd.takeSample(NULL, 1.5, true)); + + /* The maximum number of time intervals is surpassed */ + EXPECT_DEATH(svd.takeSample(NULL, 2, true), ".*"); +} + +int main(int argc, char* argv[]) +{ + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} +#else // #ifndef CAROM_HAS_GTEST +int main() +{ + std::cout << "libROM was compiled without Google Test support, so unit " + << "tests have been disabled. To enable unit tests, compile " + << "libROM with Google Test support." << std::endl; +} +#endif // #endif CAROM_HAS_GTEST From 9fdc4535a7e70ab742ddf34fe55af03b9617cd92 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Sat, 17 Feb 2024 14:32:55 -0800 Subject: [PATCH 20/50] stylization. --- lib/linalg/BasisReader.h | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/linalg/BasisReader.h b/lib/linalg/BasisReader.h index f1c209574..c56f75f70 100644 --- a/lib/linalg/BasisReader.h +++ b/lib/linalg/BasisReader.h @@ -97,7 +97,9 @@ class BasisReader { Matrix* getSpatialBasis( double time) - { return getSpatialBasis(); } + { + return getSpatialBasis(); + } /** * @@ -185,7 +187,9 @@ class BasisReader { Matrix* getTemporalBasis( double time) - { return getTemporalBasis(); } + { + return getTemporalBasis(); + } /** * @@ -374,7 +378,9 @@ class BasisReader { Matrix* getSnapshotMatrix( double time) - { return getSnapshotMatrix(); } + { + return getSnapshotMatrix(); + } /** * From 3acb60630951a4256e661767635461f10a893051 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Wed, 21 Feb 2024 13:03:39 -0800 Subject: [PATCH 21/50] changed function signature of BasisGenerator::takeSample. --- examples/misc/combine_samples.cpp | 2 +- examples/prom/dg_advection_global_rom.cpp | 4 +-- .../dg_advection_local_rom_matrix_interp.cpp | 4 +-- .../prom/linear_elasticity_global_rom.cpp | 2 +- examples/prom/maxwell_global_rom.cpp | 2 +- examples/prom/mixed_nonlinear_diffusion.cpp | 16 +++++----- .../prom/nonlinear_elasticity_global_rom.cpp | 16 ++++++---- examples/prom/poisson_global_rom.cpp | 2 +- examples/prom/poisson_local_rom_greedy.cpp | 2 +- lib/linalg/BasisGenerator.cpp | 14 ++++++-- lib/linalg/BasisGenerator.h | 4 --- unit_tests/random_test.cpp | 4 +-- unit_tests/smoke_static.cpp | 8 ++--- unit_tests/smoke_test.cpp | 4 +-- unit_tests/test_IncrementalSVDBrand.cpp | 6 ++-- unit_tests/test_RandomizedSVD.cpp | 32 +++++++++---------- unit_tests/test_StaticSVD.cpp | 18 +++++------ unit_tests/test_include.cpp | 4 +-- unit_tests/uneven_dist.cpp | 4 +-- unit_tests/weak_scaling.cpp | 2 +- 20 files changed, 79 insertions(+), 71 deletions(-) diff --git a/examples/misc/combine_samples.cpp b/examples/misc/combine_samples.cpp index 77987ab37..d4f10d4ce 100644 --- a/examples/misc/combine_samples.cpp +++ b/examples/misc/combine_samples.cpp @@ -180,7 +180,7 @@ int main(int argc, char* argv[]) CAROM::Vector snap_cur(num_rows, true); for (int col = 0; col < num_cols; col++) { snap_cur = *snapshots->getColumn(col); - static_basis_generator2->takeSample(snap_cur.getData(), 0.0, false); + static_basis_generator2->takeSample(snap_cur.getData(), false); } /*-- Compute SVD and save file --*/ diff --git a/examples/prom/dg_advection_global_rom.cpp b/examples/prom/dg_advection_global_rom.cpp index 4f28c57d9..b4fa575b5 100644 --- a/examples/prom/dg_advection_global_rom.cpp +++ b/examples/prom/dg_advection_global_rom.cpp @@ -697,7 +697,7 @@ int main(int argc, char *argv[]) Vector u_curr(*U); Vector u_centered(U->Size()); subtract(u_curr, u_init, u_centered); - bool addSample = generator->takeSample(u_centered.GetData(), t, dt); + bool addSample = generator->takeSample(u_centered.GetData()); } // 11. The merge phase @@ -831,7 +831,7 @@ int main(int argc, char *argv[]) Vector u_curr(*U); Vector u_centered(U->Size()); subtract(u_curr, u_init, u_centered); - bool addSample = generator->takeSample(u_centered.GetData(), t, dt); + bool addSample = generator->takeSample(u_centered.GetData()); } if (done || ti % vis_steps == 0) diff --git a/examples/prom/dg_advection_local_rom_matrix_interp.cpp b/examples/prom/dg_advection_local_rom_matrix_interp.cpp index 1341c63ac..7b7e73d43 100644 --- a/examples/prom/dg_advection_local_rom_matrix_interp.cpp +++ b/examples/prom/dg_advection_local_rom_matrix_interp.cpp @@ -711,7 +711,7 @@ int main(int argc, char *argv[]) Vector u_curr(*U); Vector u_centered(U->Size()); subtract(u_curr, u_init, u_centered); - bool addSample = generator->takeSample(u_centered.GetData(), t, dt); + bool addSample = generator->takeSample(u_centered.GetData()); } if (online) @@ -932,7 +932,7 @@ int main(int argc, char *argv[]) Vector u_curr(*U); Vector u_centered(U->Size()); subtract(u_curr, u_init, u_centered); - bool addSample = generator->takeSample(u_centered.GetData(), t, dt); + bool addSample = generator->takeSample(u_centered.GetData()); } if (done || ti % vis_steps == 0) diff --git a/examples/prom/linear_elasticity_global_rom.cpp b/examples/prom/linear_elasticity_global_rom.cpp index f84974ebf..a5fde9e94 100644 --- a/examples/prom/linear_elasticity_global_rom.cpp +++ b/examples/prom/linear_elasticity_global_rom.cpp @@ -367,7 +367,7 @@ int main(int argc, char* argv[]) // 18. take and write snapshot for ROM if (offline) { - bool addSample = generator->takeSample(X.GetData(), 0.0, 0.01); + bool addSample = generator->takeSample(X.GetData()); generator->writeSnapshot(); delete generator; delete options; diff --git a/examples/prom/maxwell_global_rom.cpp b/examples/prom/maxwell_global_rom.cpp index f9de7d849..dd6beabab 100644 --- a/examples/prom/maxwell_global_rom.cpp +++ b/examples/prom/maxwell_global_rom.cpp @@ -348,7 +348,7 @@ int main(int argc, char *argv[]) // 18. take and write snapshot for ROM if (offline) { - bool addSample = generator->takeSample(X.GetData(), 0.0, 0.01); + bool addSample = generator->takeSample(X.GetData()); generator->writeSnapshot(); delete generator; delete options; diff --git a/examples/prom/mixed_nonlinear_diffusion.cpp b/examples/prom/mixed_nonlinear_diffusion.cpp index 19ab15dd9..c543b0b43 100644 --- a/examples/prom/mixed_nonlinear_diffusion.cpp +++ b/examples/prom/mixed_nonlinear_diffusion.cpp @@ -1214,7 +1214,7 @@ int main(int argc, char *argv[]) if (sampleW && hyperreduce_source) { oper.GetSource(source); - basis_generator_S->takeSample(source.GetData(), t, dt); + basis_generator_S->takeSample(source.GetData()); // TODO: dfdt? In this example, one can implement the exact formula. // In general, one can use finite differences in time (dpdt is computed that way). //basis_generator_S->computeNextSampleTime(p.GetData(), dfdt.GetData(), t); @@ -1224,21 +1224,21 @@ int main(int argc, char *argv[]) { oper.CopyDpDt(dpdt); - basis_generator_R->takeSample(p.GetData(), t, dt); + basis_generator_R->takeSample(p.GetData()); basis_generator_R->computeNextSampleTime(p.GetData(), dpdt.GetData(), t); Vector p_R(p.GetData(), N1); Vector Mp(N1); oper.SetParameters(p); oper.Mult_Mmat(p_R, Mp); - basis_generator_FR->takeSample(Mp.GetData(), t, dt); + basis_generator_FR->takeSample(Mp.GetData()); } if (sampleW) { oper.CopyDpDt_W(dpdt); - basis_generator_W->takeSample(p_W->GetData(), t, dt); + basis_generator_W->takeSample(p_W->GetData()); basis_generator_W->computeNextSampleTime(p_W->GetData(), dpdt.GetData(), t); } } @@ -1403,13 +1403,13 @@ int main(int argc, char *argv[]) oper.CopyDpDt(dpdt); // R space - basis_generator_R->takeSample(p.GetData(), t, dt); + basis_generator_R->takeSample(p.GetData()); Vector p_R(p.GetData(), N1); Vector Mp(N1); oper.SetParameters(p); oper.Mult_Mmat(p_R, Mp); - basis_generator_FR->takeSample(Mp.GetData(), t, dt); + basis_generator_FR->takeSample(Mp.GetData()); // Terminate the sampling and write out information. basis_generator_R->writeSnapshot(); @@ -1418,14 +1418,14 @@ int main(int argc, char *argv[]) // W space // TODO: why call computeNextSampleTime if you just do takeSample on every step anyway? - basis_generator_W->takeSample(p_W->GetData(), t, dt); + basis_generator_W->takeSample(p_W->GetData()); basis_generator_W->writeSnapshot(); oper.GetSource(source); if (hyperreduce_source) { - basis_generator_S->takeSample(source.GetData(), t, dt); + basis_generator_S->takeSample(source.GetData()); basis_generator_S->writeSnapshot(); } diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 7936d143c..5d4eb6085 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -1262,23 +1262,25 @@ int main(int argc, char *argv[]) } // Take samples + // NOTE(kevin): I don't know why this example checks next sample time. + // IncrementalSVD is never turned on in this example and isNextSample is always true. if (x_base_only == false && basis_generator_v->isNextSample(t)) { - basis_generator_v->takeSample(vx_diff.GetBlock(0), t, dt); + basis_generator_v->takeSample(vx_diff.GetBlock(0)); basis_generator_v->computeNextSampleTime(vx_diff.GetBlock(0), dvdt.GetData(), t); - basis_generator_H->takeSample(oper.H_sp.GetData(), t, dt); + basis_generator_H->takeSample(oper.H_sp.GetData()); } if (basis_generator_x->isNextSample(t)) { - basis_generator_x->takeSample(vx_diff.GetBlock(1), t, dt); + basis_generator_x->takeSample(vx_diff.GetBlock(1)); basis_generator_x->computeNextSampleTime(vx_diff.GetBlock(1), dxdt.GetData(), t); if (x_base_only == true) { - basis_generator_H->takeSample(oper.H_sp.GetData(), t, dt); + basis_generator_H->takeSample(oper.H_sp.GetData()); } } } @@ -1363,16 +1365,16 @@ int main(int argc, char *argv[]) // Take samples if (x_base_only == false) { - basis_generator_v->takeSample(vx_diff.GetBlock(0), t, dt); + basis_generator_v->takeSample(vx_diff.GetBlock(0)); basis_generator_v->writeSnapshot(); delete basis_generator_v; } - basis_generator_H->takeSample(oper.H_sp.GetData(), t, dt); + basis_generator_H->takeSample(oper.H_sp.GetData()); basis_generator_H->writeSnapshot(); delete basis_generator_H; - basis_generator_x->takeSample(vx_diff.GetBlock(1), t, dt); + basis_generator_x->takeSample(vx_diff.GetBlock(1)); basis_generator_x->writeSnapshot(); delete basis_generator_x; diff --git a/examples/prom/poisson_global_rom.cpp b/examples/prom/poisson_global_rom.cpp index 8406466ce..0f45d1c39 100644 --- a/examples/prom/poisson_global_rom.cpp +++ b/examples/prom/poisson_global_rom.cpp @@ -341,7 +341,7 @@ int main(int argc, char *argv[]) // 18. take and write snapshot for ROM if (offline) { - bool addSample = generator->takeSample(X.GetData(), 0.0, 0.01); + bool addSample = generator->takeSample(X.GetData()); generator->writeSnapshot(); delete generator; delete options; diff --git a/examples/prom/poisson_local_rom_greedy.cpp b/examples/prom/poisson_local_rom_greedy.cpp index 72a7eb271..1145563a8 100644 --- a/examples/prom/poisson_local_rom_greedy.cpp +++ b/examples/prom/poisson_local_rom_greedy.cpp @@ -454,7 +454,7 @@ int main(int argc, char *argv[]) // 19. take and write snapshot for ROM if (offline) { - bool addSample = generator->takeSample(X.GetData(), 0.0, 0.01); + bool addSample = generator->takeSample(X.GetData()); generator->writeSnapshot(); basisIdentifiers.push_back(saveBasisName); delete generator; diff --git a/lib/linalg/BasisGenerator.cpp b/lib/linalg/BasisGenerator.cpp index dcf59d5b0..942efa9b2 100644 --- a/lib/linalg/BasisGenerator.cpp +++ b/lib/linalg/BasisGenerator.cpp @@ -129,8 +129,6 @@ BasisGenerator::isNextSample( bool BasisGenerator::takeSample( double* u_in, - double time, - double dt, bool add_without_increase) { CAROM_VERIFY(u_in != 0); @@ -143,6 +141,18 @@ BasisGenerator::takeSample( return false; } + /* + Note for previous implementation: + Previously with multiple time interval, + there was an input argument (double dt), + which is only used to reset d_dt for new time interval. + Assuming only single interval is used in practice, + resetDt(dt) was never used in takeSample, + and options.initial_dt is used for incremental svd. + */ + // if (d_svd->isNewSample()) + // resetDt(dt); + return d_svd->takeSample(u_in, add_without_increase); } diff --git a/lib/linalg/BasisGenerator.h b/lib/linalg/BasisGenerator.h index fee00f4ee..7ba7b02c5 100644 --- a/lib/linalg/BasisGenerator.h +++ b/lib/linalg/BasisGenerator.h @@ -102,8 +102,6 @@ class BasisGenerator * @pre time >= 0.0 * * @param[in] u_in The state at the specified time. - * @param[in] time The simulation time for the state. - * @param[in] dt The current simulation dt. * @param[in] add_without_increase If true, the addLinearlyDependent is * invoked. This only applies to incremental * SVD. @@ -113,8 +111,6 @@ class BasisGenerator bool takeSample( double* u_in, - double time, - double dt, bool add_without_increase = false); /** diff --git a/unit_tests/random_test.cpp b/unit_tests/random_test.cpp index b6c759c87..104f01e5e 100644 --- a/unit_tests/random_test.cpp +++ b/unit_tests/random_test.cpp @@ -190,7 +190,7 @@ main( bool status = true; for (int i = 0; i < num_samples; ++i) { if (inc_basis_generator.isNextSample(0.01*i)) { - status = inc_basis_generator.takeSample(M[i], 0.01*i, 0.01); + status = inc_basis_generator.takeSample(M[i]); if (!status) { break; } @@ -198,7 +198,7 @@ main( } if (i < num_lin_indep_samples && static_basis_generator.isNextSample(0.01*i)) { - status = static_basis_generator.takeSample(M[i], 0.01*i, 0.01); + status = static_basis_generator.takeSample(M[i]); if (!status) { break; } diff --git a/unit_tests/smoke_static.cpp b/unit_tests/smoke_static.cpp index 316bd72f8..856dc71c7 100644 --- a/unit_tests/smoke_static.cpp +++ b/unit_tests/smoke_static.cpp @@ -74,7 +74,7 @@ main( "static_smoke1")); // Take the first sample. - static_basis_generator->takeSample(&vals0[dim*rank],0,0.1); + static_basis_generator->takeSample(&vals0[dim*rank]); std::cout << "Writing sample 1" << std::endl; static_basis_generator->writeSnapshot(); static_basis_generator->endSamples(); @@ -87,7 +87,7 @@ main( "static_smoke2")); // Take the second sample. - static_basis_generator2->takeSample(&vals1[dim*rank],0,0.1); + static_basis_generator2->takeSample(&vals1[dim*rank]); static_basis_generator2->writeSnapshot(); // "_snapshot" will be added to the base file name static_basis_generator2->endSamples(); @@ -120,8 +120,8 @@ main( static_svd_options, false, "static_smoke_check")); - static_basis_generator4->takeSample(&vals0[dim*rank],0,0.1); - static_basis_generator4->takeSample(&vals1[dim*rank],0,0.1); + static_basis_generator4->takeSample(&vals0[dim*rank]); + static_basis_generator4->takeSample(&vals1[dim*rank]); static_basis_generator4->endSamples(); static_basis_generator4 = nullptr; diff --git a/unit_tests/smoke_test.cpp b/unit_tests/smoke_test.cpp index feca85ebe..0c6237ef3 100644 --- a/unit_tests/smoke_test.cpp +++ b/unit_tests/smoke_test.cpp @@ -73,7 +73,7 @@ main( // Take the first sample. if (inc_basis_generator.isNextSample(0.0)) { - status = inc_basis_generator.takeSample(&vals0[dim*rank], 0.0, 0.11); + status = inc_basis_generator.takeSample(&vals0[dim*rank]); if (status) { inc_basis_generator.computeNextSampleTime(&vals0[dim*rank], &vals0[dim*rank], @@ -83,7 +83,7 @@ main( // Take the second sample. if (status && inc_basis_generator.isNextSample(0.11)) { - status = inc_basis_generator.takeSample(&vals1[dim*rank], 0.11, 0.11); + status = inc_basis_generator.takeSample(&vals1[dim*rank]); if (status) { inc_basis_generator.computeNextSampleTime(&vals1[dim*rank], &vals1[dim*rank], diff --git a/unit_tests/test_IncrementalSVDBrand.cpp b/unit_tests/test_IncrementalSVDBrand.cpp index a18d842d7..a55345523 100644 --- a/unit_tests/test_IncrementalSVDBrand.cpp +++ b/unit_tests/test_IncrementalSVDBrand.cpp @@ -96,9 +96,9 @@ TEST(IncrementalSVDBrandTest, Test_IncrementalSVDBrand) incremental_svd_options, true, "irrelevant.txt"); - sampler.takeSample(&sample1[row_offset[d_rank]], 0, 1e-1); - sampler.takeSample(&sample2[row_offset[d_rank]], 0, 1e-1); - sampler.takeSample(&sample3[row_offset[d_rank]], 0, 1e-1); + sampler.takeSample(&sample1[row_offset[d_rank]]); + sampler.takeSample(&sample2[row_offset[d_rank]]); + sampler.takeSample(&sample3[row_offset[d_rank]]); const CAROM::Matrix* d_basis = sampler.getSpatialBasis(); const CAROM::Matrix* d_basis_right = sampler.getTemporalBasis(); diff --git a/unit_tests/test_RandomizedSVD.cpp b/unit_tests/test_RandomizedSVD.cpp index 0f52b495c..e1b1b6c81 100644 --- a/unit_tests/test_RandomizedSVD.cpp +++ b/unit_tests/test_RandomizedSVD.cpp @@ -74,9 +74,9 @@ TEST(RandomizedSVDTest, Test_RandomizedSVD) randomized_svd_options.setDebugMode(true); randomized_svd_options.setRandomizedSVD(true); CAROM::BasisGenerator sampler(randomized_svd_options, false); - sampler.takeSample(&sample1[row_offset[d_rank]], 0, 0); - sampler.takeSample(&sample2[row_offset[d_rank]], 0, 0); - sampler.takeSample(&sample3[row_offset[d_rank]], 0, 0); + sampler.takeSample(&sample1[row_offset[d_rank]]); + sampler.takeSample(&sample2[row_offset[d_rank]]); + sampler.takeSample(&sample3[row_offset[d_rank]]); const CAROM::Matrix* d_basis = sampler.getSpatialBasis(); const CAROM::Matrix* d_basis_right = sampler.getTemporalBasis(); @@ -154,11 +154,11 @@ TEST(RandomizedSVDTest, Test_RandomizedSVDTransposed) randomized_svd_options.setDebugMode(true); randomized_svd_options.setRandomizedSVD(true); CAROM::BasisGenerator sampler(randomized_svd_options, false); - sampler.takeSample(&sample1[row_offset[d_rank]], 0, 0); - sampler.takeSample(&sample2[row_offset[d_rank]], 0, 0); - sampler.takeSample(&sample3[row_offset[d_rank]], 0, 0); - sampler.takeSample(&sample4[row_offset[d_rank]], 0, 0); - sampler.takeSample(&sample5[row_offset[d_rank]], 0, 0); + sampler.takeSample(&sample1[row_offset[d_rank]]); + sampler.takeSample(&sample2[row_offset[d_rank]]); + sampler.takeSample(&sample3[row_offset[d_rank]]); + sampler.takeSample(&sample4[row_offset[d_rank]]); + sampler.takeSample(&sample5[row_offset[d_rank]]); const CAROM::Matrix* d_basis = sampler.getSpatialBasis(); const CAROM::Matrix* d_basis_right = sampler.getTemporalBasis(); @@ -232,9 +232,9 @@ TEST(RandomizedSVDTest, Test_RandomizedSVDSmallerSubspace) randomized_svd_options.setDebugMode(true); randomized_svd_options.setRandomizedSVD(true, 2); CAROM::BasisGenerator sampler(randomized_svd_options, false); - sampler.takeSample(&sample1[row_offset[d_rank]], 0, 0); - sampler.takeSample(&sample2[row_offset[d_rank]], 0, 0); - sampler.takeSample(&sample3[row_offset[d_rank]], 0, 0); + sampler.takeSample(&sample1[row_offset[d_rank]]); + sampler.takeSample(&sample2[row_offset[d_rank]]); + sampler.takeSample(&sample3[row_offset[d_rank]]); const CAROM::Matrix* d_basis = sampler.getSpatialBasis(); const CAROM::Matrix* d_basis_right = sampler.getTemporalBasis(); @@ -311,11 +311,11 @@ TEST(RandomizedSVDTest, Test_RandomizedSVDTransposedSmallerSubspace) randomized_svd_options.setDebugMode(true); randomized_svd_options.setRandomizedSVD(true, reduced_rows); CAROM::BasisGenerator sampler(randomized_svd_options, false); - sampler.takeSample(&sample1[row_offset[d_rank]], 0, 0); - sampler.takeSample(&sample2[row_offset[d_rank]], 0, 0); - sampler.takeSample(&sample3[row_offset[d_rank]], 0, 0); - sampler.takeSample(&sample4[row_offset[d_rank]], 0, 0); - sampler.takeSample(&sample5[row_offset[d_rank]], 0, 0); + sampler.takeSample(&sample1[row_offset[d_rank]]); + sampler.takeSample(&sample2[row_offset[d_rank]]); + sampler.takeSample(&sample3[row_offset[d_rank]]); + sampler.takeSample(&sample4[row_offset[d_rank]]); + sampler.takeSample(&sample5[row_offset[d_rank]]); const CAROM::Matrix* d_basis = sampler.getSpatialBasis(); const CAROM::Matrix* d_basis_right = sampler.getTemporalBasis(); diff --git a/unit_tests/test_StaticSVD.cpp b/unit_tests/test_StaticSVD.cpp index ddda7bc4c..dde4c9694 100644 --- a/unit_tests/test_StaticSVD.cpp +++ b/unit_tests/test_StaticSVD.cpp @@ -78,7 +78,7 @@ TEST(StaticSVDTest, Test_StaticSVD) std::vector similar(columns[j]); for (unsigned i = 0; i < 12; ++i) similar[i] *= sigmas[j]; - sampler.takeSample(similar.data(), 0, 0); + sampler.takeSample(similar.data()); } auto distU = sampler.getSpatialBasis(); @@ -290,9 +290,9 @@ TEST(StaticSVDTest, Test_StaticSVDClass) svd_options.setDebugMode(true); svd_options.setRandomizedSVD(false); CAROM::BasisGenerator sampler(svd_options, false); - sampler.takeSample(&sample1[row_offset[d_rank]], 0, 0); - sampler.takeSample(&sample2[row_offset[d_rank]], 0, 0); - sampler.takeSample(&sample3[row_offset[d_rank]], 0, 0); + sampler.takeSample(&sample1[row_offset[d_rank]]); + sampler.takeSample(&sample2[row_offset[d_rank]]); + sampler.takeSample(&sample3[row_offset[d_rank]]); const CAROM::Matrix* d_basis = sampler.getSpatialBasis(); const CAROM::Matrix* d_basis_right = sampler.getTemporalBasis(); @@ -370,11 +370,11 @@ TEST(StaticSVDTest, Test_StaticSVDTranspose) svd_options.setDebugMode(true); svd_options.setRandomizedSVD(false); CAROM::BasisGenerator sampler(svd_options, false); - sampler.takeSample(&sample1[row_offset[d_rank]], 0, 0); - sampler.takeSample(&sample2[row_offset[d_rank]], 0, 0); - sampler.takeSample(&sample3[row_offset[d_rank]], 0, 0); - sampler.takeSample(&sample4[row_offset[d_rank]], 0, 0); - sampler.takeSample(&sample5[row_offset[d_rank]], 0, 0); + sampler.takeSample(&sample1[row_offset[d_rank]]); + sampler.takeSample(&sample2[row_offset[d_rank]]); + sampler.takeSample(&sample3[row_offset[d_rank]]); + sampler.takeSample(&sample4[row_offset[d_rank]]); + sampler.takeSample(&sample5[row_offset[d_rank]]); const CAROM::Matrix* d_basis = sampler.getSpatialBasis(); const CAROM::Matrix* d_basis_right = sampler.getTemporalBasis(); diff --git a/unit_tests/test_include.cpp b/unit_tests/test_include.cpp index 8cbb6b3bb..2396067d3 100644 --- a/unit_tests/test_include.cpp +++ b/unit_tests/test_include.cpp @@ -73,7 +73,7 @@ main( // Take the first sample. if (inc_basis_generator.isNextSample(0.0)) { - status = inc_basis_generator.takeSample(&vals0[dim*rank], 0.0, 0.11); + status = inc_basis_generator.takeSample(&vals0[dim*rank]); if (status) { inc_basis_generator.computeNextSampleTime(&vals0[dim*rank], &vals0[dim*rank], @@ -83,7 +83,7 @@ main( // Take the second sample. if (status && inc_basis_generator.isNextSample(0.11)) { - status = inc_basis_generator.takeSample(&vals1[dim*rank], 0.11, 0.11); + status = inc_basis_generator.takeSample(&vals1[dim*rank]); if (status) { inc_basis_generator.computeNextSampleTime(&vals1[dim*rank], &vals1[dim*rank], diff --git a/unit_tests/uneven_dist.cpp b/unit_tests/uneven_dist.cpp index 7da2dfb41..6281e61e1 100644 --- a/unit_tests/uneven_dist.cpp +++ b/unit_tests/uneven_dist.cpp @@ -136,7 +136,7 @@ main( // Take the first sample. if (inc_basis_generator.isNextSample(0.0)) { - status = inc_basis_generator.takeSample(&vals0[offset], 0.0, 0.11); + status = inc_basis_generator.takeSample(&vals0[offset]); if (status) { inc_basis_generator.computeNextSampleTime(&vals0[offset], &vals0[offset], @@ -146,7 +146,7 @@ main( // Take the second sample. if (status && inc_basis_generator.isNextSample(0.11)) { - status = inc_basis_generator.takeSample(&vals1[offset], 0.11, 0.11); + status = inc_basis_generator.takeSample(&vals1[offset]); if (status) { inc_basis_generator.computeNextSampleTime(&vals1[offset], &vals1[offset], diff --git a/unit_tests/weak_scaling.cpp b/unit_tests/weak_scaling.cpp index b0e232564..34a4c2fe5 100644 --- a/unit_tests/weak_scaling.cpp +++ b/unit_tests/weak_scaling.cpp @@ -76,7 +76,7 @@ main( int samples_taken = 0; for (int i = 0; i < num_samples; ++i) { if (basis_generator.isNextSample(0.01*i)) { - status = basis_generator.takeSample(M[i], 0.01*i, 0.01); + status = basis_generator.takeSample(M[i]); if (!status) { break; } From 79702a25ffbae0696bc91766e7bfa072b2605ba7 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Wed, 21 Feb 2024 13:35:19 -0800 Subject: [PATCH 22/50] rebased to resolve conflict. --- examples/prom/de_parametric_maxwell_greedy.cpp | 2 +- examples/prom/grad_div_global_rom.cpp | 2 +- examples/prom/maxwell_local_rom_greedy.cpp | 2 +- unit_tests/test_SVD.cpp | 14 -------------- 4 files changed, 3 insertions(+), 17 deletions(-) diff --git a/examples/prom/de_parametric_maxwell_greedy.cpp b/examples/prom/de_parametric_maxwell_greedy.cpp index a0a1a2b0d..7f8720229 100644 --- a/examples/prom/de_parametric_maxwell_greedy.cpp +++ b/examples/prom/de_parametric_maxwell_greedy.cpp @@ -316,7 +316,7 @@ double simulation() // 19. take and write snapshot for ROM if (offline) { - bool addSample = generator->takeSample(X.GetData(), 0.0, 0.01); + bool addSample = generator->takeSample(X.GetData()); generator->writeSnapshot(); basisIdentifiers.push_back(saveBasisName); delete generator; diff --git a/examples/prom/grad_div_global_rom.cpp b/examples/prom/grad_div_global_rom.cpp index 22a944854..331423204 100644 --- a/examples/prom/grad_div_global_rom.cpp +++ b/examples/prom/grad_div_global_rom.cpp @@ -358,7 +358,7 @@ int main(int argc, char *argv[]) // 18. take and write snapshot for ROM if (offline) { - generator->takeSample(X.GetData(), 0.0, 0.01); + generator->takeSample(X.GetData()); generator->writeSnapshot(); delete generator; delete options; diff --git a/examples/prom/maxwell_local_rom_greedy.cpp b/examples/prom/maxwell_local_rom_greedy.cpp index 8c75a0f7b..3bf4c3283 100644 --- a/examples/prom/maxwell_local_rom_greedy.cpp +++ b/examples/prom/maxwell_local_rom_greedy.cpp @@ -469,7 +469,7 @@ int main(int argc, char *argv[]) // 19. take and write snapshot for ROM if (offline) { - bool addSample = generator->takeSample(X.GetData(), 0.0, 0.01); + bool addSample = generator->takeSample(X.GetData()); generator->writeSnapshot(); basisIdentifiers.push_back(saveBasisName); delete generator; diff --git a/unit_tests/test_SVD.cpp b/unit_tests/test_SVD.cpp index 68a7c70fc..daa17f5bb 100644 --- a/unit_tests/test_SVD.cpp +++ b/unit_tests/test_SVD.cpp @@ -212,20 +212,6 @@ TEST(SVDSerialTest, Test_getBasisIntervalStartTime) EXPECT_DOUBLE_EQ(svd.getBasisIntervalStartTime(2), 2); } - -TEST(SVDSerialTest, Test_increaseTimeInterval) -{ - FakeSVD svd(CAROM::Options(5, 2, 2)); - - ASSERT_NO_THROW(svd.takeSample(NULL, 0, true)); - ASSERT_NO_THROW(svd.takeSample(NULL, 0.5, true)); - ASSERT_NO_THROW(svd.takeSample(NULL, 1, true)); - ASSERT_NO_THROW(svd.takeSample(NULL, 1.5, true)); - - /* The maximum number of time intervals is surpassed */ - EXPECT_DEATH(svd.takeSample(NULL, 2, true), ".*"); -} - int main(int argc, char* argv[]) { ::testing::InitGoogleTest(&argc, argv); From 588930929019fcd4235520dd816bcd4f7956c4dd Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Wed, 21 Feb 2024 14:03:04 -0800 Subject: [PATCH 23/50] changed function signature for BasisReader::getSpatialBasis. --- .../prom/de_parametric_maxwell_greedy.cpp | 2 +- examples/prom/dg_advection_global_rom.cpp | 4 +- .../dg_advection_local_rom_matrix_interp.cpp | 6 +-- examples/prom/grad_div_global_rom.cpp | 2 +- .../prom/linear_elasticity_global_rom.cpp | 2 +- examples/prom/maxwell_global_rom.cpp | 2 +- examples/prom/maxwell_local_rom_greedy.cpp | 2 +- examples/prom/mixed_nonlinear_diffusion.cpp | 8 ++-- .../prom/nonlinear_elasticity_global_rom.cpp | 6 +-- examples/prom/poisson_global_rom.cpp | 2 +- examples/prom/poisson_local_rom_greedy.cpp | 2 +- lib/linalg/BasisReader.cpp | 7 +--- lib/linalg/BasisReader.h | 39 +------------------ regression_tests/basisComparator.cpp | 8 ++-- 14 files changed, 26 insertions(+), 66 deletions(-) diff --git a/examples/prom/de_parametric_maxwell_greedy.cpp b/examples/prom/de_parametric_maxwell_greedy.cpp index 7f8720229..78e226553 100644 --- a/examples/prom/de_parametric_maxwell_greedy.cpp +++ b/examples/prom/de_parametric_maxwell_greedy.cpp @@ -329,7 +329,7 @@ double simulation() // 21. read the reduced basis assembleTimer.Start(); CAROM::BasisReader reader(loadBasisName); - spatialbasis = reader.getSpatialBasis(0.0); + spatialbasis = reader.getSpatialBasis(); numRowRB = spatialbasis->numRows(); numColumnRB = spatialbasis->numColumns(); diff --git a/examples/prom/dg_advection_global_rom.cpp b/examples/prom/dg_advection_global_rom.cpp index b4fa575b5..8c3b1bac1 100644 --- a/examples/prom/dg_advection_global_rom.cpp +++ b/examples/prom/dg_advection_global_rom.cpp @@ -731,11 +731,11 @@ int main(int argc, char *argv[]) CAROM::BasisReader reader(basisName); if (rdim != -1) { - spatialbasis = reader.getSpatialBasis(0.0, rdim); + spatialbasis = reader.getSpatialBasis(rdim); } else { - spatialbasis = reader.getSpatialBasis(0.0, ef); + spatialbasis = reader.getSpatialBasis(ef); } numRowRB = spatialbasis->numRows(); numColumnRB = spatialbasis->numColumns(); diff --git a/examples/prom/dg_advection_local_rom_matrix_interp.cpp b/examples/prom/dg_advection_local_rom_matrix_interp.cpp index 7b7e73d43..cf052998c 100644 --- a/examples/prom/dg_advection_local_rom_matrix_interp.cpp +++ b/examples/prom/dg_advection_local_rom_matrix_interp.cpp @@ -723,11 +723,11 @@ int main(int argc, char *argv[]) CAROM::BasisReader reader(basisName); if (rdim != -1) { - spatialbasis = reader.getSpatialBasis(0.0, rdim); + spatialbasis = reader.getSpatialBasis(rdim); } else { - spatialbasis = reader.getSpatialBasis(0.0, ef); + spatialbasis = reader.getSpatialBasis(ef); } numRowRB = spatialbasis->numRows(); numColumnRB = spatialbasis->numColumns(); @@ -819,7 +819,7 @@ int main(int argc, char *argv[]) CAROM::BasisReader reader(parametricBasisName); MFEM_VERIFY(rdim != -1, "rdim must be used for interpolation."); - CAROM::Matrix* parametricSpatialBasis = reader.getSpatialBasis(0.0, rdim); + CAROM::Matrix* parametricSpatialBasis = reader.getSpatialBasis(rdim); numRowRB = parametricSpatialBasis->numRows(); numColumnRB = parametricSpatialBasis->numColumns(); bases.push_back(parametricSpatialBasis); diff --git a/examples/prom/grad_div_global_rom.cpp b/examples/prom/grad_div_global_rom.cpp index 331423204..5ba867afe 100644 --- a/examples/prom/grad_div_global_rom.cpp +++ b/examples/prom/grad_div_global_rom.cpp @@ -370,7 +370,7 @@ int main(int argc, char *argv[]) // 20. read the reduced basis assembleTimer.Start(); CAROM::BasisReader reader(basisName); - const CAROM::Matrix* spatialbasis = reader.getSpatialBasis(0.0); + const CAROM::Matrix* spatialbasis = reader.getSpatialBasis(); const int numRowRB = spatialbasis->numRows(); const int numColumnRB = spatialbasis->numColumns(); if (myid == 0) printf("spatial basis dimension is %d x %d\n", numRowRB, diff --git a/examples/prom/linear_elasticity_global_rom.cpp b/examples/prom/linear_elasticity_global_rom.cpp index a5fde9e94..1e44ef7b3 100644 --- a/examples/prom/linear_elasticity_global_rom.cpp +++ b/examples/prom/linear_elasticity_global_rom.cpp @@ -379,7 +379,7 @@ int main(int argc, char* argv[]) // 20. read the reduced basis assembleTimer.Start(); CAROM::BasisReader reader(basisName); - const CAROM::Matrix* spatialbasis = reader.getSpatialBasis(0.0); + const CAROM::Matrix* spatialbasis = reader.getSpatialBasis(); numRowRB = spatialbasis->numRows(); numColumnRB = spatialbasis->numColumns(); printf("On rank %d, spatial basis dimension is %d x %d\n", myid, diff --git a/examples/prom/maxwell_global_rom.cpp b/examples/prom/maxwell_global_rom.cpp index dd6beabab..68d9abaea 100644 --- a/examples/prom/maxwell_global_rom.cpp +++ b/examples/prom/maxwell_global_rom.cpp @@ -360,7 +360,7 @@ int main(int argc, char *argv[]) // 20. read the reduced basis assembleTimer.Start(); CAROM::BasisReader reader(basisName); - spatialbasis = reader.getSpatialBasis(0.0); + spatialbasis = reader.getSpatialBasis(); numRowRB = spatialbasis->numRows(); numColumnRB = spatialbasis->numColumns(); if (myid == 0) printf("spatial basis dimension is %d x %d\n", numRowRB, diff --git a/examples/prom/maxwell_local_rom_greedy.cpp b/examples/prom/maxwell_local_rom_greedy.cpp index 3bf4c3283..41d784fcb 100644 --- a/examples/prom/maxwell_local_rom_greedy.cpp +++ b/examples/prom/maxwell_local_rom_greedy.cpp @@ -482,7 +482,7 @@ int main(int argc, char *argv[]) // 21. read the reduced basis assembleTimer.Start(); CAROM::BasisReader reader(loadBasisName); - spatialbasis = reader.getSpatialBasis(0.0); + spatialbasis = reader.getSpatialBasis(); numRowRB = spatialbasis->numRows(); numColumnRB = spatialbasis->numColumns(); if (myid == 0) printf("spatial basis dimension is %d x %d\n", numRowRB, diff --git a/examples/prom/mixed_nonlinear_diffusion.cpp b/examples/prom/mixed_nonlinear_diffusion.cpp index c543b0b43..47f01f284 100644 --- a/examples/prom/mixed_nonlinear_diffusion.cpp +++ b/examples/prom/mixed_nonlinear_diffusion.cpp @@ -918,7 +918,7 @@ int main(int argc, char *argv[]) if (online) { CAROM::BasisReader readerR("basisR"); - BR_librom = readerR.getSpatialBasis(0.0); + BR_librom = readerR.getSpatialBasis(); if (rrdim == -1) rrdim = BR_librom->numColumns(); else @@ -931,7 +931,7 @@ int main(int argc, char *argv[]) printf("reduced R dim = %d\n",rrdim); CAROM::BasisReader readerW("basisW"); - BW_librom = readerW.getSpatialBasis(0.0); + BW_librom = readerW.getSpatialBasis(); if (rwdim == -1) rwdim = BW_librom->numColumns(); else @@ -953,7 +953,7 @@ int main(int argc, char *argv[]) */ CAROM::BasisReader readerFR("basisFR"); - FR_librom = readerFR.getSpatialBasis(0.0); + FR_librom = readerFR.getSpatialBasis(); if (nldim == -1) { @@ -1041,7 +1041,7 @@ int main(int argc, char *argv[]) if (hyperreduce_source) { readerS = new CAROM::BasisReader("basisS"); - S_librom = readerS->getSpatialBasis(0.0); + S_librom = readerS->getSpatialBasis(); if (nsdim == -1) { diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 5d4eb6085..677aff4ee 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -900,7 +900,7 @@ int main(int argc, char *argv[]) readerV = new CAROM::BasisReader("basisV"); } - BV_librom = readerV->getSpatialBasis(0.0); + BV_librom = readerV->getSpatialBasis(); if (rvdim == -1) // Change rvdim rvdim = BV_librom->numColumns(); @@ -914,7 +914,7 @@ int main(int argc, char *argv[]) printf("reduced V dim = %d\n", rvdim); CAROM::BasisReader readerX("basisX"); - BX_librom = readerX.getSpatialBasis(0.0); + BX_librom = readerX.getSpatialBasis(); if (rxdim == -1) // Change rxdim rxdim = BX_librom->numColumns(); @@ -929,7 +929,7 @@ int main(int argc, char *argv[]) // Hyper reduce H CAROM::BasisReader readerH("basisH"); - H_librom = readerH.getSpatialBasis(0.0); + H_librom = readerH.getSpatialBasis(); // Compute sample points if (hdim == -1) diff --git a/examples/prom/poisson_global_rom.cpp b/examples/prom/poisson_global_rom.cpp index 0f45d1c39..6ec67cdec 100644 --- a/examples/prom/poisson_global_rom.cpp +++ b/examples/prom/poisson_global_rom.cpp @@ -353,7 +353,7 @@ int main(int argc, char *argv[]) // 20. read the reduced basis assembleTimer.Start(); CAROM::BasisReader reader(basisName); - spatialbasis = reader.getSpatialBasis(0.0); + spatialbasis = reader.getSpatialBasis(); numRowRB = spatialbasis->numRows(); numColumnRB = spatialbasis->numColumns(); if (myid == 0) printf("spatial basis dimension is %d x %d\n", numRowRB, diff --git a/examples/prom/poisson_local_rom_greedy.cpp b/examples/prom/poisson_local_rom_greedy.cpp index 1145563a8..89fd50297 100644 --- a/examples/prom/poisson_local_rom_greedy.cpp +++ b/examples/prom/poisson_local_rom_greedy.cpp @@ -467,7 +467,7 @@ int main(int argc, char *argv[]) // 21. read the reduced basis assembleTimer.Start(); CAROM::BasisReader reader(loadBasisName); - spatialbasis = reader.getSpatialBasis(0.0); + spatialbasis = reader.getSpatialBasis(); numRowRB = spatialbasis->numRows(); numColumnRB = spatialbasis->numColumns(); if (myid == 0) printf("spatial basis dimension is %d x %d\n", numRowRB, diff --git a/lib/linalg/BasisReader.cpp b/lib/linalg/BasisReader.cpp index 4cecaae45..874099ccd 100644 --- a/lib/linalg/BasisReader.cpp +++ b/lib/linalg/BasisReader.cpp @@ -76,15 +76,13 @@ BasisReader::getSpatialBasis() Matrix* BasisReader::getSpatialBasis( - double time, int n) { - return getSpatialBasis(time, 1, n); + return getSpatialBasis(1, n); } Matrix* BasisReader::getSpatialBasis( - double time, int start_col, int end_col) { @@ -112,7 +110,6 @@ BasisReader::getSpatialBasis( Matrix* BasisReader::getSpatialBasis( - double time, double ef) { Vector* sv = getSingularValues(); @@ -135,7 +132,7 @@ BasisReader::getSpatialBasis( } delete sv; - return getSpatialBasis(time, num_used_singular_values); + return getSpatialBasis(num_used_singular_values); } Matrix* diff --git a/lib/linalg/BasisReader.h b/lib/linalg/BasisReader.h index c56f75f70..75ca0edc3 100644 --- a/lib/linalg/BasisReader.h +++ b/lib/linalg/BasisReader.h @@ -58,16 +58,11 @@ class BasisReader { * @brief Returns true if the basis vectors at requested time are * different from the last requested basis vectors. * - * @param[in] time Time at which we are interested in the basis vectors. - * NOTE: this argument is obsolete and remains only for backward compatibility. - * Will be removed in future. - * * @return True if the basis vectors at the requested time are different * from the last requested basis vectors. */ bool - isNewBasis( - double time) + isNewBasis() { return (d_last_basis_idx == -1); } @@ -81,26 +76,6 @@ class BasisReader { Matrix* getSpatialBasis(); - /** - * - * @brief Returns the spatial basis vectors for the requested time as a - * Matrix. - * NOTE: this function is obsolete and remains only for backward compatibility. - * Will be removed in future. - * - * @param[in] time Time for which we want the basis vectors. - * NOTE: this argument is obsolete and remains only for backward compatibility. - * Will be removed in future. - * - * @return The spatial basis vectors for the requested time. - */ - Matrix* - getSpatialBasis( - double time) - { - return getSpatialBasis(); - } - /** * * @brief Returns the first n spatial basis vectors for the requested time @@ -108,16 +83,12 @@ class BasisReader { * * @pre 0 < n <= numColumns() * - * @param[in] time Time for which we want the basis vectors. - * NOTE: this argument is obsolete and remains only for backward compatibility. - * Will be removed in future. * @param[in] n The number of spatial basis vectors desired. * * @return The spatial basis vectors for the requested time. */ Matrix* getSpatialBasis( - double time, int n); /** @@ -128,9 +99,6 @@ class BasisReader { * @pre 0 < start_col <= numColumns() * @pre start_col <= end_col <= numColumns() * - * @param[in] time Time for which we want the basis vectors. - * NOTE: this argument is obsolete and remains only for backward compatibility. - * Will be removed in future. * @param[in] start_col The starting column desired. * @param[in] end_col The starting column desired. * @@ -138,7 +106,6 @@ class BasisReader { */ Matrix* getSpatialBasis( - double time, int start_col, int end_col); @@ -149,16 +116,12 @@ class BasisReader { * * @pre 0 <= ef <= 1.0 * - * @param[in] time Time for which we want the basis vectors. - * NOTE: this argument is obsolete and remains only for backward compatibility. - * Will be removed in future. * @param[in] ef The desired energy fraction. * * @return The spatial basis vectors for the requested time. */ Matrix* getSpatialBasis( - double time, double ef); /** diff --git a/regression_tests/basisComparator.cpp b/regression_tests/basisComparator.cpp index d4aca8fad..9ae74a840 100755 --- a/regression_tests/basisComparator.cpp +++ b/regression_tests/basisComparator.cpp @@ -37,13 +37,13 @@ void compareBasis(string &baselineFile, string &targetFile, double errorBound, CAROM::BasisReader baselineReader(baselineFile); CAROM::Matrix *baselineBasis = - (CAROM::Matrix*) baselineReader.getSpatialBasis(0.0); + (CAROM::Matrix*) baselineReader.getSpatialBasis(); CAROM::Vector *baselineSV = - (CAROM::Vector*) baselineReader.getSingularValues(0.0); + (CAROM::Vector*) baselineReader.getSingularValues(); CAROM::BasisReader targetReader(targetFile); - CAROM::Matrix *targetBasis = (CAROM::Matrix*) targetReader.getSpatialBasis(0.0); + CAROM::Matrix *targetBasis = (CAROM::Matrix*) targetReader.getSpatialBasis(); CAROM::BasisReader diffReader(baselineFile); - CAROM::Matrix *diffBasis = (CAROM::Matrix*) diffReader.getSpatialBasis(0.0); + CAROM::Matrix *diffBasis = (CAROM::Matrix*) diffReader.getSpatialBasis(); // Get basis dimensions int baselineNumRows = baselineBasis->numRows(); From 5e3a418caa0e2b4c7e99355c942d54bead1e8915 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Wed, 21 Feb 2024 14:06:41 -0800 Subject: [PATCH 24/50] changed function signature for BasisReader::getTemporalBasis. --- lib/linalg/BasisReader.cpp | 7 ++----- lib/linalg/BasisReader.h | 36 ------------------------------------ 2 files changed, 2 insertions(+), 41 deletions(-) diff --git a/lib/linalg/BasisReader.cpp b/lib/linalg/BasisReader.cpp index 874099ccd..eeda217a2 100644 --- a/lib/linalg/BasisReader.cpp +++ b/lib/linalg/BasisReader.cpp @@ -155,15 +155,13 @@ BasisReader::getTemporalBasis() Matrix* BasisReader::getTemporalBasis( - double time, int n) { - return getTemporalBasis(time, 1, n); + return getTemporalBasis(1, n); } Matrix* BasisReader::getTemporalBasis( - double time, int start_col, int end_col) { @@ -191,7 +189,6 @@ BasisReader::getTemporalBasis( Matrix* BasisReader::getTemporalBasis( - double time, double ef) { Vector* sv = getSingularValues(); @@ -214,7 +211,7 @@ BasisReader::getTemporalBasis( } delete sv; - return getTemporalBasis(time, num_used_singular_values); + return getTemporalBasis(num_used_singular_values); } Vector* diff --git a/lib/linalg/BasisReader.h b/lib/linalg/BasisReader.h index 75ca0edc3..aedfe632b 100644 --- a/lib/linalg/BasisReader.h +++ b/lib/linalg/BasisReader.h @@ -134,26 +134,6 @@ class BasisReader { Matrix* getTemporalBasis(); - /** - * - * @brief Returns the temporal basis vectors for the requested time as - * a Matrix. - * NOTE: this function is obsolete and remains only for backward compatibility. - * Will be removed in future. - * - * @param[in] time Time for which we want the basis vectors. - * NOTE: this argument is obsolete and remains only for backward compatibility. - * Will be removed in future. - * - * @return The temporal basis vectors for the requested time. - */ - Matrix* - getTemporalBasis( - double time) - { - return getTemporalBasis(); - } - /** * * @brief Returns the first n temporal basis vectors for the requested time @@ -161,16 +141,12 @@ class BasisReader { * * @pre 0 < n <= numColumns() * - * @param[in] time Time for which we want the basis vectors. - * NOTE: this argument is obsolete and remains only for backward compatibility. - * Will be removed in future. * @param[in] n The number of temporal basis vectors desired. * * @return The temporal basis vectors for the requested time. */ Matrix* getTemporalBasis( - double time, int n); /** @@ -181,9 +157,6 @@ class BasisReader { * @pre 0 < start_col <= numColumns() * @pre start_col <= end_col <= numColumns() * - * @param[in] time Time for which we want the basis vectors. - * NOTE: this argument is obsolete and remains only for backward compatibility. - * Will be removed in future. * @param[in] start_col The starting column desired. * @param[in] end_col The starting column desired. * @@ -191,7 +164,6 @@ class BasisReader { */ Matrix* getTemporalBasis( - double time, int start_col, int end_col); @@ -202,26 +174,18 @@ class BasisReader { * * @pre 0 <= ef <= 1.0 * - * @param[in] time Time for which we want the basis vectors. - * NOTE: this argument is obsolete and remains only for backward compatibility. - * Will be removed in future. * @param[in] ef The desired energy fraction. * * @return The temporal basis vectors for the requested time. */ Matrix* getTemporalBasis( - double time, double ef); /** * * @brief Returns the singular values for the requested time. * - * @param[in] time Time for which we want the basis vectors. - * NOTE: this argument is obsolete and remains only for backward compatibility. - * Will be removed in future. - * * @return The temporal basis vectors for the requested time. */ Vector* From 99beadbc5e8679074988a7d926c3b1095099e5ef Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Wed, 21 Feb 2024 14:08:12 -0800 Subject: [PATCH 25/50] changed function signature for BasisReader::getSingularValues. --- lib/linalg/BasisReader.cpp | 1 - lib/linalg/BasisReader.h | 23 ----------------------- 2 files changed, 24 deletions(-) diff --git a/lib/linalg/BasisReader.cpp b/lib/linalg/BasisReader.cpp index eeda217a2..ad5013779 100644 --- a/lib/linalg/BasisReader.cpp +++ b/lib/linalg/BasisReader.cpp @@ -237,7 +237,6 @@ BasisReader::getSingularValues() Vector* BasisReader::getSingularValues( - double time, double ef) { Vector* sv = getSingularValues(); diff --git a/lib/linalg/BasisReader.h b/lib/linalg/BasisReader.h index aedfe632b..481fbb4b2 100644 --- a/lib/linalg/BasisReader.h +++ b/lib/linalg/BasisReader.h @@ -191,25 +191,6 @@ class BasisReader { Vector* getSingularValues(); - /** - * - * @brief Returns the singular values for the requested time. - * NOTE: this function is obsolete and remains only for backward compatibility. - * Will be removed in future. - * - * @param[in] time Time for which we want the basis vectors. - * NOTE: this argument is obsolete and remains only for backward compatibility. - * Will be removed in future. - * - * @return The temporal basis vectors for the requested time. - */ - Vector* - getSingularValues( - double time) - { - return getSingularValues(); - } - /** * * @brief Returns the largest singular values for the requested time @@ -217,16 +198,12 @@ class BasisReader { * * @pre 0 <= ef <= 1.0 * - * @param[in] time Time for which we want the basis vectors. - * NOTE: this argument is obsolete and remains only for backward compatibility. - * Will be removed in future. * @param[in] ef The desired energy fraction. * * @return The temporal basis vectors for the requested time. */ Vector* getSingularValues( - double time, double ef); /** From 30ff8558d16aeb1acb2849031835c78e00c218dc Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Wed, 21 Feb 2024 14:11:06 -0800 Subject: [PATCH 26/50] changed function signature for BasisReader::getSnapshotMatrix. --- examples/misc/combine_samples.cpp | 4 +-- lib/linalg/BasisReader.cpp | 4 +-- lib/linalg/BasisReader.h | 59 ------------------------------- 3 files changed, 3 insertions(+), 64 deletions(-) diff --git a/examples/misc/combine_samples.cpp b/examples/misc/combine_samples.cpp index d4f10d4ce..f191368f6 100644 --- a/examples/misc/combine_samples.cpp +++ b/examples/misc/combine_samples.cpp @@ -101,8 +101,8 @@ int main(int argc, char* argv[]) for (const auto& sample_name: sample_names) { CAROM::BasisReader reader(sample_name); - dim = reader.getDim(kind, 0); - snaps += reader.getNumSamples(kind, 0); + dim = reader.getDim(kind); + snaps += reader.getNumSamples(kind); if (dimFirst == 0) dimFirst = dim; CAROM_VERIFY(dim == diff --git a/lib/linalg/BasisReader.cpp b/lib/linalg/BasisReader.cpp index ad5013779..cb00e2309 100644 --- a/lib/linalg/BasisReader.cpp +++ b/lib/linalg/BasisReader.cpp @@ -335,15 +335,13 @@ BasisReader::getSnapshotMatrix() Matrix* BasisReader::getSnapshotMatrix( - double time, int n) { - return getSnapshotMatrix(time, 1, n); + return getSnapshotMatrix(1, n); } Matrix* BasisReader::getSnapshotMatrix( - double time, int start_col, int end_col) { diff --git a/lib/linalg/BasisReader.h b/lib/linalg/BasisReader.h index 481fbb4b2..5a7090802 100644 --- a/lib/linalg/BasisReader.h +++ b/lib/linalg/BasisReader.h @@ -216,22 +216,6 @@ class BasisReader { getDim( const std::string kind); - /** - * - * @brief Returns the dimension of the system on this processor. - * NOTE: this function is obsolete and remains only for backward compatibility. - * Will be removed in future. - * - * @return The dimension of the system on this processor. - */ - int - getDim( - const std::string kind, - double time) - { - return getDim(kind); - } - /** * * @brief Returns the number of samples (columns) in file. @@ -242,22 +226,6 @@ class BasisReader { getNumSamples( const std::string kind); - /** - * - * @brief Returns the number of samples (columns) in file. - * NOTE: this function is obsolete and remains only for backward compatibility. - * Will be removed in future. - * - * @return The number of samples in file. - */ - int - getNumSamples( - const std::string kind, - double time) - { - return getNumSamples(kind); - } - /** * * @brief Returns the snapshot matrix for the requested time. @@ -267,41 +235,18 @@ class BasisReader { Matrix* getSnapshotMatrix(); - /** - * - * @brief Returns the snapshot matrix for the requested time. - * NOTE: this function is obsolete and remains only for backward compatibility. - * Will be removed in future. - * - * @param[in] time Time for which we want the basis vectors. - * NOTE: this argument is obsolete and remains only for backward compatibility. - * Will be removed in future. - * - * @return The snapshot matrix for the requested time. - */ - Matrix* - getSnapshotMatrix( - double time) - { - return getSnapshotMatrix(); - } - /** * * @brief Returns the first n columns of the snapshot matrix for the requested time. * * @pre 0 < n <= numColumns() * - * @param[in] time Time for which we want the basis vectors. - * NOTE: this argument is obsolete and remains only for backward compatibility. - * Will be removed in future. * @param[in] n The number of basis vectors desired. * * @return The snapshot matrix for the requested time. */ Matrix* getSnapshotMatrix( - double time, int n); /** @@ -311,9 +256,6 @@ class BasisReader { * @pre 0 < start_col <= numColumns() * @pre start_col <= end_col <= numColumns() * - * @param[in] time Time for which we want the basis vectors. - * NOTE: this argument is obsolete and remains only for backward compatibility. - * Will be removed in future. * @param[in] start_col The starting column desired. * @param[in] end_col The starting column desired. * @@ -321,7 +263,6 @@ class BasisReader { */ Matrix* getSnapshotMatrix( - double time, int start_col, int end_col); From 7eb20446817f8629ef5ee9c383eadf874244eee9 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Fri, 16 Feb 2024 09:48:18 -0800 Subject: [PATCH 27/50] unit test with fapl_mpi. --- CMakeLists.txt | 1 + unit_tests/test_HDFDatabase.cpp | 65 +++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 unit_tests/test_HDFDatabase.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 9a2b489f5..b27c5c3fb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -265,6 +265,7 @@ if(GTEST_FOUND) set(unit_test_stems Vector Matrix + HDFDatabase DEIM DMD GNAT diff --git a/unit_tests/test_HDFDatabase.cpp b/unit_tests/test_HDFDatabase.cpp new file mode 100644 index 000000000..feb4777cd --- /dev/null +++ b/unit_tests/test_HDFDatabase.cpp @@ -0,0 +1,65 @@ +/****************************************************************************** + * + * Copyright (c) 2013-2023, Lawrence Livermore National Security, LLC + * and other libROM project developers. See the top-level COPYRIGHT + * file for details. + * + * SPDX-License-Identifier: (Apache-2.0 OR MIT) + * + *****************************************************************************/ + +#ifdef CAROM_HAS_GTEST + +#include +#include "linalg/Matrix.h" +#include "linalg/Vector.h" +#include "utils/HDFDatabase.h" +#include +#include +#include +#include // for memcpy +#include +#include "mpi.h" +#include "utils/mpi_utils.h" + +/** + * Simple smoke test to make sure Google Test is properly linked + */ +TEST(GoogleTestFramework, GoogleTestFrameworkFound) { + SUCCEED(); +} + +TEST(HDFDatabase, Test_file_access_property_list) +{ + hid_t access_plist; + hid_t file_id; + + access_plist = H5Pcreate(H5P_FILE_ACCESS); + H5Pset_fapl_mpi(access_plist, MPI_COMM_WORLD, MPI_INFO_NULL); + + // H5Fopen must be called collectively + file_id = H5Fopen("test.h5", H5F_ACC_RDWR, access_plist); + + // H5Fclose must be called collectively + H5Fclose(file_id); +} + +int main(int argc, char* argv[]) +{ + ::testing::InitGoogleTest(&argc, argv); + MPI_Init(&argc, &argv); + int result = RUN_ALL_TESTS(); + MPI_Finalize(); + return result; +} + +#else // #ifndef CAROM_HAS_GTEST + +int main() +{ + std::cout << "libROM was compiled without Google Test support, so unit " + << "tests have been disabled. To enable unit tests, compile " + << "libROM with Google Test support." << std::endl; +} + +#endif // #endif CAROM_HAS_GTEST \ No newline at end of file From 6e809f24401d909905885323e00ec3ea030d5f78 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Fri, 16 Feb 2024 13:22:26 -0800 Subject: [PATCH 28/50] parallel integer array writing example. --- docker/Dockerfile | 2 +- unit_tests/test_HDFDatabase.cpp | 103 +++++++++++++++++++++++++++++--- 2 files changed, 96 insertions(+), 9 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 1cfdfc866..e3b5fe563 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -10,7 +10,7 @@ WORKDIR /$ENVDIR # install packages RUN sudo apt-get install -yq git RUN sudo apt-get install --no-install-recommends -yq make gcc gfortran libssl-dev cmake -RUN sudo apt-get install -yq libopenblas-dev libmpich-dev libblas-dev liblapack-dev libscalapack-mpi-dev libhdf5-serial-dev +RUN sudo apt-get install -yq libopenblas-dev libmpich-dev libblas-dev liblapack-dev libscalapack-mpi-dev libhdf5-mpi-dev RUN sudo apt-get install -yq vim RUN sudo apt-get install -yq git-lfs RUN sudo apt-get install -yq valgrind diff --git a/unit_tests/test_HDFDatabase.cpp b/unit_tests/test_HDFDatabase.cpp index feb4777cd..d3e5a3f7c 100644 --- a/unit_tests/test_HDFDatabase.cpp +++ b/unit_tests/test_HDFDatabase.cpp @@ -29,18 +29,105 @@ TEST(GoogleTestFramework, GoogleTestFrameworkFound) { SUCCEED(); } -TEST(HDFDatabase, Test_file_access_property_list) +TEST(HDF5, Test_parallel_writing) { - hid_t access_plist; + int nproc, rank; + MPI_Comm_size(MPI_COMM_WORLD, &nproc); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + + hid_t plist_id; hid_t file_id; + herr_t errf = 0; - access_plist = H5Pcreate(H5P_FILE_ACCESS); - H5Pset_fapl_mpi(access_plist, MPI_COMM_WORLD, MPI_INFO_NULL); - - // H5Fopen must be called collectively - file_id = H5Fopen("test.h5", H5F_ACC_RDWR, access_plist); + /* + * Set up file access property list with parallel I/O access + */ + plist_id = H5Pcreate(H5P_FILE_ACCESS); + H5Pset_fapl_mpio(plist_id, MPI_COMM_WORLD, MPI_INFO_NULL); + + /* + * OPTIONAL: It is generally recommended to set collective + * metadata reads/writes on FAPL to perform metadata reads + * collectively, which usually allows datasets + * to perform better at scale, although it is not + * strictly necessary. + */ + H5Pset_all_coll_metadata_ops(plist_id, true); + H5Pset_coll_metadata_write(plist_id, true); - // H5Fclose must be called collectively + /* + * Create a new file collectively and release property list identifier. + */ + file_id = H5Fcreate("test.h5", H5F_ACC_TRUNC, H5P_DEFAULT, plist_id); + H5Pclose(plist_id); + + /* + * Create the dataspace for the dataset. + */ + const int dim_rank = 1; + const int nelements = 100000; + hsize_t dim_global[dim_rank] = { static_cast(nelements) }; + hid_t filespace = H5Screate_simple(dim_rank, dim_global, NULL); + + /* + * Create the dataset with default properties and close filespace. + */ + hid_t dset_id = + H5Dcreate(file_id, "test-int", H5T_NATIVE_INT, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + H5Sclose(filespace); + + /* + * Each process defines dataset in memory and writes it to the hyperslab + * in the file. + */ + const int nelem_local = CAROM::split_dimension(nelements, MPI_COMM_WORLD); + std::vector offsets; + int dummy = CAROM::get_global_offsets(nelem_local, offsets, MPI_COMM_WORLD); + + /* hyperslab selection parameters */ + hsize_t count[dim_rank]; + hsize_t offset[dim_rank]; + count[0] = nelem_local; + // count[1] = dimsf[1]; + offset[0] = offsets[rank]; + // offset[1] = 0; + hid_t memspace = H5Screate_simple(dim_rank, count, NULL); + + /* + * Select hyperslab in the file. + */ + filespace = H5Dget_space(dset_id); + H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset, NULL, count, NULL); + + /* + * Initialize data buffer + */ + int data[nelem_local]; + for (int d = 0; d < nelem_local; d++) + data[d] = d + offsets[rank]; + + hid_t space = H5Screate_simple(1, dim_global, 0); + CAROM_VERIFY(space >= 0); + + /* + * Create property list for collective dataset write. + */ + plist_id = H5Pcreate(H5P_DATASET_XFER); + H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE); + + /* + * Write the data collectively. + */ + errf = H5Dwrite(dset_id, H5T_NATIVE_INT, memspace, filespace, plist_id, data); + CAROM_VERIFY(errf >= 0); + + /* + * Close/release resources. + */ + H5Dclose(dset_id); + H5Sclose(filespace); + H5Sclose(memspace); + H5Pclose(plist_id); H5Fclose(file_id); } From dd17b60e87abc98b7f8d341b36a429fc8df9f6f2 Mon Sep 17 00:00:00 2001 From: Seung Whan Chung Date: Fri, 16 Feb 2024 13:52:29 -0800 Subject: [PATCH 29/50] add timing for H5DWrite. --- unit_tests/test_HDFDatabase.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/unit_tests/test_HDFDatabase.cpp b/unit_tests/test_HDFDatabase.cpp index d3e5a3f7c..d412c8dd1 100644 --- a/unit_tests/test_HDFDatabase.cpp +++ b/unit_tests/test_HDFDatabase.cpp @@ -21,6 +21,8 @@ #include #include "mpi.h" #include "utils/mpi_utils.h" +#include +using namespace std::chrono; /** * Simple smoke test to make sure Google Test is properly linked @@ -65,7 +67,7 @@ TEST(HDF5, Test_parallel_writing) * Create the dataspace for the dataset. */ const int dim_rank = 1; - const int nelements = 100000; + const int nelements = 10000000; hsize_t dim_global[dim_rank] = { static_cast(nelements) }; hid_t filespace = H5Screate_simple(dim_rank, dim_global, NULL); @@ -118,8 +120,14 @@ TEST(HDF5, Test_parallel_writing) /* * Write the data collectively. */ + MPI_Barrier(MPI_COMM_WORLD); + const auto start = steady_clock::now(); errf = H5Dwrite(dset_id, H5T_NATIVE_INT, memspace, filespace, plist_id, data); CAROM_VERIFY(errf >= 0); + MPI_Barrier(MPI_COMM_WORLD); + const auto stop = steady_clock::now(); + const auto duration = duration_cast(stop-start); + printf("rank: %d, duration: %dms\n", rank, duration); /* * Close/release resources. @@ -149,4 +157,4 @@ int main() << "libROM with Google Test support." << std::endl; } -#endif // #endif CAROM_HAS_GTEST \ No newline at end of file +#endif // #endif CAROM_HAS_GTEST From 3f43c620b2ab39d0b8373cd9a558a093acfa228b Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Fri, 16 Feb 2024 14:08:11 -0800 Subject: [PATCH 30/50] 2d integer array parallel writing example. --- docker/Dockerfile | 2 +- unit_tests/test_HDFDatabase.cpp | 51 ++++++++++++++++----------------- 2 files changed, 26 insertions(+), 27 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index e3b5fe563..b7161cbfc 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -10,7 +10,7 @@ WORKDIR /$ENVDIR # install packages RUN sudo apt-get install -yq git RUN sudo apt-get install --no-install-recommends -yq make gcc gfortran libssl-dev cmake -RUN sudo apt-get install -yq libopenblas-dev libmpich-dev libblas-dev liblapack-dev libscalapack-mpi-dev libhdf5-mpi-dev +RUN sudo apt-get install -yq libopenblas-dev libmpich-dev libblas-dev liblapack-dev libscalapack-mpi-dev libhdf5-mpi-dev hdf5-tools RUN sudo apt-get install -yq vim RUN sudo apt-get install -yq git-lfs RUN sudo apt-get install -yq valgrind diff --git a/unit_tests/test_HDFDatabase.cpp b/unit_tests/test_HDFDatabase.cpp index d412c8dd1..31e208f85 100644 --- a/unit_tests/test_HDFDatabase.cpp +++ b/unit_tests/test_HDFDatabase.cpp @@ -37,6 +37,22 @@ TEST(HDF5, Test_parallel_writing) MPI_Comm_size(MPI_COMM_WORLD, &nproc); MPI_Comm_rank(MPI_COMM_WORLD, &rank); + const int dim_rank = 2; + const int nrow = 10000, ncol = 400; + const int nrow_local = CAROM::split_dimension(nrow, MPI_COMM_WORLD); + std::vector offsets; + int dummy = CAROM::get_global_offsets(nrow_local, offsets, MPI_COMM_WORLD); + + /* + * Initialize data buffer + */ + int data[nrow_local * ncol]; + for (int d = 0; d < nrow_local * ncol; d++) + data[d] = d + offsets[rank] * ncol; + + MPI_Barrier(MPI_COMM_WORLD); + const auto start = steady_clock::now(); + hid_t plist_id; hid_t file_id; herr_t errf = 0; @@ -66,9 +82,7 @@ TEST(HDF5, Test_parallel_writing) /* * Create the dataspace for the dataset. */ - const int dim_rank = 1; - const int nelements = 10000000; - hsize_t dim_global[dim_rank] = { static_cast(nelements) }; + hsize_t dim_global[dim_rank] = { static_cast(nrow), static_cast(ncol) }; hid_t filespace = H5Screate_simple(dim_rank, dim_global, NULL); /* @@ -82,17 +96,13 @@ TEST(HDF5, Test_parallel_writing) * Each process defines dataset in memory and writes it to the hyperslab * in the file. */ - const int nelem_local = CAROM::split_dimension(nelements, MPI_COMM_WORLD); - std::vector offsets; - int dummy = CAROM::get_global_offsets(nelem_local, offsets, MPI_COMM_WORLD); - /* hyperslab selection parameters */ hsize_t count[dim_rank]; hsize_t offset[dim_rank]; - count[0] = nelem_local; - // count[1] = dimsf[1]; + count[0] = nrow_local; + count[1] = ncol; offset[0] = offsets[rank]; - // offset[1] = 0; + offset[1] = 0; hid_t memspace = H5Screate_simple(dim_rank, count, NULL); /* @@ -101,16 +111,6 @@ TEST(HDF5, Test_parallel_writing) filespace = H5Dget_space(dset_id); H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset, NULL, count, NULL); - /* - * Initialize data buffer - */ - int data[nelem_local]; - for (int d = 0; d < nelem_local; d++) - data[d] = d + offsets[rank]; - - hid_t space = H5Screate_simple(1, dim_global, 0); - CAROM_VERIFY(space >= 0); - /* * Create property list for collective dataset write. */ @@ -120,14 +120,8 @@ TEST(HDF5, Test_parallel_writing) /* * Write the data collectively. */ - MPI_Barrier(MPI_COMM_WORLD); - const auto start = steady_clock::now(); errf = H5Dwrite(dset_id, H5T_NATIVE_INT, memspace, filespace, plist_id, data); CAROM_VERIFY(errf >= 0); - MPI_Barrier(MPI_COMM_WORLD); - const auto stop = steady_clock::now(); - const auto duration = duration_cast(stop-start); - printf("rank: %d, duration: %dms\n", rank, duration); /* * Close/release resources. @@ -137,6 +131,11 @@ TEST(HDF5, Test_parallel_writing) H5Sclose(memspace); H5Pclose(plist_id); H5Fclose(file_id); + + MPI_Barrier(MPI_COMM_WORLD); + const auto stop = steady_clock::now(); + const auto duration = duration_cast(stop-start); + printf("rank: %d, duration: %dms\n", rank, duration); } int main(int argc, char* argv[]) From fc51f9e493843e3bd967fe7f80d176cd2898a3fb Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Fri, 16 Feb 2024 14:58:18 -0800 Subject: [PATCH 31/50] hdf5 parallel integer array reading example. --- unit_tests/test_HDFDatabase.cpp | 119 +++++++++++++++++++++++++++++++- 1 file changed, 117 insertions(+), 2 deletions(-) diff --git a/unit_tests/test_HDFDatabase.cpp b/unit_tests/test_HDFDatabase.cpp index 31e208f85..0a7958c9a 100644 --- a/unit_tests/test_HDFDatabase.cpp +++ b/unit_tests/test_HDFDatabase.cpp @@ -38,7 +38,7 @@ TEST(HDF5, Test_parallel_writing) MPI_Comm_rank(MPI_COMM_WORLD, &rank); const int dim_rank = 2; - const int nrow = 10000, ncol = 400; + const int nrow = 1000, ncol = 400; const int nrow_local = CAROM::split_dimension(nrow, MPI_COMM_WORLD); std::vector offsets; int dummy = CAROM::get_global_offsets(nrow_local, offsets, MPI_COMM_WORLD); @@ -56,7 +56,7 @@ TEST(HDF5, Test_parallel_writing) hid_t plist_id; hid_t file_id; herr_t errf = 0; - + /* * Set up file access property list with parallel I/O access */ @@ -138,6 +138,121 @@ TEST(HDF5, Test_parallel_writing) printf("rank: %d, duration: %dms\n", rank, duration); } +TEST(HDF5, Test_parallel_reading) +{ + int nproc, rank; + MPI_Comm_size(MPI_COMM_WORLD, &nproc); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + + const int dim_rank = 2; + const int nrow = 1000, ncol = 400; + const int nrow_local = CAROM::split_dimension(nrow, MPI_COMM_WORLD); + std::vector offsets; + int dummy = CAROM::get_global_offsets(nrow_local, offsets, MPI_COMM_WORLD); + + /* + * Initialize data buffer + */ + int data[nrow_local * ncol]; + + MPI_Barrier(MPI_COMM_WORLD); + const auto start = steady_clock::now(); + + hid_t plist_id; + hid_t file_id; + herr_t errf = 0; + + /* + * Set up file access property list with parallel I/O access + */ + plist_id = H5Pcreate(H5P_FILE_ACCESS); + H5Pset_fapl_mpio(plist_id, MPI_COMM_WORLD, MPI_INFO_NULL); + + /* + * OPTIONAL: It is generally recommended to set collective + * metadata reads/writes on FAPL to perform metadata reads + * collectively, which usually allows datasets + * to perform better at scale, although it is not + * strictly necessary. + */ + H5Pset_all_coll_metadata_ops(plist_id, true); + H5Pset_coll_metadata_write(plist_id, true); + + /* + * Open a new file collectively and release property list identifier. + */ + file_id = H5Fopen("test.h5", H5F_ACC_RDONLY, plist_id); + H5Pclose(plist_id); + + /* + * Open the dataset with default properties. + */ + hid_t dset_id = H5Dopen(file_id, "test-int", H5P_DEFAULT); + + /* + * Get filespace and read the global size. + */ + hid_t filespace = H5Dget_space(dset_id); + CAROM_VERIFY(filespace >= 0); + int ndims = H5Sget_simple_extent_ndims(filespace); + CAROM_VERIFY(ndims == dim_rank); + hsize_t dim_global[dim_rank]; + errf = H5Sget_simple_extent_dims(filespace, dim_global, NULL); + CAROM_VERIFY(errf >= 0); + CAROM_VERIFY(dim_global[0] == nrow); + CAROM_VERIFY(dim_global[1] == ncol); + + H5Sclose(filespace); + + /* + * Each process defines dataset in memory and writes it to the hyperslab + * in the file. + */ + /* hyperslab selection parameters */ + hsize_t count[dim_rank]; + hsize_t offset[dim_rank]; + count[0] = nrow_local; + count[1] = ncol; + offset[0] = offsets[rank]; + offset[1] = 0; + hid_t memspace = H5Screate_simple(dim_rank, count, NULL); + + /* + * Select hyperslab in the file. + */ + filespace = H5Dget_space(dset_id); + H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset, NULL, count, NULL); + + /* + * Create property list for collective dataset write. + */ + plist_id = H5Pcreate(H5P_DATASET_XFER); + H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE); + + /* + * Read the data collectively. + */ + errf = H5Dread(dset_id, H5T_NATIVE_INT, memspace, filespace, plist_id, data); + CAROM_VERIFY(errf >= 0); + + /* + * Close/release resources. + */ + H5Dclose(dset_id); + H5Sclose(filespace); + H5Sclose(memspace); + H5Pclose(plist_id); + H5Fclose(file_id); + + MPI_Barrier(MPI_COMM_WORLD); + const auto stop = steady_clock::now(); + const auto duration = duration_cast(stop-start); + printf("rank: %d, duration: %dms\n", rank, duration); + + for (int d = 0; d < nrow_local * ncol; d++) + EXPECT_TRUE(data[d] == d + offsets[rank] * ncol); +} + int main(int argc, char* argv[]) { ::testing::InitGoogleTest(&argc, argv); From 62366be4e6d16ea5359e1f5afd9245fe298df9d4 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Sat, 17 Feb 2024 09:25:14 -0800 Subject: [PATCH 32/50] add d_dim as const member variable of BasisGenerator/Reader. --- lib/linalg/BasisGenerator.cpp | 1 + lib/linalg/BasisGenerator.h | 9 ++++++++- lib/linalg/BasisReader.cpp | 4 +++- lib/linalg/BasisReader.h | 12 +++++++++++- lib/linalg/svd/SVD.h | 2 +- 5 files changed, 24 insertions(+), 4 deletions(-) diff --git a/lib/linalg/BasisGenerator.cpp b/lib/linalg/BasisGenerator.cpp index 942efa9b2..6b2018df7 100644 --- a/lib/linalg/BasisGenerator.cpp +++ b/lib/linalg/BasisGenerator.cpp @@ -28,6 +28,7 @@ BasisGenerator::BasisGenerator( bool incremental, const std::string& basis_file_name, Database::formats file_format) : + d_dim(options.dim), d_incremental(incremental), d_basis_writer(0), d_basis_reader(0), diff --git a/lib/linalg/BasisGenerator.h b/lib/linalg/BasisGenerator.h index 7ba7b02c5..131633415 100644 --- a/lib/linalg/BasisGenerator.h +++ b/lib/linalg/BasisGenerator.h @@ -288,7 +288,7 @@ class BasisGenerator int getDim() { - return d_svd->getDim(); + return d_dim; } /** @@ -342,6 +342,13 @@ class BasisGenerator * @brief The number of processors being run on. */ int d_num_procs; + + /** + * @brief Dimension of the system on this processor. + * + * Equivalent to d_svd->getDim(). + */ + const int d_dim; }; } diff --git a/lib/linalg/BasisReader.cpp b/lib/linalg/BasisReader.cpp index cb00e2309..c5836156c 100644 --- a/lib/linalg/BasisReader.cpp +++ b/lib/linalg/BasisReader.cpp @@ -21,7 +21,9 @@ namespace CAROM { BasisReader::BasisReader( const std::string& base_file_name, - Database::formats db_format) : + Database::formats db_format, + const int dim) : + d_dim(dim), d_last_basis_idx(-1), full_file_name(""), base_file_name_(base_file_name) diff --git a/lib/linalg/BasisReader.h b/lib/linalg/BasisReader.h index 5a7090802..6e57b3cbd 100644 --- a/lib/linalg/BasisReader.h +++ b/lib/linalg/BasisReader.h @@ -44,10 +44,13 @@ class BasisReader { * @param[in] db_format Format of the file to read. * One of the implemented file formats defined in * Database. + * @param[in] dim Dimension of the basis that will be read from a file. + * If negative, will use the dimension from the rank-specific local file. */ BasisReader( const std::string& base_file_name, - Database::formats db_format = Database::HDF5); + Database::formats db_format = Database::HDF5, + const int dim = -1); /** * @brief Destructor. @@ -304,6 +307,13 @@ class BasisReader { * @brief The last time at which basis vectors were requested. */ int d_last_basis_idx; + + /** + * @brief Dimension of the basis on this processor. + * + * If negative, use the dimension from the rank-specific local file. + */ + const int d_dim; }; } diff --git a/lib/linalg/svd/SVD.h b/lib/linalg/svd/SVD.h index 25a4662b0..fa26f6e6f 100644 --- a/lib/linalg/svd/SVD.h +++ b/lib/linalg/svd/SVD.h @@ -142,7 +142,7 @@ class SVD /** * @brief Dimension of the system. */ - int d_dim; + const int d_dim; /** * @brief Number of samples stored for the current time interval. From cd347ec3ed5b6518ec2d41d376bab0a0af5d56a5 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Sat, 17 Feb 2024 10:35:01 -0800 Subject: [PATCH 33/50] BasisReader: CSV format is never used. Enforce HDF5 format from now on. --- lib/linalg/BasisReader.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/linalg/BasisReader.cpp b/lib/linalg/BasisReader.cpp index c5836156c..76cacfa22 100644 --- a/lib/linalg/BasisReader.cpp +++ b/lib/linalg/BasisReader.cpp @@ -40,15 +40,15 @@ BasisReader::BasisReader( rank = 0; } + // Enforce hdf data format. + CAROM_VERIFY(db_format != Database::CSV); + char tmp[100]; sprintf(tmp, ".%06d", rank); full_file_name = base_file_name + tmp; if (db_format == Database::HDF5) { d_database = new HDFDatabase(); } - else if (db_format == Database::CSV) { - d_database = new CSVDatabase(); - } d_database->open(full_file_name, "r"); } From 281eb9a4fe754685af9658a4abec7236917710f2 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Sat, 17 Feb 2024 10:37:21 -0800 Subject: [PATCH 34/50] create/open_parallel function. --- lib/utils/CSVDatabase.h | 54 ++++++++++++++++++++++++++-------- lib/utils/Database.h | 37 +++++++++++++++++++++++ lib/utils/HDFDatabase.cpp | 1 - lib/utils/HDFDatabase.h | 62 ++++++++++++++++++++++++++++++++++++++- 4 files changed, 139 insertions(+), 15 deletions(-) diff --git a/lib/utils/CSVDatabase.h b/lib/utils/CSVDatabase.h index 9eefa1f8a..ba5fe333d 100644 --- a/lib/utils/CSVDatabase.h +++ b/lib/utils/CSVDatabase.h @@ -14,6 +14,7 @@ #define included_CSVDatabase_h #include "Database.h" +#include "Utilities.h" #include #include #include @@ -39,25 +40,27 @@ class CSVDatabase : public Database /** * @brief Creates a new CSV database file with the supplied name. + * NOTE: CSVDatabase does not actually create a file with this. + * This function will only print out the file_name. * * @param[in] file_name Name of CSV database file to create. * * @return True if file create was successful. */ - virtual bool create( const std::string& file_name) override; /** * @brief Opens an existing CSV database file with the supplied name. + * NOTE: CSVDatabase does not actually open a file with this. + * This function will only print out the file_name. * * @param[in] file_name Name of existing CSV database file to open. * @param[in] type Read/write type ("r"/"wr") * * @return True if file open was successful. */ - virtual bool open( const std::string& file_name, @@ -68,7 +71,6 @@ class CSVDatabase : public Database * * @return True if the file close was successful. */ - virtual bool close(); @@ -84,7 +86,6 @@ class CSVDatabase : public Database * @param[in] data The array of integer values to be written. * @param[in] nelements The number of integers in the array. */ - virtual void putIntegerArray( const std::string& file_name, @@ -103,7 +104,6 @@ class CSVDatabase : public Database * @param[in] data The array of double values to be written. * @param[in] nelements The number of doubles in the array. */ - virtual void putDoubleArray( const std::string& file_name, @@ -123,7 +123,6 @@ class CSVDatabase : public Database * @param[in] data The vector of double values to be written. * @param[in] nelements The number of doubles in the vector. */ - virtual void putDoubleVector( const std::string& file_name, @@ -142,7 +141,6 @@ class CSVDatabase : public Database * @param[in] data The vector of complex double values to be written. * @param[in] nelements The number of complex doubles in the vector. */ - virtual void putComplexVector( const std::string& file_name, @@ -161,7 +159,6 @@ class CSVDatabase : public Database * @param[in] data The vector of strings to be written. * @param[in] nelements The number of strings in the vector. */ - virtual void putStringVector( const std::string& file_name, @@ -179,7 +176,6 @@ class CSVDatabase : public Database * @param[out] data The allocated array of integer values to be read. * @param[in] nelements The number of integers in the array. */ - virtual void getIntegerArray( const std::string& file_name, @@ -211,7 +207,6 @@ class CSVDatabase : public Database * @param[in] file_name The filename associated with the array of values to be * read. */ - virtual int getDoubleArraySize(const std::string& file_name) { @@ -231,7 +226,6 @@ class CSVDatabase : public Database * @param[out] data The allocated array of double values to be read. * @param[in] nelements The number of doubles in the array. */ - virtual void getDoubleArray( const std::string& file_name, @@ -250,7 +244,6 @@ class CSVDatabase : public Database * @param[in] nelements The number of doubles in the full array. * @param[in] idx The set of indices in the sub-array. */ - virtual void getDoubleArray( const std::string& file_name, @@ -272,7 +265,6 @@ class CSVDatabase : public Database * @param[in] block_size The block size to read from the CSV dataset. * @param[in] stride The stride to read from the CSV dataset. */ - virtual void getDoubleArray( const std::string& file_name, @@ -326,6 +318,42 @@ class CSVDatabase : public Database getLineCount( const std::string& file_name); + /** + * @brief Unsupported parallel I/O function. + * + * @param[in] file_name Base Name of CSV database file to create. + * @param[in] comm MPI communicator to get the rank. + * + * @return True if file create was successful. + */ + bool + create_parallel( + const std::string& file_name, + const MPI_Comm comm) + { + CAROM_ERROR("CSVDatabase does not support parallel I/O!\n"); + return false; + } + + /** + * @brief Unsupported parallel I/O function. + * + * @param[in] file_name Name of existing CSV database file to open. + * @param[in] type Read/write type ("r"/"wr") + * @param[in] comm MPI communicator to get the rank. + * + * @return True if file open was successful. + */ + bool + open_parallel( + const std::string& file_name, + const std::string& type, + const MPI_Comm comm) + { + CAROM_ERROR("CSVDatabase does not support parallel I/O!\n"); + return false; + } + private: /** * @brief Unimplemented copy constructor. diff --git a/lib/utils/Database.h b/lib/utils/Database.h index e839da863..4320bb35b 100644 --- a/lib/utils/Database.h +++ b/lib/utils/Database.h @@ -15,6 +15,7 @@ #include #include +#include "mpi.h" namespace CAROM { @@ -51,6 +52,23 @@ class Database create( const std::string& file_name); + /** + * @brief Creates a new database file with the supplied name, with parallel I/O support. + * Supported only for HDF5 format. + * For HDFDatabase, the function is equivalent to create, + * only extending the file name with 6 digits indicating processor rank. + * For HDFDatabaseMPIO, the file is created with MPI I/O file access property. + * + * @param[in] file_name Name of database file to create. + * + * @return True if file create was successful. + */ + virtual + bool + create_parallel( + const std::string& file_name, + const MPI_Comm comm) = 0; + /** * @brief Opens an existing database file with the supplied name. * @@ -65,6 +83,25 @@ class Database const std::string& file_name, const std::string& type); + /** + * @brief Opens an existing database file with the supplied name, with parallel I/O support. + * Supported only for HDF5 format. + * For HDFDatabase, the function is equivalent to open, + * only extending the file name with 6 digits indicating processor rank. + * For HDFDatabaseMPIO, the file is opened with MPI I/O file access property. + * + * @param[in] file_name Name of existing database file to open. + * @param[in] type Read/write type ("r"/"wr") + * + * @return True if file open was successful. + */ + virtual + bool + open_parallel( + const std::string& file_name, + const std::string& type, + const MPI_Comm comm) = 0; + /** * @brief Closes the currently open database file. * diff --git a/lib/utils/HDFDatabase.cpp b/lib/utils/HDFDatabase.cpp index 60bd0ad69..f94d56ad8 100644 --- a/lib/utils/HDFDatabase.cpp +++ b/lib/utils/HDFDatabase.cpp @@ -11,7 +11,6 @@ // Description: The concrete database implementation using HDF5. #include "HDFDatabase.h" -#include "Utilities.h" #include namespace CAROM { diff --git a/lib/utils/HDFDatabase.h b/lib/utils/HDFDatabase.h index d91e707d6..3965f0255 100644 --- a/lib/utils/HDFDatabase.h +++ b/lib/utils/HDFDatabase.h @@ -14,6 +14,7 @@ #define included_HDFDatabase_h #include "Database.h" +#include "Utilities.h" #include "hdf5.h" #include @@ -62,6 +63,59 @@ class HDFDatabase : public Database const std::string& file_name, const std::string& type) override; + /** + * @brief Creates a new HDF5 database file with the supplied name, + * extending it with 6 digits indicating the process rank. + * + * @param[in] file_name Name of HDF5 database file to create. + * @param[in] comm MPI communicator to obtain the process rank. + * + * @return True if file create was successful. + */ + virtual bool + create_parallel( + const std::string& file_name, + const MPI_Comm comm) override + { + CAROM_VERIFY(!file_name.empty()); + MPI_Comm_rank(comm, &d_rank); + + std::string ext_filename(file_name); + char tmp[10]; + sprintf(tmp, ".%06d", d_rank); + ext_filename += tmp; + + return create(ext_filename); + } + + /** + * @brief Opens an existing HDF5 database file with the supplied name, + * extending it with 6 digits indicating the process rank. + * + * @param[in] file_name Name of existing HDF5 database file to open. + * @param[in] type Read/write type ("r"/"wr") + * @param[in] comm MPI communicator to obtain the process rank. + * + * @return True if file open was successful. + */ + virtual + bool + open_parallel( + const std::string& file_name, + const std::string& type, + const MPI_Comm comm) override + { + CAROM_VERIFY(!file_name.empty()); + MPI_Comm_rank(comm, &d_rank); + + std::string ext_filename(file_name); + char tmp[10]; + sprintf(tmp, ".%06d", d_rank); + ext_filename += tmp; + + return open(ext_filename, type); + } + /** * @brief Closes the currently open HDF5 database file. * @@ -312,7 +366,13 @@ class HDFDatabase : public Database /** * @brief The key representing an integer array. * */ - static const int KEY_INT_ARRAY; + static const int KEY_INT_ARRAY; + + /** + * @brief MPI process rank. + * Used only when d_parallel is true. + */ + int d_rank; }; } From a6f8c1206214cdf547938b21858491630a9dc37e Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Sat, 17 Feb 2024 15:03:21 -0800 Subject: [PATCH 35/50] stylization --- lib/linalg/BasisGenerator.h | 2 +- lib/linalg/BasisReader.h | 2 +- lib/utils/HDFDatabase.h | 2 +- unit_tests/test_HDFDatabase.cpp | 15 ++++++++------- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/lib/linalg/BasisGenerator.h b/lib/linalg/BasisGenerator.h index 131633415..17a4328b9 100644 --- a/lib/linalg/BasisGenerator.h +++ b/lib/linalg/BasisGenerator.h @@ -345,7 +345,7 @@ class BasisGenerator /** * @brief Dimension of the system on this processor. - * + * * Equivalent to d_svd->getDim(). */ const int d_dim; diff --git a/lib/linalg/BasisReader.h b/lib/linalg/BasisReader.h index 6e57b3cbd..5ed12d371 100644 --- a/lib/linalg/BasisReader.h +++ b/lib/linalg/BasisReader.h @@ -310,7 +310,7 @@ class BasisReader { /** * @brief Dimension of the basis on this processor. - * + * * If negative, use the dimension from the rank-specific local file. */ const int d_dim; diff --git a/lib/utils/HDFDatabase.h b/lib/utils/HDFDatabase.h index 3965f0255..ce1af48fa 100644 --- a/lib/utils/HDFDatabase.h +++ b/lib/utils/HDFDatabase.h @@ -366,7 +366,7 @@ class HDFDatabase : public Database /** * @brief The key representing an integer array. * */ - static const int KEY_INT_ARRAY; + static const int KEY_INT_ARRAY; /** * @brief MPI process rank. diff --git a/unit_tests/test_HDFDatabase.cpp b/unit_tests/test_HDFDatabase.cpp index 0a7958c9a..0d096319f 100644 --- a/unit_tests/test_HDFDatabase.cpp +++ b/unit_tests/test_HDFDatabase.cpp @@ -56,7 +56,7 @@ TEST(HDF5, Test_parallel_writing) hid_t plist_id; hid_t file_id; herr_t errf = 0; - + /* * Set up file access property list with parallel I/O access */ @@ -72,7 +72,7 @@ TEST(HDF5, Test_parallel_writing) */ H5Pset_all_coll_metadata_ops(plist_id, true); H5Pset_coll_metadata_write(plist_id, true); - + /* * Create a new file collectively and release property list identifier. */ @@ -89,7 +89,8 @@ TEST(HDF5, Test_parallel_writing) * Create the dataset with default properties and close filespace. */ hid_t dset_id = - H5Dcreate(file_id, "test-int", H5T_NATIVE_INT, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + H5Dcreate(file_id, "test-int", H5T_NATIVE_INT, filespace, H5P_DEFAULT, + H5P_DEFAULT, H5P_DEFAULT); H5Sclose(filespace); /* @@ -97,7 +98,7 @@ TEST(HDF5, Test_parallel_writing) * in the file. */ /* hyperslab selection parameters */ - hsize_t count[dim_rank]; + hsize_t count[dim_rank]; hsize_t offset[dim_rank]; count[0] = nrow_local; count[1] = ncol; @@ -161,7 +162,7 @@ TEST(HDF5, Test_parallel_reading) hid_t plist_id; hid_t file_id; herr_t errf = 0; - + /* * Set up file access property list with parallel I/O access */ @@ -177,7 +178,7 @@ TEST(HDF5, Test_parallel_reading) */ H5Pset_all_coll_metadata_ops(plist_id, true); H5Pset_coll_metadata_write(plist_id, true); - + /* * Open a new file collectively and release property list identifier. */ @@ -209,7 +210,7 @@ TEST(HDF5, Test_parallel_reading) * in the file. */ /* hyperslab selection parameters */ - hsize_t count[dim_rank]; + hsize_t count[dim_rank]; hsize_t offset[dim_rank]; count[0] = nrow_local; count[1] = ncol; From 472a3e5b05836f856f74470976145038ece76d56 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Tue, 20 Feb 2024 12:26:31 -0800 Subject: [PATCH 36/50] test_HDFDatabase: test for selective parallal I/O. --- unit_tests/test_HDFDatabase.cpp | 259 ++++++++++++++++++++++++++++++++ 1 file changed, 259 insertions(+) diff --git a/unit_tests/test_HDFDatabase.cpp b/unit_tests/test_HDFDatabase.cpp index 0d096319f..d8ada61f8 100644 --- a/unit_tests/test_HDFDatabase.cpp +++ b/unit_tests/test_HDFDatabase.cpp @@ -254,6 +254,265 @@ TEST(HDF5, Test_parallel_reading) EXPECT_TRUE(data[d] == d + offsets[rank] * ncol); } +TEST(HDF5, Test_selective_parallel_writing) +{ + int nproc, rank; + MPI_Comm_size(MPI_COMM_WORLD, &nproc); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + + /* Only the first 2 processes will participate in I/O. */ + const int ioproc = std::min(2, nproc); + int iorank[ioproc]; + for (int r = 0; r < ioproc; r++) + iorank[r] = r; + MPI_Group world_grp, io_grp; + MPI_Comm_group(MPI_COMM_WORLD, &world_grp); + MPI_Group_incl(world_grp, ioproc, iorank, &io_grp); + MPI_Comm io_comm; + MPI_Comm_create(MPI_COMM_WORLD, io_grp, &io_comm); + + const int dim_rank = 2; + const int nrow = 10, ncol = 4; + int nrow_local = 0; + if (rank < ioproc) + nrow_local = CAROM::split_dimension(nrow, io_comm); + printf("rank %d, I/O row size: %d\n", rank, nrow_local); + std::vector offsets; + int dummy = CAROM::get_global_offsets(nrow_local, offsets, MPI_COMM_WORLD); + for (int k = offsets.size(); k < nproc; k++) + offsets.push_back(offsets.back()); + + /* + * Initialize data buffer + */ + int data[nrow_local * ncol]; + for (int d = 0; d < nrow_local * ncol; d++) + data[d] = d + offsets[rank] * ncol; + + MPI_Barrier(MPI_COMM_WORLD); + const auto start = steady_clock::now(); + + hid_t plist_id; + hid_t file_id; + herr_t errf = 0; + + /* + * Set up file access property list with parallel I/O access + */ + plist_id = H5Pcreate(H5P_FILE_ACCESS); + H5Pset_fapl_mpio(plist_id, MPI_COMM_WORLD, MPI_INFO_NULL); + + /* + * OPTIONAL: It is generally recommended to set collective + * metadata reads/writes on FAPL to perform metadata reads + * collectively, which usually allows datasets + * to perform better at scale, although it is not + * strictly necessary. + */ + H5Pset_all_coll_metadata_ops(plist_id, true); + H5Pset_coll_metadata_write(plist_id, true); + + /* + * Create a new file collectively and release property list identifier. + */ + file_id = H5Fcreate("test.h5", H5F_ACC_TRUNC, H5P_DEFAULT, plist_id); + H5Pclose(plist_id); + + /* + * Create the dataspace for the dataset. + */ + hsize_t dim_global[dim_rank] = { static_cast(nrow), static_cast(ncol) }; + hid_t filespace = H5Screate_simple(dim_rank, dim_global, NULL); + + /* + * Create the dataset with default properties and close filespace. + */ + hid_t dset_id = + H5Dcreate(file_id, "test-int", H5T_NATIVE_INT, filespace, H5P_DEFAULT, + H5P_DEFAULT, H5P_DEFAULT); + H5Sclose(filespace); + + /* + * Each process defines dataset in memory and writes it to the hyperslab + * in the file. + */ + /* hyperslab selection parameters */ + hsize_t count[dim_rank]; + hsize_t offset[dim_rank]; + count[0] = nrow_local; + count[1] = ncol; + offset[0] = offsets[rank]; + offset[1] = 0; + hid_t memspace = H5Screate_simple(dim_rank, count, NULL); + + /* + * Select hyperslab in the file. + */ + filespace = H5Dget_space(dset_id); + if (rank < ioproc) + H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset, NULL, count, NULL); + else + H5Sselect_none(filespace); + + /* + * Create property list for collective dataset write. + */ + plist_id = H5Pcreate(H5P_DATASET_XFER); + H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE); + + /* + * Write the data collectively. + */ + errf = H5Dwrite(dset_id, H5T_NATIVE_INT, memspace, filespace, plist_id, data); + CAROM_VERIFY(errf >= 0); + + /* + * Close/release resources. + */ + H5Dclose(dset_id); + H5Sclose(filespace); + H5Sclose(memspace); + H5Pclose(plist_id); + H5Fclose(file_id); + + MPI_Barrier(MPI_COMM_WORLD); + const auto stop = steady_clock::now(); + const auto duration = duration_cast(stop-start); + printf("rank: %d, duration: %dms\n", rank, duration); +} + +TEST(HDF5, Test_selective_parallel_reading) +{ + int nproc, rank; + MPI_Comm_size(MPI_COMM_WORLD, &nproc); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + + /* Only the first 2 processes will participate in I/O. */ + const int ioproc = std::min(2, nproc); + int iorank[ioproc]; + for (int r = 0; r < ioproc; r++) + iorank[r] = r; + MPI_Group world_grp, io_grp; + MPI_Comm_group(MPI_COMM_WORLD, &world_grp); + MPI_Group_incl(world_grp, ioproc, iorank, &io_grp); + MPI_Comm io_comm; + MPI_Comm_create(MPI_COMM_WORLD, io_grp, &io_comm); + + const int dim_rank = 2; + const int nrow = 10, ncol = 4; + int nrow_local = 0; + if (rank < ioproc) + nrow_local = CAROM::split_dimension(nrow, io_comm); + printf("rank %d, I/O row size: %d\n", rank, nrow_local); + std::vector offsets; + int dummy = CAROM::get_global_offsets(nrow_local, offsets, MPI_COMM_WORLD); + + /* + * Initialize data buffer + */ + int data[nrow_local * ncol]; + + MPI_Barrier(MPI_COMM_WORLD); + const auto start = steady_clock::now(); + + hid_t plist_id; + hid_t file_id; + herr_t errf = 0; + + /* + * Set up file access property list with parallel I/O access + */ + plist_id = H5Pcreate(H5P_FILE_ACCESS); + H5Pset_fapl_mpio(plist_id, MPI_COMM_WORLD, MPI_INFO_NULL); + + /* + * OPTIONAL: It is generally recommended to set collective + * metadata reads/writes on FAPL to perform metadata reads + * collectively, which usually allows datasets + * to perform better at scale, although it is not + * strictly necessary. + */ + H5Pset_all_coll_metadata_ops(plist_id, true); + H5Pset_coll_metadata_write(plist_id, true); + + /* + * Open a new file collectively and release property list identifier. + */ + file_id = H5Fopen("test.h5", H5F_ACC_RDONLY, plist_id); + H5Pclose(plist_id); + + /* + * Open the dataset with default properties. + */ + hid_t dset_id = H5Dopen(file_id, "test-int", H5P_DEFAULT); + + /* + * Get filespace and read the global size. + */ + hid_t filespace = H5Dget_space(dset_id); + CAROM_VERIFY(filespace >= 0); + int ndims = H5Sget_simple_extent_ndims(filespace); + CAROM_VERIFY(ndims == dim_rank); + hsize_t dim_global[dim_rank]; + errf = H5Sget_simple_extent_dims(filespace, dim_global, NULL); + CAROM_VERIFY(errf >= 0); + CAROM_VERIFY(dim_global[0] == nrow); + CAROM_VERIFY(dim_global[1] == ncol); + + H5Sclose(filespace); + + /* + * Each process defines dataset in memory and writes it to the hyperslab + * in the file. + */ + /* hyperslab selection parameters */ + hsize_t count[dim_rank]; + hsize_t offset[dim_rank]; + count[0] = nrow_local; + count[1] = ncol; + offset[0] = offsets[rank]; + offset[1] = 0; + hid_t memspace = H5Screate_simple(dim_rank, count, NULL); + + /* + * Select hyperslab in the file. + */ + filespace = H5Dget_space(dset_id); + if (rank < ioproc) + H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset, NULL, count, NULL); + else + H5Sselect_none(filespace); + + /* + * Create property list for collective dataset write. + */ + plist_id = H5Pcreate(H5P_DATASET_XFER); + H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE); + + /* + * Read the data collectively. + */ + errf = H5Dread(dset_id, H5T_NATIVE_INT, memspace, filespace, plist_id, data); + CAROM_VERIFY(errf >= 0); + + /* + * Close/release resources. + */ + H5Dclose(dset_id); + H5Sclose(filespace); + H5Sclose(memspace); + H5Pclose(plist_id); + H5Fclose(file_id); + + MPI_Barrier(MPI_COMM_WORLD); + const auto stop = steady_clock::now(); + const auto duration = duration_cast(stop-start); + printf("rank: %d, duration: %dms\n", rank, duration); + + for (int d = 0; d < nrow_local * ncol; d++) + EXPECT_TRUE(data[d] == d + offsets[rank] * ncol); +} + int main(int argc, char* argv[]) { ::testing::InitGoogleTest(&argc, argv); From 17f0438cca515f058c782e5a04b49266e876b99f Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Tue, 20 Feb 2024 14:43:06 -0800 Subject: [PATCH 37/50] parallel I/O routines within basis classes. --- lib/linalg/BasisReader.cpp | 60 ++++++----- lib/linalg/BasisWriter.cpp | 25 ++--- lib/utils/CSVDatabase.h | 37 ------- lib/utils/Database.cpp | 2 + lib/utils/Database.h | 161 ++++++++++++++++++++++------- lib/utils/HDFDatabase.h | 175 ++++++++++++++++++++++---------- unit_tests/test_HDFDatabase.cpp | 72 ++++++++++++- 7 files changed, 360 insertions(+), 172 deletions(-) diff --git a/lib/linalg/BasisReader.cpp b/lib/linalg/BasisReader.cpp index 76cacfa22..e6d1c2809 100644 --- a/lib/linalg/BasisReader.cpp +++ b/lib/linalg/BasisReader.cpp @@ -43,14 +43,12 @@ BasisReader::BasisReader( // Enforce hdf data format. CAROM_VERIFY(db_format != Database::CSV); - char tmp[100]; - sprintf(tmp, ".%06d", rank); - full_file_name = base_file_name + tmp; + full_file_name = base_file_name; if (db_format == Database::HDF5) { d_database = new HDFDatabase(); } - d_database->open(full_file_name, "r"); + d_database->open_parallel(full_file_name, "r", MPI_COMM_WORLD); } BasisReader::~BasisReader() @@ -70,9 +68,9 @@ BasisReader::getSpatialBasis() // This 0 index is the remainder from time interval concept. // This remains only for backward compatibility purpose. - d_database->getDoubleArray("spatial_basis_000000", - &spatial_basis_vectors->item(0, 0), - num_rows*num_cols); + d_database->getDoubleArray_parallel("spatial_basis_000000", + &spatial_basis_vectors->item(0, 0), + num_rows*num_cols); return spatial_basis_vectors; } @@ -101,12 +99,12 @@ BasisReader::getSpatialBasis( // This 0 index is the remainder from time interval concept. // This remains only for backward compatibility purpose. sprintf(tmp, "spatial_basis_000000"); - d_database->getDoubleArray(tmp, - &spatial_basis_vectors->item(0, 0), - num_rows*num_cols_to_read, - start_col - 1, - num_cols_to_read, - num_cols); + d_database->getDoubleArray_parallel(tmp, + &spatial_basis_vectors->item(0, 0), + num_rows*num_cols_to_read, + start_col - 1, + num_cols_to_read, + num_cols); return spatial_basis_vectors; } @@ -149,9 +147,9 @@ BasisReader::getTemporalBasis() // This 0 index is the remainder from time interval concept. // This remains only for backward compatibility purpose. sprintf(tmp, "temporal_basis_000000"); - d_database->getDoubleArray(tmp, - &temporal_basis_vectors->item(0, 0), - num_rows*num_cols); + d_database->getDoubleArray_parallel(tmp, + &temporal_basis_vectors->item(0, 0), + num_rows*num_cols); return temporal_basis_vectors; } @@ -180,12 +178,12 @@ BasisReader::getTemporalBasis( // This 0 index is the remainder from time interval concept. // This remains only for backward compatibility purpose. sprintf(tmp, "temporal_basis_000000"); - d_database->getDoubleArray(tmp, - &temporal_basis_vectors->item(0, 0), - num_rows*num_cols_to_read, - start_col - 1, - num_cols_to_read, - num_cols); + d_database->getDoubleArray_parallel(tmp, + &temporal_basis_vectors->item(0, 0), + num_rows*num_cols_to_read, + start_col - 1, + num_cols_to_read, + num_cols); return temporal_basis_vectors; } @@ -329,9 +327,9 @@ BasisReader::getSnapshotMatrix() // This 0 index is the remainder from time interval concept. // This remains only for backward compatibility purpose. sprintf(tmp, "snapshot_matrix_000000"); - d_database->getDoubleArray(tmp, - &snapshots->item(0, 0), - num_rows*num_cols); + d_database->getDoubleArray_parallel(tmp, + &snapshots->item(0, 0), + num_rows*num_cols); return snapshots; } @@ -360,12 +358,12 @@ BasisReader::getSnapshotMatrix( // This 0 index is the remainder from time interval concept. // This remains only for backward compatibility purpose. sprintf(tmp, "snapshot_matrix_000000"); - d_database->getDoubleArray(tmp, - &snapshots->item(0, 0), - num_rows*num_cols_to_read, - start_col - 1, - num_cols_to_read, - num_cols); + d_database->getDoubleArray_parallel(tmp, + &snapshots->item(0, 0), + num_rows*num_cols_to_read, + start_col - 1, + num_cols_to_read, + num_cols); return snapshots; } } diff --git a/lib/linalg/BasisWriter.cpp b/lib/linalg/BasisWriter.cpp index 712d5f0c7..1e98542e5 100644 --- a/lib/linalg/BasisWriter.cpp +++ b/lib/linalg/BasisWriter.cpp @@ -45,13 +45,14 @@ BasisWriter::BasisWriter( rank = 0; } - char tmp[100]; - sprintf(tmp, ".%06d", rank); - full_file_name = base_file_name + tmp; + // char tmp[100]; + // sprintf(tmp, ".%06d", rank); + // full_file_name = base_file_name + tmp; + full_file_name = base_file_name; - char tmp2[100]; - sprintf(tmp2, "_snapshot.%06d", rank); - snap_file_name = base_file_name + tmp2; + // char tmp2[100]; + // sprintf(tmp2, "_snapshot.%06d", rank); + snap_file_name = base_file_name + "_snapshot"; // create and open snapshot/basis database CAROM_VERIFY(db_format_ == Database::HDF5); @@ -75,7 +76,7 @@ BasisWriter::writeBasis(const std::string& kind) // This remains only for backward compatibility purpose. if (kind == "basis") { - d_database->create(full_file_name); + d_database->create_parallel(full_file_name, MPI_COMM_WORLD); const Matrix* basis = d_basis_generator->getSpatialBasis(); int num_rows = basis->numRows(); @@ -85,7 +86,7 @@ BasisWriter::writeBasis(const std::string& kind) sprintf(tmp, "spatial_basis_num_cols_000000"); d_database->putInteger(tmp, num_cols); sprintf(tmp, "spatial_basis_000000"); - d_database->putDoubleArray(tmp, &basis->item(0, 0), num_rows*num_cols); + d_database->putDoubleArray_parallel(tmp, &basis->item(0, 0), num_rows*num_cols); if(d_basis_generator->updateRightSV()) { const Matrix* tbasis = d_basis_generator->getTemporalBasis(); @@ -96,7 +97,7 @@ BasisWriter::writeBasis(const std::string& kind) sprintf(tmp, "temporal_basis_num_cols_000000"); d_database->putInteger(tmp, num_cols); sprintf(tmp, "temporal_basis_000000"); - d_database->putDoubleArray(tmp, &tbasis->item(0, 0), num_rows*num_cols); + d_database->putDoubleArray_parallel(tmp, &tbasis->item(0, 0), num_rows*num_cols); } const Vector* sv = d_basis_generator->getSingularValues(); @@ -104,13 +105,13 @@ BasisWriter::writeBasis(const std::string& kind) sprintf(tmp, "singular_value_size_000000"); d_database->putInteger(tmp, sv_dim); sprintf(tmp, "singular_value_000000"); - d_database->putDoubleArray(tmp, &sv->item(0), sv_dim); + d_database->putDoubleArray_parallel(tmp, &sv->item(0), sv_dim); d_database->close(); } if (kind == "snapshot") { - d_snap_database->create(snap_file_name); + d_snap_database->create_parallel(snap_file_name, MPI_COMM_WORLD); const Matrix* snapshots = d_basis_generator->getSnapshotMatrix(); int num_rows = snapshots->numRows(); // d_dim @@ -120,7 +121,7 @@ BasisWriter::writeBasis(const std::string& kind) sprintf(tmp, "snapshot_matrix_num_cols_000000"); d_snap_database->putInteger(tmp, num_cols); sprintf(tmp, "snapshot_matrix_000000"); - d_snap_database->putDoubleArray(tmp, &snapshots->item(0,0), num_rows*num_cols); + d_snap_database->putDoubleArray_parallel(tmp, &snapshots->item(0,0), num_rows*num_cols); d_snap_database->close(); } diff --git a/lib/utils/CSVDatabase.h b/lib/utils/CSVDatabase.h index ba5fe333d..b7c655ee3 100644 --- a/lib/utils/CSVDatabase.h +++ b/lib/utils/CSVDatabase.h @@ -14,7 +14,6 @@ #define included_CSVDatabase_h #include "Database.h" -#include "Utilities.h" #include #include #include @@ -318,42 +317,6 @@ class CSVDatabase : public Database getLineCount( const std::string& file_name); - /** - * @brief Unsupported parallel I/O function. - * - * @param[in] file_name Base Name of CSV database file to create. - * @param[in] comm MPI communicator to get the rank. - * - * @return True if file create was successful. - */ - bool - create_parallel( - const std::string& file_name, - const MPI_Comm comm) - { - CAROM_ERROR("CSVDatabase does not support parallel I/O!\n"); - return false; - } - - /** - * @brief Unsupported parallel I/O function. - * - * @param[in] file_name Name of existing CSV database file to open. - * @param[in] type Read/write type ("r"/"wr") - * @param[in] comm MPI communicator to get the rank. - * - * @return True if file open was successful. - */ - bool - open_parallel( - const std::string& file_name, - const std::string& type, - const MPI_Comm comm) - { - CAROM_ERROR("CSVDatabase does not support parallel I/O!\n"); - return false; - } - private: /** * @brief Unimplemented copy constructor. diff --git a/lib/utils/Database.cpp b/lib/utils/Database.cpp index 49c6c9eb7..0ef4599da 100644 --- a/lib/utils/Database.cpp +++ b/lib/utils/Database.cpp @@ -35,6 +35,7 @@ bool Database::create(const std::string& file_name) { std::cout << "Creating file: " << file_name << std::endl; + return true; } bool @@ -43,6 +44,7 @@ Database::open( const std::string& type) { std::cout << "Opening file: " << file_name << std::endl; + return true; } } diff --git a/lib/utils/Database.h b/lib/utils/Database.h index 4320bb35b..b1eeff04d 100644 --- a/lib/utils/Database.h +++ b/lib/utils/Database.h @@ -13,6 +13,7 @@ #ifndef included_Database_h #define included_Database_h +#include "Utilities.h" #include #include #include "mpi.h" @@ -52,23 +53,6 @@ class Database create( const std::string& file_name); - /** - * @brief Creates a new database file with the supplied name, with parallel I/O support. - * Supported only for HDF5 format. - * For HDFDatabase, the function is equivalent to create, - * only extending the file name with 6 digits indicating processor rank. - * For HDFDatabaseMPIO, the file is created with MPI I/O file access property. - * - * @param[in] file_name Name of database file to create. - * - * @return True if file create was successful. - */ - virtual - bool - create_parallel( - const std::string& file_name, - const MPI_Comm comm) = 0; - /** * @brief Opens an existing database file with the supplied name. * @@ -83,25 +67,6 @@ class Database const std::string& file_name, const std::string& type); - /** - * @brief Opens an existing database file with the supplied name, with parallel I/O support. - * Supported only for HDF5 format. - * For HDFDatabase, the function is equivalent to open, - * only extending the file name with 6 digits indicating processor rank. - * For HDFDatabaseMPIO, the file is opened with MPI I/O file access property. - * - * @param[in] file_name Name of existing database file to open. - * @param[in] type Read/write type ("r"/"wr") - * - * @return True if file open was successful. - */ - virtual - bool - open_parallel( - const std::string& file_name, - const std::string& type, - const MPI_Comm comm) = 0; - /** * @brief Closes the currently open database file. * @@ -314,9 +279,131 @@ class Database */ enum formats { HDF5, - CSV + CSV, + HDF5_MPIO }; + /** + * @brief Creates a new database file with the supplied name, with parallel I/O support. + * Supported only for HDF5 format. + * For HDFDatabase, the function is equivalent to create, + * only extending the file name with 6 digits indicating processor rank. + * For HDFDatabaseMPIO, the file is created with MPI I/O file access property. + * + * @param[in] file_name Base Name of CSV database file to create. + * @param[in] comm MPI communicator to get the rank. + * + * @return True if file create was successful. + */ + virtual + bool + create_parallel( + const std::string& file_name, + const MPI_Comm comm) + { + CAROM_ERROR("Abstract method Database::create_parallel!\n"); + return false; + } + + /** + * @brief Opens an existing database file with the supplied name, with parallel I/O support. + * Supported only for HDF5 format. + * For HDFDatabase, the function is equivalent to open, + * only extending the file name with 6 digits indicating processor rank. + * For HDFDatabaseMPIO, the file is opened with MPI I/O file access property. + * + * @param[in] file_name Name of existing CSV database file to open. + * @param[in] type Read/write type ("r"/"wr") + * @param[in] comm MPI communicator to get the rank. + * + * @return True if file open was successful. + */ + virtual + bool + open_parallel( + const std::string& file_name, + const std::string& type, + const MPI_Comm comm) + { + CAROM_ERROR("Abstract method Database::open_parallel!\n"); + return false; + } + + /** + * @brief Writes a distributed array of doubles associated with the supplied key to + * the currently open database file. + * Supported only for HDF5 format. + * For HDFDatabase, the function is equivalent to putDoubleArray, + * writing the local array per each process. + * For HDFDatabaseMPIO, the global array is written into a single file via MPI I/O. + * + * @param[in] key The key associated with the array of values to be + * written. + * @param[in] data The array of double values to be written. + * @param[in] nelements The number of doubles in the array. + */ + virtual + void + putDoubleArray_parallel( + const std::string& key, + const double* const data, + int nelements) + { + CAROM_ERROR("Abstract method Database::putDoubleArray_parallel!\n"); + } + + /** + * @brief Reads a distributed array of doubles associated with the supplied key + * from the currently open database file. + * Supported only for HDF5 format. + * For HDFDatabase, the function is equivalent to getDoubleArray, + * reading the local array from a local file per each process. + * For HDFDatabaseMPIO, the local array is read from a single file via MPI I/O. + * + * @param[in] key The key associated with the array of values to be + * read. + * @param[out] data The allocated array of double values to be read. + * @param[in] nelements The number of doubles in the array. + */ + virtual + void + getDoubleArray_parallel( + const std::string& key, + double* data, + int nelements) + { + CAROM_ERROR("Abstract method Database::getDoubleArray_parallel!\n"); + } + + /** + * @brief Reads a distributed array of doubles associated with the supplied key + * from the currently open database file. + * Supported only for HDF5 format. + * For HDFDatabase, the function is equivalent to getDoubleArray, + * reading the local array from a local file per each process. + * For HDFDatabaseMPIO, the local array is read from a single file via MPI I/O. + * + * @param[in] key The key associated with the array of values to be + * read. + * @param[out] data The allocated array of double values to be read. + * @param[in] nelements The number of doubles in the array. + * @param[in] offset The initial offset in the array. + * @param[in] block_size The block size to read from the HDF5 dataset. + * @param[in] stride The stride to read from the HDF5 dataset. + */ + virtual + void + getDoubleArray_parallel( + const std::string& key, + double* data, + int nelements, + int offset, + int block_size, + int stride) + { + CAROM_ERROR("Abstract method Database::getDoubleArray_parallel!\n"); + } + private: /** * @brief Unimplemented copy constructor. diff --git a/lib/utils/HDFDatabase.h b/lib/utils/HDFDatabase.h index ce1af48fa..8a4707349 100644 --- a/lib/utils/HDFDatabase.h +++ b/lib/utils/HDFDatabase.h @@ -63,59 +63,6 @@ class HDFDatabase : public Database const std::string& file_name, const std::string& type) override; - /** - * @brief Creates a new HDF5 database file with the supplied name, - * extending it with 6 digits indicating the process rank. - * - * @param[in] file_name Name of HDF5 database file to create. - * @param[in] comm MPI communicator to obtain the process rank. - * - * @return True if file create was successful. - */ - virtual bool - create_parallel( - const std::string& file_name, - const MPI_Comm comm) override - { - CAROM_VERIFY(!file_name.empty()); - MPI_Comm_rank(comm, &d_rank); - - std::string ext_filename(file_name); - char tmp[10]; - sprintf(tmp, ".%06d", d_rank); - ext_filename += tmp; - - return create(ext_filename); - } - - /** - * @brief Opens an existing HDF5 database file with the supplied name, - * extending it with 6 digits indicating the process rank. - * - * @param[in] file_name Name of existing HDF5 database file to open. - * @param[in] type Read/write type ("r"/"wr") - * @param[in] comm MPI communicator to obtain the process rank. - * - * @return True if file open was successful. - */ - virtual - bool - open_parallel( - const std::string& file_name, - const std::string& type, - const MPI_Comm comm) override - { - CAROM_VERIFY(!file_name.empty()); - MPI_Comm_rank(comm, &d_rank); - - std::string ext_filename(file_name); - char tmp[10]; - sprintf(tmp, ".%06d", d_rank); - ext_filename += tmp; - - return open(ext_filename, type); - } - /** * @brief Closes the currently open HDF5 database file. * @@ -281,6 +228,128 @@ class HDFDatabase : public Database int block_size, int stride); + /** + * @brief Creates a new HDF5 database file with the supplied name, + * extending it with 6 digits indicating the process rank. + * + * @param[in] file_name Name of HDF5 database file to create. + * @param[in] comm MPI communicator to obtain the process rank. + * + * @return True if file create was successful. + */ + virtual bool + create_parallel( + const std::string& file_name, + const MPI_Comm comm) override + { + CAROM_VERIFY(!file_name.empty()); + MPI_Comm_rank(comm, &d_rank); + + std::string ext_filename(file_name); + char tmp[10]; + sprintf(tmp, ".%06d", d_rank); + ext_filename += tmp; + + return create(ext_filename); + } + + /** + * @brief Opens an existing HDF5 database file with the supplied name, + * extending it with 6 digits indicating the process rank. + * + * @param[in] file_name Name of existing HDF5 database file to open. + * @param[in] type Read/write type ("r"/"wr") + * @param[in] comm MPI communicator to obtain the process rank. + * + * @return True if file open was successful. + */ + virtual + bool + open_parallel( + const std::string& file_name, + const std::string& type, + const MPI_Comm comm) override + { + CAROM_VERIFY(!file_name.empty()); + MPI_Comm_rank(comm, &d_rank); + + std::string ext_filename(file_name); + char tmp[10]; + sprintf(tmp, ".%06d", d_rank); + ext_filename += tmp; + + return open(ext_filename, type); + } + + /** + * @brief Writes a distributed array of doubles associated with the supplied key to + * the currently open database file. + * For HDFDatabase, the function is equivalent to putDoubleArray, + * writing the local array per each process. + * + * @param[in] key The key associated with the array of values to be + * written. + * @param[in] data The array of double values to be written. + * @param[in] nelements The number of doubles in the array. + */ + virtual + void + putDoubleArray_parallel( + const std::string& key, + const double* const data, + int nelements) + { + putDoubleArray(key, data, nelements); + } + + /** + * @brief Reads a distributed array of doubles associated with the supplied key + * from the currently open database file. + * For HDFDatabase, the function is equivalent to getDoubleArray, + * reading the local array from a local file per each process. + * + * @param[in] key The key associated with the array of values to be + * read. + * @param[out] data The allocated array of double values to be read. + * @param[in] nelements The number of doubles in the array. + */ + virtual + void + getDoubleArray_parallel( + const std::string& key, + double* data, + int nelements) + { + getDoubleArray(key, data, nelements); + } + + /** + * @brief Reads a distributed array of doubles associated with the supplied key + * from the currently open database file. + * For HDFDatabase, the function is equivalent to getDoubleArray, + * reading the local array from a local file per each process. + * + * @param[in] key The key associated with the array of values to be + * read. + * @param[out] data The allocated array of double values to be read. + * @param[in] nelements The number of doubles in the array. + * @param[in] offset The initial offset in the array. + * @param[in] block_size The block size to read from the HDF5 dataset. + * @param[in] stride The stride to read from the HDF5 dataset. + */ + virtual + void + getDoubleArray_parallel( + const std::string& key, + double* data, + int nelements, + int offset, + int block_size, + int stride) + { + getDoubleArray(key, data, nelements, offset, block_size, stride); + } + private: /** * @brief Unimplemented copy constructor. diff --git a/unit_tests/test_HDFDatabase.cpp b/unit_tests/test_HDFDatabase.cpp index d8ada61f8..227f3df47 100644 --- a/unit_tests/test_HDFDatabase.cpp +++ b/unit_tests/test_HDFDatabase.cpp @@ -11,8 +11,7 @@ #ifdef CAROM_HAS_GTEST #include -#include "linalg/Matrix.h" -#include "linalg/Vector.h" +#include "linalg/BasisGenerator.h" #include "utils/HDFDatabase.h" #include #include @@ -513,6 +512,75 @@ TEST(HDF5, Test_selective_parallel_reading) EXPECT_TRUE(data[d] == d + offsets[rank] * ncol); } +TEST(BasisGeneratorIO, HDFDatabase) +{ + // Get the rank of this process, and the number of processors. + int mpi_init, d_rank, d_num_procs; + MPI_Initialized(&mpi_init); + if (mpi_init == 0) { + MPI_Init(nullptr, nullptr); + } + + MPI_Comm_rank(MPI_COMM_WORLD, &d_rank); + MPI_Comm_size(MPI_COMM_WORLD, &d_num_procs); + + const int nrow = 100, ncol = 10; + int nrow_local = CAROM::split_dimension(nrow, MPI_COMM_WORLD); + std::vector row_offset(d_num_procs + 1); + const int dummy = CAROM::get_global_offsets(nrow_local, row_offset, + MPI_COMM_WORLD); + EXPECT_EQ(nrow, dummy); + + std::default_random_engine generator; + generator.seed( + 1234); // fix the seed to keep the same result for different nproc. + std::uniform_real_distribution<> uniform_distribution(0.0, 1.0); + std::normal_distribution normal_distribution(0.0, 1.0); + + // distribute from a global matrix to keep the same system for different nproc. + CAROM::Matrix snapshots(nrow, ncol, false); + for (int i = 0; i < nrow; i++) + for (int j = 0; j < ncol; j++) + snapshots(i, j) = normal_distribution(generator); + snapshots.distribute(nrow_local); + + CAROM::Options svd_options = CAROM::Options(nrow_local, ncol, 1); + svd_options.setMaxBasisDimension(nrow); + svd_options.setRandomizedSVD(false); + CAROM::BasisGenerator sampler(svd_options, false, "test_basis"); + CAROM::Vector sample(nrow_local, true); + for (int s = 0; s < ncol; s++) + { + for (int d = 0; d < nrow_local; d++) + sample(d) = snapshots(d, s); + + sampler.takeSample(sample.getData(), 0, 0); + } + sampler.endSamples(); + sampler.writeSnapshot(); + + const CAROM::Matrix *spatial_basis = sampler.getSpatialBasis(); + const CAROM::Matrix *snapshot = sampler.getSnapshotMatrix(); + + CAROM::BasisReader basis_reader("test_basis"); + const CAROM::Matrix *spatial_basis1 = basis_reader.getSpatialBasis(); + + EXPECT_EQ(spatial_basis->numRows(), spatial_basis1->numRows()); + EXPECT_EQ(spatial_basis->numColumns(), spatial_basis1->numColumns()); + for (int i = 0; i < spatial_basis->numRows(); i++) + for (int j = 0; j < spatial_basis->numColumns(); j++) + EXPECT_NEAR((*spatial_basis)(i, j), (*spatial_basis1)(i, j), 1.0e-15); + + CAROM::BasisReader snapshot_reader("test_basis_snapshot"); + const CAROM::Matrix *snapshot1 = snapshot_reader.getSnapshotMatrix(); + + EXPECT_EQ(snapshot->numRows(), snapshot1->numRows()); + EXPECT_EQ(snapshot->numColumns(), snapshot1->numColumns()); + for (int i = 0; i < snapshot->numRows(); i++) + for (int j = 0; j < snapshot->numColumns(); j++) + EXPECT_NEAR((*snapshot)(i, j), (*snapshot1)(i, j), 1.0e-15); +} + int main(int argc, char* argv[]) { ::testing::InitGoogleTest(&argc, argv); From 2806f8e7ee5c082ab0a05cdc5e2c2d23c411b973 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Tue, 20 Feb 2024 21:24:48 -0800 Subject: [PATCH 38/50] HDFDatabaseMPIO class initial loading. --- lib/CMakeLists.txt | 1 + lib/utils/HDFDatabase.h | 29 +- lib/utils/HDFDatabaseMPIO.cpp | 626 ++++++++++++++++++++++++++++++++++ lib/utils/HDFDatabaseMPIO.h | 388 +++++++++++++++++++++ lib/utils/mpi_utils.cpp | 9 + lib/utils/mpi_utils.h | 9 + 6 files changed, 1049 insertions(+), 13 deletions(-) create mode 100644 lib/utils/HDFDatabaseMPIO.cpp create mode 100644 lib/utils/HDFDatabaseMPIO.h diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index c21ae8169..0e546061d 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -54,6 +54,7 @@ set(module_list hyperreduction/Hyperreduction utils/Database utils/HDFDatabase + utils/HDFDatabaseMPIO utils/CSVDatabase utils/Utilities utils/ParallelBuffer diff --git a/lib/utils/HDFDatabase.h b/lib/utils/HDFDatabase.h index 8a4707349..49c8f436f 100644 --- a/lib/utils/HDFDatabase.h +++ b/lib/utils/HDFDatabase.h @@ -350,19 +350,7 @@ class HDFDatabase : public Database getDoubleArray(key, data, nelements, offset, block_size, stride); } -private: - /** - * @brief Unimplemented copy constructor. - */ - HDFDatabase( - const HDFDatabase& other); - - /** - * @brief Unimplemented assignment operator. - */ - HDFDatabase& - operator = ( - const HDFDatabase& rhs); +protected: /** * @brief Returns true if the specified key represents an integer entry. @@ -396,6 +384,7 @@ class HDFDatabase : public Database * @param[in] type_key The attribute to be written. * @param[in] dataset_id ID of the dataset key will be written to. */ + virtual void writeAttribute( int type_key, @@ -442,6 +431,20 @@ class HDFDatabase : public Database * Used only when d_parallel is true. */ int d_rank; + +private: + /** + * @brief Unimplemented copy constructor. + */ + HDFDatabase( + const HDFDatabase& other); + + /** + * @brief Unimplemented assignment operator. + */ + HDFDatabase& + operator = ( + const HDFDatabase& rhs); }; } diff --git a/lib/utils/HDFDatabaseMPIO.cpp b/lib/utils/HDFDatabaseMPIO.cpp new file mode 100644 index 000000000..ef96f6796 --- /dev/null +++ b/lib/utils/HDFDatabaseMPIO.cpp @@ -0,0 +1,626 @@ +/****************************************************************************** + * + * Copyright (c) 2013-2023, Lawrence Livermore National Security, LLC + * and other libROM project developers. See the top-level COPYRIGHT + * file for details. + * + * SPDX-License-Identifier: (Apache-2.0 OR MIT) + * + *****************************************************************************/ + +// Description: The concrete database implementation using HDF5. + +#include "HDFDatabaseMPIO.h" +#include "utils/mpi_utils.h" +#include + +namespace CAROM { + +HDFDatabaseMPIO::HDFDatabaseMPIO() : + HDFDatabase(), + d_rank(-1) +{} + +bool +HDFDatabaseMPIO::create_parallel( + const std::string& file_name, + const MPI_Comm comm) +{ + int mpi_init; + MPI_Initialized(&mpi_init); + if (mpi_init) { + d_comm = comm; + MPI_Comm_rank(d_comm, &d_rank); + } + else + d_rank = 0; + + std::string file_name_ext(file_name + ".000000"); + if (d_rank == 0) + Database::create(file_name_ext); + CAROM_VERIFY(!file_name.empty()); + + hid_t plist_id; + /* + * Set up file access property list with parallel I/O access + */ + plist_id = H5Pcreate(H5P_FILE_ACCESS); + H5Pset_fapl_mpio(plist_id, d_comm, MPI_INFO_NULL); + /* + * OPTIONAL: It is generally recommended to set collective + * metadata reads/writes on FAPL to perform metadata reads + * collectively, which usually allows datasets + * to perform better at scale, although it is not + * strictly necessary. + */ + H5Pset_all_coll_metadata_ops(plist_id, true); + H5Pset_coll_metadata_write(plist_id, true); + + hid_t file_id = H5Fcreate(file_name_ext.c_str(), + H5F_ACC_TRUNC, + H5P_DEFAULT, + plist_id); + bool result = file_id >= 0; + CAROM_VERIFY(result); + d_is_file = true; + d_file_id = file_id; + d_group_id = file_id; + + H5Pclose(plist_id); + + return result; +} + +bool +HDFDatabaseMPIO::open_parallel( + const std::string& file_name, + const std::string& type, + const MPI_Comm comm) +{ + int mpi_init; + MPI_Initialized(&mpi_init); + if (mpi_init) { + d_comm = comm; + MPI_Comm_rank(d_comm, &d_rank); + } + else + d_rank = 0; + + std::string file_name_ext(file_name + ".000000"); + if (d_rank == 0) + Database::open(file_name_ext, type); + CAROM_VERIFY(!file_name.empty()); + CAROM_VERIFY(type == "r" || type == "wr"); + + hid_t plist_id; + /* + * Set up file access property list with parallel I/O access + */ + plist_id = H5Pcreate(H5P_FILE_ACCESS); + H5Pset_fapl_mpio(plist_id, d_comm, MPI_INFO_NULL); + /* + * OPTIONAL: It is generally recommended to set collective + * metadata reads/writes on FAPL to perform metadata reads + * collectively, which usually allows datasets + * to perform better at scale, although it is not + * strictly necessary. + */ + H5Pset_all_coll_metadata_ops(plist_id, true); + H5Pset_coll_metadata_write(plist_id, true); + + hid_t file_id; + if (type == "r") + { + file_id = H5Fopen(file_name_ext.c_str(), + H5F_ACC_RDONLY, + plist_id); + } + else if (type == "wr") + { + file_id = H5Fopen(file_name_ext.c_str(), + H5F_ACC_RDWR, + plist_id); + } + bool result = file_id >= 0; + CAROM_VERIFY(result); + d_is_file = true; + d_file_id = file_id; + d_group_id = file_id; + + H5Pclose(plist_id); + + return result; +} + +void +HDFDatabaseMPIO::putIntegerArray_parallel( + const std::string& key, + const int* const data, + int nelem_local) +{ + CAROM_VERIFY(!key.empty()); + CAROM_VERIFY(data != nullptr); + CAROM_VERIFY(nelem_local >= 0); + + /* determine global nelements and offsets */ + std::vector offsets; + int nelements = CAROM::get_global_offsets(nelem_local, offsets, d_comm); + CAROM_VERIFY(nelements > 0); + + const int dim_rank = 1; + hsize_t dim[dim_rank] = { static_cast(nelements) }; + hid_t filespace = H5Screate_simple(dim_rank, dim, NULL); + CAROM_VERIFY(filespace >= 0); + +#if (H5_VERS_MAJOR > 1) || ((H5_VERS_MAJOR == 1) && (H5_VERS_MINOR > 6)) + hid_t dataset = H5Dcreate(d_group_id, + key.c_str(), + H5T_NATIVE_INT, + filespace, + H5P_DEFAULT, + H5P_DEFAULT, + H5P_DEFAULT); +#else + CAROM_ERROR("HDFDatabaseMPIO is not compatible with current HDF5 version!\n"); +#endif + CAROM_VERIFY(dataset >= 0); + H5Sclose(filespace); + + /* + * Each process defines dataset in memory and writes it to the hyperslab + * in the file. + */ + /* hyperslab selection parameters */ + hsize_t count[dim_rank]; + hsize_t offset[dim_rank]; + count[0] = nelem_local; + offset[0] = offsets[d_rank]; + hid_t memspace = H5Screate_simple(dim_rank, count, NULL); + + /* + * Select hyperslab in the file. + */ + filespace = H5Dget_space(dataset); + if (nelem_local > 0) + H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset, NULL, count, NULL); + else + H5Sselect_none(filespace); + + /* + * Create property list for collective dataset write. + */ + hid_t plist_id = H5Pcreate(H5P_DATASET_XFER); + H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE); + + herr_t errf = H5Dwrite(dataset, + H5T_NATIVE_INT, + memspace, + filespace, + plist_id, + data); + CAROM_VERIFY(errf >= 0); + + // Write attribute so we know what kind of data this is. + writeAttribute(KEY_INT_ARRAY, dataset); + + errf = H5Sclose(filespace); + CAROM_VERIFY(errf >= 0); + + errf = H5Sclose(memspace); + CAROM_VERIFY(errf >= 0); + + errf = H5Pclose(plist_id); + CAROM_VERIFY(errf >= 0); + + errf = H5Dclose(dataset); + CAROM_VERIFY(errf >= 0); +#ifndef DEBUG_CHECK_ASSERTIONS + CAROM_NULL_USE(errf); +#endif +} + +void +HDFDatabaseMPIO::putDoubleArray_parallel( + const std::string& key, + const double* const data, + int nelem_local) +{ + CAROM_VERIFY(!key.empty()); + CAROM_VERIFY(data != nullptr); + CAROM_VERIFY(nelem_local >= 0); + + /* determine global nelements and offsets */ + std::vector offsets; + int nelements = CAROM::get_global_offsets(nelem_local, offsets, d_comm); + CAROM_VERIFY(nelements > 0); + + const int dim_rank = 1; + hsize_t dim[dim_rank] = { static_cast(nelements) }; + hid_t filespace = H5Screate_simple(dim_rank, dim, 0); + CAROM_VERIFY(filespace >= 0); + +#if (H5_VERS_MAJOR > 1) || ((H5_VERS_MAJOR == 1) && (H5_VERS_MINOR > 6)) + hid_t dataset = H5Dcreate(d_group_id, + key.c_str(), + H5T_NATIVE_DOUBLE, + filespace, + H5P_DEFAULT, + H5P_DEFAULT, + H5P_DEFAULT); +#else + CAROM_ERROR("HDFDatabaseMPIO is not compatible with current HDF5 version!\n"); +#endif + CAROM_VERIFY(dataset >= 0); + H5Sclose(filespace); + + /* + * Each process defines dataset in memory and writes it to the hyperslab + * in the file. + */ + /* hyperslab selection parameters */ + hsize_t count[dim_rank]; + hsize_t offset[dim_rank]; + count[0] = nelem_local; + offset[0] = offsets[d_rank]; + hid_t memspace = H5Screate_simple(dim_rank, count, NULL); + + /* + * Select hyperslab in the file. + */ + filespace = H5Dget_space(dataset); + if (nelem_local > 0) + H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset, NULL, count, NULL); + else + H5Sselect_none(filespace); + + /* + * Create property list for collective dataset write. + */ + hid_t plist_id = H5Pcreate(H5P_DATASET_XFER); + H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE); + + herr_t errf = H5Dwrite(dataset, + H5T_NATIVE_DOUBLE, + memspace, + filespace, + plist_id, + data); + CAROM_VERIFY(errf >= 0); + + // Write attribute so we know what kind of data this is. + writeAttribute(KEY_DOUBLE_ARRAY, dataset); + + errf = H5Sclose(filespace); + CAROM_VERIFY(errf >= 0); + + errf = H5Sclose(memspace); + CAROM_VERIFY(errf >= 0); + + errf = H5Pclose(plist_id); + CAROM_VERIFY(errf >= 0); + + errf = H5Dclose(dataset); + CAROM_VERIFY(errf >= 0); +#ifndef DEBUG_CHECK_ASSERTIONS + CAROM_NULL_USE(errf); +#endif +} + +void +HDFDatabaseMPIO::getIntegerArray_parallel( + const std::string& key, + int* data, + int nelem_local) +{ + CAROM_VERIFY(nelem_local >= 0); + /* determine global nelements and offsets */ + std::vector offsets; + int nelements = CAROM::get_global_offsets(nelem_local, offsets, d_comm); + if (nelements == 0) return; + + CAROM_VERIFY(!key.empty()); +#ifndef DEBUG_CHECK_ASSERTIONS + CAROM_NULL_USE(nelem_local); +#endif + +#if (H5_VERS_MAJOR > 1) || ((H5_VERS_MAJOR == 1) && (H5_VERS_MINOR > 6)) + hid_t dset = H5Dopen(d_group_id, key.c_str(), H5P_DEFAULT); +#else + CAROM_ERROR("HDFDatabaseMPIO is not compatible with current HDF5 version!\n"); +#endif + CAROM_VERIFY(dset >= 0); + + hid_t filespace = H5Dget_space(dset); + CAROM_VERIFY(filespace >= 0); + + hsize_t nsel = H5Sget_select_npoints(filespace); + CAROM_VERIFY(static_cast(nsel) == nelements); + + const int dim_rank = 1; + H5Sclose(filespace); + + /* + * Each process defines dataset in memory and writes it to the hyperslab + * in the file. + */ + /* hyperslab selection parameters */ + hsize_t count[dim_rank]; + hsize_t offset[dim_rank]; + count[0] = nelem_local; + offset[0] = offsets[d_rank]; + hid_t memspace = H5Screate_simple(dim_rank, count, NULL); + + /* + * Select hyperslab in the file. + */ + filespace = H5Dget_space(dset); + if (nelem_local > 0) + H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset, NULL, count, NULL); + else + H5Sselect_none(filespace); + + /* + * Create property list for collective dataset write. + */ + hid_t plist_id = H5Pcreate(H5P_DATASET_XFER); + H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE); + + herr_t errf; + errf = H5Dread(dset, H5T_NATIVE_INT, memspace, filespace, plist_id, data); + CAROM_VERIFY(errf >= 0); + + errf = H5Sclose(filespace); + CAROM_VERIFY(errf >= 0); + + errf = H5Sclose(memspace); + CAROM_VERIFY(errf >= 0); + + errf = H5Pclose(plist_id); + CAROM_VERIFY(errf >= 0); + + errf = H5Dclose(dset); + CAROM_VERIFY(errf >= 0); +#ifndef DEBUG_CHECK_ASSERTIONS + CAROM_NULL_USE(errf); +#endif +} + +void +HDFDatabaseMPIO::getDoubleArray_parallel( + const std::string& key, + double* data, + int nelem_local) +{ + CAROM_VERIFY(nelem_local >= 0); + /* determine global nelements and offsets */ + std::vector offsets; + int nelements = CAROM::get_global_offsets(nelem_local, offsets, d_comm); + if (nelements == 0) return; + + CAROM_VERIFY(!key.empty()); +#ifndef DEBUG_CHECK_ASSERTIONS + CAROM_NULL_USE(nelements); +#endif + +#if (H5_VERS_MAJOR > 1) || ((H5_VERS_MAJOR == 1) && (H5_VERS_MINOR > 6)) + hid_t dset = H5Dopen(d_group_id, key.c_str(), H5P_DEFAULT); +#else + hid_t dset = H5Dopen(d_group_id, key.c_str()); +#endif + CAROM_VERIFY(dset >= 0); + + hid_t filespace = H5Dget_space(dset); + CAROM_VERIFY(filespace >= 0); + + hsize_t nsel = H5Sget_select_npoints(filespace); + CAROM_VERIFY(static_cast(nsel) == nelements); + + const int dim_rank = 1; + H5Sclose(filespace); + + /* + * Each process defines dataset in memory and writes it to the hyperslab + * in the file. + */ + /* hyperslab selection parameters */ + hsize_t count[dim_rank]; + hsize_t offset[dim_rank]; + count[0] = nelem_local; + offset[0] = offsets[d_rank]; + hid_t memspace = H5Screate_simple(dim_rank, count, NULL); + + /* + * Select hyperslab in the file. + */ + filespace = H5Dget_space(dset); + if (nelem_local > 0) + H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset, NULL, count, NULL); + else + H5Sselect_none(filespace); + + /* + * Create property list for collective dataset write. + */ + hid_t plist_id = H5Pcreate(H5P_DATASET_XFER); + H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE); + + herr_t errf; + errf = H5Dread(dset, H5T_NATIVE_DOUBLE, memspace, filespace, plist_id, data); + CAROM_VERIFY(errf >= 0); + + errf = H5Sclose(filespace); + CAROM_VERIFY(errf >= 0); + + errf = H5Sclose(memspace); + CAROM_VERIFY(errf >= 0); + + errf = H5Pclose(plist_id); + CAROM_VERIFY(errf >= 0); + + errf = H5Dclose(dset); + CAROM_VERIFY(errf >= 0); +#ifndef DEBUG_CHECK_ASSERTIONS + CAROM_NULL_USE(errf); +#endif +} + +void +HDFDatabaseMPIO::getDoubleArray_parallel( + const std::string& key, + double* data, + int nelem_local, + const std::vector& idx_local) +{ + std::vector idx_tmp = idx_local; + if (idx_local.size() == 0) + { + for (int i = 0; i < nelem_local; i++) + idx_tmp.push_back(i); + } + + std::vector alldata(nelem_local); + getDoubleArray_parallel(key, alldata.data(), nelem_local); + int k = 0; + for (int i = 0; i < nelem_local; ++i) + { + if (idx_tmp[k] == i) + { + data[k++] = alldata[i]; + } + if (k == idx_tmp.size()) + { + break; + } + } + CAROM_VERIFY(k == idx_tmp.size()); +} + +void +HDFDatabaseMPIO::getDoubleArray_parallel( + const std::string& key, + double* data, + int nelem_local, + int block_offset_global, + int block_size_global, + int stride_global) +{ + CAROM_VERIFY(nelem_local >= 0); + CAROM_VERIFY(CAROM::is_same(stride_global, d_comm)); + CAROM_VERIFY(CAROM::is_same(block_size_global, d_comm)); + CAROM_VERIFY(CAROM::is_same(block_offset_global, d_comm)); + /* We assume stride sets the maximum block size */ + CAROM_VERIFY(block_offset_global + block_size_global <= stride_global); + /* determine global nelements and offsets */ + hsize_t num_local_blocks = nelem_local / block_size_global; + std::vector global_offsets; + int nelements = CAROM::get_global_offsets(num_local_blocks * stride_global, + global_offsets, d_comm); + + CAROM_VERIFY(!key.empty()); +#ifndef DEBUG_CHECK_ASSERTIONS + CAROM_NULL_USE(nelements); +#endif + +#if (H5_VERS_MAJOR > 1) || ((H5_VERS_MAJOR == 1) && (H5_VERS_MINOR > 6)) + hid_t dset = H5Dopen(d_group_id, key.c_str(), H5P_DEFAULT); +#else + hid_t dset = H5Dopen(d_group_id, key.c_str()); +#endif + CAROM_VERIFY(dset >= 0); + + hid_t filespace = H5Dget_space(dset); + CAROM_VERIFY(filespace >= 0); + + hsize_t nsel = H5Sget_select_npoints(filespace); + CAROM_VERIFY((nsel == 0) || (nsel == nelements)); + + const int dim_rank = 1; + H5Sclose(filespace); + + herr_t errf; + if (nsel > 0) { + /* + * Each process defines dataset in memory and writes it to the hyperslab + * in the file. + */ + /* hyperslab selection parameters */ + hsize_t num_blocks[dim_rank] = {num_local_blocks}; + hsize_t buffer_array_size[dim_rank] = {(hsize_t) nelem_local}; + hsize_t offset[dim_rank] = {(hsize_t) global_offsets[d_rank] + block_offset_global}; + hsize_t strides[dim_rank] = {(hsize_t) stride_global}; + hsize_t block_sizes[1] = {(hsize_t) block_size_global}; + hid_t memspace = H5Screate_simple(dim_rank, buffer_array_size, NULL); + + /* + * Select hyperslab in the file. + */ + filespace = H5Dget_space(dset); + if (nelem_local > 0) + H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset, + strides, num_blocks, block_sizes); + else + H5Sselect_none(filespace); + + /* + * Create property list for collective dataset write. + */ + hid_t plist_id = H5Pcreate(H5P_DATASET_XFER); + H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE); + + errf = H5Dread(dset, H5T_NATIVE_DOUBLE, memspace, filespace, plist_id, data); + CAROM_VERIFY(errf >= 0); + + errf = H5Sclose(memspace); + CAROM_VERIFY(errf >= 0); + + errf = H5Pclose(plist_id); + CAROM_VERIFY(errf >= 0); + } + + errf = H5Sclose(filespace); + CAROM_VERIFY(errf >= 0); + + errf = H5Dclose(dset); + CAROM_VERIFY(errf >= 0); +#ifndef DEBUG_CHECK_ASSERTIONS + CAROM_NULL_USE(errf); +#endif +} + +void +HDFDatabaseMPIO::writeAttribute( + int type_key, + hid_t dataset_id) +{ + CAROM_VERIFY(CAROM::is_same(type_key, d_comm)); + + hid_t attr_id = H5Screate(H5S_SCALAR); + CAROM_VERIFY(attr_id >= 0); + +#if (H5_VERS_MAJOR > 1) || ((H5_VERS_MAJOR == 1) && (H5_VERS_MINOR > 6)) + hid_t attr = H5Acreate(dataset_id, + "Type", + H5T_NATIVE_INT, + attr_id, + H5P_DEFAULT, + H5P_DEFAULT); +#else + CAROM_ERROR("HDFDatabaseMPIO is not compatible with current HDF5 version!\n"); +#endif + CAROM_VERIFY(attr >= 0); + + /* only the last process writes the attribute */ + herr_t errf = H5Awrite(attr, H5T_NATIVE_INT, &type_key); + CAROM_VERIFY(errf >= 0); + + errf = H5Aclose(attr); + CAROM_VERIFY(errf >= 0); + + errf = H5Sclose(attr_id); + CAROM_VERIFY(errf >= 0); +#ifndef DEBUG_CHECK_ASSERTIONS + CAROM_NULL_USE(errf); +#endif +} + +} diff --git a/lib/utils/HDFDatabaseMPIO.h b/lib/utils/HDFDatabaseMPIO.h new file mode 100644 index 000000000..c8d596d23 --- /dev/null +++ b/lib/utils/HDFDatabaseMPIO.h @@ -0,0 +1,388 @@ +/****************************************************************************** + * + * Copyright (c) 2013-2023, Lawrence Livermore National Security, LLC + * and other libROM project developers. See the top-level COPYRIGHT + * file for details. + * + * SPDX-License-Identifier: (Apache-2.0 OR MIT) + * + *****************************************************************************/ + +// Description: The concrete database implementation using HDF5. + +#ifndef included_HDFDatabaseMPI_h +#define included_HDFDatabaseMPI_h + +#include "HDFDatabase.h" + +namespace CAROM { + +/** + * HDFDatabaseMPIO implements Multi-Path Input/Output (MPIO) of HDF5. + */ +class HDFDatabaseMPIO : public HDFDatabase +{ +public: + /** + * @brief Default constructor. + */ + HDFDatabaseMPIO(); + + /** + * @brief Destructor. + */ + virtual + ~HDFDatabaseMPIO() {} + + /** + * @brief Creates a new HDF5 database file with the supplied name. + * + * @param[in] file_name Name of HDF5 database file to create. + * + * @return True if file create was successful. + */ + bool + create_parallel( + const std::string& file_name, + const MPI_Comm comm) override; + + /** + * @brief Creates a new HDF5 database file with the supplied name. + * + * @param[in] file_name Name of HDF5 database file to create. + * + * @return True if file create was successful. + */ + bool + create( + const std::string& file_name) override + { + return create_parallel(file_name, MPI_COMM_WORLD); + } + + /** + * @brief Opens an existing HDF5 database file with the supplied name. + * + * @param[in] file_name Name of existing HDF5 database file to open. + * @param[in] type Read/write type ("r"/"wr") + * + * @return True if file open was successful. + */ + bool + open_parallel( + const std::string& file_name, + const std::string& type, + const MPI_Comm comm) override; + + /** + * @brief Opens an existing HDF5 database file with the supplied name. + * + * @param[in] file_name Name of existing HDF5 database file to open. + * @param[in] type Read/write type ("r"/"wr") + * + * @return True if file open was successful. + */ + bool + open( + const std::string& file_name, + const std::string& type) override + { + return open_parallel(file_name, type, MPI_COMM_WORLD); + } + + /** + * @brief Writes a distributed array of integers associated with the supplied key to + * the currently open HDF5 database file. + * + * @pre !key.empty() + * @pre data != nullptr + * @pre nelem_local >= 0 + * @pre nelements > 0 + * + * @param[in] key The key associated with the array of values to be + * written. + * @param[in] data The array of integer values to be written. + * @param[in] nelem_local The local number of integers in the array. + */ + virtual + void + putIntegerArray_parallel( + const std::string& key, + const int* const data, + int nelem_local); + + /** + * @brief Writes a local array of integers only for root rank, + * with associated supplied key to + * the currently open HDF5 database file. + * + * @pre !key.empty() + * @pre data != nullptr + * @pre nelements > 0 + * + * @param[in] key The key associated with the array of values to be + * written. + * @param[in] data The array of integer values to be written. + * @param[in] nelements The number of integers in the array. + */ + void + putIntegerArray( + const std::string& key, + const int* const data, + int nelements) override + { + if (d_rank != 0) + nelements = 0; + putIntegerArray_parallel(key, data, nelements); + } + + /** + * @brief Writes an array of doubles associated with the supplied key to + * the currently open HDF5 database file. + * + * @pre !key.empty() + * @pre data != nullptr + * @pre nelem_local >= 0 + * @pre nelements > 0 + * + * @param[in] key The key associated with the array of values to be + * written. + * @param[in] data The array of double values to be written. + * @param[in] nelem_local The number of doubles in the array. + */ + virtual + void + putDoubleArray_parallel( + const std::string& key, + const double* const data, + int nelem_local); + + /** + * @brief Writes an array of doubles associated with the supplied key to + * the currently open HDF5 database file. + * + * @pre !key.empty() + * @pre data != nullptr + * @pre nelements > 0 + * + * @param[in] key The key associated with the array of values to be + * written. + * @param[in] data The array of double values to be written. + * @param[in] nelements The number of doubles in the array. + */ + void + putDoubleArray( + const std::string& key, + const double* const data, + int nelements) override + { + if (d_rank != 0) + nelements = 0; + putDoubleArray_parallel(key, data, nelements); + } + + /** + * @brief Reads a distributed array of integers associated with the supplied key + * from the currently open HDF5 database file. + * + * @pre !key.empty() + * @pre data != nullptr || nelements == 0 + * + * @param[in] key The key associated with the array of values to be + * read. + * @param[out] data The allocated array of integer values to be read. + * @param[in] nelem_local The local number of integers in the array. + */ + virtual + void + getIntegerArray_parallel( + const std::string& key, + int* data, + int nelem_local); + + /** + * @brief Reads an array of integers associated with the supplied key + * from the currently open HDF5 database file. + * + * @pre !key.empty() + * @pre data != nullptr || nelements == 0 + * + * @param[in] key The key associated with the array of values to be + * read. + * @param[out] data The allocated array of integer values to be read. + * @param[in] nelements The number of integers in the array. + */ + void + getIntegerArray( + const std::string& key, + int* data, + int nelements) override + { + if (d_rank != 0) + nelements = 0; + getIntegerArray_parallel(key, data, nelements); + + MPI_Bcast(data, nelements, MPI_INT, 0, MPI_COMM_WORLD); + } + + /** + * @brief Reads an array of doubles associated with the supplied key + * from the currently open HDF5 database file. + * + * @pre !key.empty() + * @pre data != nullptr || nelements == 0 + * + * @param[in] key The key associated with the array of values to be + * read. + * @param[out] data The allocated array of double values to be read. + * @param[in] nelem_local The number of doubles in the array. + */ + virtual + void + getDoubleArray_parallel( + const std::string& key, + double* data, + int nelem_local); + + /** + * @brief Reads an array of doubles associated with the supplied key + * from the currently open HDF5 database file. + * + * @pre !key.empty() + * @pre data != nullptr || nelements == 0 + * + * @param[in] key The key associated with the array of values to be + * read. + * @param[out] data The allocated array of double values to be read. + * @param[in] nelements The number of doubles in the array. + */ + void + getDoubleArray( + const std::string& key, + double* data, + int nelements) override + { + if (d_rank != 0) + nelements = 0; + getDoubleArray_parallel(key, data, nelements); + + MPI_Bcast(data, nelements, MPI_DOUBLE, 0, MPI_COMM_WORLD); + } + + /** + * @brief Reads a sub-array of doubles associated with the supplied key + * from the currently open HDF5 database file. + * + * @pre !key.empty() + * @pre data != nullptr || nelements == 0 + * + * @param[in] key The key associated with the array of values to be + * read. + * @param[out] data The allocated sub-array of double values to be read. + * @param[in] nelem_local The number of doubles in the full array. + * @param[in] idx_local The set of indices in the sub-array. + */ + virtual + void + getDoubleArray_parallel( + const std::string& key, + double* data, + int nelem_local, + const std::vector& idx_local); + + /** + * @brief Reads a sub-array of doubles associated with the supplied key + * from the currently open HDF5 database file. + * + * @pre !key.empty() + * @pre data != nullptr || nelements == 0 + * + * @param[in] key The key associated with the array of values to be + * read. + * @param[out] data The allocated sub-array of double values to be read. + * @param[in] nelements The number of doubles in the full array. + * @param[in] idx The set of indices in the sub-array. + */ + void + getDoubleArray( + const std::string& key, + double* data, + int nelements, + const std::vector& idx) override + { + if (d_rank != 0) + nelements = 0; + getDoubleArray_parallel(key, data, nelements, idx); + + MPI_Bcast(data, nelements, MPI_DOUBLE, 0, MPI_COMM_WORLD); + } + + /** + * @brief Reads an array of doubles associated with the supplied key + * from the currently open HDF5 database file. + * + * @pre !key.empty() + * @pre data != nullptr || nelements == 0 + * + * @param[in] key The key associated with the array of values to be + * read. + * @param[out] data The allocated array of double values to be read. + * @param[in] nelements The number of doubles in the array. + * @param[in] offset The initial offset in the array. + * @param[in] block_size The block size to read from the HDF5 dataset. + * @param[in] stride The stride to read from the HDF5 dataset. + */ + virtual + void + getDoubleArray_parallel( + const std::string& key, + double* data, + int nelem_local, + int block_offset_global, + int block_size_global, + int stride_global); + + /** + * @brief Reads an array of doubles associated with the supplied key + * from the currently open HDF5 database file. + * + * @pre !key.empty() + * @pre data != nullptr || nelements == 0 + * + * @param[in] key The key associated with the array of values to be + * read. + * @param[out] data The allocated array of double values to be read. + * @param[in] nelements The number of doubles in the array. + * @param[in] offset The initial offset in the array. + * @param[in] block_size The block size to read from the HDF5 dataset. + * @param[in] stride The stride to read from the HDF5 dataset. + */ + void + getDoubleArray( + const std::string& key, + double* data, + int nelements, + int offset, + int block_size, + int stride) override + { + if (d_rank != 0) + nelements = 0; + getDoubleArray_parallel(key, data, nelements, offset, block_size, stride); + + MPI_Bcast(data, nelements, MPI_DOUBLE, 0, MPI_COMM_WORLD); + } + + void + writeAttribute( + int type_key, + hid_t dataset_id) override; + +private: + MPI_Comm d_comm; + int d_rank; +}; + +} + +#endif diff --git a/lib/utils/mpi_utils.cpp b/lib/utils/mpi_utils.cpp index a92d23094..99caeb32d 100644 --- a/lib/utils/mpi_utils.cpp +++ b/lib/utils/mpi_utils.cpp @@ -68,4 +68,13 @@ get_global_offsets(const int local_dim, std::vector &offsets, return dim; } +bool +is_same(int x, const MPI_Comm &comm) { + int p[2]; + p[0] = -x; + p[1] = x; + MPI_Allreduce(MPI_IN_PLACE, p, 2, MPI_INT, MPI_MIN, comm); + return (p[0] == -p[1]); +} + } diff --git a/lib/utils/mpi_utils.h b/lib/utils/mpi_utils.h index bb2a02a3b..fdfd5e142 100644 --- a/lib/utils/mpi_utils.h +++ b/lib/utils/mpi_utils.h @@ -43,6 +43,15 @@ int get_global_offsets(const int local_dim, std::vector &offsets, const MPI_Comm &comm=MPI_COMM_WORLD); +/** + * @brief Check if an integer is equal over all MPI processes. + * + * @param[in] x Input integer value to test equality. + * @param[in] comm MPI communicator. default value MPI_COMM_WORLD. + */ +bool +is_same(int x, const MPI_Comm &comm=MPI_COMM_WORLD); + } #endif From bf6fa2296e8309b6004b38dcdf9d79b12c21eb6a Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Wed, 21 Feb 2024 14:22:23 -0800 Subject: [PATCH 39/50] rebase from generator-fix2 --- unit_tests/test_HDFDatabase.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unit_tests/test_HDFDatabase.cpp b/unit_tests/test_HDFDatabase.cpp index 227f3df47..cf5bde264 100644 --- a/unit_tests/test_HDFDatabase.cpp +++ b/unit_tests/test_HDFDatabase.cpp @@ -554,7 +554,7 @@ TEST(BasisGeneratorIO, HDFDatabase) for (int d = 0; d < nrow_local; d++) sample(d) = snapshots(d, s); - sampler.takeSample(sample.getData(), 0, 0); + sampler.takeSample(sample.getData()); } sampler.endSamples(); sampler.writeSnapshot(); From efe2f7158b8d6eef7afaa67010585702ff4199b6 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Wed, 21 Feb 2024 15:55:22 -0800 Subject: [PATCH 40/50] deployed HDFDatabaseMPIO to basis classes. --- lib/linalg/BasisGenerator.cpp | 2 +- lib/linalg/BasisReader.cpp | 41 +++++++-- lib/linalg/BasisReader.h | 17 ++++ lib/linalg/BasisWriter.cpp | 41 +++++++-- lib/linalg/svd/StaticSVD.cpp | 2 +- lib/utils/HDFDatabaseMPIO.cpp | 2 +- lib/utils/HDFDatabaseMPIO.h | 22 ++--- unit_tests/test_HDFDatabase.cpp | 151 ++++++++++++++++++++++++++++++-- 8 files changed, 238 insertions(+), 40 deletions(-) diff --git a/lib/linalg/BasisGenerator.cpp b/lib/linalg/BasisGenerator.cpp index 6b2018df7..3a1cc9c8e 100644 --- a/lib/linalg/BasisGenerator.cpp +++ b/lib/linalg/BasisGenerator.cpp @@ -168,7 +168,7 @@ BasisGenerator::loadSamples(const std::string& base_file_name, if (d_basis_reader) delete d_basis_reader; - d_basis_reader = new BasisReader(base_file_name, db_format); + d_basis_reader = new BasisReader(base_file_name, db_format, d_dim); const Matrix* mat; const Vector* singular_vals; diff --git a/lib/linalg/BasisReader.cpp b/lib/linalg/BasisReader.cpp index e6d1c2809..ef5340207 100644 --- a/lib/linalg/BasisReader.cpp +++ b/lib/linalg/BasisReader.cpp @@ -12,10 +12,11 @@ #include "BasisReader.h" #include "utils/HDFDatabase.h" -#include "utils/CSVDatabase.h" +#include "utils/HDFDatabaseMPIO.h" #include "Matrix.h" #include "Vector.h" #include "mpi.h" +#include "utils/mpi_utils.h" namespace CAROM { @@ -26,7 +27,8 @@ BasisReader::BasisReader( d_dim(dim), d_last_basis_idx(-1), full_file_name(""), - base_file_name_(base_file_name) + base_file_name_(base_file_name), + d_format(db_format) { CAROM_ASSERT(!base_file_name.empty()); @@ -40,13 +42,27 @@ BasisReader::BasisReader( rank = 0; } - // Enforce hdf data format. - CAROM_VERIFY(db_format != Database::CSV); - full_file_name = base_file_name; - if (db_format == Database::HDF5) { + + // Enforce hdf data format. + if (d_format == Database::HDF5) + { d_database = new HDFDatabase(); } + else if (d_format == Database::HDF5_MPIO) + { + /* + For MPIO case, local dimension needs to be specified. + We allow 0 local dimension. (global dimension still needs to be positive) + */ + std::vector tmp; + d_global_dim = get_global_offsets(d_dim, tmp, MPI_COMM_WORLD); + CAROM_VERIFY(d_dim >= 0); + CAROM_VERIFY(d_global_dim > 0); + d_database = new HDFDatabaseMPIO(); + } + else + CAROM_ERROR("BasisWriter only supports HDF5/HDF5_MPIO data format!\n"); d_database->open_parallel(full_file_name, "r", MPI_COMM_WORLD); } @@ -289,7 +305,18 @@ BasisReader::getDim( "temporal_basis_num_rows_000000"); d_database->getInteger(tmp, num_rows); - return num_rows; + /* only basis and snapshot are stored as distributed matrices */ + if ((kind != "temporal_basis") && (d_format == Database::HDF5_MPIO)) + { + /* + for MPIO database, return specified local dimension. + only checks the global dimension match. + */ + CAROM_VERIFY(d_global_dim == num_rows); + return d_dim; + } + else + return num_rows; } int diff --git a/lib/linalg/BasisReader.h b/lib/linalg/BasisReader.h index 5ed12d371..8683f6458 100644 --- a/lib/linalg/BasisReader.h +++ b/lib/linalg/BasisReader.h @@ -213,6 +213,11 @@ class BasisReader { * * @brief Returns the dimension of the system on this processor. * + * @param[in] kind The target matrix of which dimension (row) is returned. + * "basis" - local dimension of spatial basis + * "snapshot" - local dimension of snapshot matrix + * "temporal_basis" - global dimension of temporal basis + * * @return The dimension of the system on this processor. */ int @@ -293,6 +298,11 @@ class BasisReader { */ Database* d_database; + /** + * @brief The database being read from. + */ + Database::formats d_format; + /** * @brief Base file name stored for consistency between reading and writing. */ @@ -314,6 +324,13 @@ class BasisReader { * If negative, use the dimension from the rank-specific local file. */ const int d_dim; + + /** + * @brief Dimension of the basis on this processor. + * + * If negative, use the dimension from the rank-specific local file. + */ + int d_global_dim; }; } diff --git a/lib/linalg/BasisWriter.cpp b/lib/linalg/BasisWriter.cpp index 1e98542e5..e38eee0ea 100644 --- a/lib/linalg/BasisWriter.cpp +++ b/lib/linalg/BasisWriter.cpp @@ -12,6 +12,7 @@ #include "BasisWriter.h" #include "utils/HDFDatabase.h" +#include "utils/HDFDatabaseMPIO.h" #include "Matrix.h" #include "Vector.h" #include "BasisGenerator.h" @@ -55,9 +56,18 @@ BasisWriter::BasisWriter( snap_file_name = base_file_name + "_snapshot"; // create and open snapshot/basis database - CAROM_VERIFY(db_format_ == Database::HDF5); - d_snap_database = new HDFDatabase(); - d_database = new HDFDatabase(); + if (db_format_ == Database::HDF5) + { + d_snap_database = new HDFDatabase(); + d_database = new HDFDatabase(); + } + else if (db_format_ == Database::HDF5_MPIO) + { + d_snap_database = new HDFDatabaseMPIO(); + d_database = new HDFDatabaseMPIO(); + } + else + CAROM_ERROR("BasisWriter only supports HDF5/HDF5_MPIO data format!\n"); } BasisWriter::~BasisWriter() @@ -79,9 +89,14 @@ BasisWriter::writeBasis(const std::string& kind) d_database->create_parallel(full_file_name, MPI_COMM_WORLD); const Matrix* basis = d_basis_generator->getSpatialBasis(); + /* singular is always distributed */ + CAROM_VERIFY(basis->distributed()); int num_rows = basis->numRows(); + int nrows_infile = num_rows; + if (db_format_ == Database::HDF5_MPIO) + MPI_Allreduce(MPI_IN_PLACE, &nrows_infile, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); sprintf(tmp, "spatial_basis_num_rows_000000"); - d_database->putInteger(tmp, num_rows); + d_database->putInteger(tmp, nrows_infile); int num_cols = basis->numColumns(); sprintf(tmp, "spatial_basis_num_cols_000000"); d_database->putInteger(tmp, num_cols); @@ -90,6 +105,8 @@ BasisWriter::writeBasis(const std::string& kind) if(d_basis_generator->updateRightSV()) { const Matrix* tbasis = d_basis_generator->getTemporalBasis(); + /* temporal basis is always not distributed */ + CAROM_VERIFY(!tbasis->distributed()); num_rows = tbasis->numRows(); sprintf(tmp, "temporal_basis_num_rows_000000"); d_database->putInteger(tmp, num_rows); @@ -97,15 +114,17 @@ BasisWriter::writeBasis(const std::string& kind) sprintf(tmp, "temporal_basis_num_cols_000000"); d_database->putInteger(tmp, num_cols); sprintf(tmp, "temporal_basis_000000"); - d_database->putDoubleArray_parallel(tmp, &tbasis->item(0, 0), num_rows*num_cols); + d_database->putDoubleArray(tmp, &tbasis->item(0, 0), num_rows*num_cols); } const Vector* sv = d_basis_generator->getSingularValues(); + /* singular is always not distributed */ + CAROM_VERIFY(!sv->distributed()); int sv_dim = sv->dim(); sprintf(tmp, "singular_value_size_000000"); d_database->putInteger(tmp, sv_dim); sprintf(tmp, "singular_value_000000"); - d_database->putDoubleArray_parallel(tmp, &sv->item(0), sv_dim); + d_database->putDoubleArray(tmp, &sv->item(0), sv_dim); d_database->close(); } @@ -114,14 +133,20 @@ BasisWriter::writeBasis(const std::string& kind) d_snap_database->create_parallel(snap_file_name, MPI_COMM_WORLD); const Matrix* snapshots = d_basis_generator->getSnapshotMatrix(); + /* snapshot matrix is always distributed */ + CAROM_VERIFY(snapshots->distributed()); int num_rows = snapshots->numRows(); // d_dim + int nrows_infile = num_rows; + if (db_format_ == Database::HDF5_MPIO) + MPI_Allreduce(MPI_IN_PLACE, &nrows_infile, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); sprintf(tmp, "snapshot_matrix_num_rows_000000"); - d_snap_database->putInteger(tmp, num_rows); + d_snap_database->putInteger(tmp, nrows_infile); int num_cols = snapshots->numColumns(); // d_num_samples sprintf(tmp, "snapshot_matrix_num_cols_000000"); d_snap_database->putInteger(tmp, num_cols); sprintf(tmp, "snapshot_matrix_000000"); - d_snap_database->putDoubleArray_parallel(tmp, &snapshots->item(0,0), num_rows*num_cols); + d_snap_database->putDoubleArray_parallel(tmp, &snapshots->item(0,0), + num_rows*num_cols); d_snap_database->close(); } diff --git a/lib/linalg/svd/StaticSVD.cpp b/lib/linalg/svd/StaticSVD.cpp index 47e8344e1..2ea2c851e 100644 --- a/lib/linalg/svd/StaticSVD.cpp +++ b/lib/linalg/svd/StaticSVD.cpp @@ -221,7 +221,7 @@ StaticSVD::getSnapshotMatrix() " To preserve the snapshots, set Options::static_svd_preserve_snapshot to be true!\n"); if (d_snapshots) delete d_snapshots; - d_snapshots = new Matrix(d_dim, d_num_samples, false); + d_snapshots = new Matrix(d_dim, d_num_samples, true); for (int rank = 0; rank < d_num_procs; ++rank) { int nrows = d_dims[static_cast(rank)]; diff --git a/lib/utils/HDFDatabaseMPIO.cpp b/lib/utils/HDFDatabaseMPIO.cpp index ef96f6796..0be4f85fe 100644 --- a/lib/utils/HDFDatabaseMPIO.cpp +++ b/lib/utils/HDFDatabaseMPIO.cpp @@ -514,7 +514,7 @@ HDFDatabaseMPIO::getDoubleArray_parallel( hsize_t num_local_blocks = nelem_local / block_size_global; std::vector global_offsets; int nelements = CAROM::get_global_offsets(num_local_blocks * stride_global, - global_offsets, d_comm); + global_offsets, d_comm); CAROM_VERIFY(!key.empty()); #ifndef DEBUG_CHECK_ASSERTIONS diff --git a/lib/utils/HDFDatabaseMPIO.h b/lib/utils/HDFDatabaseMPIO.h index c8d596d23..b45c1f3f4 100644 --- a/lib/utils/HDFDatabaseMPIO.h +++ b/lib/utils/HDFDatabaseMPIO.h @@ -73,7 +73,7 @@ class HDFDatabaseMPIO : public HDFDatabase const std::string& file_name, const std::string& type, const MPI_Comm comm) override; - + /** * @brief Opens an existing HDF5 database file with the supplied name. * @@ -218,9 +218,8 @@ class HDFDatabaseMPIO : public HDFDatabase int* data, int nelements) override { - if (d_rank != 0) - nelements = 0; - getIntegerArray_parallel(key, data, nelements); + int read_size = (d_rank == 0) ? nelements : 0; + getIntegerArray_parallel(key, data, read_size); MPI_Bcast(data, nelements, MPI_INT, 0, MPI_COMM_WORLD); } @@ -262,9 +261,8 @@ class HDFDatabaseMPIO : public HDFDatabase double* data, int nelements) override { - if (d_rank != 0) - nelements = 0; - getDoubleArray_parallel(key, data, nelements); + int read_size = (d_rank == 0) ? nelements : 0; + getDoubleArray_parallel(key, data, read_size); MPI_Bcast(data, nelements, MPI_DOUBLE, 0, MPI_COMM_WORLD); } @@ -310,9 +308,8 @@ class HDFDatabaseMPIO : public HDFDatabase int nelements, const std::vector& idx) override { - if (d_rank != 0) - nelements = 0; - getDoubleArray_parallel(key, data, nelements, idx); + int read_size = (d_rank == 0) ? nelements : 0; + getDoubleArray_parallel(key, data, read_size, idx); MPI_Bcast(data, nelements, MPI_DOUBLE, 0, MPI_COMM_WORLD); } @@ -366,9 +363,8 @@ class HDFDatabaseMPIO : public HDFDatabase int block_size, int stride) override { - if (d_rank != 0) - nelements = 0; - getDoubleArray_parallel(key, data, nelements, offset, block_size, stride); + int read_size = (d_rank == 0) ? nelements : 0; + getDoubleArray_parallel(key, data, read_size, offset, block_size, stride); MPI_Bcast(data, nelements, MPI_DOUBLE, 0, MPI_COMM_WORLD); } diff --git a/unit_tests/test_HDFDatabase.cpp b/unit_tests/test_HDFDatabase.cpp index cf5bde264..fa9f75fd8 100644 --- a/unit_tests/test_HDFDatabase.cpp +++ b/unit_tests/test_HDFDatabase.cpp @@ -23,6 +23,9 @@ #include using namespace std::chrono; +static const int nrow = 100, ncol = 40; +static const double threshold = 1.0e-13; + /** * Simple smoke test to make sure Google Test is properly linked */ @@ -37,7 +40,6 @@ TEST(HDF5, Test_parallel_writing) MPI_Comm_rank(MPI_COMM_WORLD, &rank); const int dim_rank = 2; - const int nrow = 1000, ncol = 400; const int nrow_local = CAROM::split_dimension(nrow, MPI_COMM_WORLD); std::vector offsets; int dummy = CAROM::get_global_offsets(nrow_local, offsets, MPI_COMM_WORLD); @@ -123,6 +125,21 @@ TEST(HDF5, Test_parallel_writing) errf = H5Dwrite(dset_id, H5T_NATIVE_INT, memspace, filespace, plist_id, data); CAROM_VERIFY(errf >= 0); + /* + * Write a integer scalar attribute + */ + // Only last process attribute writes the attribute. + const int test_attr = rank + 1234; + hid_t attr_id = H5Screate(H5S_SCALAR); + hid_t attr = H5Acreate(dset_id, "test_attr", H5T_NATIVE_INT, + attr_id, H5P_DEFAULT, H5P_DEFAULT); + + // if (rank == 1) + errf = H5Awrite(attr, H5T_NATIVE_INT, &test_attr); + errf = H5Aclose(attr); + errf = H5Sclose(attr_id); + MPI_Barrier(MPI_COMM_WORLD); + /* * Close/release resources. */ @@ -145,7 +162,6 @@ TEST(HDF5, Test_parallel_reading) MPI_Comm_rank(MPI_COMM_WORLD, &rank); const int dim_rank = 2; - const int nrow = 1000, ncol = 400; const int nrow_local = CAROM::split_dimension(nrow, MPI_COMM_WORLD); std::vector offsets; int dummy = CAROM::get_global_offsets(nrow_local, offsets, MPI_COMM_WORLD); @@ -235,6 +251,17 @@ TEST(HDF5, Test_parallel_reading) errf = H5Dread(dset_id, H5T_NATIVE_INT, memspace, filespace, plist_id, data); CAROM_VERIFY(errf >= 0); + /* + * Write a integer scalar attribute + */ + int test_attr = -1; + hid_t attr = H5Aopen_name(dset_id, "test_attr"); + errf = H5Aread(attr, H5T_NATIVE_INT, &test_attr); + // MPI_Barrier(MPI_COMM_WORLD); + errf = H5Aclose(attr); + if (rank == 0) + EXPECT_EQ(test_attr, nproc - 1 + 1234); + /* * Close/release resources. */ @@ -271,7 +298,6 @@ TEST(HDF5, Test_selective_parallel_writing) MPI_Comm_create(MPI_COMM_WORLD, io_grp, &io_comm); const int dim_rank = 2; - const int nrow = 10, ncol = 4; int nrow_local = 0; if (rank < ioproc) nrow_local = CAROM::split_dimension(nrow, io_comm); @@ -398,7 +424,6 @@ TEST(HDF5, Test_selective_parallel_reading) MPI_Comm_create(MPI_COMM_WORLD, io_grp, &io_comm); const int dim_rank = 2; - const int nrow = 10, ncol = 4; int nrow_local = 0; if (rank < ioproc) nrow_local = CAROM::split_dimension(nrow, io_comm); @@ -524,11 +549,10 @@ TEST(BasisGeneratorIO, HDFDatabase) MPI_Comm_rank(MPI_COMM_WORLD, &d_rank); MPI_Comm_size(MPI_COMM_WORLD, &d_num_procs); - const int nrow = 100, ncol = 10; int nrow_local = CAROM::split_dimension(nrow, MPI_COMM_WORLD); std::vector row_offset(d_num_procs + 1); const int dummy = CAROM::get_global_offsets(nrow_local, row_offset, - MPI_COMM_WORLD); + MPI_COMM_WORLD); EXPECT_EQ(nrow, dummy); std::default_random_engine generator; @@ -547,6 +571,7 @@ TEST(BasisGeneratorIO, HDFDatabase) CAROM::Options svd_options = CAROM::Options(nrow_local, ncol, 1); svd_options.setMaxBasisDimension(nrow); svd_options.setRandomizedSVD(false); + svd_options.setDebugMode(true); CAROM::BasisGenerator sampler(svd_options, false, "test_basis"); CAROM::Vector sample(nrow_local, true); for (int s = 0; s < ncol; s++) @@ -556,12 +581,76 @@ TEST(BasisGeneratorIO, HDFDatabase) sampler.takeSample(sample.getData()); } - sampler.endSamples(); + sampler.writeSnapshot(); + const CAROM::Matrix *snapshot = sampler.getSnapshotMatrix(); + sampler.endSamples(); const CAROM::Matrix *spatial_basis = sampler.getSpatialBasis(); + + CAROM::BasisReader basis_reader("test_basis"); + const CAROM::Matrix *spatial_basis1 = basis_reader.getSpatialBasis(); + + EXPECT_EQ(spatial_basis->numRows(), spatial_basis1->numRows()); + EXPECT_EQ(spatial_basis->numColumns(), spatial_basis1->numColumns()); + for (int i = 0; i < spatial_basis->numRows(); i++) + for (int j = 0; j < spatial_basis->numColumns(); j++) + EXPECT_NEAR((*spatial_basis)(i, j), (*spatial_basis1)(i, j), threshold); + + CAROM::BasisReader snapshot_reader("test_basis_snapshot"); + const CAROM::Matrix *snapshot1 = snapshot_reader.getSnapshotMatrix(); + + EXPECT_EQ(snapshot->numRows(), snapshot1->numRows()); + EXPECT_EQ(snapshot->numColumns(), snapshot1->numColumns()); + for (int i = 0; i < snapshot->numRows(); i++) + for (int j = 0; j < snapshot->numColumns(); j++) + EXPECT_NEAR((*snapshot)(i, j), (*snapshot1)(i, j), threshold); +} + +TEST(BasisGeneratorIO, Base_MPIO_combination) +{ + // Get the rank of this process, and the number of processors. + int mpi_init, d_rank, d_num_procs; + MPI_Initialized(&mpi_init); + if (mpi_init == 0) { + MPI_Init(nullptr, nullptr); + } + + MPI_Comm_rank(MPI_COMM_WORLD, &d_rank); + MPI_Comm_size(MPI_COMM_WORLD, &d_num_procs); + + int nrow_local = CAROM::split_dimension(nrow, MPI_COMM_WORLD); + std::vector row_offset(d_num_procs + 1); + const int dummy = CAROM::get_global_offsets(nrow_local, row_offset, + MPI_COMM_WORLD); + EXPECT_EQ(nrow, dummy); + + std::string base_name = "test_basis"; + std::string mpio_name = "test_mpio"; + CAROM::Options svd_options = CAROM::Options(nrow_local, ncol, 1); + svd_options.setMaxBasisDimension(nrow); + svd_options.setRandomizedSVD(false); + svd_options.setDebugMode(true); + CAROM::BasisGenerator sampler(svd_options, false, mpio_name, + CAROM::Database::HDF5_MPIO); + + sampler.loadSamples(base_name + "_snapshot", "snapshot", 1e9, + CAROM::Database::HDF5); + sampler.writeSnapshot(); const CAROM::Matrix *snapshot = sampler.getSnapshotMatrix(); + CAROM::BasisReader snapshot_reader("test_basis_snapshot"); + const CAROM::Matrix *snapshot1 = snapshot_reader.getSnapshotMatrix(); + + EXPECT_EQ(snapshot->numRows(), snapshot1->numRows()); + EXPECT_EQ(snapshot->numColumns(), snapshot1->numColumns()); + for (int i = 0; i < snapshot->numRows(); i++) + for (int j = 0; j < snapshot->numColumns(); j++) + EXPECT_NEAR((*snapshot)(i, j), (*snapshot1)(i, j), threshold); + + sampler.endSamples(); + const CAROM::Matrix *spatial_basis = sampler.getSpatialBasis(); + CAROM::BasisReader basis_reader("test_basis"); const CAROM::Matrix *spatial_basis1 = basis_reader.getSpatialBasis(); @@ -569,7 +658,39 @@ TEST(BasisGeneratorIO, HDFDatabase) EXPECT_EQ(spatial_basis->numColumns(), spatial_basis1->numColumns()); for (int i = 0; i < spatial_basis->numRows(); i++) for (int j = 0; j < spatial_basis->numColumns(); j++) - EXPECT_NEAR((*spatial_basis)(i, j), (*spatial_basis1)(i, j), 1.0e-15); + EXPECT_NEAR((*spatial_basis)(i, j), (*spatial_basis1)(i, j), threshold); +} + +TEST(BasisGeneratorIO, MPIO_Base_combination) +{ + // Get the rank of this process, and the number of processors. + int mpi_init, d_rank, d_num_procs; + MPI_Initialized(&mpi_init); + if (mpi_init == 0) { + MPI_Init(nullptr, nullptr); + } + + MPI_Comm_rank(MPI_COMM_WORLD, &d_rank); + MPI_Comm_size(MPI_COMM_WORLD, &d_num_procs); + + int nrow_local = CAROM::split_dimension(nrow, MPI_COMM_WORLD); + std::vector row_offset(d_num_procs + 1); + const int dummy = CAROM::get_global_offsets(nrow_local, row_offset, + MPI_COMM_WORLD); + EXPECT_EQ(nrow, dummy); + + std::string test_name = "test_basis2"; + std::string mpio_name = "test_mpio"; + CAROM::Options svd_options = CAROM::Options(nrow_local, ncol, 1); + svd_options.setMaxBasisDimension(nrow); + svd_options.setRandomizedSVD(false); + svd_options.setDebugMode(true); + CAROM::BasisGenerator sampler(svd_options, false, test_name, + CAROM::Database::HDF5); + + sampler.loadSamples(mpio_name + "_snapshot", "snapshot", 1e9, + CAROM::Database::HDF5_MPIO); + const CAROM::Matrix *snapshot = sampler.getSnapshotMatrix(); CAROM::BasisReader snapshot_reader("test_basis_snapshot"); const CAROM::Matrix *snapshot1 = snapshot_reader.getSnapshotMatrix(); @@ -578,7 +699,19 @@ TEST(BasisGeneratorIO, HDFDatabase) EXPECT_EQ(snapshot->numColumns(), snapshot1->numColumns()); for (int i = 0; i < snapshot->numRows(); i++) for (int j = 0; j < snapshot->numColumns(); j++) - EXPECT_NEAR((*snapshot)(i, j), (*snapshot1)(i, j), 1.0e-15); + EXPECT_NEAR((*snapshot)(i, j), (*snapshot1)(i, j), threshold); + + sampler.endSamples(); + const CAROM::Matrix *spatial_basis = sampler.getSpatialBasis(); + + CAROM::BasisReader basis_reader("test_basis"); + const CAROM::Matrix *spatial_basis1 = basis_reader.getSpatialBasis(); + + EXPECT_EQ(spatial_basis->numRows(), spatial_basis1->numRows()); + EXPECT_EQ(spatial_basis->numColumns(), spatial_basis1->numColumns()); + for (int i = 0; i < spatial_basis->numRows(); i++) + for (int j = 0; j < spatial_basis->numColumns(); j++) + EXPECT_NEAR((*spatial_basis)(i, j), (*spatial_basis1)(i, j), threshold); } int main(int argc, char* argv[]) From 736bc11659b70f42fb522869f37824f83b86b0ca Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Wed, 21 Feb 2024 16:17:08 -0800 Subject: [PATCH 41/50] test for partial getSpatialBasis. --- unit_tests/test_HDFDatabase.cpp | 50 +++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/unit_tests/test_HDFDatabase.cpp b/unit_tests/test_HDFDatabase.cpp index fa9f75fd8..7ceb2116d 100644 --- a/unit_tests/test_HDFDatabase.cpp +++ b/unit_tests/test_HDFDatabase.cpp @@ -714,6 +714,56 @@ TEST(BasisGeneratorIO, MPIO_Base_combination) EXPECT_NEAR((*spatial_basis)(i, j), (*spatial_basis1)(i, j), threshold); } +TEST(BasisReaderIO, partial_getSpatialBasis) +{ + // Get the rank of this process, and the number of processors. + int mpi_init, d_rank, d_num_procs; + MPI_Initialized(&mpi_init); + if (mpi_init == 0) { + MPI_Init(nullptr, nullptr); + } + + MPI_Comm_rank(MPI_COMM_WORLD, &d_rank); + MPI_Comm_size(MPI_COMM_WORLD, &d_num_procs); + + int nrow_local = CAROM::split_dimension(nrow, MPI_COMM_WORLD); + std::vector row_offset(d_num_procs + 1); + const int dummy = CAROM::get_global_offsets(nrow_local, row_offset, + MPI_COMM_WORLD); + EXPECT_EQ(nrow, dummy); + + std::string base_name = "test_basis"; + std::string mpio_name = "test_mpio"; + + CAROM::BasisReader basis_reader("test_basis"); + const CAROM::Matrix *spatial_basis = basis_reader.getSpatialBasis(); + + CAROM::BasisReader basis_reader1("test_mpio", CAROM::Database::HDF5_MPIO, + nrow_local); + const CAROM::Matrix *spatial_basis1 = basis_reader1.getSpatialBasis(); + + EXPECT_EQ(spatial_basis->numRows(), spatial_basis1->numRows()); + EXPECT_EQ(spatial_basis->numColumns(), spatial_basis1->numColumns()); + for (int i = 0; i < spatial_basis->numRows(); i++) + for (int j = 0; j < spatial_basis->numColumns(); j++) + EXPECT_NEAR((*spatial_basis)(i, j), (*spatial_basis1)(i, j), threshold); + + delete spatial_basis; + delete spatial_basis1; + const int col1 = 3, col2 = 7; + spatial_basis = basis_reader.getSpatialBasis(col1, col2); + spatial_basis1 = basis_reader1.getSpatialBasis(col1, col2); + + EXPECT_EQ(spatial_basis->numRows(), spatial_basis1->numRows()); + EXPECT_EQ(spatial_basis->numColumns(), spatial_basis1->numColumns()); + for (int i = 0; i < spatial_basis->numRows(); i++) + for (int j = 0; j < spatial_basis->numColumns(); j++) + EXPECT_NEAR((*spatial_basis)(i, j), (*spatial_basis1)(i, j), threshold); + + delete spatial_basis; + delete spatial_basis1; +} + int main(int argc, char* argv[]) { ::testing::InitGoogleTest(&argc, argv); From 21238625019736b99dac4694355e499c8cf74cce Mon Sep 17 00:00:00 2001 From: Seung Whan Chung Date: Wed, 21 Feb 2024 17:15:36 -0800 Subject: [PATCH 42/50] test routien for scaling. --- unit_tests/test_HDFDatabase.cpp | 94 ++++++++++++++++++++++++++++++--- 1 file changed, 88 insertions(+), 6 deletions(-) diff --git a/unit_tests/test_HDFDatabase.cpp b/unit_tests/test_HDFDatabase.cpp index 7ceb2116d..14548f508 100644 --- a/unit_tests/test_HDFDatabase.cpp +++ b/unit_tests/test_HDFDatabase.cpp @@ -23,7 +23,7 @@ #include using namespace std::chrono; -static const int nrow = 100, ncol = 40; +static const int nrow = 123, ncol = 21; static const double threshold = 1.0e-13; /** @@ -77,7 +77,7 @@ TEST(HDF5, Test_parallel_writing) /* * Create a new file collectively and release property list identifier. */ - file_id = H5Fcreate("test.h5", H5F_ACC_TRUNC, H5P_DEFAULT, plist_id); + file_id = H5Fcreate("test1.h5", H5F_ACC_TRUNC, H5P_DEFAULT, plist_id); H5Pclose(plist_id); /* @@ -128,8 +128,9 @@ TEST(HDF5, Test_parallel_writing) /* * Write a integer scalar attribute */ - // Only last process attribute writes the attribute. - const int test_attr = rank + 1234; + // Only one process attribute writes the attribute. + // Make sure all processes have the same value. + const int test_attr = 1234; hid_t attr_id = H5Screate(H5S_SCALAR); hid_t attr = H5Acreate(dset_id, "test_attr", H5T_NATIVE_INT, attr_id, H5P_DEFAULT, H5P_DEFAULT); @@ -197,7 +198,7 @@ TEST(HDF5, Test_parallel_reading) /* * Open a new file collectively and release property list identifier. */ - file_id = H5Fopen("test.h5", H5F_ACC_RDONLY, plist_id); + file_id = H5Fopen("test1.h5", H5F_ACC_RDONLY, plist_id); H5Pclose(plist_id); /* @@ -260,7 +261,7 @@ TEST(HDF5, Test_parallel_reading) // MPI_Barrier(MPI_COMM_WORLD); errf = H5Aclose(attr); if (rank == 0) - EXPECT_EQ(test_attr, nproc - 1 + 1234); + EXPECT_EQ(test_attr, 1234); /* * Close/release resources. @@ -764,6 +765,87 @@ TEST(BasisReaderIO, partial_getSpatialBasis) delete spatial_basis1; } +TEST(BasisGeneratorIO, Scaling_test) +{ + int nproc, rank; + MPI_Comm_size(MPI_COMM_WORLD, &nproc); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + + const int scale_nrow = 500, scale_ncol = 100; + int nrow_local = CAROM::split_dimension(scale_nrow, MPI_COMM_WORLD); + std::vector row_offset(nproc + 1); + const int dummy = CAROM::get_global_offsets(nrow_local, row_offset, + MPI_COMM_WORLD); + EXPECT_EQ(scale_nrow, dummy); + + std::default_random_engine generator; + generator.seed( + 1234); // fix the seed to keep the same result for different nproc. + std::uniform_real_distribution<> uniform_distribution(0.0, 1.0); + std::normal_distribution normal_distribution(0.0, 1.0); + + // distribute from a global matrix to keep the same system for different nproc. + CAROM::Matrix snapshots(scale_nrow, scale_ncol, false); + for (int i = 0; i < scale_nrow; i++) + for (int j = 0; j < scale_ncol; j++) + snapshots(i, j) = normal_distribution(generator); + snapshots.distribute(nrow_local); + + CAROM::Options svd_options = CAROM::Options(nrow_local, scale_ncol, 1); + svd_options.setMaxBasisDimension(scale_nrow); + svd_options.setRandomizedSVD(false); + + CAROM::BasisGenerator base_sampler(svd_options, false, "base"); + CAROM::BasisGenerator mpio_sampler(svd_options, false, "mpio", + CAROM::Database::HDF5_MPIO); + + CAROM::Vector sample(nrow_local, true); + for (int s = 0; s < scale_ncol; s++) + { + for (int d = 0; d < nrow_local; d++) + sample(d) = snapshots(d, s); + + base_sampler.takeSample(sample.getData()); + mpio_sampler.takeSample(sample.getData()); + } + + MPI_Barrier(MPI_COMM_WORLD); + auto start = steady_clock::now(); + base_sampler.writeSnapshot(); + MPI_Barrier(MPI_COMM_WORLD); + auto stop = steady_clock::now(); + auto duration = duration_cast(stop-start); + if (rank == 0) + printf("Base writeSnapshot- duration: %dms\n", duration); + + MPI_Barrier(MPI_COMM_WORLD); + start = steady_clock::now(); + mpio_sampler.writeSnapshot(); + MPI_Barrier(MPI_COMM_WORLD); + stop = steady_clock::now(); + duration = duration_cast(stop-start); + if (rank == 0) + printf("MPIO writeSnapshot- duration: %dms\n", duration); + + MPI_Barrier(MPI_COMM_WORLD); + start = steady_clock::now(); + base_sampler.endSamples(); + MPI_Barrier(MPI_COMM_WORLD); + stop = steady_clock::now(); + duration = duration_cast(stop-start); + if (rank == 0) + printf("Base endSamples- duration: %dms\n", duration); + + MPI_Barrier(MPI_COMM_WORLD); + start = steady_clock::now(); + mpio_sampler.endSamples(); + MPI_Barrier(MPI_COMM_WORLD); + stop = steady_clock::now(); + duration = duration_cast(stop-start); + if (rank == 0) + printf("MPIO endSamples- duration: %dms\n", duration); +} + int main(int argc, char* argv[]) { ::testing::InitGoogleTest(&argc, argv); From 945995fe1fb5dacaa06ff71c5ac92ca59e93b74d Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Wed, 21 Feb 2024 20:48:35 -0800 Subject: [PATCH 43/50] add hdf database test in ci workflow. --- .github/workflows/run_tests/action.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/run_tests/action.yml b/.github/workflows/run_tests/action.yml index 7a7ee5b2b..9f412b585 100644 --- a/.github/workflows/run_tests/action.yml +++ b/.github/workflows/run_tests/action.yml @@ -23,6 +23,8 @@ runs: mpirun -n 3 --oversubscribe tests/test_StaticSVD ./tests/test_IncrementalSVDBrand mpirun -n 3 --oversubscribe tests/test_IncrementalSVDBrand + ./tests/test_HDFDatabase + mpirun -n 3 --oversubscribe tests/test_HDFDatabase shell: bash From f36fbb491a0deca0f22d398f8f27d9c40a0efc4c Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Thu, 29 Feb 2024 11:50:41 -0800 Subject: [PATCH 44/50] update doxygen description --- lib/utils/Database.h | 6 ++++ lib/utils/HDFDatabase.h | 3 ++ lib/utils/HDFDatabaseMPIO.cpp | 1 + lib/utils/HDFDatabaseMPIO.h | 59 ++++++++++++++++++++++++----------- 4 files changed, 50 insertions(+), 19 deletions(-) diff --git a/lib/utils/Database.h b/lib/utils/Database.h index b1eeff04d..4d5daee59 100644 --- a/lib/utils/Database.h +++ b/lib/utils/Database.h @@ -260,8 +260,11 @@ class Database * @param[out] data The allocated array of double values to be read. * @param[in] nelements The number of doubles in the array. * @param[in] offset The initial offset in the array. + * Typically, this is a column index of the matrix data. * @param[in] block_size The block size to read from the HDF5 dataset. + * Typically, this is a number of columns of the matrix data. * @param[in] stride The stride to read from the HDF5 dataset. + * Typically, this is the total number of columns of the matrix data. */ virtual void @@ -388,8 +391,11 @@ class Database * @param[out] data The allocated array of double values to be read. * @param[in] nelements The number of doubles in the array. * @param[in] offset The initial offset in the array. + * Typically, this is a column index of the matrix data. * @param[in] block_size The block size to read from the HDF5 dataset. + * Typically, this is a number of columns of the matrix data. * @param[in] stride The stride to read from the HDF5 dataset. + * Typically, this is the total number of columns of the matrix data. */ virtual void diff --git a/lib/utils/HDFDatabase.h b/lib/utils/HDFDatabase.h index 49c8f436f..8db8d88d6 100644 --- a/lib/utils/HDFDatabase.h +++ b/lib/utils/HDFDatabase.h @@ -215,8 +215,11 @@ class HDFDatabase : public Database * @param[out] data The allocated array of double values to be read. * @param[in] nelements The number of doubles in the array. * @param[in] offset The initial offset in the array. + * Typically, this is a column index of the matrix data. * @param[in] block_size The block size to read from the HDF5 dataset. + * Typically, this is a number of columns of the matrix data. * @param[in] stride The stride to read from the HDF5 dataset. + * Typically, this is the total number of columns of the matrix data. */ virtual void diff --git a/lib/utils/HDFDatabaseMPIO.cpp b/lib/utils/HDFDatabaseMPIO.cpp index 0be4f85fe..fd31267a6 100644 --- a/lib/utils/HDFDatabaseMPIO.cpp +++ b/lib/utils/HDFDatabaseMPIO.cpp @@ -35,6 +35,7 @@ HDFDatabaseMPIO::create_parallel( else d_rank = 0; + /* Create a single file with a name compatible to HDFDatabase. */ std::string file_name_ext(file_name + ".000000"); if (d_rank == 0) Database::create(file_name_ext); diff --git a/lib/utils/HDFDatabaseMPIO.h b/lib/utils/HDFDatabaseMPIO.h index b45c1f3f4..21daa9f92 100644 --- a/lib/utils/HDFDatabaseMPIO.h +++ b/lib/utils/HDFDatabaseMPIO.h @@ -35,7 +35,8 @@ class HDFDatabaseMPIO : public HDFDatabase ~HDFDatabaseMPIO() {} /** - * @brief Creates a new HDF5 database file with the supplied name. + * @brief Creates a new HDF5 database file for distributed data + * with the supplied name. * * @param[in] file_name Name of HDF5 database file to create. * @@ -47,7 +48,9 @@ class HDFDatabaseMPIO : public HDFDatabase const MPI_Comm comm) override; /** - * @brief Creates a new HDF5 database file with the supplied name. + * @brief Creates a new HDF5 database file for non-distributed data + * with the supplied name. + * For HDFDatabaseMPIO, the behavior is equilvalent to create_parallel. * * @param[in] file_name Name of HDF5 database file to create. * @@ -61,7 +64,8 @@ class HDFDatabaseMPIO : public HDFDatabase } /** - * @brief Opens an existing HDF5 database file with the supplied name. + * @brief Opens an existing HDF5 database file for distributed data + * with the supplied name. * * @param[in] file_name Name of existing HDF5 database file to open. * @param[in] type Read/write type ("r"/"wr") @@ -75,7 +79,9 @@ class HDFDatabaseMPIO : public HDFDatabase const MPI_Comm comm) override; /** - * @brief Opens an existing HDF5 database file with the supplied name. + * @brief Opens an existing HDF5 database file for non-distributed data + * with the supplied name. + * For HDFDatabaseMPIO, the behavior is equilvalent to open_parallel. * * @param[in] file_name Name of existing HDF5 database file to open. * @param[in] type Read/write type ("r"/"wr") @@ -91,8 +97,9 @@ class HDFDatabaseMPIO : public HDFDatabase } /** - * @brief Writes a distributed array of integers associated with the supplied key to - * the currently open HDF5 database file. + * @brief Writes a distributed array of integers + * associated with the supplied key + * to the currently open HDF5 database file. * * @pre !key.empty() * @pre data != nullptr @@ -137,8 +144,9 @@ class HDFDatabaseMPIO : public HDFDatabase } /** - * @brief Writes an array of doubles associated with the supplied key to - * the currently open HDF5 database file. + * @brief Writes a distributed array of doubles + * associated with the supplied key to + * the currently open HDF5 database file. * * @pre !key.empty() * @pre data != nullptr @@ -158,8 +166,9 @@ class HDFDatabaseMPIO : public HDFDatabase int nelem_local); /** - * @brief Writes an array of doubles associated with the supplied key to - * the currently open HDF5 database file. + * @brief Writes an array of doubles in the root rank + * associated with the supplied key to + * the currently open HDF5 database file. * * @pre !key.empty() * @pre data != nullptr @@ -182,8 +191,9 @@ class HDFDatabaseMPIO : public HDFDatabase } /** - * @brief Reads a distributed array of integers associated with the supplied key - * from the currently open HDF5 database file. + * @brief Reads a distributed array of integers + * associated with the supplied key + * from the currently open HDF5 database file. * * @pre !key.empty() * @pre data != nullptr || nelements == 0 @@ -203,6 +213,7 @@ class HDFDatabaseMPIO : public HDFDatabase /** * @brief Reads an array of integers associated with the supplied key * from the currently open HDF5 database file. + * All processes share the same non-distributed integer array. * * @pre !key.empty() * @pre data != nullptr || nelements == 0 @@ -225,7 +236,8 @@ class HDFDatabaseMPIO : public HDFDatabase } /** - * @brief Reads an array of doubles associated with the supplied key + * @brief Reads a distributed array of doubles + * associated with the supplied key * from the currently open HDF5 database file. * * @pre !key.empty() @@ -246,6 +258,7 @@ class HDFDatabaseMPIO : public HDFDatabase /** * @brief Reads an array of doubles associated with the supplied key * from the currently open HDF5 database file. + * All processes share the same non-distributed double array. * * @pre !key.empty() * @pre data != nullptr || nelements == 0 @@ -268,7 +281,8 @@ class HDFDatabaseMPIO : public HDFDatabase } /** - * @brief Reads a sub-array of doubles associated with the supplied key + * @brief Reads a distributed sub-array of doubles + * associated with the supplied key * from the currently open HDF5 database file. * * @pre !key.empty() @@ -291,6 +305,7 @@ class HDFDatabaseMPIO : public HDFDatabase /** * @brief Reads a sub-array of doubles associated with the supplied key * from the currently open HDF5 database file. + * All processes share the same non-distributed double array. * * @pre !key.empty() * @pre data != nullptr || nelements == 0 @@ -315,19 +330,24 @@ class HDFDatabaseMPIO : public HDFDatabase } /** - * @brief Reads an array of doubles associated with the supplied key + * @brief Reads a distributed array of doubles + * associated with the supplied key * from the currently open HDF5 database file. * * @pre !key.empty() * @pre data != nullptr || nelements == 0 + * @pre block_offset_global + block_size_global <= stride_global * * @param[in] key The key associated with the array of values to be * read. * @param[out] data The allocated array of double values to be read. - * @param[in] nelements The number of doubles in the array. - * @param[in] offset The initial offset in the array. - * @param[in] block_size The block size to read from the HDF5 dataset. - * @param[in] stride The stride to read from the HDF5 dataset. + * @param[in] nelem_local The number of doubles in the array at the local process. + * @param[in] block_offset_global The initial offset in the global array. + * Typically, this is a global column index of the matrix data. + * @param[in] block_size_global The total block size to read from the HDF5 dataset. + * Typically, this is a number of columns (in global) of the matrix data. + * @param[in] stride_global The global stride to read from the HDF5 dataset. + * Typically, this is the total number of columns of the matrix data. */ virtual void @@ -342,6 +362,7 @@ class HDFDatabaseMPIO : public HDFDatabase /** * @brief Reads an array of doubles associated with the supplied key * from the currently open HDF5 database file. + * All processes share the same non-distributed double array. * * @pre !key.empty() * @pre data != nullptr || nelements == 0 From 41d57c56ccbc4b528bd14a42f4852fecb47c4859 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Wed, 13 Mar 2024 16:17:50 -0700 Subject: [PATCH 45/50] reflecting comments 1 --- lib/linalg/BasisGenerator.h | 4 ++-- lib/linalg/BasisReader.cpp | 6 +++--- lib/linalg/BasisReader.h | 6 +++--- lib/linalg/BasisWriter.cpp | 8 ++++---- lib/linalg/BasisWriter.h | 2 +- lib/utils/CSVDatabase.h | 14 +++++++------- lib/utils/Database.h | 4 ++-- lib/utils/HDFDatabase.h | 1 - lib/utils/HDFDatabaseMPIO.cpp | 2 +- lib/utils/HDFDatabaseMPIO.h | 2 +- lib/utils/mpi_utils.cpp | 4 +--- unit_tests/test_HDFDatabase.cpp | 16 ++++++++-------- 12 files changed, 33 insertions(+), 36 deletions(-) diff --git a/lib/linalg/BasisGenerator.h b/lib/linalg/BasisGenerator.h index 17a4328b9..3ae7dde35 100644 --- a/lib/linalg/BasisGenerator.h +++ b/lib/linalg/BasisGenerator.h @@ -64,7 +64,7 @@ class BasisGenerator Options options, bool incremental, const std::string& basis_file_name = "", - Database::formats file_format = Database::HDF5); + Database::formats file_format = Database::formats::HDF5); /** * @brief Destructor. @@ -152,7 +152,7 @@ class BasisGenerator loadSamples(const std::string& base_file_name, const std::string& kind = "basis", int cut_off = 1e9, - Database::formats db_format = Database::HDF5); + Database::formats db_format = Database::formats::HDF5); /** * @brief Computes next time an svd sample is needed. diff --git a/lib/linalg/BasisReader.cpp b/lib/linalg/BasisReader.cpp index aac7f89f7..d8086bebc 100644 --- a/lib/linalg/BasisReader.cpp +++ b/lib/linalg/BasisReader.cpp @@ -44,11 +44,11 @@ BasisReader::BasisReader( full_file_name = base_file_name; // Enforce hdf data format. - if (d_format == Database::HDF5) + if (d_format == Database::formats::HDF5) { d_database = new HDFDatabase(); } - else if (d_format == Database::HDF5_MPIO) + else if (d_format == Database::formats::HDF5_MPIO) { /* For MPIO case, local dimension needs to be specified. @@ -285,7 +285,7 @@ BasisReader::getDim( d_database->getInteger(tmp, num_rows); /* only basis and snapshot are stored as distributed matrices */ - if ((kind != "temporal_basis") && (d_format == Database::HDF5_MPIO)) + if ((kind != "temporal_basis") && (d_format == Database::formats::HDF5_MPIO)) { /* for MPIO database, return specified local dimension. diff --git a/lib/linalg/BasisReader.h b/lib/linalg/BasisReader.h index 954b1aaf9..fee556e8f 100644 --- a/lib/linalg/BasisReader.h +++ b/lib/linalg/BasisReader.h @@ -44,12 +44,12 @@ class BasisReader { * @param[in] db_format Format of the file to read. * One of the implemented file formats defined in * Database. - * @param[in] dim Dimension of the basis that will be read from a file. + * @param[in] dim Number of rows of basis that will be read from a file. * If negative, will use the dimension from the rank-specific local file. */ BasisReader( const std::string& base_file_name, - Database::formats db_format = Database::HDF5, + Database::formats db_format = Database::formats::HDF5, const int dim = -1); /** @@ -200,7 +200,7 @@ class BasisReader { * * @brief Returns the dimension of the system on this processor. * - * @param[in] kind The target matrix of which dimension (row) is returned. + * @param[in] kind Type of matrix whose row-dimension is returned. * "basis" - local dimension of spatial basis * "snapshot" - local dimension of snapshot matrix * "temporal_basis" - global dimension of temporal basis diff --git a/lib/linalg/BasisWriter.cpp b/lib/linalg/BasisWriter.cpp index 68ea5903d..e5d2520d8 100644 --- a/lib/linalg/BasisWriter.cpp +++ b/lib/linalg/BasisWriter.cpp @@ -56,12 +56,12 @@ BasisWriter::BasisWriter( snap_file_name = base_file_name + "_snapshot"; // create and open snapshot/basis database - if (db_format_ == Database::HDF5) + if (db_format_ == Database::formats::HDF5) { d_snap_database = new HDFDatabase(); d_database = new HDFDatabase(); } - else if (db_format_ == Database::HDF5_MPIO) + else if (db_format_ == Database::formats::HDF5_MPIO) { d_snap_database = new HDFDatabaseMPIO(); d_database = new HDFDatabaseMPIO(); @@ -91,7 +91,7 @@ BasisWriter::writeBasis(const std::string& kind) CAROM_VERIFY(basis->distributed()); int num_rows = basis->numRows(); int nrows_infile = num_rows; - if (db_format_ == Database::HDF5_MPIO) + if (db_format_ == Database::formats::HDF5_MPIO) MPI_Allreduce(MPI_IN_PLACE, &nrows_infile, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); sprintf(tmp, "spatial_basis_num_rows"); d_database->putInteger(tmp, nrows_infile); @@ -135,7 +135,7 @@ BasisWriter::writeBasis(const std::string& kind) CAROM_VERIFY(snapshots->distributed()); int num_rows = snapshots->numRows(); // d_dim int nrows_infile = num_rows; - if (db_format_ == Database::HDF5_MPIO) + if (db_format_ == Database::formats::HDF5_MPIO) MPI_Allreduce(MPI_IN_PLACE, &nrows_infile, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); sprintf(tmp, "snapshot_matrix_num_rows"); d_snap_database->putInteger(tmp, nrows_infile); diff --git a/lib/linalg/BasisWriter.h b/lib/linalg/BasisWriter.h index d495b6ef0..9684460d3 100644 --- a/lib/linalg/BasisWriter.h +++ b/lib/linalg/BasisWriter.h @@ -42,7 +42,7 @@ class BasisWriter { BasisWriter( BasisGenerator* basis_generator, const std::string& base_file_name, - Database::formats db_format = Database::HDF5); + Database::formats db_format = Database::formats::HDF5); /** * @brief Destructor. diff --git a/lib/utils/CSVDatabase.h b/lib/utils/CSVDatabase.h index b7c655ee3..e5d63ef13 100644 --- a/lib/utils/CSVDatabase.h +++ b/lib/utils/CSVDatabase.h @@ -89,7 +89,7 @@ class CSVDatabase : public Database putIntegerArray( const std::string& file_name, const int* const data, - int nelements); + int nelements) override; /** * @brief Writes an array of doubles associated with the supplied filename. @@ -107,7 +107,7 @@ class CSVDatabase : public Database putDoubleArray( const std::string& file_name, const double* const data, - int nelements); + int nelements) override; /** * @brief Writes a vector of doubles associated with the supplied filename to @@ -126,7 +126,7 @@ class CSVDatabase : public Database putDoubleVector( const std::string& file_name, const std::vector& data, - int nelements); + int nelements) override; /** * @brief Writes a vector of complex doubles associated with the supplied filename. @@ -179,7 +179,7 @@ class CSVDatabase : public Database getIntegerArray( const std::string& file_name, int* data, - int nelements); + int nelements) override; /** * @brief Reads a vector of integers associated with the supplied filename. @@ -207,7 +207,7 @@ class CSVDatabase : public Database * read. */ int - getDoubleArraySize(const std::string& file_name) + getDoubleArraySize(const std::string& file_name) override { std::vector tmp; getDoubleVector(file_name, tmp); @@ -229,7 +229,7 @@ class CSVDatabase : public Database getDoubleArray( const std::string& file_name, double* data, - int nelements); + int nelements) override; /** * @brief Reads a sub-array of doubles associated with the supplied filename. @@ -271,7 +271,7 @@ class CSVDatabase : public Database int nelements, int offset, int block_size, - int stride); + int stride) override; /** * @brief Reads a vector of doubles associated with the supplied filename. diff --git a/lib/utils/Database.h b/lib/utils/Database.h index 4d5daee59..d95ab9a19 100644 --- a/lib/utils/Database.h +++ b/lib/utils/Database.h @@ -260,7 +260,7 @@ class Database * @param[out] data The allocated array of double values to be read. * @param[in] nelements The number of doubles in the array. * @param[in] offset The initial offset in the array. - * Typically, this is a column index of the matrix data. + * Typically, this is a zero-based column index of the matrix data. * @param[in] block_size The block size to read from the HDF5 dataset. * Typically, this is a number of columns of the matrix data. * @param[in] stride The stride to read from the HDF5 dataset. @@ -280,7 +280,7 @@ class Database * @brief Implemented database file formats. Add to this enum each time a * new database format is implemented. */ - enum formats { + enum class formats { HDF5, CSV, HDF5_MPIO diff --git a/lib/utils/HDFDatabase.h b/lib/utils/HDFDatabase.h index 8db8d88d6..c140d122e 100644 --- a/lib/utils/HDFDatabase.h +++ b/lib/utils/HDFDatabase.h @@ -431,7 +431,6 @@ class HDFDatabase : public Database /** * @brief MPI process rank. - * Used only when d_parallel is true. */ int d_rank; diff --git a/lib/utils/HDFDatabaseMPIO.cpp b/lib/utils/HDFDatabaseMPIO.cpp index fd31267a6..9f5411a1b 100644 --- a/lib/utils/HDFDatabaseMPIO.cpp +++ b/lib/utils/HDFDatabaseMPIO.cpp @@ -1,6 +1,6 @@ /****************************************************************************** * - * Copyright (c) 2013-2023, Lawrence Livermore National Security, LLC + * Copyright (c) 2013-2024, Lawrence Livermore National Security, LLC * and other libROM project developers. See the top-level COPYRIGHT * file for details. * diff --git a/lib/utils/HDFDatabaseMPIO.h b/lib/utils/HDFDatabaseMPIO.h index 21daa9f92..88ce4d2ed 100644 --- a/lib/utils/HDFDatabaseMPIO.h +++ b/lib/utils/HDFDatabaseMPIO.h @@ -1,6 +1,6 @@ /****************************************************************************** * - * Copyright (c) 2013-2023, Lawrence Livermore National Security, LLC + * Copyright (c) 2013-2024, Lawrence Livermore National Security, LLC * and other libROM project developers. See the top-level COPYRIGHT * file for details. * diff --git a/lib/utils/mpi_utils.cpp b/lib/utils/mpi_utils.cpp index 99caeb32d..7670bd5ac 100644 --- a/lib/utils/mpi_utils.cpp +++ b/lib/utils/mpi_utils.cpp @@ -70,9 +70,7 @@ get_global_offsets(const int local_dim, std::vector &offsets, bool is_same(int x, const MPI_Comm &comm) { - int p[2]; - p[0] = -x; - p[1] = x; + int p[2] = {-x, x}; MPI_Allreduce(MPI_IN_PLACE, p, 2, MPI_INT, MPI_MIN, comm); return (p[0] == -p[1]); } diff --git a/unit_tests/test_HDFDatabase.cpp b/unit_tests/test_HDFDatabase.cpp index 14548f508..1fc774051 100644 --- a/unit_tests/test_HDFDatabase.cpp +++ b/unit_tests/test_HDFDatabase.cpp @@ -1,6 +1,6 @@ /****************************************************************************** * - * Copyright (c) 2013-2023, Lawrence Livermore National Security, LLC + * Copyright (c) 2013-2024, Lawrence Livermore National Security, LLC * and other libROM project developers. See the top-level COPYRIGHT * file for details. * @@ -135,7 +135,6 @@ TEST(HDF5, Test_parallel_writing) hid_t attr = H5Acreate(dset_id, "test_attr", H5T_NATIVE_INT, attr_id, H5P_DEFAULT, H5P_DEFAULT); - // if (rank == 1) errf = H5Awrite(attr, H5T_NATIVE_INT, &test_attr); errf = H5Aclose(attr); errf = H5Sclose(attr_id); @@ -633,10 +632,10 @@ TEST(BasisGeneratorIO, Base_MPIO_combination) svd_options.setRandomizedSVD(false); svd_options.setDebugMode(true); CAROM::BasisGenerator sampler(svd_options, false, mpio_name, - CAROM::Database::HDF5_MPIO); + CAROM::Database::formats::HDF5_MPIO); sampler.loadSamples(base_name + "_snapshot", "snapshot", 1e9, - CAROM::Database::HDF5); + CAROM::Database::formats::HDF5); sampler.writeSnapshot(); const CAROM::Matrix *snapshot = sampler.getSnapshotMatrix(); @@ -687,10 +686,10 @@ TEST(BasisGeneratorIO, MPIO_Base_combination) svd_options.setRandomizedSVD(false); svd_options.setDebugMode(true); CAROM::BasisGenerator sampler(svd_options, false, test_name, - CAROM::Database::HDF5); + CAROM::Database::formats::HDF5); sampler.loadSamples(mpio_name + "_snapshot", "snapshot", 1e9, - CAROM::Database::HDF5_MPIO); + CAROM::Database::formats::HDF5_MPIO); const CAROM::Matrix *snapshot = sampler.getSnapshotMatrix(); CAROM::BasisReader snapshot_reader("test_basis_snapshot"); @@ -739,7 +738,8 @@ TEST(BasisReaderIO, partial_getSpatialBasis) CAROM::BasisReader basis_reader("test_basis"); const CAROM::Matrix *spatial_basis = basis_reader.getSpatialBasis(); - CAROM::BasisReader basis_reader1("test_mpio", CAROM::Database::HDF5_MPIO, + CAROM::BasisReader basis_reader1("test_mpio", + CAROM::Database::formats::HDF5_MPIO, nrow_local); const CAROM::Matrix *spatial_basis1 = basis_reader1.getSpatialBasis(); @@ -797,7 +797,7 @@ TEST(BasisGeneratorIO, Scaling_test) CAROM::BasisGenerator base_sampler(svd_options, false, "base"); CAROM::BasisGenerator mpio_sampler(svd_options, false, "mpio", - CAROM::Database::HDF5_MPIO); + CAROM::Database::formats::HDF5_MPIO); CAROM::Vector sample(nrow_local, true); for (int s = 0; s < scale_ncol; s++) From d91bb906e78c3eac9abfb4f1f5a4f54dd9bbbca0 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Wed, 13 Mar 2024 20:31:39 -0700 Subject: [PATCH 46/50] reflecting comments 2. changed interface. --- lib/linalg/BasisReader.cpp | 42 ++-- lib/linalg/BasisWriter.cpp | 10 +- lib/utils/CSVDatabase.cpp | 31 ++- lib/utils/CSVDatabase.h | 45 +++- lib/utils/Database.cpp | 7 +- lib/utils/Database.h | 293 +++++++++++++------------ lib/utils/HDFDatabase.cpp | 57 +++-- lib/utils/HDFDatabase.h | 297 ++++++++++++++------------ lib/utils/HDFDatabaseMPIO.cpp | 9 +- lib/utils/HDFDatabaseMPIO.h | 388 +++++++++++++++++++--------------- 10 files changed, 684 insertions(+), 495 deletions(-) diff --git a/lib/linalg/BasisReader.cpp b/lib/linalg/BasisReader.cpp index d8086bebc..1fdb11e65 100644 --- a/lib/linalg/BasisReader.cpp +++ b/lib/linalg/BasisReader.cpp @@ -63,7 +63,7 @@ BasisReader::BasisReader( else CAROM_ERROR("BasisWriter only supports HDF5/HDF5_MPIO data format!\n"); - d_database->open_parallel(full_file_name, "r", MPI_COMM_WORLD); + d_database->open(full_file_name, "r", MPI_COMM_WORLD); } BasisReader::~BasisReader() @@ -80,9 +80,10 @@ BasisReader::getSpatialBasis() Matrix* spatial_basis_vectors = new Matrix(num_rows, num_cols, true); - d_database->getDoubleArray_parallel("spatial_basis", - &spatial_basis_vectors->item(0, 0), - num_rows*num_cols); + d_database->getDoubleArray("spatial_basis", + &spatial_basis_vectors->item(0, 0), + num_rows*num_cols, + true); return spatial_basis_vectors; } @@ -108,12 +109,13 @@ BasisReader::getSpatialBasis( Matrix* spatial_basis_vectors = new Matrix(num_rows, num_cols_to_read, true); sprintf(tmp, "spatial_basis"); - d_database->getDoubleArray_parallel(tmp, - &spatial_basis_vectors->item(0, 0), - num_rows*num_cols_to_read, - start_col - 1, - num_cols_to_read, - num_cols); + d_database->getDoubleArray(tmp, + &spatial_basis_vectors->item(0, 0), + num_rows*num_cols_to_read, + start_col - 1, + num_cols_to_read, + num_cols, + true); return spatial_basis_vectors; } @@ -327,9 +329,10 @@ BasisReader::getSnapshotMatrix() char tmp[100]; Matrix* snapshots = new Matrix(num_rows, num_cols, true); sprintf(tmp, "snapshot_matrix"); - d_database->getDoubleArray_parallel(tmp, - &snapshots->item(0, 0), - num_rows*num_cols); + d_database->getDoubleArray(tmp, + &snapshots->item(0, 0), + num_rows*num_cols, + true); return snapshots; } @@ -355,12 +358,13 @@ BasisReader::getSnapshotMatrix( char tmp[100]; Matrix* snapshots = new Matrix(num_rows, num_cols_to_read, true); sprintf(tmp, "snapshot_matrix"); - d_database->getDoubleArray_parallel(tmp, - &snapshots->item(0, 0), - num_rows*num_cols_to_read, - start_col - 1, - num_cols_to_read, - num_cols); + d_database->getDoubleArray(tmp, + &snapshots->item(0, 0), + num_rows*num_cols_to_read, + start_col - 1, + num_cols_to_read, + num_cols, + true); return snapshots; } } diff --git a/lib/linalg/BasisWriter.cpp b/lib/linalg/BasisWriter.cpp index e5d2520d8..466ff6dea 100644 --- a/lib/linalg/BasisWriter.cpp +++ b/lib/linalg/BasisWriter.cpp @@ -84,7 +84,7 @@ BasisWriter::writeBasis(const std::string& kind) char tmp[100]; if (kind == "basis") { - d_database->create_parallel(full_file_name, MPI_COMM_WORLD); + d_database->create(full_file_name, MPI_COMM_WORLD); const Matrix* basis = d_basis_generator->getSpatialBasis(); /* singular is always distributed */ @@ -99,7 +99,7 @@ BasisWriter::writeBasis(const std::string& kind) sprintf(tmp, "spatial_basis_num_cols"); d_database->putInteger(tmp, num_cols); sprintf(tmp, "spatial_basis"); - d_database->putDoubleArray_parallel(tmp, &basis->item(0, 0), num_rows*num_cols); + d_database->putDoubleArray(tmp, &basis->item(0, 0), num_rows*num_cols, true); if(d_basis_generator->updateRightSV()) { const Matrix* tbasis = d_basis_generator->getTemporalBasis(); @@ -128,7 +128,7 @@ BasisWriter::writeBasis(const std::string& kind) } if (kind == "snapshot") { - d_snap_database->create_parallel(snap_file_name, MPI_COMM_WORLD); + d_snap_database->create(snap_file_name, MPI_COMM_WORLD); const Matrix* snapshots = d_basis_generator->getSnapshotMatrix(); /* snapshot matrix is always distributed */ @@ -143,8 +143,8 @@ BasisWriter::writeBasis(const std::string& kind) sprintf(tmp, "snapshot_matrix_num_cols"); d_snap_database->putInteger(tmp, num_cols); sprintf(tmp, "snapshot_matrix"); - d_snap_database->putDoubleArray_parallel(tmp, &snapshots->item(0,0), - num_rows*num_cols); + d_snap_database->putDoubleArray(tmp, &snapshots->item(0,0), num_rows*num_cols, + true); d_snap_database->close(); } diff --git a/lib/utils/CSVDatabase.cpp b/lib/utils/CSVDatabase.cpp index 99547d6a7..263447594 100644 --- a/lib/utils/CSVDatabase.cpp +++ b/lib/utils/CSVDatabase.cpp @@ -28,18 +28,20 @@ CSVDatabase::~CSVDatabase() bool CSVDatabase::create( - const std::string& file_name) + const std::string& file_name, + const MPI_Comm comm) { - Database::create(file_name); + Database::create(file_name, comm); return true; } bool CSVDatabase::open( const std::string& file_name, - const std::string& type) + const std::string& type, + const MPI_Comm comm) { - Database::open(file_name, type); + Database::open(file_name, type, comm); return true; } @@ -53,7 +55,8 @@ void CSVDatabase::putIntegerArray( const std::string& file_name, const int* const data, - int nelements) + int nelements, + const bool distributed) { CAROM_VERIFY(!file_name.empty()); CAROM_VERIFY(data != nullptr); @@ -71,7 +74,8 @@ void CSVDatabase::putDoubleArray( const std::string& file_name, const double* const data, - int nelements) + int nelements, + const bool distributed) { CAROM_VERIFY(!file_name.empty()); CAROM_VERIFY(data != nullptr); @@ -90,7 +94,8 @@ void CSVDatabase::putDoubleVector( const std::string& file_name, const std::vector& data, - int nelements) + int nelements, + const bool distributed) { CAROM_VERIFY(!file_name.empty()); CAROM_VERIFY(nelements > 0); @@ -146,7 +151,8 @@ void CSVDatabase::getIntegerArray( const std::string& file_name, int* data, - int nelements) + int nelements, + const bool distributed) { CAROM_VERIFY(!file_name.empty()); #ifndef DEBUG_CHECK_ASSERTIONS @@ -188,7 +194,8 @@ void CSVDatabase::getDoubleArray( const std::string& file_name, double* data, - int nelements) + int nelements, + const bool distributed) { CAROM_VERIFY(!file_name.empty()); #ifndef DEBUG_CHECK_ASSERTIONS @@ -211,7 +218,8 @@ CSVDatabase::getDoubleArray( const std::string& file_name, double* data, int nelements, - const std::vector& idx) + const std::vector& idx, + const bool distributed) { CAROM_VERIFY(!file_name.empty()); #ifndef DEBUG_CHECK_ASSERTIONS @@ -252,7 +260,8 @@ CSVDatabase::getDoubleArray( int nelements, int offset, int block_size, - int stride) + int stride, + const bool distributed) { CAROM_VERIFY(!file_name.empty()); #ifndef DEBUG_CHECK_ASSERTIONS diff --git a/lib/utils/CSVDatabase.h b/lib/utils/CSVDatabase.h index e5d63ef13..c9132a15f 100644 --- a/lib/utils/CSVDatabase.h +++ b/lib/utils/CSVDatabase.h @@ -43,12 +43,15 @@ class CSVDatabase : public Database * This function will only print out the file_name. * * @param[in] file_name Name of CSV database file to create. + * @param[in] comm MPI communicator for distributed data I/O. + * CSVDatabase I/O is always performed serially, regardless. * * @return True if file create was successful. */ bool create( - const std::string& file_name) override; + const std::string& file_name, + const MPI_Comm comm=MPI_COMM_NULL) override; /** * @brief Opens an existing CSV database file with the supplied name. @@ -57,13 +60,16 @@ class CSVDatabase : public Database * * @param[in] file_name Name of existing CSV database file to open. * @param[in] type Read/write type ("r"/"wr") + * @param[in] comm MPI communicator for distributed data I/O. + * CSVDatabase I/O is always performed serially, regardless. * * @return True if file open was successful. */ bool open( const std::string& file_name, - const std::string& type) override; + const std::string& type, + const MPI_Comm comm=MPI_COMM_NULL) override; /** * @brief Closes the currently open CSV database file. @@ -84,12 +90,15 @@ class CSVDatabase : public Database * written. * @param[in] data The array of integer values to be written. * @param[in] nelements The number of integers in the array. + * @param[in] distributed True if data is a distributed integer array. + * CSVDatabase writes the array serially whether or not distributed. */ void putIntegerArray( const std::string& file_name, const int* const data, - int nelements) override; + int nelements, + const bool distributed=false) override; /** * @brief Writes an array of doubles associated with the supplied filename. @@ -102,12 +111,15 @@ class CSVDatabase : public Database * written. * @param[in] data The array of double values to be written. * @param[in] nelements The number of doubles in the array. + * @param[in] distributed True if data is a distributed double array. + * CSVDatabase writes the array serially whether or not distributed. */ void putDoubleArray( const std::string& file_name, const double* const data, - int nelements) override; + int nelements, + const bool distributed=false) override; /** * @brief Writes a vector of doubles associated with the supplied filename to @@ -121,12 +133,15 @@ class CSVDatabase : public Database * written. * @param[in] data The vector of double values to be written. * @param[in] nelements The number of doubles in the vector. + * @param[in] distributed True if data is a distributed double vector. + * CSVDatabase writes the vector serially whether or not distributed. */ void putDoubleVector( const std::string& file_name, const std::vector& data, - int nelements) override; + int nelements, + const bool distributed=false) override; /** * @brief Writes a vector of complex doubles associated with the supplied filename. @@ -174,12 +189,15 @@ class CSVDatabase : public Database * read. * @param[out] data The allocated array of integer values to be read. * @param[in] nelements The number of integers in the array. + * @param[in] distributed True if data is a distributed integer array. + * CSVDatabase reads the array serially whether or not distributed. */ void getIntegerArray( const std::string& file_name, int* data, - int nelements) override; + int nelements, + const bool distributed=false) override; /** * @brief Reads a vector of integers associated with the supplied filename. @@ -224,12 +242,15 @@ class CSVDatabase : public Database * read. * @param[out] data The allocated array of double values to be read. * @param[in] nelements The number of doubles in the array. + * @param[in] distributed True if data is a distributed double array. + * CSVDatabase reads the array serially whether or not distributed. */ void getDoubleArray( const std::string& file_name, double* data, - int nelements) override; + int nelements, + const bool distributed=false) override; /** * @brief Reads a sub-array of doubles associated with the supplied filename. @@ -242,13 +263,16 @@ class CSVDatabase : public Database * @param[out] data The allocated sub-array of double values to be read. * @param[in] nelements The number of doubles in the full array. * @param[in] idx The set of indices in the sub-array. + * @param[in] distributed True if data is a distributed integer array. + * CSVDatabase reads the array serially whether or not distributed. */ void getDoubleArray( const std::string& file_name, double* data, int nelements, - const std::vector& idx); + const std::vector& idx, + const bool distributed=false) override; /** * @brief Reads an array of doubles associated with the supplied filename. @@ -263,6 +287,8 @@ class CSVDatabase : public Database * @param[in] offset The initial offset in the array. * @param[in] block_size The block size to read from the CSV dataset. * @param[in] stride The stride to read from the CSV dataset. + * @param[in] distributed True if data is a distributed integer array. + * CSVDatabase reads the array serially whether or not distributed. */ void getDoubleArray( @@ -271,7 +297,8 @@ class CSVDatabase : public Database int nelements, int offset, int block_size, - int stride) override; + int stride, + const bool distributed=false) override; /** * @brief Reads a vector of doubles associated with the supplied filename. diff --git a/lib/utils/Database.cpp b/lib/utils/Database.cpp index 0ef4599da..df2cb01b6 100644 --- a/lib/utils/Database.cpp +++ b/lib/utils/Database.cpp @@ -32,7 +32,9 @@ Database::~Database() } bool -Database::create(const std::string& file_name) +Database::create( + const std::string& file_name, + const MPI_Comm comm) { std::cout << "Creating file: " << file_name << std::endl; return true; @@ -41,7 +43,8 @@ Database::create(const std::string& file_name) bool Database::open( const std::string& file_name, - const std::string& type) + const std::string& type, + const MPI_Comm comm) { std::cout << "Opening file: " << file_name << std::endl; return true; diff --git a/lib/utils/Database.h b/lib/utils/Database.h index d95ab9a19..187b48fb5 100644 --- a/lib/utils/Database.h +++ b/lib/utils/Database.h @@ -45,19 +45,26 @@ class Database * @brief Creates a new database file with the supplied name. * * @param[in] file_name Name of database file to create. + * @param[in] comm MPI communicator for distributed data I/O. + * If MPI_COMM_NULL, the file will be created serially. + * The behavior for distributed I/O will vary by classes. * * @return True if file create was successful. */ virtual bool create( - const std::string& file_name); + const std::string& file_name, + const MPI_Comm comm=MPI_COMM_NULL); /** * @brief Opens an existing database file with the supplied name. * * @param[in] file_name Name of existing database file to open. * @param[in] type Read/write type ("r"/"wr") + * @param[in] comm MPI communicator for distributed data I/O. + * If MPI_COMM_NULL, the file will be opened serially. + * The behavior for distributed I/O will vary by classes. * * @return True if file open was successful. */ @@ -65,7 +72,8 @@ class Database bool open( const std::string& file_name, - const std::string& type); + const std::string& type, + const MPI_Comm comm=MPI_COMM_NULL); /** * @brief Closes the currently open database file. @@ -99,13 +107,16 @@ class Database * written. * @param[in] data The array of integer values to be written. * @param[in] nelements The number of integers in the array. + * @param[in] distributed If true, distributed integer array will be written. + * the distributed I/O behavior varies with classes. */ virtual void putIntegerArray( const std::string& key, const int* const data, - int nelements) = 0; + int nelements, + const bool distributed=false) = 0; /** * @brief Writes a double associated with the supplied key to currently @@ -130,13 +141,16 @@ class Database * written. * @param[in] data The array of double values to be written. * @param[in] nelements The number of doubles in the array. + * @param[in] distributed If true, distributed double array will be written. + * the distributed I/O behavior varies with classes. */ virtual void putDoubleArray( const std::string& key, const double* const data, - int nelements) = 0; + int nelements, + const bool distributed=false) = 0; /** * @brief Writes a vector of doubles associated with the supplied key to @@ -146,13 +160,16 @@ class Database * written. * @param[in] data The vector of double values to be written. * @param[in] nelements The number of doubles in the vector. + * @param[in] distributed If true, distributed double vector will be written. + * the distributed I/O behavior varies with classes. */ virtual void putDoubleVector( const std::string& key, const std::vector& data, - int nelements) = 0; + int nelements, + const bool distributed=false) = 0; /** * @brief Reads an integer associated with the supplied key from the @@ -177,13 +194,16 @@ class Database * read. * @param[out] data The allocated array of integer values to be read. * @param[in] nelements The number of integers in the array. + * @param[in] distributed If true, distributed integer array will be read. + * the distributed I/O behavior varies with classes. */ virtual void getIntegerArray( const std::string& key, int* data, - int nelements) = 0; + int nelements, + const bool distributed=false) = 0; /** * @brief Reads a double associated with the supplied key from the @@ -222,13 +242,16 @@ class Database * read. * @param[out] data The allocated array of double values to be read. * @param[in] nelements The number of doubles in the array. + * @param[in] distributed If true, distributed double array will be read. + * the distributed I/O behavior varies with classes. */ virtual void getDoubleArray( const std::string& key, double* data, - int nelements) = 0; + int nelements, + const bool distributed=false) = 0; /** * @brief Reads a sub-array of doubles associated with the supplied key @@ -242,6 +265,8 @@ class Database * @param[out] data The allocated sub-array of double values to be read. * @param[in] nelements The number of doubles in the full array. * @param[in] idx The set of indices in the sub-array. + * @param[in] distributed If true, distributed double array will be read. + * the distributed I/O behavior varies with classes. */ virtual void @@ -249,7 +274,8 @@ class Database const std::string& key, double* data, int nelements, - const std::vector& idx) = 0; + const std::vector& idx, + const bool distributed=false) = 0; /** * @brief Reads an array of doubles associated with the supplied key @@ -265,6 +291,8 @@ class Database * Typically, this is a number of columns of the matrix data. * @param[in] stride The stride to read from the HDF5 dataset. * Typically, this is the total number of columns of the matrix data. + * @param[in] distributed If true, distributed double array will be read. + * the distributed I/O behavior varies with classes. */ virtual void @@ -274,7 +302,8 @@ class Database int nelements, int offset, int block_size, - int stride) = 0; + int stride, + const bool distributed=false) = 0; /** * @brief Implemented database file formats. Add to this enum each time a @@ -286,129 +315,129 @@ class Database HDF5_MPIO }; - /** - * @brief Creates a new database file with the supplied name, with parallel I/O support. - * Supported only for HDF5 format. - * For HDFDatabase, the function is equivalent to create, - * only extending the file name with 6 digits indicating processor rank. - * For HDFDatabaseMPIO, the file is created with MPI I/O file access property. - * - * @param[in] file_name Base Name of CSV database file to create. - * @param[in] comm MPI communicator to get the rank. - * - * @return True if file create was successful. - */ - virtual - bool - create_parallel( - const std::string& file_name, - const MPI_Comm comm) - { - CAROM_ERROR("Abstract method Database::create_parallel!\n"); - return false; - } - - /** - * @brief Opens an existing database file with the supplied name, with parallel I/O support. - * Supported only for HDF5 format. - * For HDFDatabase, the function is equivalent to open, - * only extending the file name with 6 digits indicating processor rank. - * For HDFDatabaseMPIO, the file is opened with MPI I/O file access property. - * - * @param[in] file_name Name of existing CSV database file to open. - * @param[in] type Read/write type ("r"/"wr") - * @param[in] comm MPI communicator to get the rank. - * - * @return True if file open was successful. - */ - virtual - bool - open_parallel( - const std::string& file_name, - const std::string& type, - const MPI_Comm comm) - { - CAROM_ERROR("Abstract method Database::open_parallel!\n"); - return false; - } - - /** - * @brief Writes a distributed array of doubles associated with the supplied key to - * the currently open database file. - * Supported only for HDF5 format. - * For HDFDatabase, the function is equivalent to putDoubleArray, - * writing the local array per each process. - * For HDFDatabaseMPIO, the global array is written into a single file via MPI I/O. - * - * @param[in] key The key associated with the array of values to be - * written. - * @param[in] data The array of double values to be written. - * @param[in] nelements The number of doubles in the array. - */ - virtual - void - putDoubleArray_parallel( - const std::string& key, - const double* const data, - int nelements) - { - CAROM_ERROR("Abstract method Database::putDoubleArray_parallel!\n"); - } - - /** - * @brief Reads a distributed array of doubles associated with the supplied key - * from the currently open database file. - * Supported only for HDF5 format. - * For HDFDatabase, the function is equivalent to getDoubleArray, - * reading the local array from a local file per each process. - * For HDFDatabaseMPIO, the local array is read from a single file via MPI I/O. - * - * @param[in] key The key associated with the array of values to be - * read. - * @param[out] data The allocated array of double values to be read. - * @param[in] nelements The number of doubles in the array. - */ - virtual - void - getDoubleArray_parallel( - const std::string& key, - double* data, - int nelements) - { - CAROM_ERROR("Abstract method Database::getDoubleArray_parallel!\n"); - } - - /** - * @brief Reads a distributed array of doubles associated with the supplied key - * from the currently open database file. - * Supported only for HDF5 format. - * For HDFDatabase, the function is equivalent to getDoubleArray, - * reading the local array from a local file per each process. - * For HDFDatabaseMPIO, the local array is read from a single file via MPI I/O. - * - * @param[in] key The key associated with the array of values to be - * read. - * @param[out] data The allocated array of double values to be read. - * @param[in] nelements The number of doubles in the array. - * @param[in] offset The initial offset in the array. - * Typically, this is a column index of the matrix data. - * @param[in] block_size The block size to read from the HDF5 dataset. - * Typically, this is a number of columns of the matrix data. - * @param[in] stride The stride to read from the HDF5 dataset. - * Typically, this is the total number of columns of the matrix data. - */ - virtual - void - getDoubleArray_parallel( - const std::string& key, - double* data, - int nelements, - int offset, - int block_size, - int stride) - { - CAROM_ERROR("Abstract method Database::getDoubleArray_parallel!\n"); - } + // /** + // * @brief Creates a new database file with the supplied name, with parallel I/O support. + // * Supported only for HDF5 format. + // * For HDFDatabase, the function is equivalent to create, + // * only extending the file name with 6 digits indicating processor rank. + // * For HDFDatabaseMPIO, the file is created with MPI I/O file access property. + // * + // * @param[in] file_name Base Name of CSV database file to create. + // * @param[in] comm MPI communicator to get the rank. + // * + // * @return True if file create was successful. + // */ + // virtual + // bool + // create_parallel( + // const std::string& file_name, + // const MPI_Comm comm) + // { + // CAROM_ERROR("Abstract method Database::create_parallel!\n"); + // return false; + // } + + // /** + // * @brief Opens an existing database file with the supplied name, with parallel I/O support. + // * Supported only for HDF5 format. + // * For HDFDatabase, the function is equivalent to open, + // * only extending the file name with 6 digits indicating processor rank. + // * For HDFDatabaseMPIO, the file is opened with MPI I/O file access property. + // * + // * @param[in] file_name Name of existing CSV database file to open. + // * @param[in] type Read/write type ("r"/"wr") + // * @param[in] comm MPI communicator to get the rank. + // * + // * @return True if file open was successful. + // */ + // virtual + // bool + // open_parallel( + // const std::string& file_name, + // const std::string& type, + // const MPI_Comm comm) + // { + // CAROM_ERROR("Abstract method Database::open_parallel!\n"); + // return false; + // } + + // /** + // * @brief Writes a distributed array of doubles associated with the supplied key to + // * the currently open database file. + // * Supported only for HDF5 format. + // * For HDFDatabase, the function is equivalent to putDoubleArray, + // * writing the local array per each process. + // * For HDFDatabaseMPIO, the global array is written into a single file via MPI I/O. + // * + // * @param[in] key The key associated with the array of values to be + // * written. + // * @param[in] data The array of double values to be written. + // * @param[in] nelements The number of doubles in the array. + // */ + // virtual + // void + // putDoubleArray_parallel( + // const std::string& key, + // const double* const data, + // int nelements) + // { + // CAROM_ERROR("Abstract method Database::putDoubleArray_parallel!\n"); + // } + + // /** + // * @brief Reads a distributed array of doubles associated with the supplied key + // * from the currently open database file. + // * Supported only for HDF5 format. + // * For HDFDatabase, the function is equivalent to getDoubleArray, + // * reading the local array from a local file per each process. + // * For HDFDatabaseMPIO, the local array is read from a single file via MPI I/O. + // * + // * @param[in] key The key associated with the array of values to be + // * read. + // * @param[out] data The allocated array of double values to be read. + // * @param[in] nelements The number of doubles in the array. + // */ + // virtual + // void + // getDoubleArray_parallel( + // const std::string& key, + // double* data, + // int nelements) + // { + // CAROM_ERROR("Abstract method Database::getDoubleArray_parallel!\n"); + // } + + // /** + // * @brief Reads a distributed array of doubles associated with the supplied key + // * from the currently open database file. + // * Supported only for HDF5 format. + // * For HDFDatabase, the function is equivalent to getDoubleArray, + // * reading the local array from a local file per each process. + // * For HDFDatabaseMPIO, the local array is read from a single file via MPI I/O. + // * + // * @param[in] key The key associated with the array of values to be + // * read. + // * @param[out] data The allocated array of double values to be read. + // * @param[in] nelements The number of doubles in the array. + // * @param[in] offset The initial offset in the array. + // * Typically, this is a column index of the matrix data. + // * @param[in] block_size The block size to read from the HDF5 dataset. + // * Typically, this is a number of columns of the matrix data. + // * @param[in] stride The stride to read from the HDF5 dataset. + // * Typically, this is the total number of columns of the matrix data. + // */ + // virtual + // void + // getDoubleArray_parallel( + // const std::string& key, + // double* data, + // int nelements, + // int offset, + // int block_size, + // int stride) + // { + // CAROM_ERROR("Abstract method Database::getDoubleArray_parallel!\n"); + // } private: /** diff --git a/lib/utils/HDFDatabase.cpp b/lib/utils/HDFDatabase.cpp index 60bd0ad69..971914bd7 100644 --- a/lib/utils/HDFDatabase.cpp +++ b/lib/utils/HDFDatabase.cpp @@ -43,12 +43,23 @@ HDFDatabase::~HDFDatabase() bool HDFDatabase::create( - const std::string& file_name) + const std::string& file_name, + const MPI_Comm comm) { - Database::create(file_name); + Database::create(file_name, comm); CAROM_VERIFY(!file_name.empty()); - hid_t file_id = H5Fcreate(file_name.c_str(), + std::string ext_filename(file_name); + + if (comm != MPI_COMM_NULL) + { + MPI_Comm_rank(comm, &d_rank); + char tmp[10]; + sprintf(tmp, ".%06d", d_rank); + ext_filename += tmp; + } + + hid_t file_id = H5Fcreate(ext_filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); @@ -64,22 +75,33 @@ HDFDatabase::create( bool HDFDatabase::open( const std::string& file_name, - const std::string& type) + const std::string& type, + const MPI_Comm comm) { - Database::open(file_name, type); + Database::open(file_name, type, comm); CAROM_VERIFY(!file_name.empty()); + std::string ext_filename(file_name); + + if (comm != MPI_COMM_NULL) + { + MPI_Comm_rank(comm, &d_rank); + char tmp[10]; + sprintf(tmp, ".%06d", d_rank); + ext_filename += tmp; + } + CAROM_VERIFY(type == "r" || type == "wr"); hid_t file_id; if (type == "r") { - file_id = H5Fopen(file_name.c_str(), + file_id = H5Fopen(ext_filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT); } else if (type == "wr") { - file_id = H5Fopen(file_name.c_str(), + file_id = H5Fopen(ext_filename.c_str(), H5F_ACC_RDWR, H5P_DEFAULT); } @@ -114,7 +136,8 @@ void HDFDatabase::putIntegerArray( const std::string& key, const int* const data, - int nelements) + int nelements, + const bool distributed) { CAROM_VERIFY(!key.empty()); CAROM_VERIFY(data != nullptr); @@ -166,7 +189,8 @@ void HDFDatabase::putDoubleArray( const std::string& key, const double* const data, - int nelements) + int nelements, + const bool distributed) { CAROM_VERIFY(!key.empty()); CAROM_VERIFY(data != nullptr); @@ -218,7 +242,8 @@ void HDFDatabase::putDoubleVector( const std::string& key, const std::vector& data, - int nelements) + int nelements, + const bool distributed) { putDoubleArray(key, data.data(), nelements); } @@ -227,7 +252,8 @@ void HDFDatabase::getIntegerArray( const std::string& key, int* data, - int nelements) + int nelements, + const bool distributed) { if (nelements == 0) return; @@ -297,7 +323,8 @@ void HDFDatabase::getDoubleArray( const std::string& key, double* data, - int nelements) + int nelements, + const bool distributed) { if (nelements == 0) return; @@ -340,7 +367,8 @@ HDFDatabase::getDoubleArray( const std::string& key, double* data, int nelements, - const std::vector& idx) + const std::vector& idx, + const bool distributed) { if (idx.size() == 0) { @@ -373,7 +401,8 @@ HDFDatabase::getDoubleArray( int nelements, int offset, int block_size, - int stride) + int stride, + const bool distributed) { CAROM_VERIFY(!key.empty()); #ifndef DEBUG_CHECK_ASSERTIONS diff --git a/lib/utils/HDFDatabase.h b/lib/utils/HDFDatabase.h index c140d122e..1517c1ac2 100644 --- a/lib/utils/HDFDatabase.h +++ b/lib/utils/HDFDatabase.h @@ -41,19 +41,25 @@ class HDFDatabase : public Database * @brief Creates a new HDF5 database file with the supplied name. * * @param[in] file_name Name of HDF5 database file to create. + * @param[in] comm MPI communicator for distributed data I/O. + * If not MPI_COMM_NULL, each process creates a file + * whose name is file_name extended with process rank in 6 digits. * * @return True if file create was successful. */ virtual bool create( - const std::string& file_name) override; + const std::string& file_name, + const MPI_Comm comm=MPI_COMM_NULL) override; /** * @brief Opens an existing HDF5 database file with the supplied name. * * @param[in] file_name Name of existing HDF5 database file to open. * @param[in] type Read/write type ("r"/"wr") + * If not MPI_COMM_NULL, each process opens a file + * whose name is file_name extended with process rank in 6 digits. * * @return True if file open was successful. */ @@ -61,7 +67,8 @@ class HDFDatabase : public Database bool open( const std::string& file_name, - const std::string& type) override; + const std::string& type, + const MPI_Comm comm=MPI_COMM_NULL) override; /** * @brief Closes the currently open HDF5 database file. @@ -84,13 +91,17 @@ class HDFDatabase : public Database * written. * @param[in] data The array of integer values to be written. * @param[in] nelements The number of integers in the array. + * @param[in] distributed True if data is a distributed integer array. + * HDFDatabase writes the array in file-per-process, + * where each file is written serially by one process. */ virtual void putIntegerArray( const std::string& key, const int* const data, - int nelements); + int nelements, + const bool distributed=false); /** * @brief Writes an array of doubles associated with the supplied key to @@ -104,13 +115,17 @@ class HDFDatabase : public Database * written. * @param[in] data The array of double values to be written. * @param[in] nelements The number of doubles in the array. + * @param[in] distributed True if data is a distributed double array. + * HDFDatabase writes the array in file-per-process, + * where each file is written serially by one process. */ virtual void putDoubleArray( const std::string& key, const double* const data, - int nelements); + int nelements, + const bool distributed=false); /** * @brief Writes a vector of doubles associated with the supplied key to @@ -124,13 +139,17 @@ class HDFDatabase : public Database * written. * @param[in] data The vector of double values to be written. * @param[in] nelements The number of doubles in the vector. + * @param[in] distributed True if data is a distributed double array. + * HDFDatabase writes the array in file-per-process, + * where each file is written serially by one process. */ virtual void putDoubleVector( const std::string& key, const std::vector& data, - int nelements); + int nelements, + const bool distributed=false); /** * @brief Reads an array of integers associated with the supplied key @@ -143,13 +162,17 @@ class HDFDatabase : public Database * read. * @param[out] data The allocated array of integer values to be read. * @param[in] nelements The number of integers in the array. + * @param[in] distributed True if data is a distributed integer array. + * HDFDatabase reads the array in file-per-process, + * where each file is read serially by one process. */ virtual void getIntegerArray( const std::string& key, int* data, - int nelements); + int nelements, + const bool distributed=false); /** * @brief Count the number of elements in an array of doubles associated @@ -174,13 +197,17 @@ class HDFDatabase : public Database * read. * @param[out] data The allocated array of double values to be read. * @param[in] nelements The number of doubles in the array. + * @param[in] distributed True if data is a distributed double array. + * HDFDatabase reads the array in file-per-process, + * where each file is read serially by one process. */ virtual void getDoubleArray( const std::string& key, double* data, - int nelements); + int nelements, + const bool distributed=false); /** * @brief Reads a sub-array of doubles associated with the supplied key @@ -194,6 +221,9 @@ class HDFDatabase : public Database * @param[out] data The allocated sub-array of double values to be read. * @param[in] nelements The number of doubles in the full array. * @param[in] idx The set of indices in the sub-array. + * @param[in] distributed True if data is a distributed double array. + * HDFDatabase reads the array in file-per-process, + * where each file is read serially by one process. */ virtual void @@ -201,7 +231,8 @@ class HDFDatabase : public Database const std::string& key, double* data, int nelements, - const std::vector& idx); + const std::vector& idx, + const bool distributed=false); /** * @brief Reads an array of doubles associated with the supplied key @@ -220,6 +251,9 @@ class HDFDatabase : public Database * Typically, this is a number of columns of the matrix data. * @param[in] stride The stride to read from the HDF5 dataset. * Typically, this is the total number of columns of the matrix data. + * @param[in] distributed True if data is a distributed double array. + * HDFDatabase reads the array in file-per-process, + * where each file is read serially by one process. */ virtual void @@ -229,129 +263,130 @@ class HDFDatabase : public Database int nelements, int offset, int block_size, - int stride); - - /** - * @brief Creates a new HDF5 database file with the supplied name, - * extending it with 6 digits indicating the process rank. - * - * @param[in] file_name Name of HDF5 database file to create. - * @param[in] comm MPI communicator to obtain the process rank. - * - * @return True if file create was successful. - */ - virtual bool - create_parallel( - const std::string& file_name, - const MPI_Comm comm) override - { - CAROM_VERIFY(!file_name.empty()); - MPI_Comm_rank(comm, &d_rank); - - std::string ext_filename(file_name); - char tmp[10]; - sprintf(tmp, ".%06d", d_rank); - ext_filename += tmp; - - return create(ext_filename); - } - - /** - * @brief Opens an existing HDF5 database file with the supplied name, - * extending it with 6 digits indicating the process rank. - * - * @param[in] file_name Name of existing HDF5 database file to open. - * @param[in] type Read/write type ("r"/"wr") - * @param[in] comm MPI communicator to obtain the process rank. - * - * @return True if file open was successful. - */ - virtual - bool - open_parallel( - const std::string& file_name, - const std::string& type, - const MPI_Comm comm) override - { - CAROM_VERIFY(!file_name.empty()); - MPI_Comm_rank(comm, &d_rank); - - std::string ext_filename(file_name); - char tmp[10]; - sprintf(tmp, ".%06d", d_rank); - ext_filename += tmp; - - return open(ext_filename, type); - } - - /** - * @brief Writes a distributed array of doubles associated with the supplied key to - * the currently open database file. - * For HDFDatabase, the function is equivalent to putDoubleArray, - * writing the local array per each process. - * - * @param[in] key The key associated with the array of values to be - * written. - * @param[in] data The array of double values to be written. - * @param[in] nelements The number of doubles in the array. - */ - virtual - void - putDoubleArray_parallel( - const std::string& key, - const double* const data, - int nelements) - { - putDoubleArray(key, data, nelements); - } - - /** - * @brief Reads a distributed array of doubles associated with the supplied key - * from the currently open database file. - * For HDFDatabase, the function is equivalent to getDoubleArray, - * reading the local array from a local file per each process. - * - * @param[in] key The key associated with the array of values to be - * read. - * @param[out] data The allocated array of double values to be read. - * @param[in] nelements The number of doubles in the array. - */ - virtual - void - getDoubleArray_parallel( - const std::string& key, - double* data, - int nelements) - { - getDoubleArray(key, data, nelements); - } - - /** - * @brief Reads a distributed array of doubles associated with the supplied key - * from the currently open database file. - * For HDFDatabase, the function is equivalent to getDoubleArray, - * reading the local array from a local file per each process. - * - * @param[in] key The key associated with the array of values to be - * read. - * @param[out] data The allocated array of double values to be read. - * @param[in] nelements The number of doubles in the array. - * @param[in] offset The initial offset in the array. - * @param[in] block_size The block size to read from the HDF5 dataset. - * @param[in] stride The stride to read from the HDF5 dataset. - */ - virtual - void - getDoubleArray_parallel( - const std::string& key, - double* data, - int nelements, - int offset, - int block_size, - int stride) - { - getDoubleArray(key, data, nelements, offset, block_size, stride); - } + int stride, + const bool distributed=false); + + // /** + // * @brief Creates a new HDF5 database file with the supplied name, + // * extending it with 6 digits indicating the process rank. + // * + // * @param[in] file_name Name of HDF5 database file to create. + // * @param[in] comm MPI communicator to obtain the process rank. + // * + // * @return True if file create was successful. + // */ + // virtual bool + // create_parallel( + // const std::string& file_name, + // const MPI_Comm comm) override + // { + // CAROM_VERIFY(!file_name.empty()); + // MPI_Comm_rank(comm, &d_rank); + + // std::string ext_filename(file_name); + // char tmp[10]; + // sprintf(tmp, ".%06d", d_rank); + // ext_filename += tmp; + + // return create(ext_filename); + // } + + // /** + // * @brief Opens an existing HDF5 database file with the supplied name, + // * extending it with 6 digits indicating the process rank. + // * + // * @param[in] file_name Name of existing HDF5 database file to open. + // * @param[in] type Read/write type ("r"/"wr") + // * @param[in] comm MPI communicator to obtain the process rank. + // * + // * @return True if file open was successful. + // */ + // virtual + // bool + // open_parallel( + // const std::string& file_name, + // const std::string& type, + // const MPI_Comm comm) override + // { + // CAROM_VERIFY(!file_name.empty()); + // MPI_Comm_rank(comm, &d_rank); + + // std::string ext_filename(file_name); + // char tmp[10]; + // sprintf(tmp, ".%06d", d_rank); + // ext_filename += tmp; + + // return open(ext_filename, type); + // } + + // /** + // * @brief Writes a distributed array of doubles associated with the supplied key to + // * the currently open database file. + // * For HDFDatabase, the function is equivalent to putDoubleArray, + // * writing the local array per each process. + // * + // * @param[in] key The key associated with the array of values to be + // * written. + // * @param[in] data The array of double values to be written. + // * @param[in] nelements The number of doubles in the array. + // */ + // virtual + // void + // putDoubleArray_parallel( + // const std::string& key, + // const double* const data, + // int nelements) + // { + // putDoubleArray(key, data, nelements); + // } + + // /** + // * @brief Reads a distributed array of doubles associated with the supplied key + // * from the currently open database file. + // * For HDFDatabase, the function is equivalent to getDoubleArray, + // * reading the local array from a local file per each process. + // * + // * @param[in] key The key associated with the array of values to be + // * read. + // * @param[out] data The allocated array of double values to be read. + // * @param[in] nelements The number of doubles in the array. + // */ + // virtual + // void + // getDoubleArray_parallel( + // const std::string& key, + // double* data, + // int nelements) + // { + // getDoubleArray(key, data, nelements); + // } + + // /** + // * @brief Reads a distributed array of doubles associated with the supplied key + // * from the currently open database file. + // * For HDFDatabase, the function is equivalent to getDoubleArray, + // * reading the local array from a local file per each process. + // * + // * @param[in] key The key associated with the array of values to be + // * read. + // * @param[out] data The allocated array of double values to be read. + // * @param[in] nelements The number of doubles in the array. + // * @param[in] offset The initial offset in the array. + // * @param[in] block_size The block size to read from the HDF5 dataset. + // * @param[in] stride The stride to read from the HDF5 dataset. + // */ + // virtual + // void + // getDoubleArray_parallel( + // const std::string& key, + // double* data, + // int nelements, + // int offset, + // int block_size, + // int stride) + // { + // getDoubleArray(key, data, nelements, offset, block_size, stride); + // } protected: diff --git a/lib/utils/HDFDatabaseMPIO.cpp b/lib/utils/HDFDatabaseMPIO.cpp index 9f5411a1b..0ffa906a8 100644 --- a/lib/utils/HDFDatabaseMPIO.cpp +++ b/lib/utils/HDFDatabaseMPIO.cpp @@ -18,17 +18,19 @@ namespace CAROM { HDFDatabaseMPIO::HDFDatabaseMPIO() : HDFDatabase(), - d_rank(-1) + d_rank(-1), + d_comm(MPI_COMM_NULL) {} bool -HDFDatabaseMPIO::create_parallel( +HDFDatabaseMPIO::create( const std::string& file_name, const MPI_Comm comm) { int mpi_init; MPI_Initialized(&mpi_init); if (mpi_init) { + CAROM_VERIFY(comm != MPI_COMM_NULL); d_comm = comm; MPI_Comm_rank(d_comm, &d_rank); } @@ -73,7 +75,7 @@ HDFDatabaseMPIO::create_parallel( } bool -HDFDatabaseMPIO::open_parallel( +HDFDatabaseMPIO::open( const std::string& file_name, const std::string& type, const MPI_Comm comm) @@ -81,6 +83,7 @@ HDFDatabaseMPIO::open_parallel( int mpi_init; MPI_Initialized(&mpi_init); if (mpi_init) { + CAROM_VERIFY(comm != MPI_COMM_NULL); d_comm = comm; MPI_Comm_rank(d_comm, &d_rank); } diff --git a/lib/utils/HDFDatabaseMPIO.h b/lib/utils/HDFDatabaseMPIO.h index 88ce4d2ed..830a8ba28 100644 --- a/lib/utils/HDFDatabaseMPIO.h +++ b/lib/utils/HDFDatabaseMPIO.h @@ -39,29 +39,31 @@ class HDFDatabaseMPIO : public HDFDatabase * with the supplied name. * * @param[in] file_name Name of HDF5 database file to create. + * @param[in] comm MPI communicator for distributed data I/O. + * HDFDatabaseMPIO does not allow MPI_COMM_NULL. * * @return True if file create was successful. */ bool - create_parallel( + create( const std::string& file_name, - const MPI_Comm comm) override; + const MPI_Comm comm=MPI_COMM_WORLD) override; - /** - * @brief Creates a new HDF5 database file for non-distributed data - * with the supplied name. - * For HDFDatabaseMPIO, the behavior is equilvalent to create_parallel. - * - * @param[in] file_name Name of HDF5 database file to create. - * - * @return True if file create was successful. - */ - bool - create( - const std::string& file_name) override - { - return create_parallel(file_name, MPI_COMM_WORLD); - } + // /** + // * @brief Creates a new HDF5 database file for non-distributed data + // * with the supplied name. + // * For HDFDatabaseMPIO, the behavior is equilvalent to create_parallel. + // * + // * @param[in] file_name Name of HDF5 database file to create. + // * + // * @return True if file create was successful. + // */ + // bool + // create( + // const std::string& file_name) override + // { + // return create_parallel(file_name, MPI_COMM_WORLD); + // } /** * @brief Opens an existing HDF5 database file for distributed data @@ -69,54 +71,34 @@ class HDFDatabaseMPIO : public HDFDatabase * * @param[in] file_name Name of existing HDF5 database file to open. * @param[in] type Read/write type ("r"/"wr") + * @param[in] comm MPI communicator for distributed data I/O. + * HDFDatabaseMPIO does not allow MPI_COMM_NULL. * * @return True if file open was successful. */ bool - open_parallel( + open( const std::string& file_name, const std::string& type, - const MPI_Comm comm) override; + const MPI_Comm comm=MPI_COMM_WORLD) override; - /** - * @brief Opens an existing HDF5 database file for non-distributed data - * with the supplied name. - * For HDFDatabaseMPIO, the behavior is equilvalent to open_parallel. - * - * @param[in] file_name Name of existing HDF5 database file to open. - * @param[in] type Read/write type ("r"/"wr") - * - * @return True if file open was successful. - */ - bool - open( - const std::string& file_name, - const std::string& type) override - { - return open_parallel(file_name, type, MPI_COMM_WORLD); - } - - /** - * @brief Writes a distributed array of integers - * associated with the supplied key - * to the currently open HDF5 database file. - * - * @pre !key.empty() - * @pre data != nullptr - * @pre nelem_local >= 0 - * @pre nelements > 0 - * - * @param[in] key The key associated with the array of values to be - * written. - * @param[in] data The array of integer values to be written. - * @param[in] nelem_local The local number of integers in the array. - */ - virtual - void - putIntegerArray_parallel( - const std::string& key, - const int* const data, - int nelem_local); + // /** + // * @brief Opens an existing HDF5 database file for non-distributed data + // * with the supplied name. + // * For HDFDatabaseMPIO, the behavior is equilvalent to open_parallel. + // * + // * @param[in] file_name Name of existing HDF5 database file to open. + // * @param[in] type Read/write type ("r"/"wr") + // * + // * @return True if file open was successful. + // */ + // bool + // open( + // const std::string& file_name, + // const std::string& type) override + // { + // return open_parallel(file_name, type, MPI_COMM_WORLD); + // } /** * @brief Writes a local array of integers only for root rank, @@ -131,40 +113,21 @@ class HDFDatabaseMPIO : public HDFDatabase * written. * @param[in] data The array of integer values to be written. * @param[in] nelements The number of integers in the array. + * @param[in] distributed If true, distributed integer array will be written. + * If not, only the root process writes its integer array. */ void putIntegerArray( const std::string& key, const int* const data, - int nelements) override + int nelements, + const bool distributed=false) override { - if (d_rank != 0) + if ((!distributed) && (d_rank != 0)) nelements = 0; putIntegerArray_parallel(key, data, nelements); } - /** - * @brief Writes a distributed array of doubles - * associated with the supplied key to - * the currently open HDF5 database file. - * - * @pre !key.empty() - * @pre data != nullptr - * @pre nelem_local >= 0 - * @pre nelements > 0 - * - * @param[in] key The key associated with the array of values to be - * written. - * @param[in] data The array of double values to be written. - * @param[in] nelem_local The number of doubles in the array. - */ - virtual - void - putDoubleArray_parallel( - const std::string& key, - const double* const data, - int nelem_local); - /** * @brief Writes an array of doubles in the root rank * associated with the supplied key to @@ -178,22 +141,25 @@ class HDFDatabaseMPIO : public HDFDatabase * written. * @param[in] data The array of double values to be written. * @param[in] nelements The number of doubles in the array. + * @param[in] distributed If true, distributed double array will be written. + * If not, only the root process writes its double array. */ void putDoubleArray( const std::string& key, const double* const data, - int nelements) override + int nelements, + const bool distributed=false) override { - if (d_rank != 0) + if ((!distributed) && (d_rank != 0)) nelements = 0; putDoubleArray_parallel(key, data, nelements); } /** - * @brief Reads a distributed array of integers - * associated with the supplied key - * from the currently open HDF5 database file. + * @brief Reads an array of integers associated with the supplied key + * from the currently open HDF5 database file. + * All processes share the same non-distributed integer array. * * @pre !key.empty() * @pre data != nullptr || nelements == 0 @@ -201,59 +167,101 @@ class HDFDatabaseMPIO : public HDFDatabase * @param[in] key The key associated with the array of values to be * read. * @param[out] data The allocated array of integer values to be read. - * @param[in] nelem_local The local number of integers in the array. + * @param[in] nelements The number of integers in the array. + * @param[in] distributed If true, the integer array will be read in a distributed way. + * If not, the root process reads the entire array and broadcast to all processes. */ - virtual void - getIntegerArray_parallel( + getIntegerArray( const std::string& key, int* data, - int nelem_local); + int nelements, + const bool distributed=false) override + { + if (distributed) + { + getIntegerArray_parallel(key, data, nelements); + return; + } + + int read_size = (d_rank == 0) ? nelements : 0; + getIntegerArray_parallel(key, data, read_size); + + CAROM_VERIFY(d_comm != MPI_COMM_NULL); + MPI_Bcast(data, nelements, MPI_INT, 0, d_comm); + } /** - * @brief Reads an array of integers associated with the supplied key + * @brief Reads an array of doubles associated with the supplied key * from the currently open HDF5 database file. - * All processes share the same non-distributed integer array. + * All processes share the same non-distributed double array. * * @pre !key.empty() * @pre data != nullptr || nelements == 0 * * @param[in] key The key associated with the array of values to be * read. - * @param[out] data The allocated array of integer values to be read. - * @param[in] nelements The number of integers in the array. + * @param[out] data The allocated array of double values to be read. + * @param[in] nelements The number of doubles in the array. + * @param[in] distributed If true, the double array will be read in a distributed way. + * If not, the root process reads the entire array and broadcast to all processes. */ void - getIntegerArray( + getDoubleArray( const std::string& key, - int* data, - int nelements) override + double* data, + int nelements, + const bool distributed=false) override { + if (distributed) + { + getDoubleArray_parallel(key, data, nelements); + return; + } + int read_size = (d_rank == 0) ? nelements : 0; - getIntegerArray_parallel(key, data, read_size); + getDoubleArray_parallel(key, data, read_size); - MPI_Bcast(data, nelements, MPI_INT, 0, MPI_COMM_WORLD); + CAROM_VERIFY(d_comm != MPI_COMM_NULL); + MPI_Bcast(data, nelements, MPI_DOUBLE, 0, d_comm); } /** - * @brief Reads a distributed array of doubles - * associated with the supplied key + * @brief Reads a sub-array of doubles associated with the supplied key * from the currently open HDF5 database file. + * All processes share the same non-distributed double array. * * @pre !key.empty() * @pre data != nullptr || nelements == 0 * * @param[in] key The key associated with the array of values to be * read. - * @param[out] data The allocated array of double values to be read. - * @param[in] nelem_local The number of doubles in the array. + * @param[out] data The allocated sub-array of double values to be read. + * @param[in] nelements The number of doubles in the full array. + * @param[in] idx The set of indices in the sub-array. + * @param[in] distributed If true, the double array will be read in a distributed way. + * If not, the root process reads the entire array and broadcast to all processes. */ - virtual void - getDoubleArray_parallel( + getDoubleArray( const std::string& key, double* data, - int nelem_local); + int nelements, + const std::vector& idx, + const bool distributed=false) override + { + if (distributed) + { + getDoubleArray_parallel(key, data, nelements, idx); + return; + } + + int read_size = (d_rank == 0) ? nelements : 0; + getDoubleArray_parallel(key, data, read_size, idx); + + CAROM_VERIFY(d_comm != MPI_COMM_NULL); + MPI_Bcast(data, nelements, MPI_DOUBLE, 0, d_comm); + } /** * @brief Reads an array of doubles associated with the supplied key @@ -267,21 +275,110 @@ class HDFDatabaseMPIO : public HDFDatabase * read. * @param[out] data The allocated array of double values to be read. * @param[in] nelements The number of doubles in the array. + * @param[in] offset The initial offset in the array. + * @param[in] block_size The block size to read from the HDF5 dataset. + * @param[in] stride The stride to read from the HDF5 dataset. + * @param[in] distributed If true, the double array will be read in a distributed way. + * If not, the root process reads the entire array and broadcast to all processes. */ void getDoubleArray( const std::string& key, double* data, - int nelements) override + int nelements, + int offset, + int block_size, + int stride, + const bool distributed=false) override { + if (distributed) + { + getDoubleArray_parallel(key, data, nelements, offset, block_size, stride); + return; + } + int read_size = (d_rank == 0) ? nelements : 0; - getDoubleArray_parallel(key, data, read_size); + getDoubleArray_parallel(key, data, read_size, offset, block_size, stride); - MPI_Bcast(data, nelements, MPI_DOUBLE, 0, MPI_COMM_WORLD); + CAROM_VERIFY(d_comm != MPI_COMM_NULL); + MPI_Bcast(data, nelements, MPI_DOUBLE, 0, d_comm); } + void + writeAttribute( + int type_key, + hid_t dataset_id) override; + +private: + MPI_Comm d_comm; + int d_rank; + /** - * @brief Reads a distributed sub-array of doubles + * @brief Writes a distributed array of integers + * associated with the supplied key + * to the currently open HDF5 database file. + * + * @pre !key.empty() + * @pre data != nullptr + * @pre nelem_local >= 0 + * @pre nelements > 0 + * + * @param[in] key The key associated with the array of values to be + * written. + * @param[in] data The array of integer values to be written. + * @param[in] nelem_local The local number of integers in the array. + */ + virtual + void + putIntegerArray_parallel( + const std::string& key, + const int* const data, + int nelem_local); + + /** + * @brief Writes a distributed array of doubles + * associated with the supplied key to + * the currently open HDF5 database file. + * + * @pre !key.empty() + * @pre data != nullptr + * @pre nelem_local >= 0 + * @pre nelements > 0 + * + * @param[in] key The key associated with the array of values to be + * written. + * @param[in] data The array of double values to be written. + * @param[in] nelem_local The number of doubles in the array. + */ + virtual + void + putDoubleArray_parallel( + const std::string& key, + const double* const data, + int nelem_local); + + /** + * @brief Reads a distributed array of integers + * associated with the supplied key + * from the currently open HDF5 database file. + * + * @pre !key.empty() + * @pre data != nullptr || nelements == 0 + * + * @param[in] key The key associated with the array of values to be + * read. + * @param[out] data The allocated array of integer values to be read. + * @param[in] nelem_local The local number of integers in the array. + */ + virtual + void + getIntegerArray_parallel( + const std::string& key, + int* data, + int nelem_local); + + /** + * @brief Reads a distributed array of doubles * associated with the supplied key * from the currently open HDF5 database file. * @@ -290,22 +387,20 @@ class HDFDatabaseMPIO : public HDFDatabase * * @param[in] key The key associated with the array of values to be * read. - * @param[out] data The allocated sub-array of double values to be read. - * @param[in] nelem_local The number of doubles in the full array. - * @param[in] idx_local The set of indices in the sub-array. + * @param[out] data The allocated array of double values to be read. + * @param[in] nelem_local The number of doubles in the array. */ virtual void getDoubleArray_parallel( const std::string& key, double* data, - int nelem_local, - const std::vector& idx_local); + int nelem_local); /** - * @brief Reads a sub-array of doubles associated with the supplied key + * @brief Reads a distributed sub-array of doubles + * associated with the supplied key * from the currently open HDF5 database file. - * All processes share the same non-distributed double array. * * @pre !key.empty() * @pre data != nullptr || nelements == 0 @@ -313,21 +408,16 @@ class HDFDatabaseMPIO : public HDFDatabase * @param[in] key The key associated with the array of values to be * read. * @param[out] data The allocated sub-array of double values to be read. - * @param[in] nelements The number of doubles in the full array. - * @param[in] idx The set of indices in the sub-array. + * @param[in] nelem_local The number of doubles in the full array. + * @param[in] idx_local The set of indices in the sub-array. */ + virtual void - getDoubleArray( + getDoubleArray_parallel( const std::string& key, double* data, - int nelements, - const std::vector& idx) override - { - int read_size = (d_rank == 0) ? nelements : 0; - getDoubleArray_parallel(key, data, read_size, idx); - - MPI_Bcast(data, nelements, MPI_DOUBLE, 0, MPI_COMM_WORLD); - } + int nelem_local, + const std::vector& idx_local); /** * @brief Reads a distributed array of doubles @@ -358,46 +448,6 @@ class HDFDatabaseMPIO : public HDFDatabase int block_offset_global, int block_size_global, int stride_global); - - /** - * @brief Reads an array of doubles associated with the supplied key - * from the currently open HDF5 database file. - * All processes share the same non-distributed double array. - * - * @pre !key.empty() - * @pre data != nullptr || nelements == 0 - * - * @param[in] key The key associated with the array of values to be - * read. - * @param[out] data The allocated array of double values to be read. - * @param[in] nelements The number of doubles in the array. - * @param[in] offset The initial offset in the array. - * @param[in] block_size The block size to read from the HDF5 dataset. - * @param[in] stride The stride to read from the HDF5 dataset. - */ - void - getDoubleArray( - const std::string& key, - double* data, - int nelements, - int offset, - int block_size, - int stride) override - { - int read_size = (d_rank == 0) ? nelements : 0; - getDoubleArray_parallel(key, data, read_size, offset, block_size, stride); - - MPI_Bcast(data, nelements, MPI_DOUBLE, 0, MPI_COMM_WORLD); - } - - void - writeAttribute( - int type_key, - hid_t dataset_id) override; - -private: - MPI_Comm d_comm; - int d_rank; }; } From 0d883f1350c38291ba21e09c1bd409a3a36ac211 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Wed, 13 Mar 2024 20:42:33 -0700 Subject: [PATCH 47/50] minor fix --- lib/linalg/BasisGenerator.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/linalg/BasisGenerator.h b/lib/linalg/BasisGenerator.h index 24ee18945..87604447e 100644 --- a/lib/linalg/BasisGenerator.h +++ b/lib/linalg/BasisGenerator.h @@ -155,7 +155,7 @@ class BasisGenerator const std::string& kind = "basis", int col_min = 0, int col_max = 1e9, - Database::formats db_format = Database::HDF5); + Database::formats db_format = Database::formats::HDF5); /** * @brief Load previously saved sample (basis or state). From 09b82716b9bf7ea92c69759590614825c32634dc Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Mon, 18 Mar 2024 12:29:23 -0700 Subject: [PATCH 48/50] reflecting comments --- lib/linalg/BasisWriter.cpp | 10 +-- lib/utils/Database.h | 124 ------------------------------------ lib/utils/HDFDatabase.h | 122 ----------------------------------- lib/utils/HDFDatabaseMPIO.h | 40 ++---------- 4 files changed, 6 insertions(+), 290 deletions(-) diff --git a/lib/linalg/BasisWriter.cpp b/lib/linalg/BasisWriter.cpp index 466ff6dea..1257006b8 100644 --- a/lib/linalg/BasisWriter.cpp +++ b/lib/linalg/BasisWriter.cpp @@ -46,13 +46,7 @@ BasisWriter::BasisWriter( rank = 0; } - // char tmp[100]; - // sprintf(tmp, ".%06d", rank); - // full_file_name = base_file_name + tmp; full_file_name = base_file_name; - - // char tmp2[100]; - // sprintf(tmp2, "_snapshot.%06d", rank); snap_file_name = base_file_name + "_snapshot"; // create and open snapshot/basis database @@ -87,7 +81,7 @@ BasisWriter::writeBasis(const std::string& kind) d_database->create(full_file_name, MPI_COMM_WORLD); const Matrix* basis = d_basis_generator->getSpatialBasis(); - /* singular is always distributed */ + /* spatial basis is always distributed */ CAROM_VERIFY(basis->distributed()); int num_rows = basis->numRows(); int nrows_infile = num_rows; @@ -116,7 +110,7 @@ BasisWriter::writeBasis(const std::string& kind) } const Vector* sv = d_basis_generator->getSingularValues(); - /* singular is always not distributed */ + /* singular values are always not distributed */ CAROM_VERIFY(!sv->distributed()); int sv_dim = sv->dim(); sprintf(tmp, "singular_value_size"); diff --git a/lib/utils/Database.h b/lib/utils/Database.h index 187b48fb5..1008f391d 100644 --- a/lib/utils/Database.h +++ b/lib/utils/Database.h @@ -315,130 +315,6 @@ class Database HDF5_MPIO }; - // /** - // * @brief Creates a new database file with the supplied name, with parallel I/O support. - // * Supported only for HDF5 format. - // * For HDFDatabase, the function is equivalent to create, - // * only extending the file name with 6 digits indicating processor rank. - // * For HDFDatabaseMPIO, the file is created with MPI I/O file access property. - // * - // * @param[in] file_name Base Name of CSV database file to create. - // * @param[in] comm MPI communicator to get the rank. - // * - // * @return True if file create was successful. - // */ - // virtual - // bool - // create_parallel( - // const std::string& file_name, - // const MPI_Comm comm) - // { - // CAROM_ERROR("Abstract method Database::create_parallel!\n"); - // return false; - // } - - // /** - // * @brief Opens an existing database file with the supplied name, with parallel I/O support. - // * Supported only for HDF5 format. - // * For HDFDatabase, the function is equivalent to open, - // * only extending the file name with 6 digits indicating processor rank. - // * For HDFDatabaseMPIO, the file is opened with MPI I/O file access property. - // * - // * @param[in] file_name Name of existing CSV database file to open. - // * @param[in] type Read/write type ("r"/"wr") - // * @param[in] comm MPI communicator to get the rank. - // * - // * @return True if file open was successful. - // */ - // virtual - // bool - // open_parallel( - // const std::string& file_name, - // const std::string& type, - // const MPI_Comm comm) - // { - // CAROM_ERROR("Abstract method Database::open_parallel!\n"); - // return false; - // } - - // /** - // * @brief Writes a distributed array of doubles associated with the supplied key to - // * the currently open database file. - // * Supported only for HDF5 format. - // * For HDFDatabase, the function is equivalent to putDoubleArray, - // * writing the local array per each process. - // * For HDFDatabaseMPIO, the global array is written into a single file via MPI I/O. - // * - // * @param[in] key The key associated with the array of values to be - // * written. - // * @param[in] data The array of double values to be written. - // * @param[in] nelements The number of doubles in the array. - // */ - // virtual - // void - // putDoubleArray_parallel( - // const std::string& key, - // const double* const data, - // int nelements) - // { - // CAROM_ERROR("Abstract method Database::putDoubleArray_parallel!\n"); - // } - - // /** - // * @brief Reads a distributed array of doubles associated with the supplied key - // * from the currently open database file. - // * Supported only for HDF5 format. - // * For HDFDatabase, the function is equivalent to getDoubleArray, - // * reading the local array from a local file per each process. - // * For HDFDatabaseMPIO, the local array is read from a single file via MPI I/O. - // * - // * @param[in] key The key associated with the array of values to be - // * read. - // * @param[out] data The allocated array of double values to be read. - // * @param[in] nelements The number of doubles in the array. - // */ - // virtual - // void - // getDoubleArray_parallel( - // const std::string& key, - // double* data, - // int nelements) - // { - // CAROM_ERROR("Abstract method Database::getDoubleArray_parallel!\n"); - // } - - // /** - // * @brief Reads a distributed array of doubles associated with the supplied key - // * from the currently open database file. - // * Supported only for HDF5 format. - // * For HDFDatabase, the function is equivalent to getDoubleArray, - // * reading the local array from a local file per each process. - // * For HDFDatabaseMPIO, the local array is read from a single file via MPI I/O. - // * - // * @param[in] key The key associated with the array of values to be - // * read. - // * @param[out] data The allocated array of double values to be read. - // * @param[in] nelements The number of doubles in the array. - // * @param[in] offset The initial offset in the array. - // * Typically, this is a column index of the matrix data. - // * @param[in] block_size The block size to read from the HDF5 dataset. - // * Typically, this is a number of columns of the matrix data. - // * @param[in] stride The stride to read from the HDF5 dataset. - // * Typically, this is the total number of columns of the matrix data. - // */ - // virtual - // void - // getDoubleArray_parallel( - // const std::string& key, - // double* data, - // int nelements, - // int offset, - // int block_size, - // int stride) - // { - // CAROM_ERROR("Abstract method Database::getDoubleArray_parallel!\n"); - // } - private: /** * @brief Unimplemented copy constructor. diff --git a/lib/utils/HDFDatabase.h b/lib/utils/HDFDatabase.h index 1517c1ac2..3faadcf60 100644 --- a/lib/utils/HDFDatabase.h +++ b/lib/utils/HDFDatabase.h @@ -266,128 +266,6 @@ class HDFDatabase : public Database int stride, const bool distributed=false); - // /** - // * @brief Creates a new HDF5 database file with the supplied name, - // * extending it with 6 digits indicating the process rank. - // * - // * @param[in] file_name Name of HDF5 database file to create. - // * @param[in] comm MPI communicator to obtain the process rank. - // * - // * @return True if file create was successful. - // */ - // virtual bool - // create_parallel( - // const std::string& file_name, - // const MPI_Comm comm) override - // { - // CAROM_VERIFY(!file_name.empty()); - // MPI_Comm_rank(comm, &d_rank); - - // std::string ext_filename(file_name); - // char tmp[10]; - // sprintf(tmp, ".%06d", d_rank); - // ext_filename += tmp; - - // return create(ext_filename); - // } - - // /** - // * @brief Opens an existing HDF5 database file with the supplied name, - // * extending it with 6 digits indicating the process rank. - // * - // * @param[in] file_name Name of existing HDF5 database file to open. - // * @param[in] type Read/write type ("r"/"wr") - // * @param[in] comm MPI communicator to obtain the process rank. - // * - // * @return True if file open was successful. - // */ - // virtual - // bool - // open_parallel( - // const std::string& file_name, - // const std::string& type, - // const MPI_Comm comm) override - // { - // CAROM_VERIFY(!file_name.empty()); - // MPI_Comm_rank(comm, &d_rank); - - // std::string ext_filename(file_name); - // char tmp[10]; - // sprintf(tmp, ".%06d", d_rank); - // ext_filename += tmp; - - // return open(ext_filename, type); - // } - - // /** - // * @brief Writes a distributed array of doubles associated with the supplied key to - // * the currently open database file. - // * For HDFDatabase, the function is equivalent to putDoubleArray, - // * writing the local array per each process. - // * - // * @param[in] key The key associated with the array of values to be - // * written. - // * @param[in] data The array of double values to be written. - // * @param[in] nelements The number of doubles in the array. - // */ - // virtual - // void - // putDoubleArray_parallel( - // const std::string& key, - // const double* const data, - // int nelements) - // { - // putDoubleArray(key, data, nelements); - // } - - // /** - // * @brief Reads a distributed array of doubles associated with the supplied key - // * from the currently open database file. - // * For HDFDatabase, the function is equivalent to getDoubleArray, - // * reading the local array from a local file per each process. - // * - // * @param[in] key The key associated with the array of values to be - // * read. - // * @param[out] data The allocated array of double values to be read. - // * @param[in] nelements The number of doubles in the array. - // */ - // virtual - // void - // getDoubleArray_parallel( - // const std::string& key, - // double* data, - // int nelements) - // { - // getDoubleArray(key, data, nelements); - // } - - // /** - // * @brief Reads a distributed array of doubles associated with the supplied key - // * from the currently open database file. - // * For HDFDatabase, the function is equivalent to getDoubleArray, - // * reading the local array from a local file per each process. - // * - // * @param[in] key The key associated with the array of values to be - // * read. - // * @param[out] data The allocated array of double values to be read. - // * @param[in] nelements The number of doubles in the array. - // * @param[in] offset The initial offset in the array. - // * @param[in] block_size The block size to read from the HDF5 dataset. - // * @param[in] stride The stride to read from the HDF5 dataset. - // */ - // virtual - // void - // getDoubleArray_parallel( - // const std::string& key, - // double* data, - // int nelements, - // int offset, - // int block_size, - // int stride) - // { - // getDoubleArray(key, data, nelements, offset, block_size, stride); - // } - protected: /** diff --git a/lib/utils/HDFDatabaseMPIO.h b/lib/utils/HDFDatabaseMPIO.h index 830a8ba28..f8cfa914b 100644 --- a/lib/utils/HDFDatabaseMPIO.h +++ b/lib/utils/HDFDatabaseMPIO.h @@ -40,7 +40,8 @@ class HDFDatabaseMPIO : public HDFDatabase * * @param[in] file_name Name of HDF5 database file to create. * @param[in] comm MPI communicator for distributed data I/O. - * HDFDatabaseMPIO does not allow MPI_COMM_NULL. + * By default, HDFDatabaseMPIO uses MPI_COMM_WORLD + * and does not allow MPI_COMM_NULL. * * @return True if file create was successful. */ @@ -49,22 +50,6 @@ class HDFDatabaseMPIO : public HDFDatabase const std::string& file_name, const MPI_Comm comm=MPI_COMM_WORLD) override; - // /** - // * @brief Creates a new HDF5 database file for non-distributed data - // * with the supplied name. - // * For HDFDatabaseMPIO, the behavior is equilvalent to create_parallel. - // * - // * @param[in] file_name Name of HDF5 database file to create. - // * - // * @return True if file create was successful. - // */ - // bool - // create( - // const std::string& file_name) override - // { - // return create_parallel(file_name, MPI_COMM_WORLD); - // } - /** * @brief Opens an existing HDF5 database file for distributed data * with the supplied name. @@ -72,7 +57,8 @@ class HDFDatabaseMPIO : public HDFDatabase * @param[in] file_name Name of existing HDF5 database file to open. * @param[in] type Read/write type ("r"/"wr") * @param[in] comm MPI communicator for distributed data I/O. - * HDFDatabaseMPIO does not allow MPI_COMM_NULL. + * By default, HDFDatabaseMPIO uses MPI_COMM_WORLD + * and does not allow MPI_COMM_NULL. * * @return True if file open was successful. */ @@ -82,24 +68,6 @@ class HDFDatabaseMPIO : public HDFDatabase const std::string& type, const MPI_Comm comm=MPI_COMM_WORLD) override; - // /** - // * @brief Opens an existing HDF5 database file for non-distributed data - // * with the supplied name. - // * For HDFDatabaseMPIO, the behavior is equilvalent to open_parallel. - // * - // * @param[in] file_name Name of existing HDF5 database file to open. - // * @param[in] type Read/write type ("r"/"wr") - // * - // * @return True if file open was successful. - // */ - // bool - // open( - // const std::string& file_name, - // const std::string& type) override - // { - // return open_parallel(file_name, type, MPI_COMM_WORLD); - // } - /** * @brief Writes a local array of integers only for root rank, * with associated supplied key to From fdff1ff466df0970b697667846f16cb6ecf826fb Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Mon, 18 Mar 2024 15:09:43 -0700 Subject: [PATCH 49/50] hdf5 parallel is optional. set compile-time if statements. --- lib/utils/HDFDatabaseMPIO.cpp | 8 ++++++++ lib/utils/HDFDatabaseMPIO.h | 2 ++ unit_tests/test_HDFDatabase.cpp | 4 ++++ 3 files changed, 14 insertions(+) diff --git a/lib/utils/HDFDatabaseMPIO.cpp b/lib/utils/HDFDatabaseMPIO.cpp index 0ffa906a8..5e2d9bc40 100644 --- a/lib/utils/HDFDatabaseMPIO.cpp +++ b/lib/utils/HDFDatabaseMPIO.cpp @@ -17,11 +17,17 @@ namespace CAROM { HDFDatabaseMPIO::HDFDatabaseMPIO() : +#if HDF5_IS_PARALLEL HDFDatabase(), d_rank(-1), d_comm(MPI_COMM_NULL) +#else + HDFDatabase() +#endif {} +#if HDF5_IS_PARALLEL + bool HDFDatabaseMPIO::create( const std::string& file_name, @@ -627,4 +633,6 @@ HDFDatabaseMPIO::writeAttribute( #endif } +#endif + } diff --git a/lib/utils/HDFDatabaseMPIO.h b/lib/utils/HDFDatabaseMPIO.h index f8cfa914b..6686a95fe 100644 --- a/lib/utils/HDFDatabaseMPIO.h +++ b/lib/utils/HDFDatabaseMPIO.h @@ -34,6 +34,7 @@ class HDFDatabaseMPIO : public HDFDatabase virtual ~HDFDatabaseMPIO() {} +#if HDF5_IS_PARALLEL /** * @brief Creates a new HDF5 database file for distributed data * with the supplied name. @@ -416,6 +417,7 @@ class HDFDatabaseMPIO : public HDFDatabase int block_offset_global, int block_size_global, int stride_global); +#endif }; } diff --git a/unit_tests/test_HDFDatabase.cpp b/unit_tests/test_HDFDatabase.cpp index 1fc774051..68b43beb7 100644 --- a/unit_tests/test_HDFDatabase.cpp +++ b/unit_tests/test_HDFDatabase.cpp @@ -33,6 +33,8 @@ TEST(GoogleTestFramework, GoogleTestFrameworkFound) { SUCCEED(); } +#if HDF5_IS_PARALLEL + TEST(HDF5, Test_parallel_writing) { int nproc, rank; @@ -537,6 +539,8 @@ TEST(HDF5, Test_selective_parallel_reading) EXPECT_TRUE(data[d] == d + offsets[rank] * ncol); } +#endif + TEST(BasisGeneratorIO, HDFDatabase) { // Get the rank of this process, and the number of processors. From a339294ba801b200d735f8b59c51080edcb386eb Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Mon, 18 Mar 2024 16:22:46 -0700 Subject: [PATCH 50/50] minor fix --- lib/CAROM_config.h.in | 3 +++ lib/utils/HDFDatabase.cpp | 8 ++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/CAROM_config.h.in b/lib/CAROM_config.h.in index 9a0246992..569332ebd 100644 --- a/lib/CAROM_config.h.in +++ b/lib/CAROM_config.h.in @@ -57,6 +57,9 @@ /* Defined if you have HDF5 support */ #cmakedefine01 CAROM_HAVE_HDF5 +/* Defined if you have parallel HDF5 support */ +#cmakedefine01 HDF5_IS_PARALLEL + /* Define to 1 if you have the header file. */ #cmakedefine01 CAROM_HAVE_INTTYPES_H diff --git a/lib/utils/HDFDatabase.cpp b/lib/utils/HDFDatabase.cpp index 971914bd7..49e9136a8 100644 --- a/lib/utils/HDFDatabase.cpp +++ b/lib/utils/HDFDatabase.cpp @@ -46,8 +46,6 @@ HDFDatabase::create( const std::string& file_name, const MPI_Comm comm) { - Database::create(file_name, comm); - CAROM_VERIFY(!file_name.empty()); std::string ext_filename(file_name); @@ -59,6 +57,8 @@ HDFDatabase::create( ext_filename += tmp; } + Database::create(ext_filename, comm); + hid_t file_id = H5Fcreate(ext_filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, @@ -78,8 +78,6 @@ HDFDatabase::open( const std::string& type, const MPI_Comm comm) { - Database::open(file_name, type, comm); - CAROM_VERIFY(!file_name.empty()); std::string ext_filename(file_name); @@ -91,6 +89,8 @@ HDFDatabase::open( ext_filename += tmp; } + Database::open(ext_filename, type, comm); + CAROM_VERIFY(type == "r" || type == "wr"); hid_t file_id; if (type == "r")