From 2f7318d205a85503fc7e8e4bd1bc71dea56b456c Mon Sep 17 00:00:00 2001 From: hgangwar Date: Mon, 13 Jul 2026 12:10:51 -0400 Subject: [PATCH 01/12] add_global_comm_header --- src/pcms/coupler/CMakeLists.txt | 1 + src/pcms/coupler/coupler.hpp | 45 ++++++++++++++++++++++++ src/pcms/coupler/global_communicator.h | 48 ++++++++++++++++++++++++++ 3 files changed, 94 insertions(+) create mode 100644 src/pcms/coupler/global_communicator.h diff --git a/src/pcms/coupler/CMakeLists.txt b/src/pcms/coupler/CMakeLists.txt index 82afceb3..6a498c62 100644 --- a/src/pcms/coupler/CMakeLists.txt +++ b/src/pcms/coupler/CMakeLists.txt @@ -7,6 +7,7 @@ set(PCMS_COUPLER_HEADERS field_exchange_planner.h partition.h overlap_mask.h + global_communicator.h ) diff --git a/src/pcms/coupler/coupler.hpp b/src/pcms/coupler/coupler.hpp index d2d12cb1..5706d350 100644 --- a/src/pcms/coupler/coupler.hpp +++ b/src/pcms/coupler/coupler.hpp @@ -11,11 +11,47 @@ #include "pcms/utility/assert.h" #include "pcms/utility/common.h" #include "pcms/utility/profile.h" +#include "pcms/coupler/global_communicator.h" #include namespace pcms { +template +class GlobalDataInterface +{ +public: + GlobalDataInterface(const std::string& name, + MPI_Comm mpi_comm, + redev::Channel& channel) + : mpi_comm_(mpi_comm), + type_info_(typeid(T)), + comm_(name, mpi_comm_, channel) + { + PCMS_FUNCTION_TIMER; + } + + void SendData(T* msg, + std::string VarName, + size_t msg_size, + Mode mode = Mode::Synchronous) + { + PCMS_FUNCTION_TIMER; + comm_.Send(msg, std::move(VarName), msg_size, mode); + } + + std::vector ReceiveData(std::string VarName, + size_t msg_size, + Mode mode = Mode::Synchronous) + { + PCMS_FUNCTION_TIMER; + return comm_.Receive(std::move(VarName), msg_size, mode); + } +private: + MPI_Comm mpi_comm_; + const std::type_info& type_info_; + GlobalCommunicator comm_; +}; class Application; template @@ -70,6 +106,8 @@ class Application FieldHandle AddField(std::string name, Field&& field, std::unique_ptr> serializer, bool participates = true); + template + std::unique_ptr> AddData(std::string name, MPI_Comm mpi_comm); void SendField(const std::string& name, redev::Mode mode = redev::Mode::Synchronous) @@ -162,6 +200,13 @@ class Application std::map> layout_overlap_masks_; }; +template +std::unique_ptr> +Application::AddData(std::string name, MPI_Comm mpi_comm) +{ + PCMS_FUNCTION_TIMER; + return std::make_unique>(name, mpi_comm, channel_); +} class Coupler { private: diff --git a/src/pcms/coupler/global_communicator.h b/src/pcms/coupler/global_communicator.h new file mode 100644 index 00000000..b02b2187 --- /dev/null +++ b/src/pcms/coupler/global_communicator.h @@ -0,0 +1,48 @@ +#ifndef PCMS_GLOBAL_COMMUNICATOR_H +#define PCMS_GLOBAL_COMMUNICATOR_H +#include +#include +namespace pcms +{ + using redev::Mode; + template + struct GlobalCommunicator + { + using value_type = T; + public: + GlobalCommunicator(std::string name, MPI_Comm mpi_comm, redev::Channel& channel) + : mpi_comm(mpi_comm), + channel_(channel), + name_(std::move(name)) + { + PCMS_FUNCTION_TIMER; + comm_ = channel_.CreateComm(name_, mpi_comm, redev::CommType::Global ); + } + GlobalCommunicator(const GlobalCommunicator&) = delete; + GlobalCommunicator& operator=(const GlobalCommunicator&) = delete; + GlobalCommunicator(GlobalCommunicator&&)= default; + GlobalCommunicator& operator=(GlobalCommunicator&&) = default; + + void Send(T* msg, std::string VarName, size_t msg_size, Mode mode = Mode::Synchronous) + { + PCMS_FUNCTION_TIMER; + PCMS_ALWAYS_ASSERT(channel_.InSendCommunicationPhase()); + comm_.SetCommParams( VarName, msg_size); + comm_.Send(msg, mode); + } + std::vector Receive(std::string VarName, size_t msg_size, Mode mode = Mode::Synchronous) + { + PCMS_FUNCTION_TIMER; + PCMS_ALWAYS_ASSERT(channel_.InReceiveCommunicationPhase()); + comm_.SetCommParams(VarName, msg_size); + auto data = comm_.Recv(mode); + return data; + } + private: + MPI_Comm mpi_comm; + redev::Channel& channel_; + std::string name_; + redev::BidirectionalComm comm_; + }; +} +#endif // PCMS_GLOBAL_COMMUNICATOR_H \ No newline at end of file From 313df5e4991981dd91f21579fe1e369eccc773d9 Mon Sep 17 00:00:00 2001 From: hgangwar Date: Mon, 13 Jul 2026 12:19:54 -0400 Subject: [PATCH 02/12] add_test_case --- test/CMakeLists.txt | 40 ++++++++++---- test/test_GDI.cpp | 128 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 159 insertions(+), 9 deletions(-) create mode 100644 test/test_GDI.cpp diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 1c603c45..698e697b 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -96,6 +96,37 @@ if(PCMS_ENABLE_OMEGA_H) ${d3d16p} ignored) endif() + add_exe(test_GDI) + tri_mpi_test( + TESTNAME + test_GDI + TIMEOUT + 20 + NAME1 + app + EXE1 + ./test_GDI + PROCS1 + 1 + ARGS1 + 1 + NAME2 + rdv + EXE2 + ./test_GDI + PROCS2 + 1 + ARGS2 + -1 + NAME3 + app + EXE3 + ./test_GDI + PROCS3 + 1 + ARGS3 + 0 + ) set(d3d8p ${PCMS_TEST_DATA_DIR}/d3d/d3d-full_9k_sfc_p8.osh/) add_exe(test_twoClientOverlap) @@ -380,7 +411,6 @@ if(Catch2_FOUND) APPEND PCMS_UNIT_TEST_SOURCES test_error_handling.cpp - test_eqdsk.cpp test_uniform_grid.cpp test_field_evaluation.cpp test_field_interpolation.cpp @@ -401,7 +431,6 @@ if(Catch2_FOUND) test_omega_h_lagrange_field.cpp test_point_evaluator.cpp) endif() - if(PCMS_ENABLE_MESHFIELDS) list(APPEND PCMS_UNIT_TEST_SOURCES test_load_vector.cpp) @@ -426,13 +455,6 @@ if(Catch2_FOUND) target_link_libraries(unit_tests PRIVATE PETSc::PETSc) endif() - target_link_libraries(unit_tests PUBLIC - Catch2::Catch2 - pcms::core - pcms_transfer - pcms_transfer - ) - target_include_directories(unit_tests PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) add_executable(test_interpolation_on_ltx_mesh test_interpolation_on_ltx_mesh.cpp) diff --git a/test/test_GDI.cpp b/test/test_GDI.cpp new file mode 100644 index 00000000..846404c0 --- /dev/null +++ b/test/test_GDI.cpp @@ -0,0 +1,128 @@ + +#include +#include +#include "test_support.h" +#include "pcms/coupler/coupler.hpp" +#include +static constexpr bool done = true; +static constexpr int COMM_ROUNDS = 1; + +void xgc_delta_f(MPI_Comm comm) +{ + pcms::Coupler coupler("proxy_couple", comm, false, {}); + pcms::Application* app = coupler.AddApplication("proxy_couple_xgc_delta_f"); + + const auto GDI = app->AddData("global_comm", comm); + auto mean = std::vector(1); + mean[0] = 16; + do { + for (int i = 0; i < COMM_ROUNDS; ++i) { + app->BeginSendPhase(); + GDI->SendData(mean.data(), "mean", mean.size()); + app->EndSendPhase(); + printf("delta Sent mean:%d\n", mean[0]); + app->BeginReceivePhase(); + mean = GDI->ReceiveData("mean", mean.size()); + app->EndReceivePhase(); + mean[0] = mean[0]/2; + } + } while (!done); + printf("final Mean = %d\n", mean[0]); + assert(std::fabs(mean[0] - 1.0) < 1e-12); + printf("GDI test successful.\n"); +} +void xgc_total_f(MPI_Comm comm) +{ + pcms::Coupler coupler("proxy_couple", comm, false, {}); + pcms::Application* app = coupler.AddApplication("proxy_couple_xgc_total_f"); + + auto GDI = app->AddData("global_comm", comm); + auto mean = std::vector(1); + do { + for (int i = 0; i < COMM_ROUNDS; ++i) { + app->BeginReceivePhase(); + mean = GDI->ReceiveData("mean", mean.size()); + app->EndReceivePhase(); + printf("total Recieved mean:%d\n", mean[0]); + mean[0] = mean[0]/2; + app->BeginSendPhase(); + GDI->SendData(mean.data(), "mean", mean.size()); + app->EndSendPhase(); + printf("total Sent mean:%d\n", mean[0]); + } + } while (!done); +} +void xgc_coupler(MPI_Comm comm) +{ + // Define Partition + redev::LO dim = 3; + redev::LOs ranks(1); + std::iota(ranks.begin(), ranks.end(), 0); + redev::Reals cuts = {0}; + auto partition = redev::Partition{redev::RCBPtn{dim, ranks, cuts}}; + + pcms::Coupler cpl("proxy_couple", comm, true, + partition); + auto* total_f = cpl.AddApplication("proxy_couple_xgc_total_f"); + auto* delta_f = cpl.AddApplication("proxy_couple_xgc_delta_f"); + + auto GDI_total = total_f->AddData("global_comm", comm); + auto GDI_delta = delta_f->AddData("global_comm", comm); + auto mean = std::vector(1); + do { + for (int i = 0; i < COMM_ROUNDS; ++i) { + delta_f->BeginReceivePhase(); + mean = GDI_delta->ReceiveData("mean", 1); + delta_f->EndReceivePhase(); + printf("delta Received mean:%d\n", mean[0]); + mean[0] = mean[0]/2; + const auto msg_size = mean.size(); + total_f->BeginSendPhase(); + GDI_total->SendData(mean.data(), "mean", msg_size); + total_f->EndSendPhase(); + printf("total sent mean:%d\n", mean[0]); + total_f->BeginReceivePhase(); + mean = GDI_total->ReceiveData("mean", msg_size); + total_f->EndReceivePhase(); + printf("delta Received mean:%d\n", mean[0]); + mean[0] = mean[0]/2; + delta_f->BeginSendPhase(); + GDI_delta->SendData(mean.data(), "mean", msg_size); + delta_f->EndSendPhase(); + printf("detla sent mean:%d\n", mean[0]); + } + } while (!done); +} + +int main(int argc, char** argv) +{ + MPI_Init(&argc, &argv); + + OMEGA_H_CHECK(argc == 2); + + const auto clientId = std::atoi(argv[1]); + REDEV_ALWAYS_ASSERT(clientId >= -1 && clientId <= 1); + + MPI_Comm comm = MPI_COMM_WORLD; + + switch (clientId) { + case -1: + xgc_coupler(comm); + break; + + case 0: + xgc_delta_f(comm); + break; + + case 1: + xgc_total_f(comm); + break; + + default: + std::cerr << "Unhandled client id; expected -1, 0, or 1\n"; + MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); + } + + MPI_Finalize(); + return 0; +} From e8cd7cd5d8e6a08830abbb344f8d961ff355c341 Mon Sep 17 00:00:00 2001 From: hgangwar Date: Mon, 20 Jul 2026 18:43:52 -0400 Subject: [PATCH 03/12] format_test --- test/test_GDI.cpp | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/test/test_GDI.cpp b/test/test_GDI.cpp index 846404c0..d60bc678 100644 --- a/test/test_GDI.cpp +++ b/test/test_GDI.cpp @@ -24,12 +24,12 @@ void xgc_delta_f(MPI_Comm comm) app->BeginReceivePhase(); mean = GDI->ReceiveData("mean", mean.size()); app->EndReceivePhase(); - mean[0] = mean[0]/2; + mean[0] = mean[0] / 2; } } while (!done); printf("final Mean = %d\n", mean[0]); assert(std::fabs(mean[0] - 1.0) < 1e-12); - printf("GDI test successful.\n"); + printf("GDI test successful.\n"); } void xgc_total_f(MPI_Comm comm) { @@ -44,7 +44,7 @@ void xgc_total_f(MPI_Comm comm) mean = GDI->ReceiveData("mean", mean.size()); app->EndReceivePhase(); printf("total Recieved mean:%d\n", mean[0]); - mean[0] = mean[0]/2; + mean[0] = mean[0] / 2; app->BeginSendPhase(); GDI->SendData(mean.data(), "mean", mean.size()); app->EndSendPhase(); @@ -61,8 +61,7 @@ void xgc_coupler(MPI_Comm comm) redev::Reals cuts = {0}; auto partition = redev::Partition{redev::RCBPtn{dim, ranks, cuts}}; - pcms::Coupler cpl("proxy_couple", comm, true, - partition); + pcms::Coupler cpl("proxy_couple", comm, true, partition); auto* total_f = cpl.AddApplication("proxy_couple_xgc_total_f"); auto* delta_f = cpl.AddApplication("proxy_couple_xgc_delta_f"); @@ -75,8 +74,8 @@ void xgc_coupler(MPI_Comm comm) mean = GDI_delta->ReceiveData("mean", 1); delta_f->EndReceivePhase(); printf("delta Received mean:%d\n", mean[0]); - mean[0] = mean[0]/2; - const auto msg_size = mean.size(); + mean[0] = mean[0] / 2; + const auto msg_size = mean.size(); total_f->BeginSendPhase(); GDI_total->SendData(mean.data(), "mean", msg_size); total_f->EndSendPhase(); @@ -85,7 +84,7 @@ void xgc_coupler(MPI_Comm comm) mean = GDI_total->ReceiveData("mean", msg_size); total_f->EndReceivePhase(); printf("delta Received mean:%d\n", mean[0]); - mean[0] = mean[0]/2; + mean[0] = mean[0] / 2; delta_f->BeginSendPhase(); GDI_delta->SendData(mean.data(), "mean", msg_size); delta_f->EndSendPhase(); @@ -106,17 +105,11 @@ int main(int argc, char** argv) MPI_Comm comm = MPI_COMM_WORLD; switch (clientId) { - case -1: - xgc_coupler(comm); - break; + case -1: xgc_coupler(comm); break; - case 0: - xgc_delta_f(comm); - break; + case 0: xgc_delta_f(comm); break; - case 1: - xgc_total_f(comm); - break; + case 1: xgc_total_f(comm); break; default: std::cerr << "Unhandled client id; expected -1, 0, or 1\n"; From dcbc7f3dd4a3a6cc69e79535970b9d2ad3165955 Mon Sep 17 00:00:00 2001 From: hgangwar Date: Mon, 20 Jul 2026 19:14:18 -0400 Subject: [PATCH 04/12] update_redev_ref --- .github/workflows/cmake-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cmake-test.yml b/.github/workflows/cmake-test.yml index 20c58f9b..dff3030e 100644 --- a/.github/workflows/cmake-test.yml +++ b/.github/workflows/cmake-test.yml @@ -165,7 +165,7 @@ jobs: with: repo-name: 'redev' repo-path: 'SCOREC/redev' - repo-ref: 'ac09848a5f9b89493e8b679c9080b9efe5538376' + repo-ref: 'eb52569702864979f5b7d03d9c082c96f20b2bd5' cache: true cache-suffix: ${{ matrix.python_api == 'ON' && '-shared' || '' }} options: '-DCMAKE_CXX_COMPILER=`which mpicxx` From f197296401a5308344e1a88a8989988311249668 Mon Sep 17 00:00:00 2001 From: hgangwar Date: Mon, 20 Jul 2026 19:18:13 -0400 Subject: [PATCH 05/12] format --- src/pcms/coupler/coupler.hpp | 21 +++---- src/pcms/coupler/global_communicator.h | 81 +++++++++++++------------- 2 files changed, 51 insertions(+), 51 deletions(-) diff --git a/src/pcms/coupler/coupler.hpp b/src/pcms/coupler/coupler.hpp index 5706d350..ea9fd002 100644 --- a/src/pcms/coupler/coupler.hpp +++ b/src/pcms/coupler/coupler.hpp @@ -20,8 +20,7 @@ template class GlobalDataInterface { public: - GlobalDataInterface(const std::string& name, - MPI_Comm mpi_comm, + GlobalDataInterface(const std::string& name, MPI_Comm mpi_comm, redev::Channel& channel) : mpi_comm_(mpi_comm), type_info_(typeid(T)), @@ -30,18 +29,15 @@ class GlobalDataInterface PCMS_FUNCTION_TIMER; } - void SendData(T* msg, - std::string VarName, - size_t msg_size, - Mode mode = Mode::Synchronous) + void SendData(T* msg, std::string VarName, size_t msg_size, + Mode mode = Mode::Synchronous) { PCMS_FUNCTION_TIMER; comm_.Send(msg, std::move(VarName), msg_size, mode); } - std::vector ReceiveData(std::string VarName, - size_t msg_size, - Mode mode = Mode::Synchronous) + std::vector ReceiveData(std::string VarName, size_t msg_size, + Mode mode = Mode::Synchronous) { PCMS_FUNCTION_TIMER; return comm_.Receive(std::move(VarName), msg_size, mode); @@ -107,7 +103,8 @@ class Application std::unique_ptr> serializer, bool participates = true); template - std::unique_ptr> AddData(std::string name, MPI_Comm mpi_comm); + std::unique_ptr> AddData(std::string name, + MPI_Comm mpi_comm); void SendField(const std::string& name, redev::Mode mode = redev::Mode::Synchronous) @@ -201,8 +198,8 @@ class Application }; template -std::unique_ptr> -Application::AddData(std::string name, MPI_Comm mpi_comm) +std::unique_ptr> Application::AddData(std::string name, + MPI_Comm mpi_comm) { PCMS_FUNCTION_TIMER; return std::make_unique>(name, mpi_comm, channel_); diff --git a/src/pcms/coupler/global_communicator.h b/src/pcms/coupler/global_communicator.h index b02b2187..888c1ab9 100644 --- a/src/pcms/coupler/global_communicator.h +++ b/src/pcms/coupler/global_communicator.h @@ -4,45 +4,48 @@ #include namespace pcms { - using redev::Mode; - template - struct GlobalCommunicator +using redev::Mode; +template +struct GlobalCommunicator +{ + using value_type = T; + +public: + GlobalCommunicator(std::string name, MPI_Comm mpi_comm, + redev::Channel& channel) + : mpi_comm(mpi_comm), channel_(channel), name_(std::move(name)) + { + PCMS_FUNCTION_TIMER; + comm_ = channel_.CreateComm(name_, mpi_comm, redev::CommType::Global); + } + GlobalCommunicator(const GlobalCommunicator&) = delete; + GlobalCommunicator& operator=(const GlobalCommunicator&) = delete; + GlobalCommunicator(GlobalCommunicator&&) = default; + GlobalCommunicator& operator=(GlobalCommunicator&&) = default; + + void Send(T* msg, std::string VarName, size_t msg_size, + Mode mode = Mode::Synchronous) + { + PCMS_FUNCTION_TIMER; + PCMS_ALWAYS_ASSERT(channel_.InSendCommunicationPhase()); + comm_.SetCommParams(VarName, msg_size); + comm_.Send(msg, mode); + } + std::vector Receive(std::string VarName, size_t msg_size, + Mode mode = Mode::Synchronous) { - using value_type = T; - public: - GlobalCommunicator(std::string name, MPI_Comm mpi_comm, redev::Channel& channel) - : mpi_comm(mpi_comm), - channel_(channel), - name_(std::move(name)) - { - PCMS_FUNCTION_TIMER; - comm_ = channel_.CreateComm(name_, mpi_comm, redev::CommType::Global ); - } - GlobalCommunicator(const GlobalCommunicator&) = delete; - GlobalCommunicator& operator=(const GlobalCommunicator&) = delete; - GlobalCommunicator(GlobalCommunicator&&)= default; - GlobalCommunicator& operator=(GlobalCommunicator&&) = default; + PCMS_FUNCTION_TIMER; + PCMS_ALWAYS_ASSERT(channel_.InReceiveCommunicationPhase()); + comm_.SetCommParams(VarName, msg_size); + auto data = comm_.Recv(mode); + return data; + } - void Send(T* msg, std::string VarName, size_t msg_size, Mode mode = Mode::Synchronous) - { - PCMS_FUNCTION_TIMER; - PCMS_ALWAYS_ASSERT(channel_.InSendCommunicationPhase()); - comm_.SetCommParams( VarName, msg_size); - comm_.Send(msg, mode); - } - std::vector Receive(std::string VarName, size_t msg_size, Mode mode = Mode::Synchronous) - { - PCMS_FUNCTION_TIMER; - PCMS_ALWAYS_ASSERT(channel_.InReceiveCommunicationPhase()); - comm_.SetCommParams(VarName, msg_size); - auto data = comm_.Recv(mode); - return data; - } - private: - MPI_Comm mpi_comm; - redev::Channel& channel_; - std::string name_; - redev::BidirectionalComm comm_; - }; -} +private: + MPI_Comm mpi_comm; + redev::Channel& channel_; + std::string name_; + redev::BidirectionalComm comm_; +}; +} // namespace pcms #endif // PCMS_GLOBAL_COMMUNICATOR_H \ No newline at end of file From 29984dc22680fb720a2982f13d3aaed87c55defe Mon Sep 17 00:00:00 2001 From: hgangwar Date: Mon, 20 Jul 2026 19:29:35 -0400 Subject: [PATCH 06/12] fix_warnings --- test/test_GDI.cpp | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/test/test_GDI.cpp b/test/test_GDI.cpp index d60bc678..65dabe3a 100644 --- a/test/test_GDI.cpp +++ b/test/test_GDI.cpp @@ -20,14 +20,14 @@ void xgc_delta_f(MPI_Comm comm) app->BeginSendPhase(); GDI->SendData(mean.data(), "mean", mean.size()); app->EndSendPhase(); - printf("delta Sent mean:%d\n", mean[0]); + printf("delta Sent mean:%ld\n", mean[0]); app->BeginReceivePhase(); mean = GDI->ReceiveData("mean", mean.size()); app->EndReceivePhase(); mean[0] = mean[0] / 2; } } while (!done); - printf("final Mean = %d\n", mean[0]); + printf("final Mean = %ld\n", mean[0]); assert(std::fabs(mean[0] - 1.0) < 1e-12); printf("GDI test successful.\n"); } @@ -43,12 +43,12 @@ void xgc_total_f(MPI_Comm comm) app->BeginReceivePhase(); mean = GDI->ReceiveData("mean", mean.size()); app->EndReceivePhase(); - printf("total Recieved mean:%d\n", mean[0]); + printf("total Recieved mean:%ld\n", mean[0]); mean[0] = mean[0] / 2; app->BeginSendPhase(); GDI->SendData(mean.data(), "mean", mean.size()); app->EndSendPhase(); - printf("total Sent mean:%d\n", mean[0]); + printf("total Sent mean:%ld\n", mean[0]); } } while (!done); } @@ -73,22 +73,22 @@ void xgc_coupler(MPI_Comm comm) delta_f->BeginReceivePhase(); mean = GDI_delta->ReceiveData("mean", 1); delta_f->EndReceivePhase(); - printf("delta Received mean:%d\n", mean[0]); + printf("delta Received mean:%ld\n", mean[0]); mean[0] = mean[0] / 2; const auto msg_size = mean.size(); total_f->BeginSendPhase(); GDI_total->SendData(mean.data(), "mean", msg_size); total_f->EndSendPhase(); - printf("total sent mean:%d\n", mean[0]); + printf("total sent mean:%ld\n", mean[0]); total_f->BeginReceivePhase(); mean = GDI_total->ReceiveData("mean", msg_size); total_f->EndReceivePhase(); - printf("delta Received mean:%d\n", mean[0]); + printf("delta Received mean:%ld\n", mean[0]); mean[0] = mean[0] / 2; delta_f->BeginSendPhase(); GDI_delta->SendData(mean.data(), "mean", msg_size); delta_f->EndSendPhase(); - printf("detla sent mean:%d\n", mean[0]); + printf("detla sent mean:%ld\n", mean[0]); } } while (!done); } @@ -110,7 +110,6 @@ int main(int argc, char** argv) case 0: xgc_delta_f(comm); break; case 1: xgc_total_f(comm); break; - default: std::cerr << "Unhandled client id; expected -1, 0, or 1\n"; MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); From 22d089e2064801b1e9759df09fcc7a079dc46245 Mon Sep 17 00:00:00 2001 From: hgangwar Date: Mon, 20 Jul 2026 19:34:00 -0400 Subject: [PATCH 07/12] clang-tidy-redev_update --- .github/workflows/clang-tidy.yml | 2 +- test/test_GDI.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml index 784e8f71..56fd3574 100644 --- a/.github/workflows/clang-tidy.yml +++ b/.github/workflows/clang-tidy.yml @@ -165,7 +165,7 @@ jobs: with: repo-name: 'redev-openmpi' repo-path: 'SCOREC/redev' - repo-ref: 'ac09848a5f9b89493e8b679c9080b9efe5538376' + repo-ref: 'eb52569702864979f5b7d03d9c082c96f20b2bd5' cache: true options: '-DCMAKE_CXX_COMPILER=`which mpicxx` -DMPIEXEC_EXECUTABLE=`which mpirun` diff --git a/test/test_GDI.cpp b/test/test_GDI.cpp index 65dabe3a..a1281f60 100644 --- a/test/test_GDI.cpp +++ b/test/test_GDI.cpp @@ -1,9 +1,9 @@ - #include #include #include "test_support.h" #include "pcms/coupler/coupler.hpp" #include + static constexpr bool done = true; static constexpr int COMM_ROUNDS = 1; @@ -88,7 +88,7 @@ void xgc_coupler(MPI_Comm comm) delta_f->BeginSendPhase(); GDI_delta->SendData(mean.data(), "mean", msg_size); delta_f->EndSendPhase(); - printf("detla sent mean:%ld\n", mean[0]); + printf("delta sent mean:%ld\n", mean[0]); } } while (!done); } From 3761c83d411836ff7f87d9eb047f6355a1da511a Mon Sep 17 00:00:00 2001 From: hgangwar Date: Mon, 20 Jul 2026 19:52:28 -0400 Subject: [PATCH 08/12] CI_update --- .github/workflows/self-hosted.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/self-hosted.yml b/.github/workflows/self-hosted.yml index 35043edf..22c9f51b 100644 --- a/.github/workflows/self-hosted.yml +++ b/.github/workflows/self-hosted.yml @@ -111,7 +111,7 @@ jobs: # redev git clone https://github.com/SCOREC/redev.git ${workDir}/redev cd ${workDir}/redev - git checkout ac09848a5f9b89493e8b679c9080b9efe5538376 + git checkout eb52569702864979f5b7d03d9c082c96f20b2bd5 cd $workDir rdbdir=${workDir}/build-redev cmake -S ${workDir}/redev -B $rdbdir \ From e11b082adf3242c4479ae2432846985da4fe7fd5 Mon Sep 17 00:00:00 2001 From: hgangwar Date: Tue, 21 Jul 2026 16:15:19 -0400 Subject: [PATCH 09/12] add_DataHandle --- src/pcms/coupler/coupler.hpp | 116 ++++++++++++++++++++++++++++++----- test/test_GDI.cpp | 18 +++--- 2 files changed, 108 insertions(+), 26 deletions(-) diff --git a/src/pcms/coupler/coupler.hpp b/src/pcms/coupler/coupler.hpp index ea9fd002..53df72da 100644 --- a/src/pcms/coupler/coupler.hpp +++ b/src/pcms/coupler/coupler.hpp @@ -22,32 +22,34 @@ class GlobalDataInterface public: GlobalDataInterface(const std::string& name, MPI_Comm mpi_comm, redev::Channel& channel) - : mpi_comm_(mpi_comm), - type_info_(typeid(T)), - comm_(name, mpi_comm_, channel) + : mpi_comm_(mpi_comm), comm_(name, mpi_comm_, channel) { PCMS_FUNCTION_TIMER; } - void SendData(T* msg, std::string VarName, size_t msg_size, - Mode mode = Mode::Synchronous) + void SendData(T* msg, std::string variable_name, std::size_t msg_size, + redev::Mode mode = redev::Mode::Synchronous) { PCMS_FUNCTION_TIMER; - comm_.Send(msg, std::move(VarName), msg_size, mode); + comm_.Send(msg, variable_name, msg_size, mode); } - std::vector ReceiveData(std::string VarName, size_t msg_size, - Mode mode = Mode::Synchronous) + std::vector ReceiveData(std::string variable_name, std::size_t msg_size, + redev::Mode mode = redev::Mode::Synchronous) { PCMS_FUNCTION_TIMER; - return comm_.Receive(std::move(VarName), msg_size, mode); + return comm_.Receive(variable_name, msg_size, mode); } private: MPI_Comm mpi_comm_; - const std::type_info& type_info_; GlobalCommunicator comm_; }; +using GlobalDataVariant = + std::variant, GlobalDataInterface, + GlobalDataInterface, GlobalDataInterface, + GlobalDataInterface>; + class Application; template @@ -62,7 +64,27 @@ class FieldHandle void Send(redev::Mode mode = redev::Mode::Synchronous) const; void Receive(redev::Mode mode = redev::Mode::Synchronous) const; [[nodiscard]] Field& GetField() const; +private: + Application* app_; + std::string name_; +}; +template +class DataHandle +{ +public: + DataHandle(Application* app, std::string name) + : app_(app), name_(std::move(name)) + { + } + + void Send(T* msg, std::string variable_name, std::size_t msg_size, + redev::Mode mode = redev::Mode::Synchronous) const; + [[nodiscard]] std::vector Receive( + std::string variable_name, std::size_t msg_size, + redev::Mode mode = redev::Mode::Synchronous) const; + + [[nodiscard]] GlobalDataInterface& GetDataInterface() const; private: Application* app_; std::string name_; @@ -103,9 +125,10 @@ class Application std::unique_ptr> serializer, bool participates = true); template - std::unique_ptr> AddData(std::string name, - MPI_Comm mpi_comm); - + DataHandle AddData(std::string name, MPI_Comm mpi_comm); + template + [[nodiscard]] GlobalDataInterface& GetDataInterface( + const std::string& name); void SendField(const std::string& name, redev::Mode mode = redev::Mode::Synchronous) { @@ -126,6 +149,27 @@ class Application [mode](auto& field_communicator) { field_communicator->Receive(); }, detail::find_or_error(name, field_communicators_)); }; + template + void SendData(const std::string& name, T* msg, std::string variable_name, + std::size_t msg_size, + redev::Mode mode = redev::Mode::Synchronous) + { + PCMS_FUNCTION_TIMER; + PCMS_ALWAYS_ASSERT(InSendPhase()); + GetDataInterface(name).SendData(msg, std::move(variable_name), msg_size, + mode); + } + + template + std::vector ReceiveData(const std::string& name, std::string variable_name, + std::size_t msg_size, + redev::Mode mode = redev::Mode::Synchronous) + { + PCMS_FUNCTION_TIMER; + PCMS_ALWAYS_ASSERT(InReceivePhase()); + return GetDataInterface(name).ReceiveData(std::move(variable_name), + msg_size, mode); + } [[nodiscard]] bool InSendPhase() const noexcept { PCMS_FUNCTION_TIMER; @@ -195,15 +239,53 @@ class Application std::map> field_layout_communicators_; std::map> layout_overlap_masks_; + std::map global_data_interfaces_; }; - template -std::unique_ptr> Application::AddData(std::string name, - MPI_Comm mpi_comm) +DataHandle Application::AddData(std::string name, MPI_Comm mpi_comm) { PCMS_FUNCTION_TIMER; - return std::make_unique>(name, mpi_comm, channel_); + auto [it, inserted] = global_data_interfaces_.try_emplace( + name, std::in_place_type>, name, mpi_comm, channel_); + if (!inserted) { + throw pcms_error("Global data interface with this name already exists"); + } + return DataHandle{this, std::move(name)}; +} +template +GlobalDataInterface& Application::GetDataInterface(const std::string& name) +{ + auto* data_interface = std::get_if>( &detail::find_or_error(name, global_data_interfaces_)); + if (data_interface == nullptr) { + throw pcms_error( + "Global data interface stored with different type than requested"); + } + return *data_interface; +} +template +GlobalDataInterface& DataHandle::GetDataInterface() const +{ + PCMS_ALWAYS_ASSERT(app_ != nullptr); + return app_->GetDataInterface(name_); +} +template +void DataHandle::Send(T* msg, std::string variable_name, + std::size_t msg_size, redev::Mode mode) const +{ + PCMS_ALWAYS_ASSERT(app_ != nullptr); + app_->SendData(name_, msg, std::move(variable_name), msg_size, mode); +} + +template +std::vector DataHandle::Receive(std::string variable_name, + std::size_t msg_size, + redev::Mode mode) const +{ + PCMS_ALWAYS_ASSERT(app_ != nullptr); + + return app_->ReceiveData(name_, std::move(variable_name), msg_size, mode); } + class Coupler { private: diff --git a/test/test_GDI.cpp b/test/test_GDI.cpp index a1281f60..47239668 100644 --- a/test/test_GDI.cpp +++ b/test/test_GDI.cpp @@ -12,17 +12,17 @@ void xgc_delta_f(MPI_Comm comm) pcms::Coupler coupler("proxy_couple", comm, false, {}); pcms::Application* app = coupler.AddApplication("proxy_couple_xgc_delta_f"); - const auto GDI = app->AddData("global_comm", comm); + auto gdi = app->AddData("global_comm", comm); auto mean = std::vector(1); mean[0] = 16; do { for (int i = 0; i < COMM_ROUNDS; ++i) { app->BeginSendPhase(); - GDI->SendData(mean.data(), "mean", mean.size()); + gdi.Send(mean.data(), "mean", mean.size()); app->EndSendPhase(); printf("delta Sent mean:%ld\n", mean[0]); app->BeginReceivePhase(); - mean = GDI->ReceiveData("mean", mean.size()); + mean = gdi.Receive("mean", mean.size()); app->EndReceivePhase(); mean[0] = mean[0] / 2; } @@ -41,12 +41,12 @@ void xgc_total_f(MPI_Comm comm) do { for (int i = 0; i < COMM_ROUNDS; ++i) { app->BeginReceivePhase(); - mean = GDI->ReceiveData("mean", mean.size()); + mean = GDI.Receive("mean", mean.size()); app->EndReceivePhase(); printf("total Recieved mean:%ld\n", mean[0]); mean[0] = mean[0] / 2; app->BeginSendPhase(); - GDI->SendData(mean.data(), "mean", mean.size()); + GDI.Send(mean.data(), "mean", mean.size()); app->EndSendPhase(); printf("total Sent mean:%ld\n", mean[0]); } @@ -71,22 +71,22 @@ void xgc_coupler(MPI_Comm comm) do { for (int i = 0; i < COMM_ROUNDS; ++i) { delta_f->BeginReceivePhase(); - mean = GDI_delta->ReceiveData("mean", 1); + mean = GDI_delta.Receive("mean", 1); delta_f->EndReceivePhase(); printf("delta Received mean:%ld\n", mean[0]); mean[0] = mean[0] / 2; const auto msg_size = mean.size(); total_f->BeginSendPhase(); - GDI_total->SendData(mean.data(), "mean", msg_size); + GDI_total.Send(mean.data(), "mean", msg_size); total_f->EndSendPhase(); printf("total sent mean:%ld\n", mean[0]); total_f->BeginReceivePhase(); - mean = GDI_total->ReceiveData("mean", msg_size); + mean = GDI_total.Receive("mean", msg_size); total_f->EndReceivePhase(); printf("delta Received mean:%ld\n", mean[0]); mean[0] = mean[0] / 2; delta_f->BeginSendPhase(); - GDI_delta->SendData(mean.data(), "mean", msg_size); + GDI_delta.Send(mean.data(), "mean", msg_size); delta_f->EndSendPhase(); printf("delta sent mean:%ld\n", mean[0]); } From 21f0ce7d17df18596d3a7e8e3f049464cfa9c0ed Mon Sep 17 00:00:00 2001 From: hgangwar Date: Tue, 21 Jul 2026 16:20:03 -0400 Subject: [PATCH 10/12] clang_format --- src/pcms/coupler/coupler.hpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/pcms/coupler/coupler.hpp b/src/pcms/coupler/coupler.hpp index 53df72da..4c0a324b 100644 --- a/src/pcms/coupler/coupler.hpp +++ b/src/pcms/coupler/coupler.hpp @@ -64,6 +64,7 @@ class FieldHandle void Send(redev::Mode mode = redev::Mode::Synchronous) const; void Receive(redev::Mode mode = redev::Mode::Synchronous) const; [[nodiscard]] Field& GetField() const; + private: Application* app_; std::string name_; @@ -85,6 +86,7 @@ class DataHandle redev::Mode mode = redev::Mode::Synchronous) const; [[nodiscard]] GlobalDataInterface& GetDataInterface() const; + private: Application* app_; std::string name_; @@ -255,7 +257,8 @@ DataHandle Application::AddData(std::string name, MPI_Comm mpi_comm) template GlobalDataInterface& Application::GetDataInterface(const std::string& name) { - auto* data_interface = std::get_if>( &detail::find_or_error(name, global_data_interfaces_)); + auto* data_interface = std::get_if>( + &detail::find_or_error(name, global_data_interfaces_)); if (data_interface == nullptr) { throw pcms_error( "Global data interface stored with different type than requested"); From 25671dd393efa627b1358b21d011b82dbd3a205a Mon Sep 17 00:00:00 2001 From: hgangwar Date: Tue, 21 Jul 2026 16:52:41 -0400 Subject: [PATCH 11/12] add_try_catch --- test/test_GDI.cpp | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/test/test_GDI.cpp b/test/test_GDI.cpp index 47239668..053a4c6a 100644 --- a/test/test_GDI.cpp +++ b/test/test_GDI.cpp @@ -95,26 +95,36 @@ void xgc_coupler(MPI_Comm comm) int main(int argc, char** argv) { - MPI_Init(&argc, &argv); + try { + MPI_Init(&argc, &argv); - OMEGA_H_CHECK(argc == 2); + OMEGA_H_CHECK(argc == 2); - const auto clientId = std::atoi(argv[1]); - REDEV_ALWAYS_ASSERT(clientId >= -1 && clientId <= 1); + const auto clientId = std::atoi(argv[1]); + REDEV_ALWAYS_ASSERT(clientId >= -1 && clientId <= 1); - MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm comm = MPI_COMM_WORLD; - switch (clientId) { - case -1: xgc_coupler(comm); break; + switch (clientId) { + case -1: xgc_coupler(comm); break; - case 0: xgc_delta_f(comm); break; + case 0: xgc_delta_f(comm); break; - case 1: xgc_total_f(comm); break; - default: - std::cerr << "Unhandled client id; expected -1, 0, or 1\n"; - MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); - } + case 1: xgc_total_f(comm); break; + default: + std::cerr << "Unhandled client id; expected -1, 0, or 1\n"; + MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); + } - MPI_Finalize(); - return 0; + MPI_Finalize(); + return 0; + } + catch (const std::exception& e) { + std::cerr << "Exception caught in main: " << e.what() << std::endl; + return 1; + } + catch (...) { + std::cerr << "Unknown exception caught in main\n"; + return 1; + } } From a9a7267d1c9379b5d2b972cb9e6e481748bb294b Mon Sep 17 00:00:00 2001 From: hgangwar Date: Tue, 21 Jul 2026 16:57:09 -0400 Subject: [PATCH 12/12] format --- test/test_GDI.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/test_GDI.cpp b/test/test_GDI.cpp index 053a4c6a..c53b23b1 100644 --- a/test/test_GDI.cpp +++ b/test/test_GDI.cpp @@ -118,12 +118,10 @@ int main(int argc, char** argv) MPI_Finalize(); return 0; - } - catch (const std::exception& e) { + } catch (const std::exception& e) { std::cerr << "Exception caught in main: " << e.what() << std::endl; return 1; - } - catch (...) { + } catch (...) { std::cerr << "Unknown exception caught in main\n"; return 1; }